git.delta.rocks / jrsonnet / refs/commits / 74199ce77317

--- a/bindings/jsonnet/src/import.rs
+++ b/bindings/jsonnet/src/import.rs
@@ -2,7 +2,7 @@
 
 use jrsonnet_evaluator::{
 	error::{Error::*, Result},
-	throw, EvaluationState, IStr, ImportResolver,
+	throw, EvaluationState, ImportResolver,
 };
 use std::{
 	any::Any,
@@ -29,8 +29,7 @@
 pub struct CallbackImportResolver {
 	cb: JsonnetImportCallback,
 	ctx: *mut c_void,
-
-	out: RefCell<HashMap<PathBuf, IStr>>,
+	out: RefCell<HashMap<PathBuf, Vec<u8>>>,
 }
 impl ImportResolver for CallbackImportResolver {
 	fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
@@ -75,9 +74,10 @@
 
 		Ok(found_here_buf.into())
 	}
-	fn load_file_contents(&self, resolved: &Path) -> Result<IStr> {
+	fn load_file_contents(&self, resolved: &Path) -> Result<Vec<u8>> {
 		Ok(self.out.borrow().get(resolved).unwrap().clone())
 	}
+
 	unsafe fn as_any(&self) -> &dyn Any {
 		self
 	}
@@ -124,12 +124,12 @@
 			throw!(ImportFileNotFound(from.to_owned(), path.to_owned()))
 		}
 	}
-	fn load_file_contents(&self, id: &Path) -> Result<IStr> {
+	fn load_file_contents(&self, id: &Path) -> Result<Vec<u8>> {
 		let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?;
-		let mut out = String::new();
-		file.read_to_string(&mut out)
-			.map_err(|_e| ImportBadFileUtf8(id.to_owned()))?;
-		Ok(out.into())
+		let mut out = Vec::new();
+		file.read_to_end(&mut out)
+			.map_err(|e| ImportIo(e.to_string()))?;
+		Ok(out)
 	}
 	unsafe fn as_any(&self) -> &dyn Any {
 		self

--- a/crates/jrsonnet-evaluator/src/builtin/mod.rs
+++ b/crates/jrsonnet-evaluator/src/builtin/mod.rs
@@ -1,5 +1,5 @@
 use crate::function::{CallLocation, StaticBuiltin};
-use crate::typed::{Any, PositiveF64, VecVal, M1};
+use crate::typed::{Any, Bytes, PositiveF64, VecVal, M1};
 use crate::{
 	builtin::manifest::{manifest_yaml_ex, ManifestYamlOptions},
 	equals,
@@ -447,17 +447,15 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_encode_utf8(str: IStr) -> Result<VecVal> {
-	Ok(VecVal(
-		str.bytes()
-			.map(|b| Val::Num(b as f64))
-			.collect::<Vec<Val>>(),
-	))
+fn builtin_encode_utf8(str: IStr) -> Result<Bytes> {
+	Ok(Bytes(str.bytes().map(|b| b).collect::<Vec<u8>>().into()))
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_decode_utf8(arr: Vec<u8>) -> Result<String> {
-	Ok(String::from_utf8(arr).map_err(|_| RuntimeError("bad utf8".into()))?)
+fn builtin_decode_utf8(arr: Bytes) -> Result<IStr> {
+	Ok(std::str::from_utf8(&arr.0)
+		.map_err(|_| RuntimeError("bad utf8".into()))?
+		.into())
 }
 
 #[jrsonnet_macros::builtin]
@@ -483,17 +481,21 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_base64(input: Either![Vec<u8>, IStr]) -> Result<String> {
+fn builtin_base64(input: Either![Bytes, IStr]) -> Result<String> {
 	use Either2::*;
 	Ok(match input {
-		A(a) => base64::encode(a),
+		A(a) => base64::encode(a.0),
 		B(l) => base64::encode(l.bytes().collect::<Vec<_>>()),
 	})
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_base64_decode_bytes(input: IStr) -> Result<Vec<u8>> {
-	Ok(base64::decode(&input.as_bytes()).map_err(|_| RuntimeError("bad base64".into()))?)
+fn builtin_base64_decode_bytes(input: IStr) -> Result<Bytes> {
+	Ok(Bytes(
+		base64::decode(&input.as_bytes())
+			.map_err(|_| RuntimeError("bad base64".into()))?
+			.into(),
+	))
 }
 
 #[jrsonnet_macros::builtin]

--- a/crates/jrsonnet-evaluator/src/error.rs
+++ b/crates/jrsonnet-evaluator/src/error.rs
@@ -82,6 +82,8 @@
 	ResolvedFileNotFound(PathBuf),
 	#[error("imported file is not valid utf-8: {0:?}")]
 	ImportBadFileUtf8(PathBuf),
+	#[error("import io error: {0}")]
+	ImportIo(String),
 	#[error("tried to import {1} from {0}, but imports is not supported")]
 	ImportNotSupported(PathBuf, PathBuf),
 	#[error(

--- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs
+++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
@@ -701,5 +701,12 @@
 			import_location.pop();
 			Val::Str(with_state(|s| s.import_file_str(&import_location, path))?)
 		}
+		ImportBin(path) => {
+			let tmp = loc.clone().0;
+			let mut import_location = tmp.to_path_buf();
+			import_location.pop();
+			let bytes = with_state(|s| s.import_file_bin(&import_location, path))?;
+			Val::Arr(ArrValue::Bytes(bytes))
+		}
 	})
 }

--- a/crates/jrsonnet-evaluator/src/import.rs
+++ b/crates/jrsonnet-evaluator/src/import.rs
@@ -5,14 +5,12 @@
 use fs::File;
 use jrsonnet_interner::IStr;
 use std::fs;
-use std::io::Read;
 use std::{
 	any::Any,
-	cell::RefCell,
-	collections::HashMap,
 	path::{Path, PathBuf},
 	rc::Rc,
 };
+use std::{convert::TryFrom, io::Read};
 
 /// Implements file resolution logic for `import` and `importStr`
 pub trait ImportResolver {
@@ -21,9 +19,19 @@
 	/// where `${vendor}` is a library path.
 	fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>>;
 
+	fn load_file_contents(&self, resolved: &Path) -> Result<Vec<u8>>;
+
 	/// Reads file from filesystem, should be used only with path received from `resolve_file`
-	fn load_file_contents(&self, resolved: &Path) -> Result<IStr>;
+	fn load_file_str(&self, resolved: &Path) -> Result<IStr> {
+		Ok(IStr::try_from(&self.load_file_contents(resolved)? as &[u8])
+			.map_err(|_| ImportBadFileUtf8(resolved.to_path_buf()))?)
+	}
 
+	/// Reads file from filesystem, should be used only with path received from `resolve_file`
+	fn load_file_bin(&self, resolved: &Path) -> Result<Rc<[u8]>> {
+		Ok(self.load_file_contents(resolved)?.into())
+	}
+
 	/// # Safety
 	///
 	/// For use only in bindings, should not be used elsewhere.
@@ -39,8 +47,7 @@
 		throw!(ImportNotSupported(from.into(), path.into()))
 	}
 
-	fn load_file_contents(&self, _resolved: &Path) -> Result<IStr> {
-		// Can be only caused by library direct consumer, not by supplied jsonnet
+	fn load_file_contents(&self, _resolved: &Path) -> Result<Vec<u8>> {
 		panic!("dummy resolver can't load any file")
 	}
 
@@ -79,41 +86,12 @@
 			throw!(ImportFileNotFound(from.to_owned(), path.to_owned()))
 		}
 	}
-	fn load_file_contents(&self, id: &Path) -> Result<IStr> {
+	fn load_file_contents(&self, id: &Path) -> Result<Vec<u8>> {
 		let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?;
-		let mut out = String::new();
-		file.read_to_string(&mut out)
-			.map_err(|_e| ImportBadFileUtf8(id.to_owned()))?;
-		Ok(out.into())
-	}
-	unsafe fn as_any(&self) -> &dyn Any {
-		panic!("this resolver can't be used as any")
-	}
-}
-
-type ResolutionData = (PathBuf, PathBuf);
-
-/// Caches results of the underlying resolver
-pub struct CachingImportResolver {
-	resolution_cache: RefCell<HashMap<ResolutionData, Result<Rc<Path>>>>,
-	loading_cache: RefCell<HashMap<PathBuf, Result<IStr>>>,
-	inner: Box<dyn ImportResolver>,
-}
-impl ImportResolver for CachingImportResolver {
-	fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
-		self.resolution_cache
-			.borrow_mut()
-			.entry((from.to_owned(), path.to_owned()))
-			.or_insert_with(|| self.inner.resolve_file(from, path))
-			.clone()
-	}
-
-	fn load_file_contents(&self, resolved: &Path) -> Result<IStr> {
-		self.loading_cache
-			.borrow_mut()
-			.entry(resolved.to_owned())
-			.or_insert_with(|| self.inner.load_file_contents(resolved))
-			.clone()
+		let mut out = Vec::new();
+		file.read_to_end(&mut out)
+			.map_err(|e| ImportIo(e.to_string()))?;
+		Ok(out)
 	}
 	unsafe fn as_any(&self) -> &dyn Any {
 		panic!("this resolver can't be used as any")

--- a/crates/jrsonnet-evaluator/src/lib.rs
+++ b/crates/jrsonnet-evaluator/src/lib.rs
@@ -118,8 +118,9 @@
 
 	breakpoints: Breakpoints,
 	/// Contains file source codes and evaluation results for imports and pretty-printed stacktraces
-	files: HashMap<Rc<Path>, FileData>,
-	str_files: HashMap<Rc<Path>, IStr>,
+	files: GcHashMap<Rc<Path>, FileData>,
+	str_files: GcHashMap<Rc<Path>, IStr>,
+	bin_files: GcHashMap<Rc<Path>, Rc<[u8]>>,
 }
 
 pub struct FileData {
@@ -280,18 +281,26 @@
 				return self.evaluate_loaded_file_raw(&file_path);
 			}
 		}
-		let contents = self.load_file_contents(&file_path)?;
+		let contents = self.load_file_str(&file_path)?;
 		self.add_file(file_path.clone(), contents)?;
 		self.evaluate_loaded_file_raw(&file_path)
 	}
 	pub(crate) fn import_file_str(&self, from: &Path, path: &Path) -> Result<IStr> {
 		let path = self.resolve_file(from, path)?;
 		if !self.data().str_files.contains_key(&path) {
-			let file_str = self.load_file_contents(&path)?;
+			let file_str = self.load_file_str(&path)?;
 			self.data_mut().str_files.insert(path.clone(), file_str);
 		}
 		Ok(self.data().str_files.get(&path).cloned().unwrap())
 	}
+	pub(crate) fn import_file_bin(&self, from: &Path, path: &Path) -> Result<Rc<[u8]>> {
+		let path = self.resolve_file(from, path)?;
+		if !self.data().bin_files.contains_key(&path) {
+			let file_bin = self.load_file_bin(&path)?;
+			self.data_mut().bin_files.insert(path.clone(), file_bin);
+		}
+		Ok(self.data().bin_files.get(&path).cloned().unwrap())
+	}
 
 	fn evaluate_loaded_file_raw(&self, name: &Path) -> Result<Val> {
 		let expr: LocExpr = {
@@ -607,8 +616,11 @@
 	pub fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
 		self.settings().import_resolver.resolve_file(from, path)
 	}
-	pub fn load_file_contents(&self, path: &Path) -> Result<IStr> {
-		self.settings().import_resolver.load_file_contents(path)
+	pub fn load_file_str(&self, path: &Path) -> Result<IStr> {
+		self.settings().import_resolver.load_file_str(path)
+	}
+	pub fn load_file_bin(&self, path: &Path) -> Result<Rc<[u8]>> {
+		self.settings().import_resolver.load_file_bin(path)
 	}
 
 	pub fn import_resolver(&self) -> Ref<dyn ImportResolver> {
@@ -687,7 +699,6 @@
 		primitive_equals, EvaluationState,
 	};
 	use gcmodule::{Cc, Trace};
-	use jrsonnet_interner::IStr;
 	use jrsonnet_parser::*;
 	use std::{
 		path::{Path, PathBuf},
@@ -1204,19 +1215,23 @@
 		Ok(())
 	}
 
-	struct TestImportResolver(IStr);
+	struct TestImportResolver(Vec<u8>);
 	impl crate::import::ImportResolver for TestImportResolver {
 		fn resolve_file(&self, _: &Path, _: &Path) -> crate::error::Result<Rc<Path>> {
 			Ok(PathBuf::from("/test").into())
 		}
 
-		fn load_file_contents(&self, _: &Path) -> crate::error::Result<IStr> {
+		fn load_file_contents(&self, _: &Path) -> crate::error::Result<Vec<u8>> {
 			Ok(self.0.clone())
 		}
 
 		unsafe fn as_any(&self) -> &dyn std::any::Any {
 			panic!()
 		}
+
+		fn load_file_bin(&self, _resolved: &Path) -> crate::error::Result<Rc<[u8]>> {
+			panic!()
+		}
 	}
 
 	#[test]

--- a/crates/jrsonnet-evaluator/src/typed/conversions.rs
+++ b/crates/jrsonnet-evaluator/src/typed/conversions.rs
@@ -1,4 +1,7 @@
-use std::convert::{TryFrom, TryInto};
+use std::{
+	convert::{TryFrom, TryInto},
+	rc::Rc,
+};
 
 use gcmodule::Cc;
 use jrsonnet_interner::IStr;
@@ -306,6 +309,44 @@
 	}
 }
 
+/// Specialization
+pub struct Bytes(pub Rc<[u8]>);
+
+impl Typed for Bytes {
+	const TYPE: &'static ComplexValType =
+		&ComplexValType::ArrayRef(&ComplexValType::BoundedNumber(Some(0.0), Some(255.0)));
+}
+impl TryFrom<Val> for Bytes {
+	type Error = LocError;
+
+	fn try_from(value: Val) -> Result<Self> {
+		match value {
+			Val::Arr(ArrValue::Bytes(bytes)) => Ok(Self(bytes)),
+			_ => {
+				<Self as Typed>::TYPE.check(&value)?;
+				match value {
+					Val::Arr(a) => {
+						let mut out = Vec::with_capacity(a.len());
+						for e in a.iter() {
+							let r = e?;
+							out.push(u8::try_from(r)?);
+						}
+						Ok(Self(out.into()))
+					}
+					_ => unreachable!(),
+				}
+			}
+		}
+	}
+}
+impl TryFrom<Bytes> for Val {
+	type Error = LocError;
+
+	fn try_from(value: Bytes) -> Result<Self> {
+		Ok(Val::Arr(ArrValue::Bytes(value.0)))
+	}
+}
+
 pub struct M1;
 impl Typed for M1 {
 	const TYPE: &'static ComplexValType = &ComplexValType::BoundedNumber(Some(-1.0), Some(-1.0));

--- a/crates/jrsonnet-evaluator/src/val.rs
+++ b/crates/jrsonnet-evaluator/src/val.rs
@@ -174,6 +174,7 @@
 #[derive(Debug, Clone, Trace)]
 #[force_tracking]
 pub enum ArrValue {
+	Bytes(#[skip_trace] Rc<[u8]>),
 	Lazy(Cc<Vec<LazyVal>>),
 	Eager(Cc<Vec<Val>>),
 	Extended(Box<(Self, Self)>),
@@ -185,6 +186,7 @@
 
 	pub fn len(&self) -> usize {
 		match self {
+			Self::Bytes(i) => i.len(),
 			Self::Lazy(l) => l.len(),
 			Self::Eager(e) => e.len(),
 			Self::Extended(v) => v.0.len() + v.1.len(),
@@ -197,6 +199,9 @@
 
 	pub fn get(&self, index: usize) -> Result<Option<Val>> {
 		match self {
+			Self::Bytes(i) => i
+				.get(index)
+				.map_or(Ok(None), |v| Ok(Some(Val::Num(*v as f64)))),
 			Self::Lazy(vec) => {
 				if let Some(v) = vec.get(index) {
 					Ok(Some(v.evaluate()?))
@@ -218,6 +223,9 @@
 
 	pub fn get_lazy(&self, index: usize) -> Option<LazyVal> {
 		match self {
+			Self::Bytes(i) => i
+				.get(index)
+				.map(|b| LazyVal::new_resolved(Val::Num(*b as f64))),
 			Self::Lazy(vec) => vec.get(index).cloned(),
 			Self::Eager(vec) => vec.get(index).cloned().map(LazyVal::new_resolved),
 			Self::Extended(v) => {
@@ -233,6 +241,13 @@
 
 	pub fn evaluated(&self) -> Result<Cc<Vec<Val>>> {
 		Ok(match self {
+			Self::Bytes(i) => {
+				let mut out = Vec::with_capacity(i.len());
+				for v in i.iter() {
+					out.push(Val::Num(*v as f64));
+				}
+				Cc::new(out)
+			}
 			Self::Lazy(vec) => {
 				let mut out = Vec::with_capacity(vec.len());
 				for item in vec.iter() {
@@ -253,6 +268,7 @@
 
 	pub fn iter(&self) -> impl DoubleEndedIterator<Item = Result<Val>> + '_ {
 		(0..self.len()).map(move |idx| match self {
+			Self::Bytes(b) => Ok(Val::Num(b[idx] as f64)),
 			Self::Lazy(l) => l[idx].evaluate(),
 			Self::Eager(e) => Ok(e[idx].clone()),
 			Self::Extended(_) => self.get(idx).map(|e| e.unwrap()),
@@ -261,6 +277,7 @@
 
 	pub fn iter_lazy(&self) -> impl DoubleEndedIterator<Item = LazyVal> + '_ {
 		(0..self.len()).map(move |idx| match self {
+			Self::Bytes(b) => LazyVal::new_resolved(Val::Num(b[idx] as f64)),
 			Self::Lazy(l) => l[idx].clone(),
 			Self::Eager(e) => LazyVal::new_resolved(e[idx].clone()),
 			Self::Extended(_) => self.get_lazy(idx).unwrap(),
@@ -269,6 +286,11 @@
 
 	pub fn reversed(self) -> Self {
 		match self {
+			Self::Bytes(b) => {
+				let mut out = b.to_vec();
+				out.reverse();
+				Self::Bytes(out.into())
+			}
 			Self::Lazy(vec) => {
 				let mut out = (&vec as &Vec<_>).clone();
 				out.reverse();

--- a/crates/jrsonnet-interner/src/lib.rs
+++ b/crates/jrsonnet-interner/src/lib.rs
@@ -4,10 +4,12 @@
 use std::{
 	borrow::Cow,
 	cell::RefCell,
+	convert::TryFrom,
 	fmt::{self, Display},
 	hash::{BuildHasherDefault, Hash, Hasher},
 	ops::Deref,
 	rc::Rc,
+	str::Utf8Error,
 };
 
 #[derive(Clone, PartialOrd, Ord, Eq)]
@@ -85,6 +87,15 @@
 	}
 }
 
+impl TryFrom<&[u8]> for IStr {
+	type Error = Utf8Error;
+
+	fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+		let str = std::str::from_utf8(value)?;
+		Ok(str.into())
+	}
+}
+
 impl From<String> for IStr {
 	fn from(str: String) -> Self {
 		(&str as &str).into()

--- a/crates/jrsonnet-parser/src/lib.rs
+++ b/crates/jrsonnet-parser/src/lib.rs
@@ -53,7 +53,7 @@
 		rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "<number>") }} / expected!("<number>")
 
 		/// Reserved word followed by any non-alphanumberic
-		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()
+		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "importbin" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()
 		rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")
 
 		rule keyword(id: &'static str) -> ()
@@ -218,6 +218,7 @@
 			/ array_comp_expr(s)
 
 			/ keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}
+			/ keyword("importbin") _ path:string() {Expr::ImportBin(PathBuf::from(path))}
 			/ keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}
 
 			/ var_expr(s)