From 20cd69c85b05f1bda64d974b8b08dbcaef158f93 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 29 Jan 2022 17:47:03 +0000 Subject: [PATCH] feat: importbin --- --- a/bindings/jsonnet/src/import.rs +++ b/bindings/jsonnet/src/import.rs @@ -2,7 +2,7 @@ use jrsonnet_evaluator::{ error::{Error::*, Result}, - throw, EvaluationState, IStr, ImportResolver, + throw, EvaluationState, ImportResolver, }; use std::{ any::Any, @@ -29,8 +29,7 @@ pub struct CallbackImportResolver { cb: JsonnetImportCallback, ctx: *mut c_void, - - out: RefCell>, + out: RefCell>>, } impl ImportResolver for CallbackImportResolver { fn resolve_file(&self, from: &Path, path: &Path) -> Result> { @@ -75,9 +74,10 @@ Ok(found_here_buf.into()) } - fn load_file_contents(&self, resolved: &Path) -> Result { + fn load_file_contents(&self, resolved: &Path) -> Result> { Ok(self.out.borrow().get(resolved).unwrap().clone()) } + unsafe fn as_any(&self) -> &dyn Any { self } @@ -124,12 +124,12 @@ throw!(ImportFileNotFound(from.to_owned(), path.to_owned())) } } - fn load_file_contents(&self, id: &Path) -> Result { + fn load_file_contents(&self, id: &Path) -> Result> { let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?; - let mut out = String::new(); - file.read_to_string(&mut out) - .map_err(|_e| ImportBadFileUtf8(id.to_owned()))?; - Ok(out.into()) + let mut out = Vec::new(); + file.read_to_end(&mut out) + .map_err(|e| ImportIo(e.to_string()))?; + Ok(out) } unsafe fn as_any(&self) -> &dyn Any { self --- a/crates/jrsonnet-evaluator/src/error.rs +++ b/crates/jrsonnet-evaluator/src/error.rs @@ -82,6 +82,8 @@ ResolvedFileNotFound(PathBuf), #[error("imported file is not valid utf-8: {0:?}")] ImportBadFileUtf8(PathBuf), + #[error("import io error: {0}")] + ImportIo(String), #[error("tried to import {1} from {0}, but imports is not supported")] ImportNotSupported(PathBuf, PathBuf), #[error( --- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs +++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs @@ -700,5 +700,12 @@ import_location.pop(); Val::Str(with_state(|s| s.import_file_str(&import_location, path))?) } + ImportBin(path) => { + let tmp = loc.clone().0; + let mut import_location = tmp.to_path_buf(); + import_location.pop(); + let bytes = with_state(|s| s.import_file_bin(&import_location, path))?; + Val::Arr(ArrValue::Bytes(bytes)) + } }) } --- a/crates/jrsonnet-evaluator/src/import.rs +++ b/crates/jrsonnet-evaluator/src/import.rs @@ -5,14 +5,12 @@ use fs::File; use jrsonnet_interner::IStr; use std::fs; -use std::io::Read; use std::{ any::Any, - cell::RefCell, - collections::HashMap, path::{Path, PathBuf}, rc::Rc, }; +use std::{convert::TryFrom, io::Read}; /// Implements file resolution logic for `import` and `importStr` pub trait ImportResolver { @@ -21,9 +19,19 @@ /// where `${vendor}` is a library path. fn resolve_file(&self, from: &Path, path: &Path) -> Result>; + fn load_file_contents(&self, resolved: &Path) -> Result>; + /// Reads file from filesystem, should be used only with path received from `resolve_file` - fn load_file_contents(&self, resolved: &Path) -> Result; + fn load_file_str(&self, resolved: &Path) -> Result { + Ok(IStr::try_from(&self.load_file_contents(resolved)? as &[u8]) + .map_err(|_| ImportBadFileUtf8(resolved.to_path_buf()))?) + } + /// Reads file from filesystem, should be used only with path received from `resolve_file` + fn load_file_bin(&self, resolved: &Path) -> Result> { + Ok(self.load_file_contents(resolved)?.into()) + } + /// # Safety /// /// For use only in bindings, should not be used elsewhere. @@ -39,8 +47,7 @@ throw!(ImportNotSupported(from.into(), path.into())) } - fn load_file_contents(&self, _resolved: &Path) -> Result { - // Can be only caused by library direct consumer, not by supplied jsonnet + fn load_file_contents(&self, _resolved: &Path) -> Result> { panic!("dummy resolver can't load any file") } @@ -79,41 +86,12 @@ throw!(ImportFileNotFound(from.to_owned(), path.to_owned())) } } - fn load_file_contents(&self, id: &Path) -> Result { + fn load_file_contents(&self, id: &Path) -> Result> { let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?; - let mut out = String::new(); - file.read_to_string(&mut out) - .map_err(|_e| ImportBadFileUtf8(id.to_owned()))?; - Ok(out.into()) - } - unsafe fn as_any(&self) -> &dyn Any { - panic!("this resolver can't be used as any") - } -} - -type ResolutionData = (PathBuf, PathBuf); - -/// Caches results of the underlying resolver -pub struct CachingImportResolver { - resolution_cache: RefCell>>>, - loading_cache: RefCell>>, - inner: Box, -} -impl ImportResolver for CachingImportResolver { - fn resolve_file(&self, from: &Path, path: &Path) -> Result> { - self.resolution_cache - .borrow_mut() - .entry((from.to_owned(), path.to_owned())) - .or_insert_with(|| self.inner.resolve_file(from, path)) - .clone() - } - - fn load_file_contents(&self, resolved: &Path) -> Result { - self.loading_cache - .borrow_mut() - .entry(resolved.to_owned()) - .or_insert_with(|| self.inner.load_file_contents(resolved)) - .clone() + let mut out = Vec::new(); + file.read_to_end(&mut out) + .map_err(|e| ImportIo(e.to_string()))?; + Ok(out) } unsafe fn as_any(&self) -> &dyn Any { panic!("this resolver can't be used as any") --- a/crates/jrsonnet-evaluator/src/lib.rs +++ b/crates/jrsonnet-evaluator/src/lib.rs @@ -119,8 +119,9 @@ breakpoints: Breakpoints, /// Contains file source codes and evaluation results for imports and pretty-printed stacktraces - files: HashMap, FileData>, - str_files: HashMap, IStr>, + files: GcHashMap, FileData>, + str_files: GcHashMap, IStr>, + bin_files: GcHashMap, Rc<[u8]>>, } pub struct FileData { @@ -276,18 +277,26 @@ return self.evaluate_loaded_file_raw(&file_path); } } - let contents = self.load_file_contents(&file_path)?; + let contents = self.load_file_str(&file_path)?; self.add_file(file_path.clone(), contents)?; self.evaluate_loaded_file_raw(&file_path) } pub(crate) fn import_file_str(&self, from: &Path, path: &Path) -> Result { let path = self.resolve_file(from, path)?; if !self.data().str_files.contains_key(&path) { - let file_str = self.load_file_contents(&path)?; + let file_str = self.load_file_str(&path)?; self.data_mut().str_files.insert(path.clone(), file_str); } Ok(self.data().str_files.get(&path).cloned().unwrap()) } + pub(crate) fn import_file_bin(&self, from: &Path, path: &Path) -> Result> { + let path = self.resolve_file(from, path)?; + if !self.data().bin_files.contains_key(&path) { + let file_bin = self.load_file_bin(&path)?; + self.data_mut().bin_files.insert(path.clone(), file_bin); + } + Ok(self.data().bin_files.get(&path).cloned().unwrap()) + } fn evaluate_loaded_file_raw(&self, name: &Path) -> Result { let expr: LocExpr = { @@ -603,8 +612,11 @@ pub fn resolve_file(&self, from: &Path, path: &Path) -> Result> { self.settings().import_resolver.resolve_file(from, path) } - pub fn load_file_contents(&self, path: &Path) -> Result { - self.settings().import_resolver.load_file_contents(path) + pub fn load_file_str(&self, path: &Path) -> Result { + self.settings().import_resolver.load_file_str(path) + } + pub fn load_file_bin(&self, path: &Path) -> Result> { + self.settings().import_resolver.load_file_bin(path) } pub fn import_resolver(&self) -> Ref { @@ -661,7 +673,6 @@ EvaluationState, }; use gcmodule::{Cc, Trace}; - use jrsonnet_interner::IStr; use jrsonnet_parser::*; use std::{ path::{Path, PathBuf}, @@ -1165,19 +1176,23 @@ Ok(()) } - struct TestImportResolver(IStr); + struct TestImportResolver(Vec); impl crate::import::ImportResolver for TestImportResolver { fn resolve_file(&self, _: &Path, _: &Path) -> crate::error::Result> { Ok(PathBuf::from("/test").into()) } - fn load_file_contents(&self, _: &Path) -> crate::error::Result { + fn load_file_contents(&self, _: &Path) -> crate::error::Result> { Ok(self.0.clone()) } unsafe fn as_any(&self) -> &dyn std::any::Any { panic!() } + + fn load_file_bin(&self, _resolved: &Path) -> crate::error::Result> { + panic!() + } } #[test] --- a/crates/jrsonnet-evaluator/src/val.rs +++ b/crates/jrsonnet-evaluator/src/val.rs @@ -186,6 +186,7 @@ #[derive(Debug, Clone, Trace)] #[force_tracking] pub enum ArrValue { + Bytes(#[skip_trace] Rc<[u8]>), Lazy(Cc>), Eager(Cc>), Extended(Box<(Self, Self)>), @@ -197,6 +198,7 @@ pub fn len(&self) -> usize { match self { + Self::Bytes(i) => i.len(), Self::Lazy(l) => l.len(), Self::Eager(e) => e.len(), Self::Extended(v) => v.0.len() + v.1.len(), @@ -209,6 +211,9 @@ pub fn get(&self, index: usize) -> Result> { match self { + Self::Bytes(i) => i + .get(index) + .map_or(Ok(None), |v| Ok(Some(Val::Num(*v as f64)))), Self::Lazy(vec) => { if let Some(v) = vec.get(index) { Ok(Some(v.evaluate()?)) @@ -230,6 +235,9 @@ pub fn get_lazy(&self, index: usize) -> Option { match self { + Self::Bytes(i) => i + .get(index) + .map(|b| LazyVal::new_resolved(Val::Num(*b as f64))), Self::Lazy(vec) => vec.get(index).cloned(), Self::Eager(vec) => vec.get(index).cloned().map(LazyVal::new_resolved), Self::Extended(v) => { @@ -245,6 +253,13 @@ pub fn evaluated(&self) -> Result>> { Ok(match self { + Self::Bytes(i) => { + let mut out = Vec::with_capacity(i.len()); + for v in i.iter() { + out.push(Val::Num(*v as f64)); + } + Cc::new(out) + } Self::Lazy(vec) => { let mut out = Vec::with_capacity(vec.len()); for item in vec.iter() { @@ -265,6 +280,7 @@ pub fn iter(&self) -> impl DoubleEndedIterator> + '_ { (0..self.len()).map(move |idx| match self { + Self::Bytes(b) => Ok(Val::Num(b[idx] as f64)), Self::Lazy(l) => l[idx].evaluate(), Self::Eager(e) => Ok(e[idx].clone()), Self::Extended(_) => self.get(idx).map(|e| e.unwrap()), @@ -273,6 +289,7 @@ pub fn iter_lazy(&self) -> impl DoubleEndedIterator + '_ { (0..self.len()).map(move |idx| match self { + Self::Bytes(b) => LazyVal::new_resolved(Val::Num(b[idx] as f64)), Self::Lazy(l) => l[idx].clone(), Self::Eager(e) => LazyVal::new_resolved(e[idx].clone()), Self::Extended(_) => self.get_lazy(idx).unwrap(), @@ -281,6 +298,11 @@ pub fn reversed(self) -> Self { match self { + Self::Bytes(b) => { + let mut out = b.to_vec(); + out.reverse(); + Self::Bytes(out.into()) + } Self::Lazy(vec) => { let mut out = (&vec as &Vec<_>).clone(); out.reverse(); --- a/crates/jrsonnet-interner/src/lib.rs +++ b/crates/jrsonnet-interner/src/lib.rs @@ -4,10 +4,12 @@ use std::{ borrow::Cow, cell::RefCell, + convert::TryFrom, fmt::{self, Display}, hash::{BuildHasherDefault, Hash, Hasher}, ops::Deref, rc::Rc, + str::Utf8Error, }; #[derive(Clone, PartialOrd, Ord, Eq)] @@ -85,6 +87,15 @@ } } +impl TryFrom<&[u8]> for IStr { + type Error = Utf8Error; + + fn try_from(value: &[u8]) -> Result { + let str = std::str::from_utf8(value)?; + Ok(str.into()) + } +} + impl From for IStr { fn from(str: String) -> Self { (&str as &str).into() --- a/crates/jrsonnet-parser/src/expr.rs +++ b/crates/jrsonnet-parser/src/expr.rs @@ -306,6 +306,8 @@ Import(PathBuf), /// importStr "file.txt" ImportStr(PathBuf), + /// importBin "file.txt" + ImportBin(PathBuf), /// error "I'm broken" ErrorStmt(LocExpr), /// a(b, c) --- a/crates/jrsonnet-parser/src/lib.rs +++ b/crates/jrsonnet-parser/src/lib.rs @@ -53,7 +53,7 @@ rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "") }} / expected!("") /// Reserved word followed by any non-alphanumberic - rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident() + rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "importbin" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident() rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("") rule keyword(id: &'static str) -> () @@ -218,6 +218,7 @@ / array_comp_expr(s) / keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))} + / keyword("importbin") _ path:string() {Expr::ImportBin(PathBuf::from(path))} / keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))} / var_expr(s) -- gitstuff