difftreelog
Merge remote-tracking branch 'origin/feature/importbin' into gcmodule
in: master
11 files changed
bindings/jsonnet/src/import.rsdiffbeforeafterboth--- a/bindings/jsonnet/src/import.rs
+++ b/bindings/jsonnet/src/import.rs
@@ -2,7 +2,7 @@
use jrsonnet_evaluator::{
error::{Error::*, Result},
- throw, EvaluationState, IStr, ImportResolver,
+ throw, EvaluationState, ImportResolver,
};
use std::{
any::Any,
@@ -29,8 +29,7 @@
pub struct CallbackImportResolver {
cb: JsonnetImportCallback,
ctx: *mut c_void,
-
- out: RefCell<HashMap<PathBuf, IStr>>,
+ out: RefCell<HashMap<PathBuf, Vec<u8>>>,
}
impl ImportResolver for CallbackImportResolver {
fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
@@ -75,9 +74,10 @@
Ok(found_here_buf.into())
}
- fn load_file_contents(&self, resolved: &Path) -> Result<IStr> {
+ fn load_file_contents(&self, resolved: &Path) -> Result<Vec<u8>> {
Ok(self.out.borrow().get(resolved).unwrap().clone())
}
+
unsafe fn as_any(&self) -> &dyn Any {
self
}
@@ -124,12 +124,12 @@
throw!(ImportFileNotFound(from.to_owned(), path.to_owned()))
}
}
- fn load_file_contents(&self, id: &Path) -> Result<IStr> {
+ fn load_file_contents(&self, id: &Path) -> Result<Vec<u8>> {
let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?;
- let mut out = String::new();
- file.read_to_string(&mut out)
- .map_err(|_e| ImportBadFileUtf8(id.to_owned()))?;
- Ok(out.into())
+ let mut out = Vec::new();
+ file.read_to_end(&mut out)
+ .map_err(|e| ImportIo(e.to_string()))?;
+ Ok(out)
}
unsafe fn as_any(&self) -> &dyn Any {
self
crates/jrsonnet-evaluator/src/builtin/mod.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/builtin/mod.rs
+++ b/crates/jrsonnet-evaluator/src/builtin/mod.rs
@@ -1,5 +1,5 @@
use crate::function::{CallLocation, StaticBuiltin};
-use crate::typed::{Any, PositiveF64, VecVal, M1};
+use crate::typed::{Any, Bytes, PositiveF64, VecVal, M1};
use crate::{
builtin::manifest::{manifest_yaml_ex, ManifestYamlOptions},
equals,
@@ -447,17 +447,15 @@
}
#[jrsonnet_macros::builtin]
-fn builtin_encode_utf8(str: IStr) -> Result<VecVal> {
- Ok(VecVal(
- str.bytes()
- .map(|b| Val::Num(b as f64))
- .collect::<Vec<Val>>(),
- ))
+fn builtin_encode_utf8(str: IStr) -> Result<Bytes> {
+ Ok(Bytes(str.bytes().map(|b| b).collect::<Vec<u8>>().into()))
}
#[jrsonnet_macros::builtin]
-fn builtin_decode_utf8(arr: Vec<u8>) -> Result<String> {
- Ok(String::from_utf8(arr).map_err(|_| RuntimeError("bad utf8".into()))?)
+fn builtin_decode_utf8(arr: Bytes) -> Result<IStr> {
+ Ok(std::str::from_utf8(&arr.0)
+ .map_err(|_| RuntimeError("bad utf8".into()))?
+ .into())
}
#[jrsonnet_macros::builtin]
@@ -483,17 +481,21 @@
}
#[jrsonnet_macros::builtin]
-fn builtin_base64(input: Either![Vec<u8>, IStr]) -> Result<String> {
+fn builtin_base64(input: Either![Bytes, IStr]) -> Result<String> {
use Either2::*;
Ok(match input {
- A(a) => base64::encode(a),
+ A(a) => base64::encode(a.0),
B(l) => base64::encode(l.bytes().collect::<Vec<_>>()),
})
}
#[jrsonnet_macros::builtin]
-fn builtin_base64_decode_bytes(input: IStr) -> Result<Vec<u8>> {
- Ok(base64::decode(&input.as_bytes()).map_err(|_| RuntimeError("bad base64".into()))?)
+fn builtin_base64_decode_bytes(input: IStr) -> Result<Bytes> {
+ Ok(Bytes(
+ base64::decode(&input.as_bytes())
+ .map_err(|_| RuntimeError("bad base64".into()))?
+ .into(),
+ ))
}
#[jrsonnet_macros::builtin]
crates/jrsonnet-evaluator/src/error.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/error.rs
+++ b/crates/jrsonnet-evaluator/src/error.rs
@@ -82,6 +82,8 @@
ResolvedFileNotFound(PathBuf),
#[error("imported file is not valid utf-8: {0:?}")]
ImportBadFileUtf8(PathBuf),
+ #[error("import io error: {0}")]
+ ImportIo(String),
#[error("tried to import {1} from {0}, but imports is not supported")]
ImportNotSupported(PathBuf, PathBuf),
#[error(
crates/jrsonnet-evaluator/src/evaluate/mod.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs
+++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
@@ -701,5 +701,12 @@
import_location.pop();
Val::Str(with_state(|s| s.import_file_str(&import_location, path))?)
}
+ ImportBin(path) => {
+ let tmp = loc.clone().0;
+ let mut import_location = tmp.to_path_buf();
+ import_location.pop();
+ let bytes = with_state(|s| s.import_file_bin(&import_location, path))?;
+ Val::Arr(ArrValue::Bytes(bytes))
+ }
})
}
crates/jrsonnet-evaluator/src/import.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/import.rs
+++ b/crates/jrsonnet-evaluator/src/import.rs
@@ -5,14 +5,12 @@
use fs::File;
use jrsonnet_interner::IStr;
use std::fs;
-use std::io::Read;
use std::{
any::Any,
- cell::RefCell,
- collections::HashMap,
path::{Path, PathBuf},
rc::Rc,
};
+use std::{convert::TryFrom, io::Read};
/// Implements file resolution logic for `import` and `importStr`
pub trait ImportResolver {
@@ -21,9 +19,19 @@
/// where `${vendor}` is a library path.
fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>>;
+ fn load_file_contents(&self, resolved: &Path) -> Result<Vec<u8>>;
+
/// Reads file from filesystem, should be used only with path received from `resolve_file`
- fn load_file_contents(&self, resolved: &Path) -> Result<IStr>;
+ fn load_file_str(&self, resolved: &Path) -> Result<IStr> {
+ Ok(IStr::try_from(&self.load_file_contents(resolved)? as &[u8])
+ .map_err(|_| ImportBadFileUtf8(resolved.to_path_buf()))?)
+ }
+ /// Reads file from filesystem, should be used only with path received from `resolve_file`
+ fn load_file_bin(&self, resolved: &Path) -> Result<Rc<[u8]>> {
+ Ok(self.load_file_contents(resolved)?.into())
+ }
+
/// # Safety
///
/// For use only in bindings, should not be used elsewhere.
@@ -39,8 +47,7 @@
throw!(ImportNotSupported(from.into(), path.into()))
}
- fn load_file_contents(&self, _resolved: &Path) -> Result<IStr> {
- // Can be only caused by library direct consumer, not by supplied jsonnet
+ fn load_file_contents(&self, _resolved: &Path) -> Result<Vec<u8>> {
panic!("dummy resolver can't load any file")
}
@@ -79,41 +86,12 @@
throw!(ImportFileNotFound(from.to_owned(), path.to_owned()))
}
}
- fn load_file_contents(&self, id: &Path) -> Result<IStr> {
+ fn load_file_contents(&self, id: &Path) -> Result<Vec<u8>> {
let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?;
- let mut out = String::new();
- file.read_to_string(&mut out)
- .map_err(|_e| ImportBadFileUtf8(id.to_owned()))?;
- Ok(out.into())
- }
- unsafe fn as_any(&self) -> &dyn Any {
- panic!("this resolver can't be used as any")
- }
-}
-
-type ResolutionData = (PathBuf, PathBuf);
-
-/// Caches results of the underlying resolver
-pub struct CachingImportResolver {
- resolution_cache: RefCell<HashMap<ResolutionData, Result<Rc<Path>>>>,
- loading_cache: RefCell<HashMap<PathBuf, Result<IStr>>>,
- inner: Box<dyn ImportResolver>,
-}
-impl ImportResolver for CachingImportResolver {
- fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
- self.resolution_cache
- .borrow_mut()
- .entry((from.to_owned(), path.to_owned()))
- .or_insert_with(|| self.inner.resolve_file(from, path))
- .clone()
- }
-
- fn load_file_contents(&self, resolved: &Path) -> Result<IStr> {
- self.loading_cache
- .borrow_mut()
- .entry(resolved.to_owned())
- .or_insert_with(|| self.inner.load_file_contents(resolved))
- .clone()
+ let mut out = Vec::new();
+ file.read_to_end(&mut out)
+ .map_err(|e| ImportIo(e.to_string()))?;
+ Ok(out)
}
unsafe fn as_any(&self) -> &dyn Any {
panic!("this resolver can't be used as any")
crates/jrsonnet-evaluator/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/lib.rs
+++ b/crates/jrsonnet-evaluator/src/lib.rs
@@ -118,8 +118,9 @@
breakpoints: Breakpoints,
/// Contains file source codes and evaluation results for imports and pretty-printed stacktraces
- files: HashMap<Rc<Path>, FileData>,
- str_files: HashMap<Rc<Path>, IStr>,
+ files: GcHashMap<Rc<Path>, FileData>,
+ str_files: GcHashMap<Rc<Path>, IStr>,
+ bin_files: GcHashMap<Rc<Path>, Rc<[u8]>>,
}
pub struct FileData {
@@ -280,18 +281,26 @@
return self.evaluate_loaded_file_raw(&file_path);
}
}
- let contents = self.load_file_contents(&file_path)?;
+ let contents = self.load_file_str(&file_path)?;
self.add_file(file_path.clone(), contents)?;
self.evaluate_loaded_file_raw(&file_path)
}
pub(crate) fn import_file_str(&self, from: &Path, path: &Path) -> Result<IStr> {
let path = self.resolve_file(from, path)?;
if !self.data().str_files.contains_key(&path) {
- let file_str = self.load_file_contents(&path)?;
+ let file_str = self.load_file_str(&path)?;
self.data_mut().str_files.insert(path.clone(), file_str);
}
Ok(self.data().str_files.get(&path).cloned().unwrap())
}
+ pub(crate) fn import_file_bin(&self, from: &Path, path: &Path) -> Result<Rc<[u8]>> {
+ let path = self.resolve_file(from, path)?;
+ if !self.data().bin_files.contains_key(&path) {
+ let file_bin = self.load_file_bin(&path)?;
+ self.data_mut().bin_files.insert(path.clone(), file_bin);
+ }
+ Ok(self.data().bin_files.get(&path).cloned().unwrap())
+ }
fn evaluate_loaded_file_raw(&self, name: &Path) -> Result<Val> {
let expr: LocExpr = {
@@ -607,8 +616,11 @@
pub fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {
self.settings().import_resolver.resolve_file(from, path)
}
- pub fn load_file_contents(&self, path: &Path) -> Result<IStr> {
- self.settings().import_resolver.load_file_contents(path)
+ pub fn load_file_str(&self, path: &Path) -> Result<IStr> {
+ self.settings().import_resolver.load_file_str(path)
+ }
+ pub fn load_file_bin(&self, path: &Path) -> Result<Rc<[u8]>> {
+ self.settings().import_resolver.load_file_bin(path)
}
pub fn import_resolver(&self) -> Ref<dyn ImportResolver> {
@@ -687,7 +699,6 @@
primitive_equals, EvaluationState,
};
use gcmodule::{Cc, Trace};
- use jrsonnet_interner::IStr;
use jrsonnet_parser::*;
use std::{
path::{Path, PathBuf},
@@ -1204,19 +1215,23 @@
Ok(())
}
- struct TestImportResolver(IStr);
+ struct TestImportResolver(Vec<u8>);
impl crate::import::ImportResolver for TestImportResolver {
fn resolve_file(&self, _: &Path, _: &Path) -> crate::error::Result<Rc<Path>> {
Ok(PathBuf::from("/test").into())
}
- fn load_file_contents(&self, _: &Path) -> crate::error::Result<IStr> {
+ fn load_file_contents(&self, _: &Path) -> crate::error::Result<Vec<u8>> {
Ok(self.0.clone())
}
unsafe fn as_any(&self) -> &dyn std::any::Any {
panic!()
}
+
+ fn load_file_bin(&self, _resolved: &Path) -> crate::error::Result<Rc<[u8]>> {
+ panic!()
+ }
}
#[test]
crates/jrsonnet-evaluator/src/typed/conversions.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/typed/conversions.rs
+++ b/crates/jrsonnet-evaluator/src/typed/conversions.rs
@@ -1,4 +1,7 @@
-use std::convert::{TryFrom, TryInto};
+use std::{
+ convert::{TryFrom, TryInto},
+ rc::Rc,
+};
use gcmodule::Cc;
use jrsonnet_interner::IStr;
@@ -306,6 +309,44 @@
}
}
+/// Specialization
+pub struct Bytes(pub Rc<[u8]>);
+
+impl Typed for Bytes {
+ const TYPE: &'static ComplexValType =
+ &ComplexValType::ArrayRef(&ComplexValType::BoundedNumber(Some(0.0), Some(255.0)));
+}
+impl TryFrom<Val> for Bytes {
+ type Error = LocError;
+
+ fn try_from(value: Val) -> Result<Self> {
+ match value {
+ Val::Arr(ArrValue::Bytes(bytes)) => Ok(Self(bytes)),
+ _ => {
+ <Self as Typed>::TYPE.check(&value)?;
+ match value {
+ Val::Arr(a) => {
+ let mut out = Vec::with_capacity(a.len());
+ for e in a.iter() {
+ let r = e?;
+ out.push(u8::try_from(r)?);
+ }
+ Ok(Self(out.into()))
+ }
+ _ => unreachable!(),
+ }
+ }
+ }
+ }
+}
+impl TryFrom<Bytes> for Val {
+ type Error = LocError;
+
+ fn try_from(value: Bytes) -> Result<Self> {
+ Ok(Val::Arr(ArrValue::Bytes(value.0)))
+ }
+}
+
pub struct M1;
impl Typed for M1 {
const TYPE: &'static ComplexValType = &ComplexValType::BoundedNumber(Some(-1.0), Some(-1.0));
crates/jrsonnet-evaluator/src/val.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/val.rs
+++ b/crates/jrsonnet-evaluator/src/val.rs
@@ -174,6 +174,7 @@
#[derive(Debug, Clone, Trace)]
#[force_tracking]
pub enum ArrValue {
+ Bytes(#[skip_trace] Rc<[u8]>),
Lazy(Cc<Vec<LazyVal>>),
Eager(Cc<Vec<Val>>),
Extended(Box<(Self, Self)>),
@@ -185,6 +186,7 @@
pub fn len(&self) -> usize {
match self {
+ Self::Bytes(i) => i.len(),
Self::Lazy(l) => l.len(),
Self::Eager(e) => e.len(),
Self::Extended(v) => v.0.len() + v.1.len(),
@@ -197,6 +199,9 @@
pub fn get(&self, index: usize) -> Result<Option<Val>> {
match self {
+ Self::Bytes(i) => i
+ .get(index)
+ .map_or(Ok(None), |v| Ok(Some(Val::Num(*v as f64)))),
Self::Lazy(vec) => {
if let Some(v) = vec.get(index) {
Ok(Some(v.evaluate()?))
@@ -218,6 +223,9 @@
pub fn get_lazy(&self, index: usize) -> Option<LazyVal> {
match self {
+ Self::Bytes(i) => i
+ .get(index)
+ .map(|b| LazyVal::new_resolved(Val::Num(*b as f64))),
Self::Lazy(vec) => vec.get(index).cloned(),
Self::Eager(vec) => vec.get(index).cloned().map(LazyVal::new_resolved),
Self::Extended(v) => {
@@ -233,6 +241,13 @@
pub fn evaluated(&self) -> Result<Cc<Vec<Val>>> {
Ok(match self {
+ Self::Bytes(i) => {
+ let mut out = Vec::with_capacity(i.len());
+ for v in i.iter() {
+ out.push(Val::Num(*v as f64));
+ }
+ Cc::new(out)
+ }
Self::Lazy(vec) => {
let mut out = Vec::with_capacity(vec.len());
for item in vec.iter() {
@@ -253,6 +268,7 @@
pub fn iter(&self) -> impl DoubleEndedIterator<Item = Result<Val>> + '_ {
(0..self.len()).map(move |idx| match self {
+ Self::Bytes(b) => Ok(Val::Num(b[idx] as f64)),
Self::Lazy(l) => l[idx].evaluate(),
Self::Eager(e) => Ok(e[idx].clone()),
Self::Extended(_) => self.get(idx).map(|e| e.unwrap()),
@@ -261,6 +277,7 @@
pub fn iter_lazy(&self) -> impl DoubleEndedIterator<Item = LazyVal> + '_ {
(0..self.len()).map(move |idx| match self {
+ Self::Bytes(b) => LazyVal::new_resolved(Val::Num(b[idx] as f64)),
Self::Lazy(l) => l[idx].clone(),
Self::Eager(e) => LazyVal::new_resolved(e[idx].clone()),
Self::Extended(_) => self.get_lazy(idx).unwrap(),
@@ -269,6 +286,11 @@
pub fn reversed(self) -> Self {
match self {
+ Self::Bytes(b) => {
+ let mut out = b.to_vec();
+ out.reverse();
+ Self::Bytes(out.into())
+ }
Self::Lazy(vec) => {
let mut out = (&vec as &Vec<_>).clone();
out.reverse();
crates/jrsonnet-interner/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-interner/src/lib.rs
+++ b/crates/jrsonnet-interner/src/lib.rs
@@ -4,10 +4,12 @@
use std::{
borrow::Cow,
cell::RefCell,
+ convert::TryFrom,
fmt::{self, Display},
hash::{BuildHasherDefault, Hash, Hasher},
ops::Deref,
rc::Rc,
+ str::Utf8Error,
};
#[derive(Clone, PartialOrd, Ord, Eq)]
@@ -85,6 +87,15 @@
}
}
+impl TryFrom<&[u8]> for IStr {
+ type Error = Utf8Error;
+
+ fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+ let str = std::str::from_utf8(value)?;
+ Ok(str.into())
+ }
+}
+
impl From<String> for IStr {
fn from(str: String) -> Self {
(&str as &str).into()
crates/jrsonnet-parser/src/expr.rsdiffbeforeafterboth--- a/crates/jrsonnet-parser/src/expr.rs
+++ b/crates/jrsonnet-parser/src/expr.rs
@@ -285,6 +285,8 @@
Import(PathBuf),
/// importStr "file.txt"
ImportStr(PathBuf),
+ /// importBin "file.txt"
+ ImportBin(PathBuf),
/// error "I'm broken"
ErrorStmt(LocExpr),
/// a(b, c)
crates/jrsonnet-parser/src/lib.rsdiffbeforeafterboth1#![allow(clippy::redundant_closure_call)]23use peg::parser;4use std::{5 path::{Path, PathBuf},6 rc::Rc,7};8mod expr;9pub use expr::*;10pub use jrsonnet_interner::IStr;11pub use peg;12mod unescape;1314pub struct ParserSettings {15 pub file_name: Rc<Path>,16}1718macro_rules! expr_bin {19 ($a:ident $op:ident $b:ident) => {20 Expr::BinaryOp($a, $op, $b)21 };22}23macro_rules! expr_un {24 ($op:ident $a:ident) => {25 Expr::UnaryOp($op, $a)26 };27}2829parser! {30 grammar jsonnet_parser() for str {31 use peg::ParseLiteral;3233 rule eof() = quiet!{![_]} / expected!("<eof>")34 rule eol() = "\n" / eof()3536 /// Standard C-like comments37 rule comment()38 = "//" (!eol()[_])* eol()39 / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/"40 / "#" (!eol()[_])* eol()4142 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")43 rule _() = quiet!{([' ' | '\r' | '\n' | '\t']+) / comment()}* / expected!("<whitespace>")4445 /// For comma-delimited elements46 rule comma() = quiet!{_ "," _} / expected!("<comma>")47 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}48 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}49 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']50 /// Sequence of digits51 rule uint_str() -> &'input str = a:$(digit()+) { a }52 /// Number in scientific notation format53 rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "<number>") }} / expected!("<number>")5455 /// Reserved word followed by any non-alphanumberic56 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()57 rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")5859 rule keyword(id: &'static str) -> ()60 = ##parse_string_literal(id) end_of_ident()6162 pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) }63 pub rule params(s: &ParserSettings) -> expr::ParamsDesc64 = params:param(s) ** comma() comma()? { expr::ParamsDesc(Rc::new(params)) }65 / { expr::ParamsDesc(Rc::new(Vec::new())) }6667 pub rule arg(s: &ParserSettings) -> (Option<IStr>, LocExpr)68 = quiet! { name:(s:$(id()) _ "=" _ {s})? expr:expr(s) {(name.map(Into::into), expr)} }69 / expected!("<argument>")7071 pub rule args(s: &ParserSettings) -> expr::ArgsDesc72 = args:arg(s)**comma() comma()? {?73 let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();74 let mut unnamed = Vec::with_capacity(unnamed_count);75 let mut named = Vec::with_capacity(args.len() - unnamed_count);76 let mut named_started = false;77 for (name, value) in args {78 if let Some(name) = name {79 named_started = true;80 named.push((name, value));81 } else {82 if named_started {83 return Err("<named argument>")84 }85 unnamed.push(value);86 }87 }88 Ok(expr::ArgsDesc::new(unnamed, named))89 }9091 pub rule bind(s: &ParserSettings) -> expr::BindSpec92 = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}}93 / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}}94 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt95 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }9697 pub rule whole_line() -> &'input str98 = str:$((!['\n'][_])* "\n") {str}99 pub rule string_block() -> String100 = "|||" (!['\n']single_whitespace())* "\n"101 empty_lines:$(['\n']*)102 prefix:[' ' | '\t']+ first_line:whole_line()103 lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*104 [' ' | '\t']*<, {prefix.len() - 1}> "|||"105 {let mut l = empty_lines.to_owned(); l.push_str(first_line); l.extend(lines); l}106107 rule hex_char()108 = quiet! { ['0'..='9' | 'a'..='f' | 'A'..='F'] } / expected!("<hex char>")109110 rule string_char(c: rule<()>)111 = (!['\\']!c()[_])+112 / "\\\\"113 / "\\u" hex_char() hex_char() hex_char() hex_char()114 / "\\x" hex_char() hex_char()115 / ['\\'] (quiet! { ['b' | 'f' | 'n' | 'r' | 't'] / c() } / expected!("<escape character>"))116 pub rule string() -> String117 = ['"'] str:$(string_char(<"\"">)*) ['"'] {? unescape::unescape(str).ok_or("<escaped string>")}118 / ['\''] str:$(string_char(<"\'">)*) ['\''] {? unescape::unescape(str).ok_or("<escaped string>")}119 / quiet!{ "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}120 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}121 / string_block() } / expected!("<string>")122123 pub rule field_name(s: &ParserSettings) -> expr::FieldName124 = name:$(id()) {expr::FieldName::Fixed(name.into())}125 / name:string() {expr::FieldName::Fixed(name.into())}126 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}127 pub rule visibility() -> expr::Visibility128 = ":::" {expr::Visibility::Unhide}129 / "::" {expr::Visibility::Hidden}130 / ":" {expr::Visibility::Normal}131 pub rule field(s: &ParserSettings) -> expr::FieldMember132 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{133 name,134 plus: plus.is_some(),135 params: None,136 visibility,137 value,138 }}139 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{140 name,141 plus: false,142 params: Some(params),143 visibility,144 value,145 }}146 pub rule obj_local(s: &ParserSettings) -> BindSpec147 = keyword("local") _ bind:bind(s) {bind}148 pub rule member(s: &ParserSettings) -> expr::Member149 = bind:obj_local(s) {expr::Member::BindStmt(bind)}150 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}151 / field:field(s) {expr::Member::Field(field)}152 pub rule objinside(s: &ParserSettings) -> expr::ObjBody153 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ plus:"+"? _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {154 let mut compspecs = vec![CompSpec::ForSpec(forspec)];155 compspecs.extend(others.unwrap_or_default());156 expr::ObjBody::ObjComp(expr::ObjComp{157 pre_locals,158 key,159 plus: plus.is_some(),160 value,161 post_locals,162 compspecs,163 })164 }165 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}166 pub rule ifspec(s: &ParserSettings) -> IfSpecData167 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}168 pub rule forspec(s: &ParserSettings) -> ForSpecData169 = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)}170 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>171 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}172 pub rule local_expr(s: &ParserSettings) -> Expr173 = keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }174 pub rule string_expr(s: &ParserSettings) -> Expr175 = s:string() {Expr::Str(s.into())}176 pub rule obj_expr(s: &ParserSettings) -> Expr177 = "{" _ body:objinside(s) _ "}" {Expr::Obj(body)}178 pub rule array_expr(s: &ParserSettings) -> Expr179 = "[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}180 pub rule array_comp_expr(s: &ParserSettings) -> Expr181 = "[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {182 let mut specs = vec![CompSpec::ForSpec(forspec)];183 specs.extend(others.unwrap_or_default());184 Expr::ArrComp(expr, specs)185 }186 pub rule number_expr(s: &ParserSettings) -> Expr187 = n:number() { expr::Expr::Num(n) }188 pub rule var_expr(s: &ParserSettings) -> Expr189 = n:$(id()) { expr::Expr::Var(n.into()) }190 pub rule id_loc(s: &ParserSettings) -> LocExpr191 = a:position!() n:$(id()) b:position!() { LocExpr(Rc::new(expr::Expr::Str(n.into())), ExprLocation(s.file_name.clone(), a,b)) }192 pub rule if_then_else_expr(s: &ParserSettings) -> Expr193 = cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{194 cond,195 cond_then,196 cond_else,197 }}198199 pub rule literal(s: &ParserSettings) -> Expr200 = v:(201 keyword("null") {LiteralType::Null}202 / keyword("true") {LiteralType::True}203 / keyword("false") {LiteralType::False}204 / keyword("self") {LiteralType::This}205 / keyword("$") {LiteralType::Dollar}206 / keyword("super") {LiteralType::Super}207 ) {Expr::Literal(v)}208209 pub rule expr_basic(s: &ParserSettings) -> Expr210 = literal(s)211212 / quiet!{"$intrinsic(" name:$(id()) ")" {Expr::Intrinsic(name.into())}}213214 / string_expr(s) / number_expr(s)215 / array_expr(s)216 / obj_expr(s)217 / array_expr(s)218 / array_comp_expr(s)219220 / keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}221 / keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}222223 / var_expr(s)224 / local_expr(s)225 / if_then_else_expr(s)226227 / keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}228 / assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }229230 / keyword("error") _ expr:expr(s) { Expr::ErrorStmt(expr) }231232 rule slice_part(s: &ParserSettings) -> Option<LocExpr>233 = e:(_ e:expr(s) _{e})? {e}234 pub rule slice_desc(s: &ParserSettings) -> SliceDesc235 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {236 let (end, step) = if let Some((end, step)) = pair {237 (end, step)238 }else{239 (None, None)240 };241242 SliceDesc { start, end, step }243 }244245 rule binop(x: rule<()>) -> ()246 = quiet!{ x() } / expected!("<binary op>")247 rule unaryop(x: rule<()>) -> ()248 = quiet!{ x() } / expected!("<unary op>")249250251 use BinaryOpType::*;252 use UnaryOpType::*;253 rule expr(s: &ParserSettings) -> LocExpr254 = precedence! {255 start:position!() v:@ end:position!() { LocExpr(Rc::new(v), ExprLocation(s.file_name.clone(), start, end)) }256 --257 a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}258 --259 a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}260 --261 a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}262 --263 a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}264 --265 a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}266 --267 a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}268 a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}269 --270 a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}271 a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}272 a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}273 a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}274 a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}275 --276 a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}277 a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}278 --279 a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}280 a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}281 --282 a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}283 a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}284 a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}285 --286 unaryop(<"-">) _ b:@ {expr_un!(Minus b)}287 unaryop(<"!">) _ b:@ {expr_un!(Not b)}288 unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}289 --290 a:(@) _ "[" _ e:slice_desc(s) _ "]" {Expr::Slice(a, e)}291 a:(@) _ "." _ a:position!() e:id_loc(s) b:position!() {Expr::Index(a, e)}292 a:(@) _ "[" _ e:expr(s) _ "]" {Expr::Index(a, e)}293 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {Expr::Apply(a, args, ts.is_some())}294 a:(@) _ "{" _ body:objinside(s) _ "}" {Expr::ObjExtend(a, body)}295 --296 e:expr_basic(s) {e}297 "(" _ e:expr(s) _ ")" {Expr::Parened(e)}298 }299300 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}301 }302}303304pub type ParseError = peg::error::ParseError<peg::str::LineCol>;305pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {306 jsonnet_parser::jsonnet(str, settings)307}308/// Used for importstr values309pub fn string_to_expr(str: IStr, settings: &ParserSettings) -> LocExpr {310 let len = str.len();311 LocExpr(312 Rc::new(Expr::Str(str)),313 ExprLocation(settings.file_name.clone(), 0, len),314 )315}316317#[cfg(test)]318pub mod tests {319 use super::{expr::*, parse};320 use crate::ParserSettings;321 use std::path::PathBuf;322 use BinaryOpType::*;323324 macro_rules! parse {325 ($s:expr) => {326 parse(327 $s,328 &ParserSettings {329 file_name: PathBuf::from("test.jsonnet").into(),330 },331 )332 .unwrap()333 };334 }335336 macro_rules! el {337 ($expr:expr, $from:expr, $to:expr$(,)?) => {338 LocExpr(339 std::rc::Rc::new($expr),340 ExprLocation(PathBuf::from("test.jsonnet").into(), $from, $to),341 )342 };343 }344345 #[test]346 fn multiline_string() {347 assert_eq!(348 parse!("|||\n Hello world!\n a\n|||"),349 el!(Expr::Str("Hello world!\n a\n".into()), 0, 31),350 );351 assert_eq!(352 parse!("|||\n Hello world!\n a\n|||"),353 el!(Expr::Str("Hello world!\n a\n".into()), 0, 27),354 );355 assert_eq!(356 parse!("|||\n\t\tHello world!\n\t\t\ta\n|||"),357 el!(Expr::Str("Hello world!\n\ta\n".into()), 0, 27),358 );359 assert_eq!(360 parse!("|||\n Hello world!\n a\n |||"),361 el!(Expr::Str("Hello world!\n a\n".into()), 0, 30),362 );363 }364365 #[test]366 fn slice() {367 parse!("a[1:]");368 parse!("a[1::]");369 parse!("a[:1:]");370 parse!("a[::1]");371 parse!("str[:len - 1]");372 }373374 #[test]375 fn string_escaping() {376 assert_eq!(377 parse!(r#""Hello, \"world\"!""#),378 el!(Expr::Str(r#"Hello, "world"!"#.into()), 0, 19),379 );380 assert_eq!(381 parse!(r#"'Hello \'world\'!'"#),382 el!(Expr::Str("Hello 'world'!".into()), 0, 18),383 );384 assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into()), 0, 6));385 }386387 #[test]388 fn string_unescaping() {389 assert_eq!(390 parse!(r#""Hello\nWorld""#),391 el!(Expr::Str("Hello\nWorld".into()), 0, 14),392 );393 }394395 #[test]396 fn string_verbantim() {397 assert_eq!(398 parse!(r#"@"Hello\n""World""""#),399 el!(Expr::Str("Hello\\n\"World\"".into()), 0, 19),400 );401 }402403 #[test]404 fn imports() {405 assert_eq!(406 parse!("import \"hello\""),407 el!(Expr::Import(PathBuf::from("hello")), 0, 14),408 );409 assert_eq!(410 parse!("importstr \"garnish.txt\""),411 el!(Expr::ImportStr(PathBuf::from("garnish.txt")), 0, 23)412 );413 }414415 #[test]416 fn empty_object() {417 assert_eq!(418 parse!("{}"),419 el!(Expr::Obj(ObjBody::MemberList(vec![])), 0, 2)420 );421 }422423 #[test]424 fn basic_math() {425 assert_eq!(426 parse!("2+2*2"),427 el!(428 Expr::BinaryOp(429 el!(Expr::Num(2.0), 0, 1),430 Add,431 el!(432 Expr::BinaryOp(el!(Expr::Num(2.0), 2, 3), Mul, el!(Expr::Num(2.0), 4, 5)),433 2,434 5435 )436 ),437 0,438 5439 )440 );441 }442443 #[test]444 fn basic_math_with_indents() {445 assert_eq!(446 parse!("2 + 2 * 2 "),447 el!(448 Expr::BinaryOp(449 el!(Expr::Num(2.0), 0, 1),450 Add,451 el!(452 Expr::BinaryOp(el!(Expr::Num(2.0), 7, 8), Mul, el!(Expr::Num(2.0), 13, 14),),453 7,454 14455 ),456 ),457 0,458 14459 )460 );461 }462463 #[test]464 fn basic_math_parened() {465 assert_eq!(466 parse!("2+(2+2*2)"),467 el!(468 Expr::BinaryOp(469 el!(Expr::Num(2.0), 0, 1),470 Add,471 el!(472 Expr::Parened(el!(473 Expr::BinaryOp(474 el!(Expr::Num(2.0), 3, 4),475 Add,476 el!(477 Expr::BinaryOp(478 el!(Expr::Num(2.0), 5, 6),479 Mul,480 el!(Expr::Num(2.0), 7, 8),481 ),482 5,483 8484 ),485 ),486 3,487 8488 )),489 2,490 9491 ),492 ),493 0,494 9495 )496 );497 }498499 /// Comments should not affect parsing500 #[test]501 fn comments() {502 assert_eq!(503 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),504 el!(505 Expr::BinaryOp(506 el!(Expr::Num(2.0), 0, 1),507 Add,508 el!(509 Expr::BinaryOp(510 el!(Expr::Num(3.0), 22, 23),511 Mul,512 el!(Expr::Num(4.0), 40, 41)513 ),514 22,515 41516 )517 ),518 0,519 41520 )521 );522 }523524 /// Comments should be able to be escaped525 #[test]526 fn comment_escaping() {527 assert_eq!(528 parse!("2/*\\*/+*/ - 22"),529 el!(530 Expr::BinaryOp(el!(Expr::Num(2.0), 0, 1), Sub, el!(Expr::Num(22.0), 12, 14)),531 0,532 14533 )534 );535 }536537 #[test]538 fn suffix() {539 // assert_eq!(parse!("std.test"), el!(Expr::Num(2.2)));540 // assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2)));541 // assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2)));542 // assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2)))543 }544545 #[test]546 fn array_comp() {547 use Expr::*;548 /*549 `ArrComp(Apply(Index(Var("std") from "test.jsonnet":1-4, Var("deepJoin") from "test.jsonnet":5-13) from "test.jsonnet":1-13, ArgsDesc { unnamed: [Var("x") from "test.jsonnet":14-15], named: [] }, false) from "test.jsonnet":1-16, [ForSpec(ForSpecData("x", Var("arr") from "test.jsonnet":26-29))]) from "test.jsonnet":0-30`,550 `ArrComp(Apply(Index(Var("std") from "test.jsonnet":1-4, Str("deepJoin") from "test.jsonnet":5-13) from "test.jsonnet":1-13, ArgsDesc { unnamed: [Var("x") from "test.jsonnet":14-15], named: [] }, false) from "test.jsonnet":1-16, [ForSpec(ForSpecData("x", Var("arr") from "test.jsonnet":26-29))]) from "test.jsonnet":0-30`551 */552 assert_eq!(553 parse!("[std.deepJoin(x) for x in arr]"),554 el!(555 ArrComp(556 el!(557 Apply(558 el!(559 Index(560 el!(Var("std".into()), 1, 4),561 el!(Str("deepJoin".into()), 5, 13)562 ),563 1,564 13565 ),566 ArgsDesc::new(vec![el!(Var("x".into()), 14, 15)], vec![]),567 false,568 ),569 1,570 16571 ),572 vec![CompSpec::ForSpec(ForSpecData(573 "x".into(),574 el!(Var("arr".into()), 26, 29)575 ))]576 ),577 0,578 30579 ),580 )581 }582583 #[test]584 fn reserved() {585 use Expr::*;586 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null), 0, 4));587 assert_eq!(parse!("nulla"), el!(Var("nulla".into()), 0, 5));588 }589590 #[test]591 fn multiple_args_buf() {592 parse!("a(b, null_fields)");593 }594595 #[test]596 fn infix_precedence() {597 use Expr::*;598 assert_eq!(599 parse!("!a && !b"),600 el!(601 BinaryOp(602 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()), 1, 2)), 0, 2),603 And,604 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()), 7, 8)), 6, 8)605 ),606 0,607 8608 )609 );610 }611612 #[test]613 fn infix_precedence_division() {614 use Expr::*;615 assert_eq!(616 parse!("!a / !b"),617 el!(618 BinaryOp(619 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()), 1, 2)), 0, 2),620 Div,621 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()), 6, 7)), 5, 7)622 ),623 0,624 7625 )626 );627 }628629 #[test]630 fn double_negation() {631 use Expr::*;632 assert_eq!(633 parse!("!!a"),634 el!(635 UnaryOp(636 UnaryOpType::Not,637 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()), 2, 3)), 1, 3)638 ),639 0,640 3641 )642 )643 }644645 #[test]646 fn array_test_error() {647 parse!("[a for a in b if c for e in f]");648 // ^^^^ failed code649 }650651 #[test]652 fn missing_newline_between_comment_and_eof() {653 parse!(654 "{a:1}655656 //+213"657 );658 }659660 #[test]661 fn default_param_before_nondefault() {662 parse!("local x(foo = 'foo', bar) = null; null");663 }664665 #[test]666 fn can_parse_stdlib() {667 parse!(jrsonnet_stdlib::STDLIB_STR);668 }669670 #[test]671 fn add_location_info_to_all_sub_expressions() {672 use Expr::*;673674 let file_name: std::rc::Rc<std::path::Path> = PathBuf::from("test.jsonnet").into();675 let expr = parse(676 "{} { local x = 1, x: x } + {}",677 &ParserSettings {678 file_name: file_name.clone(),679 },680 )681 .unwrap();682 assert_eq!(683 expr,684 el!(685 BinaryOp(686 el!(687 ObjExtend(688 el!(Obj(ObjBody::MemberList(vec![])), 0, 2),689 ObjBody::MemberList(vec![690 Member::BindStmt(BindSpec {691 name: "x".into(),692 params: None,693 value: el!(Num(1.0), 15, 16)694 }),695 Member::Field(FieldMember {696 name: FieldName::Fixed("x".into()),697 plus: false,698 params: None,699 visibility: Visibility::Normal,700 value: el!(Var("x".into()), 21, 22),701 })702 ])703 ),704 0,705 24706 ),707 BinaryOpType::Add,708 el!(Obj(ObjBody::MemberList(vec![])), 27, 29),709 ),710 0,711 29712 ),713 );714 }715 // From source code716 /*717 #[bench]718 fn bench_parse_peg(b: &mut Bencher) {719 b.iter(|| parse!(jrsonnet_stdlib::STDLIB_STR))720 }721 */722}