From 1e470a761e6e408669718a1f5c4a0911fd2767cf Mon Sep 17 00:00:00 2001 From: Лач Date: Sat, 27 Jun 2020 13:14:40 +0000 Subject: [PATCH] chore: rename crates for publish --- --- a/Cargo.lock +++ b/Cargo.lock @@ -127,29 +127,29 @@ "annotate-snippets", "clap", "jemallocator", - "jsonnet-evaluator", - "jsonnet-parser", + "jrsonnet-evaluator", + "jrsonnet-parser", ] [[package]] -name = "jsonnet-evaluator" +name = "jrsonnet-evaluator" version = "0.1.0" dependencies = [ "bincode", "closure", "indexmap", - "jsonnet-parser", - "jsonnet-stdlib", + "jrsonnet-parser", + "jrsonnet-stdlib", "md5", "serde", "structdump", ] [[package]] -name = "jsonnet-parser" +name = "jrsonnet-parser" version = "0.1.0" dependencies = [ - "jsonnet-stdlib", + "jrsonnet-stdlib", "peg", "serde", "structdump", @@ -158,7 +158,7 @@ ] [[package]] -name = "jsonnet-stdlib" +name = "jrsonnet-stdlib" version = "0.1.0" [[package]] --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [workspace] members = [ - "crates/jsonnet-parser", - "crates/jsonnet-evaluator", - "crates/jsonnet-stdlib", + "crates/jrsonnet-parser", + "crates/jrsonnet-evaluator", + "crates/jrsonnet-stdlib", "cmds/jrsonnet" ] --- a/cmds/jrsonnet/Cargo.toml +++ b/cmds/jrsonnet/Cargo.toml @@ -7,8 +7,8 @@ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -jsonnet-evaluator = { path = "../../crates/jsonnet-evaluator" } -jsonnet-parser = { path = "../../crates/jsonnet-parser" } +jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator" } +jrsonnet-parser = { path = "../../crates/jrsonnet-parser" } annotate-snippets = "0.8.0" jemallocator = "0.3.2" --- a/cmds/jrsonnet/src/main.rs +++ b/cmds/jrsonnet/src/main.rs @@ -1,7 +1,7 @@ pub mod location; use clap::Clap; -use jsonnet_evaluator::{EvaluationSettings, EvaluationState, LocError, StackTrace, Val}; +use jrsonnet_evaluator::{EvaluationSettings, EvaluationState, LocError, StackTrace, Val}; use jsonnet_parser::{el, Arg, ArgsDesc, Expr, LocExpr, ParserSettings}; use location::{offset_to_location, CodeLocation}; use std::env::current_dir; @@ -118,12 +118,12 @@ fn main() { let opts: Opts = Opts::parse(); - let evaluator = jsonnet_evaluator::EvaluationState::new( + let evaluator = jrsonnet_evaluator::EvaluationState::new( EvaluationSettings { max_stack_trace_size: opts.max_trace, max_stack_frames: opts.max_stack, }, - Box::new(jsonnet_evaluator::FileImportResolver { + Box::new(jrsonnet_evaluator::FileImportResolver { library_paths: opts.jpath.clone(), }), ); --- /dev/null +++ b/crates/jrsonnet-evaluator/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "jrsonnet-evaluator" +version = "0.1.0" +authors = ["Лач "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["serialized-stdlib", "faster"] +# Serializes standard library AST instead of parsing them every run +serialized-stdlib = ["serde", "bincode", "jrsonnet-parser/deserialize"] +# Same as above, but with generated code instead of serde. Reduces memory usage, but increases binary size and compilation time +codegenerated-stdlib = [] +# Replace some standard library functions with faster implementations (I.e manifestJsonEx) +# Library works fine without this feature, but requires more memory and time for std function calls +faster = [] + +[dependencies] +jrsonnet-parser = { path = "../jrsonnet-parser" } +closure = "0.3.0" +jrsonnet-stdlib = { path = "../jrsonnet-stdlib" } +indexmap = "1.4.0" +md5 = "0.7.0" + +serde = { version = "1.0.114", optional = true } +bincode = { version = "1.3.1", optional = true } + +[build-dependencies] +jrsonnet-parser = { path = "../jrsonnet-parser", features = ["dump", "serialize", "deserialize"] } +jrsonnet-stdlib = { path = "../jrsonnet-stdlib" } +structdump = "0.1.2" +serde = "1.0.114" +bincode = "1.3.1" --- /dev/null +++ b/crates/jrsonnet-evaluator/README.md @@ -0,0 +1,40 @@ +# jrsonnet-evaluator + +Interpreter for parsed jsonnet tree + +## Standard library + +jsonnet stdlib is embedded into evaluator, but there is different modes for this: + +- `codegenerated-stdlib` + - generates source code for reproducing stdlib AST ([Example](https://gist.githubusercontent.com/CertainLach/7b3149df556f3406f5e9368aaa9f32ec/raw/0c80d8ab9aa7b9288c6219a2779cb2ab37287669/a.rs)) + - fastest on interpretation, slowest on compilation (it takes more than 5 minutes to optimize them by llvm) +- `serialized-stdlib` + - serializes standard library AST using serde + - slower than `codegenerated-stdlib` at runtime, but have no compilation speed penality +- none + - leaves only stdlib source code in binary, processing them same way as user supplied data + - slowest (as it involves parsing of standard library source code) + +Because of `codegenerated-stdlib` compilation slowdown, `serialized-stdlib` is used by default + +### Benchmark + +Can also be run via `cargo bench` + +```md +# codegenerated-stdlib +test tests::bench_codegen ... bench: 401,696 ns/iter (+/- 38,521) +# serialized-stdlib +test tests::bench_serialize ... bench: 1,763,999 ns/iter (+/- 76,211) +# none +test tests::bench_parse ... bench: 7,206,164 ns/iter (+/- 1,067,418) +``` + +## Intristics + +Some functions from stdlib are implemented as intristics + +### Intristic handling + +If indexed jsonnet object has field '__intristic_namespace__' of type 'string', then any not found field/method is resolved as `Val::Intristic(__intristic_namespace__, name)` --- /dev/null +++ b/crates/jrsonnet-evaluator/build.rs @@ -0,0 +1,65 @@ +use bincode::serialize; +use jrsonnet_parser::{ + parse, Expr, FieldMember, FieldName, LocExpr, Member, ObjBody, ParserSettings, +}; +use jrsonnet_stdlib::STDLIB_STR; +use std::{ + env, + fs::File, + io::Write, + path::{Path, PathBuf}, + rc::Rc, +}; +use structdump::CodegenResult; + +fn main() { + let parsed = parse( + STDLIB_STR, + &ParserSettings { + file_name: Rc::new(PathBuf::from("std.jsonnet")), + loc_data: true, + }, + ) + .expect("parse"); + + let parsed = if cfg!(feature = "faster") { + let LocExpr(expr, location) = parsed; + LocExpr( + Rc::new(match Rc::try_unwrap(expr).unwrap() { + Expr::Obj(ObjBody::MemberList(members)) => Expr::Obj(ObjBody::MemberList( + members + .into_iter() + .filter(|p| { + !matches!( + p, + Member::Field(FieldMember { + name: FieldName::Fixed(name), + .. + }) if **name == *"join" || **name == *"manifestJsonEx" || **name == *"escapeStringJson" + ) + }) + .collect(), + )), + _ => panic!("std value should be object"), + }), + location, + ) + } else { + parsed + }; + { + let mut codegen = CodegenResult::default(); + let code = codegen.codegen(&parsed); + + let out_dir = env::var("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("stdlib.rs"); + let mut f = File::create(&dest_path).unwrap(); + f.write_all(&code.as_bytes()).unwrap(); + } + { + let out_dir = env::var("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("stdlib.bincode"); + let mut f = File::create(&dest_path).unwrap(); + f.write_all(&serialize(&parsed).unwrap()).unwrap(); + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/ctx.rs @@ -0,0 +1,131 @@ +use crate::{ + create_error, future_wrapper, map::LayeredHashMap, rc_fn_helper, resolved_lazy_val, Error, + LazyBinding, LazyVal, ObjValue, Result, Val, +}; +use std::{cell::RefCell, collections::HashMap, fmt::Debug, rc::Rc}; + +rc_fn_helper!( + ContextCreator, + context_creator, + dyn Fn(Option, Option) -> Result +); + +future_wrapper!(Context, FutureContext); + +struct ContextInternals { + dollar: Option, + this: Option, + super_obj: Option, + bindings: LayeredHashMap, LazyVal>, +} +pub struct Context(Rc); +impl Debug for Context { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Context") + .field("this", &self.0.this.as_ref().map(|e| Rc::as_ptr(&e.0))) + .field("bindings", &self.0.bindings) + .finish() + } +} +impl Context { + pub fn new_future() -> FutureContext { + FutureContext(Rc::new(RefCell::new(None))) + } + + pub fn dollar(&self) -> &Option { + &self.0.dollar + } + + pub fn this(&self) -> &Option { + &self.0.this + } + + pub fn super_obj(&self) -> &Option { + &self.0.super_obj + } + + pub fn new() -> Context { + Context(Rc::new(ContextInternals { + dollar: None, + this: None, + super_obj: None, + bindings: LayeredHashMap::default(), + })) + } + + pub fn binding(&self, name: Rc) -> Result { + self.0 + .bindings + .get(&name) + .cloned() + .ok_or_else(|| create_error(Error::UnknownVariable(name))) + } + pub fn into_future(self, ctx: FutureContext) -> Context { + { + ctx.0.borrow_mut().replace(self); + } + ctx.unwrap() + } + + pub fn with_var(&self, name: Rc, value: Val) -> Result { + let mut new_bindings = HashMap::with_capacity(1); + new_bindings.insert(name, resolved_lazy_val!(value)); + self.extend(new_bindings, None, None, None) + } + + pub fn extend( + &self, + new_bindings: HashMap, LazyVal>, + new_dollar: Option, + new_this: Option, + new_super_obj: Option, + ) -> Result { + let dollar = new_dollar.or_else(|| self.0.dollar.clone()); + let this = new_this.or_else(|| self.0.this.clone()); + let super_obj = new_super_obj.or_else(|| self.0.super_obj.clone()); + let bindings = if new_bindings.is_empty() { + self.0.bindings.clone() + } else { + self.0.bindings.extend(new_bindings) + }; + Ok(Context(Rc::new(ContextInternals { + dollar, + this, + super_obj, + bindings, + }))) + } + pub fn extend_unbound( + &self, + new_bindings: HashMap, LazyBinding>, + new_dollar: Option, + new_this: Option, + new_super_obj: Option, + ) -> Result { + let this = new_this.or_else(|| self.0.this.clone()); + let super_obj = new_super_obj.or_else(|| self.0.super_obj.clone()); + let mut new = HashMap::with_capacity(new_bindings.len()); + for (k, v) in new_bindings.into_iter() { + new.insert(k, v.evaluate(this.clone(), super_obj.clone())?); + } + self.extend(new, new_dollar, this, super_obj) + } +} + +impl Default for Context { + fn default() -> Self { + Self::new() + } +} + +impl PartialEq for Context { + fn eq(&self, other: &Self) -> bool { + Rc::ptr_eq(&self.0, &other.0) + } +} + +impl Clone for Context { + fn clone(&self) -> Self { + Context(self.0.clone()) + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/dynamic.rs @@ -0,0 +1,55 @@ +#[macro_export] +macro_rules! future_wrapper { + ($orig: ty, $wrapper: ident) => { + #[derive(Debug, Clone)] + pub struct $wrapper(pub std::rc::Rc>>); + impl $wrapper { + pub fn unwrap(self) -> $orig { + self.0.borrow().as_ref().map(|e| e.clone()).unwrap() + } + pub fn new() -> Self { + $wrapper(std::rc::Rc::new(std::cell::RefCell::new(None))) + } + pub fn fill(self, val: $orig) -> $orig { + if self.0.borrow().is_some() { + panic!("wrapper is filled already"); + } + { + self.0.borrow_mut().replace(val); + } + self.unwrap() + } + } + impl Default for $wrapper { + fn default() -> Self { + Self::new() + } + } + }; +} + +#[macro_export] +macro_rules! rc_fn_helper { + ($name: ident, $macro_name: ident, $fn: ty) => { + #[derive(Clone)] + #[doc = "Function wrapper"] + pub struct $name(pub std::rc::Rc<$fn>); + impl std::fmt::Debug for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct(std::stringify!($name)).finish() + } + } + impl std::cmp::PartialEq for $name { + fn eq(&self, other: &$name) -> bool { + std::ptr::eq(&self.0, &other.0) + } + } + #[doc = "Macro to ease wrapper creation"] + #[macro_export] + macro_rules! $macro_name { + ($val: expr) => { + $crate::$name(std::rc::Rc::new($val)) + }; + } + }; +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/error.rs @@ -0,0 +1,64 @@ +use crate::{Val, ValType}; +use jrsonnet_parser::{BinaryOpType, ExprLocation, UnaryOpType}; +use std::{path::PathBuf, rc::Rc}; + +#[derive(Debug, Clone)] +pub enum Error { + IntristicNotFound(Rc, Rc), + IntristicArgumentReorderingIsNotSupportedYet, + + UnaryOperatorDoesNotOperateOnType(UnaryOpType, ValType), + BinaryOperatorDoesNotOperateOnValues(BinaryOpType, ValType, ValType), + + NoTopLevelObjectFound, + CantUseSelfOutsideOfObject, + CantUseSuperOutsideOfObject, + + InComprehensionCanOnlyIterateOverArray, + + ArrayBoundsError(usize, usize), + + AssertionFailed(Val), + + VariableIsNotDefined(String), + TypeMismatch(&'static str, Vec, ValType), + NoSuchField(Rc), + + UnknownVariable(Rc), + + OnlyFunctionsCanBeCalledGot(ValType), + UnknownFunctionParameter(String), + BindingParameterASecondTime(Rc), + TooManyArgsFunctionHas(usize), + FunctionParameterNotBoundInCall(Rc), + + UndefinedExternalVariable(Rc), + + FieldMustBeStringGot(ValType), + + AttemptedIndexAnArrayWithString(Rc), + ValueIndexMustBeTypeGot(ValType, ValType, ValType), + CantIndexInto(ValType), + + StandaloneSuper, + + ImportFileNotFound(PathBuf, PathBuf), + ResolvedFileNotFound(PathBuf), + ImportBadFileUtf8(PathBuf), + ImportNotSupported(PathBuf, PathBuf), + ImportSyntaxError(jrsonnet_parser::ParseError), + + RuntimeError(Rc), + StackOverflow, + FractionalIndex, + DivisionByZero, +} + +#[derive(Clone, Debug)] +pub struct StackTraceElement(pub ExprLocation, pub String); +#[derive(Debug, Clone)] +pub struct StackTrace(pub Vec); + +#[derive(Debug, Clone)] +pub struct LocError(pub Error, pub StackTrace); +pub type Result = std::result::Result; --- /dev/null +++ b/crates/jrsonnet-evaluator/src/evaluate.rs @@ -0,0 +1,802 @@ +use crate::{ + context_creator, create_error, create_error_result, escape_string_json, future_wrapper, + lazy_val, manifest_json_ex, parse_args, push, with_state, Context, ContextCreator, Error, + FuncDesc, LazyBinding, LazyVal, ObjMember, ObjValue, Result, Val, ValType, +}; +use closure::closure; +use jrsonnet_parser::{ + AssertStmt, BinaryOpType, BindSpec, CompSpec, Expr, FieldMember, ForSpecData, IfSpecData, + LiteralType, LocExpr, Member, ObjBody, ParamsDesc, UnaryOpType, Visibility, +}; +use std::{ + collections::{BTreeMap, HashMap}, + rc::Rc, +}; + +pub fn evaluate_binding(b: &BindSpec, context_creator: ContextCreator) -> (Rc, LazyBinding) { + let b = b.clone(); + if let Some(params) = &b.params { + let params = params.clone(); + ( + b.name.clone(), + LazyBinding::Bindable(Rc::new(move |this, super_obj| { + Ok(lazy_val!( + closure!(clone b, clone params, clone context_creator, || Ok(evaluate_method( + context_creator.0(this.clone(), super_obj.clone())?, + params.clone(), + b.value.clone(), + ))) + )) + })), + ) + } else { + ( + b.name.clone(), + LazyBinding::Bindable(Rc::new(move |this, super_obj| { + Ok(lazy_val!(closure!(clone context_creator, clone b, || + push(&b.value.1, "thunk", ||{ + evaluate( + context_creator.0(this.clone(), super_obj.clone())?, + &b.value + ) + }) + ))) + })), + ) + } +} + +pub fn evaluate_method(ctx: Context, params: ParamsDesc, body: LocExpr) -> Val { + Val::Func(FuncDesc { ctx, params, body }) +} + +pub fn evaluate_field_name( + context: Context, + field_name: &jrsonnet_parser::FieldName, +) -> Result>> { + Ok(match field_name { + jrsonnet_parser::FieldName::Fixed(n) => Some(n.clone()), + jrsonnet_parser::FieldName::Dyn(expr) => { + let lazy = evaluate(context, expr)?; + let value = lazy.unwrap_if_lazy()?; + if matches!(value, Val::Null) { + None + } else { + Some(value.try_cast_str("dynamic field name")?) + } + } + }) +} + +pub fn evaluate_unary_op(op: UnaryOpType, b: &Val) -> Result { + Ok(match (op, b) { + (o, Val::Lazy(l)) => evaluate_unary_op(o, &l.evaluate()?)?, + (UnaryOpType::Not, Val::Bool(v)) => Val::Bool(!v), + (UnaryOpType::Minus, Val::Num(n)) => Val::Num(-*n), + (UnaryOpType::BitNot, Val::Num(n)) => Val::Num(!(*n as i32) as f64), + (op, o) => create_error_result(Error::UnaryOperatorDoesNotOperateOnType( + op, + o.value_type()?, + ))?, + }) +} + +pub(crate) fn evaluate_add_op(a: &Val, b: &Val) -> Result { + Ok(match (a, b) { + (Val::Str(v1), Val::Str(v2)) => Val::Str(((**v1).to_owned() + &v2).into()), + + // Can't use generic json serialization way, because it depends on number to string concatenation (std.jsonnet:890) + (Val::Num(n), Val::Str(o)) => Val::Str(format!("{}{}", n, o).into()), + (Val::Str(o), Val::Num(n)) => Val::Str(format!("{}{}", o, n).into()), + + (Val::Str(s), o) => Val::Str(format!("{}{}", s, o.clone().into_json(0)?).into()), + (o, Val::Str(s)) => Val::Str(format!("{}{}", o.clone().into_json(0)?, s).into()), + + (Val::Obj(v1), Val::Obj(v2)) => Val::Obj(v2.with_super(v1.clone())), + (Val::Arr(a), Val::Arr(b)) => Val::Arr(Rc::new([&a[..], &b[..]].concat())), + (Val::Num(v1), Val::Num(v2)) => Val::Num(v1 + v2), + _ => create_error_result(Error::BinaryOperatorDoesNotOperateOnValues( + BinaryOpType::Add, + a.value_type()?, + b.value_type()?, + ))?, + }) +} + +pub fn evaluate_binary_op_special( + context: Context, + a: &LocExpr, + op: BinaryOpType, + b: &LocExpr, +) -> Result { + Ok( + match (evaluate(context.clone(), &a)?.unwrap_if_lazy()?, op, b) { + (Val::Bool(true), BinaryOpType::Or, _o) => Val::Bool(true), + (Val::Bool(false), BinaryOpType::And, _o) => Val::Bool(false), + (a, op, eb) => { + evaluate_binary_op_normal(&a, op, &evaluate(context, eb)?.unwrap_if_lazy()?)? + } + }, + ) +} + +pub fn evaluate_binary_op_normal(a: &Val, op: BinaryOpType, b: &Val) -> Result { + Ok(match (a, op, b) { + (a, BinaryOpType::Add, b) => evaluate_add_op(a, b)?, + + (Val::Str(v1), BinaryOpType::Mul, Val::Num(v2)) => Val::Str(v1.repeat(*v2 as usize).into()), + + // Bool X Bool + (Val::Bool(a), BinaryOpType::And, Val::Bool(b)) => Val::Bool(*a && *b), + (Val::Bool(a), BinaryOpType::Or, Val::Bool(b)) => Val::Bool(*a || *b), + + // Str X Str + (Val::Str(v1), BinaryOpType::Lt, Val::Str(v2)) => Val::Bool(v1 < v2), + (Val::Str(v1), BinaryOpType::Gt, Val::Str(v2)) => Val::Bool(v1 > v2), + (Val::Str(v1), BinaryOpType::Lte, Val::Str(v2)) => Val::Bool(v1 <= v2), + (Val::Str(v1), BinaryOpType::Gte, Val::Str(v2)) => Val::Bool(v1 >= v2), + + // Num X Num + (Val::Num(v1), BinaryOpType::Mul, Val::Num(v2)) => Val::Num(v1 * v2), + (Val::Num(v1), BinaryOpType::Div, Val::Num(v2)) => { + if *v2 <= f64::EPSILON { + create_error_result(crate::Error::DivisionByZero)? + } + Val::Num(v1 / v2) + } + + (Val::Num(v1), BinaryOpType::Sub, Val::Num(v2)) => Val::Num(v1 - v2), + + (Val::Num(v1), BinaryOpType::Lt, Val::Num(v2)) => Val::Bool(v1 < v2), + (Val::Num(v1), BinaryOpType::Gt, Val::Num(v2)) => Val::Bool(v1 > v2), + (Val::Num(v1), BinaryOpType::Lte, Val::Num(v2)) => Val::Bool(v1 <= v2), + (Val::Num(v1), BinaryOpType::Gte, Val::Num(v2)) => Val::Bool(v1 >= v2), + + (Val::Num(v1), BinaryOpType::BitAnd, Val::Num(v2)) => { + Val::Num(((*v1 as i32) & (*v2 as i32)) as f64) + } + (Val::Num(v1), BinaryOpType::BitOr, Val::Num(v2)) => { + Val::Num(((*v1 as i32) | (*v2 as i32)) as f64) + } + (Val::Num(v1), BinaryOpType::BitXor, Val::Num(v2)) => { + Val::Num(((*v1 as i32) ^ (*v2 as i32)) as f64) + } + (Val::Num(v1), BinaryOpType::Lhs, Val::Num(v2)) => { + Val::Num(((*v1 as i32) << (*v2 as i32)) as f64) + } + (Val::Num(v1), BinaryOpType::Rhs, Val::Num(v2)) => { + Val::Num(((*v1 as i32) >> (*v2 as i32)) as f64) + } + + _ => create_error_result(Error::BinaryOperatorDoesNotOperateOnValues( + op, + a.value_type()?, + b.value_type()?, + ))?, + }) +} + +future_wrapper!(HashMap, LazyBinding>, FutureNewBindings); +future_wrapper!(ObjValue, FutureObjValue); + +pub fn evaluate_comp( + context: Context, + value: &impl Fn(Context) -> Result, + specs: &[CompSpec], +) -> Result>> { + Ok(match specs.get(0) { + None => Some(vec![value(context)?]), + Some(CompSpec::IfSpec(IfSpecData(cond))) => { + if evaluate(context.clone(), &cond)?.try_cast_bool("if spec")? { + evaluate_comp(context, value, &specs[1..])? + } else { + None + } + } + Some(CompSpec::ForSpec(ForSpecData(var, expr))) => { + match evaluate(context.clone(), &expr)?.unwrap_if_lazy()? { + Val::Arr(list) => { + let mut out = Vec::new(); + for item in list.iter() { + let item = item.unwrap_if_lazy()?; + out.push(evaluate_comp( + context.with_var(var.clone(), item.clone())?, + value, + &specs[1..], + )?); + } + Some(out.into_iter().flatten().flatten().collect()) + } + _ => create_error_result(Error::InComprehensionCanOnlyIterateOverArray)?, + } + } + }) +} + +pub fn evaluate_member_list_object(context: Context, members: &[Member]) -> Result { + let new_bindings = FutureNewBindings::new(); + let future_this = FutureObjValue::new(); + let context_creator = context_creator!( + closure!(clone context, clone new_bindings, |this: Option, super_obj: Option| { + Ok(context.extend_unbound( + new_bindings.clone().unwrap(), + context.dollar().clone().or_else(||this.clone()), + Some(this.unwrap()), + super_obj + )?) + }) + ); + { + let mut bindings: HashMap, LazyBinding> = HashMap::new(); + for (n, b) in members + .iter() + .filter_map(|m| match m { + Member::BindStmt(b) => Some(b.clone()), + _ => None, + }) + .map(|b| evaluate_binding(&b, context_creator.clone())) + { + bindings.insert(n, b); + } + new_bindings.fill(bindings); + } + + let mut new_members = BTreeMap::new(); + for member in members.iter() { + match member { + Member::Field(FieldMember { + name, + plus, + params: None, + visibility, + value, + }) => { + let name = evaluate_field_name(context.clone(), &name)?; + if name.is_none() { + continue; + } + let name = name.unwrap(); + new_members.insert( + name.clone(), + ObjMember { + add: *plus, + visibility: *visibility, + invoke: LazyBinding::Bindable(Rc::new( + closure!(clone name, clone value, clone context_creator, |this, super_obj| { + Ok(LazyVal::new_resolved(push(&value.1, "object field", ||{ + let context = context_creator.0(this, super_obj)?; + evaluate( + context, + &value, + ) + })?)) + }), + )), + }, + ); + } + Member::Field(FieldMember { + name, + params: Some(params), + value, + .. + }) => { + let name = evaluate_field_name(context.clone(), &name)?; + if name.is_none() { + continue; + } + let name = name.unwrap(); + new_members.insert( + name, + ObjMember { + add: false, + visibility: Visibility::Hidden, + invoke: LazyBinding::Bindable(Rc::new( + closure!(clone value, clone context_creator, clone params, |this, super_obj| { + // TODO: Assert + Ok(LazyVal::new_resolved(evaluate_method( + context_creator.0(this, super_obj)?, + params.clone(), + value.clone(), + ))) + }), + )), + }, + ); + } + Member::BindStmt(_) => {} + Member::AssertStmt(_) => {} + } + } + Ok(future_this.fill(ObjValue::new(None, Rc::new(new_members)))) +} + +pub fn evaluate_object(context: Context, object: &ObjBody) -> Result { + Ok(match object { + ObjBody::MemberList(members) => evaluate_member_list_object(context, &members)?, + ObjBody::ObjComp(obj) => { + let future_this = FutureObjValue::new(); + let mut new_members = BTreeMap::new(); + for (k, v) in evaluate_comp( + context.clone(), + &|ctx| { + let new_bindings = FutureNewBindings::new(); + let context_creator = context_creator!( + closure!(clone context, clone new_bindings, |this: Option, super_obj: Option| { + Ok(context.extend_unbound( + new_bindings.clone().unwrap(), + context.dollar().clone().or_else(||this.clone()), + None, + super_obj + )?) + }) + ); + let mut bindings: HashMap, LazyBinding> = HashMap::new(); + for (n, b) in obj + .pre_locals + .iter() + .chain(obj.post_locals.iter()) + .map(|b| evaluate_binding(b, context_creator.clone())) + { + bindings.insert(n, b); + } + let bindings = new_bindings.fill(bindings); + let ctx = ctx.extend_unbound(bindings, None, None, None)?; + let key = evaluate(ctx.clone(), &obj.key)?; + let value = LazyBinding::Bindable(Rc::new( + closure!(clone ctx, clone obj.value, |this, _super_obj| { + Ok(LazyVal::new_resolved(evaluate(ctx.extend(HashMap::new(), None, this, None)?, &value)?)) + }), + )); + + Ok((key, value)) + }, + &obj.compspecs, + )? + .unwrap() + { + match k { + Val::Null => {} + Val::Str(n) => { + new_members.insert( + n, + ObjMember { + add: false, + visibility: Visibility::Normal, + invoke: v, + }, + ); + } + v => create_error_result(Error::FieldMustBeStringGot(v.value_type()?))?, + } + } + + future_this.fill(ObjValue::new(None, Rc::new(new_members))) + } + }) +} + +pub fn evaluate(context: Context, expr: &LocExpr) -> Result { + use Expr::*; + let LocExpr(expr, loc) = expr; + Ok(match &**expr { + Literal(LiteralType::This) => Val::Obj( + context + .this() + .clone() + .ok_or_else(|| create_error(crate::Error::CantUseSelfOutsideOfObject))?, + ), + Literal(LiteralType::Dollar) => Val::Obj( + context + .dollar() + .clone() + .ok_or_else(|| create_error(crate::Error::NoTopLevelObjectFound))?, + ), + Literal(LiteralType::True) => Val::Bool(true), + Literal(LiteralType::False) => Val::Bool(false), + Literal(LiteralType::Null) => Val::Null, + Parened(e) => evaluate(context, e)?, + Str(v) => Val::Str(v.clone()), + Num(v) => Val::Num(*v), + BinaryOp(v1, o, v2) => evaluate_binary_op_special(context, &v1, *o, &v2)?, + UnaryOp(o, v) => evaluate_unary_op(*o, &evaluate(context, v)?)?, + Var(name) => push(loc, "var", || { + Ok(Val::Lazy(context.binding(name.clone())?).unwrap_if_lazy()?) + })?, + Index(LocExpr(v, _), index) if matches!(&**v, Expr::Literal(LiteralType::Super)) => { + let name = evaluate(context.clone(), index)?.try_cast_str("object index")?; + context + .super_obj() + .clone() + .expect("no super found") + .get_raw(&name, &context.this().clone().expect("no this found"))? + .expect("value not found") + } + Index(value, index) => { + match ( + evaluate(context.clone(), value)?.unwrap_if_lazy()?, + evaluate(context, index)?, + ) { + (Val::Obj(v), Val::Str(s)) => { + if let Some(v) = v.get(s.clone())? { + v.unwrap_if_lazy()? + } else if let Some(Val::Str(n)) = v.get("__intristic_namespace__".into())? { + Val::Intristic(n, s) + } else { + create_error_result(crate::Error::NoSuchField(s))? + } + } + (Val::Obj(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( + ValType::Obj, + ValType::Str, + n.value_type()?, + ))?, + + (Val::Arr(v), Val::Num(n)) => { + if n.fract() > f64::EPSILON { + create_error_result(crate::Error::FractionalIndex)? + } + v.get(n as usize) + .ok_or_else(|| { + create_error(crate::Error::ArrayBoundsError(n as usize, v.len())) + })? + .clone() + .unwrap_if_lazy()? + } + (Val::Arr(_), Val::Str(n)) => { + create_error_result(crate::Error::AttemptedIndexAnArrayWithString(n))? + } + (Val::Arr(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( + ValType::Arr, + ValType::Num, + n.value_type()?, + ))?, + + (Val::Str(s), Val::Num(n)) => Val::Str( + s.chars() + .skip(n as usize) + .take(1) + .collect::() + .into(), + ), + (Val::Str(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( + ValType::Str, + ValType::Num, + n.value_type()?, + ))?, + + (v, _) => create_error_result(crate::Error::CantIndexInto(v.value_type()?))?, + } + } + LocalExpr(bindings, returned) => { + let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); + let future_context = Context::new_future(); + + let context_creator = context_creator!( + closure!(clone future_context, |_, _| Ok(future_context.clone().unwrap())) + ); + + for (k, v) in bindings + .iter() + .map(|b| evaluate_binding(b, context_creator.clone())) + { + new_bindings.insert(k, v); + } + + let context = context + .extend_unbound(new_bindings, None, None, None)? + .into_future(future_context); + evaluate(context, &returned.clone())? + } + Arr(items) => { + let mut out = Vec::with_capacity(items.len()); + for item in items { + out.push(Val::Lazy(lazy_val!( + closure!(clone context, clone item, || { + evaluate(context.clone(), &item) + }) + ))); + } + Val::Arr(Rc::new(out)) + } + ArrComp(expr, compspecs) => Val::Arr( + // First compspec should be forspec, so no "None" possible here + Rc::new(evaluate_comp(context, &|ctx| evaluate(ctx, expr), compspecs)?.unwrap()), + ), + Obj(body) => Val::Obj(evaluate_object(context, body)?), + ObjExtend(s, t) => evaluate_add_op( + &evaluate(context.clone(), s)?, + &Val::Obj(evaluate_object(context, t)?), + )?, + Apply(value, args, tailstrict) => { + let lazy = evaluate(context.clone(), value)?; + let value = lazy.unwrap_if_lazy()?; + match value { + Val::Intristic(ns, name) => match (&ns as &str, &name as &str) { + // arr/string/function + ("std", "length") => parse_args!(context, "std.length", args, 1, [ + 0, x: [Val::Str|Val::Arr|Val::Obj], vec![ValType::Str, ValType::Arr, ValType::Obj]; + ], { + match x { + Val::Str(n) => Val::Num(n.chars().count() as f64), + Val::Arr(i) => Val::Num(i.len() as f64), + Val::Obj(o) => Val::Num( + o.fields_visibility() + .into_iter() + .filter(|(_k, v)| *v) + .count() as f64, + ), + _ => unreachable!(), + } + }), + // any + ("std", "type") => parse_args!(context, "std.type", args, 1, [ + 0, x, vec![]; + ], { + Val::Str(x.value_type()?.name().into()) + }), + // length, idx=>any + ("std", "makeArray") => parse_args!(context, "std.makeArray", args, 2, [ + 0, sz: [Val::Num]!!Val::Num, vec![ValType::Num]; + 1, func: [Val::Func]!!Val::Func, vec![ValType::Func]; + ], { + assert!(sz >= 0.0); + let mut out = Vec::with_capacity(sz as usize); + for i in 0..sz as usize { + out.push(func.evaluate_values( + Context::new(), + &[Val::Num(i as f64)] + )?) + } + Val::Arr(Rc::new(out)) + }), + // string + ("std", "codepoint") => parse_args!(context, "std.codepoint", args, 1, [ + 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + assert!( + str.chars().count() == 1, + "std.codepoint should receive single char string" + ); + Val::Num(str.chars().take(1).next().unwrap() as u32 as f64) + }), + // object, includeHidden + ("std", "objectFieldsEx") => { + parse_args!(context, "std.objectFieldsEx",args, 2, [ + 0, obj: [Val::Obj]!!Val::Obj, vec![ValType::Obj]; + 1, inc_hidden: [Val::Bool]!!Val::Bool, vec![ValType::Bool]; + ], { + Val::Arr(Rc::new( + obj.fields_visibility() + .into_iter() + .filter(|(_k, v)| *v || inc_hidden) + .map(|(k, _v)| Val::Str(k)) + .collect(), + )) + }) + } + // object, field, includeHidden + ("std", "objectHasEx") => parse_args!(context, "std.objectHasEx", args, 3, [ + 0, obj: [Val::Obj]!!Val::Obj, vec![ValType::Obj]; + 1, f: [Val::Str]!!Val::Str, vec![ValType::Str]; + 2, inc_hidden: [Val::Bool]!!Val::Bool, vec![ValType::Bool]; + ], { + Val::Bool( + obj.fields_visibility() + .into_iter() + .filter(|(_k, v)| *v || inc_hidden) + .any(|(k, _v)| *k == *f), + ) + }), + ("std", "primitiveEquals") => { + parse_args!(context, "std.primitiveEquals", args, 2, [ + 0, a, vec![]; + 1, b, vec![]; + ], { + Val::Bool(a == b) + }) + } + ("std", "modulo") => parse_args!(context, "std.modulo", args, 2, [ + 0, a: [Val::Num]!!Val::Num, vec![ValType::Num]; + 1, b: [Val::Num]!!Val::Num, vec![ValType::Num]; + ], { + Val::Num(a % b) + }), + ("std", "floor") => parse_args!(context, "std.floor", args, 1, [ + 0, x: [Val::Num]!!Val::Num, vec![ValType::Num]; + ], { + Val::Num(x.floor()) + }), + ("std", "trace") => parse_args!(context, "std.trace", args, 2, [ + 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; + 1, rest, vec![]; + ], { + // TODO: Line numbers as in original jsonnet + println!("TRACE: {}", str); + rest + }), + ("std", "pow") => parse_args!(context, "std.modulo", args, 2, [ + 0, x: [Val::Num]!!Val::Num, vec![ValType::Num]; + 1, n: [Val::Num]!!Val::Num, vec![ValType::Num]; + ], { + Val::Num(x.powf(n)) + }), + ("std", "extVar") => parse_args!(context, "std.extVar", args, 2, [ + 0, x: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + with_state(|s| s.0.ext_vars.borrow().get(&x).cloned()).ok_or_else( + || create_error(crate::Error::UndefinedExternalVariable(x)), + )? + }), + ("std", "filter") => parse_args!(context, "std.filter", args, 2, [ + 0, func: [Val::Func]!!Val::Func, vec![ValType::Func]; + 1, arr: [Val::Arr]!!Val::Arr, vec![ValType::Arr]; + ], { + Val::Arr(Rc::new( + arr.iter() + .cloned() + .filter(|e| { + func + .evaluate_values(context.clone(), &[e.clone()]) + .unwrap() + .try_cast_bool("filter predicate") + .unwrap() + }) + .collect(), + )) + }), + ("std", "char") => parse_args!(context, "std.char", args, 1, [ + 0, n: [Val::Num]!!Val::Num, vec![ValType::Num]; + ], { + let mut out = String::new(); + out.push(std::char::from_u32(n as u32).unwrap()); + Val::Str(out.into()) + }), + ("std", "encodeUTF8") => parse_args!(context, "std.encodeUtf8", args, 1, [ + 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + Val::Arr(Rc::new(str.bytes().map(|b| Val::Num(b as f64)).collect())) + }), + ("std", "md5") => parse_args!(context, "std.md5", args, 1, [ + 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + Val::Str(format!("{:x}", md5::compute(str.as_bytes())).into()) + }), + // faster + ("std", "join") => parse_args!(context, "std.join", args, 2, [ + 0, sep: [Val::Str|Val::Arr], vec![ValType::Str, ValType::Arr]; + 1, arr: [Val::Arr]!!Val::Arr, vec![ValType::Arr]; + ], { + match sep { + Val::Arr(joiner_items) => { + let mut out = Vec::new(); + + let mut first = true; + for item in arr.iter().cloned() { + if let Val::Arr(items) = item.unwrap_if_lazy()? { + if !first { + out.reserve(joiner_items.len()); + out.extend(joiner_items.iter().cloned()); + } + first = false; + out.reserve(items.len()); + out.extend(items.iter().cloned()); + } else { + create_error_result(crate::Error::RuntimeError("in std.join all items should be arrays".into()))?; + } + } + + Val::Arr(Rc::new(out)) + }, + Val::Str(sep) => { + let mut out = String::new(); + + let mut first = true; + for item in arr.iter().cloned() { + if let Val::Str(item) = item.unwrap_if_lazy()? { + if !first { + out += &sep; + } + first = false; + out += &item; + } else { + create_error_result(crate::Error::RuntimeError("in std.join all items should be strings".into()))?; + } + } + + Val::Str(out.into()) + }, + _ => unreachable!() + } + }), + // Faster + ("std", "escapeStringJson") => { + parse_args!(context, "std.escapeStringJson", args, 1, [ + 0, str_: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + Val::Str(escape_string_json(&str_).into()) + }) + } + // Faster + ("std", "manifestJsonEx") => { + parse_args!(context, "std.manifestJsonEx", args, 2, [ + 0, value, vec![]; + 1, indent: [Val::Str]!!Val::Str, vec![ValType::Str]; + ], { + Val::Str(manifest_json_ex(&value, &indent)?.into()) + }) + } + (ns, name) => create_error_result(crate::Error::IntristicNotFound( + ns.into(), + name.into(), + ))?, + }, + Val::Func(f) => { + let body = || f.evaluate(context, args, *tailstrict); + if *tailstrict { + body()? + } else { + push(loc, "function call", body)? + } + } + v => create_error_result(crate::Error::OnlyFunctionsCanBeCalledGot( + v.value_type()?, + ))?, + } + } + Function(params, body) => evaluate_method(context, params.clone(), body.clone()), + AssertExpr(AssertStmt(value, msg), returned) => { + let assertion_result = push(&value.1, "assertion condition", || { + evaluate(context.clone(), &value)? + .try_cast_bool("assertion condition should be boolean") + })?; + if assertion_result { + evaluate(context, returned)? + } else if let Some(msg) = msg { + create_error_result(crate::Error::AssertionFailed(evaluate(context, msg)?))? + } else { + create_error_result(crate::Error::AssertionFailed(Val::Null))? + } + } + Error(e) => create_error_result(crate::Error::RuntimeError( + evaluate(context, e)?.try_cast_str("error text should be string")?, + ))?, + IfElse { + cond, + cond_then, + cond_else, + } => { + if evaluate(context.clone(), &cond.0)? + .try_cast_bool("if condition should be boolean")? + { + evaluate(context, cond_then)? + } else { + match cond_else { + Some(v) => evaluate(context, v)?, + None => Val::Null, + } + } + } + Import(path) => { + let mut tmp = loc + .clone() + .expect("imports can't be used without loc_data") + .0; + let import_location = Rc::make_mut(&mut tmp); + import_location.pop(); + with_state(|s| s.import_file(&import_location, path))? + } + ImportStr(path) => { + let mut tmp = loc + .clone() + .expect("imports can't be used without loc_data") + .0; + let import_location = Rc::make_mut(&mut tmp); + import_location.pop(); + Val::Str(with_state(|s| s.import_file_str(&import_location, path))?) + } + Literal(LiteralType::Super) => { + return create_error_result(crate::Error::StandaloneSuper) + } + }) +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/function.rs @@ -0,0 +1,157 @@ +use crate::{ + create_error, create_error_result, evaluate, lazy_val, resolved_lazy_val, Context, Error, + Result, Val, +}; +use closure::closure; +use jrsonnet_parser::{ArgsDesc, ParamsDesc}; +use std::collections::HashMap; + +/// Creates correct [context](Context) for function body evaluation, returning error on invalid call +/// +/// * `ctx` used for passed argument expressions execution, and for body execution (if `body_ctx` is not set) +/// * `body_ctx` used for default parameter values execution, and for body execution (if set) +/// * `params` function parameters definition +/// * `args` passed function arguments +/// * `tailstruct` if true - function arguments is eager executed, otherwise - lazy +pub fn parse_function_call( + ctx: Context, + body_ctx: Option, + params: &ParamsDesc, + args: &ArgsDesc, + tailstrict: bool, +) -> Result { + let mut out = HashMap::new(); + let mut positioned_args = vec![None; params.0.len()]; + for (id, arg) in args.iter().enumerate() { + let idx = if let Some(name) = &arg.0 { + params + .iter() + .position(|p| *p.0 == *name) + .ok_or_else(|| create_error(Error::UnknownFunctionParameter(name.clone())))? + } else { + id + }; + + if idx >= params.len() { + create_error_result(Error::TooManyArgsFunctionHas(params.len()))?; + } + if positioned_args[idx].is_some() { + create_error_result(Error::BindingParameterASecondTime(params[idx].0.clone()))?; + } + positioned_args[idx] = Some(arg.1.clone()); + } + // Fill defaults + for (id, p) in params.iter().enumerate() { + let (ctx, expr) = if let Some(arg) = &positioned_args[id] { + (ctx.clone(), arg) + } else if let Some(default) = &p.1 { + ( + body_ctx + .clone() + .expect("no default context set for call with defined default parameter value"), + default, + ) + } else { + create_error_result(Error::FunctionParameterNotBoundInCall(p.0.clone()))?; + unreachable!() + }; + let val = if tailstrict { + resolved_lazy_val!(evaluate(ctx, expr)?) + } else { + lazy_val!(closure!(clone ctx, clone expr, ||evaluate(ctx.clone(), &expr))) + }; + out.insert(p.0.clone(), val); + } + + Ok(body_ctx.unwrap_or(ctx).extend(out, None, None, None)?) +} + +pub(crate) fn place_args( + ctx: Context, + body_ctx: Option, + params: &ParamsDesc, + args: &[Val], +) -> Result { + let mut out = HashMap::new(); + let mut positioned_args = vec![None; params.0.len()]; + for (id, arg) in args.iter().enumerate() { + if id >= params.len() { + create_error_result(Error::TooManyArgsFunctionHas(params.len()))?; + } + positioned_args[id] = Some(arg); + } + // Fill defaults + for (id, p) in params.iter().enumerate() { + let val = if let Some(arg) = &positioned_args[id] { + (*arg).clone() + } else if let Some(default) = &p.1 { + evaluate(ctx.clone(), default)? + } else { + create_error_result(Error::FunctionParameterNotBoundInCall(p.0.clone()))?; + unreachable!() + }; + out.insert(p.0.clone(), resolved_lazy_val!(val)); + } + + Ok(body_ctx.unwrap_or(ctx).extend(out, None, None, None)?) +} + +#[macro_export] +macro_rules! parse_args { + ($ctx: expr, $fn_name: expr, $args: expr, $total_args: expr, [ + $($id: expr, $name: ident $(: [$($p: path)|+] $(!! $a: path)?)?, $nt: expr);+ $(;)? + ], $handler:block) => {{ + use crate::Error; + let args = $args; + if args.len() > $total_args { + create_error_result(Error::TooManyArgsFunctionHas($total_args))?; + } + $( + if args.len() <= $id { + create_error_result(Error::FunctionParameterNotBoundInCall(stringify!($name).into()))?; + } + let $name = &args[$id]; + if $name.0.is_some() { + if $name.0.as_ref().unwrap() != stringify!($name) { + create_error_result(Error::IntristicArgumentReorderingIsNotSupportedYet)?; + } + } + let $name = evaluate($ctx.clone(), &$name.1)?; + $( + match $name { + $($p(_))|+ => {}, + _ => create_error_result(Error::TypeMismatch(concat!($fn_name, " ", stringify!($id), "nd argument"), $nt, $name.value_type()?))?, + }; + $( + let $name = match $name { + $a(v) => v, + _ => create_error_result(Error::TypeMismatch(concat!($fn_name, " ", stringify!($id), "nd argument"), $nt, $name.value_type()?))?, + }; + )* + )* + )+ + $handler + }}; +} + +#[test] +fn test() -> Result<()> { + use jrsonnet_parser::*; + use crate::val::ValType; + let state = crate::EvaluationState::default(); + let evaluator = state.with_stdlib(); + let ctx = evaluator.create_default_context()?; + evaluator.run_in_state(|| { + parse_args!(ctx, "test", ArgsDesc(vec![ + Arg(None, el!(Expr::Num(2.0))), + Arg(Some("b".into()), el!(Expr::Num(1.0))), + ]), 2, [ + 0, a: [Val::Num]!!Val::Num, vec![ValType::Num]; + 1, b: [Val::Num]!!Val::Num, vec![ValType::Num]; + ], { + assert!((a - 2.0).abs() <= f64::EPSILON); + assert!((b - 1.0).abs() <= f64::EPSILON); + }); + Ok(()) + }) +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/import.rs @@ -0,0 +1,83 @@ +use crate::create_error_result; +use crate::{ + create_error, + error::{Error, Result}, +}; +use fs::File; +use std::fs; +use std::io::Read; +use std::{cell::RefCell, collections::HashMap, path::PathBuf, rc::Rc}; + +pub trait ImportResolver { + fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result>; + fn load_file_contents(&self, resolved: &PathBuf) -> Result>; +} + +pub struct DummyImportResolver; +impl ImportResolver for DummyImportResolver { + fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { + create_error_result(Error::ImportNotSupported(from.clone(), path.clone())) + } + fn load_file_contents(&self, _resolved: &PathBuf) -> Result> { + // Can be only caused by library direct consumer, not by supplied jsonnet + panic!("dummy resolver can't load any file") + } +} +impl Default for Box { + fn default() -> Self { + Box::new(DummyImportResolver) + } +} + +pub struct FileImportResolver { + pub library_paths: Vec, +} +impl ImportResolver for FileImportResolver { + fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { + let mut new_path = from.clone(); + new_path.push(path); + if new_path.exists() { + Ok(Rc::new(new_path)) + } else { + for library_path in self.library_paths.iter() { + let mut cloned = library_path.clone(); + cloned.push(path); + if cloned.exists() { + return Ok(Rc::new(cloned)); + } + } + create_error_result(Error::ImportFileNotFound(from.clone(), path.clone())) + } + } + fn load_file_contents(&self, id: &PathBuf) -> Result> { + let mut file = + File::open(id).map_err(|_e| create_error(Error::ResolvedFileNotFound(id.clone())))?; + let mut out = String::new(); + file.read_to_string(&mut out) + .map_err(|_e| create_error(Error::ImportBadFileUtf8(id.clone())))?; + Ok(out.into()) + } +} + +type ResolutionData = (PathBuf, PathBuf); +pub struct CachingImportResolver { + resolution_cache: RefCell>>>, + loading_cache: RefCell>>>, + inner: Box, +} +impl ImportResolver for CachingImportResolver { + fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { + self.resolution_cache + .borrow_mut() + .entry((from.clone(), path.clone())) + .or_insert_with(|| self.inner.resolve_file(from, path)) + .clone() + } + fn load_file_contents(&self, resolved: &PathBuf) -> Result> { + self.loading_cache + .borrow_mut() + .entry(resolved.clone()) + .or_insert_with(|| self.inner.load_file_contents(resolved)) + .clone() + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/lib.rs @@ -0,0 +1,730 @@ +#![feature(box_syntax, box_patterns)] +#![feature(type_alias_impl_trait)] +#![feature(debug_non_exhaustive)] +#![feature(test)] +#![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)] + +extern crate test; + +mod ctx; +mod dynamic; +mod error; +mod evaluate; +mod function; +mod import; +mod map; +mod obj; +mod val; + +pub use ctx::*; +pub use dynamic::*; +pub use error::*; +pub use evaluate::*; +pub use function::parse_function_call; +pub use import::*; +use jrsonnet_parser::*; +pub use obj::*; +use std::{cell::RefCell, collections::HashMap, fmt::Debug, path::PathBuf, rc::Rc}; +pub use val::*; + +type BindableFn = dyn Fn(Option, Option) -> Result; +#[derive(Clone)] +pub enum LazyBinding { + Bindable(Rc), + Bound(LazyVal), +} + +impl Debug for LazyBinding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "LazyBinding") + } +} +impl LazyBinding { + pub fn evaluate(&self, this: Option, super_obj: Option) -> Result { + match self { + LazyBinding::Bindable(v) => v(this, super_obj), + LazyBinding::Bound(v) => Ok(v.clone()), + } + } +} + +pub struct EvaluationSettings { + pub max_stack_frames: usize, + pub max_stack_trace_size: usize, +} +impl Default for EvaluationSettings { + fn default() -> Self { + EvaluationSettings { + max_stack_frames: 200, + max_stack_trace_size: 20, + } + } +} + +pub struct FileData(Rc, LocExpr, Option); +#[derive(Default)] +pub struct EvaluationStateInternals { + /// Used for stack-overflows and stacktraces + stack: RefCell>, + /// Contains file source codes and evaluated results for imports and pretty + /// printing stacktraces + files: RefCell, FileData>>, + str_files: RefCell, Rc>>, + globals: RefCell, Val>>, + + /// Values to use with std.extVar + ext_vars: RefCell, Val>>, + + settings: EvaluationSettings, + import_resolver: Box, +} + +thread_local! { + /// Contains state for currently executing file + /// Global state is fine there + pub(crate) static EVAL_STATE: RefCell> = RefCell::new(None) +} +pub(crate) fn with_state(f: impl FnOnce(&EvaluationState) -> T) -> T { + EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap())) +} +pub(crate) fn create_error(err: Error) -> LocError { + with_state(|s| s.error(err)) +} +pub(crate) fn create_error_result(err: Error) -> Result { + Err(with_state(|s| s.error(err))) +} +pub(crate) fn push( + e: &Option, + comment: &str, + f: impl FnOnce() -> Result, +) -> Result { + if e.is_some() { + with_state(|s| s.push(e.clone().unwrap(), comment.to_owned(), f)) + } else { + f() + } +} + +/// Maintains stack trace and import resolution +#[derive(Default, Clone)] +pub struct EvaluationState(Rc); +impl EvaluationState { + pub fn new(settings: EvaluationSettings, import_resolver: Box) -> Self { + EvaluationState(Rc::new(EvaluationStateInternals { + settings, + import_resolver, + ..Default::default() + })) + } + pub fn add_file( + &self, + name: Rc, + code: Rc, + ) -> std::result::Result<(), ParseError> { + self.0.files.borrow_mut().insert( + name.clone(), + FileData( + code.clone(), + parse( + &code, + &ParserSettings { + file_name: name, + loc_data: true, + }, + )?, + None, + ), + ); + + Ok(()) + } + pub fn add_parsed_file( + &self, + name: Rc, + code: Rc, + parsed: LocExpr, + ) -> std::result::Result<(), ()> { + self.0 + .files + .borrow_mut() + .insert(name, FileData(code, parsed, None)); + + Ok(()) + } + pub fn get_source(&self, name: &PathBuf) -> Option> { + let ro_map = self.0.files.borrow(); + ro_map.get(name).map(|value| value.0.clone()) + } + pub fn evaluate_file(&self, name: &PathBuf) -> Result { + self.run_in_state(|| { + let expr: LocExpr = { + let ro_map = self.0.files.borrow(); + let value = ro_map + .get(name) + .unwrap_or_else(|| panic!("file not added: {:?}", name)); + if value.2.is_some() { + return Ok(value.2.clone().unwrap()); + } + value.1.clone() + }; + let value = evaluate(self.create_default_context()?, &expr)?; + { + self.0 + .files + .borrow_mut() + .get_mut(name) + .unwrap() + .2 + .replace(value.clone()); + } + Ok(value) + }) + } + pub(crate) fn import_file(&self, from: &PathBuf, path: &PathBuf) -> Result { + let file_path = self.0.import_resolver.resolve_file(from, path)?; + { + let files = self.0.files.borrow(); + if files.contains_key(&file_path) { + return self.evaluate_file(&file_path); + } + } + let contents = self.0.import_resolver.load_file_contents(&file_path)?; + self.add_file(file_path.clone(), contents).map_err(|e| { + create_error(Error::ImportSyntaxError(e)) + })?; + self.evaluate_file(&file_path) + } + pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result> { + let path = self.0.import_resolver.resolve_file(from, path)?; + if !self.0.str_files.borrow().contains_key(&path) { + let file_str = self.0.import_resolver.load_file_contents(&path)?; + self.0 + .str_files + .borrow_mut() + .insert(path.clone(), file_str); + } + Ok(self.0.str_files.borrow().get(&path).cloned().unwrap()) + } + + pub fn parse_evaluate_raw(&self, code: &str) -> Result { + let parsed = parse( + &code, + &ParserSettings { + file_name: Rc::new(PathBuf::from("raw.jsonnet")), + loc_data: true, + }, + ) + .unwrap(); + self.evaluate_raw(parsed) + } + + pub fn evaluate_raw(&self, code: LocExpr) -> Result { + self.run_in_state(|| evaluate(self.create_default_context()?, &code)) + } + + pub fn add_global(&self, name: Rc, value: Val) { + self.0.globals.borrow_mut().insert(name, value); + } + pub fn add_ext_var(&self, name: Rc, value: Val) { + self.0.ext_vars.borrow_mut().insert(name, value); + } + + pub fn with_stdlib(&self) -> &Self { + let std_path = Rc::new(PathBuf::from("std.jsonnet")); + self.run_in_state(|| { + use jrsonnet_stdlib::STDLIB_STR; + let mut parsed = false; + #[cfg(feature = "codegenerated-stdlib")] + if !parsed { + parsed = true; + #[allow(clippy::all)] + let stdlib = { + use jrsonnet_parser::*; + include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) + }; + self.add_parsed_file(std_path.clone(), STDLIB_STR.to_owned().into(), stdlib) + .unwrap(); + } + + #[cfg(feature = "serialized-stdlib")] + if !parsed { + parsed = true; + self.add_parsed_file( + std_path.clone(), + STDLIB_STR.to_owned().into(), + bincode::deserialize(include_bytes!(concat!( + env!("OUT_DIR"), + "/stdlib.bincode" + ))) + .expect("deserialize stdlib"), + ) + .unwrap(); + } + + if !parsed { + self.add_file(std_path, STDLIB_STR.to_owned().into()) + .unwrap(); + } + let val = self.evaluate_file(&PathBuf::from("std.jsonnet")).unwrap(); + self.add_global("std".into(), val); + }); + self + } + + pub fn create_default_context(&self) -> Result { + let globals = self.0.globals.borrow(); + let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); + for (name, value) in globals.iter() { + new_bindings.insert( + name.clone(), + LazyBinding::Bound(resolved_lazy_val!(value.clone())), + ); + } + Context::new().extend_unbound(new_bindings, None, None, None) + } + + pub fn push( + &self, + e: ExprLocation, + comment: String, + f: impl FnOnce() -> Result, + ) -> Result { + { + let mut stack = self.0.stack.borrow_mut(); + if stack.len() > self.0.settings.max_stack_frames { + drop(stack); + return Err(self.error(Error::StackOverflow)); + } else { + stack.push(StackTraceElement(e, comment)); + } + } + let result = f(); + self.0.stack.borrow_mut().pop(); + result + } + pub fn print_stack_trace(&self) { + for e in self.stack_trace().0 { + println!("{:?} - {:?}", e.0, e.1) + } + } + pub fn stack_trace(&self) -> StackTrace { + StackTrace( + self.0 + .stack + .borrow() + .iter() + .rev() + .take(self.0.settings.max_stack_trace_size) + .cloned() + .collect(), + ) + } + pub fn error(&self, err: Error) -> LocError { + LocError(err, self.stack_trace()) + } + + pub fn run_in_state(&self, f: impl FnOnce() -> T) -> T { + EVAL_STATE.with(|v| { + let has_state = v.borrow().is_some(); + if !has_state { + v.borrow_mut().replace(self.clone()); + } + let result = f(); + if !has_state { + v.borrow_mut().take(); + } + result + }) + } +} + +#[cfg(test)] +pub mod tests { + use super::Val; + use crate::EvaluationState; + use jrsonnet_parser::*; + use std::{path::PathBuf, rc::Rc}; + + #[test] + fn eval_state_stacktrace() { + let state = EvaluationState::default(); + state + .push( + ExprLocation(Rc::new(PathBuf::from("test1.jsonnet")), 10, 20), + "outer".to_owned(), + || { + state.push( + ExprLocation(Rc::new(PathBuf::from("test2.jsonnet")), 30, 40), + "inner".to_owned(), + || { + state.print_stack_trace(); + Ok(()) + }, + )?; + Ok(()) + }, + ) + .unwrap(); + } + + #[test] + fn eval_state_standard() { + let state = EvaluationState::default(); + state.with_stdlib(); + assert_eq!( + state + .parse_evaluate_raw(r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#) + .unwrap(), + Val::Bool(true) + ); + } + + macro_rules! eval { + ($str: expr) => { + EvaluationState::default() + .with_stdlib() + .parse_evaluate_raw($str) + .unwrap() + }; + } + macro_rules! eval_json { + ($str: expr) => {{ + let evaluator = EvaluationState::default(); + evaluator.with_stdlib(); + evaluator.run_in_state(||{ + evaluator + .parse_evaluate_raw($str) + .unwrap() + .into_json(0) + .unwrap() + .replace("\n", "") + }) + }} + } + + /// Asserts given code returns `true` + macro_rules! assert_eval { + ($str: expr) => { + assert_eq!(eval!($str), Val::Bool(true)) + }; + } + + /// Asserts given code returns `false` + macro_rules! assert_eval_neg { + ($str: expr) => { + assert_eq!(eval!($str), Val::Bool(false)) + }; + } + macro_rules! assert_json { + ($str: expr, $out: expr) => { + assert_eq!(eval_json!($str), $out.replace("\t", "")) + }; + } + + /// Sanity checking, before trusting to another tests + #[test] + fn equality_operator() { + assert_eval!("2 == 2"); + assert_eval_neg!("2 != 2"); + assert_eval!("2 != 3"); + assert_eval_neg!("2 == 3"); + assert_eval!("'Hello' == 'Hello'"); + assert_eval_neg!("'Hello' != 'Hello'"); + assert_eval!("'Hello' != 'World'"); + assert_eval_neg!("'Hello' == 'World'"); + } + + #[test] + fn math_evaluation() { + assert_eval!("2 + 2 * 2 == 6"); + assert_eval!("3 + (2 + 2 * 2) == 9"); + } + + #[test] + fn string_concat() { + assert_eval!("'Hello' + 'World' == 'HelloWorld'"); + assert_eval!("'Hello' * 3 == 'HelloHelloHello'"); + assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'"); + } + + #[test] + fn faster_join() { + assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]"); + assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'"); + } + + #[test] + fn function_contexts() { + assert_eval!( + r#" + local k = { + t(name = self.h): [self.h, name], + h: 3, + }; + local f = { + t: k.t(), + h: 4, + }; + f.t[0] == f.t[1] + "# + ); + } + + #[test] + fn local() { + assert_eval!("local a = 2; local b = 3; a + b == 5"); + assert_eval!("local a = 1, b = a + 1; a + b == 3"); + assert_eval!("local a = 1; local a = 2; a == 2"); + } + + #[test] + fn object_lazyness() { + assert_json!("local a = {a:error 'test'}; {}", r#"{}"#); + } + + #[test] + fn object_inheritance() { + assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#); + } + + #[test] + fn object_assertion_success() { + eval!("{assert \"a\" in self} + {a:2}"); + } + + #[test] + fn object_assertion_error() { + eval!("{assert \"a\" in self}"); + } + + #[test] + fn lazy_args() { + eval!("local test(a) = 2; test(error '3')"); + } + + #[test] + #[should_panic] + fn tailstrict_args() { + eval!("local test(a) = 2; test(error '3') tailstrict"); + } + + #[test] + #[should_panic] + fn no_binding_error() { + eval!("a"); + } + + #[test] + fn test_object() { + assert_json!("{a:2}", r#"{"a": 2}"#); + assert_json!("{a:2+2}", r#"{"a": 4}"#); + assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#); + assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#); + assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#); + assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#); + assert_json!( + r#" + { + name: "Alice", + welcome: "Hello " + self.name + "!", + } + "#, + r#"{"name": "Alice","welcome": "Hello Alice!"}"# + ); + assert_json!( + r#" + { + name: "Alice", + welcome: "Hello " + self.name + "!", + } + { + name: "Bob" + } + "#, + r#"{"name": "Bob","welcome": "Hello Bob!"}"# + ); + } + + #[test] + fn functions() { + assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4"); + assert_json!( + r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#, + r#""HelloDearWorld""# + ); + } + + #[test] + fn local_methods() { + assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4"); + assert_json!( + r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#, + r#""HelloDearWorld""# + ); + } + + #[test] + fn object_locals() { + assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#); + assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#); + assert_json!( + r#"{local a = function (b) {[b]:4}, test: a("test")}"#, + r#"{"test": {"test": 4}}"# + ); + } + + #[test] + fn object_comp() { + assert_json!( + r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#, + "{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}" + ) + } + + #[test] + fn direct_self() { + println!( + "{:#?}", + eval!( + r#" + { + local me = self, + a: 3, + b(): me.a, + } + "# + ) + ); + } + + #[test] + fn indirect_self() { + // `self` assigned to `me` was lost when being + // referenced from field + eval!( + r#"{ + local me = self, + a: 3, + b: me.a, + }.b"# + ); + } + + // We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly + #[test] + fn std_assert_ok() { + eval!("std.assertEqual(4.5 << 2, 16)"); + } + + #[test] + #[should_panic] + fn std_assert_failure() { + eval!("std.assertEqual(4.5 << 2, 15)"); + } + + #[test] + fn string_is_string() { + assert_eq!( + eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"), + Val::Bool(false) + ); + } + + #[test] + fn base64_works() { + assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#); + } + + #[test] + fn utf8_chars() { + assert_json!( + r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#, + r#"{"c": 128526,"l": 1}"# + ) + } + + #[test] + fn json() { + assert_json!( + r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#, + r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""# + ); + } + + #[test] + fn test() { + assert_json!( + r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#, + "[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]" + ); + } + + #[test] + fn sjsonnet() { + eval!( + r#" + local x0 = {k: 1}; + local x1 = {k: x0.k + x0.k}; + local x2 = {k: x1.k + x1.k}; + local x3 = {k: x2.k + x2.k}; + local x4 = {k: x3.k + x3.k}; + local x5 = {k: x4.k + x4.k}; + local x6 = {k: x5.k + x5.k}; + local x7 = {k: x6.k + x6.k}; + local x8 = {k: x7.k + x7.k}; + local x9 = {k: x8.k + x8.k}; + local x10 = {k: x9.k + x9.k}; + local x11 = {k: x10.k + x10.k}; + local x12 = {k: x11.k + x11.k}; + local x13 = {k: x12.k + x12.k}; + local x14 = {k: x13.k + x13.k}; + local x15 = {k: x14.k + x14.k}; + local x16 = {k: x15.k + x15.k}; + local x17 = {k: x16.k + x16.k}; + local x18 = {k: x17.k + x17.k}; + local x19 = {k: x18.k + x18.k}; + local x20 = {k: x19.k + x19.k}; + local x21 = {k: x20.k + x20.k}; + x21.k + "# + ); + } + + use test::Bencher; + + // This test is commented out by default, because of huge compilation slowdown + // #[bench] + // fn bench_codegen(b: &mut Bencher) { + // b.iter(|| { + // #[allow(clippy::all)] + // let stdlib = { + // use jrsonnet_parser::*; + // include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) + // }; + // stdlib + // }) + // } + + #[bench] + fn bench_serialize(b: &mut Bencher) { + b.iter(|| { + bincode::deserialize::(include_bytes!(concat!( + env!("OUT_DIR"), + "/stdlib.bincode" + ))) + .expect("deserialize stdlib") + }) + } + + #[bench] + fn bench_parse(b: &mut Bencher) { + b.iter(|| { + jrsonnet_parser::parse( + jrsonnet_stdlib::STDLIB_STR, + &jrsonnet_parser::ParserSettings { + loc_data: true, + file_name: Rc::new(PathBuf::from("std.jsonnet")), + }, + ) + }) + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/map.rs @@ -0,0 +1,46 @@ +use std::{borrow::Borrow, collections::HashMap, hash::Hash, rc::Rc}; + +#[derive(Default, Debug)] +struct LayeredHashMapInternals { + parent: Option>, + current: HashMap, +} + +#[derive(Debug)] +pub struct LayeredHashMap(Rc>); + +impl LayeredHashMap { + pub fn extend(&self, new_layer: HashMap) -> Self { + let super_map = self.clone(); + LayeredHashMap(Rc::new(LayeredHashMapInternals { + parent: Some(super_map), + current: new_layer, + })) + } + + pub fn get(&self, key: &Q) -> Option<&V> + where + K: Borrow, + Q: Hash + Eq, + { + (self.0) + .current + .get(&key) + .or_else(|| self.0.parent.as_ref().and_then(|p| p.get(key))) + } +} + +impl Clone for LayeredHashMap { + fn clone(&self) -> Self { + LayeredHashMap(self.0.clone()) + } +} + +impl Default for LayeredHashMap { + fn default() -> Self { + LayeredHashMap(Rc::new(LayeredHashMapInternals { + parent: None, + current: HashMap::new(), + })) + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/obj.rs @@ -0,0 +1,138 @@ +use crate::{evaluate_add_op, LazyBinding, Result, Val}; +use indexmap::IndexMap; +use jrsonnet_parser::Visibility; +use std::{ + cell::RefCell, + collections::{BTreeMap, HashMap}, + fmt::Debug, + rc::Rc, +}; + +#[derive(Debug)] +pub struct ObjMember { + pub add: bool, + pub visibility: Visibility, + pub invoke: LazyBinding, +} + +#[derive(Debug)] +pub struct ObjValueInternals { + super_obj: Option, + this_entries: Rc, ObjMember>>, + value_cache: RefCell, Val>>, +} +#[derive(Clone)] +pub struct ObjValue(pub(crate) Rc); +impl Debug for ObjValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(super_obj) = self.0.super_obj.as_ref() { + if f.alternate() { + write!(f, "{:#?}", super_obj)?; + } else { + write!(f, "{:?}", super_obj)?; + } + write!(f, " + ")?; + } + let mut debug = f.debug_struct("ObjValue"); + for (name, member) in self.0.this_entries.iter() { + debug.field(name, member); + } + debug.finish_non_exhaustive() + } +} + +impl ObjValue { + pub fn new( + super_obj: Option, + this_entries: Rc, ObjMember>>, + ) -> ObjValue { + ObjValue(Rc::new(ObjValueInternals { + super_obj, + this_entries, + value_cache: RefCell::new(HashMap::new()), + })) + } + pub fn with_super(&self, super_obj: ObjValue) -> ObjValue { + match &self.0.super_obj { + None => ObjValue::new(Some(super_obj), self.0.this_entries.clone()), + Some(v) => ObjValue::new(Some(v.with_super(super_obj)), self.0.this_entries.clone()), + } + } + pub fn enum_fields(&self, handler: &impl Fn(&Rc, &Visibility)) { + if let Some(s) = &self.0.super_obj { + s.enum_fields(handler); + } + for (name, member) in self.0.this_entries.iter() { + handler(&name, &member.visibility); + } + } + pub fn fields_visibility(&self) -> IndexMap, bool> { + let out = Rc::new(RefCell::new(IndexMap::new())); + self.enum_fields(&|name, visibility| { + let mut out = out.borrow_mut(); + match visibility { + Visibility::Normal => { + if !out.contains_key(name) { + out.insert(name.to_owned(), true); + } + } + Visibility::Hidden => { + out.insert(name.to_owned(), false); + } + Visibility::Unhide => { + out.insert(name.to_owned(), true); + } + }; + }); + Rc::try_unwrap(out).unwrap().into_inner() + } + pub fn visible_fields(&self) -> Vec> { + self.fields_visibility() + .into_iter() + .filter(|(_k, v)| *v) + .map(|(k, _)| k) + .collect() + } + pub fn get(&self, key: Rc) -> Result> { + if let Some(v) = self.0.value_cache.borrow().get(&key) { + return Ok(Some(v.clone())); + } + if let Some(v) = self.get_raw(&key, self)? { + let v = v.unwrap_if_lazy()?; + self.0.value_cache.borrow_mut().insert(key, v.clone()); + Ok(Some(v)) + } else { + Ok(None) + } + } + pub(crate) fn get_raw(&self, key: &str, real_this: &ObjValue) -> Result> { + match (self.0.this_entries.get(key), &self.0.super_obj) { + (Some(k), None) => Ok(Some( + k.invoke + .evaluate(Some(real_this.clone()), self.0.super_obj.clone())? + .evaluate()?, + )), + (Some(k), Some(s)) => { + let lazy = k + .invoke + .evaluate(Some(real_this.clone()), self.0.super_obj.clone())?; + let our = lazy.evaluate()?; + if k.add { + s.get_raw(key, real_this)? + .map_or(Ok(Some(our.clone())), |v| { + Ok(Some(evaluate_add_op(&v, &our)?)) + }) + } else { + Ok(Some(our)) + } + } + (None, Some(s)) => s.get_raw(key, real_this), + (None, None) => Ok(None), + } + } +} +impl PartialEq for ObjValue { + fn eq(&self, other: &Self) -> bool { + Rc::ptr_eq(&self.0, &other.0) + } +} --- /dev/null +++ b/crates/jrsonnet-evaluator/src/val.rs @@ -0,0 +1,299 @@ +use crate::{ + create_error_result, evaluate, + function::{parse_function_call, place_args}, + Context, Error, ObjValue, Result, +}; +use jrsonnet_parser::{ArgsDesc, LocExpr, ParamsDesc}; +use std::{ + cell::RefCell, + fmt::{Debug, Display}, + rc::Rc, +}; + +enum LazyValInternals { + Computed(Val), + Waiting(Box Result>), +} +#[derive(Clone)] +pub struct LazyVal(Rc>); +impl LazyVal { + pub fn new(f: Box Result>) -> Self { + LazyVal(Rc::new(RefCell::new(LazyValInternals::Waiting(f)))) + } + pub fn new_resolved(val: Val) -> Self { + LazyVal(Rc::new(RefCell::new(LazyValInternals::Computed(val)))) + } + pub fn evaluate(&self) -> Result { + let new_value = match &*self.0.borrow() { + LazyValInternals::Computed(v) => return Ok(v.clone()), + LazyValInternals::Waiting(f) => f()?, + }; + *self.0.borrow_mut() = LazyValInternals::Computed(new_value.clone()); + Ok(new_value) + } +} + +#[macro_export] +macro_rules! lazy_val { + ($f: expr) => { + $crate::LazyVal::new(Box::new($f)) + }; +} +#[macro_export] +macro_rules! resolved_lazy_val { + ($f: expr) => { + $crate::LazyVal::new_resolved($f) + }; +} +impl Debug for LazyVal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Lazy") + } +} +impl PartialEq for LazyVal { + fn eq(&self, other: &Self) -> bool { + Rc::ptr_eq(&self.0, &other.0) + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct FuncDesc { + pub ctx: Context, + pub params: ParamsDesc, + pub body: LocExpr, +} +impl FuncDesc { + /// This function is always inlined to make tailstrict work + pub fn evaluate(&self, call_ctx: Context, args: &ArgsDesc, tailstrict: bool) -> Result { + let ctx = parse_function_call( + call_ctx, + Some(self.ctx.clone()), + &self.params, + args, + tailstrict, + )?; + evaluate(ctx, &self.body) + } + + pub fn evaluate_values(&self, call_ctx: Context, args: &[Val]) -> Result { + let ctx = place_args(call_ctx, Some(self.ctx.clone()), &self.params, args)?; + evaluate(ctx, &self.body) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ValType { + Bool, + Null, + Str, + Num, + Arr, + Obj, + Func, +} +impl ValType { + pub fn name(&self) -> &'static str { + use ValType::*; + match self { + Bool => "boolean", + Null => "null", + Str => "string", + Num => "number", + Arr => "array", + Obj => "object", + Func => "function", + } + } +} +impl Display for ValType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum Val { + Bool(bool), + Null, + Str(Rc), + Num(f64), + Lazy(LazyVal), + Arr(Rc>), + Obj(ObjValue), + Func(FuncDesc), + + // Library functions implemented in native + Intristic(Rc, Rc), +} +macro_rules! matches_unwrap { + ($e: expr, $p: pat, $r: expr) => { + match $e { + $p => $r, + _ => panic!("no match"), + } + }; +} +impl Val { + pub fn assert_type(&self, context: &'static str, val_type: ValType) -> Result<()> { + let this_type = self.value_type()?; + if this_type != val_type { + create_error_result(Error::TypeMismatch(context, vec![val_type], this_type)) + } else { + Ok(()) + } + } + pub fn try_cast_bool(self, context: &'static str) -> Result { + self.assert_type(context, ValType::Bool)?; + Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Bool(v), v)) + } + pub fn try_cast_str(self, context: &'static str) -> Result> { + self.assert_type(context, ValType::Str)?; + Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Str(v), v)) + } + pub fn try_cast_num(self, context: &'static str) -> Result { + self.assert_type(context, ValType::Num)?; + Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Num(v), v)) + } + pub fn unwrap_if_lazy(&self) -> Result { + Ok(if let Val::Lazy(v) = self { + v.evaluate()?.unwrap_if_lazy()? + } else { + self.clone() + }) + } + pub fn value_type(&self) -> Result { + Ok(match self { + Val::Str(..) => ValType::Str, + Val::Num(..) => ValType::Num, + Val::Arr(..) => ValType::Arr, + Val::Obj(..) => ValType::Obj, + Val::Func(..) => ValType::Func, + Val::Bool(_) => ValType::Bool, + Val::Null => ValType::Null, + Val::Intristic(_, _) => ValType::Func, + Val::Lazy(_) => self.clone().unwrap_if_lazy()?.value_type()?, + }) + } + #[cfg(feature = "faster")] + pub fn into_json(self, padding: usize) -> Result> { + manifest_json_ex(&self, &" ".repeat(padding)).map(|s| s.into()) + } + #[cfg(not(feature = "faster"))] + pub fn into_json(self, padding: usize) -> Result> { + with_state(|s| { + let ctx = s + .create_default_context()? + .with_var("__tmp__to_json__".into(), self)?; + Ok(evaluate( + ctx, + &el!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("manifestJsonEx".into())) + )), + ArgsDesc(vec![ + Arg(None, el!(Expr::Var("__tmp__to_json__".into()))), + Arg(None, el!(Expr::Str(" ".repeat(padding).into()))) + ]), + false + )), + )? + .try_cast_str("to json")?) + }) + } +} + +pub fn manifest_json_ex(val: &Val, padding: &str) -> Result { + let mut out = String::new(); + manifest_json_ex_buf(val, &mut out, padding, &mut String::new())?; + Ok(out) +} +fn manifest_json_ex_buf( + val: &Val, + buf: &mut String, + padding: &str, + cur_padding: &mut String, +) -> Result<()> { + use std::fmt::Write; + match val.unwrap_if_lazy()? { + Val::Bool(v) => { + if v { + buf.push_str("true"); + } else { + buf.push_str("false"); + } + } + Val::Null => buf.push_str("null"), + Val::Str(s) => buf.push_str(&escape_string_json(&s)), + Val::Num(n) => write!(buf, "{}", n).unwrap(), + Val::Arr(items) => { + buf.push_str("[\n"); + if !items.is_empty() { + let old_len = cur_padding.len(); + cur_padding.push_str(padding); + for (i, item) in items.iter().enumerate() { + if i != 0 { + buf.push_str(",\n") + } + buf.push_str(cur_padding); + manifest_json_ex_buf(item, buf, padding, cur_padding)?; + } + cur_padding.truncate(old_len); + } + buf.push('\n'); + buf.push_str(cur_padding); + buf.push(']'); + } + Val::Obj(obj) => { + buf.push_str("{\n"); + let fields = obj.visible_fields(); + if !fields.is_empty() { + let old_len = cur_padding.len(); + cur_padding.push_str(padding); + for (i, field) in fields.into_iter().enumerate() { + if i != 0 { + buf.push_str(",\n") + } + buf.push_str(cur_padding); + buf.push_str(&escape_string_json(&field)); + buf.push_str(": "); + manifest_json_ex_buf(&obj.get(field)?.unwrap(), buf, padding, cur_padding)?; + } + cur_padding.truncate(old_len); + } + buf.push('\n'); + buf.push_str(cur_padding); + buf.push('}'); + } + Val::Func(_) | Val::Intristic(_, _) => create_error_result(Error::RuntimeError("tried to manifest function".into()))?, + Val::Lazy(_) => unreachable!(), + }; + Ok(()) +} +pub fn escape_string_json(s: &str) -> String { + use std::fmt::Write; + let mut out = String::new(); + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\u{0008}' => out.push_str("\\b"), + '\u{000c}' => out.push_str("\\f"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c < 32 as char || (c >= 127 as char && c <= 159 as char) => { + write!(out, "\\u{:04x}", c as u32).unwrap() + } + c => out.push(c), + } + } + out.push('"'); + out +} + +#[test] +fn json_test() { + assert_eq!(escape_string_json("\u{001f}"), "\"\\u001f\"") +} --- /dev/null +++ b/crates/jrsonnet-parser/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "jrsonnet-parser" +version = "0.1.0" +authors = ["Лач "] +edition = "2018" + +[features] +default = [] +serialize = ["serde"] +deserialize = ["serde"] +# Adds ability to dump AST as source code for easy embedding +dump = ["structdump", "structdump-derive"] + +[dependencies] +peg = "0.6.2" +unescape = "0.1.0" + +serde = { version = "1.0.114", features = ["derive", "rc"], optional = true } +structdump = { version = "0.1.2", optional = true } +structdump-derive = { version = "0.1.2", optional = true } + +[dev-dependencies] +jrsonnet-stdlib = { version = "0.1.0", path = "../jrsonnet-stdlib" } --- /dev/null +++ b/crates/jrsonnet-parser/README.md @@ -0,0 +1,3 @@ +# jrsonnet-parser + +Parser for jsonnet language --- /dev/null +++ b/crates/jrsonnet-parser/src/expr.rs @@ -0,0 +1,321 @@ +use std::{fmt::Debug, ops::Deref, path::PathBuf, rc::Rc}; +#[cfg(feature = "dump")] +use structdump_derive::Codegen; +#[cfg(feature = "serialize")] +use serde::Serialize; +#[cfg(feature = "deserialize")] +use serde::Deserialize; + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub enum FieldName { + /// {fixed: 2} + Fixed(Rc), + /// {["dyn"+"amic"]: 3} + Dyn(LocExpr), +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Visibility { + /// : + Normal, + /// :: + Hidden, + /// ::: + Unhide, +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct AssertStmt(pub LocExpr, pub Option); + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct FieldMember { + pub name: FieldName, + pub plus: bool, + pub params: Option, + pub visibility: Visibility, + pub value: LocExpr, +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub enum Member { + Field(FieldMember), + BindStmt(BindSpec), + AssertStmt(AssertStmt), +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum UnaryOpType { + Plus, + Minus, + BitNot, + Not, +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum BinaryOpType { + Mul, + Div, + + Add, + Sub, + + Lhs, + Rhs, + + Lt, + Gt, + Lte, + Gte, + + BitAnd, + BitOr, + BitXor, + + And, + Or, +} + +/// name, default value +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct Param(pub Rc, pub Option); + +/// Defined function parameters +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, Clone, PartialEq)] +pub struct ParamsDesc(pub Rc>); +impl Deref for ParamsDesc { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct Arg(pub Option, pub LocExpr); + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct ArgsDesc(pub Vec); +impl Deref for ArgsDesc { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, Clone, PartialEq)] +pub struct BindSpec { + pub name: Rc, + pub params: Option, + pub value: LocExpr, +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct IfSpecData(pub LocExpr); + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct ForSpecData(pub Rc, pub LocExpr); + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub enum CompSpec { + IfSpec(IfSpecData), + ForSpec(ForSpecData), +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub struct ObjComp { + pub pre_locals: Vec, + pub key: LocExpr, + pub value: LocExpr, + pub post_locals: Vec, + pub compspecs: Vec, +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub enum ObjBody { + MemberList(Vec), + ObjComp(ObjComp), +} + +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum LiteralType { + This, + Super, + Dollar, + Null, + True, + False, +} + +#[derive(Debug, PartialEq)] +pub struct SliceDesc { + pub start: Option, + pub end: Option, + pub step: Option, +} + +/// Syntax base +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Debug, PartialEq)] +pub enum Expr { + Literal(LiteralType), + + /// String value: "hello" + Str(Rc), + /// Number: 1, 2.0, 2e+20 + Num(f64), + /// Variable name: test + Var(Rc), + + /// Array of expressions: [1, 2, "Hello"] + Arr(Vec), + /// Array comprehension: + /// ```jsonnet + /// ingredients: [ + /// { kind: kind, qty: 4 / 3 } + /// for kind in [ + /// 'Honey Syrup', + /// 'Lemon Juice', + /// 'Farmers Gin', + /// ] + /// ], + /// ``` + ArrComp(LocExpr, Vec), + + /// Object: {a: 2} + Obj(ObjBody), + /// Object extension: var1 {b: 2} + ObjExtend(LocExpr, ObjBody), + + /// (obj) + Parened(LocExpr), + + /// -2 + UnaryOp(UnaryOpType, LocExpr), + /// 2 - 2 + BinaryOp(LocExpr, BinaryOpType, LocExpr), + /// assert 2 == 2 : "Math is broken" + AssertExpr(AssertStmt, LocExpr), + /// local a = 2; { b: a } + LocalExpr(Vec, LocExpr), + + /// import "hello" + Import(PathBuf), + /// importStr "file.txt" + ImportStr(PathBuf), + /// error "I'm broken" + Error(LocExpr), + /// a(b, c) + Apply(LocExpr, ArgsDesc, bool), + /// a[b] + Index(LocExpr, LocExpr), + /// function(x) x + Function(ParamsDesc, LocExpr), + /// if true == false then 1 else 2 + IfElse { + cond: IfSpecData, + cond_then: LocExpr, + cond_else: Option, + }, +} + +/// file, begin offset, end offset +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Clone, PartialEq)] +pub struct ExprLocation(pub Rc, pub usize, pub usize); +impl Debug for ExprLocation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}:{:?}-{:?}", self.0, self.1, self.2) + } +} + +/// Holds AST expression and its location in source file +#[cfg_attr(feature = "dump", derive(Codegen))] +#[cfg_attr(feature = "serialize", derive(Serialize))] +#[cfg_attr(feature = "deserialize", derive(Deserialize))] +#[derive(Clone, PartialEq)] +pub struct LocExpr(pub Rc, pub Option); +impl Debug for LocExpr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?} from {:?}", self.0, self.1) + } +} + +/// Creates LocExpr from Expr and ExprLocation components +#[macro_export] +macro_rules! loc_expr { + ($expr:expr, $need_loc:expr,($name:expr, $start:expr, $end:expr)) => { + LocExpr( + std::rc::Rc::new($expr), + if $need_loc { + Some(ExprLocation($name, $start, $end)) + } else { + None + }, + ) + }; +} + +/// Creates LocExpr without location info +#[macro_export] +macro_rules! loc_expr_todo { + ($expr:expr) => { + LocExpr(Rc::new($expr), None) + }; +} --- /dev/null +++ b/crates/jrsonnet-parser/src/lib.rs @@ -0,0 +1,578 @@ +#![feature(box_syntax)] +#![feature(test)] + +extern crate test; + +use peg::parser; +use std::{path::PathBuf, rc::Rc}; +mod expr; +pub use expr::*; +pub use peg; + +pub struct ParserSettings { + pub loc_data: bool, + pub file_name: Rc, +} + +parser! { + grammar jsonnet_parser() for str { + use peg::ParseLiteral; + + /// Standard C-like comments + rule comment() + = "//" (!['\n'][_])* "\n" + / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/" + / "#" (!['\n'][_])* "\n" + + rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("") + rule _() = single_whitespace()* + + /// For comma-delimited elements + rule comma() = quiet!{_ "," _} / expected!("") + rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()} + rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()} + rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z'] + /// Sequence of digits + rule uint() -> u64 = a:$(digit()+) { a.parse().unwrap() } + /// Number in scientific notation format + rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("") + + /// Reserved word followed by any non-alphanumberic + rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident() + rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("") + + rule keyword(id: &'static str) + = ##parse_string_literal(id) end_of_ident() + // Adds location data information to existing expression + rule l(s: &ParserSettings, x: rule) -> LocExpr + = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))} + + pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) } + pub rule params(s: &ParserSettings) -> expr::ParamsDesc + = params:param(s) ** comma() comma()? { + let mut defaults_started = false; + for param in ¶ms { + defaults_started = defaults_started || param.1.is_some(); + assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals"); + } + expr::ParamsDesc(Rc::new(params)) + } + / { expr::ParamsDesc(Rc::new(Vec::new())) } + + pub rule arg(s: &ParserSettings) -> expr::Arg + = name:$(id()) _ "=" _ expr:expr(s) {expr::Arg(Some(name.into()), expr)} + / expr:expr(s) {expr::Arg(None, expr)} + pub rule args(s: &ParserSettings) -> expr::ArgsDesc + = args:arg(s) ** comma() comma()? { + let mut named_started = false; + for arg in &args { + named_started = named_started || arg.0.is_some(); + assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals"); + } + expr::ArgsDesc(args) + } + / { expr::ArgsDesc(Vec::new()) } + + pub rule bind(s: &ParserSettings) -> expr::BindSpec + = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}} + / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}} + pub rule assertion(s: &ParserSettings) -> expr::AssertStmt + = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) } + + pub rule whole_line() -> &'input str + = str:$((!['\n'][_])* "\n") {str} + pub rule string_block() -> String + = "|||" (!['\n']single_whitespace())* "\n" + prefix:[' ']+ first_line:whole_line() + lines:([' ']*<{prefix.len()}> s:whole_line() {s})* + [' ']*<, {prefix.len() - 1}> "|||" + {let mut l = first_line.to_owned(); l.extend(lines); l} + pub rule string() -> String + = "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()} + / "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()} + / "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")} + / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")} + / string_block() + + pub rule field_name(s: &ParserSettings) -> expr::FieldName + = name:$(id()) {expr::FieldName::Fixed(name.into())} + / name:string() {expr::FieldName::Fixed(name.into())} + / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)} + pub rule visibility() -> expr::Visibility + = ":::" {expr::Visibility::Unhide} + / "::" {expr::Visibility::Hidden} + / ":" {expr::Visibility::Normal} + pub rule field(s: &ParserSettings) -> expr::FieldMember + = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{ + name, + plus: plus.is_some(), + params: None, + visibility, + value, + }} + / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{ + name, + plus: false, + params: Some(params), + visibility, + value, + }} + pub rule obj_local(s: &ParserSettings) -> BindSpec + = keyword("local") _ bind:bind(s) {bind} + pub rule member(s: &ParserSettings) -> expr::Member + = bind:obj_local(s) {expr::Member::BindStmt(bind)} + / assertion:assertion(s) {expr::Member::AssertStmt(assertion)} + / field:field(s) {expr::Member::Field(field)} + pub rule objinside(s: &ParserSettings) -> expr::ObjBody + = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? { + let mut compspecs = vec![CompSpec::ForSpec(forspec)]; + compspecs.extend(others.unwrap_or_default()); + expr::ObjBody::ObjComp(expr::ObjComp{ + pre_locals, + key, + value, + post_locals, + compspecs, + }) + } + / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)} + pub rule ifspec(s: &ParserSettings) -> IfSpecData + = keyword("if") _ expr:expr(s) {IfSpecData(expr)} + pub rule forspec(s: &ParserSettings) -> ForSpecData + = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)} + pub rule compspec(s: &ParserSettings) -> Vec + = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s} + pub rule local_expr(s: &ParserSettings) -> LocExpr + = l(s,) + pub rule string_expr(s: &ParserSettings) -> LocExpr + = l(s, ) + pub rule obj_expr(s: &ParserSettings) -> LocExpr + = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>) + pub rule array_expr(s: &ParserSettings) -> LocExpr + = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>) + pub rule array_comp_expr(s: &ParserSettings) -> LocExpr + = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" { + let mut specs = vec![CompSpec::ForSpec(forspec)]; + specs.extend(others.unwrap_or_default()); + Expr::ArrComp(expr, specs) + }>) + pub rule number_expr(s: &ParserSettings) -> LocExpr + = l(s,) + pub rule var_expr(s: &ParserSettings) -> LocExpr + = l(s,) + pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr + = l(s,) + + pub rule literal(s: &ParserSettings) -> LocExpr + = l(s,) + + pub rule expr_basic(s: &ParserSettings) -> LocExpr + = literal(s) + + / string_expr(s) / number_expr(s) + / array_expr(s) + / obj_expr(s) + / array_expr(s) + / array_comp_expr(s) + + / l(s,) + / l(s,) + + / var_expr(s) + / local_expr(s) + / if_then_else_expr(s) + + / l(s,) + / l(s,) + + / l(s,) + + rule slice_part(s: &ParserSettings) -> Option + = e:(_ e:expr(s) _{e})? {e} + pub rule slice_desc(s: &ParserSettings) -> SliceDesc + = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? { + let (end, step) = if let Some((end, step)) = pair { + (end, step) + }else{ + (None, None) + }; + + SliceDesc { start, end, step } + } + + rule expr(s: &ParserSettings) -> LocExpr + = start:position!() a:precedence! { + a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))} + -- + a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))} + -- + a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))} + -- + a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))} + -- + a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))} + -- + a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("equals".into())) + )), + ArgsDesc(vec![Arg(None, a), Arg(None, b)]), + true + ))} + a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("equals".into())) + )), + ArgsDesc(vec![Arg(None, a), Arg(None, b)]), + true + ))))} + -- + a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))} + a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))} + a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))} + a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))} + a:(@) _ keyword("in") _ b:@ {loc_expr_todo!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("objectHasEx".into())) + )), ArgsDesc(vec![Arg(None, b), Arg(None, a), Arg(None, el!(Expr::Literal(LiteralType::True)))]), + true + ))} + -- + a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))} + a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))} + -- + a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))} + a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))} + -- + a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))} + a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))} + a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("mod".into())) + )), ArgsDesc(vec![Arg(None, a), Arg(None, b)]), + true + ))} + -- + "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))} + "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))} + "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) } + -- + a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Apply( + el!(Expr::Index( + el!(Expr::Var("std".into())), + el!(Expr::Str("slice".into())), + )), + ArgsDesc(vec![ + Arg(None, a), + Arg(None, s.start.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), + Arg(None, s.end.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), + Arg(None, s.step.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), + ]), + true, + ))} + a:(@) _ "." _ s:$(id()) {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s.into()))))} + a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))} + a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args, ts.is_some()))} + a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))} + -- + e:expr_basic(s) {e} + "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))} + } end:position!() { + let LocExpr(e, _) = a; + LocExpr(e, if s.loc_data { + Some(ExprLocation(s.file_name.clone(), start, end)) + } else { + None + }) + } + / e:expr_basic(s) {e} + + pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e} + } +} + +pub type ParseError = peg::error::ParseError; +pub fn parse(str: &str, settings: &ParserSettings) -> Result { + jsonnet_parser::jsonnet(str, settings) +} + +#[macro_export] +macro_rules! el { + ($expr:expr) => { + LocExpr(std::rc::Rc::new($expr), None) + }; +} + +#[cfg(test)] +pub mod tests { + use super::{expr::*, parse}; + use crate::ParserSettings; + use std::path::PathBuf; + use std::rc::Rc; + + macro_rules! parse { + ($s:expr) => { + parse( + $s, + &ParserSettings { + loc_data: false, + file_name: Rc::new(PathBuf::from("/test.jsonnet")), + }, + ) + .unwrap() + }; + } + + mod expressions { + use super::*; + + pub fn basic_math() -> LocExpr { + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Add, + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Mul, + el!(Expr::Num(2.0)), + )), + )) + } + } + + #[test] + fn multiline_string() { + assert_eq!( + parse!("|||\n Hello world!\n a\n|||"), + el!(Expr::Str("Hello world!\n a\n".into())), + ) + } + + #[test] + fn slice() { + parse!("a[1:]"); + parse!("a[1::]"); + parse!("a[:1:]"); + parse!("a[::1]"); + parse!("str[:len - 1]"); + } + + #[test] + fn string_escaping() { + assert_eq!( + parse!(r#""Hello, \"world\"!""#), + el!(Expr::Str(r#"Hello, "world"!"#.into())), + ); + assert_eq!( + parse!(r#"'Hello \'world\'!'"#), + el!(Expr::Str("Hello 'world'!".into())), + ); + assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),); + } + + #[test] + fn string_unescaping() { + assert_eq!( + parse!(r#""Hello\nWorld""#), + el!(Expr::Str("Hello\nWorld".into())), + ); + } + + #[test] + fn string_verbantim() { + assert_eq!( + parse!(r#"@"Hello\n""World""""#), + el!(Expr::Str("Hello\\n\"World\"".into())), + ); + } + + #[test] + fn imports() { + assert_eq!( + parse!("import \"hello\""), + el!(Expr::Import(PathBuf::from("hello"))), + ); + assert_eq!( + parse!("importstr \"garnish.txt\""), + el!(Expr::ImportStr(PathBuf::from("garnish.txt"))) + ); + } + + #[test] + fn empty_object() { + assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![])))); + } + + #[test] + fn basic_math() { + assert_eq!( + parse!("2+2*2"), + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Add, + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Mul, + el!(Expr::Num(2.0)) + )) + )) + ); + } + + #[test] + fn basic_math_with_indents() { + assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math()); + } + + #[test] + fn basic_math_parened() { + assert_eq!( + parse!("2+(2+2*2)"), + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Add, + el!(Expr::Parened(expressions::basic_math())), + )) + ); + } + + /// Comments should not affect parsing + #[test] + fn comments() { + assert_eq!( + parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"), + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Add, + el!(Expr::BinaryOp( + el!(Expr::Num(3.0)), + BinaryOpType::Mul, + el!(Expr::Num(4.0)) + )) + )) + ); + } + + /// Comments should be able to be escaped + #[test] + fn comment_escaping() { + assert_eq!( + parse!("2/*\\*/+*/ - 22"), + el!(Expr::BinaryOp( + el!(Expr::Num(2.0)), + BinaryOpType::Sub, + el!(Expr::Num(22.0)) + )) + ); + } + + #[test] + fn suffix() { + // assert_eq!(parse!("std.test"), el!(Expr::Num(2.2))); + // assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2))); + // assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2))); + // assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2))) + } + + #[test] + fn array_comp() { + use Expr::*; + assert_eq!( + parse!("[std.deepJoin(x) for x in arr]"), + el!(ArrComp( + el!(Apply( + el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))), + ArgsDesc(vec![Arg(None, el!(Var("x".into())))]), + false, + )), + vec![CompSpec::ForSpec(ForSpecData( + "x".into(), + el!(Var("arr".into())) + ))] + )), + ) + } + + #[test] + fn reserved() { + use Expr::*; + assert_eq!(parse!("null"), el!(Literal(LiteralType::Null))); + assert_eq!(parse!("nulla"), el!(Var("nulla".into()))); + } + + #[test] + fn multiple_args_buf() { + parse!("a(b, null_fields)"); + } + + #[test] + fn infix_precedence() { + use Expr::*; + assert_eq!( + parse!("!a && !b"), + el!(BinaryOp( + el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))), + BinaryOpType::And, + el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into())))) + )) + ); + } + + #[test] + fn infix_precedence_division() { + use Expr::*; + assert_eq!( + parse!("!a / !b"), + el!(BinaryOp( + el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))), + BinaryOpType::Div, + el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into())))) + )) + ); + } + + #[test] + fn double_negation() { + use Expr::*; + assert_eq!( + parse!("!!a"), + el!(UnaryOp( + UnaryOpType::Not, + el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))) + )) + ) + } + + #[test] + fn array_test_error() { + parse!("[a for a in b if c for e in f]"); + // ^^^^ failed code + } + + #[test] + fn can_parse_stdlib() { + parse!(jrsonnet_stdlib::STDLIB_STR); + } + + use test::Bencher; + + // From source code + #[bench] + fn bench_parse_peg(b: &mut Bencher) { + b.iter(|| parse!(jrsonnet_stdlib::STDLIB_STR)) + } +} --- /dev/null +++ b/crates/jrsonnet-stdlib/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "jrsonnet-stdlib" +version = "0.1.0" +authors = ["Лач "] +edition = "2018" + +[features] + +[dependencies] --- /dev/null +++ b/crates/jrsonnet-stdlib/README.md @@ -0,0 +1,3 @@ +# jrsonnet-stdlib + +Jsonnet standard library packaged as crate --- /dev/null +++ b/crates/jrsonnet-stdlib/src/lib.rs @@ -0,0 +1 @@ +pub const STDLIB_STR: &str = include_str!("./std.jsonnet"); --- /dev/null +++ b/crates/jrsonnet-stdlib/src/std.jsonnet @@ -0,0 +1,1346 @@ +{ + __intristic_namespace__: 'std', + + local std = self, + local id = function(x) x, + + isString(v):: std.type(v) == 'string', + isNumber(v):: std.type(v) == 'number', + isBoolean(v):: std.type(v) == 'boolean', + isObject(v):: std.type(v) == 'object', + isArray(v):: std.type(v) == 'array', + isFunction(v):: std.type(v) == 'function', + + toString(a):: + if std.type(a) == 'string' then a else '' + a, + + substr(str, from, len):: + assert std.isString(str) : 'substr first parameter should be a string, got ' + std.type(str); + assert std.isNumber(from) : 'substr second parameter should be a string, got ' + std.type(from); + assert std.isNumber(len) : 'substr third parameter should be a string, got ' + std.type(len); + assert len >= 0 : 'substr third parameter should be greater than zero, got ' + len; + std.join('', std.makeArray(std.max(0, std.min(len, std.length(str) - from)), function(i) str[i + from])), + + startsWith(a, b):: + if std.length(a) < std.length(b) then + false + else + std.substr(a, 0, std.length(b)) == b, + + endsWith(a, b):: + if std.length(a) < std.length(b) then + false + else + std.substr(a, std.length(a) - std.length(b), std.length(b)) == b, + + lstripChars(str, chars):: + if std.length(str) > 0 && std.member(chars, str[0]) then + std.lstripChars(str[1:], chars) + else + str, + + rstripChars(str, chars):: + local len = std.length(str); + if len > 0 && std.member(chars, str[len - 1]) then + std.rstripChars(str[:len - 1], chars) + else + str, + + stripChars(str, chars):: + std.lstripChars(std.rstripChars(str, chars), chars), + + stringChars(str):: + std.makeArray(std.length(str), function(i) str[i]), + + local parse_nat(str, base) = + assert base > 0 && base <= 16 : 'integer base %d invalid' % base; + // These codepoints are in ascending order: + local zero_code = std.codepoint('0'); + local upper_a_code = std.codepoint('A'); + local lower_a_code = std.codepoint('a'); + local addDigit(aggregate, char) = + local code = std.codepoint(char); + local digit = if code >= lower_a_code then + code - lower_a_code + 10 + else if code >= upper_a_code then + code - upper_a_code + 10 + else + code - zero_code; + assert digit >= 0 && digit < base : '%s is not a base %d integer' % [str, base]; + base * aggregate + digit; + std.foldl(addDigit, std.stringChars(str), 0), + + parseInt(str):: + assert std.isString(str) : 'Expected string, got ' + std.type(str); + assert std.length(str) > 0 && str != '-' : 'Not an integer: "%s"' % [str]; + if str[0] == '-' then + -parse_nat(str[1:], 10) + else + parse_nat(str, 10), + + parseOctal(str):: + assert std.isString(str) : 'Expected string, got ' + std.type(str); + assert std.length(str) > 0 : 'Not an octal number: ""'; + parse_nat(str, 8), + + parseHex(str):: + assert std.isString(str) : 'Expected string, got ' + std.type(str); + assert std.length(str) > 0 : 'Not hexadecimal: ""'; + parse_nat(str, 16), + + split(str, c):: + assert std.isString(str) : 'std.split first parameter should be a string, got ' + std.type(str); + assert std.isString(c) : 'std.split second parameter should be a string, got ' + std.type(c); + assert std.length(c) == 1 : 'std.split second parameter should have length 1, got ' + std.length(c); + std.splitLimit(str, c, -1), + + splitLimit(str, c, maxsplits):: + assert std.isString(str) : 'std.splitLimit first parameter should be a string, got ' + std.type(str); + assert std.isString(c) : 'std.splitLimit second parameter should be a string, got ' + std.type(c); + assert std.length(c) == 1 : 'std.splitLimit second parameter should have length 1, got ' + std.length(c); + assert std.isNumber(maxsplits) : 'std.splitLimit third parameter should be a number, got ' + std.type(maxsplits); + local aux(str, delim, i, arr, v) = + local c = str[i]; + local i2 = i + 1; + if i >= std.length(str) then + arr + [v] + else if c == delim && (maxsplits == -1 || std.length(arr) < maxsplits) then + aux(str, delim, i2, arr + [v], '') tailstrict + else + aux(str, delim, i2, arr, v + c) tailstrict; + aux(str, c, 0, [], ''), + + strReplace(str, from, to):: + assert std.isString(str); + assert std.isString(from); + assert std.isString(to); + assert from != '' : "'from' string must not be zero length."; + + // Cache for performance. + local str_len = std.length(str); + local from_len = std.length(from); + + // True if from is at str[i]. + local found_at(i) = str[i:i + from_len] == from; + + // Return the remainder of 'str' starting with 'start_index' where + // all occurrences of 'from' after 'curr_index' are replaced with 'to'. + local replace_after(start_index, curr_index, acc) = + if curr_index > str_len then + acc + str[start_index:curr_index] + else if found_at(curr_index) then + local new_index = curr_index + std.length(from); + replace_after(new_index, new_index, acc + str[start_index:curr_index] + to) tailstrict + else + replace_after(start_index, curr_index + 1, acc) tailstrict; + + // if from_len==1, then we replace by splitting and rejoining the + // string which is much faster than recursing on replace_after + if from_len == 1 then + std.join(to, std.split(str, from)) + else + replace_after(0, 0, ''), + + asciiUpper(str):: + local cp = std.codepoint; + local up_letter(c) = if cp(c) >= 97 && cp(c) < 123 then + std.char(cp(c) - 32) + else + c; + std.join('', std.map(up_letter, std.stringChars(str))), + + asciiLower(str):: + local cp = std.codepoint; + local down_letter(c) = if cp(c) >= 65 && cp(c) < 91 then + std.char(cp(c) + 32) + else + c; + std.join('', std.map(down_letter, std.stringChars(str))), + + range(from, to):: + std.makeArray(to - from + 1, function(i) i + from), + + repeat(what, count):: + local joiner = + if std.isString(what) then '' + else if std.isArray(what) then [] + else error 'std.repeat first argument must be an array or a string'; + std.join(joiner, std.makeArray(count, function(i) what)), + + slice(indexable, index, end, step):: + local invar = + // loop invariant with defaults applied + { + indexable: indexable, + index: + if index == null then 0 + else index, + end: + if end == null then std.length(indexable) + else end, + step: + if step == null then 1 + else step, + length: std.length(indexable), + type: std.type(indexable), + }; + assert invar.index >= 0 && invar.end >= 0 && invar.step >= 0 : 'got [%s:%s:%s] but negative index, end, and steps are not supported' % [invar.index, invar.end, invar.step]; + assert step != 0 : 'got %s but step must be greater than 0' % step; + assert std.isString(indexable) || std.isArray(indexable) : 'std.slice accepts a string or an array, but got: %s' % std.type(indexable); + local build(slice, cur) = + if cur >= invar.end || cur >= invar.length then + slice + else + build( + if invar.type == 'string' then + slice + invar.indexable[cur] + else + slice + [invar.indexable[cur]], + cur + invar.step + ) tailstrict; + build(if invar.type == 'string' then '' else [], invar.index), + + member(arr, x):: + if std.isArray(arr) then + std.count(arr, x) > 0 + else if std.isString(arr) then + std.length(std.findSubstr(x, arr)) > 0 + else error 'std.member first argument must be an array or a string', + + count(arr, x):: std.length(std.filter(function(v) v == x, arr)), + + mod(a, b):: + if std.isNumber(a) && std.isNumber(b) then + std.modulo(a, b) + else if std.isString(a) then + std.format(a, b) + else + error 'Operator % cannot be used on types ' + std.type(a) + ' and ' + std.type(b) + '.', + + map(func, arr):: + if !std.isFunction(func) then + error ('std.map first param must be function, got ' + std.type(func)) + else if !std.isArray(arr) && !std.isString(arr) then + error ('std.map second param must be array / string, got ' + std.type(arr)) + else + std.makeArray(std.length(arr), function(i) func(arr[i])), + + mapWithIndex(func, arr):: + if !std.isFunction(func) then + error ('std.mapWithIndex first param must be function, got ' + std.type(func)) + else if !std.isArray(arr) && !std.isString(arr) then + error ('std.mapWithIndex second param must be array, got ' + std.type(arr)) + else + std.makeArray(std.length(arr), function(i) func(i, arr[i])), + + mapWithKey(func, obj):: + if !std.isFunction(func) then + error ('std.mapWithKey first param must be function, got ' + std.type(func)) + else if !std.isObject(obj) then + error ('std.mapWithKey second param must be object, got ' + std.type(obj)) + else + { [k]: func(k, obj[k]) for k in std.objectFields(obj) }, + + flatMap(func, arr):: + if !std.isFunction(func) then + error ('std.flatMap first param must be function, got ' + std.type(func)) + else if std.isArray(arr) then + std.flattenArrays(std.makeArray(std.length(arr), function(i) func(arr[i]))) + else if std.isString(arr) then + std.join('', std.makeArray(std.length(arr), function(i) func(arr[i]))) + else error ('std.flatMap second param must be array / string, got ' + std.type(arr)), + + join(sep, arr):: + local aux(arr, i, first, running) = + if i >= std.length(arr) then + running + else if arr[i] == null then + aux(arr, i + 1, first, running) tailstrict + else if std.type(arr[i]) != std.type(sep) then + error 'expected %s but arr[%d] was %s ' % [std.type(sep), i, std.type(arr[i])] + else if first then + aux(arr, i + 1, false, running + arr[i]) tailstrict + else + aux(arr, i + 1, false, running + sep + arr[i]) tailstrict; + if !std.isArray(arr) then + error 'join second parameter should be array, got ' + std.type(arr) + else if std.isString(sep) then + aux(arr, 0, true, '') + else if std.isArray(sep) then + aux(arr, 0, true, []) + else + error 'join first parameter should be string or array, got ' + std.type(sep), + + lines(arr):: + std.join('\n', arr + ['']), + + deepJoin(arr):: + if std.isString(arr) then + arr + else if std.isArray(arr) then + std.join('', [std.deepJoin(x) for x in arr]) + else + error 'Expected string or array, got %s' % std.type(arr), + + + format(str, vals):: + + ///////////////////////////// + // Parse the mini-language // + ///////////////////////////// + + local try_parse_mapping_key(str, i) = + assert i < std.length(str) : 'Truncated format code.'; + local c = str[i]; + if c == '(' then + local consume(str, j, v) = + if j >= std.length(str) then + error 'Truncated format code.' + else + local c = str[j]; + if c != ')' then + consume(str, j + 1, v + c) + else + { i: j + 1, v: v }; + consume(str, i + 1, '') + else + { i: i, v: null }; + + local try_parse_cflags(str, i) = + local consume(str, j, v) = + assert j < std.length(str) : 'Truncated format code.'; + local c = str[j]; + if c == '#' then + consume(str, j + 1, v { alt: true }) + else if c == '0' then + consume(str, j + 1, v { zero: true }) + else if c == '-' then + consume(str, j + 1, v { left: true }) + else if c == ' ' then + consume(str, j + 1, v { blank: true }) + else if c == '+' then + consume(str, j + 1, v { sign: true }) + else + { i: j, v: v }; + consume(str, i, { alt: false, zero: false, left: false, blank: false, sign: false }); + + local try_parse_field_width(str, i) = + if i < std.length(str) && str[i] == '*' then + { i: i + 1, v: '*' } + else + local consume(str, j, v) = + assert j < std.length(str) : 'Truncated format code.'; + local c = str[j]; + if c == '0' then + consume(str, j + 1, v * 10 + 0) + else if c == '1' then + consume(str, j + 1, v * 10 + 1) + else if c == '2' then + consume(str, j + 1, v * 10 + 2) + else if c == '3' then + consume(str, j + 1, v * 10 + 3) + else if c == '4' then + consume(str, j + 1, v * 10 + 4) + else if c == '5' then + consume(str, j + 1, v * 10 + 5) + else if c == '6' then + consume(str, j + 1, v * 10 + 6) + else if c == '7' then + consume(str, j + 1, v * 10 + 7) + else if c == '8' then + consume(str, j + 1, v * 10 + 8) + else if c == '9' then + consume(str, j + 1, v * 10 + 9) + else + { i: j, v: v }; + consume(str, i, 0); + + local try_parse_precision(str, i) = + assert i < std.length(str) : 'Truncated format code.'; + local c = str[i]; + if c == '.' then + try_parse_field_width(str, i + 1) + else + { i: i, v: null }; + + // Ignored, if it exists. + local try_parse_length_modifier(str, i) = + assert i < std.length(str) : 'Truncated format code.'; + local c = str[i]; + if c == 'h' || c == 'l' || c == 'L' then + i + 1 + else + i; + + local parse_conv_type(str, i) = + assert i < std.length(str) : 'Truncated format code.'; + local c = str[i]; + if c == 'd' || c == 'i' || c == 'u' then + { i: i + 1, v: 'd', caps: false } + else if c == 'o' then + { i: i + 1, v: 'o', caps: false } + else if c == 'x' then + { i: i + 1, v: 'x', caps: false } + else if c == 'X' then + { i: i + 1, v: 'x', caps: true } + else if c == 'e' then + { i: i + 1, v: 'e', caps: false } + else if c == 'E' then + { i: i + 1, v: 'e', caps: true } + else if c == 'f' then + { i: i + 1, v: 'f', caps: false } + else if c == 'F' then + { i: i + 1, v: 'f', caps: true } + else if c == 'g' then + { i: i + 1, v: 'g', caps: false } + else if c == 'G' then + { i: i + 1, v: 'g', caps: true } + else if c == 'c' then + { i: i + 1, v: 'c', caps: false } + else if c == 's' then + { i: i + 1, v: 's', caps: false } + else if c == '%' then + { i: i + 1, v: '%', caps: false } + else + error 'Unrecognised conversion type: ' + c; + + + // Parsed initial %, now the rest. + local parse_code(str, i) = + assert i < std.length(str) : 'Truncated format code.'; + local mkey = try_parse_mapping_key(str, i); + local cflags = try_parse_cflags(str, mkey.i); + local fw = try_parse_field_width(str, cflags.i); + local prec = try_parse_precision(str, fw.i); + local len_mod = try_parse_length_modifier(str, prec.i); + local ctype = parse_conv_type(str, len_mod); + { + i: ctype.i, + code: { + mkey: mkey.v, + cflags: cflags.v, + fw: fw.v, + prec: prec.v, + ctype: ctype.v, + caps: ctype.caps, + }, + }; + + // Parse a format string (containing none or more % format tags). + local parse_codes(str, i, out, cur) = + if i >= std.length(str) then + out + [cur] + else + local c = str[i]; + if c == '%' then + local r = parse_code(str, i + 1); + parse_codes(str, r.i, out + [cur, r.code], '') tailstrict + else + parse_codes(str, i + 1, out, cur + c) tailstrict; + + local codes = parse_codes(str, 0, [], ''); + + + /////////////////////// + // Format the values // + /////////////////////// + + // Useful utilities + local padding(w, s) = + local aux(w, v) = + if w <= 0 then + v + else + aux(w - 1, v + s); + aux(w, ''); + + // Add s to the left of str so that its length is at least w. + local pad_left(str, w, s) = + padding(w - std.length(str), s) + str; + + // Add s to the right of str so that its length is at least w. + local pad_right(str, w, s) = + str + padding(w - std.length(str), s); + + // Render an integer (e.g., decimal or octal). + local render_int(n__, min_chars, min_digits, blank, sign, radix, zero_prefix) = + local n_ = std.abs(n__); + local aux(n) = + if n == 0 then + zero_prefix + else + aux(std.floor(n / radix)) + (n % radix); + local dec = if std.floor(n_) == 0 then '0' else aux(std.floor(n_)); + local neg = n__ < 0; + local zp = min_chars - (if neg || blank || sign then 1 else 0); + local zp2 = std.max(zp, min_digits); + local dec2 = pad_left(dec, zp2, '0'); + (if neg then '-' else if sign then '+' else if blank then ' ' else '') + dec2; + + // Render an integer in hexadecimal. + local render_hex(n__, min_chars, min_digits, blank, sign, add_zerox, capitals) = + local numerals = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + if capitals then ['A', 'B', 'C', 'D', 'E', 'F'] + else ['a', 'b', 'c', 'd', 'e', 'f']; + local n_ = std.abs(n__); + local aux(n) = + if n == 0 then + '' + else + aux(std.floor(n / 16)) + numerals[n % 16]; + local hex = if std.floor(n_) == 0 then '0' else aux(std.floor(n_)); + local neg = n__ < 0; + local zp = min_chars - (if neg || blank || sign then 1 else 0) + - (if add_zerox then 2 else 0); + local zp2 = std.max(zp, min_digits); + local hex2 = (if add_zerox then (if capitals then '0X' else '0x') else '') + + pad_left(hex, zp2, '0'); + (if neg then '-' else if sign then '+' else if blank then ' ' else '') + hex2; + + local strip_trailing_zero(str) = + local aux(str, i) = + if i < 0 then + '' + else + if str[i] == '0' then + aux(str, i - 1) + else + std.substr(str, 0, i + 1); + aux(str, std.length(str) - 1); + + // Render floating point in decimal form + local render_float_dec(n__, zero_pad, blank, sign, ensure_pt, trailing, prec) = + local n_ = std.abs(n__); + local whole = std.floor(n_); + local dot_size = if prec == 0 && !ensure_pt then 0 else 1; + local zp = zero_pad - prec - dot_size; + local str = render_int(std.sign(n__) * whole, zp, 0, blank, sign, 10, ''); + if prec == 0 then + str + if ensure_pt then '.' else '' + else + local frac = std.floor((n_ - whole) * std.pow(10, prec) + 0.5); + if trailing || frac > 0 then + local frac_str = render_int(frac, prec, 0, false, false, 10, ''); + str + '.' + if !trailing then strip_trailing_zero(frac_str) else frac_str + else + str; + + // Render floating point in scientific form + local render_float_sci(n__, zero_pad, blank, sign, ensure_pt, trailing, caps, prec) = + local exponent = if n__ == 0 then 0 else std.floor(std.log(std.abs(n__)) / std.log(10)); + local suff = (if caps then 'E' else 'e') + + render_int(exponent, 3, 0, false, true, 10, ''); + local mantissa = if exponent == -324 then + // Avoid a rounding error where std.pow(10, -324) is 0 + // -324 is the smallest exponent possible. + n__ * 10 / std.pow(10, exponent + 1) + else + n__ / std.pow(10, exponent); + local zp2 = zero_pad - std.length(suff); + render_float_dec(mantissa, zp2, blank, sign, ensure_pt, trailing, prec) + suff; + + // Render a value with an arbitrary format code. + local format_code(val, code, fw, prec_or_null, i) = + local cflags = code.cflags; + local fpprec = if prec_or_null != null then prec_or_null else 6; + local iprec = if prec_or_null != null then prec_or_null else 0; + local zp = if cflags.zero && !cflags.left then fw else 0; + if code.ctype == 's' then + std.toString(val) + else if code.ctype == 'd' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + render_int(val, zp, iprec, cflags.blank, cflags.sign, 10, '') + else if code.ctype == 'o' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + local zero_prefix = if cflags.alt then '0' else ''; + render_int(val, zp, iprec, cflags.blank, cflags.sign, 8, zero_prefix) + else if code.ctype == 'x' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + render_hex(val, + zp, + iprec, + cflags.blank, + cflags.sign, + cflags.alt, + code.caps) + else if code.ctype == 'f' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + render_float_dec(val, + zp, + cflags.blank, + cflags.sign, + cflags.alt, + true, + fpprec) + else if code.ctype == 'e' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + render_float_sci(val, + zp, + cflags.blank, + cflags.sign, + cflags.alt, + true, + code.caps, + fpprec) + else if code.ctype == 'g' then + if std.type(val) != 'number' then + error 'Format required number at ' + + i + ', got ' + std.type(val) + else + local exponent = std.floor(std.log(std.abs(val)) / std.log(10)); + if exponent < -4 || exponent >= fpprec then + render_float_sci(val, + zp, + cflags.blank, + cflags.sign, + cflags.alt, + cflags.alt, + code.caps, + fpprec - 1) + else + local digits_before_pt = std.max(1, exponent + 1); + render_float_dec(val, + zp, + cflags.blank, + cflags.sign, + cflags.alt, + cflags.alt, + fpprec - digits_before_pt) + else if code.ctype == 'c' then + if std.type(val) == 'number' then + std.char(val) + else if std.type(val) == 'string' then + if std.length(val) == 1 then + val + else + error '%c expected 1-sized string got: ' + std.length(val) + else + error '%c expected number / string, got: ' + std.type(val) + else + error 'Unknown code: ' + code.ctype; + + // Render a parsed format string with an array of values. + local format_codes_arr(codes, arr, i, j, v) = + if i >= std.length(codes) then + if j < std.length(arr) then + error ('Too many values to format: ' + std.length(arr) + ', expected ' + j) + else + v + else + local code = codes[i]; + if std.type(code) == 'string' then + format_codes_arr(codes, arr, i + 1, j, v + code) tailstrict + else + local tmp = if code.fw == '*' then { + j: j + 1, + fw: if j >= std.length(arr) then + error ('Not enough values to format: ' + std.length(arr) + ', expected at least ' + j) + else + arr[j], + } else { + j: j, + fw: code.fw, + }; + local tmp2 = if code.prec == '*' then { + j: tmp.j + 1, + prec: if tmp.j >= std.length(arr) then + error ('Not enough values to format: ' + std.length(arr) + ', expected at least ' + tmp.j) + else + arr[tmp.j], + } else { + j: tmp.j, + prec: code.prec, + }; + local j2 = tmp2.j; + local val = + if j2 < std.length(arr) then + arr[j2] + else + error ('Not enough values to format: ' + std.length(arr) + ', expected more than ' + j2); + local s = + if code.ctype == '%' then + '%' + else + format_code(val, code, tmp.fw, tmp2.prec, j2); + local s_padded = + if code.cflags.left then + pad_right(s, tmp.fw, ' ') + else + pad_left(s, tmp.fw, ' '); + local j3 = + if code.ctype == '%' then + j2 + else + j2 + 1; + format_codes_arr(codes, arr, i + 1, j3, v + s_padded) tailstrict; + + // Render a parsed format string with an object of values. + local format_codes_obj(codes, obj, i, v) = + if i >= std.length(codes) then + v + else + local code = codes[i]; + if std.type(code) == 'string' then + format_codes_obj(codes, obj, i + 1, v + code) tailstrict + else + local f = + if code.mkey == null then + error 'Mapping keys required.' + else + code.mkey; + local fw = + if code.fw == '*' then + error 'Cannot use * field width with object.' + else + code.fw; + local prec = + if code.prec == '*' then + error 'Cannot use * precision with object.' + else + code.prec; + local val = + if std.objectHasAll(obj, f) then + obj[f] + else + error 'No such field: ' + f; + local s = + if code.ctype == '%' then + '%' + else + format_code(val, code, fw, prec, f); + local s_padded = + if code.cflags.left then + pad_right(s, fw, ' ') + else + pad_left(s, fw, ' '); + format_codes_obj(codes, obj, i + 1, v + s_padded) tailstrict; + + if std.isArray(vals) then + format_codes_arr(codes, vals, 0, 0, '') + else if std.isObject(vals) then + format_codes_obj(codes, vals, 0, '') + else + format_codes_arr(codes, [vals], 0, 0, ''), + + foldr(func, arr, init):: + local aux(func, arr, running, idx) = + if idx < 0 then + running + else + aux(func, arr, func(arr[idx], running), idx - 1) tailstrict; + aux(func, arr, init, std.length(arr) - 1), + + foldl(func, arr, init):: + local aux(func, arr, running, idx) = + if idx >= std.length(arr) then + running + else + aux(func, arr, func(running, arr[idx]), idx + 1) tailstrict; + aux(func, arr, init, 0), + + + filterMap(filter_func, map_func, arr):: + if !std.isFunction(filter_func) then + error ('std.filterMap first param must be function, got ' + std.type(filter_func)) + else if !std.isFunction(map_func) then + error ('std.filterMap second param must be function, got ' + std.type(map_func)) + else if !std.isArray(arr) then + error ('std.filterMap third param must be array, got ' + std.type(arr)) + else + std.map(map_func, std.filter(filter_func, arr)), + + assertEqual(a, b):: + if a == b then + true + else + error 'Assertion failed. ' + a + ' != ' + b, + + abs(n):: + if !std.isNumber(n) then + error 'std.abs expected number, got ' + std.type(n) + else + if n > 0 then n else -n, + + sign(n):: + if !std.isNumber(n) then + error 'std.sign expected number, got ' + std.type(n) + else + if n > 0 then + 1 + else if n < 0 then + -1 + else 0, + + max(a, b):: + if !std.isNumber(a) then + error 'std.max first param expected number, got ' + std.type(a) + else if !std.isNumber(b) then + error 'std.max second param expected number, got ' + std.type(b) + else + if a > b then a else b, + + min(a, b):: + if !std.isNumber(a) then + error 'std.max first param expected number, got ' + std.type(a) + else if !std.isNumber(b) then + error 'std.max second param expected number, got ' + std.type(b) + else + if a < b then a else b, + + clamp(x, minVal, maxVal):: + if x < minVal then minVal + else if x > maxVal then maxVal + else x, + + flattenArrays(arrs):: + std.foldl(function(a, b) a + b, arrs, []), + + manifestIni(ini):: + local body_lines(body) = + std.join([], [ + local value_or_values = body[k]; + if std.isArray(value_or_values) then + ['%s = %s' % [k, value] for value in value_or_values] + else + ['%s = %s' % [k, value_or_values]] + + for k in std.objectFields(body) + ]); + + local section_lines(sname, sbody) = ['[%s]' % [sname]] + body_lines(sbody), + main_body = if std.objectHas(ini, 'main') then body_lines(ini.main) else [], + all_sections = [ + section_lines(k, ini.sections[k]) + for k in std.objectFields(ini.sections) + ]; + std.join('\n', main_body + std.flattenArrays(all_sections) + ['']), + + escapeStringJson(str_):: + local str = std.toString(str_); + local trans(ch) = + if ch == '"' then + '\\"' + else if ch == '\\' then + '\\\\' + else if ch == '\b' then + '\\b' + else if ch == '\f' then + '\\f' + else if ch == '\n' then + '\\n' + else if ch == '\r' then + '\\r' + else if ch == '\t' then + '\\t' + else + local cp = std.codepoint(ch); + if cp < 32 || (cp >= 127 && cp <= 159) then + '\\u%04x' % [cp] + else + ch; + '"%s"' % std.join('', [trans(ch) for ch in std.stringChars(str)]), + + escapeStringPython(str):: + std.escapeStringJson(str), + + escapeStringBash(str_):: + local str = std.toString(str_); + local trans(ch) = + if ch == "'" then + "'\"'\"'" + else + ch; + "'%s'" % std.join('', [trans(ch) for ch in std.stringChars(str)]), + + escapeStringDollars(str_):: + local str = std.toString(str_); + local trans(ch) = + if ch == '$' then + '$$' + else + ch; + std.foldl(function(a, b) a + trans(b), std.stringChars(str), ''), + + manifestJson(value):: std.manifestJsonEx(value, ' '), + + manifestJsonEx(value, indent):: + local aux(v, path, cindent) = + if v == true then + 'true' + else if v == false then + 'false' + else if v == null then + 'null' + else if std.isNumber(v) then + '' + v + else if std.isString(v) then + std.escapeStringJson(v) + else if std.isFunction(v) then + error 'Tried to manifest function at ' + path + else if std.isArray(v) then + local range = std.range(0, std.length(v) - 1); + local new_indent = cindent + indent; + local lines = ['[\n'] + + std.join([',\n'], + [ + [new_indent + aux(v[i], path + [i], new_indent)] + for i in range + ]) + + ['\n' + cindent + ']']; + std.join('', lines) + else if std.isObject(v) then + local lines = ['{\n'] + + std.join([',\n'], + [ + [cindent + indent + std.escapeStringJson(k) + ': ' + + aux(v[k], path + [k], cindent + indent)] + for k in std.objectFields(v) + ]) + + ['\n' + cindent + '}']; + std.join('', lines); + aux(value, [], ''), + + manifestYamlDoc(value, indent_array_in_object=false):: + local aux(v, path, cindent) = + if v == true then + 'true' + else if v == false then + 'false' + else if v == null then + 'null' + else if std.isNumber(v) then + '' + v + else if std.isString(v) then + local len = std.length(v); + if len == 0 then + '""' + else if v[len - 1] == '\n' then + local split = std.split(v, '\n'); + std.join('\n' + cindent + ' ', ['|'] + split[0:std.length(split) - 1]) + else + std.escapeStringJson(v) + else if std.isFunction(v) then + error 'Tried to manifest function at ' + path + else if std.isArray(v) then + if std.length(v) == 0 then + '[]' + else + local params(value) = + if std.isArray(value) && std.length(value) > 0 then { + // While we could avoid the new line, it yields YAML that is + // hard to read, e.g.: + // - - - 1 + // - 2 + // - - 3 + // - 4 + new_indent: cindent + ' ', + space: '\n' + self.new_indent, + } else if std.isObject(value) && std.length(value) > 0 then { + new_indent: cindent + ' ', + // In this case we can start on the same line as the - because the indentation + // matches up then. The converse is not true, because fields are not always + // 1 character long. + space: ' ', + } else { + // In this case, new_indent is only used in the case of multi-line strings. + new_indent: cindent, + space: ' ', + }; + local range = std.range(0, std.length(v) - 1); + local parts = [ + '-' + param.space + aux(v[i], path + [i], param.new_indent) + for i in range + for param in [params(v[i])] + ]; + std.join('\n' + cindent, parts) + else if std.isObject(v) then + if std.length(v) == 0 then + '{}' + else + local params(value) = + if std.isArray(value) && std.length(value) > 0 then { + // Not indenting allows e.g. + // ports: + // - 80 + // instead of + // ports: + // - 80 + new_indent: if indent_array_in_object then cindent + ' ' else cindent, + space: '\n' + self.new_indent, + } else if std.isObject(value) && std.length(value) > 0 then { + new_indent: cindent + ' ', + space: '\n' + self.new_indent, + } else { + // In this case, new_indent is only used in the case of multi-line strings. + new_indent: cindent, + space: ' ', + }; + local lines = [ + std.escapeStringJson(k) + ':' + param.space + aux(v[k], path + [k], param.new_indent) + for k in std.objectFields(v) + for param in [params(v[k])] + ]; + std.join('\n' + cindent, lines); + aux(value, [], ''), + + manifestYamlStream(value, indent_array_in_object=false, c_document_end=true):: + if !std.isArray(value) then + error 'manifestYamlStream only takes arrays, got ' + std.type(value) + else + '---\n' + std.join( + '\n---\n', [std.manifestYamlDoc(e, indent_array_in_object) for e in value] + ) + if c_document_end then '\n...\n' else '\n', + + + manifestPython(v):: + if std.isObject(v) then + local fields = [ + '%s: %s' % [std.escapeStringPython(k), std.manifestPython(v[k])] + for k in std.objectFields(v) + ]; + '{%s}' % [std.join(', ', fields)] + else if std.isArray(v) then + '[%s]' % [std.join(', ', [std.manifestPython(v2) for v2 in v])] + else if std.isString(v) then + '%s' % [std.escapeStringPython(v)] + else if std.isFunction(v) then + error 'cannot manifest function' + else if std.isNumber(v) then + std.toString(v) + else if v == true then + 'True' + else if v == false then + 'False' + else if v == null then + 'None', + + manifestPythonVars(conf):: + local vars = ['%s = %s' % [k, std.manifestPython(conf[k])] for k in std.objectFields(conf)]; + std.join('\n', vars + ['']), + + manifestXmlJsonml(value):: + if !std.isArray(value) then + error 'Expected a JSONML value (an array), got %s' % std.type(value) + else + local aux(v) = + if std.isString(v) then + v + else + local tag = v[0]; + local has_attrs = std.length(v) > 1 && std.isObject(v[1]); + local attrs = if has_attrs then v[1] else {}; + local children = if has_attrs then v[2:] else v[1:]; + local attrs_str = + std.join('', [' %s="%s"' % [k, attrs[k]] for k in std.objectFields(attrs)]); + std.deepJoin(['<', tag, attrs_str, '>', [aux(x) for x in children], '']); + + aux(value), + + local base64_table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', + local base64_inv = { [base64_table[i]]: i for i in std.range(0, 63) }, + + base64(input):: + local bytes = + if std.isString(input) then + std.map(function(c) std.codepoint(c), input) + else + input; + + local aux(arr, i, r) = + if i >= std.length(arr) then + r + else if i + 1 >= std.length(arr) then + local str = + // 6 MSB of i + base64_table[(arr[i] & 252) >> 2] + + // 2 LSB of i + base64_table[(arr[i] & 3) << 4] + + '=='; + aux(arr, i + 3, r + str) tailstrict + else if i + 2 >= std.length(arr) then + local str = + // 6 MSB of i + base64_table[(arr[i] & 252) >> 2] + + // 2 LSB of i, 4 MSB of i+1 + base64_table[(arr[i] & 3) << 4 | (arr[i + 1] & 240) >> 4] + + // 4 LSB of i+1 + base64_table[(arr[i + 1] & 15) << 2] + + '='; + aux(arr, i + 3, r + str) tailstrict + else + local str = + // 6 MSB of i + base64_table[(arr[i] & 252) >> 2] + + // 2 LSB of i, 4 MSB of i+1 + base64_table[(arr[i] & 3) << 4 | (arr[i + 1] & 240) >> 4] + + // 4 LSB of i+1, 2 MSB of i+2 + base64_table[(arr[i + 1] & 15) << 2 | (arr[i + 2] & 192) >> 6] + + // 6 LSB of i+2 + base64_table[(arr[i + 2] & 63)]; + aux(arr, i + 3, r + str) tailstrict; + + local sanity = std.foldl(function(r, a) r && (a < 256), bytes, true); + if !sanity then + error 'Can only base64 encode strings / arrays of single bytes.' + else + aux(bytes, 0, ''), + + + base64DecodeBytes(str):: + if std.length(str) % 4 != 0 then + error 'Not a base64 encoded string "%s"' % str + else + local aux(str, i, r) = + if i >= std.length(str) then + r + else + // all 6 bits of i, 2 MSB of i+1 + local n1 = [base64_inv[str[i]] << 2 | (base64_inv[str[i + 1]] >> 4)]; + // 4 LSB of i+1, 4MSB of i+2 + local n2 = + if str[i + 2] == '=' then [] + else [(base64_inv[str[i + 1]] & 15) << 4 | (base64_inv[str[i + 2]] >> 2)]; + // 2 LSB of i+2, all 6 bits of i+3 + local n3 = + if str[i + 3] == '=' then [] + else [(base64_inv[str[i + 2]] & 3) << 6 | base64_inv[str[i + 3]]]; + aux(str, i + 4, r + n1 + n2 + n3) tailstrict; + aux(str, 0, []), + + base64Decode(str):: + local bytes = std.base64DecodeBytes(str); + std.join('', std.map(function(b) std.char(b), bytes)), + + reverse(arr):: + local l = std.length(arr); + std.makeArray(l, function(i) arr[l - i - 1]), + + // Merge-sort for long arrays and naive quicksort for shorter ones + sort(arr, keyF=id):: + local quickSort(arr, keyF=id) = + local l = std.length(arr); + if std.length(arr) <= 1 then + arr + else + local pos = 0; + local pivot = keyF(arr[pos]); + local rest = std.makeArray(l - 1, function(i) if i < pos then arr[i] else arr[i + 1]); + local left = std.filter(function(x) keyF(x) < pivot, rest); + local right = std.filter(function(x) keyF(x) >= pivot, rest); + quickSort(left, keyF) + [arr[pos]] + quickSort(right, keyF); + + local merge(a, b) = + local la = std.length(a), lb = std.length(b); + local aux(i, j, prefix) = + if i == la then + prefix + b[j:] + else if j == lb then + prefix + a[i:] + else + if keyF(a[i]) <= keyF(b[j]) then + aux(i + 1, j, prefix + [a[i]]) tailstrict + else + aux(i, j + 1, prefix + [b[j]]) tailstrict; + aux(0, 0, []); + + local l = std.length(arr); + if std.length(arr) <= 30 then + quickSort(arr, keyF=keyF) + else + local mid = std.floor(l / 2); + local left = arr[:mid], right = arr[mid:]; + merge(std.sort(left, keyF=keyF), std.sort(right, keyF=keyF)), + + uniq(arr, keyF=id):: + local f(a, b) = + if std.length(a) == 0 then + [b] + else if keyF(a[std.length(a) - 1]) == keyF(b) then + a + else + a + [b]; + std.foldl(f, arr, []), + + set(arr, keyF=id):: + std.uniq(std.sort(arr, keyF), keyF), + + setMember(x, arr, keyF=id):: + // TODO(dcunnin): Binary chop for O(log n) complexity + std.length(std.setInter([x], arr, keyF)) > 0, + + setUnion(a, b, keyF=id):: + // NOTE: order matters, values in `a` win + local aux(a, b, i, j, acc) = + if i >= std.length(a) then + acc + b[j:] + else if j >= std.length(b) then + acc + a[i:] + else + local ak = keyF(a[i]); + local bk = keyF(b[j]); + if ak == bk then + aux(a, b, i + 1, j + 1, acc + [a[i]]) tailstrict + else if ak < bk then + aux(a, b, i + 1, j, acc + [a[i]]) tailstrict + else + aux(a, b, i, j + 1, acc + [b[j]]) tailstrict; + aux(a, b, 0, 0, []), + + setInter(a, b, keyF=id):: + local aux(a, b, i, j, acc) = + if i >= std.length(a) || j >= std.length(b) then + acc + else + if keyF(a[i]) == keyF(b[j]) then + aux(a, b, i + 1, j + 1, acc + [a[i]]) tailstrict + else if keyF(a[i]) < keyF(b[j]) then + aux(a, b, i + 1, j, acc) tailstrict + else + aux(a, b, i, j + 1, acc) tailstrict; + aux(a, b, 0, 0, []) tailstrict, + + setDiff(a, b, keyF=id):: + local aux(a, b, i, j, acc) = + if i >= std.length(a) then + acc + else if j >= std.length(b) then + acc + a[i:] + else + if keyF(a[i]) == keyF(b[j]) then + aux(a, b, i + 1, j + 1, acc) tailstrict + else if keyF(a[i]) < keyF(b[j]) then + aux(a, b, i + 1, j, acc + [a[i]]) tailstrict + else + aux(a, b, i, j + 1, acc) tailstrict; + aux(a, b, 0, 0, []) tailstrict, + + mergePatch(target, patch):: + if std.isObject(patch) then + local target_object = + if std.isObject(target) then target else {}; + + local target_fields = + if std.isObject(target_object) then std.objectFields(target_object) else []; + + local null_fields = [k for k in std.objectFields(patch) if patch[k] == null]; + local both_fields = std.setUnion(target_fields, std.objectFields(patch)); + + { + [k]: + if !std.objectHas(patch, k) then + target_object[k] + else if !std.objectHas(target_object, k) then + std.mergePatch(null, patch[k]) tailstrict + else + std.mergePatch(target_object[k], patch[k]) tailstrict + for k in std.setDiff(both_fields, null_fields) + } + else + patch, + + objectFields(o):: + std.objectFieldsEx(o, false), + + objectFieldsAll(o):: + std.objectFieldsEx(o, true), + + objectHas(o, f):: + std.objectHasEx(o, f, false), + + objectHasAll(o, f):: + std.objectHasEx(o, f, true), + + equals(a, b):: + local ta = std.type(a); + local tb = std.type(b); + if !std.primitiveEquals(ta, tb) then + false + else + if std.primitiveEquals(ta, 'array') then + local la = std.length(a); + if !std.primitiveEquals(la, std.length(b)) then + false + else + local aux(a, b, i) = + if i >= la then + true + else if a[i] != b[i] then + false + else + aux(a, b, i + 1) tailstrict; + aux(a, b, 0) + else if std.primitiveEquals(ta, 'object') then + local fields = std.objectFields(a); + local lfields = std.length(fields); + if fields != std.objectFields(b) then + false + else + local aux(a, b, i) = + if i >= lfields then + true + else if local f = fields[i]; a[f] != b[f] then + false + else + aux(a, b, i + 1) tailstrict; + aux(a, b, 0) + else + std.primitiveEquals(a, b), + + + resolvePath(f, r):: + local arr = std.split(f, '/'); + std.join('/', std.makeArray(std.length(arr) - 1, function(i) arr[i]) + [r]), + + prune(a):: + local isContent(b) = + if b == null then + false + else if std.isArray(b) then + std.length(b) > 0 + else if std.isObject(b) then + std.length(b) > 0 + else + true; + if std.isArray(a) then + [std.prune(x) for x in a if isContent($.prune(x))] + else if std.isObject(a) then { + [x]: $.prune(a[x]) + for x in std.objectFields(a) + if isContent(std.prune(a[x])) + } else + a, + + findSubstr(pat, str):: + if !std.isString(pat) then + error 'findSubstr first parameter should be a string, got ' + std.type(pat) + else if !std.isString(str) then + error 'findSubstr second parameter should be a string, got ' + std.type(str) + else + local pat_len = std.length(pat); + local str_len = std.length(str); + if pat_len == 0 || str_len == 0 || pat_len > str_len then + [] + else + std.filter(function(i) str[i:i + pat_len] == pat, std.range(0, str_len - pat_len)), + + find(value, arr):: + if !std.isArray(arr) then + error 'find second parameter should be an array, got ' + std.type(arr) + else + std.filter(function(i) arr[i] == value, std.range(0, std.length(arr) - 1)), +} --- a/crates/jsonnet-evaluator/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "jsonnet-evaluator" -version = "0.1.0" -authors = ["Лач "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[features] -default = ["serialized-stdlib", "faster"] -# Serializes standard library AST instead of parsing them every run -serialized-stdlib = ["serde", "bincode", "jsonnet-parser/deserialize"] -# Same as above, but with generated code instead of serde. Reduces memory usage, but increases binary size and compilation time -codegenerated-stdlib = [] -# Replace some standard library functions with faster implementations (I.e manifestJsonEx) -# Library works fine without this feature, but requires more memory and time for std function calls -faster = [] - -[dependencies] -jsonnet-parser = { path = "../jsonnet-parser" } -closure = "0.3.0" -jsonnet-stdlib = { path = "../jsonnet-stdlib" } -indexmap = "1.4.0" -md5 = "0.7.0" - -serde = { version = "1.0.114", optional = true } -bincode = { version = "1.3.1", optional = true } - -[build-dependencies] -jsonnet-parser = { path = "../jsonnet-parser", features = ["dump", "serialize", "deserialize"] } -jsonnet-stdlib = { path = "../jsonnet-stdlib" } -structdump = "0.1.2" -serde = "1.0.114" -bincode = "1.3.1" --- a/crates/jsonnet-evaluator/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# jsonnet-evaluator - -Interpreter for parsed jsonnet tree - -## Standard library - -jsonnet stdlib is embedded into evaluator, but there is different modes for this: - -- `codegenerated-stdlib` - - generates source code for reproducing stdlib AST ([Example](https://gist.githubusercontent.com/CertainLach/7b3149df556f3406f5e9368aaa9f32ec/raw/0c80d8ab9aa7b9288c6219a2779cb2ab37287669/a.rs)) - - fastest on interpretation, slowest on compilation (it takes more than 5 minutes to optimize them by llvm) -- `serialized-stdlib` - - serializes standard library AST using serde - - slower than `codegenerated-stdlib` at runtime, but have no compilation speed penality -- none - - leaves only stdlib source code in binary, processing them same way as user supplied data - - slowest (as it involves parsing of standard library source code) - -Because of `codegenerated-stdlib` compilation slowdown, `serialized-stdlib` is used by default - -### Benchmark - -Can also be run via `cargo bench` - -```md -# codegenerated-stdlib -test tests::bench_codegen ... bench: 401,696 ns/iter (+/- 38,521) -# serialized-stdlib -test tests::bench_serialize ... bench: 1,763,999 ns/iter (+/- 76,211) -# none -test tests::bench_parse ... bench: 7,206,164 ns/iter (+/- 1,067,418) -``` - -## Intristics - -Some functions from stdlib are implemented as intristics - -### Intristic handling - -If indexed jsonnet object has field '__intristic_namespace__' of type 'string', then any not found field/method is resolved as `Val::Intristic(__intristic_namespace__, name)` --- a/crates/jsonnet-evaluator/build.rs +++ /dev/null @@ -1,65 +0,0 @@ -use bincode::serialize; -use jsonnet_parser::{ - parse, Expr, FieldMember, FieldName, LocExpr, Member, ObjBody, ParserSettings, -}; -use jsonnet_stdlib::STDLIB_STR; -use std::{ - env, - fs::File, - io::Write, - path::{Path, PathBuf}, - rc::Rc, -}; -use structdump::CodegenResult; - -fn main() { - let parsed = parse( - STDLIB_STR, - &ParserSettings { - file_name: Rc::new(PathBuf::from("std.jsonnet")), - loc_data: true, - }, - ) - .expect("parse"); - - let parsed = if cfg!(feature = "faster") { - let LocExpr(expr, location) = parsed; - LocExpr( - Rc::new(match Rc::try_unwrap(expr).unwrap() { - Expr::Obj(ObjBody::MemberList(members)) => Expr::Obj(ObjBody::MemberList( - members - .into_iter() - .filter(|p| { - !matches!( - p, - Member::Field(FieldMember { - name: FieldName::Fixed(name), - .. - }) if **name == *"join" || **name == *"manifestJsonEx" || **name == *"escapeStringJson" - ) - }) - .collect(), - )), - _ => panic!("std value should be object"), - }), - location, - ) - } else { - parsed - }; - { - let mut codegen = CodegenResult::default(); - let code = codegen.codegen(&parsed); - - let out_dir = env::var("OUT_DIR").unwrap(); - let dest_path = Path::new(&out_dir).join("stdlib.rs"); - let mut f = File::create(&dest_path).unwrap(); - f.write_all(&code.as_bytes()).unwrap(); - } - { - let out_dir = env::var("OUT_DIR").unwrap(); - let dest_path = Path::new(&out_dir).join("stdlib.bincode"); - let mut f = File::create(&dest_path).unwrap(); - f.write_all(&serialize(&parsed).unwrap()).unwrap(); - } -} --- a/crates/jsonnet-evaluator/src/ctx.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::{ - create_error, future_wrapper, map::LayeredHashMap, rc_fn_helper, resolved_lazy_val, Error, - LazyBinding, LazyVal, ObjValue, Result, Val, -}; -use std::{cell::RefCell, collections::HashMap, fmt::Debug, rc::Rc}; - -rc_fn_helper!( - ContextCreator, - context_creator, - dyn Fn(Option, Option) -> Result -); - -future_wrapper!(Context, FutureContext); - -struct ContextInternals { - dollar: Option, - this: Option, - super_obj: Option, - bindings: LayeredHashMap, LazyVal>, -} -pub struct Context(Rc); -impl Debug for Context { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Context") - .field("this", &self.0.this.as_ref().map(|e| Rc::as_ptr(&e.0))) - .field("bindings", &self.0.bindings) - .finish() - } -} -impl Context { - pub fn new_future() -> FutureContext { - FutureContext(Rc::new(RefCell::new(None))) - } - - pub fn dollar(&self) -> &Option { - &self.0.dollar - } - - pub fn this(&self) -> &Option { - &self.0.this - } - - pub fn super_obj(&self) -> &Option { - &self.0.super_obj - } - - pub fn new() -> Context { - Context(Rc::new(ContextInternals { - dollar: None, - this: None, - super_obj: None, - bindings: LayeredHashMap::default(), - })) - } - - pub fn binding(&self, name: Rc) -> Result { - self.0 - .bindings - .get(&name) - .cloned() - .ok_or_else(|| create_error(Error::UnknownVariable(name))) - } - pub fn into_future(self, ctx: FutureContext) -> Context { - { - ctx.0.borrow_mut().replace(self); - } - ctx.unwrap() - } - - pub fn with_var(&self, name: Rc, value: Val) -> Result { - let mut new_bindings = HashMap::with_capacity(1); - new_bindings.insert(name, resolved_lazy_val!(value)); - self.extend(new_bindings, None, None, None) - } - - pub fn extend( - &self, - new_bindings: HashMap, LazyVal>, - new_dollar: Option, - new_this: Option, - new_super_obj: Option, - ) -> Result { - let dollar = new_dollar.or_else(|| self.0.dollar.clone()); - let this = new_this.or_else(|| self.0.this.clone()); - let super_obj = new_super_obj.or_else(|| self.0.super_obj.clone()); - let bindings = if new_bindings.is_empty() { - self.0.bindings.clone() - } else { - self.0.bindings.extend(new_bindings) - }; - Ok(Context(Rc::new(ContextInternals { - dollar, - this, - super_obj, - bindings, - }))) - } - pub fn extend_unbound( - &self, - new_bindings: HashMap, LazyBinding>, - new_dollar: Option, - new_this: Option, - new_super_obj: Option, - ) -> Result { - let this = new_this.or_else(|| self.0.this.clone()); - let super_obj = new_super_obj.or_else(|| self.0.super_obj.clone()); - let mut new = HashMap::with_capacity(new_bindings.len()); - for (k, v) in new_bindings.into_iter() { - new.insert(k, v.evaluate(this.clone(), super_obj.clone())?); - } - self.extend(new, new_dollar, this, super_obj) - } -} - -impl Default for Context { - fn default() -> Self { - Self::new() - } -} - -impl PartialEq for Context { - fn eq(&self, other: &Self) -> bool { - Rc::ptr_eq(&self.0, &other.0) - } -} - -impl Clone for Context { - fn clone(&self) -> Self { - Context(self.0.clone()) - } -} --- a/crates/jsonnet-evaluator/src/dynamic.rs +++ /dev/null @@ -1,55 +0,0 @@ -#[macro_export] -macro_rules! future_wrapper { - ($orig: ty, $wrapper: ident) => { - #[derive(Debug, Clone)] - pub struct $wrapper(pub std::rc::Rc>>); - impl $wrapper { - pub fn unwrap(self) -> $orig { - self.0.borrow().as_ref().map(|e| e.clone()).unwrap() - } - pub fn new() -> Self { - $wrapper(std::rc::Rc::new(std::cell::RefCell::new(None))) - } - pub fn fill(self, val: $orig) -> $orig { - if self.0.borrow().is_some() { - panic!("wrapper is filled already"); - } - { - self.0.borrow_mut().replace(val); - } - self.unwrap() - } - } - impl Default for $wrapper { - fn default() -> Self { - Self::new() - } - } - }; -} - -#[macro_export] -macro_rules! rc_fn_helper { - ($name: ident, $macro_name: ident, $fn: ty) => { - #[derive(Clone)] - #[doc = "Function wrapper"] - pub struct $name(pub std::rc::Rc<$fn>); - impl std::fmt::Debug for $name { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct(std::stringify!($name)).finish() - } - } - impl std::cmp::PartialEq for $name { - fn eq(&self, other: &$name) -> bool { - std::ptr::eq(&self.0, &other.0) - } - } - #[doc = "Macro to ease wrapper creation"] - #[macro_export] - macro_rules! $macro_name { - ($val: expr) => { - $crate::$name(std::rc::Rc::new($val)) - }; - } - }; -} --- a/crates/jsonnet-evaluator/src/error.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::{Val, ValType}; -use jsonnet_parser::{BinaryOpType, ExprLocation, UnaryOpType}; -use std::{path::PathBuf, rc::Rc}; - -#[derive(Debug, Clone)] -pub enum Error { - IntristicNotFound(Rc, Rc), - IntristicArgumentReorderingIsNotSupportedYet, - - UnaryOperatorDoesNotOperateOnType(UnaryOpType, ValType), - BinaryOperatorDoesNotOperateOnValues(BinaryOpType, ValType, ValType), - - NoTopLevelObjectFound, - CantUseSelfOutsideOfObject, - CantUseSuperOutsideOfObject, - - InComprehensionCanOnlyIterateOverArray, - - ArrayBoundsError(usize, usize), - - AssertionFailed(Val), - - VariableIsNotDefined(String), - TypeMismatch(&'static str, Vec, ValType), - NoSuchField(Rc), - - UnknownVariable(Rc), - - OnlyFunctionsCanBeCalledGot(ValType), - UnknownFunctionParameter(String), - BindingParameterASecondTime(Rc), - TooManyArgsFunctionHas(usize), - FunctionParameterNotBoundInCall(Rc), - - UndefinedExternalVariable(Rc), - - FieldMustBeStringGot(ValType), - - AttemptedIndexAnArrayWithString(Rc), - ValueIndexMustBeTypeGot(ValType, ValType, ValType), - CantIndexInto(ValType), - - StandaloneSuper, - - ImportFileNotFound(PathBuf, PathBuf), - ResolvedFileNotFound(PathBuf), - ImportBadFileUtf8(PathBuf), - ImportNotSupported(PathBuf, PathBuf), - ImportSyntaxError(jsonnet_parser::ParseError), - - RuntimeError(Rc), - StackOverflow, - FractionalIndex, - DivisionByZero, -} - -#[derive(Clone, Debug)] -pub struct StackTraceElement(pub ExprLocation, pub String); -#[derive(Debug, Clone)] -pub struct StackTrace(pub Vec); - -#[derive(Debug, Clone)] -pub struct LocError(pub Error, pub StackTrace); -pub type Result = std::result::Result; --- a/crates/jsonnet-evaluator/src/evaluate.rs +++ /dev/null @@ -1,802 +0,0 @@ -use crate::{ - context_creator, create_error, create_error_result, escape_string_json, future_wrapper, - lazy_val, manifest_json_ex, parse_args, push, with_state, Context, ContextCreator, Error, - FuncDesc, LazyBinding, LazyVal, ObjMember, ObjValue, Result, Val, ValType, -}; -use closure::closure; -use jsonnet_parser::{ - AssertStmt, BinaryOpType, BindSpec, CompSpec, Expr, FieldMember, ForSpecData, IfSpecData, - LiteralType, LocExpr, Member, ObjBody, ParamsDesc, UnaryOpType, Visibility, -}; -use std::{ - collections::{BTreeMap, HashMap}, - rc::Rc, -}; - -pub fn evaluate_binding(b: &BindSpec, context_creator: ContextCreator) -> (Rc, LazyBinding) { - let b = b.clone(); - if let Some(params) = &b.params { - let params = params.clone(); - ( - b.name.clone(), - LazyBinding::Bindable(Rc::new(move |this, super_obj| { - Ok(lazy_val!( - closure!(clone b, clone params, clone context_creator, || Ok(evaluate_method( - context_creator.0(this.clone(), super_obj.clone())?, - params.clone(), - b.value.clone(), - ))) - )) - })), - ) - } else { - ( - b.name.clone(), - LazyBinding::Bindable(Rc::new(move |this, super_obj| { - Ok(lazy_val!(closure!(clone context_creator, clone b, || - push(&b.value.1, "thunk", ||{ - evaluate( - context_creator.0(this.clone(), super_obj.clone())?, - &b.value - ) - }) - ))) - })), - ) - } -} - -pub fn evaluate_method(ctx: Context, params: ParamsDesc, body: LocExpr) -> Val { - Val::Func(FuncDesc { ctx, params, body }) -} - -pub fn evaluate_field_name( - context: Context, - field_name: &jsonnet_parser::FieldName, -) -> Result>> { - Ok(match field_name { - jsonnet_parser::FieldName::Fixed(n) => Some(n.clone()), - jsonnet_parser::FieldName::Dyn(expr) => { - let lazy = evaluate(context, expr)?; - let value = lazy.unwrap_if_lazy()?; - if matches!(value, Val::Null) { - None - } else { - Some(value.try_cast_str("dynamic field name")?) - } - } - }) -} - -pub fn evaluate_unary_op(op: UnaryOpType, b: &Val) -> Result { - Ok(match (op, b) { - (o, Val::Lazy(l)) => evaluate_unary_op(o, &l.evaluate()?)?, - (UnaryOpType::Not, Val::Bool(v)) => Val::Bool(!v), - (UnaryOpType::Minus, Val::Num(n)) => Val::Num(-*n), - (UnaryOpType::BitNot, Val::Num(n)) => Val::Num(!(*n as i32) as f64), - (op, o) => create_error_result(Error::UnaryOperatorDoesNotOperateOnType( - op, - o.value_type()?, - ))?, - }) -} - -pub(crate) fn evaluate_add_op(a: &Val, b: &Val) -> Result { - Ok(match (a, b) { - (Val::Str(v1), Val::Str(v2)) => Val::Str(((**v1).to_owned() + &v2).into()), - - // Can't use generic json serialization way, because it depends on number to string concatenation (std.jsonnet:890) - (Val::Num(n), Val::Str(o)) => Val::Str(format!("{}{}", n, o).into()), - (Val::Str(o), Val::Num(n)) => Val::Str(format!("{}{}", o, n).into()), - - (Val::Str(s), o) => Val::Str(format!("{}{}", s, o.clone().into_json(0)?).into()), - (o, Val::Str(s)) => Val::Str(format!("{}{}", o.clone().into_json(0)?, s).into()), - - (Val::Obj(v1), Val::Obj(v2)) => Val::Obj(v2.with_super(v1.clone())), - (Val::Arr(a), Val::Arr(b)) => Val::Arr(Rc::new([&a[..], &b[..]].concat())), - (Val::Num(v1), Val::Num(v2)) => Val::Num(v1 + v2), - _ => create_error_result(Error::BinaryOperatorDoesNotOperateOnValues( - BinaryOpType::Add, - a.value_type()?, - b.value_type()?, - ))?, - }) -} - -pub fn evaluate_binary_op_special( - context: Context, - a: &LocExpr, - op: BinaryOpType, - b: &LocExpr, -) -> Result { - Ok( - match (evaluate(context.clone(), &a)?.unwrap_if_lazy()?, op, b) { - (Val::Bool(true), BinaryOpType::Or, _o) => Val::Bool(true), - (Val::Bool(false), BinaryOpType::And, _o) => Val::Bool(false), - (a, op, eb) => { - evaluate_binary_op_normal(&a, op, &evaluate(context, eb)?.unwrap_if_lazy()?)? - } - }, - ) -} - -pub fn evaluate_binary_op_normal(a: &Val, op: BinaryOpType, b: &Val) -> Result { - Ok(match (a, op, b) { - (a, BinaryOpType::Add, b) => evaluate_add_op(a, b)?, - - (Val::Str(v1), BinaryOpType::Mul, Val::Num(v2)) => Val::Str(v1.repeat(*v2 as usize).into()), - - // Bool X Bool - (Val::Bool(a), BinaryOpType::And, Val::Bool(b)) => Val::Bool(*a && *b), - (Val::Bool(a), BinaryOpType::Or, Val::Bool(b)) => Val::Bool(*a || *b), - - // Str X Str - (Val::Str(v1), BinaryOpType::Lt, Val::Str(v2)) => Val::Bool(v1 < v2), - (Val::Str(v1), BinaryOpType::Gt, Val::Str(v2)) => Val::Bool(v1 > v2), - (Val::Str(v1), BinaryOpType::Lte, Val::Str(v2)) => Val::Bool(v1 <= v2), - (Val::Str(v1), BinaryOpType::Gte, Val::Str(v2)) => Val::Bool(v1 >= v2), - - // Num X Num - (Val::Num(v1), BinaryOpType::Mul, Val::Num(v2)) => Val::Num(v1 * v2), - (Val::Num(v1), BinaryOpType::Div, Val::Num(v2)) => { - if *v2 <= f64::EPSILON { - create_error_result(crate::Error::DivisionByZero)? - } - Val::Num(v1 / v2) - } - - (Val::Num(v1), BinaryOpType::Sub, Val::Num(v2)) => Val::Num(v1 - v2), - - (Val::Num(v1), BinaryOpType::Lt, Val::Num(v2)) => Val::Bool(v1 < v2), - (Val::Num(v1), BinaryOpType::Gt, Val::Num(v2)) => Val::Bool(v1 > v2), - (Val::Num(v1), BinaryOpType::Lte, Val::Num(v2)) => Val::Bool(v1 <= v2), - (Val::Num(v1), BinaryOpType::Gte, Val::Num(v2)) => Val::Bool(v1 >= v2), - - (Val::Num(v1), BinaryOpType::BitAnd, Val::Num(v2)) => { - Val::Num(((*v1 as i32) & (*v2 as i32)) as f64) - } - (Val::Num(v1), BinaryOpType::BitOr, Val::Num(v2)) => { - Val::Num(((*v1 as i32) | (*v2 as i32)) as f64) - } - (Val::Num(v1), BinaryOpType::BitXor, Val::Num(v2)) => { - Val::Num(((*v1 as i32) ^ (*v2 as i32)) as f64) - } - (Val::Num(v1), BinaryOpType::Lhs, Val::Num(v2)) => { - Val::Num(((*v1 as i32) << (*v2 as i32)) as f64) - } - (Val::Num(v1), BinaryOpType::Rhs, Val::Num(v2)) => { - Val::Num(((*v1 as i32) >> (*v2 as i32)) as f64) - } - - _ => create_error_result(Error::BinaryOperatorDoesNotOperateOnValues( - op, - a.value_type()?, - b.value_type()?, - ))?, - }) -} - -future_wrapper!(HashMap, LazyBinding>, FutureNewBindings); -future_wrapper!(ObjValue, FutureObjValue); - -pub fn evaluate_comp( - context: Context, - value: &impl Fn(Context) -> Result, - specs: &[CompSpec], -) -> Result>> { - Ok(match specs.get(0) { - None => Some(vec![value(context)?]), - Some(CompSpec::IfSpec(IfSpecData(cond))) => { - if evaluate(context.clone(), &cond)?.try_cast_bool("if spec")? { - evaluate_comp(context, value, &specs[1..])? - } else { - None - } - } - Some(CompSpec::ForSpec(ForSpecData(var, expr))) => { - match evaluate(context.clone(), &expr)?.unwrap_if_lazy()? { - Val::Arr(list) => { - let mut out = Vec::new(); - for item in list.iter() { - let item = item.unwrap_if_lazy()?; - out.push(evaluate_comp( - context.with_var(var.clone(), item.clone())?, - value, - &specs[1..], - )?); - } - Some(out.into_iter().flatten().flatten().collect()) - } - _ => create_error_result(Error::InComprehensionCanOnlyIterateOverArray)?, - } - } - }) -} - -pub fn evaluate_member_list_object(context: Context, members: &[Member]) -> Result { - let new_bindings = FutureNewBindings::new(); - let future_this = FutureObjValue::new(); - let context_creator = context_creator!( - closure!(clone context, clone new_bindings, |this: Option, super_obj: Option| { - Ok(context.extend_unbound( - new_bindings.clone().unwrap(), - context.dollar().clone().or_else(||this.clone()), - Some(this.unwrap()), - super_obj - )?) - }) - ); - { - let mut bindings: HashMap, LazyBinding> = HashMap::new(); - for (n, b) in members - .iter() - .filter_map(|m| match m { - Member::BindStmt(b) => Some(b.clone()), - _ => None, - }) - .map(|b| evaluate_binding(&b, context_creator.clone())) - { - bindings.insert(n, b); - } - new_bindings.fill(bindings); - } - - let mut new_members = BTreeMap::new(); - for member in members.iter() { - match member { - Member::Field(FieldMember { - name, - plus, - params: None, - visibility, - value, - }) => { - let name = evaluate_field_name(context.clone(), &name)?; - if name.is_none() { - continue; - } - let name = name.unwrap(); - new_members.insert( - name.clone(), - ObjMember { - add: *plus, - visibility: *visibility, - invoke: LazyBinding::Bindable(Rc::new( - closure!(clone name, clone value, clone context_creator, |this, super_obj| { - Ok(LazyVal::new_resolved(push(&value.1, "object field", ||{ - let context = context_creator.0(this, super_obj)?; - evaluate( - context, - &value, - ) - })?)) - }), - )), - }, - ); - } - Member::Field(FieldMember { - name, - params: Some(params), - value, - .. - }) => { - let name = evaluate_field_name(context.clone(), &name)?; - if name.is_none() { - continue; - } - let name = name.unwrap(); - new_members.insert( - name, - ObjMember { - add: false, - visibility: Visibility::Hidden, - invoke: LazyBinding::Bindable(Rc::new( - closure!(clone value, clone context_creator, clone params, |this, super_obj| { - // TODO: Assert - Ok(LazyVal::new_resolved(evaluate_method( - context_creator.0(this, super_obj)?, - params.clone(), - value.clone(), - ))) - }), - )), - }, - ); - } - Member::BindStmt(_) => {} - Member::AssertStmt(_) => {} - } - } - Ok(future_this.fill(ObjValue::new(None, Rc::new(new_members)))) -} - -pub fn evaluate_object(context: Context, object: &ObjBody) -> Result { - Ok(match object { - ObjBody::MemberList(members) => evaluate_member_list_object(context, &members)?, - ObjBody::ObjComp(obj) => { - let future_this = FutureObjValue::new(); - let mut new_members = BTreeMap::new(); - for (k, v) in evaluate_comp( - context.clone(), - &|ctx| { - let new_bindings = FutureNewBindings::new(); - let context_creator = context_creator!( - closure!(clone context, clone new_bindings, |this: Option, super_obj: Option| { - Ok(context.extend_unbound( - new_bindings.clone().unwrap(), - context.dollar().clone().or_else(||this.clone()), - None, - super_obj - )?) - }) - ); - let mut bindings: HashMap, LazyBinding> = HashMap::new(); - for (n, b) in obj - .pre_locals - .iter() - .chain(obj.post_locals.iter()) - .map(|b| evaluate_binding(b, context_creator.clone())) - { - bindings.insert(n, b); - } - let bindings = new_bindings.fill(bindings); - let ctx = ctx.extend_unbound(bindings, None, None, None)?; - let key = evaluate(ctx.clone(), &obj.key)?; - let value = LazyBinding::Bindable(Rc::new( - closure!(clone ctx, clone obj.value, |this, _super_obj| { - Ok(LazyVal::new_resolved(evaluate(ctx.extend(HashMap::new(), None, this, None)?, &value)?)) - }), - )); - - Ok((key, value)) - }, - &obj.compspecs, - )? - .unwrap() - { - match k { - Val::Null => {} - Val::Str(n) => { - new_members.insert( - n, - ObjMember { - add: false, - visibility: Visibility::Normal, - invoke: v, - }, - ); - } - v => create_error_result(Error::FieldMustBeStringGot(v.value_type()?))?, - } - } - - future_this.fill(ObjValue::new(None, Rc::new(new_members))) - } - }) -} - -pub fn evaluate(context: Context, expr: &LocExpr) -> Result { - use Expr::*; - let LocExpr(expr, loc) = expr; - Ok(match &**expr { - Literal(LiteralType::This) => Val::Obj( - context - .this() - .clone() - .ok_or_else(|| create_error(crate::Error::CantUseSelfOutsideOfObject))?, - ), - Literal(LiteralType::Dollar) => Val::Obj( - context - .dollar() - .clone() - .ok_or_else(|| create_error(crate::Error::NoTopLevelObjectFound))?, - ), - Literal(LiteralType::True) => Val::Bool(true), - Literal(LiteralType::False) => Val::Bool(false), - Literal(LiteralType::Null) => Val::Null, - Parened(e) => evaluate(context, e)?, - Str(v) => Val::Str(v.clone()), - Num(v) => Val::Num(*v), - BinaryOp(v1, o, v2) => evaluate_binary_op_special(context, &v1, *o, &v2)?, - UnaryOp(o, v) => evaluate_unary_op(*o, &evaluate(context, v)?)?, - Var(name) => push(loc, "var", || { - Ok(Val::Lazy(context.binding(name.clone())?).unwrap_if_lazy()?) - })?, - Index(LocExpr(v, _), index) if matches!(&**v, Expr::Literal(LiteralType::Super)) => { - let name = evaluate(context.clone(), index)?.try_cast_str("object index")?; - context - .super_obj() - .clone() - .expect("no super found") - .get_raw(&name, &context.this().clone().expect("no this found"))? - .expect("value not found") - } - Index(value, index) => { - match ( - evaluate(context.clone(), value)?.unwrap_if_lazy()?, - evaluate(context, index)?, - ) { - (Val::Obj(v), Val::Str(s)) => { - if let Some(v) = v.get(s.clone())? { - v.unwrap_if_lazy()? - } else if let Some(Val::Str(n)) = v.get("__intristic_namespace__".into())? { - Val::Intristic(n, s) - } else { - create_error_result(crate::Error::NoSuchField(s))? - } - } - (Val::Obj(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( - ValType::Obj, - ValType::Str, - n.value_type()?, - ))?, - - (Val::Arr(v), Val::Num(n)) => { - if n.fract() > f64::EPSILON { - create_error_result(crate::Error::FractionalIndex)? - } - v.get(n as usize) - .ok_or_else(|| { - create_error(crate::Error::ArrayBoundsError(n as usize, v.len())) - })? - .clone() - .unwrap_if_lazy()? - } - (Val::Arr(_), Val::Str(n)) => { - create_error_result(crate::Error::AttemptedIndexAnArrayWithString(n))? - } - (Val::Arr(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( - ValType::Arr, - ValType::Num, - n.value_type()?, - ))?, - - (Val::Str(s), Val::Num(n)) => Val::Str( - s.chars() - .skip(n as usize) - .take(1) - .collect::() - .into(), - ), - (Val::Str(_), n) => create_error_result(crate::Error::ValueIndexMustBeTypeGot( - ValType::Str, - ValType::Num, - n.value_type()?, - ))?, - - (v, _) => create_error_result(crate::Error::CantIndexInto(v.value_type()?))?, - } - } - LocalExpr(bindings, returned) => { - let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); - let future_context = Context::new_future(); - - let context_creator = context_creator!( - closure!(clone future_context, |_, _| Ok(future_context.clone().unwrap())) - ); - - for (k, v) in bindings - .iter() - .map(|b| evaluate_binding(b, context_creator.clone())) - { - new_bindings.insert(k, v); - } - - let context = context - .extend_unbound(new_bindings, None, None, None)? - .into_future(future_context); - evaluate(context, &returned.clone())? - } - Arr(items) => { - let mut out = Vec::with_capacity(items.len()); - for item in items { - out.push(Val::Lazy(lazy_val!( - closure!(clone context, clone item, || { - evaluate(context.clone(), &item) - }) - ))); - } - Val::Arr(Rc::new(out)) - } - ArrComp(expr, compspecs) => Val::Arr( - // First compspec should be forspec, so no "None" possible here - Rc::new(evaluate_comp(context, &|ctx| evaluate(ctx, expr), compspecs)?.unwrap()), - ), - Obj(body) => Val::Obj(evaluate_object(context, body)?), - ObjExtend(s, t) => evaluate_add_op( - &evaluate(context.clone(), s)?, - &Val::Obj(evaluate_object(context, t)?), - )?, - Apply(value, args, tailstrict) => { - let lazy = evaluate(context.clone(), value)?; - let value = lazy.unwrap_if_lazy()?; - match value { - Val::Intristic(ns, name) => match (&ns as &str, &name as &str) { - // arr/string/function - ("std", "length") => parse_args!(context, "std.length", args, 1, [ - 0, x: [Val::Str|Val::Arr|Val::Obj], vec![ValType::Str, ValType::Arr, ValType::Obj]; - ], { - match x { - Val::Str(n) => Val::Num(n.chars().count() as f64), - Val::Arr(i) => Val::Num(i.len() as f64), - Val::Obj(o) => Val::Num( - o.fields_visibility() - .into_iter() - .filter(|(_k, v)| *v) - .count() as f64, - ), - _ => unreachable!(), - } - }), - // any - ("std", "type") => parse_args!(context, "std.type", args, 1, [ - 0, x, vec![]; - ], { - Val::Str(x.value_type()?.name().into()) - }), - // length, idx=>any - ("std", "makeArray") => parse_args!(context, "std.makeArray", args, 2, [ - 0, sz: [Val::Num]!!Val::Num, vec![ValType::Num]; - 1, func: [Val::Func]!!Val::Func, vec![ValType::Func]; - ], { - assert!(sz >= 0.0); - let mut out = Vec::with_capacity(sz as usize); - for i in 0..sz as usize { - out.push(func.evaluate_values( - Context::new(), - &[Val::Num(i as f64)] - )?) - } - Val::Arr(Rc::new(out)) - }), - // string - ("std", "codepoint") => parse_args!(context, "std.codepoint", args, 1, [ - 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - assert!( - str.chars().count() == 1, - "std.codepoint should receive single char string" - ); - Val::Num(str.chars().take(1).next().unwrap() as u32 as f64) - }), - // object, includeHidden - ("std", "objectFieldsEx") => { - parse_args!(context, "std.objectFieldsEx",args, 2, [ - 0, obj: [Val::Obj]!!Val::Obj, vec![ValType::Obj]; - 1, inc_hidden: [Val::Bool]!!Val::Bool, vec![ValType::Bool]; - ], { - Val::Arr(Rc::new( - obj.fields_visibility() - .into_iter() - .filter(|(_k, v)| *v || inc_hidden) - .map(|(k, _v)| Val::Str(k)) - .collect(), - )) - }) - } - // object, field, includeHidden - ("std", "objectHasEx") => parse_args!(context, "std.objectHasEx", args, 3, [ - 0, obj: [Val::Obj]!!Val::Obj, vec![ValType::Obj]; - 1, f: [Val::Str]!!Val::Str, vec![ValType::Str]; - 2, inc_hidden: [Val::Bool]!!Val::Bool, vec![ValType::Bool]; - ], { - Val::Bool( - obj.fields_visibility() - .into_iter() - .filter(|(_k, v)| *v || inc_hidden) - .any(|(k, _v)| *k == *f), - ) - }), - ("std", "primitiveEquals") => { - parse_args!(context, "std.primitiveEquals", args, 2, [ - 0, a, vec![]; - 1, b, vec![]; - ], { - Val::Bool(a == b) - }) - } - ("std", "modulo") => parse_args!(context, "std.modulo", args, 2, [ - 0, a: [Val::Num]!!Val::Num, vec![ValType::Num]; - 1, b: [Val::Num]!!Val::Num, vec![ValType::Num]; - ], { - Val::Num(a % b) - }), - ("std", "floor") => parse_args!(context, "std.floor", args, 1, [ - 0, x: [Val::Num]!!Val::Num, vec![ValType::Num]; - ], { - Val::Num(x.floor()) - }), - ("std", "trace") => parse_args!(context, "std.trace", args, 2, [ - 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; - 1, rest, vec![]; - ], { - // TODO: Line numbers as in original jsonnet - println!("TRACE: {}", str); - rest - }), - ("std", "pow") => parse_args!(context, "std.modulo", args, 2, [ - 0, x: [Val::Num]!!Val::Num, vec![ValType::Num]; - 1, n: [Val::Num]!!Val::Num, vec![ValType::Num]; - ], { - Val::Num(x.powf(n)) - }), - ("std", "extVar") => parse_args!(context, "std.extVar", args, 2, [ - 0, x: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - with_state(|s| s.0.ext_vars.borrow().get(&x).cloned()).ok_or_else( - || create_error(crate::Error::UndefinedExternalVariable(x)), - )? - }), - ("std", "filter") => parse_args!(context, "std.filter", args, 2, [ - 0, func: [Val::Func]!!Val::Func, vec![ValType::Func]; - 1, arr: [Val::Arr]!!Val::Arr, vec![ValType::Arr]; - ], { - Val::Arr(Rc::new( - arr.iter() - .cloned() - .filter(|e| { - func - .evaluate_values(context.clone(), &[e.clone()]) - .unwrap() - .try_cast_bool("filter predicate") - .unwrap() - }) - .collect(), - )) - }), - ("std", "char") => parse_args!(context, "std.char", args, 1, [ - 0, n: [Val::Num]!!Val::Num, vec![ValType::Num]; - ], { - let mut out = String::new(); - out.push(std::char::from_u32(n as u32).unwrap()); - Val::Str(out.into()) - }), - ("std", "encodeUTF8") => parse_args!(context, "std.encodeUtf8", args, 1, [ - 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - Val::Arr(Rc::new(str.bytes().map(|b| Val::Num(b as f64)).collect())) - }), - ("std", "md5") => parse_args!(context, "std.md5", args, 1, [ - 0, str: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - Val::Str(format!("{:x}", md5::compute(str.as_bytes())).into()) - }), - // faster - ("std", "join") => parse_args!(context, "std.join", args, 2, [ - 0, sep: [Val::Str|Val::Arr], vec![ValType::Str, ValType::Arr]; - 1, arr: [Val::Arr]!!Val::Arr, vec![ValType::Arr]; - ], { - match sep { - Val::Arr(joiner_items) => { - let mut out = Vec::new(); - - let mut first = true; - for item in arr.iter().cloned() { - if let Val::Arr(items) = item.unwrap_if_lazy()? { - if !first { - out.reserve(joiner_items.len()); - out.extend(joiner_items.iter().cloned()); - } - first = false; - out.reserve(items.len()); - out.extend(items.iter().cloned()); - } else { - create_error_result(crate::Error::RuntimeError("in std.join all items should be arrays".into()))?; - } - } - - Val::Arr(Rc::new(out)) - }, - Val::Str(sep) => { - let mut out = String::new(); - - let mut first = true; - for item in arr.iter().cloned() { - if let Val::Str(item) = item.unwrap_if_lazy()? { - if !first { - out += &sep; - } - first = false; - out += &item; - } else { - create_error_result(crate::Error::RuntimeError("in std.join all items should be strings".into()))?; - } - } - - Val::Str(out.into()) - }, - _ => unreachable!() - } - }), - // Faster - ("std", "escapeStringJson") => { - parse_args!(context, "std.escapeStringJson", args, 1, [ - 0, str_: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - Val::Str(escape_string_json(&str_).into()) - }) - } - // Faster - ("std", "manifestJsonEx") => { - parse_args!(context, "std.manifestJsonEx", args, 2, [ - 0, value, vec![]; - 1, indent: [Val::Str]!!Val::Str, vec![ValType::Str]; - ], { - Val::Str(manifest_json_ex(&value, &indent)?.into()) - }) - } - (ns, name) => create_error_result(crate::Error::IntristicNotFound( - ns.into(), - name.into(), - ))?, - }, - Val::Func(f) => { - let body = || f.evaluate(context, args, *tailstrict); - if *tailstrict { - body()? - } else { - push(loc, "function call", body)? - } - } - v => create_error_result(crate::Error::OnlyFunctionsCanBeCalledGot( - v.value_type()?, - ))?, - } - } - Function(params, body) => evaluate_method(context, params.clone(), body.clone()), - AssertExpr(AssertStmt(value, msg), returned) => { - let assertion_result = push(&value.1, "assertion condition", || { - evaluate(context.clone(), &value)? - .try_cast_bool("assertion condition should be boolean") - })?; - if assertion_result { - evaluate(context, returned)? - } else if let Some(msg) = msg { - create_error_result(crate::Error::AssertionFailed(evaluate(context, msg)?))? - } else { - create_error_result(crate::Error::AssertionFailed(Val::Null))? - } - } - Error(e) => create_error_result(crate::Error::RuntimeError( - evaluate(context, e)?.try_cast_str("error text should be string")?, - ))?, - IfElse { - cond, - cond_then, - cond_else, - } => { - if evaluate(context.clone(), &cond.0)? - .try_cast_bool("if condition should be boolean")? - { - evaluate(context, cond_then)? - } else { - match cond_else { - Some(v) => evaluate(context, v)?, - None => Val::Null, - } - } - } - Import(path) => { - let mut tmp = loc - .clone() - .expect("imports can't be used without loc_data") - .0; - let import_location = Rc::make_mut(&mut tmp); - import_location.pop(); - with_state(|s| s.import_file(&import_location, path))? - } - ImportStr(path) => { - let mut tmp = loc - .clone() - .expect("imports can't be used without loc_data") - .0; - let import_location = Rc::make_mut(&mut tmp); - import_location.pop(); - Val::Str(with_state(|s| s.import_file_str(&import_location, path))?) - } - Literal(LiteralType::Super) => { - return create_error_result(crate::Error::StandaloneSuper) - } - }) -} --- a/crates/jsonnet-evaluator/src/function.rs +++ /dev/null @@ -1,157 +0,0 @@ -use crate::{ - create_error, create_error_result, evaluate, lazy_val, resolved_lazy_val, Context, Error, - Result, Val, -}; -use closure::closure; -use jsonnet_parser::{ArgsDesc, ParamsDesc}; -use std::collections::HashMap; - -/// Creates correct [context](Context) for function body evaluation, returning error on invalid call -/// -/// * `ctx` used for passed argument expressions execution, and for body execution (if `body_ctx` is not set) -/// * `body_ctx` used for default parameter values execution, and for body execution (if set) -/// * `params` function parameters definition -/// * `args` passed function arguments -/// * `tailstruct` if true - function arguments is eager executed, otherwise - lazy -pub fn parse_function_call( - ctx: Context, - body_ctx: Option, - params: &ParamsDesc, - args: &ArgsDesc, - tailstrict: bool, -) -> Result { - let mut out = HashMap::new(); - let mut positioned_args = vec![None; params.0.len()]; - for (id, arg) in args.iter().enumerate() { - let idx = if let Some(name) = &arg.0 { - params - .iter() - .position(|p| *p.0 == *name) - .ok_or_else(|| create_error(Error::UnknownFunctionParameter(name.clone())))? - } else { - id - }; - - if idx >= params.len() { - create_error_result(Error::TooManyArgsFunctionHas(params.len()))?; - } - if positioned_args[idx].is_some() { - create_error_result(Error::BindingParameterASecondTime(params[idx].0.clone()))?; - } - positioned_args[idx] = Some(arg.1.clone()); - } - // Fill defaults - for (id, p) in params.iter().enumerate() { - let (ctx, expr) = if let Some(arg) = &positioned_args[id] { - (ctx.clone(), arg) - } else if let Some(default) = &p.1 { - ( - body_ctx - .clone() - .expect("no default context set for call with defined default parameter value"), - default, - ) - } else { - create_error_result(Error::FunctionParameterNotBoundInCall(p.0.clone()))?; - unreachable!() - }; - let val = if tailstrict { - resolved_lazy_val!(evaluate(ctx, expr)?) - } else { - lazy_val!(closure!(clone ctx, clone expr, ||evaluate(ctx.clone(), &expr))) - }; - out.insert(p.0.clone(), val); - } - - Ok(body_ctx.unwrap_or(ctx).extend(out, None, None, None)?) -} - -pub(crate) fn place_args( - ctx: Context, - body_ctx: Option, - params: &ParamsDesc, - args: &[Val], -) -> Result { - let mut out = HashMap::new(); - let mut positioned_args = vec![None; params.0.len()]; - for (id, arg) in args.iter().enumerate() { - if id >= params.len() { - create_error_result(Error::TooManyArgsFunctionHas(params.len()))?; - } - positioned_args[id] = Some(arg); - } - // Fill defaults - for (id, p) in params.iter().enumerate() { - let val = if let Some(arg) = &positioned_args[id] { - (*arg).clone() - } else if let Some(default) = &p.1 { - evaluate(ctx.clone(), default)? - } else { - create_error_result(Error::FunctionParameterNotBoundInCall(p.0.clone()))?; - unreachable!() - }; - out.insert(p.0.clone(), resolved_lazy_val!(val)); - } - - Ok(body_ctx.unwrap_or(ctx).extend(out, None, None, None)?) -} - -#[macro_export] -macro_rules! parse_args { - ($ctx: expr, $fn_name: expr, $args: expr, $total_args: expr, [ - $($id: expr, $name: ident $(: [$($p: path)|+] $(!! $a: path)?)?, $nt: expr);+ $(;)? - ], $handler:block) => {{ - use crate::Error; - let args = $args; - if args.len() > $total_args { - create_error_result(Error::TooManyArgsFunctionHas($total_args))?; - } - $( - if args.len() <= $id { - create_error_result(Error::FunctionParameterNotBoundInCall(stringify!($name).into()))?; - } - let $name = &args[$id]; - if $name.0.is_some() { - if $name.0.as_ref().unwrap() != stringify!($name) { - create_error_result(Error::IntristicArgumentReorderingIsNotSupportedYet)?; - } - } - let $name = evaluate($ctx.clone(), &$name.1)?; - $( - match $name { - $($p(_))|+ => {}, - _ => create_error_result(Error::TypeMismatch(concat!($fn_name, " ", stringify!($id), "nd argument"), $nt, $name.value_type()?))?, - }; - $( - let $name = match $name { - $a(v) => v, - _ => create_error_result(Error::TypeMismatch(concat!($fn_name, " ", stringify!($id), "nd argument"), $nt, $name.value_type()?))?, - }; - )* - )* - )+ - $handler - }}; -} - -#[test] -fn test() -> Result<()> { - use jsonnet_parser::*; - use crate::val::ValType; - let state = crate::EvaluationState::default(); - let evaluator = state.with_stdlib(); - let ctx = evaluator.create_default_context()?; - evaluator.run_in_state(|| { - parse_args!(ctx, "test", ArgsDesc(vec![ - Arg(None, el!(Expr::Num(2.0))), - Arg(Some("b".into()), el!(Expr::Num(1.0))), - ]), 2, [ - 0, a: [Val::Num]!!Val::Num, vec![ValType::Num]; - 1, b: [Val::Num]!!Val::Num, vec![ValType::Num]; - ], { - assert!((a - 2.0).abs() <= f64::EPSILON); - assert!((b - 1.0).abs() <= f64::EPSILON); - }); - Ok(()) - }) -} --- a/crates/jsonnet-evaluator/src/import.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::create_error_result; -use crate::{ - create_error, - error::{Error, Result}, -}; -use fs::File; -use std::fs; -use std::io::Read; -use std::{cell::RefCell, collections::HashMap, path::PathBuf, rc::Rc}; - -pub trait ImportResolver { - fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result>; - fn load_file_contents(&self, resolved: &PathBuf) -> Result>; -} - -pub struct DummyImportResolver; -impl ImportResolver for DummyImportResolver { - fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { - create_error_result(Error::ImportNotSupported(from.clone(), path.clone())) - } - fn load_file_contents(&self, _resolved: &PathBuf) -> Result> { - // Can be only caused by library direct consumer, not by supplied jsonnet - panic!("dummy resolver can't load any file") - } -} -impl Default for Box { - fn default() -> Self { - Box::new(DummyImportResolver) - } -} - -pub struct FileImportResolver { - pub library_paths: Vec, -} -impl ImportResolver for FileImportResolver { - fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { - let mut new_path = from.clone(); - new_path.push(path); - if new_path.exists() { - Ok(Rc::new(new_path)) - } else { - for library_path in self.library_paths.iter() { - let mut cloned = library_path.clone(); - cloned.push(path); - if cloned.exists() { - return Ok(Rc::new(cloned)); - } - } - create_error_result(Error::ImportFileNotFound(from.clone(), path.clone())) - } - } - fn load_file_contents(&self, id: &PathBuf) -> Result> { - let mut file = - File::open(id).map_err(|_e| create_error(Error::ResolvedFileNotFound(id.clone())))?; - let mut out = String::new(); - file.read_to_string(&mut out) - .map_err(|_e| create_error(Error::ImportBadFileUtf8(id.clone())))?; - Ok(out.into()) - } -} - -type ResolutionData = (PathBuf, PathBuf); -pub struct CachingImportResolver { - resolution_cache: RefCell>>>, - loading_cache: RefCell>>>, - inner: Box, -} -impl ImportResolver for CachingImportResolver { - fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { - self.resolution_cache - .borrow_mut() - .entry((from.clone(), path.clone())) - .or_insert_with(|| self.inner.resolve_file(from, path)) - .clone() - } - fn load_file_contents(&self, resolved: &PathBuf) -> Result> { - self.loading_cache - .borrow_mut() - .entry(resolved.clone()) - .or_insert_with(|| self.inner.load_file_contents(resolved)) - .clone() - } -} --- a/crates/jsonnet-evaluator/src/lib.rs +++ /dev/null @@ -1,730 +0,0 @@ -#![feature(box_syntax, box_patterns)] -#![feature(type_alias_impl_trait)] -#![feature(debug_non_exhaustive)] -#![feature(test)] -#![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)] - -extern crate test; - -mod ctx; -mod dynamic; -mod error; -mod evaluate; -mod function; -mod import; -mod map; -mod obj; -mod val; - -pub use ctx::*; -pub use dynamic::*; -pub use error::*; -pub use evaluate::*; -pub use function::parse_function_call; -pub use import::*; -use jsonnet_parser::*; -pub use obj::*; -use std::{cell::RefCell, collections::HashMap, fmt::Debug, path::PathBuf, rc::Rc}; -pub use val::*; - -type BindableFn = dyn Fn(Option, Option) -> Result; -#[derive(Clone)] -pub enum LazyBinding { - Bindable(Rc), - Bound(LazyVal), -} - -impl Debug for LazyBinding { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "LazyBinding") - } -} -impl LazyBinding { - pub fn evaluate(&self, this: Option, super_obj: Option) -> Result { - match self { - LazyBinding::Bindable(v) => v(this, super_obj), - LazyBinding::Bound(v) => Ok(v.clone()), - } - } -} - -pub struct EvaluationSettings { - pub max_stack_frames: usize, - pub max_stack_trace_size: usize, -} -impl Default for EvaluationSettings { - fn default() -> Self { - EvaluationSettings { - max_stack_frames: 200, - max_stack_trace_size: 20, - } - } -} - -pub struct FileData(Rc, LocExpr, Option); -#[derive(Default)] -pub struct EvaluationStateInternals { - /// Used for stack-overflows and stacktraces - stack: RefCell>, - /// Contains file source codes and evaluated results for imports and pretty - /// printing stacktraces - files: RefCell, FileData>>, - str_files: RefCell, Rc>>, - globals: RefCell, Val>>, - - /// Values to use with std.extVar - ext_vars: RefCell, Val>>, - - settings: EvaluationSettings, - import_resolver: Box, -} - -thread_local! { - /// Contains state for currently executing file - /// Global state is fine there - pub(crate) static EVAL_STATE: RefCell> = RefCell::new(None) -} -pub(crate) fn with_state(f: impl FnOnce(&EvaluationState) -> T) -> T { - EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap())) -} -pub(crate) fn create_error(err: Error) -> LocError { - with_state(|s| s.error(err)) -} -pub(crate) fn create_error_result(err: Error) -> Result { - Err(with_state(|s| s.error(err))) -} -pub(crate) fn push( - e: &Option, - comment: &str, - f: impl FnOnce() -> Result, -) -> Result { - if e.is_some() { - with_state(|s| s.push(e.clone().unwrap(), comment.to_owned(), f)) - } else { - f() - } -} - -/// Maintains stack trace and import resolution -#[derive(Default, Clone)] -pub struct EvaluationState(Rc); -impl EvaluationState { - pub fn new(settings: EvaluationSettings, import_resolver: Box) -> Self { - EvaluationState(Rc::new(EvaluationStateInternals { - settings, - import_resolver, - ..Default::default() - })) - } - pub fn add_file( - &self, - name: Rc, - code: Rc, - ) -> std::result::Result<(), ParseError> { - self.0.files.borrow_mut().insert( - name.clone(), - FileData( - code.clone(), - parse( - &code, - &ParserSettings { - file_name: name, - loc_data: true, - }, - )?, - None, - ), - ); - - Ok(()) - } - pub fn add_parsed_file( - &self, - name: Rc, - code: Rc, - parsed: LocExpr, - ) -> std::result::Result<(), ()> { - self.0 - .files - .borrow_mut() - .insert(name, FileData(code, parsed, None)); - - Ok(()) - } - pub fn get_source(&self, name: &PathBuf) -> Option> { - let ro_map = self.0.files.borrow(); - ro_map.get(name).map(|value| value.0.clone()) - } - pub fn evaluate_file(&self, name: &PathBuf) -> Result { - self.run_in_state(|| { - let expr: LocExpr = { - let ro_map = self.0.files.borrow(); - let value = ro_map - .get(name) - .unwrap_or_else(|| panic!("file not added: {:?}", name)); - if value.2.is_some() { - return Ok(value.2.clone().unwrap()); - } - value.1.clone() - }; - let value = evaluate(self.create_default_context()?, &expr)?; - { - self.0 - .files - .borrow_mut() - .get_mut(name) - .unwrap() - .2 - .replace(value.clone()); - } - Ok(value) - }) - } - pub(crate) fn import_file(&self, from: &PathBuf, path: &PathBuf) -> Result { - let file_path = self.0.import_resolver.resolve_file(from, path)?; - { - let files = self.0.files.borrow(); - if files.contains_key(&file_path) { - return self.evaluate_file(&file_path); - } - } - let contents = self.0.import_resolver.load_file_contents(&file_path)?; - self.add_file(file_path.clone(), contents).map_err(|e| { - create_error(Error::ImportSyntaxError(e)) - })?; - self.evaluate_file(&file_path) - } - pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result> { - let path = self.0.import_resolver.resolve_file(from, path)?; - if !self.0.str_files.borrow().contains_key(&path) { - let file_str = self.0.import_resolver.load_file_contents(&path)?; - self.0 - .str_files - .borrow_mut() - .insert(path.clone(), file_str); - } - Ok(self.0.str_files.borrow().get(&path).cloned().unwrap()) - } - - pub fn parse_evaluate_raw(&self, code: &str) -> Result { - let parsed = parse( - &code, - &ParserSettings { - file_name: Rc::new(PathBuf::from("raw.jsonnet")), - loc_data: true, - }, - ) - .unwrap(); - self.evaluate_raw(parsed) - } - - pub fn evaluate_raw(&self, code: LocExpr) -> Result { - self.run_in_state(|| evaluate(self.create_default_context()?, &code)) - } - - pub fn add_global(&self, name: Rc, value: Val) { - self.0.globals.borrow_mut().insert(name, value); - } - pub fn add_ext_var(&self, name: Rc, value: Val) { - self.0.ext_vars.borrow_mut().insert(name, value); - } - - pub fn with_stdlib(&self) -> &Self { - let std_path = Rc::new(PathBuf::from("std.jsonnet")); - self.run_in_state(|| { - use jsonnet_stdlib::STDLIB_STR; - let mut parsed = false; - #[cfg(feature = "codegenerated-stdlib")] - if !parsed { - parsed = true; - #[allow(clippy::all)] - let stdlib = { - use jsonnet_parser::*; - include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) - }; - self.add_parsed_file(std_path.clone(), STDLIB_STR.to_owned().into(), stdlib) - .unwrap(); - } - - #[cfg(feature = "serialized-stdlib")] - if !parsed { - parsed = true; - self.add_parsed_file( - std_path.clone(), - STDLIB_STR.to_owned().into(), - bincode::deserialize(include_bytes!(concat!( - env!("OUT_DIR"), - "/stdlib.bincode" - ))) - .expect("deserialize stdlib"), - ) - .unwrap(); - } - - if !parsed { - self.add_file(std_path, STDLIB_STR.to_owned().into()) - .unwrap(); - } - let val = self.evaluate_file(&PathBuf::from("std.jsonnet")).unwrap(); - self.add_global("std".into(), val); - }); - self - } - - pub fn create_default_context(&self) -> Result { - let globals = self.0.globals.borrow(); - let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); - for (name, value) in globals.iter() { - new_bindings.insert( - name.clone(), - LazyBinding::Bound(resolved_lazy_val!(value.clone())), - ); - } - Context::new().extend_unbound(new_bindings, None, None, None) - } - - pub fn push( - &self, - e: ExprLocation, - comment: String, - f: impl FnOnce() -> Result, - ) -> Result { - { - let mut stack = self.0.stack.borrow_mut(); - if stack.len() > self.0.settings.max_stack_frames { - drop(stack); - return Err(self.error(Error::StackOverflow)); - } else { - stack.push(StackTraceElement(e, comment)); - } - } - let result = f(); - self.0.stack.borrow_mut().pop(); - result - } - pub fn print_stack_trace(&self) { - for e in self.stack_trace().0 { - println!("{:?} - {:?}", e.0, e.1) - } - } - pub fn stack_trace(&self) -> StackTrace { - StackTrace( - self.0 - .stack - .borrow() - .iter() - .rev() - .take(self.0.settings.max_stack_trace_size) - .cloned() - .collect(), - ) - } - pub fn error(&self, err: Error) -> LocError { - LocError(err, self.stack_trace()) - } - - pub fn run_in_state(&self, f: impl FnOnce() -> T) -> T { - EVAL_STATE.with(|v| { - let has_state = v.borrow().is_some(); - if !has_state { - v.borrow_mut().replace(self.clone()); - } - let result = f(); - if !has_state { - v.borrow_mut().take(); - } - result - }) - } -} - -#[cfg(test)] -pub mod tests { - use super::Val; - use crate::EvaluationState; - use jsonnet_parser::*; - use std::{path::PathBuf, rc::Rc}; - - #[test] - fn eval_state_stacktrace() { - let state = EvaluationState::default(); - state - .push( - ExprLocation(Rc::new(PathBuf::from("test1.jsonnet")), 10, 20), - "outer".to_owned(), - || { - state.push( - ExprLocation(Rc::new(PathBuf::from("test2.jsonnet")), 30, 40), - "inner".to_owned(), - || { - state.print_stack_trace(); - Ok(()) - }, - )?; - Ok(()) - }, - ) - .unwrap(); - } - - #[test] - fn eval_state_standard() { - let state = EvaluationState::default(); - state.with_stdlib(); - assert_eq!( - state - .parse_evaluate_raw(r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#) - .unwrap(), - Val::Bool(true) - ); - } - - macro_rules! eval { - ($str: expr) => { - EvaluationState::default() - .with_stdlib() - .parse_evaluate_raw($str) - .unwrap() - }; - } - macro_rules! eval_json { - ($str: expr) => {{ - let evaluator = EvaluationState::default(); - evaluator.with_stdlib(); - evaluator.run_in_state(||{ - evaluator - .parse_evaluate_raw($str) - .unwrap() - .into_json(0) - .unwrap() - .replace("\n", "") - }) - }} - } - - /// Asserts given code returns `true` - macro_rules! assert_eval { - ($str: expr) => { - assert_eq!(eval!($str), Val::Bool(true)) - }; - } - - /// Asserts given code returns `false` - macro_rules! assert_eval_neg { - ($str: expr) => { - assert_eq!(eval!($str), Val::Bool(false)) - }; - } - macro_rules! assert_json { - ($str: expr, $out: expr) => { - assert_eq!(eval_json!($str), $out.replace("\t", "")) - }; - } - - /// Sanity checking, before trusting to another tests - #[test] - fn equality_operator() { - assert_eval!("2 == 2"); - assert_eval_neg!("2 != 2"); - assert_eval!("2 != 3"); - assert_eval_neg!("2 == 3"); - assert_eval!("'Hello' == 'Hello'"); - assert_eval_neg!("'Hello' != 'Hello'"); - assert_eval!("'Hello' != 'World'"); - assert_eval_neg!("'Hello' == 'World'"); - } - - #[test] - fn math_evaluation() { - assert_eval!("2 + 2 * 2 == 6"); - assert_eval!("3 + (2 + 2 * 2) == 9"); - } - - #[test] - fn string_concat() { - assert_eval!("'Hello' + 'World' == 'HelloWorld'"); - assert_eval!("'Hello' * 3 == 'HelloHelloHello'"); - assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'"); - } - - #[test] - fn faster_join() { - assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]"); - assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'"); - } - - #[test] - fn function_contexts() { - assert_eval!( - r#" - local k = { - t(name = self.h): [self.h, name], - h: 3, - }; - local f = { - t: k.t(), - h: 4, - }; - f.t[0] == f.t[1] - "# - ); - } - - #[test] - fn local() { - assert_eval!("local a = 2; local b = 3; a + b == 5"); - assert_eval!("local a = 1, b = a + 1; a + b == 3"); - assert_eval!("local a = 1; local a = 2; a == 2"); - } - - #[test] - fn object_lazyness() { - assert_json!("local a = {a:error 'test'}; {}", r#"{}"#); - } - - #[test] - fn object_inheritance() { - assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#); - } - - #[test] - fn object_assertion_success() { - eval!("{assert \"a\" in self} + {a:2}"); - } - - #[test] - fn object_assertion_error() { - eval!("{assert \"a\" in self}"); - } - - #[test] - fn lazy_args() { - eval!("local test(a) = 2; test(error '3')"); - } - - #[test] - #[should_panic] - fn tailstrict_args() { - eval!("local test(a) = 2; test(error '3') tailstrict"); - } - - #[test] - #[should_panic] - fn no_binding_error() { - eval!("a"); - } - - #[test] - fn test_object() { - assert_json!("{a:2}", r#"{"a": 2}"#); - assert_json!("{a:2+2}", r#"{"a": 4}"#); - assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#); - assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#); - assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#); - assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#); - assert_json!( - r#" - { - name: "Alice", - welcome: "Hello " + self.name + "!", - } - "#, - r#"{"name": "Alice","welcome": "Hello Alice!"}"# - ); - assert_json!( - r#" - { - name: "Alice", - welcome: "Hello " + self.name + "!", - } + { - name: "Bob" - } - "#, - r#"{"name": "Bob","welcome": "Hello Bob!"}"# - ); - } - - #[test] - fn functions() { - assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4"); - assert_json!( - r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#, - r#""HelloDearWorld""# - ); - } - - #[test] - fn local_methods() { - assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4"); - assert_json!( - r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#, - r#""HelloDearWorld""# - ); - } - - #[test] - fn object_locals() { - assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#); - assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#); - assert_json!( - r#"{local a = function (b) {[b]:4}, test: a("test")}"#, - r#"{"test": {"test": 4}}"# - ); - } - - #[test] - fn object_comp() { - assert_json!( - r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#, - "{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}" - ) - } - - #[test] - fn direct_self() { - println!( - "{:#?}", - eval!( - r#" - { - local me = self, - a: 3, - b(): me.a, - } - "# - ) - ); - } - - #[test] - fn indirect_self() { - // `self` assigned to `me` was lost when being - // referenced from field - eval!( - r#"{ - local me = self, - a: 3, - b: me.a, - }.b"# - ); - } - - // We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly - #[test] - fn std_assert_ok() { - eval!("std.assertEqual(4.5 << 2, 16)"); - } - - #[test] - #[should_panic] - fn std_assert_failure() { - eval!("std.assertEqual(4.5 << 2, 15)"); - } - - #[test] - fn string_is_string() { - assert_eq!( - eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"), - Val::Bool(false) - ); - } - - #[test] - fn base64_works() { - assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#); - } - - #[test] - fn utf8_chars() { - assert_json!( - r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#, - r#"{"c": 128526,"l": 1}"# - ) - } - - #[test] - fn json() { - assert_json!( - r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#, - r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""# - ); - } - - #[test] - fn test() { - assert_json!( - r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#, - "[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]" - ); - } - - #[test] - fn sjsonnet() { - eval!( - r#" - local x0 = {k: 1}; - local x1 = {k: x0.k + x0.k}; - local x2 = {k: x1.k + x1.k}; - local x3 = {k: x2.k + x2.k}; - local x4 = {k: x3.k + x3.k}; - local x5 = {k: x4.k + x4.k}; - local x6 = {k: x5.k + x5.k}; - local x7 = {k: x6.k + x6.k}; - local x8 = {k: x7.k + x7.k}; - local x9 = {k: x8.k + x8.k}; - local x10 = {k: x9.k + x9.k}; - local x11 = {k: x10.k + x10.k}; - local x12 = {k: x11.k + x11.k}; - local x13 = {k: x12.k + x12.k}; - local x14 = {k: x13.k + x13.k}; - local x15 = {k: x14.k + x14.k}; - local x16 = {k: x15.k + x15.k}; - local x17 = {k: x16.k + x16.k}; - local x18 = {k: x17.k + x17.k}; - local x19 = {k: x18.k + x18.k}; - local x20 = {k: x19.k + x19.k}; - local x21 = {k: x20.k + x20.k}; - x21.k - "# - ); - } - - use test::Bencher; - - // This test is commented out by default, because of huge compilation slowdown - // #[bench] - // fn bench_codegen(b: &mut Bencher) { - // b.iter(|| { - // #[allow(clippy::all)] - // let stdlib = { - // use jsonnet_parser::*; - // include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) - // }; - // stdlib - // }) - // } - - #[bench] - fn bench_serialize(b: &mut Bencher) { - b.iter(|| { - bincode::deserialize::(include_bytes!(concat!( - env!("OUT_DIR"), - "/stdlib.bincode" - ))) - .expect("deserialize stdlib") - }) - } - - #[bench] - fn bench_parse(b: &mut Bencher) { - b.iter(|| { - jsonnet_parser::parse( - jsonnet_stdlib::STDLIB_STR, - &jsonnet_parser::ParserSettings { - loc_data: true, - file_name: Rc::new(PathBuf::from("std.jsonnet")), - }, - ) - }) - } -} --- a/crates/jsonnet-evaluator/src/map.rs +++ /dev/null @@ -1,46 +0,0 @@ -use std::{borrow::Borrow, collections::HashMap, hash::Hash, rc::Rc}; - -#[derive(Default, Debug)] -struct LayeredHashMapInternals { - parent: Option>, - current: HashMap, -} - -#[derive(Debug)] -pub struct LayeredHashMap(Rc>); - -impl LayeredHashMap { - pub fn extend(&self, new_layer: HashMap) -> Self { - let super_map = self.clone(); - LayeredHashMap(Rc::new(LayeredHashMapInternals { - parent: Some(super_map), - current: new_layer, - })) - } - - pub fn get(&self, key: &Q) -> Option<&V> - where - K: Borrow, - Q: Hash + Eq, - { - (self.0) - .current - .get(&key) - .or_else(|| self.0.parent.as_ref().and_then(|p| p.get(key))) - } -} - -impl Clone for LayeredHashMap { - fn clone(&self) -> Self { - LayeredHashMap(self.0.clone()) - } -} - -impl Default for LayeredHashMap { - fn default() -> Self { - LayeredHashMap(Rc::new(LayeredHashMapInternals { - parent: None, - current: HashMap::new(), - })) - } -} --- a/crates/jsonnet-evaluator/src/obj.rs +++ /dev/null @@ -1,138 +0,0 @@ -use crate::{evaluate_add_op, LazyBinding, Result, Val}; -use indexmap::IndexMap; -use jsonnet_parser::Visibility; -use std::{ - cell::RefCell, - collections::{BTreeMap, HashMap}, - fmt::Debug, - rc::Rc, -}; - -#[derive(Debug)] -pub struct ObjMember { - pub add: bool, - pub visibility: Visibility, - pub invoke: LazyBinding, -} - -#[derive(Debug)] -pub struct ObjValueInternals { - super_obj: Option, - this_entries: Rc, ObjMember>>, - value_cache: RefCell, Val>>, -} -#[derive(Clone)] -pub struct ObjValue(pub(crate) Rc); -impl Debug for ObjValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(super_obj) = self.0.super_obj.as_ref() { - if f.alternate() { - write!(f, "{:#?}", super_obj)?; - } else { - write!(f, "{:?}", super_obj)?; - } - write!(f, " + ")?; - } - let mut debug = f.debug_struct("ObjValue"); - for (name, member) in self.0.this_entries.iter() { - debug.field(name, member); - } - debug.finish_non_exhaustive() - } -} - -impl ObjValue { - pub fn new( - super_obj: Option, - this_entries: Rc, ObjMember>>, - ) -> ObjValue { - ObjValue(Rc::new(ObjValueInternals { - super_obj, - this_entries, - value_cache: RefCell::new(HashMap::new()), - })) - } - pub fn with_super(&self, super_obj: ObjValue) -> ObjValue { - match &self.0.super_obj { - None => ObjValue::new(Some(super_obj), self.0.this_entries.clone()), - Some(v) => ObjValue::new(Some(v.with_super(super_obj)), self.0.this_entries.clone()), - } - } - pub fn enum_fields(&self, handler: &impl Fn(&Rc, &Visibility)) { - if let Some(s) = &self.0.super_obj { - s.enum_fields(handler); - } - for (name, member) in self.0.this_entries.iter() { - handler(&name, &member.visibility); - } - } - pub fn fields_visibility(&self) -> IndexMap, bool> { - let out = Rc::new(RefCell::new(IndexMap::new())); - self.enum_fields(&|name, visibility| { - let mut out = out.borrow_mut(); - match visibility { - Visibility::Normal => { - if !out.contains_key(name) { - out.insert(name.to_owned(), true); - } - } - Visibility::Hidden => { - out.insert(name.to_owned(), false); - } - Visibility::Unhide => { - out.insert(name.to_owned(), true); - } - }; - }); - Rc::try_unwrap(out).unwrap().into_inner() - } - pub fn visible_fields(&self) -> Vec> { - self.fields_visibility() - .into_iter() - .filter(|(_k, v)| *v) - .map(|(k, _)| k) - .collect() - } - pub fn get(&self, key: Rc) -> Result> { - if let Some(v) = self.0.value_cache.borrow().get(&key) { - return Ok(Some(v.clone())); - } - if let Some(v) = self.get_raw(&key, self)? { - let v = v.unwrap_if_lazy()?; - self.0.value_cache.borrow_mut().insert(key, v.clone()); - Ok(Some(v)) - } else { - Ok(None) - } - } - pub(crate) fn get_raw(&self, key: &str, real_this: &ObjValue) -> Result> { - match (self.0.this_entries.get(key), &self.0.super_obj) { - (Some(k), None) => Ok(Some( - k.invoke - .evaluate(Some(real_this.clone()), self.0.super_obj.clone())? - .evaluate()?, - )), - (Some(k), Some(s)) => { - let lazy = k - .invoke - .evaluate(Some(real_this.clone()), self.0.super_obj.clone())?; - let our = lazy.evaluate()?; - if k.add { - s.get_raw(key, real_this)? - .map_or(Ok(Some(our.clone())), |v| { - Ok(Some(evaluate_add_op(&v, &our)?)) - }) - } else { - Ok(Some(our)) - } - } - (None, Some(s)) => s.get_raw(key, real_this), - (None, None) => Ok(None), - } - } -} -impl PartialEq for ObjValue { - fn eq(&self, other: &Self) -> bool { - Rc::ptr_eq(&self.0, &other.0) - } -} --- a/crates/jsonnet-evaluator/src/val.rs +++ /dev/null @@ -1,299 +0,0 @@ -use crate::{ - create_error_result, evaluate, - function::{parse_function_call, place_args}, - Context, Error, ObjValue, Result, -}; -use jsonnet_parser::{ArgsDesc, LocExpr, ParamsDesc}; -use std::{ - cell::RefCell, - fmt::{Debug, Display}, - rc::Rc, -}; - -enum LazyValInternals { - Computed(Val), - Waiting(Box Result>), -} -#[derive(Clone)] -pub struct LazyVal(Rc>); -impl LazyVal { - pub fn new(f: Box Result>) -> Self { - LazyVal(Rc::new(RefCell::new(LazyValInternals::Waiting(f)))) - } - pub fn new_resolved(val: Val) -> Self { - LazyVal(Rc::new(RefCell::new(LazyValInternals::Computed(val)))) - } - pub fn evaluate(&self) -> Result { - let new_value = match &*self.0.borrow() { - LazyValInternals::Computed(v) => return Ok(v.clone()), - LazyValInternals::Waiting(f) => f()?, - }; - *self.0.borrow_mut() = LazyValInternals::Computed(new_value.clone()); - Ok(new_value) - } -} - -#[macro_export] -macro_rules! lazy_val { - ($f: expr) => { - $crate::LazyVal::new(Box::new($f)) - }; -} -#[macro_export] -macro_rules! resolved_lazy_val { - ($f: expr) => { - $crate::LazyVal::new_resolved($f) - }; -} -impl Debug for LazyVal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Lazy") - } -} -impl PartialEq for LazyVal { - fn eq(&self, other: &Self) -> bool { - Rc::ptr_eq(&self.0, &other.0) - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct FuncDesc { - pub ctx: Context, - pub params: ParamsDesc, - pub body: LocExpr, -} -impl FuncDesc { - /// This function is always inlined to make tailstrict work - pub fn evaluate(&self, call_ctx: Context, args: &ArgsDesc, tailstrict: bool) -> Result { - let ctx = parse_function_call( - call_ctx, - Some(self.ctx.clone()), - &self.params, - args, - tailstrict, - )?; - evaluate(ctx, &self.body) - } - - pub fn evaluate_values(&self, call_ctx: Context, args: &[Val]) -> Result { - let ctx = place_args(call_ctx, Some(self.ctx.clone()), &self.params, args)?; - evaluate(ctx, &self.body) - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ValType { - Bool, - Null, - Str, - Num, - Arr, - Obj, - Func, -} -impl ValType { - pub fn name(&self) -> &'static str { - use ValType::*; - match self { - Bool => "boolean", - Null => "null", - Str => "string", - Num => "number", - Arr => "array", - Obj => "object", - Func => "function", - } - } -} -impl Display for ValType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.name()) - } -} - -#[derive(Debug, PartialEq, Clone)] -pub enum Val { - Bool(bool), - Null, - Str(Rc), - Num(f64), - Lazy(LazyVal), - Arr(Rc>), - Obj(ObjValue), - Func(FuncDesc), - - // Library functions implemented in native - Intristic(Rc, Rc), -} -macro_rules! matches_unwrap { - ($e: expr, $p: pat, $r: expr) => { - match $e { - $p => $r, - _ => panic!("no match"), - } - }; -} -impl Val { - pub fn assert_type(&self, context: &'static str, val_type: ValType) -> Result<()> { - let this_type = self.value_type()?; - if this_type != val_type { - create_error_result(Error::TypeMismatch(context, vec![val_type], this_type)) - } else { - Ok(()) - } - } - pub fn try_cast_bool(self, context: &'static str) -> Result { - self.assert_type(context, ValType::Bool)?; - Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Bool(v), v)) - } - pub fn try_cast_str(self, context: &'static str) -> Result> { - self.assert_type(context, ValType::Str)?; - Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Str(v), v)) - } - pub fn try_cast_num(self, context: &'static str) -> Result { - self.assert_type(context, ValType::Num)?; - Ok(matches_unwrap!(self.unwrap_if_lazy()?, Val::Num(v), v)) - } - pub fn unwrap_if_lazy(&self) -> Result { - Ok(if let Val::Lazy(v) = self { - v.evaluate()?.unwrap_if_lazy()? - } else { - self.clone() - }) - } - pub fn value_type(&self) -> Result { - Ok(match self { - Val::Str(..) => ValType::Str, - Val::Num(..) => ValType::Num, - Val::Arr(..) => ValType::Arr, - Val::Obj(..) => ValType::Obj, - Val::Func(..) => ValType::Func, - Val::Bool(_) => ValType::Bool, - Val::Null => ValType::Null, - Val::Intristic(_, _) => ValType::Func, - Val::Lazy(_) => self.clone().unwrap_if_lazy()?.value_type()?, - }) - } - #[cfg(feature = "faster")] - pub fn into_json(self, padding: usize) -> Result> { - manifest_json_ex(&self, &" ".repeat(padding)).map(|s| s.into()) - } - #[cfg(not(feature = "faster"))] - pub fn into_json(self, padding: usize) -> Result> { - with_state(|s| { - let ctx = s - .create_default_context()? - .with_var("__tmp__to_json__".into(), self)?; - Ok(evaluate( - ctx, - &el!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("manifestJsonEx".into())) - )), - ArgsDesc(vec![ - Arg(None, el!(Expr::Var("__tmp__to_json__".into()))), - Arg(None, el!(Expr::Str(" ".repeat(padding).into()))) - ]), - false - )), - )? - .try_cast_str("to json")?) - }) - } -} - -pub fn manifest_json_ex(val: &Val, padding: &str) -> Result { - let mut out = String::new(); - manifest_json_ex_buf(val, &mut out, padding, &mut String::new())?; - Ok(out) -} -fn manifest_json_ex_buf( - val: &Val, - buf: &mut String, - padding: &str, - cur_padding: &mut String, -) -> Result<()> { - use std::fmt::Write; - match val.unwrap_if_lazy()? { - Val::Bool(v) => { - if v { - buf.push_str("true"); - } else { - buf.push_str("false"); - } - } - Val::Null => buf.push_str("null"), - Val::Str(s) => buf.push_str(&escape_string_json(&s)), - Val::Num(n) => write!(buf, "{}", n).unwrap(), - Val::Arr(items) => { - buf.push_str("[\n"); - if !items.is_empty() { - let old_len = cur_padding.len(); - cur_padding.push_str(padding); - for (i, item) in items.iter().enumerate() { - if i != 0 { - buf.push_str(",\n") - } - buf.push_str(cur_padding); - manifest_json_ex_buf(item, buf, padding, cur_padding)?; - } - cur_padding.truncate(old_len); - } - buf.push('\n'); - buf.push_str(cur_padding); - buf.push(']'); - } - Val::Obj(obj) => { - buf.push_str("{\n"); - let fields = obj.visible_fields(); - if !fields.is_empty() { - let old_len = cur_padding.len(); - cur_padding.push_str(padding); - for (i, field) in fields.into_iter().enumerate() { - if i != 0 { - buf.push_str(",\n") - } - buf.push_str(cur_padding); - buf.push_str(&escape_string_json(&field)); - buf.push_str(": "); - manifest_json_ex_buf(&obj.get(field)?.unwrap(), buf, padding, cur_padding)?; - } - cur_padding.truncate(old_len); - } - buf.push('\n'); - buf.push_str(cur_padding); - buf.push('}'); - } - Val::Func(_) | Val::Intristic(_, _) => create_error_result(Error::RuntimeError("tried to manifest function".into()))?, - Val::Lazy(_) => unreachable!(), - }; - Ok(()) -} -pub fn escape_string_json(s: &str) -> String { - use std::fmt::Write; - let mut out = String::new(); - out.push('"'); - for c in s.chars() { - match c { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\u{0008}' => out.push_str("\\b"), - '\u{000c}' => out.push_str("\\f"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - c if c < 32 as char || (c >= 127 as char && c <= 159 as char) => { - write!(out, "\\u{:04x}", c as u32).unwrap() - } - c => out.push(c), - } - } - out.push('"'); - out -} - -#[test] -fn json_test() { - assert_eq!(escape_string_json("\u{001f}"), "\"\\u001f\"") -} --- a/crates/jsonnet-parser/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "jsonnet-parser" -version = "0.1.0" -authors = ["Лач "] -edition = "2018" - -[features] -default = [] -serialize = ["serde"] -deserialize = ["serde"] -# Adds ability to dump AST as source code for easy embedding -dump = ["structdump", "structdump-derive"] - -[dependencies] -peg = "0.6.2" -unescape = "0.1.0" - -serde = { version = "1.0.114", features = ["derive", "rc"], optional = true } -structdump = { version = "0.1.2", optional = true } -structdump-derive = { version = "0.1.2", optional = true } - -[dev-dependencies] -jsonnet-stdlib = { version = "0.1.0", path = "../jsonnet-stdlib" } --- a/crates/jsonnet-parser/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# jsonnet-parser - -Parser for jsonnet language --- a/crates/jsonnet-parser/src/expr.rs +++ /dev/null @@ -1,321 +0,0 @@ -use std::{fmt::Debug, ops::Deref, path::PathBuf, rc::Rc}; -#[cfg(feature = "dump")] -use structdump_derive::Codegen; -#[cfg(feature = "serialize")] -use serde::Serialize; -#[cfg(feature = "deserialize")] -use serde::Deserialize; - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub enum FieldName { - /// {fixed: 2} - Fixed(Rc), - /// {["dyn"+"amic"]: 3} - Dyn(LocExpr), -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum Visibility { - /// : - Normal, - /// :: - Hidden, - /// ::: - Unhide, -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct AssertStmt(pub LocExpr, pub Option); - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct FieldMember { - pub name: FieldName, - pub plus: bool, - pub params: Option, - pub visibility: Visibility, - pub value: LocExpr, -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub enum Member { - Field(FieldMember), - BindStmt(BindSpec), - AssertStmt(AssertStmt), -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum UnaryOpType { - Plus, - Minus, - BitNot, - Not, -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum BinaryOpType { - Mul, - Div, - - Add, - Sub, - - Lhs, - Rhs, - - Lt, - Gt, - Lte, - Gte, - - BitAnd, - BitOr, - BitXor, - - And, - Or, -} - -/// name, default value -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct Param(pub Rc, pub Option); - -/// Defined function parameters -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, Clone, PartialEq)] -pub struct ParamsDesc(pub Rc>); -impl Deref for ParamsDesc { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct Arg(pub Option, pub LocExpr); - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct ArgsDesc(pub Vec); -impl Deref for ArgsDesc { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, Clone, PartialEq)] -pub struct BindSpec { - pub name: Rc, - pub params: Option, - pub value: LocExpr, -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct IfSpecData(pub LocExpr); - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct ForSpecData(pub Rc, pub LocExpr); - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub enum CompSpec { - IfSpec(IfSpecData), - ForSpec(ForSpecData), -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub struct ObjComp { - pub pre_locals: Vec, - pub key: LocExpr, - pub value: LocExpr, - pub post_locals: Vec, - pub compspecs: Vec, -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub enum ObjBody { - MemberList(Vec), - ObjComp(ObjComp), -} - -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum LiteralType { - This, - Super, - Dollar, - Null, - True, - False, -} - -#[derive(Debug, PartialEq)] -pub struct SliceDesc { - pub start: Option, - pub end: Option, - pub step: Option, -} - -/// Syntax base -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Debug, PartialEq)] -pub enum Expr { - Literal(LiteralType), - - /// String value: "hello" - Str(Rc), - /// Number: 1, 2.0, 2e+20 - Num(f64), - /// Variable name: test - Var(Rc), - - /// Array of expressions: [1, 2, "Hello"] - Arr(Vec), - /// Array comprehension: - /// ```jsonnet - /// ingredients: [ - /// { kind: kind, qty: 4 / 3 } - /// for kind in [ - /// 'Honey Syrup', - /// 'Lemon Juice', - /// 'Farmers Gin', - /// ] - /// ], - /// ``` - ArrComp(LocExpr, Vec), - - /// Object: {a: 2} - Obj(ObjBody), - /// Object extension: var1 {b: 2} - ObjExtend(LocExpr, ObjBody), - - /// (obj) - Parened(LocExpr), - - /// -2 - UnaryOp(UnaryOpType, LocExpr), - /// 2 - 2 - BinaryOp(LocExpr, BinaryOpType, LocExpr), - /// assert 2 == 2 : "Math is broken" - AssertExpr(AssertStmt, LocExpr), - /// local a = 2; { b: a } - LocalExpr(Vec, LocExpr), - - /// import "hello" - Import(PathBuf), - /// importStr "file.txt" - ImportStr(PathBuf), - /// error "I'm broken" - Error(LocExpr), - /// a(b, c) - Apply(LocExpr, ArgsDesc, bool), - /// a[b] - Index(LocExpr, LocExpr), - /// function(x) x - Function(ParamsDesc, LocExpr), - /// if true == false then 1 else 2 - IfElse { - cond: IfSpecData, - cond_then: LocExpr, - cond_else: Option, - }, -} - -/// file, begin offset, end offset -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Clone, PartialEq)] -pub struct ExprLocation(pub Rc, pub usize, pub usize); -impl Debug for ExprLocation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}:{:?}-{:?}", self.0, self.1, self.2) - } -} - -/// Holds AST expression and its location in source file -#[cfg_attr(feature = "dump", derive(Codegen))] -#[cfg_attr(feature = "serialize", derive(Serialize))] -#[cfg_attr(feature = "deserialize", derive(Deserialize))] -#[derive(Clone, PartialEq)] -pub struct LocExpr(pub Rc, pub Option); -impl Debug for LocExpr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?} from {:?}", self.0, self.1) - } -} - -/// Creates LocExpr from Expr and ExprLocation components -#[macro_export] -macro_rules! loc_expr { - ($expr:expr, $need_loc:expr,($name:expr, $start:expr, $end:expr)) => { - LocExpr( - std::rc::Rc::new($expr), - if $need_loc { - Some(ExprLocation($name, $start, $end)) - } else { - None - }, - ) - }; -} - -/// Creates LocExpr without location info -#[macro_export] -macro_rules! loc_expr_todo { - ($expr:expr) => { - LocExpr(Rc::new($expr), None) - }; -} --- a/crates/jsonnet-parser/src/lib.rs +++ /dev/null @@ -1,578 +0,0 @@ -#![feature(box_syntax)] -#![feature(test)] - -extern crate test; - -use peg::parser; -use std::{path::PathBuf, rc::Rc}; -mod expr; -pub use expr::*; -pub use peg; - -pub struct ParserSettings { - pub loc_data: bool, - pub file_name: Rc, -} - -parser! { - grammar jsonnet_parser() for str { - use peg::ParseLiteral; - - /// Standard C-like comments - rule comment() - = "//" (!['\n'][_])* "\n" - / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/" - / "#" (!['\n'][_])* "\n" - - rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("") - rule _() = single_whitespace()* - - /// For comma-delimited elements - rule comma() = quiet!{_ "," _} / expected!("") - rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()} - rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()} - rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z'] - /// Sequence of digits - rule uint() -> u64 = a:$(digit()+) { a.parse().unwrap() } - /// Number in scientific notation format - rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("") - - /// Reserved word followed by any non-alphanumberic - rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident() - rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("") - - rule keyword(id: &'static str) - = ##parse_string_literal(id) end_of_ident() - // Adds location data information to existing expression - rule l(s: &ParserSettings, x: rule) -> LocExpr - = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))} - - pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) } - pub rule params(s: &ParserSettings) -> expr::ParamsDesc - = params:param(s) ** comma() comma()? { - let mut defaults_started = false; - for param in ¶ms { - defaults_started = defaults_started || param.1.is_some(); - assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals"); - } - expr::ParamsDesc(Rc::new(params)) - } - / { expr::ParamsDesc(Rc::new(Vec::new())) } - - pub rule arg(s: &ParserSettings) -> expr::Arg - = name:$(id()) _ "=" _ expr:expr(s) {expr::Arg(Some(name.into()), expr)} - / expr:expr(s) {expr::Arg(None, expr)} - pub rule args(s: &ParserSettings) -> expr::ArgsDesc - = args:arg(s) ** comma() comma()? { - let mut named_started = false; - for arg in &args { - named_started = named_started || arg.0.is_some(); - assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals"); - } - expr::ArgsDesc(args) - } - / { expr::ArgsDesc(Vec::new()) } - - pub rule bind(s: &ParserSettings) -> expr::BindSpec - = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}} - / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}} - pub rule assertion(s: &ParserSettings) -> expr::AssertStmt - = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) } - - pub rule whole_line() -> &'input str - = str:$((!['\n'][_])* "\n") {str} - pub rule string_block() -> String - = "|||" (!['\n']single_whitespace())* "\n" - prefix:[' ']+ first_line:whole_line() - lines:([' ']*<{prefix.len()}> s:whole_line() {s})* - [' ']*<, {prefix.len() - 1}> "|||" - {let mut l = first_line.to_owned(); l.extend(lines); l} - pub rule string() -> String - = "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()} - / "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()} - / "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")} - / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")} - / string_block() - - pub rule field_name(s: &ParserSettings) -> expr::FieldName - = name:$(id()) {expr::FieldName::Fixed(name.into())} - / name:string() {expr::FieldName::Fixed(name.into())} - / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)} - pub rule visibility() -> expr::Visibility - = ":::" {expr::Visibility::Unhide} - / "::" {expr::Visibility::Hidden} - / ":" {expr::Visibility::Normal} - pub rule field(s: &ParserSettings) -> expr::FieldMember - = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{ - name, - plus: plus.is_some(), - params: None, - visibility, - value, - }} - / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{ - name, - plus: false, - params: Some(params), - visibility, - value, - }} - pub rule obj_local(s: &ParserSettings) -> BindSpec - = keyword("local") _ bind:bind(s) {bind} - pub rule member(s: &ParserSettings) -> expr::Member - = bind:obj_local(s) {expr::Member::BindStmt(bind)} - / assertion:assertion(s) {expr::Member::AssertStmt(assertion)} - / field:field(s) {expr::Member::Field(field)} - pub rule objinside(s: &ParserSettings) -> expr::ObjBody - = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? { - let mut compspecs = vec![CompSpec::ForSpec(forspec)]; - compspecs.extend(others.unwrap_or_default()); - expr::ObjBody::ObjComp(expr::ObjComp{ - pre_locals, - key, - value, - post_locals, - compspecs, - }) - } - / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)} - pub rule ifspec(s: &ParserSettings) -> IfSpecData - = keyword("if") _ expr:expr(s) {IfSpecData(expr)} - pub rule forspec(s: &ParserSettings) -> ForSpecData - = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)} - pub rule compspec(s: &ParserSettings) -> Vec - = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s} - pub rule local_expr(s: &ParserSettings) -> LocExpr - = l(s,) - pub rule string_expr(s: &ParserSettings) -> LocExpr - = l(s, ) - pub rule obj_expr(s: &ParserSettings) -> LocExpr - = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>) - pub rule array_expr(s: &ParserSettings) -> LocExpr - = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>) - pub rule array_comp_expr(s: &ParserSettings) -> LocExpr - = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" { - let mut specs = vec![CompSpec::ForSpec(forspec)]; - specs.extend(others.unwrap_or_default()); - Expr::ArrComp(expr, specs) - }>) - pub rule number_expr(s: &ParserSettings) -> LocExpr - = l(s,) - pub rule var_expr(s: &ParserSettings) -> LocExpr - = l(s,) - pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr - = l(s,) - - pub rule literal(s: &ParserSettings) -> LocExpr - = l(s,) - - pub rule expr_basic(s: &ParserSettings) -> LocExpr - = literal(s) - - / string_expr(s) / number_expr(s) - / array_expr(s) - / obj_expr(s) - / array_expr(s) - / array_comp_expr(s) - - / l(s,) - / l(s,) - - / var_expr(s) - / local_expr(s) - / if_then_else_expr(s) - - / l(s,) - / l(s,) - - / l(s,) - - rule slice_part(s: &ParserSettings) -> Option - = e:(_ e:expr(s) _{e})? {e} - pub rule slice_desc(s: &ParserSettings) -> SliceDesc - = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? { - let (end, step) = if let Some((end, step)) = pair { - (end, step) - }else{ - (None, None) - }; - - SliceDesc { start, end, step } - } - - rule expr(s: &ParserSettings) -> LocExpr - = start:position!() a:precedence! { - a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))} - -- - a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))} - -- - a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))} - -- - a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))} - -- - a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))} - -- - a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("equals".into())) - )), - ArgsDesc(vec![Arg(None, a), Arg(None, b)]), - true - ))} - a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("equals".into())) - )), - ArgsDesc(vec![Arg(None, a), Arg(None, b)]), - true - ))))} - -- - a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))} - a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))} - a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))} - a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))} - a:(@) _ keyword("in") _ b:@ {loc_expr_todo!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("objectHasEx".into())) - )), ArgsDesc(vec![Arg(None, b), Arg(None, a), Arg(None, el!(Expr::Literal(LiteralType::True)))]), - true - ))} - -- - a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))} - a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))} - -- - a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))} - a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))} - -- - a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))} - a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))} - a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("mod".into())) - )), ArgsDesc(vec![Arg(None, a), Arg(None, b)]), - true - ))} - -- - "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))} - "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))} - "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) } - -- - a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Apply( - el!(Expr::Index( - el!(Expr::Var("std".into())), - el!(Expr::Str("slice".into())), - )), - ArgsDesc(vec![ - Arg(None, a), - Arg(None, s.start.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), - Arg(None, s.end.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), - Arg(None, s.step.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))), - ]), - true, - ))} - a:(@) _ "." _ s:$(id()) {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s.into()))))} - a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))} - a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args, ts.is_some()))} - a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))} - -- - e:expr_basic(s) {e} - "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))} - } end:position!() { - let LocExpr(e, _) = a; - LocExpr(e, if s.loc_data { - Some(ExprLocation(s.file_name.clone(), start, end)) - } else { - None - }) - } - / e:expr_basic(s) {e} - - pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e} - } -} - -pub type ParseError = peg::error::ParseError; -pub fn parse(str: &str, settings: &ParserSettings) -> Result { - jsonnet_parser::jsonnet(str, settings) -} - -#[macro_export] -macro_rules! el { - ($expr:expr) => { - LocExpr(std::rc::Rc::new($expr), None) - }; -} - -#[cfg(test)] -pub mod tests { - use super::{expr::*, parse}; - use crate::ParserSettings; - use std::path::PathBuf; - use std::rc::Rc; - - macro_rules! parse { - ($s:expr) => { - parse( - $s, - &ParserSettings { - loc_data: false, - file_name: Rc::new(PathBuf::from("/test.jsonnet")), - }, - ) - .unwrap() - }; - } - - mod expressions { - use super::*; - - pub fn basic_math() -> LocExpr { - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Add, - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Mul, - el!(Expr::Num(2.0)), - )), - )) - } - } - - #[test] - fn multiline_string() { - assert_eq!( - parse!("|||\n Hello world!\n a\n|||"), - el!(Expr::Str("Hello world!\n a\n".into())), - ) - } - - #[test] - fn slice() { - parse!("a[1:]"); - parse!("a[1::]"); - parse!("a[:1:]"); - parse!("a[::1]"); - parse!("str[:len - 1]"); - } - - #[test] - fn string_escaping() { - assert_eq!( - parse!(r#""Hello, \"world\"!""#), - el!(Expr::Str(r#"Hello, "world"!"#.into())), - ); - assert_eq!( - parse!(r#"'Hello \'world\'!'"#), - el!(Expr::Str("Hello 'world'!".into())), - ); - assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),); - } - - #[test] - fn string_unescaping() { - assert_eq!( - parse!(r#""Hello\nWorld""#), - el!(Expr::Str("Hello\nWorld".into())), - ); - } - - #[test] - fn string_verbantim() { - assert_eq!( - parse!(r#"@"Hello\n""World""""#), - el!(Expr::Str("Hello\\n\"World\"".into())), - ); - } - - #[test] - fn imports() { - assert_eq!( - parse!("import \"hello\""), - el!(Expr::Import(PathBuf::from("hello"))), - ); - assert_eq!( - parse!("importstr \"garnish.txt\""), - el!(Expr::ImportStr(PathBuf::from("garnish.txt"))) - ); - } - - #[test] - fn empty_object() { - assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![])))); - } - - #[test] - fn basic_math() { - assert_eq!( - parse!("2+2*2"), - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Add, - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Mul, - el!(Expr::Num(2.0)) - )) - )) - ); - } - - #[test] - fn basic_math_with_indents() { - assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math()); - } - - #[test] - fn basic_math_parened() { - assert_eq!( - parse!("2+(2+2*2)"), - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Add, - el!(Expr::Parened(expressions::basic_math())), - )) - ); - } - - /// Comments should not affect parsing - #[test] - fn comments() { - assert_eq!( - parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"), - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Add, - el!(Expr::BinaryOp( - el!(Expr::Num(3.0)), - BinaryOpType::Mul, - el!(Expr::Num(4.0)) - )) - )) - ); - } - - /// Comments should be able to be escaped - #[test] - fn comment_escaping() { - assert_eq!( - parse!("2/*\\*/+*/ - 22"), - el!(Expr::BinaryOp( - el!(Expr::Num(2.0)), - BinaryOpType::Sub, - el!(Expr::Num(22.0)) - )) - ); - } - - #[test] - fn suffix() { - // assert_eq!(parse!("std.test"), el!(Expr::Num(2.2))); - // assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2))); - // assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2))); - // assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2))) - } - - #[test] - fn array_comp() { - use Expr::*; - assert_eq!( - parse!("[std.deepJoin(x) for x in arr]"), - el!(ArrComp( - el!(Apply( - el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))), - ArgsDesc(vec![Arg(None, el!(Var("x".into())))]), - false, - )), - vec![CompSpec::ForSpec(ForSpecData( - "x".into(), - el!(Var("arr".into())) - ))] - )), - ) - } - - #[test] - fn reserved() { - use Expr::*; - assert_eq!(parse!("null"), el!(Literal(LiteralType::Null))); - assert_eq!(parse!("nulla"), el!(Var("nulla".into()))); - } - - #[test] - fn multiple_args_buf() { - parse!("a(b, null_fields)"); - } - - #[test] - fn infix_precedence() { - use Expr::*; - assert_eq!( - parse!("!a && !b"), - el!(BinaryOp( - el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))), - BinaryOpType::And, - el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into())))) - )) - ); - } - - #[test] - fn infix_precedence_division() { - use Expr::*; - assert_eq!( - parse!("!a / !b"), - el!(BinaryOp( - el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))), - BinaryOpType::Div, - el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into())))) - )) - ); - } - - #[test] - fn double_negation() { - use Expr::*; - assert_eq!( - parse!("!!a"), - el!(UnaryOp( - UnaryOpType::Not, - el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))) - )) - ) - } - - #[test] - fn array_test_error() { - parse!("[a for a in b if c for e in f]"); - // ^^^^ failed code - } - - #[test] - fn can_parse_stdlib() { - parse!(jsonnet_stdlib::STDLIB_STR); - } - - use test::Bencher; - - // From source code - #[bench] - fn bench_parse_peg(b: &mut Bencher) { - b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR)) - } -} --- a/crates/jsonnet-stdlib/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "jsonnet-stdlib" -version = "0.1.0" -authors = ["Лач "] -edition = "2018" - -[features] - -[dependencies] --- a/crates/jsonnet-stdlib/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# jsonnet-stdlib - -Jsonnet standard library packaged as crate --- a/crates/jsonnet-stdlib/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub const STDLIB_STR: &str = include_str!("./std.jsonnet"); --- a/crates/jsonnet-stdlib/src/std.jsonnet +++ /dev/null @@ -1,1346 +0,0 @@ -{ - __intristic_namespace__: 'std', - - local std = self, - local id = function(x) x, - - isString(v):: std.type(v) == 'string', - isNumber(v):: std.type(v) == 'number', - isBoolean(v):: std.type(v) == 'boolean', - isObject(v):: std.type(v) == 'object', - isArray(v):: std.type(v) == 'array', - isFunction(v):: std.type(v) == 'function', - - toString(a):: - if std.type(a) == 'string' then a else '' + a, - - substr(str, from, len):: - assert std.isString(str) : 'substr first parameter should be a string, got ' + std.type(str); - assert std.isNumber(from) : 'substr second parameter should be a string, got ' + std.type(from); - assert std.isNumber(len) : 'substr third parameter should be a string, got ' + std.type(len); - assert len >= 0 : 'substr third parameter should be greater than zero, got ' + len; - std.join('', std.makeArray(std.max(0, std.min(len, std.length(str) - from)), function(i) str[i + from])), - - startsWith(a, b):: - if std.length(a) < std.length(b) then - false - else - std.substr(a, 0, std.length(b)) == b, - - endsWith(a, b):: - if std.length(a) < std.length(b) then - false - else - std.substr(a, std.length(a) - std.length(b), std.length(b)) == b, - - lstripChars(str, chars):: - if std.length(str) > 0 && std.member(chars, str[0]) then - std.lstripChars(str[1:], chars) - else - str, - - rstripChars(str, chars):: - local len = std.length(str); - if len > 0 && std.member(chars, str[len - 1]) then - std.rstripChars(str[:len - 1], chars) - else - str, - - stripChars(str, chars):: - std.lstripChars(std.rstripChars(str, chars), chars), - - stringChars(str):: - std.makeArray(std.length(str), function(i) str[i]), - - local parse_nat(str, base) = - assert base > 0 && base <= 16 : 'integer base %d invalid' % base; - // These codepoints are in ascending order: - local zero_code = std.codepoint('0'); - local upper_a_code = std.codepoint('A'); - local lower_a_code = std.codepoint('a'); - local addDigit(aggregate, char) = - local code = std.codepoint(char); - local digit = if code >= lower_a_code then - code - lower_a_code + 10 - else if code >= upper_a_code then - code - upper_a_code + 10 - else - code - zero_code; - assert digit >= 0 && digit < base : '%s is not a base %d integer' % [str, base]; - base * aggregate + digit; - std.foldl(addDigit, std.stringChars(str), 0), - - parseInt(str):: - assert std.isString(str) : 'Expected string, got ' + std.type(str); - assert std.length(str) > 0 && str != '-' : 'Not an integer: "%s"' % [str]; - if str[0] == '-' then - -parse_nat(str[1:], 10) - else - parse_nat(str, 10), - - parseOctal(str):: - assert std.isString(str) : 'Expected string, got ' + std.type(str); - assert std.length(str) > 0 : 'Not an octal number: ""'; - parse_nat(str, 8), - - parseHex(str):: - assert std.isString(str) : 'Expected string, got ' + std.type(str); - assert std.length(str) > 0 : 'Not hexadecimal: ""'; - parse_nat(str, 16), - - split(str, c):: - assert std.isString(str) : 'std.split first parameter should be a string, got ' + std.type(str); - assert std.isString(c) : 'std.split second parameter should be a string, got ' + std.type(c); - assert std.length(c) == 1 : 'std.split second parameter should have length 1, got ' + std.length(c); - std.splitLimit(str, c, -1), - - splitLimit(str, c, maxsplits):: - assert std.isString(str) : 'std.splitLimit first parameter should be a string, got ' + std.type(str); - assert std.isString(c) : 'std.splitLimit second parameter should be a string, got ' + std.type(c); - assert std.length(c) == 1 : 'std.splitLimit second parameter should have length 1, got ' + std.length(c); - assert std.isNumber(maxsplits) : 'std.splitLimit third parameter should be a number, got ' + std.type(maxsplits); - local aux(str, delim, i, arr, v) = - local c = str[i]; - local i2 = i + 1; - if i >= std.length(str) then - arr + [v] - else if c == delim && (maxsplits == -1 || std.length(arr) < maxsplits) then - aux(str, delim, i2, arr + [v], '') tailstrict - else - aux(str, delim, i2, arr, v + c) tailstrict; - aux(str, c, 0, [], ''), - - strReplace(str, from, to):: - assert std.isString(str); - assert std.isString(from); - assert std.isString(to); - assert from != '' : "'from' string must not be zero length."; - - // Cache for performance. - local str_len = std.length(str); - local from_len = std.length(from); - - // True if from is at str[i]. - local found_at(i) = str[i:i + from_len] == from; - - // Return the remainder of 'str' starting with 'start_index' where - // all occurrences of 'from' after 'curr_index' are replaced with 'to'. - local replace_after(start_index, curr_index, acc) = - if curr_index > str_len then - acc + str[start_index:curr_index] - else if found_at(curr_index) then - local new_index = curr_index + std.length(from); - replace_after(new_index, new_index, acc + str[start_index:curr_index] + to) tailstrict - else - replace_after(start_index, curr_index + 1, acc) tailstrict; - - // if from_len==1, then we replace by splitting and rejoining the - // string which is much faster than recursing on replace_after - if from_len == 1 then - std.join(to, std.split(str, from)) - else - replace_after(0, 0, ''), - - asciiUpper(str):: - local cp = std.codepoint; - local up_letter(c) = if cp(c) >= 97 && cp(c) < 123 then - std.char(cp(c) - 32) - else - c; - std.join('', std.map(up_letter, std.stringChars(str))), - - asciiLower(str):: - local cp = std.codepoint; - local down_letter(c) = if cp(c) >= 65 && cp(c) < 91 then - std.char(cp(c) + 32) - else - c; - std.join('', std.map(down_letter, std.stringChars(str))), - - range(from, to):: - std.makeArray(to - from + 1, function(i) i + from), - - repeat(what, count):: - local joiner = - if std.isString(what) then '' - else if std.isArray(what) then [] - else error 'std.repeat first argument must be an array or a string'; - std.join(joiner, std.makeArray(count, function(i) what)), - - slice(indexable, index, end, step):: - local invar = - // loop invariant with defaults applied - { - indexable: indexable, - index: - if index == null then 0 - else index, - end: - if end == null then std.length(indexable) - else end, - step: - if step == null then 1 - else step, - length: std.length(indexable), - type: std.type(indexable), - }; - assert invar.index >= 0 && invar.end >= 0 && invar.step >= 0 : 'got [%s:%s:%s] but negative index, end, and steps are not supported' % [invar.index, invar.end, invar.step]; - assert step != 0 : 'got %s but step must be greater than 0' % step; - assert std.isString(indexable) || std.isArray(indexable) : 'std.slice accepts a string or an array, but got: %s' % std.type(indexable); - local build(slice, cur) = - if cur >= invar.end || cur >= invar.length then - slice - else - build( - if invar.type == 'string' then - slice + invar.indexable[cur] - else - slice + [invar.indexable[cur]], - cur + invar.step - ) tailstrict; - build(if invar.type == 'string' then '' else [], invar.index), - - member(arr, x):: - if std.isArray(arr) then - std.count(arr, x) > 0 - else if std.isString(arr) then - std.length(std.findSubstr(x, arr)) > 0 - else error 'std.member first argument must be an array or a string', - - count(arr, x):: std.length(std.filter(function(v) v == x, arr)), - - mod(a, b):: - if std.isNumber(a) && std.isNumber(b) then - std.modulo(a, b) - else if std.isString(a) then - std.format(a, b) - else - error 'Operator % cannot be used on types ' + std.type(a) + ' and ' + std.type(b) + '.', - - map(func, arr):: - if !std.isFunction(func) then - error ('std.map first param must be function, got ' + std.type(func)) - else if !std.isArray(arr) && !std.isString(arr) then - error ('std.map second param must be array / string, got ' + std.type(arr)) - else - std.makeArray(std.length(arr), function(i) func(arr[i])), - - mapWithIndex(func, arr):: - if !std.isFunction(func) then - error ('std.mapWithIndex first param must be function, got ' + std.type(func)) - else if !std.isArray(arr) && !std.isString(arr) then - error ('std.mapWithIndex second param must be array, got ' + std.type(arr)) - else - std.makeArray(std.length(arr), function(i) func(i, arr[i])), - - mapWithKey(func, obj):: - if !std.isFunction(func) then - error ('std.mapWithKey first param must be function, got ' + std.type(func)) - else if !std.isObject(obj) then - error ('std.mapWithKey second param must be object, got ' + std.type(obj)) - else - { [k]: func(k, obj[k]) for k in std.objectFields(obj) }, - - flatMap(func, arr):: - if !std.isFunction(func) then - error ('std.flatMap first param must be function, got ' + std.type(func)) - else if std.isArray(arr) then - std.flattenArrays(std.makeArray(std.length(arr), function(i) func(arr[i]))) - else if std.isString(arr) then - std.join('', std.makeArray(std.length(arr), function(i) func(arr[i]))) - else error ('std.flatMap second param must be array / string, got ' + std.type(arr)), - - join(sep, arr):: - local aux(arr, i, first, running) = - if i >= std.length(arr) then - running - else if arr[i] == null then - aux(arr, i + 1, first, running) tailstrict - else if std.type(arr[i]) != std.type(sep) then - error 'expected %s but arr[%d] was %s ' % [std.type(sep), i, std.type(arr[i])] - else if first then - aux(arr, i + 1, false, running + arr[i]) tailstrict - else - aux(arr, i + 1, false, running + sep + arr[i]) tailstrict; - if !std.isArray(arr) then - error 'join second parameter should be array, got ' + std.type(arr) - else if std.isString(sep) then - aux(arr, 0, true, '') - else if std.isArray(sep) then - aux(arr, 0, true, []) - else - error 'join first parameter should be string or array, got ' + std.type(sep), - - lines(arr):: - std.join('\n', arr + ['']), - - deepJoin(arr):: - if std.isString(arr) then - arr - else if std.isArray(arr) then - std.join('', [std.deepJoin(x) for x in arr]) - else - error 'Expected string or array, got %s' % std.type(arr), - - - format(str, vals):: - - ///////////////////////////// - // Parse the mini-language // - ///////////////////////////// - - local try_parse_mapping_key(str, i) = - assert i < std.length(str) : 'Truncated format code.'; - local c = str[i]; - if c == '(' then - local consume(str, j, v) = - if j >= std.length(str) then - error 'Truncated format code.' - else - local c = str[j]; - if c != ')' then - consume(str, j + 1, v + c) - else - { i: j + 1, v: v }; - consume(str, i + 1, '') - else - { i: i, v: null }; - - local try_parse_cflags(str, i) = - local consume(str, j, v) = - assert j < std.length(str) : 'Truncated format code.'; - local c = str[j]; - if c == '#' then - consume(str, j + 1, v { alt: true }) - else if c == '0' then - consume(str, j + 1, v { zero: true }) - else if c == '-' then - consume(str, j + 1, v { left: true }) - else if c == ' ' then - consume(str, j + 1, v { blank: true }) - else if c == '+' then - consume(str, j + 1, v { sign: true }) - else - { i: j, v: v }; - consume(str, i, { alt: false, zero: false, left: false, blank: false, sign: false }); - - local try_parse_field_width(str, i) = - if i < std.length(str) && str[i] == '*' then - { i: i + 1, v: '*' } - else - local consume(str, j, v) = - assert j < std.length(str) : 'Truncated format code.'; - local c = str[j]; - if c == '0' then - consume(str, j + 1, v * 10 + 0) - else if c == '1' then - consume(str, j + 1, v * 10 + 1) - else if c == '2' then - consume(str, j + 1, v * 10 + 2) - else if c == '3' then - consume(str, j + 1, v * 10 + 3) - else if c == '4' then - consume(str, j + 1, v * 10 + 4) - else if c == '5' then - consume(str, j + 1, v * 10 + 5) - else if c == '6' then - consume(str, j + 1, v * 10 + 6) - else if c == '7' then - consume(str, j + 1, v * 10 + 7) - else if c == '8' then - consume(str, j + 1, v * 10 + 8) - else if c == '9' then - consume(str, j + 1, v * 10 + 9) - else - { i: j, v: v }; - consume(str, i, 0); - - local try_parse_precision(str, i) = - assert i < std.length(str) : 'Truncated format code.'; - local c = str[i]; - if c == '.' then - try_parse_field_width(str, i + 1) - else - { i: i, v: null }; - - // Ignored, if it exists. - local try_parse_length_modifier(str, i) = - assert i < std.length(str) : 'Truncated format code.'; - local c = str[i]; - if c == 'h' || c == 'l' || c == 'L' then - i + 1 - else - i; - - local parse_conv_type(str, i) = - assert i < std.length(str) : 'Truncated format code.'; - local c = str[i]; - if c == 'd' || c == 'i' || c == 'u' then - { i: i + 1, v: 'd', caps: false } - else if c == 'o' then - { i: i + 1, v: 'o', caps: false } - else if c == 'x' then - { i: i + 1, v: 'x', caps: false } - else if c == 'X' then - { i: i + 1, v: 'x', caps: true } - else if c == 'e' then - { i: i + 1, v: 'e', caps: false } - else if c == 'E' then - { i: i + 1, v: 'e', caps: true } - else if c == 'f' then - { i: i + 1, v: 'f', caps: false } - else if c == 'F' then - { i: i + 1, v: 'f', caps: true } - else if c == 'g' then - { i: i + 1, v: 'g', caps: false } - else if c == 'G' then - { i: i + 1, v: 'g', caps: true } - else if c == 'c' then - { i: i + 1, v: 'c', caps: false } - else if c == 's' then - { i: i + 1, v: 's', caps: false } - else if c == '%' then - { i: i + 1, v: '%', caps: false } - else - error 'Unrecognised conversion type: ' + c; - - - // Parsed initial %, now the rest. - local parse_code(str, i) = - assert i < std.length(str) : 'Truncated format code.'; - local mkey = try_parse_mapping_key(str, i); - local cflags = try_parse_cflags(str, mkey.i); - local fw = try_parse_field_width(str, cflags.i); - local prec = try_parse_precision(str, fw.i); - local len_mod = try_parse_length_modifier(str, prec.i); - local ctype = parse_conv_type(str, len_mod); - { - i: ctype.i, - code: { - mkey: mkey.v, - cflags: cflags.v, - fw: fw.v, - prec: prec.v, - ctype: ctype.v, - caps: ctype.caps, - }, - }; - - // Parse a format string (containing none or more % format tags). - local parse_codes(str, i, out, cur) = - if i >= std.length(str) then - out + [cur] - else - local c = str[i]; - if c == '%' then - local r = parse_code(str, i + 1); - parse_codes(str, r.i, out + [cur, r.code], '') tailstrict - else - parse_codes(str, i + 1, out, cur + c) tailstrict; - - local codes = parse_codes(str, 0, [], ''); - - - /////////////////////// - // Format the values // - /////////////////////// - - // Useful utilities - local padding(w, s) = - local aux(w, v) = - if w <= 0 then - v - else - aux(w - 1, v + s); - aux(w, ''); - - // Add s to the left of str so that its length is at least w. - local pad_left(str, w, s) = - padding(w - std.length(str), s) + str; - - // Add s to the right of str so that its length is at least w. - local pad_right(str, w, s) = - str + padding(w - std.length(str), s); - - // Render an integer (e.g., decimal or octal). - local render_int(n__, min_chars, min_digits, blank, sign, radix, zero_prefix) = - local n_ = std.abs(n__); - local aux(n) = - if n == 0 then - zero_prefix - else - aux(std.floor(n / radix)) + (n % radix); - local dec = if std.floor(n_) == 0 then '0' else aux(std.floor(n_)); - local neg = n__ < 0; - local zp = min_chars - (if neg || blank || sign then 1 else 0); - local zp2 = std.max(zp, min_digits); - local dec2 = pad_left(dec, zp2, '0'); - (if neg then '-' else if sign then '+' else if blank then ' ' else '') + dec2; - - // Render an integer in hexadecimal. - local render_hex(n__, min_chars, min_digits, blank, sign, add_zerox, capitals) = - local numerals = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - + if capitals then ['A', 'B', 'C', 'D', 'E', 'F'] - else ['a', 'b', 'c', 'd', 'e', 'f']; - local n_ = std.abs(n__); - local aux(n) = - if n == 0 then - '' - else - aux(std.floor(n / 16)) + numerals[n % 16]; - local hex = if std.floor(n_) == 0 then '0' else aux(std.floor(n_)); - local neg = n__ < 0; - local zp = min_chars - (if neg || blank || sign then 1 else 0) - - (if add_zerox then 2 else 0); - local zp2 = std.max(zp, min_digits); - local hex2 = (if add_zerox then (if capitals then '0X' else '0x') else '') - + pad_left(hex, zp2, '0'); - (if neg then '-' else if sign then '+' else if blank then ' ' else '') + hex2; - - local strip_trailing_zero(str) = - local aux(str, i) = - if i < 0 then - '' - else - if str[i] == '0' then - aux(str, i - 1) - else - std.substr(str, 0, i + 1); - aux(str, std.length(str) - 1); - - // Render floating point in decimal form - local render_float_dec(n__, zero_pad, blank, sign, ensure_pt, trailing, prec) = - local n_ = std.abs(n__); - local whole = std.floor(n_); - local dot_size = if prec == 0 && !ensure_pt then 0 else 1; - local zp = zero_pad - prec - dot_size; - local str = render_int(std.sign(n__) * whole, zp, 0, blank, sign, 10, ''); - if prec == 0 then - str + if ensure_pt then '.' else '' - else - local frac = std.floor((n_ - whole) * std.pow(10, prec) + 0.5); - if trailing || frac > 0 then - local frac_str = render_int(frac, prec, 0, false, false, 10, ''); - str + '.' + if !trailing then strip_trailing_zero(frac_str) else frac_str - else - str; - - // Render floating point in scientific form - local render_float_sci(n__, zero_pad, blank, sign, ensure_pt, trailing, caps, prec) = - local exponent = if n__ == 0 then 0 else std.floor(std.log(std.abs(n__)) / std.log(10)); - local suff = (if caps then 'E' else 'e') - + render_int(exponent, 3, 0, false, true, 10, ''); - local mantissa = if exponent == -324 then - // Avoid a rounding error where std.pow(10, -324) is 0 - // -324 is the smallest exponent possible. - n__ * 10 / std.pow(10, exponent + 1) - else - n__ / std.pow(10, exponent); - local zp2 = zero_pad - std.length(suff); - render_float_dec(mantissa, zp2, blank, sign, ensure_pt, trailing, prec) + suff; - - // Render a value with an arbitrary format code. - local format_code(val, code, fw, prec_or_null, i) = - local cflags = code.cflags; - local fpprec = if prec_or_null != null then prec_or_null else 6; - local iprec = if prec_or_null != null then prec_or_null else 0; - local zp = if cflags.zero && !cflags.left then fw else 0; - if code.ctype == 's' then - std.toString(val) - else if code.ctype == 'd' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - render_int(val, zp, iprec, cflags.blank, cflags.sign, 10, '') - else if code.ctype == 'o' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - local zero_prefix = if cflags.alt then '0' else ''; - render_int(val, zp, iprec, cflags.blank, cflags.sign, 8, zero_prefix) - else if code.ctype == 'x' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - render_hex(val, - zp, - iprec, - cflags.blank, - cflags.sign, - cflags.alt, - code.caps) - else if code.ctype == 'f' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - render_float_dec(val, - zp, - cflags.blank, - cflags.sign, - cflags.alt, - true, - fpprec) - else if code.ctype == 'e' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - render_float_sci(val, - zp, - cflags.blank, - cflags.sign, - cflags.alt, - true, - code.caps, - fpprec) - else if code.ctype == 'g' then - if std.type(val) != 'number' then - error 'Format required number at ' - + i + ', got ' + std.type(val) - else - local exponent = std.floor(std.log(std.abs(val)) / std.log(10)); - if exponent < -4 || exponent >= fpprec then - render_float_sci(val, - zp, - cflags.blank, - cflags.sign, - cflags.alt, - cflags.alt, - code.caps, - fpprec - 1) - else - local digits_before_pt = std.max(1, exponent + 1); - render_float_dec(val, - zp, - cflags.blank, - cflags.sign, - cflags.alt, - cflags.alt, - fpprec - digits_before_pt) - else if code.ctype == 'c' then - if std.type(val) == 'number' then - std.char(val) - else if std.type(val) == 'string' then - if std.length(val) == 1 then - val - else - error '%c expected 1-sized string got: ' + std.length(val) - else - error '%c expected number / string, got: ' + std.type(val) - else - error 'Unknown code: ' + code.ctype; - - // Render a parsed format string with an array of values. - local format_codes_arr(codes, arr, i, j, v) = - if i >= std.length(codes) then - if j < std.length(arr) then - error ('Too many values to format: ' + std.length(arr) + ', expected ' + j) - else - v - else - local code = codes[i]; - if std.type(code) == 'string' then - format_codes_arr(codes, arr, i + 1, j, v + code) tailstrict - else - local tmp = if code.fw == '*' then { - j: j + 1, - fw: if j >= std.length(arr) then - error ('Not enough values to format: ' + std.length(arr) + ', expected at least ' + j) - else - arr[j], - } else { - j: j, - fw: code.fw, - }; - local tmp2 = if code.prec == '*' then { - j: tmp.j + 1, - prec: if tmp.j >= std.length(arr) then - error ('Not enough values to format: ' + std.length(arr) + ', expected at least ' + tmp.j) - else - arr[tmp.j], - } else { - j: tmp.j, - prec: code.prec, - }; - local j2 = tmp2.j; - local val = - if j2 < std.length(arr) then - arr[j2] - else - error ('Not enough values to format: ' + std.length(arr) + ', expected more than ' + j2); - local s = - if code.ctype == '%' then - '%' - else - format_code(val, code, tmp.fw, tmp2.prec, j2); - local s_padded = - if code.cflags.left then - pad_right(s, tmp.fw, ' ') - else - pad_left(s, tmp.fw, ' '); - local j3 = - if code.ctype == '%' then - j2 - else - j2 + 1; - format_codes_arr(codes, arr, i + 1, j3, v + s_padded) tailstrict; - - // Render a parsed format string with an object of values. - local format_codes_obj(codes, obj, i, v) = - if i >= std.length(codes) then - v - else - local code = codes[i]; - if std.type(code) == 'string' then - format_codes_obj(codes, obj, i + 1, v + code) tailstrict - else - local f = - if code.mkey == null then - error 'Mapping keys required.' - else - code.mkey; - local fw = - if code.fw == '*' then - error 'Cannot use * field width with object.' - else - code.fw; - local prec = - if code.prec == '*' then - error 'Cannot use * precision with object.' - else - code.prec; - local val = - if std.objectHasAll(obj, f) then - obj[f] - else - error 'No such field: ' + f; - local s = - if code.ctype == '%' then - '%' - else - format_code(val, code, fw, prec, f); - local s_padded = - if code.cflags.left then - pad_right(s, fw, ' ') - else - pad_left(s, fw, ' '); - format_codes_obj(codes, obj, i + 1, v + s_padded) tailstrict; - - if std.isArray(vals) then - format_codes_arr(codes, vals, 0, 0, '') - else if std.isObject(vals) then - format_codes_obj(codes, vals, 0, '') - else - format_codes_arr(codes, [vals], 0, 0, ''), - - foldr(func, arr, init):: - local aux(func, arr, running, idx) = - if idx < 0 then - running - else - aux(func, arr, func(arr[idx], running), idx - 1) tailstrict; - aux(func, arr, init, std.length(arr) - 1), - - foldl(func, arr, init):: - local aux(func, arr, running, idx) = - if idx >= std.length(arr) then - running - else - aux(func, arr, func(running, arr[idx]), idx + 1) tailstrict; - aux(func, arr, init, 0), - - - filterMap(filter_func, map_func, arr):: - if !std.isFunction(filter_func) then - error ('std.filterMap first param must be function, got ' + std.type(filter_func)) - else if !std.isFunction(map_func) then - error ('std.filterMap second param must be function, got ' + std.type(map_func)) - else if !std.isArray(arr) then - error ('std.filterMap third param must be array, got ' + std.type(arr)) - else - std.map(map_func, std.filter(filter_func, arr)), - - assertEqual(a, b):: - if a == b then - true - else - error 'Assertion failed. ' + a + ' != ' + b, - - abs(n):: - if !std.isNumber(n) then - error 'std.abs expected number, got ' + std.type(n) - else - if n > 0 then n else -n, - - sign(n):: - if !std.isNumber(n) then - error 'std.sign expected number, got ' + std.type(n) - else - if n > 0 then - 1 - else if n < 0 then - -1 - else 0, - - max(a, b):: - if !std.isNumber(a) then - error 'std.max first param expected number, got ' + std.type(a) - else if !std.isNumber(b) then - error 'std.max second param expected number, got ' + std.type(b) - else - if a > b then a else b, - - min(a, b):: - if !std.isNumber(a) then - error 'std.max first param expected number, got ' + std.type(a) - else if !std.isNumber(b) then - error 'std.max second param expected number, got ' + std.type(b) - else - if a < b then a else b, - - clamp(x, minVal, maxVal):: - if x < minVal then minVal - else if x > maxVal then maxVal - else x, - - flattenArrays(arrs):: - std.foldl(function(a, b) a + b, arrs, []), - - manifestIni(ini):: - local body_lines(body) = - std.join([], [ - local value_or_values = body[k]; - if std.isArray(value_or_values) then - ['%s = %s' % [k, value] for value in value_or_values] - else - ['%s = %s' % [k, value_or_values]] - - for k in std.objectFields(body) - ]); - - local section_lines(sname, sbody) = ['[%s]' % [sname]] + body_lines(sbody), - main_body = if std.objectHas(ini, 'main') then body_lines(ini.main) else [], - all_sections = [ - section_lines(k, ini.sections[k]) - for k in std.objectFields(ini.sections) - ]; - std.join('\n', main_body + std.flattenArrays(all_sections) + ['']), - - escapeStringJson(str_):: - local str = std.toString(str_); - local trans(ch) = - if ch == '"' then - '\\"' - else if ch == '\\' then - '\\\\' - else if ch == '\b' then - '\\b' - else if ch == '\f' then - '\\f' - else if ch == '\n' then - '\\n' - else if ch == '\r' then - '\\r' - else if ch == '\t' then - '\\t' - else - local cp = std.codepoint(ch); - if cp < 32 || (cp >= 127 && cp <= 159) then - '\\u%04x' % [cp] - else - ch; - '"%s"' % std.join('', [trans(ch) for ch in std.stringChars(str)]), - - escapeStringPython(str):: - std.escapeStringJson(str), - - escapeStringBash(str_):: - local str = std.toString(str_); - local trans(ch) = - if ch == "'" then - "'\"'\"'" - else - ch; - "'%s'" % std.join('', [trans(ch) for ch in std.stringChars(str)]), - - escapeStringDollars(str_):: - local str = std.toString(str_); - local trans(ch) = - if ch == '$' then - '$$' - else - ch; - std.foldl(function(a, b) a + trans(b), std.stringChars(str), ''), - - manifestJson(value):: std.manifestJsonEx(value, ' '), - - manifestJsonEx(value, indent):: - local aux(v, path, cindent) = - if v == true then - 'true' - else if v == false then - 'false' - else if v == null then - 'null' - else if std.isNumber(v) then - '' + v - else if std.isString(v) then - std.escapeStringJson(v) - else if std.isFunction(v) then - error 'Tried to manifest function at ' + path - else if std.isArray(v) then - local range = std.range(0, std.length(v) - 1); - local new_indent = cindent + indent; - local lines = ['[\n'] - + std.join([',\n'], - [ - [new_indent + aux(v[i], path + [i], new_indent)] - for i in range - ]) - + ['\n' + cindent + ']']; - std.join('', lines) - else if std.isObject(v) then - local lines = ['{\n'] - + std.join([',\n'], - [ - [cindent + indent + std.escapeStringJson(k) + ': ' - + aux(v[k], path + [k], cindent + indent)] - for k in std.objectFields(v) - ]) - + ['\n' + cindent + '}']; - std.join('', lines); - aux(value, [], ''), - - manifestYamlDoc(value, indent_array_in_object=false):: - local aux(v, path, cindent) = - if v == true then - 'true' - else if v == false then - 'false' - else if v == null then - 'null' - else if std.isNumber(v) then - '' + v - else if std.isString(v) then - local len = std.length(v); - if len == 0 then - '""' - else if v[len - 1] == '\n' then - local split = std.split(v, '\n'); - std.join('\n' + cindent + ' ', ['|'] + split[0:std.length(split) - 1]) - else - std.escapeStringJson(v) - else if std.isFunction(v) then - error 'Tried to manifest function at ' + path - else if std.isArray(v) then - if std.length(v) == 0 then - '[]' - else - local params(value) = - if std.isArray(value) && std.length(value) > 0 then { - // While we could avoid the new line, it yields YAML that is - // hard to read, e.g.: - // - - - 1 - // - 2 - // - - 3 - // - 4 - new_indent: cindent + ' ', - space: '\n' + self.new_indent, - } else if std.isObject(value) && std.length(value) > 0 then { - new_indent: cindent + ' ', - // In this case we can start on the same line as the - because the indentation - // matches up then. The converse is not true, because fields are not always - // 1 character long. - space: ' ', - } else { - // In this case, new_indent is only used in the case of multi-line strings. - new_indent: cindent, - space: ' ', - }; - local range = std.range(0, std.length(v) - 1); - local parts = [ - '-' + param.space + aux(v[i], path + [i], param.new_indent) - for i in range - for param in [params(v[i])] - ]; - std.join('\n' + cindent, parts) - else if std.isObject(v) then - if std.length(v) == 0 then - '{}' - else - local params(value) = - if std.isArray(value) && std.length(value) > 0 then { - // Not indenting allows e.g. - // ports: - // - 80 - // instead of - // ports: - // - 80 - new_indent: if indent_array_in_object then cindent + ' ' else cindent, - space: '\n' + self.new_indent, - } else if std.isObject(value) && std.length(value) > 0 then { - new_indent: cindent + ' ', - space: '\n' + self.new_indent, - } else { - // In this case, new_indent is only used in the case of multi-line strings. - new_indent: cindent, - space: ' ', - }; - local lines = [ - std.escapeStringJson(k) + ':' + param.space + aux(v[k], path + [k], param.new_indent) - for k in std.objectFields(v) - for param in [params(v[k])] - ]; - std.join('\n' + cindent, lines); - aux(value, [], ''), - - manifestYamlStream(value, indent_array_in_object=false, c_document_end=true):: - if !std.isArray(value) then - error 'manifestYamlStream only takes arrays, got ' + std.type(value) - else - '---\n' + std.join( - '\n---\n', [std.manifestYamlDoc(e, indent_array_in_object) for e in value] - ) + if c_document_end then '\n...\n' else '\n', - - - manifestPython(v):: - if std.isObject(v) then - local fields = [ - '%s: %s' % [std.escapeStringPython(k), std.manifestPython(v[k])] - for k in std.objectFields(v) - ]; - '{%s}' % [std.join(', ', fields)] - else if std.isArray(v) then - '[%s]' % [std.join(', ', [std.manifestPython(v2) for v2 in v])] - else if std.isString(v) then - '%s' % [std.escapeStringPython(v)] - else if std.isFunction(v) then - error 'cannot manifest function' - else if std.isNumber(v) then - std.toString(v) - else if v == true then - 'True' - else if v == false then - 'False' - else if v == null then - 'None', - - manifestPythonVars(conf):: - local vars = ['%s = %s' % [k, std.manifestPython(conf[k])] for k in std.objectFields(conf)]; - std.join('\n', vars + ['']), - - manifestXmlJsonml(value):: - if !std.isArray(value) then - error 'Expected a JSONML value (an array), got %s' % std.type(value) - else - local aux(v) = - if std.isString(v) then - v - else - local tag = v[0]; - local has_attrs = std.length(v) > 1 && std.isObject(v[1]); - local attrs = if has_attrs then v[1] else {}; - local children = if has_attrs then v[2:] else v[1:]; - local attrs_str = - std.join('', [' %s="%s"' % [k, attrs[k]] for k in std.objectFields(attrs)]); - std.deepJoin(['<', tag, attrs_str, '>', [aux(x) for x in children], '']); - - aux(value), - - local base64_table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', - local base64_inv = { [base64_table[i]]: i for i in std.range(0, 63) }, - - base64(input):: - local bytes = - if std.isString(input) then - std.map(function(c) std.codepoint(c), input) - else - input; - - local aux(arr, i, r) = - if i >= std.length(arr) then - r - else if i + 1 >= std.length(arr) then - local str = - // 6 MSB of i - base64_table[(arr[i] & 252) >> 2] + - // 2 LSB of i - base64_table[(arr[i] & 3) << 4] + - '=='; - aux(arr, i + 3, r + str) tailstrict - else if i + 2 >= std.length(arr) then - local str = - // 6 MSB of i - base64_table[(arr[i] & 252) >> 2] + - // 2 LSB of i, 4 MSB of i+1 - base64_table[(arr[i] & 3) << 4 | (arr[i + 1] & 240) >> 4] + - // 4 LSB of i+1 - base64_table[(arr[i + 1] & 15) << 2] + - '='; - aux(arr, i + 3, r + str) tailstrict - else - local str = - // 6 MSB of i - base64_table[(arr[i] & 252) >> 2] + - // 2 LSB of i, 4 MSB of i+1 - base64_table[(arr[i] & 3) << 4 | (arr[i + 1] & 240) >> 4] + - // 4 LSB of i+1, 2 MSB of i+2 - base64_table[(arr[i + 1] & 15) << 2 | (arr[i + 2] & 192) >> 6] + - // 6 LSB of i+2 - base64_table[(arr[i + 2] & 63)]; - aux(arr, i + 3, r + str) tailstrict; - - local sanity = std.foldl(function(r, a) r && (a < 256), bytes, true); - if !sanity then - error 'Can only base64 encode strings / arrays of single bytes.' - else - aux(bytes, 0, ''), - - - base64DecodeBytes(str):: - if std.length(str) % 4 != 0 then - error 'Not a base64 encoded string "%s"' % str - else - local aux(str, i, r) = - if i >= std.length(str) then - r - else - // all 6 bits of i, 2 MSB of i+1 - local n1 = [base64_inv[str[i]] << 2 | (base64_inv[str[i + 1]] >> 4)]; - // 4 LSB of i+1, 4MSB of i+2 - local n2 = - if str[i + 2] == '=' then [] - else [(base64_inv[str[i + 1]] & 15) << 4 | (base64_inv[str[i + 2]] >> 2)]; - // 2 LSB of i+2, all 6 bits of i+3 - local n3 = - if str[i + 3] == '=' then [] - else [(base64_inv[str[i + 2]] & 3) << 6 | base64_inv[str[i + 3]]]; - aux(str, i + 4, r + n1 + n2 + n3) tailstrict; - aux(str, 0, []), - - base64Decode(str):: - local bytes = std.base64DecodeBytes(str); - std.join('', std.map(function(b) std.char(b), bytes)), - - reverse(arr):: - local l = std.length(arr); - std.makeArray(l, function(i) arr[l - i - 1]), - - // Merge-sort for long arrays and naive quicksort for shorter ones - sort(arr, keyF=id):: - local quickSort(arr, keyF=id) = - local l = std.length(arr); - if std.length(arr) <= 1 then - arr - else - local pos = 0; - local pivot = keyF(arr[pos]); - local rest = std.makeArray(l - 1, function(i) if i < pos then arr[i] else arr[i + 1]); - local left = std.filter(function(x) keyF(x) < pivot, rest); - local right = std.filter(function(x) keyF(x) >= pivot, rest); - quickSort(left, keyF) + [arr[pos]] + quickSort(right, keyF); - - local merge(a, b) = - local la = std.length(a), lb = std.length(b); - local aux(i, j, prefix) = - if i == la then - prefix + b[j:] - else if j == lb then - prefix + a[i:] - else - if keyF(a[i]) <= keyF(b[j]) then - aux(i + 1, j, prefix + [a[i]]) tailstrict - else - aux(i, j + 1, prefix + [b[j]]) tailstrict; - aux(0, 0, []); - - local l = std.length(arr); - if std.length(arr) <= 30 then - quickSort(arr, keyF=keyF) - else - local mid = std.floor(l / 2); - local left = arr[:mid], right = arr[mid:]; - merge(std.sort(left, keyF=keyF), std.sort(right, keyF=keyF)), - - uniq(arr, keyF=id):: - local f(a, b) = - if std.length(a) == 0 then - [b] - else if keyF(a[std.length(a) - 1]) == keyF(b) then - a - else - a + [b]; - std.foldl(f, arr, []), - - set(arr, keyF=id):: - std.uniq(std.sort(arr, keyF), keyF), - - setMember(x, arr, keyF=id):: - // TODO(dcunnin): Binary chop for O(log n) complexity - std.length(std.setInter([x], arr, keyF)) > 0, - - setUnion(a, b, keyF=id):: - // NOTE: order matters, values in `a` win - local aux(a, b, i, j, acc) = - if i >= std.length(a) then - acc + b[j:] - else if j >= std.length(b) then - acc + a[i:] - else - local ak = keyF(a[i]); - local bk = keyF(b[j]); - if ak == bk then - aux(a, b, i + 1, j + 1, acc + [a[i]]) tailstrict - else if ak < bk then - aux(a, b, i + 1, j, acc + [a[i]]) tailstrict - else - aux(a, b, i, j + 1, acc + [b[j]]) tailstrict; - aux(a, b, 0, 0, []), - - setInter(a, b, keyF=id):: - local aux(a, b, i, j, acc) = - if i >= std.length(a) || j >= std.length(b) then - acc - else - if keyF(a[i]) == keyF(b[j]) then - aux(a, b, i + 1, j + 1, acc + [a[i]]) tailstrict - else if keyF(a[i]) < keyF(b[j]) then - aux(a, b, i + 1, j, acc) tailstrict - else - aux(a, b, i, j + 1, acc) tailstrict; - aux(a, b, 0, 0, []) tailstrict, - - setDiff(a, b, keyF=id):: - local aux(a, b, i, j, acc) = - if i >= std.length(a) then - acc - else if j >= std.length(b) then - acc + a[i:] - else - if keyF(a[i]) == keyF(b[j]) then - aux(a, b, i + 1, j + 1, acc) tailstrict - else if keyF(a[i]) < keyF(b[j]) then - aux(a, b, i + 1, j, acc + [a[i]]) tailstrict - else - aux(a, b, i, j + 1, acc) tailstrict; - aux(a, b, 0, 0, []) tailstrict, - - mergePatch(target, patch):: - if std.isObject(patch) then - local target_object = - if std.isObject(target) then target else {}; - - local target_fields = - if std.isObject(target_object) then std.objectFields(target_object) else []; - - local null_fields = [k for k in std.objectFields(patch) if patch[k] == null]; - local both_fields = std.setUnion(target_fields, std.objectFields(patch)); - - { - [k]: - if !std.objectHas(patch, k) then - target_object[k] - else if !std.objectHas(target_object, k) then - std.mergePatch(null, patch[k]) tailstrict - else - std.mergePatch(target_object[k], patch[k]) tailstrict - for k in std.setDiff(both_fields, null_fields) - } - else - patch, - - objectFields(o):: - std.objectFieldsEx(o, false), - - objectFieldsAll(o):: - std.objectFieldsEx(o, true), - - objectHas(o, f):: - std.objectHasEx(o, f, false), - - objectHasAll(o, f):: - std.objectHasEx(o, f, true), - - equals(a, b):: - local ta = std.type(a); - local tb = std.type(b); - if !std.primitiveEquals(ta, tb) then - false - else - if std.primitiveEquals(ta, 'array') then - local la = std.length(a); - if !std.primitiveEquals(la, std.length(b)) then - false - else - local aux(a, b, i) = - if i >= la then - true - else if a[i] != b[i] then - false - else - aux(a, b, i + 1) tailstrict; - aux(a, b, 0) - else if std.primitiveEquals(ta, 'object') then - local fields = std.objectFields(a); - local lfields = std.length(fields); - if fields != std.objectFields(b) then - false - else - local aux(a, b, i) = - if i >= lfields then - true - else if local f = fields[i]; a[f] != b[f] then - false - else - aux(a, b, i + 1) tailstrict; - aux(a, b, 0) - else - std.primitiveEquals(a, b), - - - resolvePath(f, r):: - local arr = std.split(f, '/'); - std.join('/', std.makeArray(std.length(arr) - 1, function(i) arr[i]) + [r]), - - prune(a):: - local isContent(b) = - if b == null then - false - else if std.isArray(b) then - std.length(b) > 0 - else if std.isObject(b) then - std.length(b) > 0 - else - true; - if std.isArray(a) then - [std.prune(x) for x in a if isContent($.prune(x))] - else if std.isObject(a) then { - [x]: $.prune(a[x]) - for x in std.objectFields(a) - if isContent(std.prune(a[x])) - } else - a, - - findSubstr(pat, str):: - if !std.isString(pat) then - error 'findSubstr first parameter should be a string, got ' + std.type(pat) - else if !std.isString(str) then - error 'findSubstr second parameter should be a string, got ' + std.type(str) - else - local pat_len = std.length(pat); - local str_len = std.length(str); - if pat_len == 0 || str_len == 0 || pat_len > str_len then - [] - else - std.filter(function(i) str[i:i + pat_len] == pat, std.range(0, str_len - pat_len)), - - find(value, arr):: - if !std.isArray(arr) then - error 'find second parameter should be an array, got ' + std.type(arr) - else - std.filter(function(i) arr[i] == value, std.range(0, std.length(arr) - 1)), -} -- gitstuff