git.delta.rocks / jrsonnet / refs/commits / 4ec63264e784

difftreelog

refactor prefer builtins over natives

Yaroslav Bolyukin2022-03-06parent: #2d3e912.patch.diff
in: master

8 files changed

modifiedbindings/jsonnet/src/native.rsdiffbeforeafterboth
--- a/bindings/jsonnet/src/native.rs
+++ b/bindings/jsonnet/src/native.rs
@@ -1,11 +1,11 @@
 use gcmodule::Cc;
 use jrsonnet_evaluator::{
 	error::{Error, LocError},
+	function::BuiltinParam,
 	gc::TraceBox,
 	native::{NativeCallback, NativeCallbackHandler},
 	EvaluationState, IStr, Val,
 };
-use jrsonnet_parser::{Param, ParamsDesc};
 use std::{
 	convert::TryFrom,
 	ffi::{c_void, CStr},
@@ -28,7 +28,7 @@
 	cb: JsonnetNativeCallback,
 }
 impl NativeCallbackHandler for JsonnetNativeCallbackHandler {
-	fn call(&self, _from: Rc<Path>, args: &[Val]) -> Result<Val, LocError> {
+	fn call(&self, _from: Option<Rc<Path>>, args: &[Val]) -> Result<Val, LocError> {
 		let mut n_args = Vec::new();
 		for a in args {
 			n_args.push(Some(Box::new(a.clone())));
@@ -68,16 +68,19 @@
 			break;
 		}
 		let param = CStr::from_ptr(*raw_params).to_str().expect("not utf8");
-		params.push(Param(param.into(), None));
+		params.push(BuiltinParam {
+			name: param.into(),
+			has_default: false,
+		});
 		raw_params = raw_params.offset(1);
 	}
-	let params = ParamsDesc(Rc::new(params));
 
 	vm.add_native(
 		name,
-		Cc::new(NativeCallback::new(
+		#[allow(deprecated)]
+		Cc::new(TraceBox(Box::new(NativeCallback::new(
 			params,
 			TraceBox(Box::new(JsonnetNativeCallbackHandler { ctx, cb })),
-		)),
+		)))),
 	)
 }
modifiedcrates/jrsonnet-evaluator/src/builtin/mod.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/builtin/mod.rs
+++ b/crates/jrsonnet-evaluator/src/builtin/mod.rs
@@ -11,7 +11,6 @@
 };
 use crate::{Either, ObjValue};
 use format::{format_arr, format_obj};
-use gcmodule::Cc;
 use jrsonnet_interner::IStr;
 use jrsonnet_parser::ExprLocation;
 use serde::Deserialize;
@@ -142,7 +141,7 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_length(x: Either![IStr, VecVal, ObjValue, Cc<FuncVal>]) -> Result<usize> {
+fn builtin_length(x: Either![IStr, VecVal, ObjValue, FuncVal]) -> Result<usize> {
 	use Either4::*;
 	Ok(match x {
 		A(x) => x.chars().count(),
@@ -162,7 +161,7 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_make_array(sz: usize, func: Cc<FuncVal>) -> Result<VecVal> {
+fn builtin_make_array(sz: usize, func: FuncVal) -> Result<VecVal> {
 	let mut out = Vec::with_capacity(sz);
 	for i in 0..sz {
 		out.push(func.evaluate_simple(&[i as f64].as_slice())?)
@@ -345,24 +344,24 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_native(name: IStr) -> Result<Cc<FuncVal>> {
+fn builtin_native(name: IStr) -> Result<FuncVal> {
 	Ok(with_state(|s| s.settings().ext_natives.get(&name).cloned())
-		.map(|v| Cc::new(FuncVal::NativeExt(name.clone(), v)))
+		.map(|v| FuncVal::Builtin(v.clone()))
 		.ok_or(UndefinedExternalFunction(name))?)
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_filter(func: Cc<FuncVal>, arr: ArrValue) -> Result<ArrValue> {
+fn builtin_filter(func: FuncVal, arr: ArrValue) -> Result<ArrValue> {
 	arr.filter(|val| bool::try_from(func.evaluate_simple(&[Any(val.clone())].as_slice())?))
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_map(func: Cc<FuncVal>, arr: ArrValue) -> Result<ArrValue> {
+fn builtin_map(func: FuncVal, arr: ArrValue) -> Result<ArrValue> {
 	arr.map(|val| func.evaluate_simple(&[Any(val)].as_slice()))
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_flatmap(func: Cc<FuncVal>, arr: IndexableVal) -> Result<IndexableVal> {
+fn builtin_flatmap(func: FuncVal, arr: IndexableVal) -> Result<IndexableVal> {
 	match arr {
 		IndexableVal::Str(s) => {
 			let mut out = String::new();
@@ -397,7 +396,7 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_foldl(func: Cc<FuncVal>, arr: ArrValue, init: Any) -> Result<Any> {
+fn builtin_foldl(func: FuncVal, arr: ArrValue, init: Any) -> Result<Any> {
 	let mut acc = init.0;
 	for i in arr.iter() {
 		acc = func.evaluate_simple(&[Any(acc), Any(i?)].as_slice())?;
@@ -406,7 +405,7 @@
 }
 
 #[jrsonnet_macros::builtin]
-fn builtin_foldr(func: Cc<FuncVal>, arr: ArrValue, init: Any) -> Result<Any> {
+fn builtin_foldr(func: FuncVal, arr: ArrValue, init: Any) -> Result<Any> {
 	let mut acc = init.0;
 	for i in arr.iter().rev() {
 		acc = func.evaluate_simple(&[Any(i?), Any(acc)].as_slice())?;
@@ -416,13 +415,13 @@
 
 #[jrsonnet_macros::builtin]
 #[allow(non_snake_case)]
-fn builtin_sort(arr: ArrValue, keyF: Option<Cc<FuncVal>>) -> Result<ArrValue> {
+fn builtin_sort(arr: ArrValue, keyF: Option<FuncVal>) -> Result<ArrValue> {
 	if arr.len() <= 1 {
 		return Ok(arr);
 	}
 	Ok(ArrValue::Eager(sort::sort(
 		arr.evaluated()?,
-		keyF.as_deref(),
+		keyF.as_ref(),
 	)?))
 }
 
modifiedcrates/jrsonnet-evaluator/src/evaluate/mod.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs
+++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
@@ -177,7 +177,7 @@
 }
 
 pub fn evaluate_method(ctx: Context, name: IStr, params: ParamsDesc, body: LocExpr) -> Val {
-	Val::Func(Cc::new(FuncVal::Normal(FuncDesc {
+	Val::Func(FuncVal::Normal(Cc::new(FuncDesc {
 		name,
 		ctx,
 		params,
@@ -630,11 +630,11 @@
 		Function(params, body) => {
 			evaluate_method(context, "anonymous".into(), params.clone(), body.clone())
 		}
-		Intrinsic(name) => Val::Func(Cc::new(FuncVal::StaticBuiltin(
+		Intrinsic(name) => Val::Func(FuncVal::StaticBuiltin(
 			BUILTINS
 				.with(|b| b.get(name).copied())
 				.ok_or_else(|| IntrinsicNotFound(name.clone()))?,
-		))),
+		)),
 		AssertExpr(assert, returned) => {
 			evaluate_assert(context.clone(), assert)?;
 			evaluate(context, returned)?
modifiedcrates/jrsonnet-evaluator/src/function.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/function.rs
+++ b/crates/jrsonnet-evaluator/src/function.rs
@@ -371,7 +371,7 @@
 
 type BuiltinParamName = Cow<'static, str>;
 
-#[derive(Clone)]
+#[derive(Clone, Trace)]
 pub struct BuiltinParam {
 	pub name: BuiltinParamName,
 	pub has_default: bool,
modifiedcrates/jrsonnet-evaluator/src/lib.rsdiffbeforeafterboth
before · crates/jrsonnet-evaluator/src/lib.rs
1#![cfg_attr(feature = "unstable", feature(stmt_expr_attributes))]2#![warn(clippy::all, clippy::nursery)]3#![allow(4	macro_expanded_macro_exports_accessed_by_absolute_paths,5	clippy::ptr_arg6)]78// For jrsonnet-macros9extern crate self as jrsonnet_evaluator;1011mod builtin;12mod ctx;13mod dynamic;14pub mod error;15mod evaluate;16mod function;17mod import;18mod integrations;19mod map;20pub mod native;21mod obj;22pub mod trace;23pub mod typed;24mod val;2526pub use ctx::*;27pub use dynamic::*;28use error::{Error::*, LocError, Result, StackTraceElement};29pub use evaluate::*;30pub use function::parse_function_call;31use function::TlaArg;32use gc::{GcHashMap, TraceBox};33use gcmodule::{Cc, Trace};34pub use import::*;35pub use jrsonnet_interner::IStr;36use jrsonnet_parser::*;37use native::NativeCallback;38pub use obj::*;39use std::{40	cell::{Ref, RefCell, RefMut},41	collections::HashMap,42	fmt::Debug,43	path::{Path, PathBuf},44	rc::Rc,45};46use trace::{location_to_offset, offset_to_location, CodeLocation, CompactFormat, TraceFormat};47pub use val::*;48pub mod gc;4950pub trait Bindable: Trace + 'static {51	fn bind(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal>;52}5354#[derive(Clone, Trace)]55pub enum LazyBinding {56	Bindable(Cc<TraceBox<dyn Bindable>>),57	Bound(LazyVal),58}5960impl Debug for LazyBinding {61	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {62		write!(f, "LazyBinding")63	}64}65impl LazyBinding {66	pub fn evaluate(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal> {67		match self {68			Self::Bindable(v) => v.bind(this, super_obj),69			Self::Bound(v) => Ok(v.clone()),70		}71	}72}7374pub struct EvaluationSettings {75	/// Limits recursion by limiting the number of stack frames76	pub max_stack: usize,77	/// Limits amount of stack trace items preserved78	pub max_trace: usize,79	/// Used for s`td.extVar`80	pub ext_vars: HashMap<IStr, Val>,81	/// Used for ext.native82	pub ext_natives: HashMap<IStr, Cc<NativeCallback>>,83	/// TLA vars84	pub tla_vars: HashMap<IStr, TlaArg>,85	/// Global variables are inserted in default context86	pub globals: HashMap<IStr, Val>,87	/// Used to resolve file locations/contents88	pub import_resolver: Box<dyn ImportResolver>,89	/// Used in manifestification functions90	pub manifest_format: ManifestFormat,91	/// Used for bindings92	pub trace_format: Box<dyn TraceFormat>,93}94impl Default for EvaluationSettings {95	fn default() -> Self {96		Self {97			max_stack: 200,98			max_trace: 20,99			globals: Default::default(),100			ext_vars: Default::default(),101			ext_natives: Default::default(),102			tla_vars: Default::default(),103			import_resolver: Box::new(DummyImportResolver),104			manifest_format: ManifestFormat::Json(4),105			trace_format: Box::new(CompactFormat {106				padding: 4,107				resolver: trace::PathResolver::Absolute,108			}),109		}110	}111}112113#[derive(Default)]114struct EvaluationData {115	/// Used for stack overflow detection, stacktrace is populated on unwind116	stack_depth: usize,117	/// Updated every time stack entry is popt118	stack_generation: usize,119120	breakpoints: Breakpoints,121	/// Contains file source codes and evaluation results for imports and pretty-printed stacktraces122	files: HashMap<Rc<Path>, FileData>,123	str_files: HashMap<Rc<Path>, IStr>,124}125126pub struct FileData {127	source_code: IStr,128	parsed: LocExpr,129	evaluated: Option<Val>,130}131132#[allow(clippy::type_complexity)]133pub struct Breakpoint {134	loc: ExprLocation,135	collected: RefCell<HashMap<usize, (usize, Vec<Result<Val>>)>>,136}137#[derive(Default)]138struct Breakpoints(Vec<Rc<Breakpoint>>);139impl Breakpoints {140	fn insert(141		&self,142		stack_depth: usize,143		stack_generation: usize,144		loc: &ExprLocation,145		result: Result<Val>,146	) -> Result<Val> {147		if self.0.is_empty() {148			return result;149		}150		for item in self.0.iter() {151			if item.loc.belongs_to(loc) {152				let mut collected = item.collected.borrow_mut();153				let (depth, vals) = collected.entry(stack_generation).or_default();154				if stack_depth > *depth {155					vals.clear();156				}157				vals.push(result.clone());158			}159		}160		result161	}162}163164#[derive(Default)]165pub struct EvaluationStateInternals {166	/// Internal state167	data: RefCell<EvaluationData>,168	/// Settings, safe to change at runtime169	settings: RefCell<EvaluationSettings>,170}171172thread_local! {173	/// Contains the state for a currently executed file.174	/// Global state is fine here.175	pub(crate) static EVAL_STATE: RefCell<Option<EvaluationState>> = RefCell::new(None)176}177pub(crate) fn with_state<T>(f: impl FnOnce(&EvaluationState) -> T) -> T {178	EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap()))179}180pub(crate) fn push_frame<T>(181	e: Option<&ExprLocation>,182	frame_desc: impl FnOnce() -> String,183	f: impl FnOnce() -> Result<T>,184) -> Result<T> {185	with_state(|s| s.push(e, frame_desc, f))186}187188#[allow(dead_code)]189pub(crate) fn push_val_frame(190	e: &ExprLocation,191	frame_desc: impl FnOnce() -> String,192	f: impl FnOnce() -> Result<Val>,193) -> Result<Val> {194	with_state(|s| s.push_val(e, frame_desc, f))195}196#[allow(dead_code)]197pub(crate) fn push_description_frame<T>(198	frame_desc: impl FnOnce() -> String,199	f: impl FnOnce() -> Result<T>,200) -> Result<T> {201	with_state(|s| s.push_description(frame_desc, f))202}203204/// Maintains stack trace and import resolution205#[derive(Default, Clone)]206pub struct EvaluationState(Rc<EvaluationStateInternals>);207208impl EvaluationState {209	/// Parses and adds file as loaded210	pub fn add_file(&self, path: Rc<Path>, source_code: IStr) -> Result<LocExpr> {211		let parsed = parse(212			&source_code,213			&ParserSettings {214				file_name: path.clone(),215			},216		)217		.map_err(|error| ImportSyntaxError {218			error: Box::new(error),219			path: path.to_owned(),220			source_code: source_code.clone(),221		})?;222		self.add_parsed_file(path, source_code, parsed.clone())?;223224		Ok(parsed)225	}226227	pub fn reset_evaluation_state(&self, name: &Path) {228		self.data_mut()229			.files230			.get_mut(name)231			.unwrap()232			.evaluated233			.take();234	}235236	/// Adds file by source code and parsed expr237	pub fn add_parsed_file(238		&self,239		name: Rc<Path>,240		source_code: IStr,241		parsed: LocExpr,242	) -> Result<()> {243		self.data_mut().files.insert(244			name,245			FileData {246				source_code,247				parsed,248				evaluated: None,249			},250		);251252		Ok(())253	}254	pub fn get_source(&self, name: &Path) -> Option<IStr> {255		let ro_map = &self.data().files;256		ro_map.get(name).map(|value| value.source_code.clone())257	}258	pub fn map_source_locations(&self, file: &Path, locs: &[usize]) -> Vec<CodeLocation> {259		offset_to_location(&self.get_source(file).unwrap_or_else(|| "".into()), locs)260	}261	pub fn map_from_source_location(262		&self,263		file: &Path,264		line: usize,265		column: usize,266	) -> Option<usize> {267		location_to_offset(&self.get_source(file).unwrap(), line, column)268	}269	pub fn import_file(&self, from: &Path, path: &Path) -> Result<Val> {270		let file_path = self.resolve_file(from, path)?;271		{272			let data = self.data();273			let files = &data.files;274			if files.contains_key(&file_path as &Path) {275				drop(data);276				return self.evaluate_loaded_file_raw(&file_path);277			}278		}279		let contents = self.load_file_contents(&file_path)?;280		self.add_file(file_path.clone(), contents)?;281		self.evaluate_loaded_file_raw(&file_path)282	}283	pub(crate) fn import_file_str(&self, from: &Path, path: &Path) -> Result<IStr> {284		let path = self.resolve_file(from, path)?;285		if !self.data().str_files.contains_key(&path) {286			let file_str = self.load_file_contents(&path)?;287			self.data_mut().str_files.insert(path.clone(), file_str);288		}289		Ok(self.data().str_files.get(&path).cloned().unwrap())290	}291292	fn evaluate_loaded_file_raw(&self, name: &Path) -> Result<Val> {293		let expr: LocExpr = {294			let ro_map = &self.data().files;295			let value = ro_map296				.get(name)297				.unwrap_or_else(|| panic!("file not added: {:?}", name));298			if let Some(ref evaluated) = value.evaluated {299				return Ok(evaluated.clone());300			}301			value.parsed.clone()302		};303		let value = evaluate(self.create_default_context(), &expr)?;304		{305			self.data_mut()306				.files307				.get_mut(name)308				.unwrap()309				.evaluated310				.replace(value.clone());311		}312		Ok(value)313	}314315	/// Adds standard library global variable (std) to this evaluator316	pub fn with_stdlib(&self) -> &Self {317		use jrsonnet_stdlib::STDLIB_STR;318		let std_path: Rc<Path> = PathBuf::from("std.jsonnet").into();319		self.run_in_state(|| {320			self.add_parsed_file(321				std_path.clone(),322				STDLIB_STR.to_owned().into(),323				builtin::get_parsed_stdlib(),324			)325			.unwrap();326			let val = self.evaluate_loaded_file_raw(&std_path).unwrap();327			self.settings_mut().globals.insert("std".into(), val);328		});329		self330	}331332	/// Creates context with all passed global variables333	pub fn create_default_context(&self) -> Context {334		let globals = &self.settings().globals;335		let mut new_bindings = GcHashMap::with_capacity(globals.len());336		for (name, value) in globals.iter() {337			new_bindings.insert(name.clone(), LazyVal::new_resolved(value.clone()));338		}339		Context::new().extend_bound(new_bindings)340	}341342	/// Executes code creating a new stack frame343	pub fn push<T>(344		&self,345		e: Option<&ExprLocation>,346		frame_desc: impl FnOnce() -> String,347		f: impl FnOnce() -> Result<T>,348	) -> Result<T> {349		{350			let mut data = self.data_mut();351			let stack_depth = &mut data.stack_depth;352			if *stack_depth > self.max_stack() {353				// Error creation uses data, so i drop guard here354				drop(data);355				throw!(StackOverflow);356			} else {357				*stack_depth += 1;358			}359		}360		let result = f();361		{362			let mut data = self.data_mut();363			data.stack_depth -= 1;364			data.stack_generation += 1;365		}366		if let Err(mut err) = result {367			err.trace_mut().0.push(StackTraceElement {368				location: e.cloned(),369				desc: frame_desc(),370			});371			return Err(err);372		}373		result374	}375376	/// Executes code creating a new stack frame377	pub fn push_val(378		&self,379		e: &ExprLocation,380		frame_desc: impl FnOnce() -> String,381		f: impl FnOnce() -> Result<Val>,382	) -> Result<Val> {383		{384			let mut data = self.data_mut();385			let stack_depth = &mut data.stack_depth;386			if *stack_depth > self.max_stack() {387				// Error creation uses data, so i drop guard here388				drop(data);389				throw!(StackOverflow);390			} else {391				*stack_depth += 1;392			}393		}394		let mut result = f();395		{396			let mut data = self.data_mut();397			data.stack_depth -= 1;398			data.stack_generation += 1;399			result = data400				.breakpoints401				.insert(data.stack_depth, data.stack_generation, e, result);402		}403		if let Err(mut err) = result {404			err.trace_mut().0.push(StackTraceElement {405				location: Some(e.clone()),406				desc: frame_desc(),407			});408			return Err(err);409		}410		result411	}412	/// Executes code creating a new stack frame413	pub fn push_description<T>(414		&self,415		frame_desc: impl FnOnce() -> String,416		f: impl FnOnce() -> Result<T>,417	) -> Result<T> {418		{419			let mut data = self.data_mut();420			let stack_depth = &mut data.stack_depth;421			if *stack_depth > self.max_stack() {422				// Error creation uses data, so i drop guard here423				drop(data);424				throw!(StackOverflow);425			} else {426				*stack_depth += 1;427			}428		}429		let result = f();430		{431			let mut data = self.data_mut();432			data.stack_depth -= 1;433			data.stack_generation += 1;434		}435		if let Err(mut err) = result {436			err.trace_mut().0.push(StackTraceElement {437				location: None,438				desc: frame_desc(),439			});440			return Err(err);441		}442		result443	}444445	/// Runs passed function in state (required if function needs to modify stack trace)446	pub fn run_in_state<T>(&self, f: impl FnOnce() -> T) -> T {447		EVAL_STATE.with(|v| {448			let has_state = v.borrow().is_some();449			if !has_state {450				v.borrow_mut().replace(self.clone());451			}452			let result = f();453			if !has_state {454				v.borrow_mut().take();455			}456			result457		})458	}459	pub fn run_in_state_with_breakpoint(460		&self,461		bp: Rc<Breakpoint>,462		f: impl FnOnce() -> Result<()>,463	) -> Result<()> {464		{465			let mut data = self.data_mut();466			data.breakpoints.0.push(bp);467		}468469		let result = self.run_in_state(f);470471		{472			let mut data = self.data_mut();473			data.breakpoints.0.pop();474		}475476		result477	}478479	pub fn stringify_err(&self, e: &LocError) -> String {480		let mut out = String::new();481		self.settings()482			.trace_format483			.write_trace(&mut out, self, e)484			.unwrap();485		out486	}487488	pub fn manifest(&self, val: Val) -> Result<IStr> {489		self.run_in_state(|| {490			push_description_frame(491				|| "manifestification".to_string(),492				|| val.manifest(&self.manifest_format()),493			)494		})495	}496	pub fn manifest_multi(&self, val: Val) -> Result<Vec<(IStr, IStr)>> {497		self.run_in_state(|| val.manifest_multi(&self.manifest_format()))498	}499	pub fn manifest_stream(&self, val: Val) -> Result<Vec<IStr>> {500		self.run_in_state(|| val.manifest_stream(&self.manifest_format()))501	}502503	/// If passed value is function then call with set TLA504	pub fn with_tla(&self, val: Val) -> Result<Val> {505		self.run_in_state(|| {506			Ok(match val {507				Val::Func(func) => push_description_frame(508					|| "during TLA call".to_owned(),509					|| {510						func.evaluate(511							self.create_default_context(),512							None,513							&self.settings().tla_vars,514							true,515						)516					},517				)?,518				v => v,519			})520		})521	}522}523524/// Internals525impl EvaluationState {526	fn data(&self) -> Ref<EvaluationData> {527		self.0.data.borrow()528	}529	fn data_mut(&self) -> RefMut<EvaluationData> {530		self.0.data.borrow_mut()531	}532	pub fn settings(&self) -> Ref<EvaluationSettings> {533		self.0.settings.borrow()534	}535	pub fn settings_mut(&self) -> RefMut<EvaluationSettings> {536		self.0.settings.borrow_mut()537	}538}539540/// Raw methods evaluate passed values but don't perform TLA execution541impl EvaluationState {542	pub fn evaluate_file_raw(&self, name: &Path) -> Result<Val> {543		self.run_in_state(|| self.import_file(&std::env::current_dir().expect("cwd"), name))544	}545	pub fn evaluate_file_raw_nocwd(&self, name: &Path) -> Result<Val> {546		self.run_in_state(|| self.import_file(&PathBuf::from("."), name))547	}548	/// Parses and evaluates the given snippet549	pub fn evaluate_snippet_raw(&self, source: Rc<Path>, code: IStr) -> Result<Val> {550		let parsed = parse(551			&code,552			&ParserSettings {553				file_name: source.clone(),554			},555		)556		.map_err(|e| ImportSyntaxError {557			path: source.clone(),558			source_code: code.clone(),559			error: Box::new(e),560		})?;561		self.add_parsed_file(source, code, parsed.clone())?;562		self.evaluate_expr_raw(parsed)563	}564	/// Evaluates the parsed expression565	pub fn evaluate_expr_raw(&self, code: LocExpr) -> Result<Val> {566		self.run_in_state(|| evaluate(self.create_default_context(), &code))567	}568}569570/// Settings utilities571impl EvaluationState {572	pub fn add_ext_var(&self, name: IStr, value: Val) {573		self.settings_mut().ext_vars.insert(name, value);574	}575	pub fn add_ext_str(&self, name: IStr, value: IStr) {576		self.add_ext_var(name, Val::Str(value));577	}578	pub fn add_ext_code(&self, name: IStr, code: IStr) -> Result<()> {579		let value =580			self.evaluate_snippet_raw(PathBuf::from(format!("ext_code {}", name)).into(), code)?;581		self.add_ext_var(name, value);582		Ok(())583	}584585	pub fn add_tla(&self, name: IStr, value: Val) {586		self.settings_mut()587			.tla_vars588			.insert(name, TlaArg::Val(value));589	}590	pub fn add_tla_str(&self, name: IStr, value: IStr) {591		self.settings_mut()592			.tla_vars593			.insert(name, TlaArg::String(value));594	}595	pub fn add_tla_code(&self, name: IStr, code: IStr) -> Result<()> {596		let parsed = self.add_file(PathBuf::from(format!("tla_code {}", name)).into(), code)?;597		self.settings_mut()598			.tla_vars599			.insert(name, TlaArg::Code(parsed));600		Ok(())601	}602603	pub fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {604		self.settings().import_resolver.resolve_file(from, path)605	}606	pub fn load_file_contents(&self, path: &Path) -> Result<IStr> {607		self.settings().import_resolver.load_file_contents(path)608	}609610	pub fn import_resolver(&self) -> Ref<dyn ImportResolver> {611		Ref::map(self.settings(), |s| &*s.import_resolver)612	}613	pub fn set_import_resolver(&self, resolver: Box<dyn ImportResolver>) {614		self.settings_mut().import_resolver = resolver;615	}616617	pub fn add_native(&self, name: IStr, cb: Cc<NativeCallback>) {618		self.settings_mut().ext_natives.insert(name, cb);619	}620621	pub fn manifest_format(&self) -> ManifestFormat {622		self.settings().manifest_format.clone()623	}624	pub fn set_manifest_format(&self, format: ManifestFormat) {625		self.settings_mut().manifest_format = format;626	}627628	pub fn trace_format(&self) -> Ref<dyn TraceFormat> {629		Ref::map(self.settings(), |s| &*s.trace_format)630	}631	pub fn set_trace_format(&self, format: Box<dyn TraceFormat>) {632		self.settings_mut().trace_format = format;633	}634635	pub fn max_trace(&self) -> usize {636		self.settings().max_trace637	}638	pub fn set_max_trace(&self, trace: usize) {639		self.settings_mut().max_trace = trace;640	}641642	pub fn max_stack(&self) -> usize {643		self.settings().max_stack644	}645	pub fn set_max_stack(&self, trace: usize) {646		self.settings_mut().max_stack = trace;647	}648}649650pub fn cc_ptr_eq<T>(a: &Cc<T>, b: &Cc<T>) -> bool {651	let a = a as &T;652	let b = b as &T;653	std::ptr::eq(a, b)654}655656#[cfg(test)]657pub mod tests {658	use super::Val;659	use crate::{660		error::Error::*, gc::TraceBox, native::NativeCallbackHandler, primitive_equals,661		EvaluationState,662	};663	use gcmodule::{Cc, Trace};664	use jrsonnet_interner::IStr;665	use jrsonnet_parser::*;666	use std::{667		path::{Path, PathBuf},668		rc::Rc,669	};670671	#[test]672	#[should_panic]673	fn eval_state_stacktrace() {674		let state = EvaluationState::default();675		state.run_in_state(|| {676			state677				.push(678					Some(&ExprLocation(PathBuf::from("test1.jsonnet").into(), 10, 20)),679					|| "outer".to_owned(),680					|| {681						state.push(682							Some(&ExprLocation(PathBuf::from("test2.jsonnet").into(), 30, 40)),683							|| "inner".to_owned(),684							|| Err(RuntimeError("".into()).into()),685						)?;686						Ok(Val::Null)687					},688				)689				.unwrap();690		});691	}692693	#[test]694	fn eval_state_standard() {695		let state = EvaluationState::default();696		state.with_stdlib();697		assert!(primitive_equals(698			&state699				.evaluate_snippet_raw(700					PathBuf::from("raw.jsonnet").into(),701					r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#.into()702				)703				.unwrap(),704			&Val::Bool(true),705		)706		.unwrap());707	}708709	macro_rules! eval {710		($str: expr) => {711			EvaluationState::default()712				.with_stdlib()713				.evaluate_snippet_raw(PathBuf::from("raw.jsonnet").into(), $str.into())714				.unwrap()715		};716	}717	macro_rules! eval_json {718		($str: expr) => {{719			let evaluator = EvaluationState::default();720			evaluator.with_stdlib();721			evaluator.run_in_state(|| {722				evaluator723					.evaluate_snippet_raw(PathBuf::from("raw.jsonnet").into(), $str.into())724					.unwrap()725					.to_json(0)726					.unwrap()727					.replace("\n", "")728			})729		}};730	}731732	/// Asserts given code returns `true`733	macro_rules! assert_eval {734		($str: expr) => {735			assert!(primitive_equals(&eval!($str), &Val::Bool(true)).unwrap())736		};737	}738739	/// Asserts given code returns `false`740	macro_rules! assert_eval_neg {741		($str: expr) => {742			assert!(primitive_equals(&eval!($str), &Val::Bool(false)).unwrap())743		};744	}745	macro_rules! assert_json {746		($str: expr, $out: expr) => {747			assert_eq!(eval_json!($str), $out.replace("\t", ""))748		};749	}750751	/// Sanity checking, before trusting to another tests752	#[test]753	fn equality_operator() {754		assert_eval!("2 == 2");755		assert_eval_neg!("2 != 2");756		assert_eval!("2 != 3");757		assert_eval_neg!("2 == 3");758		assert_eval!("'Hello' == 'Hello'");759		assert_eval_neg!("'Hello' != 'Hello'");760		assert_eval!("'Hello' != 'World'");761		assert_eval_neg!("'Hello' == 'World'");762	}763764	#[test]765	fn math_evaluation() {766		assert_eval!("2 + 2 * 2 == 6");767		assert_eval!("3 + (2 + 2 * 2) == 9");768	}769770	#[test]771	fn string_concat() {772		assert_eval!("'Hello' + 'World' == 'HelloWorld'");773		assert_eval!("'Hello' * 3 == 'HelloHelloHello'");774		assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'");775	}776777	#[test]778	fn faster_join() {779		assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]");780		assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'");781	}782783	#[test]784	fn function_contexts() {785		assert_eval!(786			r#"787				local k = {788					t(name = self.h): [self.h, name],789					h: 3,790				};791				local f = {792					t: k.t(),793					h: 4,794				};795				f.t[0] == f.t[1]796			"#797		);798	}799800	#[test]801	fn local() {802		assert_eval!("local a = 2; local b = 3; a + b == 5");803		assert_eval!("local a = 1, b = a + 1; a + b == 3");804		assert_eval!("local a = 1; local a = 2; a == 2");805	}806807	#[test]808	fn object_lazyness() {809		assert_json!("local a = {a:error 'test'}; {}", r#"{}"#);810	}811812	#[test]813	fn object_inheritance() {814		assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#);815	}816817	#[test]818	fn object_assertion_success() {819		eval!("{assert \"a\" in self} + {a:2}");820	}821822	#[test]823	fn object_assertion_error() {824		eval!("{assert \"a\" in self}");825	}826827	#[test]828	fn lazy_args() {829		eval!("local test(a) = 2; test(error '3')");830	}831832	#[test]833	#[should_panic]834	fn tailstrict_args() {835		eval!("local test(a) = 2; test(error '3') tailstrict");836	}837838	#[test]839	#[should_panic]840	fn no_binding_error() {841		eval!("a");842	}843844	#[test]845	fn test_object() {846		assert_json!("{a:2}", r#"{"a": 2}"#);847		assert_json!("{a:2+2}", r#"{"a": 4}"#);848		assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#);849		assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#);850		assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#);851		assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#);852		assert_json!(853			r#"854				{855					name: "Alice",856					welcome: "Hello " + self.name + "!",857				}858			"#,859			r#"{"name": "Alice","welcome": "Hello Alice!"}"#860		);861		assert_json!(862			r#"863				{864					name: "Alice",865					welcome: "Hello " + self.name + "!",866				} + {867					name: "Bob"868				}869			"#,870			r#"{"name": "Bob","welcome": "Hello Bob!"}"#871		);872	}873874	#[test]875	fn functions() {876		assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4");877		assert_json!(878			r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#,879			r#""HelloDearWorld""#880		);881	}882883	#[test]884	fn local_methods() {885		assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4");886		assert_json!(887			r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#,888			r#""HelloDearWorld""#889		);890	}891892	#[test]893	fn object_locals() {894		assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#);895		assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#);896		assert_json!(897			r#"{local a = function (b) {[b]:4}, test: a("test")}"#,898			r#"{"test": {"test": 4}}"#899		);900	}901902	#[test]903	fn object_comp() {904		assert_json!(905			r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#,906			"{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}"907		)908	}909910	#[test]911	fn direct_self() {912		println!(913			"{:#?}",914			eval!(915				r#"916					{917						local me = self,918						a: 3,919						b(): me.a,920					}921				"#922			)923		);924	}925926	#[test]927	fn indirect_self() {928		// `self` assigned to `me` was lost when being929		// referenced from field930		eval!(931			r#"{932				local me = self,933				a: 3,934				b: me.a,935			}.b"#936		);937	}938939	// We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly940	#[test]941	fn std_assert_ok() {942		eval!("std.assertEqual(4.5 << 2, 16)");943	}944945	#[test]946	#[should_panic]947	fn std_assert_failure() {948		eval!("std.assertEqual(4.5 << 2, 15)");949	}950951	#[test]952	fn string_is_string() {953		assert!(primitive_equals(954			&eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"),955			&Val::Bool(false),956		)957		.unwrap());958	}959960	#[test]961	fn base64_works() {962		assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#);963	}964965	#[test]966	fn utf8_chars() {967		assert_json!(968			r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#,969			r#"{"c": 128526,"l": 1}"#970		)971	}972973	#[test]974	fn json() {975		assert_json!(976			r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#,977			r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""#978		);979	}980981	#[test]982	fn json_minified() {983		assert_json!(984			r#"std.manifestJsonMinified({a:3, b:4, c:6})"#,985			r#""{\"a\":3,\"b\":4,\"c\":6}""#986		);987	}988989	#[test]990	fn parse_json() {991		assert_json!(992			r#"std.parseJson('{"a": -1,"b": 1,"c": 3.141,"d": []}')"#,993			r#"{"a": -1,"b": 1,"c": 3.141,"d": []}"#994		);995	}996997	#[test]998	fn test() {999		assert_json!(1000			r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#,1001			"[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]"1002		);1003	}10041005	#[test]1006	fn sjsonnet() {1007		eval!(1008			r#"1009			local x0 = {k: 1};1010			local x1 = {k: x0.k + x0.k};1011			local x2 = {k: x1.k + x1.k};1012			local x3 = {k: x2.k + x2.k};1013			local x4 = {k: x3.k + x3.k};1014			local x5 = {k: x4.k + x4.k};1015			local x6 = {k: x5.k + x5.k};1016			local x7 = {k: x6.k + x6.k};1017			local x8 = {k: x7.k + x7.k};1018			local x9 = {k: x8.k + x8.k};1019			local x10 = {k: x9.k + x9.k};1020			local x11 = {k: x10.k + x10.k};1021			local x12 = {k: x11.k + x11.k};1022			local x13 = {k: x12.k + x12.k};1023			local x14 = {k: x13.k + x13.k};1024			local x15 = {k: x14.k + x14.k};1025			local x16 = {k: x15.k + x15.k};1026			local x17 = {k: x16.k + x16.k};1027			local x18 = {k: x17.k + x17.k};1028			local x19 = {k: x18.k + x18.k};1029			local x20 = {k: x19.k + x19.k};1030			local x21 = {k: x20.k + x20.k};1031			x21.k1032		"#1033		);1034	}10351036	// This test is commented out by default, because of huge compilation slowdown1037	// #[bench]1038	// fn bench_codegen(b: &mut Bencher) {1039	// 	b.iter(|| {1040	// 		#[allow(clippy::all)]1041	// 		let stdlib = {1042	// 			use jrsonnet_parser::*;1043	// 			include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))1044	// 		};1045	// 		stdlib1046	// 	})1047	// }10481049	/*1050	#[bench]1051	fn bench_serialize(b: &mut Bencher) {1052		b.iter(|| {1053			bincode::deserialize::<jrsonnet_parser::LocExpr>(include_bytes!(concat!(1054				env!("OUT_DIR"),1055				"/stdlib.bincode"1056			)))1057			.expect("deserialize stdlib")1058		})1059	}10601061	#[bench]1062	fn bench_parse(b: &mut Bencher) {1063		b.iter(|| {1064			jrsonnet_parser::parse(1065				jrsonnet_stdlib::STDLIB_STR,1066				&jrsonnet_parser::ParserSettings {1067					loc_data: true,1068					file_name: Rc::new(PathBuf::from("std.jsonnet")),1069				},1070			)1071		})1072	}1073	*/10741075	#[test]1076	fn equality() {1077		println!(1078			"{:?}",1079			jrsonnet_parser::parse(1080				"{ x: 1, y: 2 } == { x: 1, y: 2 }",1081				&ParserSettings {1082					file_name: PathBuf::from("equality").into(),1083				}1084			)1085		);1086		assert_eval!("{ x: 1, y: 2 } == { x: 1, y: 2 }")1087	}10881089	#[test]1090	fn native_ext() -> crate::error::Result<()> {1091		use super::native::NativeCallback;1092		let evaluator = EvaluationState::default();10931094		evaluator.with_stdlib();10951096		#[derive(Trace)]1097		struct NativeAdd;1098		impl NativeCallbackHandler for NativeAdd {1099			fn call(&self, from: Rc<Path>, args: &[Val]) -> crate::error::Result<Val> {1100				assert_eq!(&from as &Path, &PathBuf::from("native_caller.jsonnet"));1101				match (&args[0], &args[1]) {1102					(Val::Num(a), Val::Num(b)) => Ok(Val::Num(a + b)),1103					(_, _) => unreachable!(),1104				}1105			}1106		}1107		evaluator.settings_mut().ext_natives.insert(1108			"native_add".into(),1109			Cc::new(NativeCallback::new(1110				ParamsDesc(Rc::new(vec![1111					Param("a".into(), None),1112					Param("b".into(), None),1113				])),1114				TraceBox(Box::new(NativeAdd)),1115			)),1116		);1117		evaluator.evaluate_snippet_raw(1118			PathBuf::from("native_caller.jsonnet").into(),1119			"std.assertEqual(std.native(\"native_add\")(1, 2), 3)".into(),1120		)?;1121		Ok(())1122	}11231124	#[test]1125	fn constant_intrinsic() -> crate::error::Result<()> {1126		assert_eval!(1127			"local std2 = std; local std = std2 { primitiveEquals(a, b):: false }; 1 == 1"1128		);1129		Ok(())1130	}11311132	#[test]1133	fn standalone_super() -> crate::error::Result<()> {1134		assert_eval!(1135			r#"1136			local obj = {1137				a: 1,1138				b: 2,1139				c: 3,1140			};1141			local test = obj + {1142				fields: std.objectFields(super),1143				d: 5,1144			};1145			test.fields == ['a', 'b', 'c']1146		"#1147		);1148		Ok(())1149	}11501151	#[test]1152	fn comp_self() -> crate::error::Result<()> {1153		assert_eval!(1154			r#"1155			std.objectFields({1156				a:{1157					[name]: name for name in std.objectFields(self)1158				},1159				b: 2,1160				c: 3,1161			}.a) == ['a', 'b', 'c']1162			"#1163		);11641165		Ok(())1166	}11671168	struct TestImportResolver(IStr);1169	impl crate::import::ImportResolver for TestImportResolver {1170		fn resolve_file(&self, _: &Path, _: &Path) -> crate::error::Result<Rc<Path>> {1171			Ok(PathBuf::from("/test").into())1172		}11731174		fn load_file_contents(&self, _: &Path) -> crate::error::Result<IStr> {1175			Ok(self.0.clone())1176		}11771178		unsafe fn as_any(&self) -> &dyn std::any::Any {1179			panic!()1180		}1181	}11821183	#[test]1184	fn issue_23() {1185		let state = EvaluationState::default();1186		state.set_import_resolver(Box::new(TestImportResolver(r#"import "/test""#.into())));1187		let _ = state.evaluate_file_raw(&PathBuf::from("/test"));1188	}11891190	#[test]1191	fn issue_40() {1192		let state = EvaluationState::default();1193		state.with_stdlib();11941195		let error = state1196			.evaluate_snippet_raw(1197				PathBuf::from("issue40.jsonnet").into(),1198				r#"1199				local conf = {1200					n: ""1201				};12021203				local result = conf + {1204					assert std.isNumber(self.n): "is number"1205				};12061207				std.manifestJsonEx(result, "")1208			"#1209				.into(),1210			)1211			.unwrap_err();1212		assert_eq!(error.error().to_string(), "assert failed: is number");1213	}12141215	#[test]1216	fn test_ascii_upper_lower() {1217		assert_eval!(r#"std.assertEqual(std.asciiUpper("aBc😀"), "ABC😀")"#);1218		assert_eval!(r#"std.assertEqual(std.asciiLower("aBc😀"), "abc😀")"#);1219	}12201221	#[test]1222	fn test_member() {1223		assert_eval!(r#"!std.member("", "")"#);1224		assert_eval!(r#"std.member("abc", "a")"#);1225		assert_eval!(r#"!std.member("abc", "d")"#);1226		assert_eval!(r#"!std.member([], "")"#);1227		assert_eval!(r#"std.member(["a", "b", "c"], "a")"#);1228		assert_eval!(r#"!std.member(["a", "b", "c"], "d")"#);1229	}12301231	#[test]1232	fn test_count() {1233		assert_eval!(r#"std.assertEqual(std.count([], ""), 0)"#);1234		assert_eval!(r#"std.assertEqual(std.count(["a", "b", "a"], "d"), 0)"#);1235		assert_eval!(r#"std.assertEqual(std.count(["a", "b", "a"], "a"), 2)"#);1236	}1237}
after · crates/jrsonnet-evaluator/src/lib.rs
1#![cfg_attr(feature = "unstable", feature(stmt_expr_attributes))]2#![warn(clippy::all, clippy::nursery)]3#![allow(4	macro_expanded_macro_exports_accessed_by_absolute_paths,5	clippy::ptr_arg6)]78// For jrsonnet-macros9extern crate self as jrsonnet_evaluator;1011mod builtin;12mod ctx;13mod dynamic;14pub mod error;15mod evaluate;16pub mod function;17mod import;18mod integrations;19mod map;20pub mod native;21mod obj;22pub mod trace;23pub mod typed;24mod val;2526pub use ctx::*;27pub use dynamic::*;28use error::{Error::*, LocError, Result, StackTraceElement};29pub use evaluate::*;30use function::{Builtin, TlaArg};31use gc::{GcHashMap, TraceBox};32use gcmodule::{Cc, Trace};33pub use import::*;34pub use jrsonnet_interner::IStr;35use jrsonnet_parser::*;36pub use obj::*;37use std::{38	cell::{Ref, RefCell, RefMut},39	collections::HashMap,40	fmt::Debug,41	path::{Path, PathBuf},42	rc::Rc,43};44use trace::{location_to_offset, offset_to_location, CodeLocation, CompactFormat, TraceFormat};45pub use val::*;46pub mod gc;4748pub trait Bindable: Trace + 'static {49	fn bind(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal>;50}5152#[derive(Clone, Trace)]53pub enum LazyBinding {54	Bindable(Cc<TraceBox<dyn Bindable>>),55	Bound(LazyVal),56}5758impl Debug for LazyBinding {59	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {60		write!(f, "LazyBinding")61	}62}63impl LazyBinding {64	pub fn evaluate(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal> {65		match self {66			Self::Bindable(v) => v.bind(this, super_obj),67			Self::Bound(v) => Ok(v.clone()),68		}69	}70}7172pub struct EvaluationSettings {73	/// Limits recursion by limiting the number of stack frames74	pub max_stack: usize,75	/// Limits amount of stack trace items preserved76	pub max_trace: usize,77	/// Used for s`td.extVar`78	pub ext_vars: HashMap<IStr, Val>,79	/// Used for ext.native80	pub ext_natives: HashMap<IStr, Cc<TraceBox<dyn Builtin>>>,81	/// TLA vars82	pub tla_vars: HashMap<IStr, TlaArg>,83	/// Global variables are inserted in default context84	pub globals: HashMap<IStr, Val>,85	/// Used to resolve file locations/contents86	pub import_resolver: Box<dyn ImportResolver>,87	/// Used in manifestification functions88	pub manifest_format: ManifestFormat,89	/// Used for bindings90	pub trace_format: Box<dyn TraceFormat>,91}92impl Default for EvaluationSettings {93	fn default() -> Self {94		Self {95			max_stack: 200,96			max_trace: 20,97			globals: Default::default(),98			ext_vars: Default::default(),99			ext_natives: Default::default(),100			tla_vars: Default::default(),101			import_resolver: Box::new(DummyImportResolver),102			manifest_format: ManifestFormat::Json(4),103			trace_format: Box::new(CompactFormat {104				padding: 4,105				resolver: trace::PathResolver::Absolute,106			}),107		}108	}109}110111#[derive(Default)]112struct EvaluationData {113	/// Used for stack overflow detection, stacktrace is populated on unwind114	stack_depth: usize,115	/// Updated every time stack entry is popt116	stack_generation: usize,117118	breakpoints: Breakpoints,119	/// Contains file source codes and evaluation results for imports and pretty-printed stacktraces120	files: HashMap<Rc<Path>, FileData>,121	str_files: HashMap<Rc<Path>, IStr>,122}123124pub struct FileData {125	source_code: IStr,126	parsed: LocExpr,127	evaluated: Option<Val>,128}129130#[allow(clippy::type_complexity)]131pub struct Breakpoint {132	loc: ExprLocation,133	collected: RefCell<HashMap<usize, (usize, Vec<Result<Val>>)>>,134}135#[derive(Default)]136struct Breakpoints(Vec<Rc<Breakpoint>>);137impl Breakpoints {138	fn insert(139		&self,140		stack_depth: usize,141		stack_generation: usize,142		loc: &ExprLocation,143		result: Result<Val>,144	) -> Result<Val> {145		if self.0.is_empty() {146			return result;147		}148		for item in self.0.iter() {149			if item.loc.belongs_to(loc) {150				let mut collected = item.collected.borrow_mut();151				let (depth, vals) = collected.entry(stack_generation).or_default();152				if stack_depth > *depth {153					vals.clear();154				}155				vals.push(result.clone());156			}157		}158		result159	}160}161162#[derive(Default)]163pub struct EvaluationStateInternals {164	/// Internal state165	data: RefCell<EvaluationData>,166	/// Settings, safe to change at runtime167	settings: RefCell<EvaluationSettings>,168}169170thread_local! {171	/// Contains the state for a currently executed file.172	/// Global state is fine here.173	pub(crate) static EVAL_STATE: RefCell<Option<EvaluationState>> = RefCell::new(None)174}175pub(crate) fn with_state<T>(f: impl FnOnce(&EvaluationState) -> T) -> T {176	EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap()))177}178pub(crate) fn push_frame<T>(179	e: Option<&ExprLocation>,180	frame_desc: impl FnOnce() -> String,181	f: impl FnOnce() -> Result<T>,182) -> Result<T> {183	with_state(|s| s.push(e, frame_desc, f))184}185186#[allow(dead_code)]187pub(crate) fn push_val_frame(188	e: &ExprLocation,189	frame_desc: impl FnOnce() -> String,190	f: impl FnOnce() -> Result<Val>,191) -> Result<Val> {192	with_state(|s| s.push_val(e, frame_desc, f))193}194#[allow(dead_code)]195pub(crate) fn push_description_frame<T>(196	frame_desc: impl FnOnce() -> String,197	f: impl FnOnce() -> Result<T>,198) -> Result<T> {199	with_state(|s| s.push_description(frame_desc, f))200}201202/// Maintains stack trace and import resolution203#[derive(Default, Clone)]204pub struct EvaluationState(Rc<EvaluationStateInternals>);205206impl EvaluationState {207	/// Parses and adds file as loaded208	pub fn add_file(&self, path: Rc<Path>, source_code: IStr) -> Result<LocExpr> {209		let parsed = parse(210			&source_code,211			&ParserSettings {212				file_name: path.clone(),213			},214		)215		.map_err(|error| ImportSyntaxError {216			error: Box::new(error),217			path: path.to_owned(),218			source_code: source_code.clone(),219		})?;220		self.add_parsed_file(path, source_code, parsed.clone())?;221222		Ok(parsed)223	}224225	pub fn reset_evaluation_state(&self, name: &Path) {226		self.data_mut()227			.files228			.get_mut(name)229			.unwrap()230			.evaluated231			.take();232	}233234	/// Adds file by source code and parsed expr235	pub fn add_parsed_file(236		&self,237		name: Rc<Path>,238		source_code: IStr,239		parsed: LocExpr,240	) -> Result<()> {241		self.data_mut().files.insert(242			name,243			FileData {244				source_code,245				parsed,246				evaluated: None,247			},248		);249250		Ok(())251	}252	pub fn get_source(&self, name: &Path) -> Option<IStr> {253		let ro_map = &self.data().files;254		ro_map.get(name).map(|value| value.source_code.clone())255	}256	pub fn map_source_locations(&self, file: &Path, locs: &[usize]) -> Vec<CodeLocation> {257		offset_to_location(&self.get_source(file).unwrap_or_else(|| "".into()), locs)258	}259	pub fn map_from_source_location(260		&self,261		file: &Path,262		line: usize,263		column: usize,264	) -> Option<usize> {265		location_to_offset(&self.get_source(file).unwrap(), line, column)266	}267	pub fn import_file(&self, from: &Path, path: &Path) -> Result<Val> {268		let file_path = self.resolve_file(from, path)?;269		{270			let data = self.data();271			let files = &data.files;272			if files.contains_key(&file_path as &Path) {273				drop(data);274				return self.evaluate_loaded_file_raw(&file_path);275			}276		}277		let contents = self.load_file_contents(&file_path)?;278		self.add_file(file_path.clone(), contents)?;279		self.evaluate_loaded_file_raw(&file_path)280	}281	pub(crate) fn import_file_str(&self, from: &Path, path: &Path) -> Result<IStr> {282		let path = self.resolve_file(from, path)?;283		if !self.data().str_files.contains_key(&path) {284			let file_str = self.load_file_contents(&path)?;285			self.data_mut().str_files.insert(path.clone(), file_str);286		}287		Ok(self.data().str_files.get(&path).cloned().unwrap())288	}289290	fn evaluate_loaded_file_raw(&self, name: &Path) -> Result<Val> {291		let expr: LocExpr = {292			let ro_map = &self.data().files;293			let value = ro_map294				.get(name)295				.unwrap_or_else(|| panic!("file not added: {:?}", name));296			if let Some(ref evaluated) = value.evaluated {297				return Ok(evaluated.clone());298			}299			value.parsed.clone()300		};301		let value = evaluate(self.create_default_context(), &expr)?;302		{303			self.data_mut()304				.files305				.get_mut(name)306				.unwrap()307				.evaluated308				.replace(value.clone());309		}310		Ok(value)311	}312313	/// Adds standard library global variable (std) to this evaluator314	pub fn with_stdlib(&self) -> &Self {315		use jrsonnet_stdlib::STDLIB_STR;316		let std_path: Rc<Path> = PathBuf::from("std.jsonnet").into();317		self.run_in_state(|| {318			self.add_parsed_file(319				std_path.clone(),320				STDLIB_STR.to_owned().into(),321				builtin::get_parsed_stdlib(),322			)323			.unwrap();324			let val = self.evaluate_loaded_file_raw(&std_path).unwrap();325			self.settings_mut().globals.insert("std".into(), val);326		});327		self328	}329330	/// Creates context with all passed global variables331	pub fn create_default_context(&self) -> Context {332		let globals = &self.settings().globals;333		let mut new_bindings = GcHashMap::with_capacity(globals.len());334		for (name, value) in globals.iter() {335			new_bindings.insert(name.clone(), LazyVal::new_resolved(value.clone()));336		}337		Context::new().extend_bound(new_bindings)338	}339340	/// Executes code creating a new stack frame341	pub fn push<T>(342		&self,343		e: Option<&ExprLocation>,344		frame_desc: impl FnOnce() -> String,345		f: impl FnOnce() -> Result<T>,346	) -> Result<T> {347		{348			let mut data = self.data_mut();349			let stack_depth = &mut data.stack_depth;350			if *stack_depth > self.max_stack() {351				// Error creation uses data, so i drop guard here352				drop(data);353				throw!(StackOverflow);354			} else {355				*stack_depth += 1;356			}357		}358		let result = f();359		{360			let mut data = self.data_mut();361			data.stack_depth -= 1;362			data.stack_generation += 1;363		}364		if let Err(mut err) = result {365			err.trace_mut().0.push(StackTraceElement {366				location: e.cloned(),367				desc: frame_desc(),368			});369			return Err(err);370		}371		result372	}373374	/// Executes code creating a new stack frame375	pub fn push_val(376		&self,377		e: &ExprLocation,378		frame_desc: impl FnOnce() -> String,379		f: impl FnOnce() -> Result<Val>,380	) -> Result<Val> {381		{382			let mut data = self.data_mut();383			let stack_depth = &mut data.stack_depth;384			if *stack_depth > self.max_stack() {385				// Error creation uses data, so i drop guard here386				drop(data);387				throw!(StackOverflow);388			} else {389				*stack_depth += 1;390			}391		}392		let mut result = f();393		{394			let mut data = self.data_mut();395			data.stack_depth -= 1;396			data.stack_generation += 1;397			result = data398				.breakpoints399				.insert(data.stack_depth, data.stack_generation, e, result);400		}401		if let Err(mut err) = result {402			err.trace_mut().0.push(StackTraceElement {403				location: Some(e.clone()),404				desc: frame_desc(),405			});406			return Err(err);407		}408		result409	}410	/// Executes code creating a new stack frame411	pub fn push_description<T>(412		&self,413		frame_desc: impl FnOnce() -> String,414		f: impl FnOnce() -> Result<T>,415	) -> Result<T> {416		{417			let mut data = self.data_mut();418			let stack_depth = &mut data.stack_depth;419			if *stack_depth > self.max_stack() {420				// Error creation uses data, so i drop guard here421				drop(data);422				throw!(StackOverflow);423			} else {424				*stack_depth += 1;425			}426		}427		let result = f();428		{429			let mut data = self.data_mut();430			data.stack_depth -= 1;431			data.stack_generation += 1;432		}433		if let Err(mut err) = result {434			err.trace_mut().0.push(StackTraceElement {435				location: None,436				desc: frame_desc(),437			});438			return Err(err);439		}440		result441	}442443	/// Runs passed function in state (required if function needs to modify stack trace)444	pub fn run_in_state<T>(&self, f: impl FnOnce() -> T) -> T {445		EVAL_STATE.with(|v| {446			let has_state = v.borrow().is_some();447			if !has_state {448				v.borrow_mut().replace(self.clone());449			}450			let result = f();451			if !has_state {452				v.borrow_mut().take();453			}454			result455		})456	}457	pub fn run_in_state_with_breakpoint(458		&self,459		bp: Rc<Breakpoint>,460		f: impl FnOnce() -> Result<()>,461	) -> Result<()> {462		{463			let mut data = self.data_mut();464			data.breakpoints.0.push(bp);465		}466467		let result = self.run_in_state(f);468469		{470			let mut data = self.data_mut();471			data.breakpoints.0.pop();472		}473474		result475	}476477	pub fn stringify_err(&self, e: &LocError) -> String {478		let mut out = String::new();479		self.settings()480			.trace_format481			.write_trace(&mut out, self, e)482			.unwrap();483		out484	}485486	pub fn manifest(&self, val: Val) -> Result<IStr> {487		self.run_in_state(|| {488			push_description_frame(489				|| "manifestification".to_string(),490				|| val.manifest(&self.manifest_format()),491			)492		})493	}494	pub fn manifest_multi(&self, val: Val) -> Result<Vec<(IStr, IStr)>> {495		self.run_in_state(|| val.manifest_multi(&self.manifest_format()))496	}497	pub fn manifest_stream(&self, val: Val) -> Result<Vec<IStr>> {498		self.run_in_state(|| val.manifest_stream(&self.manifest_format()))499	}500501	/// If passed value is function then call with set TLA502	pub fn with_tla(&self, val: Val) -> Result<Val> {503		self.run_in_state(|| {504			Ok(match val {505				Val::Func(func) => push_description_frame(506					|| "during TLA call".to_owned(),507					|| {508						func.evaluate(509							self.create_default_context(),510							None,511							&self.settings().tla_vars,512							true,513						)514					},515				)?,516				v => v,517			})518		})519	}520}521522/// Internals523impl EvaluationState {524	fn data(&self) -> Ref<EvaluationData> {525		self.0.data.borrow()526	}527	fn data_mut(&self) -> RefMut<EvaluationData> {528		self.0.data.borrow_mut()529	}530	pub fn settings(&self) -> Ref<EvaluationSettings> {531		self.0.settings.borrow()532	}533	pub fn settings_mut(&self) -> RefMut<EvaluationSettings> {534		self.0.settings.borrow_mut()535	}536}537538/// Raw methods evaluate passed values but don't perform TLA execution539impl EvaluationState {540	pub fn evaluate_file_raw(&self, name: &Path) -> Result<Val> {541		self.run_in_state(|| self.import_file(&std::env::current_dir().expect("cwd"), name))542	}543	pub fn evaluate_file_raw_nocwd(&self, name: &Path) -> Result<Val> {544		self.run_in_state(|| self.import_file(&PathBuf::from("."), name))545	}546	/// Parses and evaluates the given snippet547	pub fn evaluate_snippet_raw(&self, source: Rc<Path>, code: IStr) -> Result<Val> {548		let parsed = parse(549			&code,550			&ParserSettings {551				file_name: source.clone(),552			},553		)554		.map_err(|e| ImportSyntaxError {555			path: source.clone(),556			source_code: code.clone(),557			error: Box::new(e),558		})?;559		self.add_parsed_file(source, code, parsed.clone())?;560		self.evaluate_expr_raw(parsed)561	}562	/// Evaluates the parsed expression563	pub fn evaluate_expr_raw(&self, code: LocExpr) -> Result<Val> {564		self.run_in_state(|| evaluate(self.create_default_context(), &code))565	}566}567568/// Settings utilities569impl EvaluationState {570	pub fn add_ext_var(&self, name: IStr, value: Val) {571		self.settings_mut().ext_vars.insert(name, value);572	}573	pub fn add_ext_str(&self, name: IStr, value: IStr) {574		self.add_ext_var(name, Val::Str(value));575	}576	pub fn add_ext_code(&self, name: IStr, code: IStr) -> Result<()> {577		let value =578			self.evaluate_snippet_raw(PathBuf::from(format!("ext_code {}", name)).into(), code)?;579		self.add_ext_var(name, value);580		Ok(())581	}582583	pub fn add_tla(&self, name: IStr, value: Val) {584		self.settings_mut()585			.tla_vars586			.insert(name, TlaArg::Val(value));587	}588	pub fn add_tla_str(&self, name: IStr, value: IStr) {589		self.settings_mut()590			.tla_vars591			.insert(name, TlaArg::String(value));592	}593	pub fn add_tla_code(&self, name: IStr, code: IStr) -> Result<()> {594		let parsed = self.add_file(PathBuf::from(format!("tla_code {}", name)).into(), code)?;595		self.settings_mut()596			.tla_vars597			.insert(name, TlaArg::Code(parsed));598		Ok(())599	}600601	pub fn resolve_file(&self, from: &Path, path: &Path) -> Result<Rc<Path>> {602		self.settings().import_resolver.resolve_file(from, path)603	}604	pub fn load_file_contents(&self, path: &Path) -> Result<IStr> {605		self.settings().import_resolver.load_file_contents(path)606	}607608	pub fn import_resolver(&self) -> Ref<dyn ImportResolver> {609		Ref::map(self.settings(), |s| &*s.import_resolver)610	}611	pub fn set_import_resolver(&self, resolver: Box<dyn ImportResolver>) {612		self.settings_mut().import_resolver = resolver;613	}614615	pub fn add_native(&self, name: IStr, cb: Cc<TraceBox<dyn Builtin>>) {616		self.settings_mut().ext_natives.insert(name, cb);617	}618619	pub fn manifest_format(&self) -> ManifestFormat {620		self.settings().manifest_format.clone()621	}622	pub fn set_manifest_format(&self, format: ManifestFormat) {623		self.settings_mut().manifest_format = format;624	}625626	pub fn trace_format(&self) -> Ref<dyn TraceFormat> {627		Ref::map(self.settings(), |s| &*s.trace_format)628	}629	pub fn set_trace_format(&self, format: Box<dyn TraceFormat>) {630		self.settings_mut().trace_format = format;631	}632633	pub fn max_trace(&self) -> usize {634		self.settings().max_trace635	}636	pub fn set_max_trace(&self, trace: usize) {637		self.settings_mut().max_trace = trace;638	}639640	pub fn max_stack(&self) -> usize {641		self.settings().max_stack642	}643	pub fn set_max_stack(&self, trace: usize) {644		self.settings_mut().max_stack = trace;645	}646}647648pub fn cc_ptr_eq<T>(a: &Cc<T>, b: &Cc<T>) -> bool {649	let a = a as &T;650	let b = b as &T;651	std::ptr::eq(a, b)652}653654#[cfg(test)]655pub mod tests {656	use super::Val;657	use crate::{658		error::Error::*, function::BuiltinParam, gc::TraceBox, native::NativeCallbackHandler,659		primitive_equals, EvaluationState,660	};661	use gcmodule::{Cc, Trace};662	use jrsonnet_interner::IStr;663	use jrsonnet_parser::*;664	use std::{665		path::{Path, PathBuf},666		rc::Rc,667	};668669	#[test]670	#[should_panic]671	fn eval_state_stacktrace() {672		let state = EvaluationState::default();673		state.run_in_state(|| {674			state675				.push(676					Some(&ExprLocation(PathBuf::from("test1.jsonnet").into(), 10, 20)),677					|| "outer".to_owned(),678					|| {679						state.push(680							Some(&ExprLocation(PathBuf::from("test2.jsonnet").into(), 30, 40)),681							|| "inner".to_owned(),682							|| Err(RuntimeError("".into()).into()),683						)?;684						Ok(Val::Null)685					},686				)687				.unwrap();688		});689	}690691	#[test]692	fn eval_state_standard() {693		let state = EvaluationState::default();694		state.with_stdlib();695		assert!(primitive_equals(696			&state697				.evaluate_snippet_raw(698					PathBuf::from("raw.jsonnet").into(),699					r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#.into()700				)701				.unwrap(),702			&Val::Bool(true),703		)704		.unwrap());705	}706707	macro_rules! eval {708		($str: expr) => {709			EvaluationState::default()710				.with_stdlib()711				.evaluate_snippet_raw(PathBuf::from("raw.jsonnet").into(), $str.into())712				.unwrap()713		};714	}715	macro_rules! eval_json {716		($str: expr) => {{717			let evaluator = EvaluationState::default();718			evaluator.with_stdlib();719			evaluator.run_in_state(|| {720				evaluator721					.evaluate_snippet_raw(PathBuf::from("raw.jsonnet").into(), $str.into())722					.unwrap()723					.to_json(0)724					.unwrap()725					.replace("\n", "")726			})727		}};728	}729730	/// Asserts given code returns `true`731	macro_rules! assert_eval {732		($str: expr) => {733			assert!(primitive_equals(&eval!($str), &Val::Bool(true)).unwrap())734		};735	}736737	/// Asserts given code returns `false`738	macro_rules! assert_eval_neg {739		($str: expr) => {740			assert!(primitive_equals(&eval!($str), &Val::Bool(false)).unwrap())741		};742	}743	macro_rules! assert_json {744		($str: expr, $out: expr) => {745			assert_eq!(eval_json!($str), $out.replace("\t", ""))746		};747	}748749	/// Sanity checking, before trusting to another tests750	#[test]751	fn equality_operator() {752		assert_eval!("2 == 2");753		assert_eval_neg!("2 != 2");754		assert_eval!("2 != 3");755		assert_eval_neg!("2 == 3");756		assert_eval!("'Hello' == 'Hello'");757		assert_eval_neg!("'Hello' != 'Hello'");758		assert_eval!("'Hello' != 'World'");759		assert_eval_neg!("'Hello' == 'World'");760	}761762	#[test]763	fn math_evaluation() {764		assert_eval!("2 + 2 * 2 == 6");765		assert_eval!("3 + (2 + 2 * 2) == 9");766	}767768	#[test]769	fn string_concat() {770		assert_eval!("'Hello' + 'World' == 'HelloWorld'");771		assert_eval!("'Hello' * 3 == 'HelloHelloHello'");772		assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'");773	}774775	#[test]776	fn faster_join() {777		assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]");778		assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'");779	}780781	#[test]782	fn function_contexts() {783		assert_eval!(784			r#"785				local k = {786					t(name = self.h): [self.h, name],787					h: 3,788				};789				local f = {790					t: k.t(),791					h: 4,792				};793				f.t[0] == f.t[1]794			"#795		);796	}797798	#[test]799	fn local() {800		assert_eval!("local a = 2; local b = 3; a + b == 5");801		assert_eval!("local a = 1, b = a + 1; a + b == 3");802		assert_eval!("local a = 1; local a = 2; a == 2");803	}804805	#[test]806	fn object_lazyness() {807		assert_json!("local a = {a:error 'test'}; {}", r#"{}"#);808	}809810	#[test]811	fn object_inheritance() {812		assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#);813	}814815	#[test]816	fn object_assertion_success() {817		eval!("{assert \"a\" in self} + {a:2}");818	}819820	#[test]821	fn object_assertion_error() {822		eval!("{assert \"a\" in self}");823	}824825	#[test]826	fn lazy_args() {827		eval!("local test(a) = 2; test(error '3')");828	}829830	#[test]831	#[should_panic]832	fn tailstrict_args() {833		eval!("local test(a) = 2; test(error '3') tailstrict");834	}835836	#[test]837	#[should_panic]838	fn no_binding_error() {839		eval!("a");840	}841842	#[test]843	fn test_object() {844		assert_json!("{a:2}", r#"{"a": 2}"#);845		assert_json!("{a:2+2}", r#"{"a": 4}"#);846		assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#);847		assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#);848		assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#);849		assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#);850		assert_json!(851			r#"852				{853					name: "Alice",854					welcome: "Hello " + self.name + "!",855				}856			"#,857			r#"{"name": "Alice","welcome": "Hello Alice!"}"#858		);859		assert_json!(860			r#"861				{862					name: "Alice",863					welcome: "Hello " + self.name + "!",864				} + {865					name: "Bob"866				}867			"#,868			r#"{"name": "Bob","welcome": "Hello Bob!"}"#869		);870	}871872	#[test]873	fn functions() {874		assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4");875		assert_json!(876			r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#,877			r#""HelloDearWorld""#878		);879	}880881	#[test]882	fn local_methods() {883		assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4");884		assert_json!(885			r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#,886			r#""HelloDearWorld""#887		);888	}889890	#[test]891	fn object_locals() {892		assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#);893		assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#);894		assert_json!(895			r#"{local a = function (b) {[b]:4}, test: a("test")}"#,896			r#"{"test": {"test": 4}}"#897		);898	}899900	#[test]901	fn object_comp() {902		assert_json!(903			r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#,904			"{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}"905		)906	}907908	#[test]909	fn direct_self() {910		println!(911			"{:#?}",912			eval!(913				r#"914					{915						local me = self,916						a: 3,917						b(): me.a,918					}919				"#920			)921		);922	}923924	#[test]925	fn indirect_self() {926		// `self` assigned to `me` was lost when being927		// referenced from field928		eval!(929			r#"{930				local me = self,931				a: 3,932				b: me.a,933			}.b"#934		);935	}936937	// We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly938	#[test]939	fn std_assert_ok() {940		eval!("std.assertEqual(4.5 << 2, 16)");941	}942943	#[test]944	#[should_panic]945	fn std_assert_failure() {946		eval!("std.assertEqual(4.5 << 2, 15)");947	}948949	#[test]950	fn string_is_string() {951		assert!(primitive_equals(952			&eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"),953			&Val::Bool(false),954		)955		.unwrap());956	}957958	#[test]959	fn base64_works() {960		assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#);961	}962963	#[test]964	fn utf8_chars() {965		assert_json!(966			r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#,967			r#"{"c": 128526,"l": 1}"#968		)969	}970971	#[test]972	fn json() {973		assert_json!(974			r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#,975			r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""#976		);977	}978979	#[test]980	fn json_minified() {981		assert_json!(982			r#"std.manifestJsonMinified({a:3, b:4, c:6})"#,983			r#""{\"a\":3,\"b\":4,\"c\":6}""#984		);985	}986987	#[test]988	fn parse_json() {989		assert_json!(990			r#"std.parseJson('{"a": -1,"b": 1,"c": 3.141,"d": []}')"#,991			r#"{"a": -1,"b": 1,"c": 3.141,"d": []}"#992		);993	}994995	#[test]996	fn test() {997		assert_json!(998			r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#,999			"[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]"1000		);1001	}10021003	#[test]1004	fn sjsonnet() {1005		eval!(1006			r#"1007			local x0 = {k: 1};1008			local x1 = {k: x0.k + x0.k};1009			local x2 = {k: x1.k + x1.k};1010			local x3 = {k: x2.k + x2.k};1011			local x4 = {k: x3.k + x3.k};1012			local x5 = {k: x4.k + x4.k};1013			local x6 = {k: x5.k + x5.k};1014			local x7 = {k: x6.k + x6.k};1015			local x8 = {k: x7.k + x7.k};1016			local x9 = {k: x8.k + x8.k};1017			local x10 = {k: x9.k + x9.k};1018			local x11 = {k: x10.k + x10.k};1019			local x12 = {k: x11.k + x11.k};1020			local x13 = {k: x12.k + x12.k};1021			local x14 = {k: x13.k + x13.k};1022			local x15 = {k: x14.k + x14.k};1023			local x16 = {k: x15.k + x15.k};1024			local x17 = {k: x16.k + x16.k};1025			local x18 = {k: x17.k + x17.k};1026			local x19 = {k: x18.k + x18.k};1027			local x20 = {k: x19.k + x19.k};1028			local x21 = {k: x20.k + x20.k};1029			x21.k1030		"#1031		);1032	}10331034	// This test is commented out by default, because of huge compilation slowdown1035	// #[bench]1036	// fn bench_codegen(b: &mut Bencher) {1037	// 	b.iter(|| {1038	// 		#[allow(clippy::all)]1039	// 		let stdlib = {1040	// 			use jrsonnet_parser::*;1041	// 			include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))1042	// 		};1043	// 		stdlib1044	// 	})1045	// }10461047	/*1048	#[bench]1049	fn bench_serialize(b: &mut Bencher) {1050		b.iter(|| {1051			bincode::deserialize::<jrsonnet_parser::LocExpr>(include_bytes!(concat!(1052				env!("OUT_DIR"),1053				"/stdlib.bincode"1054			)))1055			.expect("deserialize stdlib")1056		})1057	}10581059	#[bench]1060	fn bench_parse(b: &mut Bencher) {1061		b.iter(|| {1062			jrsonnet_parser::parse(1063				jrsonnet_stdlib::STDLIB_STR,1064				&jrsonnet_parser::ParserSettings {1065					loc_data: true,1066					file_name: Rc::new(PathBuf::from("std.jsonnet")),1067				},1068			)1069		})1070	}1071	*/10721073	#[test]1074	fn equality() {1075		println!(1076			"{:?}",1077			jrsonnet_parser::parse(1078				"{ x: 1, y: 2 } == { x: 1, y: 2 }",1079				&ParserSettings {1080					file_name: PathBuf::from("equality").into(),1081				}1082			)1083		);1084		assert_eval!("{ x: 1, y: 2 } == { x: 1, y: 2 }")1085	}10861087	#[test]1088	fn native_ext() -> crate::error::Result<()> {1089		use super::native::NativeCallback;1090		let evaluator = EvaluationState::default();10911092		evaluator.with_stdlib();10931094		#[derive(Trace)]1095		struct NativeAdd;1096		impl NativeCallbackHandler for NativeAdd {1097			fn call(&self, from: Option<Rc<Path>>, args: &[Val]) -> crate::error::Result<Val> {1098				assert_eq!(1099					&from.unwrap() as &Path,1100					&PathBuf::from("native_caller.jsonnet")1101				);1102				match (&args[0], &args[1]) {1103					(Val::Num(a), Val::Num(b)) => Ok(Val::Num(a + b)),1104					(_, _) => unreachable!(),1105				}1106			}1107		}1108		evaluator.settings_mut().ext_natives.insert(1109			"native_add".into(),1110			#[allow(deprecated)]1111			Cc::new(TraceBox(Box::new(NativeCallback::new(1112				vec![1113					BuiltinParam {1114						name: "a".into(),1115						has_default: false,1116					},1117					BuiltinParam {1118						name: "b".into(),1119						has_default: false,1120					},1121				],1122				TraceBox(Box::new(NativeAdd)),1123			)))),1124		);1125		evaluator.evaluate_snippet_raw(1126			PathBuf::from("native_caller.jsonnet").into(),1127			"std.assertEqual(std.native(\"native_add\")(1, 2), 3)".into(),1128		)?;1129		Ok(())1130	}11311132	#[test]1133	fn constant_intrinsic() -> crate::error::Result<()> {1134		assert_eval!(1135			"local std2 = std; local std = std2 { primitiveEquals(a, b):: false }; 1 == 1"1136		);1137		Ok(())1138	}11391140	#[test]1141	fn standalone_super() -> crate::error::Result<()> {1142		assert_eval!(1143			r#"1144			local obj = {1145				a: 1,1146				b: 2,1147				c: 3,1148			};1149			local test = obj + {1150				fields: std.objectFields(super),1151				d: 5,1152			};1153			test.fields == ['a', 'b', 'c']1154		"#1155		);1156		Ok(())1157	}11581159	#[test]1160	fn comp_self() -> crate::error::Result<()> {1161		assert_eval!(1162			r#"1163			std.objectFields({1164				a:{1165					[name]: name for name in std.objectFields(self)1166				},1167				b: 2,1168				c: 3,1169			}.a) == ['a', 'b', 'c']1170			"#1171		);11721173		Ok(())1174	}11751176	struct TestImportResolver(IStr);1177	impl crate::import::ImportResolver for TestImportResolver {1178		fn resolve_file(&self, _: &Path, _: &Path) -> crate::error::Result<Rc<Path>> {1179			Ok(PathBuf::from("/test").into())1180		}11811182		fn load_file_contents(&self, _: &Path) -> crate::error::Result<IStr> {1183			Ok(self.0.clone())1184		}11851186		unsafe fn as_any(&self) -> &dyn std::any::Any {1187			panic!()1188		}1189	}11901191	#[test]1192	fn issue_23() {1193		let state = EvaluationState::default();1194		state.set_import_resolver(Box::new(TestImportResolver(r#"import "/test""#.into())));1195		let _ = state.evaluate_file_raw(&PathBuf::from("/test"));1196	}11971198	#[test]1199	fn issue_40() {1200		let state = EvaluationState::default();1201		state.with_stdlib();12021203		let error = state1204			.evaluate_snippet_raw(1205				PathBuf::from("issue40.jsonnet").into(),1206				r#"1207				local conf = {1208					n: ""1209				};12101211				local result = conf + {1212					assert std.isNumber(self.n): "is number"1213				};12141215				std.manifestJsonEx(result, "")1216			"#1217				.into(),1218			)1219			.unwrap_err();1220		assert_eq!(error.error().to_string(), "assert failed: is number");1221	}12221223	#[test]1224	fn test_ascii_upper_lower() {1225		assert_eval!(r#"std.assertEqual(std.asciiUpper("aBc😀"), "ABC😀")"#);1226		assert_eval!(r#"std.assertEqual(std.asciiLower("aBc😀"), "abc😀")"#);1227	}12281229	#[test]1230	fn test_member() {1231		assert_eval!(r#"!std.member("", "")"#);1232		assert_eval!(r#"std.member("abc", "a")"#);1233		assert_eval!(r#"!std.member("abc", "d")"#);1234		assert_eval!(r#"!std.member([], "")"#);1235		assert_eval!(r#"std.member(["a", "b", "c"], "a")"#);1236		assert_eval!(r#"!std.member(["a", "b", "c"], "d")"#);1237	}12381239	#[test]1240	fn test_count() {1241		assert_eval!(r#"std.assertEqual(std.count([], ""), 0)"#);1242		assert_eval!(r#"std.assertEqual(std.count(["a", "b", "a"], "d"), 0)"#);1243		assert_eval!(r#"std.assertEqual(std.count(["a", "b", "a"], "a"), 2)"#);1244	}1245}
modifiedcrates/jrsonnet-evaluator/src/native.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/native.rs
+++ b/crates/jrsonnet-evaluator/src/native.rs
@@ -1,33 +1,52 @@
 #![allow(clippy::type_complexity)]
 
+use crate::function::{parse_builtin_call, ArgsLike, Builtin, BuiltinParam};
 use crate::gc::TraceBox;
+use crate::Context;
 use crate::{error::Result, Val};
 use gcmodule::Trace;
-use jrsonnet_parser::ParamsDesc;
-use std::fmt::Debug;
+use jrsonnet_parser::ExprLocation;
 use std::path::Path;
 use std::rc::Rc;
 
-#[deprecated(note = "Use builtins instead")]
-pub trait NativeCallbackHandler: Trace {
-	fn call(&self, from: Rc<Path>, args: &[Val]) -> Result<Val>;
-}
-
 #[derive(Trace)]
 pub struct NativeCallback {
-	pub params: ParamsDesc,
+	pub(crate) params: Vec<BuiltinParam>,
 	handler: TraceBox<dyn NativeCallbackHandler>,
 }
 impl NativeCallback {
-	pub fn new(params: ParamsDesc, handler: TraceBox<dyn NativeCallbackHandler>) -> Self {
+	#[deprecated = "prefer using builtins directly, use this interface only for bindings"]
+	pub fn new(params: Vec<BuiltinParam>, handler: TraceBox<dyn NativeCallbackHandler>) -> Self {
 		Self { params, handler }
-	}
-	pub fn call(&self, caller: Rc<Path>, args: &[Val]) -> Result<Val> {
-		self.handler.call(caller, args)
 	}
 }
-impl Debug for NativeCallback {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		f.debug_struct("NativeCallback").finish()
+
+impl Builtin for NativeCallback {
+	fn name(&self) -> &str {
+		// TODO: standard natives gets their names from definition
+		// But builitins should already have them
+		"<native>"
+	}
+
+	fn params(&self) -> &[BuiltinParam] {
+		&self.params
+	}
+
+	fn call(
+		&self,
+		context: Context,
+		loc: Option<&ExprLocation>,
+		args: &dyn ArgsLike,
+	) -> Result<Val> {
+		let args = parse_builtin_call(context, &self.params, args, true)?;
+		let mut out_args = Vec::with_capacity(self.params.len());
+		for p in self.params.iter() {
+			out_args.push(args[&p.name].evaluate()?);
+		}
+		self.handler.call(loc.map(|l| l.0.clone()), &out_args)
 	}
 }
+
+pub trait NativeCallbackHandler: Trace {
+	fn call(&self, from: Option<Rc<Path>>, args: &[Val]) -> Result<Val>;
+}
modifiedcrates/jrsonnet-evaluator/src/typed/conversions.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/typed/conversions.rs
+++ b/crates/jrsonnet-evaluator/src/typed/conversions.rs
@@ -1,6 +1,5 @@
 use std::convert::{TryFrom, TryInto};
 
-use gcmodule::Cc;
 use jrsonnet_interner::IStr;
 use jrsonnet_types::{ComplexValType, ValType};
 
@@ -400,10 +399,10 @@
 	}
 }
 
-impl Typed for Cc<FuncVal> {
+impl Typed for FuncVal {
 	const TYPE: &'static ComplexValType = &ComplexValType::Simple(ValType::Func);
 }
-impl TryFrom<Val> for Cc<FuncVal> {
+impl TryFrom<Val> for FuncVal {
 	type Error = LocError;
 
 	fn try_from(value: Val) -> Result<Self> {
@@ -414,10 +413,10 @@
 		}
 	}
 }
-impl TryFrom<Cc<FuncVal>> for Val {
+impl TryFrom<FuncVal> for Val {
 	type Error = LocError;
 
-	fn try_from(value: Cc<FuncVal>) -> Result<Self> {
+	fn try_from(value: FuncVal) -> Result<Self> {
 		Ok(Self::Func(value))
 	}
 }
modifiedcrates/jrsonnet-evaluator/src/val.rsdiffbeforeafterboth
--- a/crates/jrsonnet-evaluator/src/val.rs
+++ b/crates/jrsonnet-evaluator/src/val.rs
@@ -7,7 +7,6 @@
 	evaluate,
 	function::{parse_function_call, ArgsLike, Builtin, StaticBuiltin},
 	gc::TraceBox,
-	native::NativeCallback,
 	throw, Context, ObjValue, Result,
 };
 use gcmodule::{Cc, Trace};
@@ -86,16 +85,14 @@
 	pub body: LocExpr,
 }
 
-#[derive(Trace)]
+#[derive(Trace, Clone)]
 pub enum FuncVal {
 	/// Plain function implemented in jsonnet
-	Normal(FuncDesc),
+	Normal(Cc<FuncDesc>),
 	/// Standard library function
 	StaticBuiltin(#[skip_trace] &'static dyn StaticBuiltin),
 
-	Builtin(TraceBox<dyn Builtin>),
-	/// Library functions implemented in native
-	NativeExt(IStr, Cc<NativeCallback>),
+	Builtin(Cc<TraceBox<dyn Builtin>>),
 }
 
 impl Debug for FuncVal {
@@ -104,9 +101,6 @@
 			Self::Normal(arg0) => f.debug_tuple("Normal").field(arg0).finish(),
 			Self::StaticBuiltin(arg0) => f.debug_tuple("Intrinsic").field(&arg0.name()).finish(),
 			Self::Builtin(arg0) => f.debug_tuple("Intrinsic").field(&arg0.name()).finish(),
-			Self::NativeExt(arg0, arg1) => {
-				f.debug_tuple("NativeExt").field(arg0).field(arg1).finish()
-			}
 		}
 	}
 }
@@ -116,7 +110,6 @@
 		match (self, other) {
 			(Self::Normal(a), Self::Normal(b)) => a == b,
 			(Self::StaticBuiltin(an), Self::StaticBuiltin(bn)) => std::ptr::eq(*an, *bn),
-			(Self::NativeExt(an, _), Self::NativeExt(bn, _)) => an == bn,
 			(..) => false,
 		}
 	}
@@ -127,7 +120,6 @@
 			Self::Normal(n) => n.params.iter().filter(|p| p.1.is_none()).count(),
 			Self::StaticBuiltin(i) => i.params().iter().filter(|p| !p.has_default).count(),
 			Self::Builtin(i) => i.params().iter().filter(|p| !p.has_default).count(),
-			Self::NativeExt(_, n) => n.params.iter().filter(|p| p.1.is_none()).count(),
 		}
 	}
 	pub fn name(&self) -> IStr {
@@ -135,7 +127,6 @@
 			Self::Normal(normal) => normal.name.clone(),
 			Self::StaticBuiltin(builtin) => builtin.name().into(),
 			Self::Builtin(builtin) => builtin.name().into(),
-			Self::NativeExt(n, _) => format!("native.{}", n).into(),
 		}
 	}
 	pub fn evaluate(
@@ -158,15 +149,6 @@
 			}
 			Self::StaticBuiltin(name) => name.call(call_ctx, loc, args),
 			Self::Builtin(b) => b.call(call_ctx, loc, args),
-			Self::NativeExt(_name, handler) => {
-				let args =
-					parse_function_call(call_ctx, Context::new(), &handler.params, args, true)?;
-				let mut out_args = Vec::with_capacity(handler.params.len());
-				for p in handler.params.0.iter() {
-					out_args.push(args.binding(p.0.clone())?.evaluate()?);
-				}
-				Ok(handler.call(loc.expect("todo").0.clone(), &out_args)?)
-			}
 		}
 	}
 	pub fn evaluate_simple(&self, args: &dyn ArgsLike) -> Result<Val> {
@@ -352,7 +334,7 @@
 	Num(f64),
 	Arr(ArrValue),
 	Obj(ObjValue),
-	Func(Cc<FuncVal>),
+	Func(FuncVal),
 }
 
 impl Val {