git.delta.rocks / jrsonnet / refs/commits / 433adfa9b8ae

1#![feature(box_syntax, box_patterns)]2#![feature(type_alias_impl_trait)]3#![feature(debug_non_exhaustive)]4#![feature(test)]5#![feature(stmt_expr_attributes)]6#![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)]78extern crate test;910mod builtin;11mod ctx;12mod dynamic;13mod error;14mod evaluate;15mod function;16mod import;17mod map;18mod obj;19pub mod trace;20mod val;2122pub use ctx::*;23pub use dynamic::*;24pub use error::*;25pub use evaluate::*;26pub use function::parse_function_call;27pub use import::*;28use jrsonnet_parser::*;29pub use obj::*;30use std::{31	cell::{Ref, RefCell, RefMut},32	collections::HashMap,33	fmt::Debug,34	path::PathBuf,35	rc::Rc,36};37use trace::{offset_to_location, CodeLocation};38pub use val::*;3940type BindableFn = dyn Fn(Option<ObjValue>, Option<ObjValue>) -> Result<LazyVal>;41#[derive(Clone)]42pub enum LazyBinding {43	Bindable(Rc<BindableFn>),44	Bound(LazyVal),45}4647impl Debug for LazyBinding {48	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {49		write!(f, "LazyBinding")50	}51}52impl LazyBinding {53	pub fn evaluate(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal> {54		match self {55			LazyBinding::Bindable(v) => v(this, super_obj),56			LazyBinding::Bound(v) => Ok(v.clone()),57		}58	}59}6061pub struct EvaluationSettings {62	/// Limits recursion by limiting stack frames63	pub max_stack: usize,64	/// Limit amount of stack trace items preserved65	pub max_trace: usize,66	/// Used for std.extVar67	pub ext_vars: HashMap<Rc<str>, Val>,68	/// Global variables are inserted in default context69	pub globals: HashMap<Rc<str>, Val>,70	/// Used to resolve file locations/contents71	pub import_resolver: Box<dyn ImportResolver>,72}73impl Default for EvaluationSettings {74	fn default() -> Self {75		EvaluationSettings {76			max_stack: 200,77			max_trace: 20,78			globals: Default::default(),79			ext_vars: Default::default(),80			import_resolver: Box::new(DummyImportResolver),81		}82	}83}8485#[derive(Default)]86struct EvaluationData {87	/// Used for stack overflow detection, stacktrace is now populated on unwind88	stack_depth: usize,89	/// Contains file source codes and evaluated results for imports and pretty90	/// printing stacktraces91	files: HashMap<Rc<PathBuf>, FileData>,92	str_files: HashMap<Rc<PathBuf>, Rc<str>>,93}9495pub struct FileData {96	source_code: Rc<str>,97	parsed: LocExpr,98	evaluated: Option<Val>,99}100#[derive(Default)]101pub struct EvaluationStateInternals {102	/// Internal state103	data: RefCell<EvaluationData>,104	/// Settings, safe to change at runtime105	settings: RefCell<EvaluationSettings>,106}107108thread_local! {109	/// Contains state for currently executing file110	/// Global state is fine there111	pub(crate) static EVAL_STATE: RefCell<Option<EvaluationState>> = RefCell::new(None)112}113pub(crate) fn with_state<T>(f: impl FnOnce(&EvaluationState) -> T) -> T {114	EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap()))115}116pub fn create_error(err: Error) -> LocError {117	with_state(|s| s.error(err))118}119pub fn create_error_result<T>(err: Error) -> Result<T> {120	Err(with_state(|s| s.error(err)))121}122pub(crate) fn push<T>(123	e: &Option<ExprLocation>,124	frame_desc: impl FnOnce() -> String,125	f: impl FnOnce() -> Result<T>,126) -> Result<T> {127	if let Some(v) = e {128		with_state(|s| s.push(&v, frame_desc, f))129	} else {130		f()131	}132}133134/// Maintains stack trace and import resolution135#[derive(Default, Clone)]136pub struct EvaluationState(Rc<EvaluationStateInternals>);137impl EvaluationState {138	fn data(&self) -> Ref<EvaluationData> {139		self.0.data.borrow()140	}141	fn data_mut(&self) -> RefMut<EvaluationData> {142		self.0.data.borrow_mut()143	}144	pub fn settings(&self) -> Ref<EvaluationSettings> {145		self.0.settings.borrow()146	}147	pub fn settings_mut(&self) -> RefMut<EvaluationSettings> {148		self.0.settings.borrow_mut()149	}150151	pub fn evaluate_file_to_json(&self, path: &PathBuf) -> std::result::Result<Rc<str>, LocError> {152		self.import_file(&PathBuf::new(), &path)153			.and_then(|v| v.into_json(4))154	}155	pub fn evaluate_snippet_to_json(156		&self,157		path: &PathBuf,158		snippet: &str,159	) -> std::result::Result<Rc<str>, LocError> {160		self.parse_evaluate_raw(Rc::new(path.clone()), snippet)161			.and_then(|v| v.into_json(4))162	}163164	/// Parses and adds file to loaded165	pub fn add_file(&self, path: Rc<PathBuf>, source_code: Rc<str>) -> Result<()> {166		self.add_parsed_file(167			path.clone(),168			source_code.clone(),169			parse(170				&source_code,171				&ParserSettings {172					file_name: path.clone(),173					loc_data: true,174				},175			)176			.map_err(|error| {177				create_error(Error::ImportSyntaxError {178					error,179					path,180					source_code,181				})182			})?,183		)?;184185		Ok(())186	}187188	/// Adds file by source code and parsed expr189	pub fn add_parsed_file(190		&self,191		name: Rc<PathBuf>,192		source_code: Rc<str>,193		parsed: LocExpr,194	) -> Result<()> {195		self.data_mut().files.insert(196			name,197			FileData {198				source_code,199				parsed,200				evaluated: None,201			},202		);203204		Ok(())205	}206	pub fn get_source(&self, name: &PathBuf) -> Option<Rc<str>> {207		let ro_map = &self.data().files;208		ro_map.get(name).map(|value| value.source_code.clone())209	}210	pub fn map_source_locations(&self, file: &PathBuf, locs: &[usize]) -> Vec<CodeLocation> {211		offset_to_location(&self.get_source(file).unwrap(), locs)212	}213214	pub fn evaluate_file(&self, name: &PathBuf) -> Result<Val> {215		self.run_in_state(|| {216			let expr: LocExpr = {217				let ro_map = &self.data().files;218				let value = ro_map219					.get(name)220					.unwrap_or_else(|| panic!("file not added: {:?}", name));221				if value.evaluated.is_some() {222					return Ok(value.evaluated.clone().unwrap());223				}224				value.parsed.clone()225			};226			let value = evaluate(self.create_default_context()?, &expr)?;227			{228				self.0229					.data230					.borrow_mut()231					.files232					.get_mut(name)233					.unwrap()234					.evaluated235					.replace(value.clone());236			}237			Ok(value)238		})239	}240	pub fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result<Rc<PathBuf>> {241		Ok(self.settings().import_resolver.resolve_file(from, path)?)242	}243	pub fn load_file_contents(&self, path: &PathBuf) -> Result<Rc<str>> {244		Ok(self.settings().import_resolver.load_file_contents(path)?)245	}246	pub(crate) fn import_file(&self, from: &PathBuf, path: &PathBuf) -> Result<Val> {247		let file_path = self.resolve_file(from, path)?;248		{249			let files = &self.data().files;250			if files.contains_key(&file_path) {251				return self.evaluate_file(&file_path);252			}253		}254		let contents = self.load_file_contents(&file_path)?;255		self.add_file(file_path.clone(), contents)?;256		self.evaluate_file(&file_path)257	}258	pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result<Rc<str>> {259		let path = self.resolve_file(from, path)?;260		if !self.data().str_files.contains_key(&path) {261			let file_str = self.load_file_contents(&path)?;262			self.data_mut().str_files.insert(path.clone(), file_str);263		}264		Ok(self.data().str_files.get(&path).cloned().unwrap())265	}266267	/// Parses and evaluates snippet268	pub fn parse_evaluate_raw(&self, source: Rc<PathBuf>, code: &str) -> Result<Val> {269		let parsed = parse(270			&code,271			&ParserSettings {272				file_name: source,273				loc_data: true,274			},275		)276		.unwrap();277		self.evaluate_raw(parsed)278	}279	/// Evaluates parsed expression280	pub fn evaluate_raw(&self, code: LocExpr) -> Result<Val> {281		self.run_in_state(|| evaluate(self.create_default_context()?, &code))282	}283284	/// Adds standard library global variable (std) to this evaluator285	pub fn with_stdlib(&self) -> &Self {286		use jrsonnet_stdlib::STDLIB_STR;287		let std_path = Rc::new(PathBuf::from("std.jsonnet"));288		self.run_in_state(|| {289			self.add_parsed_file(290				std_path.clone(),291				STDLIB_STR.to_owned().into(),292				builtin::get_parsed_stdlib(),293			)294			.unwrap();295			let val = self.evaluate_file(&std_path).unwrap();296			self.settings_mut().globals.insert("std".into(), val);297		});298		self299	}300301	/// Creates context with all passed global variables302	pub fn create_default_context(&self) -> Result<Context> {303		let globals = &self.settings().globals;304		let mut new_bindings: HashMap<Rc<str>, LazyBinding> = HashMap::new();305		for (name, value) in globals.iter() {306			new_bindings.insert(307				name.clone(),308				LazyBinding::Bound(resolved_lazy_val!(value.clone())),309			);310		}311		Context::new().extend_unbound(new_bindings, None, None, None)312	}313314	/// Executes code, creating new stack frame315	pub fn push<T>(316		&self,317		e: &ExprLocation,318		frame_desc: impl FnOnce() -> String,319		f: impl FnOnce() -> Result<T>,320	) -> Result<T> {321		{322			let mut data = self.data_mut();323			let stack_depth = &mut data.stack_depth;324			if *stack_depth > self.settings().max_stack {325				// Error creation uses data, so i drop guard here326				drop(data);327				return Err(self.error(Error::StackOverflow));328			} else {329				*stack_depth += 1;330			}331		}332		let result = f();333		self.data_mut().stack_depth -= 1;334		if let Err(mut err) = result {335			(err.1).0.push(StackTraceElement(e.clone(), frame_desc()));336			return Err(err);337		}338		result339	}340341	/// Creates error with stack trace342	pub fn error(&self, err: Error) -> LocError {343		LocError(err, StackTrace(vec![]))344	}345346	/// Runs passed function in state (required, if function needs to modify stack trace)347	pub fn run_in_state<T>(&self, f: impl FnOnce() -> T) -> T {348		EVAL_STATE.with(|v| {349			let has_state = v.borrow().is_some();350			if !has_state {351				v.borrow_mut().replace(self.clone());352			}353			let result = f();354			if !has_state {355				v.borrow_mut().take();356			}357			result358		})359	}360}361362#[cfg(test)]363pub mod tests {364	use super::Val;365	use crate::{create_error, primitive_equals, EvaluationState};366	use jrsonnet_parser::*;367	use std::{path::PathBuf, rc::Rc};368369	#[test]370	fn eval_state_stacktrace() {371		let state = EvaluationState::default();372		state.run_in_state(|| {373			state374				.push(375					&ExprLocation(Rc::new(PathBuf::from("test1.jsonnet")), 10, 20),376					|| "outer".to_owned(),377					|| {378						state.push(379							&ExprLocation(Rc::new(PathBuf::from("test2.jsonnet")), 30, 40),380							|| "inner".to_owned(),381							|| Err(create_error(crate::error::Error::RuntimeError("".into()))),382						)?;383						Ok(())384					},385				)386				.unwrap();387		});388	}389390	#[test]391	fn eval_state_standard() {392		let state = EvaluationState::default();393		state.with_stdlib();394		assert!(primitive_equals(395			&state396				.parse_evaluate_raw(397					Rc::new(PathBuf::from("raw.jsonnet")),398					r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#399				)400				.unwrap(),401			&Val::Bool(true),402		)403		.unwrap());404	}405406	macro_rules! eval {407		($str: expr) => {408			EvaluationState::default()409				.with_stdlib()410				.parse_evaluate_raw(Rc::new(PathBuf::from("raw.jsonnet")), $str)411				.unwrap()412		};413	}414	macro_rules! eval_json {415		($str: expr) => {{416			let evaluator = EvaluationState::default();417			evaluator.with_stdlib();418			evaluator.run_in_state(|| {419				evaluator420					.parse_evaluate_raw(Rc::new(PathBuf::from("raw.jsonnet")), $str)421					.unwrap()422					.into_json(0)423					.unwrap()424					.replace("\n", "")425				})426			}};427	}428429	/// Asserts given code returns `true`430	macro_rules! assert_eval {431		($str: expr) => {432			assert!(primitive_equals(&eval!($str), &Val::Bool(true)).unwrap())433		};434	}435436	/// Asserts given code returns `false`437	macro_rules! assert_eval_neg {438		($str: expr) => {439			assert!(primitive_equals(&eval!($str), &Val::Bool(false)).unwrap())440		};441	}442	macro_rules! assert_json {443		($str: expr, $out: expr) => {444			assert_eq!(eval_json!($str), $out.replace("\t", ""))445		};446	}447448	/// Sanity checking, before trusting to another tests449	#[test]450	fn equality_operator() {451		assert_eval!("2 == 2");452		assert_eval_neg!("2 != 2");453		assert_eval!("2 != 3");454		assert_eval_neg!("2 == 3");455		assert_eval!("'Hello' == 'Hello'");456		assert_eval_neg!("'Hello' != 'Hello'");457		assert_eval!("'Hello' != 'World'");458		assert_eval_neg!("'Hello' == 'World'");459	}460461	#[test]462	fn math_evaluation() {463		assert_eval!("2 + 2 * 2 == 6");464		assert_eval!("3 + (2 + 2 * 2) == 9");465	}466467	#[test]468	fn string_concat() {469		assert_eval!("'Hello' + 'World' == 'HelloWorld'");470		assert_eval!("'Hello' * 3 == 'HelloHelloHello'");471		assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'");472	}473474	#[test]475	fn faster_join() {476		assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]");477		assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'");478	}479480	#[test]481	fn function_contexts() {482		assert_eval!(483			r#"484				local k = {485					t(name = self.h): [self.h, name],486					h: 3,487				};488				local f = {489					t: k.t(),490					h: 4,491				};492				f.t[0] == f.t[1]493			"#494		);495	}496497	#[test]498	fn local() {499		assert_eval!("local a = 2; local b = 3; a + b == 5");500		assert_eval!("local a = 1, b = a + 1; a + b == 3");501		assert_eval!("local a = 1; local a = 2; a == 2");502	}503504	#[test]505	fn object_lazyness() {506		assert_json!("local a = {a:error 'test'}; {}", r#"{}"#);507	}508509	#[test]510	fn object_inheritance() {511		assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#);512	}513514	#[test]515	fn object_assertion_success() {516		eval!("{assert \"a\" in self} + {a:2}");517	}518519	#[test]520	fn object_assertion_error() {521		eval!("{assert \"a\" in self}");522	}523524	#[test]525	fn lazy_args() {526		eval!("local test(a) = 2; test(error '3')");527	}528529	#[test]530	#[should_panic]531	fn tailstrict_args() {532		eval!("local test(a) = 2; test(error '3') tailstrict");533	}534535	#[test]536	#[should_panic]537	fn no_binding_error() {538		eval!("a");539	}540541	#[test]542	fn test_object() {543		assert_json!("{a:2}", r#"{"a": 2}"#);544		assert_json!("{a:2+2}", r#"{"a": 4}"#);545		assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#);546		assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#);547		assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#);548		assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#);549		assert_json!(550			r#"551				{552					name: "Alice",553					welcome: "Hello " + self.name + "!",554				}555			"#,556			r#"{"name": "Alice","welcome": "Hello Alice!"}"#557		);558		assert_json!(559			r#"560				{561					name: "Alice",562					welcome: "Hello " + self.name + "!",563				} + {564					name: "Bob"565				}566			"#,567			r#"{"name": "Bob","welcome": "Hello Bob!"}"#568		);569	}570571	#[test]572	fn functions() {573		assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4");574		assert_json!(575			r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#,576			r#""HelloDearWorld""#577		);578	}579580	#[test]581	fn local_methods() {582		assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4");583		assert_json!(584			r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#,585			r#""HelloDearWorld""#586		);587	}588589	#[test]590	fn object_locals() {591		assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#);592		assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#);593		assert_json!(594			r#"{local a = function (b) {[b]:4}, test: a("test")}"#,595			r#"{"test": {"test": 4}}"#596		);597	}598599	#[test]600	fn object_comp() {601		assert_json!(602			r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#,603			"{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}"604		)605	}606607	#[test]608	fn direct_self() {609		println!(610			"{:#?}",611			eval!(612				r#"613					{614						local me = self,615						a: 3,616						b(): me.a,617					}618				"#619			)620		);621	}622623	#[test]624	fn indirect_self() {625		// `self` assigned to `me` was lost when being626		// referenced from field627		eval!(628			r#"{629				local me = self,630				a: 3,631				b: me.a,632			}.b"#633		);634	}635636	// We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly637	#[test]638	fn std_assert_ok() {639		eval!("std.assertEqual(4.5 << 2, 16)");640	}641642	#[test]643	#[should_panic]644	fn std_assert_failure() {645		eval!("std.assertEqual(4.5 << 2, 15)");646	}647648	#[test]649	fn string_is_string() {650		assert!(primitive_equals(651			&eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"),652			&Val::Bool(false),653		)654		.unwrap());655	}656657	#[test]658	fn base64_works() {659		assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#);660	}661662	#[test]663	fn utf8_chars() {664		assert_json!(665			r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#,666			r#"{"c": 128526,"l": 1}"#667		)668	}669670	#[test]671	fn json() {672		assert_json!(673			r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#,674			r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""#675		);676	}677678	#[test]679	fn test() {680		assert_json!(681			r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#,682			"[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]"683		);684	}685686	#[test]687	fn sjsonnet() {688		eval!(689			r#"690			local x0 = {k: 1};691			local x1 = {k: x0.k + x0.k};692			local x2 = {k: x1.k + x1.k};693			local x3 = {k: x2.k + x2.k};694			local x4 = {k: x3.k + x3.k};695			local x5 = {k: x4.k + x4.k};696			local x6 = {k: x5.k + x5.k};697			local x7 = {k: x6.k + x6.k};698			local x8 = {k: x7.k + x7.k};699			local x9 = {k: x8.k + x8.k};700			local x10 = {k: x9.k + x9.k};701			local x11 = {k: x10.k + x10.k};702			local x12 = {k: x11.k + x11.k};703			local x13 = {k: x12.k + x12.k};704			local x14 = {k: x13.k + x13.k};705			local x15 = {k: x14.k + x14.k};706			local x16 = {k: x15.k + x15.k};707			local x17 = {k: x16.k + x16.k};708			local x18 = {k: x17.k + x17.k};709			local x19 = {k: x18.k + x18.k};710			local x20 = {k: x19.k + x19.k};711			local x21 = {k: x20.k + x20.k};712			x21.k713		"#714		);715	}716717	use test::Bencher;718719	// This test is commented out by default, because of huge compilation slowdown720	// #[bench]721	// fn bench_codegen(b: &mut Bencher) {722	// 	b.iter(|| {723	// 		#[allow(clippy::all)]724	// 		let stdlib = {725	// 			use jrsonnet_parser::*;726	// 			include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))727	// 		};728	// 		stdlib729	// 	})730	// }731732	#[bench]733	fn bench_serialize(b: &mut Bencher) {734		b.iter(|| {735			bincode::deserialize::<jrsonnet_parser::LocExpr>(include_bytes!(concat!(736				env!("OUT_DIR"),737				"/stdlib.bincode"738			)))739			.expect("deserialize stdlib")740		})741	}742743	#[bench]744	fn bench_parse(b: &mut Bencher) {745		b.iter(|| {746			jrsonnet_parser::parse(747				jrsonnet_stdlib::STDLIB_STR,748				&jrsonnet_parser::ParserSettings {749					loc_data: true,750					file_name: Rc::new(PathBuf::from("std.jsonnet")),751				},752			)753		})754	}755756	#[test]757	fn equality() {758		println!(759			"{:?}",760			jrsonnet_parser::parse(761				"{ x: 1, y: 2 } == { x: 1, y: 2 }",762				&ParserSettings::default()763			)764		);765		assert_eval!("{ x: 1, y: 2 } == { x: 1, y: 2 }")766	}767}