git.delta.rocks / jrsonnet / refs/commits / a94a1150b4f0

1#![feature(box_syntax, box_patterns)]2#![feature(type_alias_impl_trait)]3#![feature(debug_non_exhaustive)]4#![feature(test)]5#![feature(stmt_expr_attributes)]6#![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)]78extern crate test;910mod ctx;11mod dynamic;12mod error;13mod evaluate;14mod function;15mod import;16mod map;17mod obj;18mod val;19pub mod trace;2021pub use ctx::*;22pub use dynamic::*;23pub use error::*;24pub use evaluate::*;25pub use function::parse_function_call;26pub use import::*;27use jrsonnet_parser::*;28pub use obj::*;29use std::{cell::{Ref, RefCell, RefMut}, collections::HashMap, fmt::Debug, path::PathBuf, rc::Rc};30pub use val::*;31use trace::{offset_to_location, CodeLocation};3233type BindableFn = dyn Fn(Option<ObjValue>, Option<ObjValue>) -> Result<LazyVal>;34#[derive(Clone)]35pub enum LazyBinding {36	Bindable(Rc<BindableFn>),37	Bound(LazyVal),38}3940impl Debug for LazyBinding {41	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {42		write!(f, "LazyBinding")43	}44}45impl LazyBinding {46	pub fn evaluate(&self, this: Option<ObjValue>, super_obj: Option<ObjValue>) -> Result<LazyVal> {47		match self {48			LazyBinding::Bindable(v) => v(this, super_obj),49			LazyBinding::Bound(v) => Ok(v.clone()),50		}51	}52}5354pub struct EvaluationSettings {55	/// Limits recursion by limiting stack frames56	pub max_stack: usize,57	/// Limit amount of stack trace items preserved58	pub max_trace: usize,59	/// Used for std.extVar60	pub ext_vars: HashMap<Rc<str>, Val>,61	/// Global variables are inserted in default context62	pub globals: HashMap<Rc<str>, Val>,63	/// Used to resolve file locations/contents64	pub import_resolver: Box<dyn ImportResolver>,65}66impl Default for EvaluationSettings {67	fn default() -> Self {68		EvaluationSettings {69			max_stack: 200,70			max_trace: 20,71			globals: Default::default(),72			ext_vars: Default::default(),73			import_resolver: Box::new(DummyImportResolver),74		}75	}76}7778#[derive(Default)]79struct EvaluationData {80	/// Used for stack overflow detection, stacktrace is now populated on unwind81	stack_depth: usize,82	/// Contains file source codes and evaluated results for imports and pretty83	/// printing stacktraces84	files: HashMap<Rc<PathBuf>, FileData>,85	str_files: HashMap<Rc<PathBuf>, Rc<str>>,86}8788pub struct FileData {89	source_code: Rc<str>,90	parsed: LocExpr,91	evaluated: Option<Val>,92}93#[derive(Default)]94pub struct EvaluationStateInternals {95	/// Internal state96	data: RefCell<EvaluationData>,97	/// Settings, safe to change at runtime98	settings: RefCell<EvaluationSettings>,99}100101thread_local! {102	/// Contains state for currently executing file103	/// Global state is fine there104	pub(crate) static EVAL_STATE: RefCell<Option<EvaluationState>> = RefCell::new(None)105}106pub(crate) fn with_state<T>(f: impl FnOnce(&EvaluationState) -> T) -> T {107	EVAL_STATE.with(|s| f(s.borrow().as_ref().unwrap()))108}109pub fn create_error(err: Error) -> LocError {110	with_state(|s| s.error(err))111}112pub fn create_error_result<T>(err: Error) -> Result<T> {113	Err(with_state(|s| s.error(err)))114}115pub(crate) fn push<T>(116	e: &Option<ExprLocation>,117	frame_desc: impl FnOnce() -> String,118	f: impl FnOnce() -> Result<T>,119) -> Result<T> {120	if let Some(v) = e {121		with_state(|s| s.push(&v, frame_desc, f))122	} else {123		f()124	}125}126127/// Maintains stack trace and import resolution128#[derive(Default, Clone)]129pub struct EvaluationState(Rc<EvaluationStateInternals>);130impl EvaluationState {131	fn data(&self) -> Ref<EvaluationData> {132		self.0.data.borrow()133	}134	fn data_mut(&self) -> RefMut<EvaluationData> {135		self.0.data.borrow_mut()136	}137	pub fn settings(&self) -> Ref<EvaluationSettings> {138		self.0.settings.borrow()139	}140	pub fn settings_mut(&self) -> RefMut<EvaluationSettings> {141		self.0.settings.borrow_mut()142	}143144	pub fn evaluate_file_to_json(&self, path: &PathBuf) -> std::result::Result<Rc<str>, LocError> {145		self.import_file(&PathBuf::new(), &path)146			.and_then(|v| v.into_json(4))147	}148	pub fn evaluate_snippet_to_json(149		&self,150		path: &PathBuf,151		snippet: &str,152	) -> std::result::Result<Rc<str>, LocError> {153		self.parse_evaluate_raw(Rc::new(path.clone()), snippet)154			.and_then(|v| v.into_json(4))155	}156157	/// Parses and adds file to loaded158	pub fn add_file(&self, path: Rc<PathBuf>, source_code: Rc<str>) -> Result<()> {159		self.add_parsed_file(160			path.clone(),161			source_code.clone(),162			parse(163				&source_code,164				&ParserSettings {165					file_name: path.clone(),166					loc_data: true,167				},168			)169			.map_err(|error| {170				create_error(Error::ImportSyntaxError {171					error,172					path,173					source_code,174				})175			})?,176		)?;177178		Ok(())179	}180181	/// Adds file by source code and parsed expr182	pub fn add_parsed_file(183		&self,184		name: Rc<PathBuf>,185		source_code: Rc<str>,186		parsed: LocExpr,187	) -> Result<()> {188		self.data_mut().files.insert(189			name,190			FileData {191				source_code,192				parsed,193				evaluated: None,194			},195		);196197		Ok(())198	}199	pub fn get_source(&self, name: &PathBuf) -> Option<Rc<str>> {200		let ro_map = &self.data().files;201		ro_map.get(name).map(|value| value.source_code.clone())202	}203	pub fn map_source_locations(&self, file: &PathBuf, locs: &[usize]) -> Vec<CodeLocation> {204		offset_to_location(&self.get_source(file).unwrap(), locs)205	}206207	pub fn evaluate_file(&self, name: &PathBuf) -> Result<Val> {208		self.run_in_state(|| {209			let expr: LocExpr = {210				let ro_map = &self.data().files;211				let value = ro_map212					.get(name)213					.unwrap_or_else(|| panic!("file not added: {:?}", name));214				if value.evaluated.is_some() {215					return Ok(value.evaluated.clone().unwrap());216				}217				value.parsed.clone()218			};219			let value = evaluate(self.create_default_context()?, &expr)?;220			{221				self.0222					.data223					.borrow_mut()224					.files225					.get_mut(name)226					.unwrap()227					.evaluated228					.replace(value.clone());229			}230			Ok(value)231		})232	}233	pub fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result<Rc<PathBuf>> {234		Ok(self.settings().import_resolver.resolve_file(from, path)?)235	}236	pub fn load_file_contents(&self, path: &PathBuf) -> Result<Rc<str>> {237		Ok(self.settings().import_resolver.load_file_contents(path)?)238	}239	pub(crate) fn import_file(&self, from: &PathBuf, path: &PathBuf) -> Result<Val> {240		let file_path = self.resolve_file(from, path)?;241		{242			let files = &self.data().files;243			if files.contains_key(&file_path) {244				return self.evaluate_file(&file_path);245			}246		}247		let contents = self.load_file_contents(&file_path)?;248		self.add_file(file_path.clone(), contents)?;249		self.evaluate_file(&file_path)250	}251	pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result<Rc<str>> {252		let path = self.resolve_file(from, path)?;253		if !self.data().str_files.contains_key(&path) {254			let file_str = self.load_file_contents(&path)?;255			self.data_mut().str_files.insert(path.clone(), file_str);256		}257		Ok(self.data().str_files.get(&path).cloned().unwrap())258	}259260	/// Parses and evaluates snippet261	pub fn parse_evaluate_raw(&self, source: Rc<PathBuf>, code: &str) -> Result<Val> {262		let parsed = parse(263			&code,264			&ParserSettings {265				file_name: source,266				loc_data: true,267			},268		)269		.unwrap();270		self.evaluate_raw(parsed)271	}272	/// Evaluates parsed expression273	pub fn evaluate_raw(&self, code: LocExpr) -> Result<Val> {274		self.run_in_state(|| evaluate(self.create_default_context()?, &code))275	}276277	/// Adds standard library global variable (std) to this evaluator278	pub fn with_stdlib(&self) -> &Self {279		use jrsonnet_stdlib::STDLIB_STR;280		let std_path = Rc::new(PathBuf::from("std.jsonnet"));281		self.run_in_state(|| {282			self.add_parsed_file(283				std_path.clone(),284				STDLIB_STR.to_owned().into(),285				builtin::get_parsed_stdlib(),286			)287			.unwrap();288			let val = self.evaluate_file(&std_path).unwrap();289			self.settings_mut().globals.insert("std".into(), val);290		});291		self292	}293294	/// Creates context with all passed global variables295	pub fn create_default_context(&self) -> Result<Context> {296		let globals = &self.settings().globals;297		let mut new_bindings: HashMap<Rc<str>, LazyBinding> = HashMap::new();298		for (name, value) in globals.iter() {299			new_bindings.insert(300				name.clone(),301				LazyBinding::Bound(resolved_lazy_val!(value.clone())),302			);303		}304		Context::new().extend_unbound(new_bindings, None, None, None)305	}306307	/// Executes code, creating new stack frame308	pub fn push<T>(309		&self,310		e: &ExprLocation,311		frame_desc: impl FnOnce() -> String,312		f: impl FnOnce() -> Result<T>,313	) -> Result<T> {314		{315			let mut data = self.data_mut();316			let stack_depth = &mut data.stack_depth;317			if *stack_depth > self.settings().max_stack {318				// Error creation uses data, so i drop guard here319				drop(data);320				return Err(self.error(Error::StackOverflow));321			} else {322				*stack_depth+=1;323			}324		}325		let result = f();326		self.data_mut().stack_depth -= 1;327		if let Err(mut err) = result {328			(err.1).0.push(StackTraceElement(e.clone(), frame_desc()));329			return Err(err);330		}331		result332	}333334	/// Creates error with stack trace335	pub fn error(&self, err: Error) -> LocError {336		LocError(err, StackTrace(vec![]))337	}338339	/// Runs passed function in state (required, if function needs to modify stack trace)340	pub fn run_in_state<T>(&self, f: impl FnOnce() -> T) -> T {341		EVAL_STATE.with(|v| {342			let has_state = v.borrow().is_some();343			if !has_state {344				v.borrow_mut().replace(self.clone());345			}346			let result = f();347			if !has_state {348				v.borrow_mut().take();349			}350			result351		})352	}353}354355#[cfg(test)]356pub mod tests {357	use super::Val;358	use crate::{create_error, EvaluationState, primitive_equals};359	use jrsonnet_parser::*;360	use std::{path::PathBuf, rc::Rc};361362	#[test]363	fn eval_state_stacktrace() {364		let state = EvaluationState::default();365		state.run_in_state(||{366			state367			.push(368				&ExprLocation(Rc::new(PathBuf::from("test1.jsonnet")), 10, 20),369				|| "outer".to_owned(),370				|| {371					state.push(372						&ExprLocation(Rc::new(PathBuf::from("test2.jsonnet")), 30, 40),373						|| "inner".to_owned(),374						|| {375							Err(create_error(crate::error::Error::RuntimeError("".into())))376						},377					)?;378					Ok(())379				},380			)381			.unwrap();382		});383	}384385	#[test]386	fn eval_state_standard() {387		let state = EvaluationState::default();388		state.with_stdlib();389		assert!(390			primitive_equals(391				&state.parse_evaluate_raw(r#"std.assertEqual(std.base64("test"), "dGVzdA==")"#).unwrap(),392				&Val::Bool(true),393			).unwrap()394		);395	}396397	macro_rules! eval {398		($str: expr) => {399			EvaluationState::default()400				.with_stdlib()401				.parse_evaluate_raw($str)402				.unwrap()403		};404	}405	macro_rules! eval_json {406		($str: expr) => {{407			let evaluator = EvaluationState::default();408			evaluator.with_stdlib();409			evaluator.run_in_state(||{410				evaluator411					.parse_evaluate_raw($str)412					.unwrap()413					.into_json(0)414					.unwrap()415					.replace("\n", "")416			})417		}}418	}419420	/// Asserts given code returns `true`421	macro_rules! assert_eval {422		($str: expr) => {423			assert!(primitive_equals(&eval!($str), &Val::Bool(true)).unwrap())424		};425	}426427	/// Asserts given code returns `false`428	macro_rules! assert_eval_neg {429		($str: expr) => {430			assert!(primitive_equals(&eval!($str), &Val::Bool(false)).unwrap())431		};432	}433	macro_rules! assert_json {434		($str: expr, $out: expr) => {435			assert_eq!(eval_json!($str), $out.replace("\t", ""))436		};437	}438439	/// Sanity checking, before trusting to another tests440	#[test]441	fn equality_operator() {442		assert_eval!("2 == 2");443		assert_eval_neg!("2 != 2");444		assert_eval!("2 != 3");445		assert_eval_neg!("2 == 3");446		assert_eval!("'Hello' == 'Hello'");447		assert_eval_neg!("'Hello' != 'Hello'");448		assert_eval!("'Hello' != 'World'");449		assert_eval_neg!("'Hello' == 'World'");450	}451452	#[test]453	fn math_evaluation() {454		assert_eval!("2 + 2 * 2 == 6");455		assert_eval!("3 + (2 + 2 * 2) == 9");456	}457458	#[test]459	fn string_concat() {460		assert_eval!("'Hello' + 'World' == 'HelloWorld'");461		assert_eval!("'Hello' * 3 == 'HelloHelloHello'");462		assert_eval!("'Hello' + 'World' * 3 == 'HelloWorldWorldWorld'");463	}464465	#[test]466	fn faster_join() {467		assert_eval!("std.join([0,0], [[1,2],[3,4],[5,6]]) == [1,2,0,0,3,4,0,0,5,6]");468		assert_eval!("std.join(',', ['1','2','3','4']) == '1,2,3,4'");469	}470471	#[test]472	fn function_contexts() {473		assert_eval!(474			r#"475				local k = {476					t(name = self.h): [self.h, name],477					h: 3,478				};479				local f = {480					t: k.t(),481					h: 4,482				};483				f.t[0] == f.t[1]484			"#485		);486	}487488	#[test]489	fn local() {490		assert_eval!("local a = 2; local b = 3; a + b == 5");491		assert_eval!("local a = 1, b = a + 1; a + b == 3");492		assert_eval!("local a = 1; local a = 2; a == 2");493	}494495	#[test]496	fn object_lazyness() {497		assert_json!("local a = {a:error 'test'}; {}", r#"{}"#);498	}499500	#[test]501	fn object_inheritance() {502		assert_json!("{a: self.b} + {b:3}", r#"{"a": 3,"b": 3}"#);503	}504505	#[test]506	fn object_assertion_success() {507		eval!("{assert \"a\" in self} + {a:2}");508	}509510	#[test]511	fn object_assertion_error() {512		eval!("{assert \"a\" in self}");513	}514515	#[test]516	fn lazy_args() {517		eval!("local test(a) = 2; test(error '3')");518	}519520	#[test]521	#[should_panic]522	fn tailstrict_args() {523		eval!("local test(a) = 2; test(error '3') tailstrict");524	}525526	#[test]527	#[should_panic]528	fn no_binding_error() {529		eval!("a");530	}531532	#[test]533	fn test_object() {534		assert_json!("{a:2}", r#"{"a": 2}"#);535		assert_json!("{a:2+2}", r#"{"a": 4}"#);536		assert_json!("{a:2}+{b:2}", r#"{"a": 2,"b": 2}"#);537		assert_json!("{b:3}+{b:2}", r#"{"b": 2}"#);538		assert_json!("{b:3}+{b+:2}", r#"{"b": 5}"#);539		assert_json!("local test='a'; {[test]:2}", r#"{"a": 2}"#);540		assert_json!(541			r#"542				{543					name: "Alice",544					welcome: "Hello " + self.name + "!",545				}546			"#,547			r#"{"name": "Alice","welcome": "Hello Alice!"}"#548		);549		assert_json!(550			r#"551				{552					name: "Alice",553					welcome: "Hello " + self.name + "!",554				} + {555					name: "Bob"556				}557			"#,558			r#"{"name": "Bob","welcome": "Hello Bob!"}"#559		);560	}561562	#[test]563	fn functions() {564		assert_json!(r#"local a = function(b, c = 2) b + c; a(2)"#, "4");565		assert_json!(566			r#"local a = function(b, c = "Dear") b + c + d, d = "World"; a("Hello")"#,567			r#""HelloDearWorld""#568		);569	}570571	#[test]572	fn local_methods() {573		assert_json!(r#"local a(b, c = 2) = b + c; a(2)"#, "4");574		assert_json!(575			r#"local a(b, c = "Dear") = b + c + d, d = "World"; a("Hello")"#,576			r#""HelloDearWorld""#577		);578	}579580	#[test]581	fn object_locals() {582		assert_json!(r#"{local a = 3, b: a}"#, r#"{"b": 3}"#);583		assert_json!(r#"{local a = 3, local c = a, b: c}"#, r#"{"b": 3}"#);584		assert_json!(585			r#"{local a = function (b) {[b]:4}, test: a("test")}"#,586			r#"{"test": {"test": 4}}"#587		);588	}589590	#[test]591	fn object_comp() {592		assert_json!(593			r#"{local t = "a", ["h"+i+"_"+z]: if "h"+(i-1)+"_"+z in self then t+1 else 0+t for i in [1,2,3] for z in [2,3,4] if z != i}"#,594			"{\"h1_2\": \"0a\",\"h1_3\": \"0a\",\"h1_4\": \"0a\",\"h2_3\": \"a1\",\"h2_4\": \"a1\",\"h3_2\": \"0a\",\"h3_4\": \"a1\"}"595		)596	}597598	#[test]599	fn direct_self() {600		println!(601			"{:#?}",602			eval!(603				r#"604					{605						local me = self,606						a: 3,607						b(): me.a,608					}609				"#610			)611		);612	}613614	#[test]615	fn indirect_self() {616		// `self` assigned to `me` was lost when being617		// referenced from field618		eval!(619			r#"{620				local me = self,621				a: 3,622				b: me.a,623			}.b"#624		);625	}626627	// We can't trust other tests (And official jsonnet testsuite), if assert is not working correctly628	#[test]629	fn std_assert_ok() {630		eval!("std.assertEqual(4.5 << 2, 16)");631	}632633	#[test]634	#[should_panic]635	fn std_assert_failure() {636		eval!("std.assertEqual(4.5 << 2, 15)");637	}638639	#[test]640	fn string_is_string() {641		assert!(642			primitive_equals(643				&eval!("local arr = 'hello'; (!std.isArray(arr)) && (!std.isString(arr))"),644				&Val::Bool(false),645			).unwrap()646		);647	}648649	#[test]650	fn base64_works() {651		assert_json!(r#"std.base64("test")"#, r#""dGVzdA==""#);652	}653654	#[test]655	fn utf8_chars() {656		assert_json!(657			r#"local c="😎";{c:std.codepoint(c),l:std.length(c)}"#,658			r#"{"c": 128526,"l": 1}"#659		)660	}661662	#[test]663	fn json() {664		assert_json!(665			r#"std.manifestJsonEx({a:3, b:4, c:6},"")"#,666			r#""{\n\"a\": 3,\n\"b\": 4,\n\"c\": 6\n}""#667		);668	}669670	#[test]671	fn test() {672		assert_json!(673			r#"[[a, b] for a in [1,2,3] for b in [4,5,6]]"#,674			"[[1,4],[1,5],[1,6],[2,4],[2,5],[2,6],[3,4],[3,5],[3,6]]"675		);676	}677678	#[test]679	fn sjsonnet() {680		eval!(681			r#"682			local x0 = {k: 1};683			local x1 = {k: x0.k + x0.k};684			local x2 = {k: x1.k + x1.k};685			local x3 = {k: x2.k + x2.k};686			local x4 = {k: x3.k + x3.k};687			local x5 = {k: x4.k + x4.k};688			local x6 = {k: x5.k + x5.k};689			local x7 = {k: x6.k + x6.k};690			local x8 = {k: x7.k + x7.k};691			local x9 = {k: x8.k + x8.k};692			local x10 = {k: x9.k + x9.k};693			local x11 = {k: x10.k + x10.k};694			local x12 = {k: x11.k + x11.k};695			local x13 = {k: x12.k + x12.k};696			local x14 = {k: x13.k + x13.k};697			local x15 = {k: x14.k + x14.k};698			local x16 = {k: x15.k + x15.k};699			local x17 = {k: x16.k + x16.k};700			local x18 = {k: x17.k + x17.k};701			local x19 = {k: x18.k + x18.k};702			local x20 = {k: x19.k + x19.k};703			local x21 = {k: x20.k + x20.k};704			x21.k705		"#706		);707	}708709	use test::Bencher;710711	// This test is commented out by default, because of huge compilation slowdown712	// #[bench]713	// fn bench_codegen(b: &mut Bencher) {714	// 	b.iter(|| {715	// 		#[allow(clippy::all)]716	// 		let stdlib = {717	// 			use jrsonnet_parser::*;718	// 			include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))719	// 		};720	// 		stdlib721	// 	})722	// }723724	#[bench]725	fn bench_serialize(b: &mut Bencher) {726		b.iter(|| {727			bincode::deserialize::<jrsonnet_parser::LocExpr>(include_bytes!(concat!(728				env!("OUT_DIR"),729				"/stdlib.bincode"730			)))731			.expect("deserialize stdlib")732		})733	}734735	#[bench]736	fn bench_parse(b: &mut Bencher) {737		b.iter(|| {738			jrsonnet_parser::parse(739				jrsonnet_stdlib::STDLIB_STR,740				&jrsonnet_parser::ParserSettings {741					loc_data: true,742					file_name: Rc::new(PathBuf::from("std.jsonnet")),743				},744			)745		})746	}747748	#[test]749	fn equality(){750		println!("{:?}", jrsonnet_parser::parse("{ x: 1, y: 2 } == { x: 1, y: 2 }", &ParserSettings::default()));751		assert_eval!("{ x: 1, y: 2 } == { x: 1, y: 2 }")752	}753}