git.delta.rocks / jrsonnet / refs/commits / 6d7ca685fe74

7edition = "2021"7edition = "2021"
88
9[features]9[features]
10default = []
11# Serializes standard library AST, and deserialize on start, instead of parsing it every run from text
12serialized-stdlib = ["bincode", "jrsonnet-parser/serde"]
13# Enables legacy `std.thisFile` support, at the cost of worse caching
14legacy-this-file = []
15# Add order preservation flag to some functions
16exp-preserve-order = ["jrsonnet-evaluator/exp-preserve-order"]
17# Preserve order for files parsed via `std.parseJson`
18# Shame it isn't possible to enable per parse call, instead of globally
19exp-serde-preserve-order = [
20    "serde_json/preserve_order",
21    "jrsonnet-evaluator/exp-serde-preserve-order",
22]
1023
11[dependencies]24[dependencies]
25jrsonnet-evaluator = { path = "../jrsonnet-evaluator", features = [
26    # std.parseJson parses file via serde, then converts Value to evaluator Val
27    "serde_json",
28], version = "0.4.2" }
29jrsonnet-macros = { path = "../jrsonnet-macros", version = "0.4.2" }
30jrsonnet-parser = { path = "../jrsonnet-parser", version = "0.4.2" }
31jrsonnet-gcmodule = "0.3.4"
32
33# Used for stdlib AST serialization
34bincode = { version = "1.3", optional = true }
35# Used both for stdlib AST serialization and std.parseJson/std.parseYaml
36serde = "1.0"
37
38# std.md5
39md5 = "0.7.0"
40# std.base64
41base64 = "0.13.0"
42# std.parseJson
43serde_json = "1.0"
44# std.parseYaml, custom library fork is used for C++/golang compatibility
45serde_yaml_with_quirks = "0.8.24"
46
47[build-dependencies]
48jrsonnet-parser = { path = "../jrsonnet-parser", version = "0.4.2", features = [
49    "serde",
50] }
51serde = "1.0"
52bincode = "1.3"
1253

1use std::{
2	borrow::Cow,
3	cell::{Ref, RefCell, RefMut},
4	collections::HashMap,
5	rc::Rc,
6};
7
8use jrsonnet_evaluator::{
9	error::{Error::*, Result},
10	function::{builtin::Builtin, ArgLike, CallLocation, FuncVal, TlaArg},
11	gc::TraceBox,
12	tb,
13	typed::{Any, Either, Either2, Either4, VecVal, M1},
14	val::ArrValue,
15	Context, ContextBuilder, IStr, ObjValue, ObjValueBuilder, State, Thunk, Val,
16};
17use jrsonnet_gcmodule::Cc;
18use jrsonnet_macros::builtin;
19use jrsonnet_parser::Source;
20
21mod expr;
22mod types;
23pub use types::*;
24mod arrays;
25pub use arrays::*;
26mod math;
27pub use math::*;
28mod operator;
29pub use operator::*;
30mod sort;
31pub use sort::*;
32mod hash;
33pub use hash::*;
34mod encoding;
35pub use encoding::*;
36mod objects;
37pub use objects::*;
38mod manifest;
39pub use manifest::*;
40mod parse;
41pub use parse::*;
42
43pub fn stdlib_uncached(s: State, settings: Rc<RefCell<Settings>>) -> ObjValue {
44	let mut builder = ObjValueBuilder::new();
45
46	let expr = expr::stdlib_expr();
47	let eval = jrsonnet_evaluator::evaluate(s.clone(), Context::default(), &expr)
48		.expect("stdlib.jsonnet should have no errors")
49		.as_obj()
50		.expect("stdlib.jsonnet should evaluate to object");
51
52	builder.with_super(eval);
53
54	for (name, builtin) in [
55		("length".into(), builtin_length::INST),
56		// Types
57		("type".into(), builtin_type::INST),
58		("isString".into(), builtin_is_string::INST),
59		("isNumber".into(), builtin_is_number::INST),
60		("isBoolean".into(), builtin_is_boolean::INST),
61		("isObject".into(), builtin_is_object::INST),
62		("isArray".into(), builtin_is_array::INST),
63		("isFunction".into(), builtin_is_function::INST),
64		// Arrays
65		("makeArray".into(), builtin_make_array::INST),
66		("slice".into(), builtin_slice::INST),
67		("map".into(), builtin_map::INST),
68		("flatMap".into(), builtin_flatmap::INST),
69		("filter".into(), builtin_filter::INST),
70		("foldl".into(), builtin_foldl::INST),
71		("foldr".into(), builtin_foldr::INST),
72		("range".into(), builtin_range::INST),
73		("join".into(), builtin_join::INST),
74		("reverse".into(), builtin_reverse::INST),
75		("any".into(), builtin_any::INST),
76		("all".into(), builtin_all::INST),
77		("member".into(), builtin_member::INST),
78		("count".into(), builtin_count::INST),
79		// Math
80		("modulo".into(), builtin_modulo::INST),
81		("floor".into(), builtin_floor::INST),
82		("ceil".into(), builtin_ceil::INST),
83		("log".into(), builtin_log::INST),
84		("pow".into(), builtin_pow::INST),
85		("sqrt".into(), builtin_sqrt::INST),
86		("sin".into(), builtin_sin::INST),
87		("cos".into(), builtin_cos::INST),
88		("tan".into(), builtin_tan::INST),
89		("asin".into(), builtin_asin::INST),
90		("acos".into(), builtin_acos::INST),
91		("atan".into(), builtin_atan::INST),
92		("exp".into(), builtin_exp::INST),
93		("mantissa".into(), builtin_mantissa::INST),
94		("exponent".into(), builtin_exponent::INST),
95		// Operator
96		("mod".into(), builtin_mod::INST),
97		("primitiveEquals".into(), builtin_primitive_equals::INST),
98		("equals".into(), builtin_equals::INST),
99		("format".into(), builtin_format::INST),
100		// Sort
101		("sort".into(), builtin_sort::INST),
102		// Hash
103		("md5".into(), builtin_md5::INST),
104		// Encoding
105		("encodeUTF8".into(), builtin_encode_utf8::INST),
106		("decodeUTF8".into(), builtin_decode_utf8::INST),
107		("base64".into(), builtin_base64::INST),
108		("base64Decode".into(), builtin_base64_decode::INST),
109		(
110			"base64DecodeBytes".into(),
111			builtin_base64_decode_bytes::INST,
112		),
113		// Objects
114		("objectFieldsEx".into(), builtin_object_fields_ex::INST),
115		("objectHasEx".into(), builtin_object_has_ex::INST),
116		// Manifest
117		("escapeStringJson".into(), builtin_escape_string_json::INST),
118		("manifestJsonEx".into(), builtin_manifest_json_ex::INST),
119		("manifestYamlDoc".into(), builtin_manifest_yaml_doc::INST),
120		// Parsing
121		("parseJson".into(), builtin_parse_json::INST),
122		("parseYaml".into(), builtin_parse_yaml::INST),
123		// Misc
124		("codepoint".into(), builtin_codepoint::INST),
125		("substr".into(), builtin_substr::INST),
126		("char".into(), builtin_char::INST),
127		("strReplace".into(), builtin_str_replace::INST),
128		("splitLimit".into(), builtin_splitlimit::INST),
129		("asciiUpper".into(), builtin_ascii_upper::INST),
130		("asciiLower".into(), builtin_ascii_lower::INST),
131	]
132	.iter()
133	.cloned()
134	{
135		builder
136			.member(name)
137			.hide()
138			.value(s.clone(), Val::Func(FuncVal::StaticBuiltin(builtin)))
139			.expect("no conflict");
140	}
141
142	builder
143		.member("extVar".into())
144		.hide()
145		.value(
146			s.clone(),
147			Val::Func(FuncVal::Builtin(Cc::new(tb!(builtin_ext_var {
148				settings: settings.clone()
149			})))),
150		)
151		.expect("no conflict");
152	builder
153		.member("native".into())
154		.hide()
155		.value(
156			s.clone(),
157			Val::Func(FuncVal::Builtin(Cc::new(tb!(builtin_native {
158				settings: settings.clone()
159			})))),
160		)
161		.expect("no conflict");
162	builder
163		.member("trace".into())
164		.hide()
165		.value(
166			s.clone(),
167			Val::Func(FuncVal::Builtin(Cc::new(tb!(builtin_trace { settings })))),
168		)
169		.expect("no conflict");
170
171	builder
172		.member("id".into())
173		.hide()
174		.value(s, Val::Func(FuncVal::Id))
175		.expect("no conflict");
176
177	builder.build()
178}
179
180pub trait TracePrinter {
181	fn print_trace(&self, s: State, loc: CallLocation, value: IStr);
182}
183
184pub struct StdTracePrinter;
185impl TracePrinter for StdTracePrinter {
186	fn print_trace(&self, s: State, loc: CallLocation, value: IStr) {
187		eprint!("TRACE:");
188		if let Some(loc) = loc.0 {
189			let locs = s.map_source_locations(loc.0.clone(), &[loc.1]);
190			eprint!(" {}:{}", loc.0.short_display(), locs[0].line);
191		}
192		eprintln!(" {}", value);
193	}
194}
195
196pub struct Settings {
197	/// Used for `std.extVar`
198	pub ext_vars: HashMap<IStr, TlaArg>,
199	/// Used for `std.native`
200	pub ext_natives: HashMap<IStr, Cc<TraceBox<dyn Builtin>>>,
201	/// Used for `std.trace`
202	pub trace_printer: Box<dyn TracePrinter>,
203}
204
205impl Default for Settings {
206	fn default() -> Self {
207		Self {
208			ext_vars: Default::default(),
209			ext_natives: Default::default(),
210			trace_printer: Box::new(StdTracePrinter),
211		}
212	}
213}
214
1pub const STDLIB_STR: &str = include_str!("./std.jsonnet");215pub fn extvar_source(name: &str) -> Source {
216	let source_name = format!("<extvar:{}>", name);
217	Source::new_virtual(Cow::Owned(source_name))
218}
219
220pub struct ContextInitializer {
221	// When we don't need to support legacy-this-file, we can reuse same context for all files
222	#[cfg(not(feature = "legacy-this-file"))]
223	context: Context,
224	// Otherwise, we can only keep first stdlib layer, and then stack thisFile on top of it
225	#[cfg(feature = "legacy-this-file")]
226	stdlib_obj: ObjValue,
227	settings: Rc<RefCell<Settings>>,
228}
229impl ContextInitializer {
230	pub fn new(s: State) -> Self {
231		let settings = Rc::new(RefCell::new(Settings::default()));
232		Self {
233			#[cfg(not(feature = "legacy-this-file"))]
234			context: {
235				let mut context = ContextBuilder::with_capacity(1);
236				context.bind(
237					"std".into(),
238					Thunk::evaluated(Val::Obj(stdlib_uncached(s, settings.clone()))),
239				);
240				context.build()
241			},
242			#[cfg(feature = "legacy-this-file")]
243			stdlib_obj: stdlib_uncached(s, settings.clone()),
244			settings,
245		}
246	}
247	pub fn settings(&self) -> Ref<Settings> {
248		self.settings.borrow()
249	}
250	pub fn settings_mut(&self) -> RefMut<Settings> {
251		self.settings.borrow_mut()
252	}
253	pub fn add_ext_var(&self, name: IStr, value: Val) {
254		self.settings_mut()
255			.ext_vars
256			.insert(name, TlaArg::Val(value));
257	}
258	pub fn add_ext_str(&self, name: IStr, value: IStr) {
259		self.settings_mut()
260			.ext_vars
261			.insert(name, TlaArg::String(value));
262	}
263	pub fn add_ext_code(&self, name: &str, code: String) -> Result<()> {
264		let source = extvar_source(name);
265		let parsed = jrsonnet_parser::parse(
266			&code,
267			&jrsonnet_parser::ParserSettings {
268				file_name: source.clone(),
269			},
270		)
271		.map_err(|e| ImportSyntaxError {
272			path: source,
273			source_code: code.clone().into(),
274			error: Box::new(e),
275		})?;
276		// self.data_mut().volatile_files.insert(source_name, code);
277		self.settings_mut()
278			.ext_vars
279			.insert(name.into(), TlaArg::Code(parsed));
280		Ok(())
281	}
282	pub fn add_native(&self, name: IStr, cb: Cc<TraceBox<dyn Builtin>>) {
283		self.settings_mut().ext_natives.insert(name, cb);
284	}
285}
286impl jrsonnet_evaluator::ContextInitializer for ContextInitializer {
287	#[cfg(not(feature = "legacy-this-file"))]
288	fn initialize(&self, _s: State, _source: Source) -> jrsonnet_evaluator::Context {
289		self.context.clone()
290	}
291	#[cfg(feature = "legacy-this-file")]
292	fn initialize(&self, s: State, source: Source) -> jrsonnet_evaluator::Context {
293		let mut builder = ObjValueBuilder::new();
294		builder.with_super(self.stdlib_obj.clone());
295		builder
296			.member("thisFile".into())
297			.hide()
298			.value(
299				s,
300				Val::Str(match source.repr() {
301					Ok(p) => p.display().to_string().into(),
302					// Virtual files end up as empty strings in std.thisFile
303					Err(_e) => "".into(),
304				}),
305			)
306			.expect("this object builder is empty");
307		let stdlib_with_this_file = builder.build();
308
309		let mut context = ContextBuilder::with_capacity(1);
310		context.bind(
311			"std".into(),
312			Thunk::evaluated(Val::Obj(stdlib_with_this_file)),
313		);
314		context.build()
315	}
316	unsafe fn as_any(&self) -> &dyn std::any::Any {
317		self
318	}
319}
320
321#[builtin]
322fn builtin_length(x: Either![IStr, ArrValue, ObjValue, FuncVal]) -> Result<usize> {
323	use Either4::*;
324	Ok(match x {
325		A(x) => x.chars().count(),
326		B(x) => x.len(),
327		C(x) => x.len(),
328		D(f) => f.params_len(),
329	})
330}
331
332#[builtin]
333const fn builtin_codepoint(str: char) -> Result<u32> {
334	Ok(str as u32)
335}
336
337#[builtin]
338fn builtin_substr(str: IStr, from: usize, len: usize) -> Result<String> {
339	Ok(str.chars().skip(from as usize).take(len as usize).collect())
340}
341
342#[builtin(fields(
343	settings: Rc<RefCell<Settings>>,
344))]
345fn builtin_ext_var(this: &builtin_ext_var, s: State, x: IStr) -> Result<Any> {
346	let ctx = s.create_default_context(extvar_source(&x));
347	Ok(Any(this
348		.settings
349		.borrow()
350		.ext_vars
351		.get(&x)
352		.cloned()
353		.ok_or(UndefinedExternalVariable(x))?
354		.evaluate_arg(s.clone(), ctx, true)?
355		.evaluate(s)?))
356}
357
358#[builtin(fields(
359	settings: Rc<RefCell<Settings>>,
360))]
361fn builtin_native(this: &builtin_native, name: IStr) -> Result<Any> {
362	Ok(Any(this
363		.settings
364		.borrow()
365		.ext_natives
366		.get(&name)
367		.cloned()
368		.map_or(Val::Null, |v| {
369			Val::Func(FuncVal::Builtin(v.clone()))
370		})))
371}
372
373#[builtin]
374fn builtin_char(n: u32) -> Result<char> {
375	Ok(std::char::from_u32(n as u32).ok_or(InvalidUnicodeCodepointGot(n as u32))?)
376}
377
378#[builtin(fields(
379	settings: Rc<RefCell<Settings>>,
380))]
381fn builtin_trace(
382	this: &builtin_trace,
383	s: State,
384	loc: CallLocation,
385	str: IStr,
386	rest: Any,
387) -> Result<Any> {
388	this.settings
389		.borrow()
390		.trace_printer
391		.print_trace(s, loc, str);
392	Ok(rest) as Result<Any>
393}
394
395#[builtin]
396fn builtin_str_replace(str: String, from: IStr, to: IStr) -> Result<String> {
397	Ok(str.replace(&from as &str, &to as &str))
398}
399
400#[builtin]
401fn builtin_splitlimit(str: IStr, c: IStr, maxsplits: Either![usize, M1]) -> Result<VecVal> {
402	use Either2::*;
403	Ok(VecVal(Cc::new(match maxsplits {
404		A(n) => str
405			.splitn(n + 1, &c as &str)
406			.map(|s| Val::Str(s.into()))
407			.collect(),
408		B(_) => str.split(&c as &str).map(|s| Val::Str(s.into())).collect(),
409	})))
410}
411
412#[builtin]
413fn builtin_ascii_upper(str: IStr) -> Result<String> {
414	Ok(str.to_ascii_uppercase())
415}
416
417#[builtin]
418fn builtin_ascii_lower(str: IStr) -> Result<String> {
419	Ok(str.to_ascii_lowercase())
420}
2421

2  local std = self,2  local std = self,
3  local id = std.id,3  local id = std.id,
44
5  # Magic legacy field5  thisFile:: error 'std.thisFile is deprecated, to enable its support in jrsonnet - recompile it with "legacy-this-file" support. This will slow down stdlib caching a bit, though',
6  thisFile:: $intrinsicThisFile,
7  id:: $intrinsicId,
86
9  # Those functions aren't normally located in stdlib
10  length:: $intrinsic(length),
11  type:: $intrinsic(type),
12  makeArray:: $intrinsic(makeArray),
13  codepoint:: $intrinsic(codepoint),
14  objectFieldsEx:: $intrinsic(objectFieldsEx),
15  objectHasEx:: $intrinsic(objectHasEx),
16  primitiveEquals:: $intrinsic(primitiveEquals),
17  modulo:: $intrinsic(modulo),
18  floor:: $intrinsic(floor),
19  ceil:: $intrinsic(ceil),
20  extVar:: $intrinsic(extVar),
21  native:: $intrinsic(native),
22  filter:: $intrinsic(filter),
23  char:: $intrinsic(char),
24  encodeUTF8:: $intrinsic(encodeUTF8),
25  decodeUTF8:: $intrinsic(decodeUTF8),
26  md5:: $intrinsic(md5),
27  trace:: $intrinsic(trace),
28  parseJson:: $intrinsic(parseJson),
29  parseYaml:: $intrinsic(parseYaml),
30
31  log:: $intrinsic(log),
32  pow:: $intrinsic(pow),
33  sqrt:: $intrinsic(sqrt),
34
35  sin:: $intrinsic(sin),
36  cos:: $intrinsic(cos),
37  tan:: $intrinsic(tan),
38  asin:: $intrinsic(asin),
39  acos:: $intrinsic(acos),
40  atan:: $intrinsic(atan),
41
42  exp:: $intrinsic(exp),
43  mantissa:: $intrinsic(mantissa),
44  exponent:: $intrinsic(exponent),
45
46  any:: $intrinsic(any),
47  all:: $intrinsic(all),
48
49  isString(v):: std.type(v) == 'string',
50  isNumber(v):: std.type(v) == 'number',
51  isBoolean(v):: std.type(v) == 'boolean',
52  isObject(v):: std.type(v) == 'object',
53  isArray(v):: std.type(v) == 'array',
54  isFunction(v):: std.type(v) == 'function',
55
56  toString(a)::7  toString(a)::
57    if std.type(a) == 'string' then a else '' + a,8    if std.type(a) == 'string' then a else '' + a,
589
59  substr:: $intrinsic(substr),
60
61  startsWith(a, b)::10  startsWith(a, b)::
62    if std.length(a) < std.length(b) then11    if std.length(a) < std.length(b) then
63      false12      false
12776
128  split(str, c):: std.splitLimit(str, c, -1),77  split(str, c):: std.splitLimit(str, c, -1),
12978
130  splitLimit:: $intrinsic(splitLimit),
131
132  strReplace:: $intrinsic(strReplace),
133
134  asciiUpper:: $intrinsic(asciiUpper),
135
136  asciiLower:: $intrinsic(asciiLower),
137
138  range:: $intrinsic(range),
139
140  repeat(what, count)::79  repeat(what, count)::
141    local joiner =80    local joiner =
142      if std.isString(what) then ''81      if std.isString(what) then ''
143      else if std.isArray(what) then []82      else if std.isArray(what) then []
144      else error 'std.repeat first argument must be an array or a string';83      else error 'std.repeat first argument must be an array or a string';
145    std.join(joiner, std.makeArray(count, function(i) what)),84    std.join(joiner, std.makeArray(count, function(i) what)),
14685
147  slice:: $intrinsic(slice),
148
149  member:: $intrinsic(member),
150
151  count:: $intrinsic(count),
152
153  mod:: $intrinsic(mod),
154
155  map:: $intrinsic(map),
156
157  mapWithIndex(func, arr)::86  mapWithIndex(func, arr)::
158    if !std.isFunction(func) then87    if !std.isFunction(func) then
159      error ('std.mapWithIndex first param must be function, got ' + std.type(func))88      error ('std.mapWithIndex first param must be function, got ' + std.type(func))
170    else99    else
171      { [k]: func(k, obj[k]) for k in std.objectFields(obj) },100      { [k]: func(k, obj[k]) for k in std.objectFields(obj) },
172101
173  flatMap:: $intrinsic(flatMap),
174
175  join:: $intrinsic(join),
176
177  lines(arr)::102  lines(arr)::
178    std.join('\n', arr + ['']),103    std.join('\n', arr + ['']),
179104
185    else110    else
186      error 'Expected string or array, got %s' % std.type(arr),111      error 'Expected string or array, got %s' % std.type(arr),
187112
188
189  format:: $intrinsic(format),
190
191  foldr:: $intrinsic(foldr),
192
193  foldl:: $intrinsic(foldl),
194
195  filterMap(filter_func, map_func, arr)::113  filterMap(filter_func, map_func, arr)::
196    if !std.isFunction(filter_func) then114    if !std.isFunction(filter_func) then
197      error ('std.filterMap first param must be function, got ' + std.type(filter_func))115      error ('std.filterMap first param must be function, got ' + std.type(filter_func))
350    else268    else
351      error 'TOML body must be an object. Got ' + std.type(value),269      error 'TOML body must be an object. Got ' + std.type(value),
352270
353  escapeStringJson:: $intrinsic(escapeStringJson),
354
355  escapeStringPython(str)::271  escapeStringPython(str)::
356    std.escapeStringJson(str),272    std.escapeStringJson(str),
357273
377293
378  manifestJsonMinified(value):: std.manifestJsonEx(value, '', '', ':'),294  manifestJsonMinified(value):: std.manifestJsonEx(value, '', '', ':'),
379295
380  manifestJsonEx:: $intrinsic(manifestJsonEx),
381
382  manifestYamlDoc:: $intrinsic(manifestYamlDoc),
383
384  manifestYamlStream(value, indent_array_in_object=false, c_document_end=true)::296  manifestYamlStream(value, indent_array_in_object=false, c_document_end=true)::
385    if !std.isArray(value) then297    if !std.isArray(value) then
386      error 'manifestYamlStream only takes arrays, got ' + std.type(value)298      error 'manifestYamlStream only takes arrays, got ' + std.type(value)
434346
435      aux(value),347      aux(value),
436348
437  local base64_table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/',
438  local base64_inv = { [base64_table[i]]: i for i in std.range(0, 63) },
439
440  base64:: $intrinsic(base64),
441
442  base64DecodeBytes:: $intrinsic(base64DecodeBytes),
443
444  base64Decode:: $intrinsic(base64Decode),
445
446  reverse:: $intrinsic(reverse),
447
448  sort:: $intrinsic(sort),
449
450  uniq(arr, keyF=id)::349  uniq(arr, keyF=id)::
451    local f(a, b) =350    local f(a, b) =
452      if std.length(a) == 0 then351      if std.length(a) == 0 then
534    else433    else
535      patch,434      patch,
536435
537  get(o, f, default = null, inc_hidden = true)::436  get(o, f, default=null, inc_hidden=true)::
538    if std.objectHasEx(o, f, inc_hidden) then o[f] else default,437    if std.objectHasEx(o, f, inc_hidden) then o[f] else default,
539438
540  objectFields(o)::439  objectFields(o)::
554453
555  objectValuesAll(o)::454  objectValuesAll(o)::
556    [o[k] for k in std.objectFieldsAll(o)],455    [o[k] for k in std.objectFieldsAll(o)],
557
558  equals:: $intrinsic(equals),
559456
560  resolvePath(f, r)::457  resolvePath(f, r)::
561    local arr = std.split(f, '/');458    local arr = std.split(f, '/');