git.delta.rocks / jrsonnet / refs/commits / a735ba768480

1#![allow(clippy::redundant_closure_call)]23use peg::parser;4use std::{5	path::{Path, PathBuf},6	rc::Rc,7};8mod expr;9pub use expr::*;10pub use jrsonnet_interner::IStr;11pub use peg;1213pub struct ParserSettings {14	pub loc_data: bool,15	pub file_name: Rc<Path>,16}1718macro_rules! expr_bin {19	($a:ident $op:ident $b:ident) => {20		Expr::BinaryOp($a, $op, $b)21	};22}23macro_rules! expr_un {24	($op:ident $a:ident) => {25		Expr::UnaryOp($op, $a)26	};27}2829parser! {30	grammar jsonnet_parser() for str {31		use peg::ParseLiteral;3233		rule eof() = quiet!{![_]} / expected!("<eof>")34		rule eol() = "\n" / eof()3536		/// Standard C-like comments37		rule comment()38			= "//" (!eol()[_])* eol()39			/ "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/"40			/ "#" (!eol()[_])* eol()4142		rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")43		rule _() = single_whitespace()*4445		/// For comma-delimited elements46		rule comma() = quiet!{_ "," _} / expected!("<comma>")47		rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}48		rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}49		rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']50		/// Sequence of digits51		rule uint_str() -> &'input str = a:$(digit()+) { a }52		/// Number in scientific notation format53		rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "<number>") }} / expected!("<number>")5455		/// Reserved word followed by any non-alphanumberic56		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()57		rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")5859		rule keyword(id: &'static str) -> ()60			= ##parse_string_literal(id) end_of_ident()6162		pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) }63		pub rule params(s: &ParserSettings) -> expr::ParamsDesc64			= params:param(s) ** comma() comma()? { expr::ParamsDesc(Rc::new(params)) }65			/ { expr::ParamsDesc(Rc::new(Vec::new())) }6667		pub rule arg(s: &ParserSettings) -> (Option<IStr>, LocExpr)68			= quiet! { name:(s:$(id()) _ "=" _ {s})? expr:expr(s) {(name.map(Into::into), expr)} }69			/ expected!("<argument>")7071		pub rule args(s: &ParserSettings) -> expr::ArgsDesc72			= args:arg(s)**comma() comma()? {?73				let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();74				let mut unnamed = Vec::with_capacity(unnamed_count);75				let mut named = Vec::with_capacity(args.len() - unnamed_count);76				let mut named_started = false;77				for (name, value) in args {78					if let Some(name) = name {79						named_started = true;80						named.push((name, value));81					} else {82						if named_started {83							return Err("<named argument>")84						}85						unnamed.push(value);86					}87				}88				Ok(expr::ArgsDesc::new(unnamed, named))89			}9091		pub rule bind(s: &ParserSettings) -> expr::BindSpec92			= name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}}93			/ name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}}94		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt95			= keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }9697		pub rule whole_line() -> &'input str98			= str:$((!['\n'][_])* "\n") {str}99		pub rule string_block() -> String100			= "|||" (!['\n']single_whitespace())* "\n"101			  empty_lines:$(['\n']*)102			  prefix:[' ' | '\t']+ first_line:whole_line()103			  lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*104			  [' ' | '\t']*<, {prefix.len() - 1}> "|||"105			  {let mut l = empty_lines.to_owned(); l.push_str(first_line); l.extend(lines); l}106		pub rule string() -> String107			= quiet!{ "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()}108			/ "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()}109			/ "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}110			/ "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}111			/ string_block() } / expected!("<string>")112113		pub rule field_name(s: &ParserSettings) -> expr::FieldName114			= name:$(id()) {expr::FieldName::Fixed(name.into())}115			/ name:string() {expr::FieldName::Fixed(name.into())}116			/ "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}117		pub rule visibility() -> expr::Visibility118			= ":::" {expr::Visibility::Unhide}119			/ "::" {expr::Visibility::Hidden}120			/ ":" {expr::Visibility::Normal}121		pub rule field(s: &ParserSettings) -> expr::FieldMember122			= name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{123				name,124				plus: plus.is_some(),125				params: None,126				visibility,127				value,128			}}129			/ name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{130				name,131				plus: false,132				params: Some(params),133				visibility,134				value,135			}}136		pub rule obj_local(s: &ParserSettings) -> BindSpec137			= keyword("local") _ bind:bind(s) {bind}138		pub rule member(s: &ParserSettings) -> expr::Member139			= bind:obj_local(s) {expr::Member::BindStmt(bind)}140			/ assertion:assertion(s) {expr::Member::AssertStmt(assertion)}141			/ field:field(s) {expr::Member::Field(field)}142		pub rule objinside(s: &ParserSettings) -> expr::ObjBody143			= pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ plus:"+"? _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {144				let mut compspecs = vec![CompSpec::ForSpec(forspec)];145				compspecs.extend(others.unwrap_or_default());146				expr::ObjBody::ObjComp(expr::ObjComp{147					pre_locals,148					key,149					plus: plus.is_some(),150					value,151					post_locals,152					compspecs,153				})154			}155			/ members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}156		pub rule ifspec(s: &ParserSettings) -> IfSpecData157			= keyword("if") _ expr:expr(s) {IfSpecData(expr)}158		pub rule forspec(s: &ParserSettings) -> ForSpecData159			= keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)}160		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>161			= s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}162		pub rule local_expr(s: &ParserSettings) -> Expr163			= keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }164		pub rule string_expr(s: &ParserSettings) -> Expr165			= s:string() {Expr::Str(s.into())}166		pub rule obj_expr(s: &ParserSettings) -> Expr167			= "{" _ body:objinside(s) _ "}" {Expr::Obj(body)}168		pub rule array_expr(s: &ParserSettings) -> Expr169			= "[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}170		pub rule array_comp_expr(s: &ParserSettings) -> Expr171			= "[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {172				let mut specs = vec![CompSpec::ForSpec(forspec)];173				specs.extend(others.unwrap_or_default());174				Expr::ArrComp(expr, specs)175			}176		pub rule number_expr(s: &ParserSettings) -> Expr177			= n:number() { expr::Expr::Num(n) }178		pub rule var_expr(s: &ParserSettings) -> Expr179			= n:$(id()) { expr::Expr::Var(n.into()) }180		pub rule if_then_else_expr(s: &ParserSettings) -> Expr181			= cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{182				cond,183				cond_then,184				cond_else,185			}}186187		pub rule literal(s: &ParserSettings) -> Expr188			= v:(189				keyword("null") {LiteralType::Null}190				/ keyword("true") {LiteralType::True}191				/ keyword("false") {LiteralType::False}192				/ keyword("self") {LiteralType::This}193				/ keyword("$") {LiteralType::Dollar}194				/ keyword("super") {LiteralType::Super}195			) {Expr::Literal(v)}196197		pub rule expr_basic(s: &ParserSettings) -> Expr198			= literal(s)199200			/ quiet!{"$intrinsic(" name:$(id()) ")" {Expr::Intrinsic(name.into())}}201202			/ string_expr(s) / number_expr(s)203			/ array_expr(s)204			/ obj_expr(s)205			/ array_expr(s)206			/ array_comp_expr(s)207208			/ keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}209			/ keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}210211			/ var_expr(s)212			/ local_expr(s)213			/ if_then_else_expr(s)214215			/ keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}216			/ assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }217218			/ keyword("error") _ expr:expr(s) { Expr::ErrorStmt(expr) }219220		rule slice_part(s: &ParserSettings) -> Option<LocExpr>221			= e:(_ e:expr(s) _{e})? {e}222		pub rule slice_desc(s: &ParserSettings) -> SliceDesc223			= start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {224				let (end, step) = if let Some((end, step)) = pair {225					(end, step)226				}else{227					(None, None)228				};229230				SliceDesc { start, end, step }231			}232233		rule binop(x: rule<()>) -> ()234			= quiet!{ x() } / expected!("<binary op>")235		rule unaryop(x: rule<()>) -> ()236			= quiet!{ x() } / expected!("<unary op>")237238239		use BinaryOpType::*;240		use UnaryOpType::*;241		rule expr(s: &ParserSettings) -> LocExpr242			= precedence! {243				start:position!() v:@ end:position!() { loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end)) }244				--245				a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}246				--247				a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}248				--249				a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}250				--251				a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}252				--253				a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}254				--255				a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}256				a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}257				--258				a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}259				a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}260				a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}261				a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}262				a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}263				--264				a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}265				a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}266				--267				a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}268				a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}269				--270				a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}271				a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}272				a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}273				--274						unaryop(<"-">) _ b:@ {expr_un!(Minus b)}275						unaryop(<"!">) _ b:@ {expr_un!(Not b)}276						unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}277				--278				a:(@) _ "[" _ e:slice_desc(s) _ "]" {Expr::Slice(a, e)}279				a:(@) _ "." _ e:$(id()) {Expr::Index(a, el!(Expr::Str(e.into())))}280				a:(@) _ "[" _ e:expr(s) _ "]" {Expr::Index(a, e)}281				a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {Expr::Apply(a, args, ts.is_some())}282				a:(@) _ "{" _ body:objinside(s) _ "}" {Expr::ObjExtend(a, body)}283				--284				e:expr_basic(s) {e}285				"(" _ e:expr(s) _ ")" {Expr::Parened(e)}286			}287288		pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}289	}290}291292pub type ParseError = peg::error::ParseError<peg::str::LineCol>;293pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {294	jsonnet_parser::jsonnet(str, settings)295}296297#[macro_export]298macro_rules! el {299	($expr:expr) => {300		LocExpr(std::rc::Rc::new($expr), None)301	};302}303304#[cfg(test)]305pub mod tests {306	use super::{expr::*, parse};307	use crate::ParserSettings;308	use std::path::PathBuf;309	use BinaryOpType::*;310311	macro_rules! parse {312		($s:expr) => {313			parse(314				$s,315				&ParserSettings {316					loc_data: false,317					file_name: PathBuf::from("/test.jsonnet").into(),318				},319			)320			.unwrap()321		};322	}323324	macro_rules! el_loc {325		($expr:expr, $loc:expr$(,)?) => {326			LocExpr(std::rc::Rc::new($expr), Some($loc))327		};328	}329330	mod expressions {331		use super::*;332333		pub fn basic_math() -> LocExpr {334			el!(Expr::BinaryOp(335				el!(Expr::Num(2.0)),336				Add,337				el!(Expr::BinaryOp(338					el!(Expr::Num(2.0)),339					Mul,340					el!(Expr::Num(2.0)),341				)),342			))343		}344	}345346	#[test]347	fn multiline_string() {348		assert_eq!(349			parse!("|||\n    Hello world!\n     a\n|||"),350			el!(Expr::Str("Hello world!\n a\n".into())),351		);352		assert_eq!(353			parse!("|||\n  Hello world!\n   a\n|||"),354			el!(Expr::Str("Hello world!\n a\n".into())),355		);356		assert_eq!(357			parse!("|||\n\t\tHello world!\n\t\t\ta\n|||"),358			el!(Expr::Str("Hello world!\n\ta\n".into())),359		);360		assert_eq!(361			parse!("|||\n   Hello world!\n    a\n |||"),362			el!(Expr::Str("Hello world!\n a\n".into())),363		);364	}365366	#[test]367	fn slice() {368		parse!("a[1:]");369		parse!("a[1::]");370		parse!("a[:1:]");371		parse!("a[::1]");372		parse!("str[:len - 1]");373	}374375	#[test]376	fn string_escaping() {377		assert_eq!(378			parse!(r#""Hello, \"world\"!""#),379			el!(Expr::Str(r#"Hello, "world"!"#.into())),380		);381		assert_eq!(382			parse!(r#"'Hello \'world\'!'"#),383			el!(Expr::Str("Hello 'world'!".into())),384		);385		assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),);386	}387388	#[test]389	fn string_unescaping() {390		assert_eq!(391			parse!(r#""Hello\nWorld""#),392			el!(Expr::Str("Hello\nWorld".into())),393		);394	}395396	#[test]397	fn string_verbantim() {398		assert_eq!(399			parse!(r#"@"Hello\n""World""""#),400			el!(Expr::Str("Hello\\n\"World\"".into())),401		);402	}403404	#[test]405	fn imports() {406		assert_eq!(407			parse!("import \"hello\""),408			el!(Expr::Import(PathBuf::from("hello"))),409		);410		assert_eq!(411			parse!("importstr \"garnish.txt\""),412			el!(Expr::ImportStr(PathBuf::from("garnish.txt")))413		);414	}415416	#[test]417	fn empty_object() {418		assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));419	}420421	#[test]422	fn basic_math() {423		assert_eq!(424			parse!("2+2*2"),425			el!(Expr::BinaryOp(426				el!(Expr::Num(2.0)),427				Add,428				el!(Expr::BinaryOp(429					el!(Expr::Num(2.0)),430					Mul,431					el!(Expr::Num(2.0))432				))433			))434		);435	}436437	#[test]438	fn basic_math_with_indents() {439		assert_eq!(parse!("2	+ 	  2	  *	2   	"), expressions::basic_math());440	}441442	#[test]443	fn basic_math_parened() {444		assert_eq!(445			parse!("2+(2+2*2)"),446			el!(Expr::BinaryOp(447				el!(Expr::Num(2.0)),448				Add,449				el!(Expr::Parened(expressions::basic_math())),450			))451		);452	}453454	/// Comments should not affect parsing455	#[test]456	fn comments() {457		assert_eq!(458			parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),459			el!(Expr::BinaryOp(460				el!(Expr::Num(2.0)),461				Add,462				el!(Expr::BinaryOp(463					el!(Expr::Num(3.0)),464					Mul,465					el!(Expr::Num(4.0))466				))467			))468		);469	}470471	/// Comments should be able to be escaped472	#[test]473	fn comment_escaping() {474		assert_eq!(475			parse!("2/*\\*/+*/ - 22"),476			el!(Expr::BinaryOp(477				el!(Expr::Num(2.0)),478				Sub,479				el!(Expr::Num(22.0))480			))481		);482	}483484	#[test]485	fn suffix() {486		// assert_eq!(parse!("std.test"), el!(Expr::Num(2.2)));487		// assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2)));488		// assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2)));489		// assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2)))490	}491492	#[test]493	fn array_comp() {494		use Expr::*;495		assert_eq!(496			parse!("[std.deepJoin(x) for x in arr]"),497			el!(ArrComp(498				el!(Apply(499					el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))),500					ArgsDesc::new(vec![el!(Var("x".into()))], vec![]),501					false,502				)),503				vec![CompSpec::ForSpec(ForSpecData(504					"x".into(),505					el!(Var("arr".into()))506				))]507			)),508		)509	}510511	#[test]512	fn reserved() {513		use Expr::*;514		assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));515		assert_eq!(parse!("nulla"), el!(Var("nulla".into())));516	}517518	#[test]519	fn multiple_args_buf() {520		parse!("a(b, null_fields)");521	}522523	#[test]524	fn infix_precedence() {525		use Expr::*;526		assert_eq!(527			parse!("!a && !b"),528			el!(BinaryOp(529				el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),530				And,531				el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))532			))533		);534	}535536	#[test]537	fn infix_precedence_division() {538		use Expr::*;539		assert_eq!(540			parse!("!a / !b"),541			el!(BinaryOp(542				el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),543				Div,544				el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))545			))546		);547	}548549	#[test]550	fn double_negation() {551		use Expr::*;552		assert_eq!(553			parse!("!!a"),554			el!(UnaryOp(555				UnaryOpType::Not,556				el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()))))557			))558		)559	}560561	#[test]562	fn array_test_error() {563		parse!("[a for a in b if c for e in f]");564		//                    ^^^^ failed code565	}566567	#[test]568	fn missing_newline_between_comment_and_eof() {569		parse!(570			"{a:1}571572			//+213"573		);574	}575576	#[test]577	fn default_param_before_nondefault() {578		parse!("local x(foo = 'foo', bar) = null; null");579	}580581	#[test]582	fn can_parse_stdlib() {583		parse!(jrsonnet_stdlib::STDLIB_STR);584	}585586	#[test]587	fn add_location_info_to_all_sub_expressions() {588		use Expr::*;589590		let file_name: std::rc::Rc<std::path::Path> = PathBuf::from("/test.jsonnet").into();591		let expr = parse(592			"{} { local x = 1, x: x } + {}",593			&ParserSettings {594				loc_data: true,595				file_name: file_name.clone(),596			},597		)598		.unwrap();599		assert_eq!(600			expr,601			el_loc!(602				BinaryOp(603					el_loc!(604						ObjExtend(605							el_loc!(606								Obj(ObjBody::MemberList(vec![])),607								ExprLocation(file_name.clone(), 0, 2)608							),609							ObjBody::MemberList(vec![610								Member::BindStmt(BindSpec {611									name: "x".into(),612									params: None,613									value: el_loc!(614										Num(1.0),615										ExprLocation(file_name.clone(), 15, 16)616									)617								}),618								Member::Field(FieldMember {619									name: FieldName::Fixed("x".into()),620									plus: false,621									params: None,622									visibility: Visibility::Normal,623									value: el_loc!(624										Var("x".into()),625										ExprLocation(file_name.clone(), 21, 22)626									),627								})628							])629						),630						ExprLocation(file_name.clone(), 0, 24)631					),632					BinaryOpType::Add,633					el_loc!(634						Obj(ObjBody::MemberList(vec![])),635						ExprLocation(file_name.clone(), 27, 29)636					),637				),638				ExprLocation(file_name.clone(), 0, 29),639			),640		);641	}642	// From source code643	/*644	#[bench]645	fn bench_parse_peg(b: &mut Bencher) {646		b.iter(|| parse!(jrsonnet_stdlib::STDLIB_STR))647	}648	*/649}