1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::{path::PathBuf, rc::Rc};8mod expr;9pub use expr::*;10pub use peg;1112pub struct ParserSettings {13 pub loc_data: bool,14 pub file_name: Rc<PathBuf>,15}1617parser! {18 grammar jsonnet_parser() for str {19 use peg::ParseLiteral;2021 22 rule comment()23 = "//" (!['\n'][_])* "\n"24 / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/"25 / "#" (!['\n'][_])* "\n"2627 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")28 rule _() = single_whitespace()*2930 31 rule comma() = quiet!{_ "," _} / expected!("<comma>")32 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}33 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}34 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']35 36 rule uint() -> u64 = a:$(digit()+) { a.parse().unwrap() }37 38 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")3940 41 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()42 rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")4344 rule keyword(id: &'static str)45 = ##parse_string_literal(id) end_of_ident()46 47 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr48 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}4950 pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) }51 pub rule params(s: &ParserSettings) -> expr::ParamsDesc52 = params:param(s) ** comma() comma()? {53 let mut defaults_started = false;54 for param in ¶ms {55 defaults_started = defaults_started || param.1.is_some();56 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");57 }58 expr::ParamsDesc(Rc::new(params))59 }60 / { expr::ParamsDesc(Rc::new(Vec::new())) }6162 pub rule arg(s: &ParserSettings) -> expr::Arg63 = name:$(id()) _ "=" _ expr:expr(s) {expr::Arg(Some(name.into()), expr)}64 / expr:expr(s) {expr::Arg(None, expr)}65 pub rule args(s: &ParserSettings) -> expr::ArgsDesc66 = args:arg(s) ** comma() comma()? {67 let mut named_started = false;68 for arg in &args {69 named_started = named_started || arg.0.is_some();70 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");71 }72 expr::ArgsDesc(args)73 }74 / { expr::ArgsDesc(Vec::new()) }7576 pub rule bind(s: &ParserSettings) -> expr::BindSpec77 = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}}78 / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}}79 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt80 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }8182 pub rule whole_line() -> &'input str83 = str:$((!['\n'][_])* "\n") {str}84 pub rule string_block() -> String85 = "|||" (!['\n']single_whitespace())* "\n"86 prefix:[' ']+ first_line:whole_line()87 lines:([' ']*<{prefix.len()}> s:whole_line() {s})*88 [' ']*<, {prefix.len() - 1}> "|||"89 {let mut l = first_line.to_owned(); l.extend(lines); l}90 pub rule string() -> String91 = "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()}92 / "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()}93 / "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}94 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}95 / string_block()9697 pub rule field_name(s: &ParserSettings) -> expr::FieldName98 = name:$(id()) {expr::FieldName::Fixed(name.into())}99 / name:string() {expr::FieldName::Fixed(name.into())}100 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}101 pub rule visibility() -> expr::Visibility102 = ":::" {expr::Visibility::Unhide}103 / "::" {expr::Visibility::Hidden}104 / ":" {expr::Visibility::Normal}105 pub rule field(s: &ParserSettings) -> expr::FieldMember106 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{107 name,108 plus: plus.is_some(),109 params: None,110 visibility,111 value,112 }}113 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{114 name,115 plus: false,116 params: Some(params),117 visibility,118 value,119 }}120 pub rule obj_local(s: &ParserSettings) -> BindSpec121 = keyword("local") _ bind:bind(s) {bind}122 pub rule member(s: &ParserSettings) -> expr::Member123 = bind:obj_local(s) {expr::Member::BindStmt(bind)}124 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}125 / field:field(s) {expr::Member::Field(field)}126 pub rule objinside(s: &ParserSettings) -> expr::ObjBody127 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {128 let mut compspecs = vec![CompSpec::ForSpec(forspec)];129 compspecs.extend(others.unwrap_or_default());130 expr::ObjBody::ObjComp(expr::ObjComp{131 pre_locals,132 key,133 value,134 post_locals,135 compspecs,136 })137 }138 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}139 pub rule ifspec(s: &ParserSettings) -> IfSpecData140 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}141 pub rule forspec(s: &ParserSettings) -> ForSpecData142 = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)}143 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>144 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}145 pub rule local_expr(s: &ParserSettings) -> LocExpr146 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)147 pub rule string_expr(s: &ParserSettings) -> LocExpr148 = l(s, <s:string() {Expr::Str(s.into())}>)149 pub rule obj_expr(s: &ParserSettings) -> LocExpr150 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)151 pub rule array_expr(s: &ParserSettings) -> LocExpr152 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)153 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr154 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {155 let mut specs = vec![CompSpec::ForSpec(forspec)];156 specs.extend(others.unwrap_or_default());157 Expr::ArrComp(expr, specs)158 }>)159 pub rule number_expr(s: &ParserSettings) -> LocExpr160 = l(s,<n:number() { expr::Expr::Num(n) }>)161 pub rule var_expr(s: &ParserSettings) -> LocExpr162 = l(s,<n:$(id()) { expr::Expr::Var(n.into()) }>)163 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr164 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{165 cond,166 cond_then,167 cond_else,168 }}>)169170 pub rule literal(s: &ParserSettings) -> LocExpr171 = l(s,<v:(172 keyword("null") {LiteralType::Null}173 / keyword("true") {LiteralType::True}174 / keyword("false") {LiteralType::False}175 / keyword("self") {LiteralType::This}176 / keyword("$") {LiteralType::Dollar}177 / keyword("super") {LiteralType::Super}178 ) {Expr::Literal(v)}>)179180 pub rule expr_basic(s: &ParserSettings) -> LocExpr181 = literal(s)182183 / string_expr(s) / number_expr(s)184 / array_expr(s)185 / obj_expr(s)186 / array_expr(s)187 / array_comp_expr(s)188189 / l(s,<keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}>)190 / l(s,<keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}>)191192 / var_expr(s)193 / local_expr(s)194 / if_then_else_expr(s)195196 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)197 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)198199 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)200201 rule slice_part(s: &ParserSettings) -> Option<LocExpr>202 = e:(_ e:expr(s) _{e})? {e}203 pub rule slice_desc(s: &ParserSettings) -> SliceDesc204 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {205 let (end, step) = if let Some((end, step)) = pair {206 (end, step)207 }else{208 (None, None)209 };210211 SliceDesc { start, end, step }212 }213214 rule expr(s: &ParserSettings) -> LocExpr215 = start:position!() a:precedence! {216 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}217 --218 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}219 --220 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}221 --222 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}223 --224 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}225 --226 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply(227 el!(Expr::Index(228 el!(Expr::Var("std".into())),229 el!(Expr::Str("equals".into()))230 )),231 ArgsDesc(vec![Arg(None, a), Arg(None, b)]),232 true233 ))}234 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply(235 el!(Expr::Index(236 el!(Expr::Var("std".into())),237 el!(Expr::Str("equals".into()))238 )),239 ArgsDesc(vec![Arg(None, a), Arg(None, b)]),240 true241 ))))}242 --243 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}244 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}245 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}246 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}247 a:(@) _ keyword("in") _ b:@ {loc_expr_todo!(Expr::Apply(248 el!(Expr::Index(249 el!(Expr::Var("std".into())),250 el!(Expr::Str("objectHasEx".into()))251 )), ArgsDesc(vec![Arg(None, b), Arg(None, a), Arg(None, el!(Expr::Literal(LiteralType::True)))]),252 true253 ))}254 --255 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}256 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}257 --258 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}259 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}260 --261 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}262 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}263 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply(264 el!(Expr::Index(265 el!(Expr::Var("std".into())),266 el!(Expr::Str("mod".into()))267 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)]),268 true269 ))}270 --271 "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))}272 "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))}273 "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) }274 --275 a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Apply(276 el!(Expr::Index(277 el!(Expr::Var("std".into())),278 el!(Expr::Str("slice".into())),279 )),280 ArgsDesc(vec![281 Arg(None, a),282 Arg(None, s.start.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))),283 Arg(None, s.end.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))),284 Arg(None, s.step.unwrap_or_else(||el!(Expr::Literal(LiteralType::Null)))),285 ]),286 true,287 ))}288 a:(@) _ "." _ s:$(id()) {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s.into()))))}289 a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))}290 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args, ts.is_some()))}291 a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))}292 --293 e:expr_basic(s) {e}294 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}295 } end:position!() {296 let LocExpr(e, _) = a;297 LocExpr(e, if s.loc_data {298 Some(ExprLocation(s.file_name.clone(), start, end))299 } else {300 None301 })302 }303 / e:expr_basic(s) {e}304305 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}306 }307}308309pub type ParseError = peg::error::ParseError<peg::str::LineCol>;310pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {311 jsonnet_parser::jsonnet(str, settings)312}313314#[macro_export]315macro_rules! el {316 ($expr:expr) => {317 LocExpr(std::rc::Rc::new($expr), None)318 };319}320321#[cfg(test)]322pub mod tests {323 use super::{expr::*, parse};324 use crate::ParserSettings;325 use std::path::PathBuf;326 use std::rc::Rc;327328 macro_rules! parse {329 ($s:expr) => {330 parse(331 $s,332 &ParserSettings {333 loc_data: false,334 file_name: Rc::new(PathBuf::from("/test.jsonnet")),335 },336 )337 .unwrap()338 };339 }340341 mod expressions {342 use super::*;343344 pub fn basic_math() -> LocExpr {345 el!(Expr::BinaryOp(346 el!(Expr::Num(2.0)),347 BinaryOpType::Add,348 el!(Expr::BinaryOp(349 el!(Expr::Num(2.0)),350 BinaryOpType::Mul,351 el!(Expr::Num(2.0)),352 )),353 ))354 }355 }356357 #[test]358 fn multiline_string() {359 assert_eq!(360 parse!("|||\n Hello world!\n a\n|||"),361 el!(Expr::Str("Hello world!\n a\n".into())),362 )363 }364365 #[test]366 fn slice() {367 parse!("a[1:]");368 parse!("a[1::]");369 parse!("a[:1:]");370 parse!("a[::1]");371 parse!("str[:len - 1]");372 }373374 #[test]375 fn string_escaping() {376 assert_eq!(377 parse!(r#""Hello, \"world\"!""#),378 el!(Expr::Str(r#"Hello, "world"!"#.into())),379 );380 assert_eq!(381 parse!(r#"'Hello \'world\'!'"#),382 el!(Expr::Str("Hello 'world'!".into())),383 );384 assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),);385 }386387 #[test]388 fn string_unescaping() {389 assert_eq!(390 parse!(r#""Hello\nWorld""#),391 el!(Expr::Str("Hello\nWorld".into())),392 );393 }394395 #[test]396 fn string_verbantim() {397 assert_eq!(398 parse!(r#"@"Hello\n""World""""#),399 el!(Expr::Str("Hello\\n\"World\"".into())),400 );401 }402403 #[test]404 fn imports() {405 assert_eq!(406 parse!("import \"hello\""),407 el!(Expr::Import(PathBuf::from("hello"))),408 );409 assert_eq!(410 parse!("importstr \"garnish.txt\""),411 el!(Expr::ImportStr(PathBuf::from("garnish.txt")))412 );413 }414415 #[test]416 fn empty_object() {417 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));418 }419420 #[test]421 fn basic_math() {422 assert_eq!(423 parse!("2+2*2"),424 el!(Expr::BinaryOp(425 el!(Expr::Num(2.0)),426 BinaryOpType::Add,427 el!(Expr::BinaryOp(428 el!(Expr::Num(2.0)),429 BinaryOpType::Mul,430 el!(Expr::Num(2.0))431 ))432 ))433 );434 }435436 #[test]437 fn basic_math_with_indents() {438 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());439 }440441 #[test]442 fn basic_math_parened() {443 assert_eq!(444 parse!("2+(2+2*2)"),445 el!(Expr::BinaryOp(446 el!(Expr::Num(2.0)),447 BinaryOpType::Add,448 el!(Expr::Parened(expressions::basic_math())),449 ))450 );451 }452453 454 #[test]455 fn comments() {456 assert_eq!(457 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),458 el!(Expr::BinaryOp(459 el!(Expr::Num(2.0)),460 BinaryOpType::Add,461 el!(Expr::BinaryOp(462 el!(Expr::Num(3.0)),463 BinaryOpType::Mul,464 el!(Expr::Num(4.0))465 ))466 ))467 );468 }469470 471 #[test]472 fn comment_escaping() {473 assert_eq!(474 parse!("2/*\\*/+*/ - 22"),475 el!(Expr::BinaryOp(476 el!(Expr::Num(2.0)),477 BinaryOpType::Sub,478 el!(Expr::Num(22.0))479 ))480 );481 }482483 #[test]484 fn suffix() {485 486 487 488 489 }490491 #[test]492 fn array_comp() {493 use Expr::*;494 assert_eq!(495 parse!("[std.deepJoin(x) for x in arr]"),496 el!(ArrComp(497 el!(Apply(498 el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))),499 ArgsDesc(vec![Arg(None, el!(Var("x".into())))]),500 false,501 )),502 vec![CompSpec::ForSpec(ForSpecData(503 "x".into(),504 el!(Var("arr".into()))505 ))]506 )),507 )508 }509510 #[test]511 fn reserved() {512 use Expr::*;513 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));514 assert_eq!(parse!("nulla"), el!(Var("nulla".into())));515 }516517 #[test]518 fn multiple_args_buf() {519 parse!("a(b, null_fields)");520 }521522 #[test]523 fn infix_precedence() {524 use Expr::*;525 assert_eq!(526 parse!("!a && !b"),527 el!(BinaryOp(528 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),529 BinaryOpType::And,530 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))531 ))532 );533 }534535 #[test]536 fn infix_precedence_division() {537 use Expr::*;538 assert_eq!(539 parse!("!a / !b"),540 el!(BinaryOp(541 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),542 BinaryOpType::Div,543 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))544 ))545 );546 }547548 #[test]549 fn double_negation() {550 use Expr::*;551 assert_eq!(552 parse!("!!a"),553 el!(UnaryOp(554 UnaryOpType::Not,555 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()))))556 ))557 )558 }559560 #[test]561 fn array_test_error() {562 parse!("[a for a in b if c for e in f]");563 564 }565566 #[test]567 fn can_parse_stdlib() {568 parse!(jsonnet_stdlib::STDLIB_STR);569 }570571 use test::Bencher;572573 574 #[bench]575 fn bench_parse_peg(b: &mut Bencher) {576 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))577 }578}