1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::{path::PathBuf, rc::Rc};8mod expr;9pub use expr::*;1011enum Suffix {12 String(String),13 Slice(SliceDesc),14 Expression(LocExpr),15 Apply(expr::ArgsDesc),16 Extend(expr::ObjBody),17}18struct LocSuffix(Suffix, ExprLocation);1920pub struct ParserSettings {21 pub loc_data: bool,22 pub file_name: PathBuf,23}2425parser! {26 grammar jsonnet_parser() for str {27 use peg::ParseLiteral;2829 30 rule comment()31 = "//" (!['\n'][_])* "\n"32 / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"33 / "#" (!['\n'][_])* "\n"3435 rule _() = ([' ' | '\n' | '\t'] / comment())*3637 38 rule comma() = quiet!{_ "," _} / expected!("<comma>")39 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}40 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}41 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']42 43 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }44 45 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")4647 48 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()49 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")5051 rule keyword(id: &'static str)52 = ##parse_string_literal(id) end_of_ident()53 54 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr55 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}5657 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }58 pub rule params(s: &ParserSettings) -> expr::ParamsDesc59 = params:(param(s) ** comma()) {60 let mut defaults_started = false;61 for param in ¶ms {62 defaults_started = defaults_started || param.1.is_some();63 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");64 }65 expr::ParamsDesc(params)66 }67 / { expr::ParamsDesc(Vec::new()) }6869 pub rule arg(s: &ParserSettings) -> expr::Arg70 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}71 / expr:expr(s) {expr::Arg(None, expr)}72 pub rule args(s: &ParserSettings) -> expr::ArgsDesc73 = args:arg(s) ** comma() comma()? {74 let mut named_started = false;75 for arg in &args {76 named_started = named_started || arg.0.is_some();77 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");78 }79 expr::ArgsDesc(args)80 }81 / { expr::ArgsDesc(Vec::new()) }8283 pub rule bind(s: &ParserSettings) -> expr::BindSpec84 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}85 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}86 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt87 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }88 pub rule string() -> String89 = v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}90 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}91 pub rule field_name(s: &ParserSettings) -> expr::FieldName92 = name:id() {expr::FieldName::Fixed(name)}93 / name:string() {expr::FieldName::Fixed(name)}94 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}95 pub rule visibility() -> expr::Visibility96 = ":::" {expr::Visibility::Unhide}97 / "::" {expr::Visibility::Hidden}98 / ":" {expr::Visibility::Normal}99 pub rule field(s: &ParserSettings) -> expr::FieldMember100 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{101 name,102 plus: plus.is_some(),103 params: None,104 visibility,105 value,106 }}107 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{108 name,109 plus: false,110 params: Some(params),111 visibility,112 value,113 }}114 pub rule obj_local(s: &ParserSettings) -> BindSpec115 = keyword("local") _ bind:bind(s) {bind}116 pub rule member(s: &ParserSettings) -> expr::Member117 = bind:obj_local(s) {expr::Member::BindStmt(bind)}118 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}119 / field:field(s) {expr::Member::Field(field)}120 pub rule objinside(s: &ParserSettings) -> expr::ObjBody121 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {122 expr::ObjBody::ObjComp {123 pre_locals,124 key,125 value,126 post_locals,127 rest: [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat(),128 }129 }130 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}131 pub rule ifspec(s: &ParserSettings) -> IfSpecData132 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}133 pub rule forspec(s: &ParserSettings) -> ForSpecData134 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}135 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>136 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}137 pub rule local_expr(s: &ParserSettings) -> LocExpr138 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)139 pub rule string_expr(s: &ParserSettings) -> LocExpr140 = l(s, <s:string() {Expr::Str(s)}>)141 pub rule obj_expr(s: &ParserSettings) -> LocExpr142 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)143 pub rule array_expr(s: &ParserSettings) -> LocExpr144 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)145 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr146 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)147 pub rule number_expr(s: &ParserSettings) -> LocExpr148 = l(s,<n:number() { expr::Expr::Num(n) }>)149 pub rule var_expr(s: &ParserSettings) -> LocExpr150 = l(s,<n:id() { expr::Expr::Var(n) }>)151 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr152 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{153 cond,154 cond_then,155 cond_else,156 }}>)157158 pub rule literal(s: &ParserSettings) -> LocExpr159 = l(s,<v:(160 keyword("null") {LiteralType::Null}161 / keyword("true") {LiteralType::True}162 / keyword("false") {LiteralType::False}163 / keyword("self") {LiteralType::This}164 / keyword("$") {LiteralType::Dollar}165 / keyword("super") {LiteralType::Super}166 ) {Expr::Literal(v)}>)167168 pub rule expr_basic(s: &ParserSettings) -> LocExpr169 = literal(s)170171 / string_expr(s) / number_expr(s)172 / array_expr(s)173 / obj_expr(s)174 / array_expr(s)175 / array_comp_expr(s)176177 / var_expr(s)178 / local_expr(s)179 / if_then_else_expr(s)180181 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)182 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)183184 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)185186 rule expr_basic_with_suffix(s: &ParserSettings) -> LocExpr187 = a:expr_basic(s) suffixes:(_ suffix:l_expr_suffix(s) {suffix})* {188 let mut cur = a;189 for suffix in suffixes {190 let LocSuffix(suffix, location) = suffix;191 cur = LocExpr(Rc::new(match suffix {192 Suffix::String(index) => Expr::Index(cur, loc_expr!(Expr::Str(index), s.loc_data, (s.file_name.clone(), location.1, location.2))),193 Suffix::Slice(desc) => Expr::Slice(cur, desc),194 Suffix::Expression(index) => Expr::Index(cur, index),195 Suffix::Apply(args) => Expr::Apply(cur, args),196 Suffix::Extend(body) => Expr::ObjExtend(cur, body),197 }), if s.loc_data { Some(Rc::new(location)) } else { None })198 }199 cur200 }201202 pub rule slice_desc(s: &ParserSettings) -> SliceDesc203 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {204 if let Some((end, step)) = pair {205 SliceDesc { start, end, step }206 }else{207 SliceDesc { start, end: None, step: None }208 }209 }210211 rule expr_suffix(s: &ParserSettings) -> Suffix212 = "." _ s:id() { Suffix::String(s) }213 / "[" _ s:slice_desc(s) _ "]" { Suffix::Slice(s) }214 / "[" _ s:expr(s) _ "]" { Suffix::Expression(s) }215 / "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }216 / "{" _ body:objinside(s) _ "}" { Suffix::Extend(body) }217 rule l_expr_suffix(s: &ParserSettings) -> LocSuffix218 = start:position!() suffix:expr_suffix(s) end:position!() {LocSuffix(suffix, ExprLocation(s.file_name.clone(), start, end))}219220 rule expr(s: &ParserSettings) -> LocExpr221 = start:position!() a:precedence! {222 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}223 --224 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}225 --226 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}227 --228 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}229 --230 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}231 --232 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply(233 el!(Expr::Index(234 el!(Expr::Var("std".to_owned())),235 el!(Expr::Str("equals".to_owned()))236 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])237 ))}238 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply(239 el!(Expr::Index(240 el!(Expr::Var("std".to_owned())),241 el!(Expr::Str("equals".to_owned()))242 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])243 ))))}244 --245 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}246 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}247 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}248 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}249 --250 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}251 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}252 --253 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}254 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}255 --256 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}257 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}258 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply(259 el!(Expr::Index(260 el!(Expr::Var("std".to_owned())),261 el!(Expr::Str("mod".to_owned()))262 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])263 ))}264 --265 "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))}266 "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))}267 "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) }268 --269 e:expr_basic_with_suffix(s) {e}270 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}271 } end:position!() {272 let LocExpr(e, _) = a;273 LocExpr(e, if s.loc_data {274 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))275 } else {276 None277 })278 }279 / e:expr_basic_with_suffix(s) {e}280281 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}282 }283}284285pub fn parse(286 str: &str,287 settings: &ParserSettings,288) -> Result<LocExpr, peg::error::ParseError<peg::str::LineCol>> {289 jsonnet_parser::jsonnet(str, settings)290}291292#[macro_export]293macro_rules! el {294 ($expr:expr) => {295 LocExpr(std::rc::Rc::new($expr), None)296 };297}298299#[cfg(test)]300pub mod tests {301 use super::{expr::*, parse};302 use crate::ParserSettings;303 use std::path::PathBuf;304305 macro_rules! parse {306 ($s:expr) => {307 parse(308 $s,309 &ParserSettings {310 loc_data: false,311 file_name: PathBuf::from("/test.jsonnet"),312 },313 )314 .unwrap()315 };316 }317318 mod expressions {319 use super::*;320321 pub fn basic_math() -> LocExpr {322 el!(Expr::BinaryOp(323 el!(Expr::Num(2.0)),324 BinaryOpType::Add,325 el!(Expr::BinaryOp(326 el!(Expr::Num(2.0)),327 BinaryOpType::Mul,328 el!(Expr::Num(2.0)),329 )),330 ))331 }332 }333334 #[test]335 fn empty_object() {336 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));337 }338339 #[test]340 fn basic_math() {341 assert_eq!(342 parse!("2+2*2"),343 el!(Expr::BinaryOp(344 el!(Expr::Num(2.0)),345 BinaryOpType::Add,346 el!(Expr::BinaryOp(347 el!(Expr::Num(2.0)),348 BinaryOpType::Mul,349 el!(Expr::Num(2.0))350 ))351 ))352 );353 }354355 #[test]356 fn basic_math_with_indents() {357 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());358 }359360 #[test]361 fn basic_math_parened() {362 assert_eq!(363 parse!("2+(2+2*2)"),364 el!(Expr::BinaryOp(365 el!(Expr::Num(2.0)),366 BinaryOpType::Add,367 el!(Expr::Parened(expressions::basic_math())),368 ))369 );370 }371372 373 #[test]374 fn comments() {375 assert_eq!(376 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),377 el!(Expr::BinaryOp(378 el!(Expr::Num(2.0)),379 BinaryOpType::Add,380 el!(Expr::BinaryOp(381 el!(Expr::Num(3.0)),382 BinaryOpType::Mul,383 el!(Expr::Num(4.0))384 ))385 ))386 );387 }388389 390 #[test]391 fn comment_escaping() {392 assert_eq!(393 parse!("2/*\\*/+*/ - 22"),394 el!(Expr::BinaryOp(395 el!(Expr::Num(2.0)),396 BinaryOpType::Sub,397 el!(Expr::Num(22.0))398 ))399 );400 }401402 #[test]403 fn array_comp() {404 use Expr::*;405 assert_eq!(406 parse!("[std.deepJoin(x) for x in arr]"),407 el!(ArrComp(408 el!(Apply(409 el!(Index(410 el!(Var("std".to_owned())),411 el!(Str("deepJoin".to_owned()))412 )),413 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])414 )),415 vec![CompSpec::ForSpec(ForSpecData(416 "x".to_owned(),417 el!(Var("arr".to_owned()))418 ))]419 )),420 )421 }422423 #[test]424 fn reserved() {425 use Expr::*;426 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));427 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));428 }429430 #[test]431 fn multiple_args_buf() {432 parse!("a(b, null_fields)");433 }434435 #[test]436 fn infix_precedence() {437 use Expr::*;438 assert_eq!(439 parse!("!a && !b"),440 el!(BinaryOp(441 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),442 BinaryOpType::And,443 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))444 ))445 );446 }447448 #[test]449 fn infix_precedence_division() {450 use Expr::*;451 assert_eq!(452 parse!("!a / !b"),453 el!(BinaryOp(454 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),455 BinaryOpType::Div,456 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))457 ))458 );459 }460461 #[test]462 fn double_negation() {463 use Expr::*;464 assert_eq!(465 parse!("!!a"),466 el!(UnaryOp(467 UnaryOpType::Not,468 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned()))))469 ))470 )471 }472473 #[test]474 fn array_test_error() {475 parse!("[a for a in b if c for e in f]");476 477 }478479 #[test]480 fn can_parse_stdlib() {481 parse!(jsonnet_stdlib::STDLIB_STR);482 }483484 use test::Bencher;485486 487 #[bench]488 fn bench_parse_peg(b: &mut Bencher) {489 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))490 }491492 493 #[bench]494 fn bench_parse_serde_bincode(b: &mut Bencher) {495 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();496 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))497 }498}