1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::{path::PathBuf, rc::Rc};8mod expr;9pub use expr::*;10pub use peg;1112pub struct ParserSettings {13 pub loc_data: bool,14 pub file_name: PathBuf,15}1617parser! {18 grammar jsonnet_parser() for str {19 use peg::ParseLiteral;2021 22 rule comment()23 = "//" (!['\n'][_])* "\n"24 / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"25 / "#" (!['\n'][_])* "\n"2627 rule _() = ([' ' | '\n' | '\t'] / comment())*2829 30 rule comma() = quiet!{_ "," _} / expected!("<comma>")31 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}32 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}33 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']34 35 rule uint() -> u64 = a:$(digit()+) { a.parse().unwrap() }36 37 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")3839 40 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()41 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")4243 rule keyword(id: &'static str)44 = ##parse_string_literal(id) end_of_ident()45 46 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr47 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}4849 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }50 pub rule params(s: &ParserSettings) -> expr::ParamsDesc51 = params:(param(s) ** comma()) {52 let mut defaults_started = false;53 for param in ¶ms {54 defaults_started = defaults_started || param.1.is_some();55 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");56 }57 expr::ParamsDesc(params)58 }59 / { expr::ParamsDesc(Vec::new()) }6061 pub rule arg(s: &ParserSettings) -> expr::Arg62 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}63 / expr:expr(s) {expr::Arg(None, expr)}64 pub rule args(s: &ParserSettings) -> expr::ArgsDesc65 = args:arg(s) ** comma() comma()? {66 let mut named_started = false;67 for arg in &args {68 named_started = named_started || arg.0.is_some();69 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");70 }71 expr::ArgsDesc(args)72 }73 / { expr::ArgsDesc(Vec::new()) }7475 pub rule bind(s: &ParserSettings) -> expr::BindSpec76 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}77 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}78 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt79 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }80 pub rule string() -> String81 = v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}82 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}83 pub rule field_name(s: &ParserSettings) -> expr::FieldName84 = name:id() {expr::FieldName::Fixed(name)}85 / name:string() {expr::FieldName::Fixed(name)}86 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}87 pub rule visibility() -> expr::Visibility88 = ":::" {expr::Visibility::Unhide}89 / "::" {expr::Visibility::Hidden}90 / ":" {expr::Visibility::Normal}91 pub rule field(s: &ParserSettings) -> expr::FieldMember92 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{93 name,94 plus: plus.is_some(),95 params: None,96 visibility,97 value,98 }}99 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{100 name,101 plus: false,102 params: Some(params),103 visibility,104 value,105 }}106 pub rule obj_local(s: &ParserSettings) -> BindSpec107 = keyword("local") _ bind:bind(s) {bind}108 pub rule member(s: &ParserSettings) -> expr::Member109 = bind:obj_local(s) {expr::Member::BindStmt(bind)}110 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}111 / field:field(s) {expr::Member::Field(field)}112 pub rule objinside(s: &ParserSettings) -> expr::ObjBody113 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {114 expr::ObjBody::ObjComp {115 pre_locals,116 key,117 value,118 post_locals,119 rest: [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat(),120 }121 }122 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}123 pub rule ifspec(s: &ParserSettings) -> IfSpecData124 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}125 pub rule forspec(s: &ParserSettings) -> ForSpecData126 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}127 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>128 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}129 pub rule local_expr(s: &ParserSettings) -> LocExpr130 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)131 pub rule string_expr(s: &ParserSettings) -> LocExpr132 = l(s, <s:string() {Expr::Str(s)}>)133 pub rule obj_expr(s: &ParserSettings) -> LocExpr134 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)135 pub rule array_expr(s: &ParserSettings) -> LocExpr136 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)137 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr138 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)139 pub rule number_expr(s: &ParserSettings) -> LocExpr140 = l(s,<n:number() { expr::Expr::Num(n) }>)141 pub rule var_expr(s: &ParserSettings) -> LocExpr142 = l(s,<n:id() { expr::Expr::Var(n) }>)143 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr144 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{145 cond,146 cond_then,147 cond_else,148 }}>)149150 pub rule literal(s: &ParserSettings) -> LocExpr151 = l(s,<v:(152 keyword("null") {LiteralType::Null}153 / keyword("true") {LiteralType::True}154 / keyword("false") {LiteralType::False}155 / keyword("self") {LiteralType::This}156 / keyword("$") {LiteralType::Dollar}157 / keyword("super") {LiteralType::Super}158 ) {Expr::Literal(v)}>)159160 pub rule expr_basic(s: &ParserSettings) -> LocExpr161 = literal(s)162163 / string_expr(s) / number_expr(s)164 / array_expr(s)165 / obj_expr(s)166 / array_expr(s)167 / array_comp_expr(s)168169 / var_expr(s)170 / local_expr(s)171 / if_then_else_expr(s)172173 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)174 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)175176 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)177178 pub rule slice_desc(s: &ParserSettings) -> SliceDesc179 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {180 if let Some((end, step)) = pair {181 SliceDesc { start, end, step }182 }else{183 SliceDesc { start, end: None, step: None }184 }185 }186187 rule expr(s: &ParserSettings) -> LocExpr188 = start:position!() a:precedence! {189 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}190 --191 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}192 --193 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}194 --195 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}196 --197 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}198 --199 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply(200 el!(Expr::Index(201 el!(Expr::Var("std".to_owned())),202 el!(Expr::Str("equals".to_owned()))203 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])204 ))}205 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply(206 el!(Expr::Index(207 el!(Expr::Var("std".to_owned())),208 el!(Expr::Str("equals".to_owned()))209 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])210 ))))}211 --212 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}213 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}214 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}215 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}216 --217 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}218 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}219 --220 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}221 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}222 --223 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}224 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}225 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply(226 el!(Expr::Index(227 el!(Expr::Var("std".to_owned())),228 el!(Expr::Str("mod".to_owned()))229 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])230 ))}231 --232 "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))}233 "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))}234 "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) }235 --236 a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Slice(a, s))}237 a:(@) _ "." _ s:id() {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s))))}238 a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))}239 a:(@) _ "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args))}240 a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))}241 --242 e:expr_basic(s) {e}243 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}244 } end:position!() {245 let LocExpr(e, _) = a;246 LocExpr(e, if s.loc_data {247 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))248 } else {249 None250 })251 }252 / e:expr_basic(s) {e}253254 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}255 }256}257258pub type ParseError = peg::error::ParseError<peg::str::LineCol>;259pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {260 jsonnet_parser::jsonnet(str, settings)261}262263#[macro_export]264macro_rules! el {265 ($expr:expr) => {266 LocExpr(std::rc::Rc::new($expr), None)267 };268}269270#[cfg(test)]271pub mod tests {272 use super::{expr::*, parse};273 use crate::ParserSettings;274 use std::path::PathBuf;275276 macro_rules! parse {277 ($s:expr) => {278 parse(279 $s,280 &ParserSettings {281 loc_data: false,282 file_name: PathBuf::from("/test.jsonnet"),283 },284 )285 .unwrap()286 };287 }288289 mod expressions {290 use super::*;291292 pub fn basic_math() -> LocExpr {293 el!(Expr::BinaryOp(294 el!(Expr::Num(2.0)),295 BinaryOpType::Add,296 el!(Expr::BinaryOp(297 el!(Expr::Num(2.0)),298 BinaryOpType::Mul,299 el!(Expr::Num(2.0)),300 )),301 ))302 }303 }304305 #[test]306 fn empty_object() {307 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));308 }309310 #[test]311 fn basic_math() {312 assert_eq!(313 parse!("2+2*2"),314 el!(Expr::BinaryOp(315 el!(Expr::Num(2.0)),316 BinaryOpType::Add,317 el!(Expr::BinaryOp(318 el!(Expr::Num(2.0)),319 BinaryOpType::Mul,320 el!(Expr::Num(2.0))321 ))322 ))323 );324 }325326 #[test]327 fn basic_math_with_indents() {328 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());329 }330331 #[test]332 fn basic_math_parened() {333 assert_eq!(334 parse!("2+(2+2*2)"),335 el!(Expr::BinaryOp(336 el!(Expr::Num(2.0)),337 BinaryOpType::Add,338 el!(Expr::Parened(expressions::basic_math())),339 ))340 );341 }342343 344 #[test]345 fn comments() {346 assert_eq!(347 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),348 el!(Expr::BinaryOp(349 el!(Expr::Num(2.0)),350 BinaryOpType::Add,351 el!(Expr::BinaryOp(352 el!(Expr::Num(3.0)),353 BinaryOpType::Mul,354 el!(Expr::Num(4.0))355 ))356 ))357 );358 }359360 361 #[test]362 fn comment_escaping() {363 assert_eq!(364 parse!("2/*\\*/+*/ - 22"),365 el!(Expr::BinaryOp(366 el!(Expr::Num(2.0)),367 BinaryOpType::Sub,368 el!(Expr::Num(22.0))369 ))370 );371 }372373 #[test]374 fn suffix() {375 376 377 378 379 }380381 #[test]382 fn array_comp() {383 use Expr::*;384 assert_eq!(385 parse!("[std.deepJoin(x) for x in arr]"),386 el!(ArrComp(387 el!(Apply(388 el!(Index(389 el!(Var("std".to_owned())),390 el!(Str("deepJoin".to_owned()))391 )),392 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])393 )),394 vec![CompSpec::ForSpec(ForSpecData(395 "x".to_owned(),396 el!(Var("arr".to_owned()))397 ))]398 )),399 )400 }401402 #[test]403 fn reserved() {404 use Expr::*;405 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));406 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));407 }408409 #[test]410 fn multiple_args_buf() {411 parse!("a(b, null_fields)");412 }413414 #[test]415 fn infix_precedence() {416 use Expr::*;417 assert_eq!(418 parse!("!a && !b"),419 el!(BinaryOp(420 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),421 BinaryOpType::And,422 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))423 ))424 );425 }426427 #[test]428 fn infix_precedence_division() {429 use Expr::*;430 assert_eq!(431 parse!("!a / !b"),432 el!(BinaryOp(433 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),434 BinaryOpType::Div,435 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))436 ))437 );438 }439440 #[test]441 fn double_negation() {442 use Expr::*;443 assert_eq!(444 parse!("!!a"),445 el!(UnaryOp(446 UnaryOpType::Not,447 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned()))))448 ))449 )450 }451452 #[test]453 fn array_test_error() {454 parse!("[a for a in b if c for e in f]");455 456 }457458 #[test]459 fn can_parse_stdlib() {460 parse!(jsonnet_stdlib::STDLIB_STR);461 }462463 use test::Bencher;464465 466 #[bench]467 fn bench_parse_peg(b: &mut Bencher) {468 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))469 }470471 472 #[bench]473 fn bench_parse_serde_bincode(b: &mut Bencher) {474 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();475 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))476 }477}