1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::{path::PathBuf, rc::Rc};8mod expr;9mod string_processing;10pub use expr::*;11pub use peg;12use string_processing::deent;1314pub struct ParserSettings {15 pub loc_data: bool,16 pub file_name: PathBuf,17}1819parser! {20 grammar jsonnet_parser() for str {21 use peg::ParseLiteral;2223 24 rule comment()25 = "//" (!['\n'][_])* "\n"26 / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"27 / "#" (!['\n'][_])* "\n"2829 rule _() = ([' ' | '\n' | '\t'] / comment())*3031 32 rule comma() = quiet!{_ "," _} / expected!("<comma>")33 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}34 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}35 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']36 37 rule uint() -> u64 = a:$(digit()+) { a.parse().unwrap() }38 39 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")4041 42 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()43 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")4445 rule keyword(id: &'static str)46 = ##parse_string_literal(id) end_of_ident()47 48 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr49 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}5051 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }52 pub rule params(s: &ParserSettings) -> expr::ParamsDesc53 = params:(param(s) ** comma()) {54 let mut defaults_started = false;55 for param in ¶ms {56 defaults_started = defaults_started || param.1.is_some();57 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");58 }59 expr::ParamsDesc(params)60 }61 / { expr::ParamsDesc(Vec::new()) }6263 pub rule arg(s: &ParserSettings) -> expr::Arg64 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}65 / expr:expr(s) {expr::Arg(None, expr)}66 pub rule args(s: &ParserSettings) -> expr::ArgsDesc67 = args:arg(s) ** comma() comma()? {68 let mut named_started = false;69 for arg in &args {70 named_started = named_started || arg.0.is_some();71 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");72 }73 expr::ArgsDesc(args)74 }75 / { expr::ArgsDesc(Vec::new()) }7677 pub rule bind(s: &ParserSettings) -> expr::BindSpec78 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}79 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}80 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt81 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }8283 pub rule whole_line() -> String84 = str:$((!['\n'][_])* "\n") {str.to_owned()}85 pub rule string() -> String86 = "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}87 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}88 89 / "|||" "\n" str:$((" "*<1, 1> whole_line())+) " "*<0, 0> "|||" {deent(str)}90 / "|||" "\n" str:$((" "*<2, 2> whole_line())+) " "*<1, 1> "|||" {deent(str)}91 / "|||" "\n" str:$((" "*<3, 3> whole_line())+) " "*<2, 2> "|||" {deent(str)}92 / "|||" "\n" str:$((" "*<4, 4> whole_line())+) " "*<3, 3> "|||" {deent(str)}93 / "|||" "\n" str:$((" "*<5, 5> whole_line())+) " "*<4, 4> "|||" {deent(str)}94 / "|||" "\n" str:$((" "*<6, 6> whole_line())+) " "*<5, 5> "|||" {deent(str)}95 / "|||" "\n" str:$((" "*<7, 7> whole_line())+) " "*<6, 6> "|||" {deent(str)}96 / "|||" "\n" str:$((" "*<8, 8> whole_line())+) " "*<7, 7> "|||" {deent(str)}97 / "|||" "\n" str:$((" "*<9, 9> whole_line())+) " "*<8, 8> "|||" {deent(str)}98 / "|||" "\n" str:$((" "*<10, 10> whole_line())+) " "*<9, 9> "|||" {deent(str)}99 / "|||" "\n" str:$((" "*<11, 11> whole_line())+) " "*<10, 10> "|||" {deent(str)}100 / "|||" "\n" str:$((" "*<12, 12> whole_line())+) " "*<11, 10> "|||" {deent(str)}101102 pub rule field_name(s: &ParserSettings) -> expr::FieldName103 = name:id() {expr::FieldName::Fixed(name)}104 / name:string() {expr::FieldName::Fixed(name)}105 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}106 pub rule visibility() -> expr::Visibility107 = ":::" {expr::Visibility::Unhide}108 / "::" {expr::Visibility::Hidden}109 / ":" {expr::Visibility::Normal}110 pub rule field(s: &ParserSettings) -> expr::FieldMember111 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{112 name,113 plus: plus.is_some(),114 params: None,115 visibility,116 value,117 }}118 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{119 name,120 plus: false,121 params: Some(params),122 visibility,123 value,124 }}125 pub rule obj_local(s: &ParserSettings) -> BindSpec126 = keyword("local") _ bind:bind(s) {bind}127 pub rule member(s: &ParserSettings) -> expr::Member128 = bind:obj_local(s) {expr::Member::BindStmt(bind)}129 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}130 / field:field(s) {expr::Member::Field(field)}131 pub rule objinside(s: &ParserSettings) -> expr::ObjBody132 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {133 expr::ObjBody::ObjComp {134 pre_locals,135 key,136 value,137 post_locals,138 rest: [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat(),139 }140 }141 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}142 pub rule ifspec(s: &ParserSettings) -> IfSpecData143 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}144 pub rule forspec(s: &ParserSettings) -> ForSpecData145 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}146 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>147 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}148 pub rule local_expr(s: &ParserSettings) -> LocExpr149 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)150 pub rule string_expr(s: &ParserSettings) -> LocExpr151 = l(s, <s:string() {Expr::Str(s)}>)152 pub rule obj_expr(s: &ParserSettings) -> LocExpr153 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)154 pub rule array_expr(s: &ParserSettings) -> LocExpr155 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)156 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr157 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)158 pub rule number_expr(s: &ParserSettings) -> LocExpr159 = l(s,<n:number() { expr::Expr::Num(n) }>)160 pub rule var_expr(s: &ParserSettings) -> LocExpr161 = l(s,<n:id() { expr::Expr::Var(n) }>)162 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr163 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{164 cond,165 cond_then,166 cond_else,167 }}>)168169 pub rule literal(s: &ParserSettings) -> LocExpr170 = l(s,<v:(171 keyword("null") {LiteralType::Null}172 / keyword("true") {LiteralType::True}173 / keyword("false") {LiteralType::False}174 / keyword("self") {LiteralType::This}175 / keyword("$") {LiteralType::Dollar}176 / keyword("super") {LiteralType::Super}177 ) {Expr::Literal(v)}>)178179 pub rule expr_basic(s: &ParserSettings) -> LocExpr180 = literal(s)181182 / string_expr(s) / number_expr(s)183 / array_expr(s)184 / obj_expr(s)185 / array_expr(s)186 / array_comp_expr(s)187188 / var_expr(s)189 / local_expr(s)190 / if_then_else_expr(s)191192 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)193 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)194195 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)196197 pub rule slice_desc(s: &ParserSettings) -> SliceDesc198 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {199 if let Some((end, step)) = pair {200 SliceDesc { start, end, step }201 }else{202 SliceDesc { start, end: None, step: None }203 }204 }205206 rule expr(s: &ParserSettings) -> LocExpr207 = start:position!() a:precedence! {208 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}209 --210 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}211 --212 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}213 --214 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}215 --216 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}217 --218 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply(219 el!(Expr::Index(220 el!(Expr::Var("std".to_owned())),221 el!(Expr::Str("equals".to_owned()))222 )),223 ArgsDesc(vec![Arg(None, a), Arg(None, b)]),224 true225 ))}226 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply(227 el!(Expr::Index(228 el!(Expr::Var("std".to_owned())),229 el!(Expr::Str("equals".to_owned()))230 )),231 ArgsDesc(vec![Arg(None, a), Arg(None, b)]),232 true233 ))))}234 --235 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}236 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}237 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}238 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}239 --240 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}241 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}242 --243 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}244 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}245 --246 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}247 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}248 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply(249 el!(Expr::Index(250 el!(Expr::Var("std".to_owned())),251 el!(Expr::Str("mod".to_owned()))252 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)]),253 true254 ))}255 --256 "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))}257 "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))}258 "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) }259 --260 a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Slice(a, s))}261 a:(@) _ "." _ s:id() {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s))))}262 a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))}263 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args, ts.is_some()))}264 a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))}265 --266 e:expr_basic(s) {e}267 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}268 } end:position!() {269 let LocExpr(e, _) = a;270 LocExpr(e, if s.loc_data {271 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))272 } else {273 None274 })275 }276 / e:expr_basic(s) {e}277278 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}279 }280}281282pub type ParseError = peg::error::ParseError<peg::str::LineCol>;283pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {284 jsonnet_parser::jsonnet(str, settings)285}286287#[macro_export]288macro_rules! el {289 ($expr:expr) => {290 LocExpr(std::rc::Rc::new($expr), None)291 };292}293294#[cfg(test)]295pub mod tests {296 use super::{expr::*, parse};297 use crate::ParserSettings;298 use std::path::PathBuf;299300 macro_rules! parse {301 ($s:expr) => {302 parse(303 $s,304 &ParserSettings {305 loc_data: false,306 file_name: PathBuf::from("/test.jsonnet"),307 },308 )309 .unwrap()310 };311 }312313 mod expressions {314 use super::*;315316 pub fn basic_math() -> LocExpr {317 el!(Expr::BinaryOp(318 el!(Expr::Num(2.0)),319 BinaryOpType::Add,320 el!(Expr::BinaryOp(321 el!(Expr::Num(2.0)),322 BinaryOpType::Mul,323 el!(Expr::Num(2.0)),324 )),325 ))326 }327 }328329 #[test]330 fn multiline_string() {331 assert_eq!(332 parse!("|||\n Hello world!\n a\n|||"),333 el!(Expr::Str(" Hello world!\na\n".to_owned())),334 )335 }336337 #[test]338 fn empty_object() {339 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));340 }341342 #[test]343 fn basic_math() {344 assert_eq!(345 parse!("2+2*2"),346 el!(Expr::BinaryOp(347 el!(Expr::Num(2.0)),348 BinaryOpType::Add,349 el!(Expr::BinaryOp(350 el!(Expr::Num(2.0)),351 BinaryOpType::Mul,352 el!(Expr::Num(2.0))353 ))354 ))355 );356 }357358 #[test]359 fn basic_math_with_indents() {360 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());361 }362363 #[test]364 fn basic_math_parened() {365 assert_eq!(366 parse!("2+(2+2*2)"),367 el!(Expr::BinaryOp(368 el!(Expr::Num(2.0)),369 BinaryOpType::Add,370 el!(Expr::Parened(expressions::basic_math())),371 ))372 );373 }374375 376 #[test]377 fn comments() {378 assert_eq!(379 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),380 el!(Expr::BinaryOp(381 el!(Expr::Num(2.0)),382 BinaryOpType::Add,383 el!(Expr::BinaryOp(384 el!(Expr::Num(3.0)),385 BinaryOpType::Mul,386 el!(Expr::Num(4.0))387 ))388 ))389 );390 }391392 393 #[test]394 fn comment_escaping() {395 assert_eq!(396 parse!("2/*\\*/+*/ - 22"),397 el!(Expr::BinaryOp(398 el!(Expr::Num(2.0)),399 BinaryOpType::Sub,400 el!(Expr::Num(22.0))401 ))402 );403 }404405 #[test]406 fn suffix() {407 408 409 410 411 }412413 #[test]414 fn array_comp() {415 use Expr::*;416 assert_eq!(417 parse!("[std.deepJoin(x) for x in arr]"),418 el!(ArrComp(419 el!(Apply(420 el!(Index(421 el!(Var("std".to_owned())),422 el!(Str("deepJoin".to_owned()))423 )),424 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))]),425 false,426 )),427 vec![CompSpec::ForSpec(ForSpecData(428 "x".to_owned(),429 el!(Var("arr".to_owned()))430 ))]431 )),432 )433 }434435 #[test]436 fn reserved() {437 use Expr::*;438 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));439 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));440 }441442 #[test]443 fn multiple_args_buf() {444 parse!("a(b, null_fields)");445 }446447 #[test]448 fn infix_precedence() {449 use Expr::*;450 assert_eq!(451 parse!("!a && !b"),452 el!(BinaryOp(453 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),454 BinaryOpType::And,455 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))456 ))457 );458 }459460 #[test]461 fn infix_precedence_division() {462 use Expr::*;463 assert_eq!(464 parse!("!a / !b"),465 el!(BinaryOp(466 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),467 BinaryOpType::Div,468 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))469 ))470 );471 }472473 #[test]474 fn double_negation() {475 use Expr::*;476 assert_eq!(477 parse!("!!a"),478 el!(UnaryOp(479 UnaryOpType::Not,480 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned()))))481 ))482 )483 }484485 #[test]486 fn array_test_error() {487 parse!("[a for a in b if c for e in f]");488 489 }490491 #[test]492 fn can_parse_stdlib() {493 parse!(jsonnet_stdlib::STDLIB_STR);494 }495496 use test::Bencher;497498 499 #[bench]500 fn bench_parse_peg(b: &mut Bencher) {501 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))502 }503504 505 #[bench]506 fn bench_parse_serde_bincode(b: &mut Bencher) {507 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();508 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))509 }510}