1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::{path::PathBuf, rc::Rc};8mod expr;9pub use expr::*;10pub use peg;1112enum Suffix {13 String(String),14 Slice(SliceDesc),15 Expression(LocExpr),16 Apply(expr::ArgsDesc),17 Extend(expr::ObjBody),18}19struct LocSuffix(Suffix, ExprLocation);2021pub struct ParserSettings {22 pub loc_data: bool,23 pub file_name: PathBuf,24}2526parser! {27 grammar jsonnet_parser() for str {28 use peg::ParseLiteral;2930 31 rule comment()32 = "//" (!['\n'][_])* "\n"33 / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"34 / "#" (!['\n'][_])* "\n"3536 rule _() = ([' ' | '\n' | '\t'] / comment())*3738 39 rule comma() = quiet!{_ "," _} / expected!("<comma>")40 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}41 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}42 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']43 44 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }45 46 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")4748 49 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()50 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")5152 rule keyword(id: &'static str)53 = ##parse_string_literal(id) end_of_ident()54 55 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr56 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}5758 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }59 pub rule params(s: &ParserSettings) -> expr::ParamsDesc60 = params:(param(s) ** comma()) {61 let mut defaults_started = false;62 for param in ¶ms {63 defaults_started = defaults_started || param.1.is_some();64 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");65 }66 expr::ParamsDesc(params)67 }68 / { expr::ParamsDesc(Vec::new()) }6970 pub rule arg(s: &ParserSettings) -> expr::Arg71 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}72 / expr:expr(s) {expr::Arg(None, expr)}73 pub rule args(s: &ParserSettings) -> expr::ArgsDesc74 = args:arg(s) ** comma() comma()? {75 let mut named_started = false;76 for arg in &args {77 named_started = named_started || arg.0.is_some();78 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");79 }80 expr::ArgsDesc(args)81 }82 / { expr::ArgsDesc(Vec::new()) }8384 pub rule bind(s: &ParserSettings) -> expr::BindSpec85 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}86 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}87 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt88 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }89 pub rule string() -> String90 = v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}91 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}92 pub rule field_name(s: &ParserSettings) -> expr::FieldName93 = name:id() {expr::FieldName::Fixed(name)}94 / name:string() {expr::FieldName::Fixed(name)}95 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}96 pub rule visibility() -> expr::Visibility97 = ":::" {expr::Visibility::Unhide}98 / "::" {expr::Visibility::Hidden}99 / ":" {expr::Visibility::Normal}100 pub rule field(s: &ParserSettings) -> expr::FieldMember101 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{102 name,103 plus: plus.is_some(),104 params: None,105 visibility,106 value,107 }}108 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{109 name,110 plus: false,111 params: Some(params),112 visibility,113 value,114 }}115 pub rule obj_local(s: &ParserSettings) -> BindSpec116 = keyword("local") _ bind:bind(s) {bind}117 pub rule member(s: &ParserSettings) -> expr::Member118 = bind:obj_local(s) {expr::Member::BindStmt(bind)}119 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}120 / field:field(s) {expr::Member::Field(field)}121 pub rule objinside(s: &ParserSettings) -> expr::ObjBody122 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {123 expr::ObjBody::ObjComp {124 pre_locals,125 key,126 value,127 post_locals,128 rest: [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat(),129 }130 }131 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}132 pub rule ifspec(s: &ParserSettings) -> IfSpecData133 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}134 pub rule forspec(s: &ParserSettings) -> ForSpecData135 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}136 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>137 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}138 pub rule local_expr(s: &ParserSettings) -> LocExpr139 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)140 pub rule string_expr(s: &ParserSettings) -> LocExpr141 = l(s, <s:string() {Expr::Str(s)}>)142 pub rule obj_expr(s: &ParserSettings) -> LocExpr143 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)144 pub rule array_expr(s: &ParserSettings) -> LocExpr145 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)146 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr147 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)148 pub rule number_expr(s: &ParserSettings) -> LocExpr149 = l(s,<n:number() { expr::Expr::Num(n) }>)150 pub rule var_expr(s: &ParserSettings) -> LocExpr151 = l(s,<n:id() { expr::Expr::Var(n) }>)152 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr153 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{154 cond,155 cond_then,156 cond_else,157 }}>)158159 pub rule literal(s: &ParserSettings) -> LocExpr160 = l(s,<v:(161 keyword("null") {LiteralType::Null}162 / keyword("true") {LiteralType::True}163 / keyword("false") {LiteralType::False}164 / keyword("self") {LiteralType::This}165 / keyword("$") {LiteralType::Dollar}166 / keyword("super") {LiteralType::Super}167 ) {Expr::Literal(v)}>)168169 pub rule expr_basic(s: &ParserSettings) -> LocExpr170 = literal(s)171172 / string_expr(s) / number_expr(s)173 / array_expr(s)174 / obj_expr(s)175 / array_expr(s)176 / array_comp_expr(s)177178 / var_expr(s)179 / local_expr(s)180 / if_then_else_expr(s)181182 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)183 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)184185 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)186187 rule expr_basic_with_suffix(s: &ParserSettings) -> LocExpr188 = a:expr_basic(s) suffixes:(_ suffix:l_expr_suffix(s) {suffix})* {189 let mut cur = a;190 for suffix in suffixes {191 let LocSuffix(suffix, location) = suffix;192 cur = LocExpr(Rc::new(match suffix {193 Suffix::String(index) => Expr::Index(cur, loc_expr!(Expr::Str(index), s.loc_data, (s.file_name.clone(), location.1, location.2))),194 Suffix::Slice(desc) => Expr::Slice(cur, desc),195 Suffix::Expression(index) => Expr::Index(cur, index),196 Suffix::Apply(args) => Expr::Apply(cur, args),197 Suffix::Extend(body) => Expr::ObjExtend(cur, body),198 }), if s.loc_data { Some(Rc::new(location)) } else { None })199 }200 cur201 }202203 pub rule slice_desc(s: &ParserSettings) -> SliceDesc204 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {205 if let Some((end, step)) = pair {206 SliceDesc { start, end, step }207 }else{208 SliceDesc { start, end: None, step: None }209 }210 }211212 rule expr_suffix(s: &ParserSettings) -> Suffix213 = "." _ s:id() { Suffix::String(s) }214 / "[" _ s:slice_desc(s) _ "]" { Suffix::Slice(s) }215 / "[" _ s:expr(s) _ "]" { Suffix::Expression(s) }216 / "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }217 / "{" _ body:objinside(s) _ "}" { Suffix::Extend(body) }218 rule l_expr_suffix(s: &ParserSettings) -> LocSuffix219 = start:position!() suffix:expr_suffix(s) end:position!() {LocSuffix(suffix, ExprLocation(s.file_name.clone(), start, end))}220221 rule expr(s: &ParserSettings) -> LocExpr222 = start:position!() a:precedence! {223 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}224 --225 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}226 --227 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}228 --229 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}230 --231 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}232 --233 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::Apply(234 el!(Expr::Index(235 el!(Expr::Var("std".to_owned())),236 el!(Expr::Str("equals".to_owned()))237 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])238 ))}239 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, el!(Expr::Apply(240 el!(Expr::Index(241 el!(Expr::Var("std".to_owned())),242 el!(Expr::Str("equals".to_owned()))243 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])244 ))))}245 --246 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}247 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}248 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}249 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}250 --251 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}252 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}253 --254 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}255 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}256 --257 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}258 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}259 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::Apply(260 el!(Expr::Index(261 el!(Expr::Var("std".to_owned())),262 el!(Expr::Str("mod".to_owned()))263 )), ArgsDesc(vec![Arg(None, a), Arg(None, b)])264 ))}265 --266 "-" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, b))}267 "!" _ b:@ {loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, b))}268 "~" _ b:@ { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, b)) }269 --270 e:expr_basic_with_suffix(s) {e}271 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}272 } end:position!() {273 let LocExpr(e, _) = a;274 LocExpr(e, if s.loc_data {275 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))276 } else {277 None278 })279 }280 / e:expr_basic_with_suffix(s) {e}281282 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}283 }284}285286pub type ParseError = peg::error::ParseError<peg::str::LineCol>;287pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {288 jsonnet_parser::jsonnet(str, settings)289}290291#[macro_export]292macro_rules! el {293 ($expr:expr) => {294 LocExpr(std::rc::Rc::new($expr), None)295 };296}297298#[cfg(test)]299pub mod tests {300 use super::{expr::*, parse};301 use crate::ParserSettings;302 use std::path::PathBuf;303304 macro_rules! parse {305 ($s:expr) => {306 parse(307 $s,308 &ParserSettings {309 loc_data: false,310 file_name: PathBuf::from("/test.jsonnet"),311 },312 )313 .unwrap()314 };315 }316317 mod expressions {318 use super::*;319320 pub fn basic_math() -> LocExpr {321 el!(Expr::BinaryOp(322 el!(Expr::Num(2.0)),323 BinaryOpType::Add,324 el!(Expr::BinaryOp(325 el!(Expr::Num(2.0)),326 BinaryOpType::Mul,327 el!(Expr::Num(2.0)),328 )),329 ))330 }331 }332333 #[test]334 fn empty_object() {335 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));336 }337338 #[test]339 fn basic_math() {340 assert_eq!(341 parse!("2+2*2"),342 el!(Expr::BinaryOp(343 el!(Expr::Num(2.0)),344 BinaryOpType::Add,345 el!(Expr::BinaryOp(346 el!(Expr::Num(2.0)),347 BinaryOpType::Mul,348 el!(Expr::Num(2.0))349 ))350 ))351 );352 }353354 #[test]355 fn basic_math_with_indents() {356 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());357 }358359 #[test]360 fn basic_math_parened() {361 assert_eq!(362 parse!("2+(2+2*2)"),363 el!(Expr::BinaryOp(364 el!(Expr::Num(2.0)),365 BinaryOpType::Add,366 el!(Expr::Parened(expressions::basic_math())),367 ))368 );369 }370371 372 #[test]373 fn comments() {374 assert_eq!(375 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),376 el!(Expr::BinaryOp(377 el!(Expr::Num(2.0)),378 BinaryOpType::Add,379 el!(Expr::BinaryOp(380 el!(Expr::Num(3.0)),381 BinaryOpType::Mul,382 el!(Expr::Num(4.0))383 ))384 ))385 );386 }387388 389 #[test]390 fn comment_escaping() {391 assert_eq!(392 parse!("2/*\\*/+*/ - 22"),393 el!(Expr::BinaryOp(394 el!(Expr::Num(2.0)),395 BinaryOpType::Sub,396 el!(Expr::Num(22.0))397 ))398 );399 }400401 #[test]402 fn array_comp() {403 use Expr::*;404 assert_eq!(405 parse!("[std.deepJoin(x) for x in arr]"),406 el!(ArrComp(407 el!(Apply(408 el!(Index(409 el!(Var("std".to_owned())),410 el!(Str("deepJoin".to_owned()))411 )),412 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])413 )),414 vec![CompSpec::ForSpec(ForSpecData(415 "x".to_owned(),416 el!(Var("arr".to_owned()))417 ))]418 )),419 )420 }421422 #[test]423 fn reserved() {424 use Expr::*;425 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));426 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));427 }428429 #[test]430 fn multiple_args_buf() {431 parse!("a(b, null_fields)");432 }433434 #[test]435 fn infix_precedence() {436 use Expr::*;437 assert_eq!(438 parse!("!a && !b"),439 el!(BinaryOp(440 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),441 BinaryOpType::And,442 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))443 ))444 );445 }446447 #[test]448 fn infix_precedence_division() {449 use Expr::*;450 assert_eq!(451 parse!("!a / !b"),452 el!(BinaryOp(453 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),454 BinaryOpType::Div,455 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))456 ))457 );458 }459460 #[test]461 fn double_negation() {462 use Expr::*;463 assert_eq!(464 parse!("!!a"),465 el!(UnaryOp(466 UnaryOpType::Not,467 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned()))))468 ))469 )470 }471472 #[test]473 fn array_test_error() {474 parse!("[a for a in b if c for e in f]");475 476 }477478 #[test]479 fn can_parse_stdlib() {480 parse!(jsonnet_stdlib::STDLIB_STR);481 }482483 use test::Bencher;484485 486 #[bench]487 fn bench_parse_peg(b: &mut Bencher) {488 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))489 }490491 492 #[bench]493 fn bench_parse_serde_bincode(b: &mut Bencher) {494 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();495 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))496 }497}