difftreelog
feat(parser) bitnot, python comments
in: master
1 file changed
crates/jsonnet-parser/src/lib.rsdiffbeforeafterboth1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::rc::Rc;8mod expr;9pub use expr::*;1011enum Suffix {12 String(String),13 Slice(SliceDesc),14 Expression(LocExpr),15 Apply(expr::ArgsDesc),16 Extend(expr::ObjBody),17}18struct LocSuffix(Suffix, ExprLocation);1920pub struct ParserSettings {21 pub loc_data: bool,22 pub file_name: String,23}2425parser! {26 grammar jsonnet_parser() for str {27 use peg::ParseLiteral;2829 /// Standard C-like comments30 rule comment() = "//" (!['\n'][_])* "\n" / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"31 rule _() = ([' ' | '\n' | '\t'] / comment())*3233 /// For comma-delimited elements34 rule comma() = quiet!{_ "," _} / expected!("<comma>")35 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}36 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}37 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']38 /// Sequence of digits39 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }40 /// Number in scientific notation format41 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")4243 /// Reserved word followed by any non-alphanumberic44 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()45 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")4647 rule keyword(id: &'static str)48 = ##parse_string_literal(id) end_of_ident()49 // Adds location data information to existing expression50 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr51 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}5253 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }54 pub rule params(s: &ParserSettings) -> expr::ParamsDesc55 = params:(param(s) ** comma()) {56 let mut defaults_started = false;57 for param in ¶ms {58 defaults_started = defaults_started || param.1.is_some();59 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");60 }61 expr::ParamsDesc(params)62 }63 / { expr::ParamsDesc(Vec::new()) }6465 pub rule arg(s: &ParserSettings) -> expr::Arg66 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}67 / expr:expr(s) {expr::Arg(None, expr)}68 pub rule args(s: &ParserSettings) -> expr::ArgsDesc69 = args:arg(s) ** comma() comma()? {70 let mut named_started = false;71 for arg in &args {72 named_started = named_started || arg.0.is_some();73 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");74 }75 expr::ArgsDesc(args)76 }77 / { expr::ArgsDesc(Vec::new()) }7879 pub rule bind(s: &ParserSettings) -> expr::BindSpec80 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}81 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}82 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt83 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }84 pub rule string() -> String85 = v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}86 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}87 pub rule field_name(s: &ParserSettings) -> expr::FieldName88 = name:id() {expr::FieldName::Fixed(name)}89 / name:string() {expr::FieldName::Fixed(name)}90 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}91 pub rule visibility() -> expr::Visibility92 = ":::" {expr::Visibility::Unhide}93 / "::" {expr::Visibility::Hidden}94 / ":" {expr::Visibility::Normal}95 pub rule field(s: &ParserSettings) -> expr::FieldMember96 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{97 name,98 plus: plus.is_some(),99 params: None,100 visibility,101 value,102 }}103 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{104 name,105 plus: false,106 params: Some(params),107 visibility,108 value,109 }}110 pub rule obj_local(s: &ParserSettings) -> BindSpec111 = keyword("local") _ bind:bind(s) {bind}112 pub rule member(s: &ParserSettings) -> expr::Member113 = bind:obj_local(s) {expr::Member::BindStmt(bind)}114 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}115 / field:field(s) {expr::Member::Field(field)}116 pub rule objinside(s: &ParserSettings) -> expr::ObjBody117 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ first:forspec(s) rest:(_ rest:compspec(s) {rest})? {118 expr::ObjBody::ObjComp {119 pre_locals,120 key,121 value,122 post_locals,123 first,124 rest: rest.unwrap_or_default(),125 }126 }127 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}128 pub rule ifspec(s: &ParserSettings) -> IfSpecData129 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}130 pub rule forspec(s: &ParserSettings) -> ForSpecData131 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}132 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>133 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}134 pub rule local_expr(s: &ParserSettings) -> LocExpr135 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)136 pub rule string_expr(s: &ParserSettings) -> LocExpr137 = l(s, <s:string() {Expr::Str(s)}>)138 pub rule obj_expr(s: &ParserSettings) -> LocExpr139 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)140 pub rule array_expr(s: &ParserSettings) -> LocExpr141 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)142 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr143 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)144 pub rule number_expr(s: &ParserSettings) -> LocExpr145 = l(s,<n:number() { expr::Expr::Num(n) }>)146 pub rule var_expr(s: &ParserSettings) -> LocExpr147 = l(s,<n:id() { expr::Expr::Var(n) }>)148 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr149 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{150 cond,151 cond_then,152 cond_else,153 }}>)154155 pub rule literal(s: &ParserSettings) -> LocExpr156 = l(s,<v:(157 keyword("null") {LiteralType::Null}158 / keyword("true") {LiteralType::True}159 / keyword("false") {LiteralType::False}160 / keyword("self") {LiteralType::This}161 / keyword("$") {LiteralType::Dollar}162 / keyword("super") {LiteralType::Super}163 ) {Expr::Literal(v)}>)164165 pub rule expr_basic(s: &ParserSettings) -> LocExpr166 = literal(s)167168 / string_expr(s) / number_expr(s)169 / array_expr(s)170 / obj_expr(s)171 / array_expr(s)172 / array_comp_expr(s)173174 / var_expr(s)175 / local_expr(s)176 / if_then_else_expr(s)177178 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)179 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)180181 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)182183 rule expr_basic_with_suffix(s: &ParserSettings) -> LocExpr184 = a:expr_basic(s) suffixes:(_ suffix:l_expr_suffix(s) {suffix})* {185 let mut cur = a;186 for suffix in suffixes {187 let LocSuffix(suffix, location) = suffix;188 cur = LocExpr(Rc::new(match suffix {189 Suffix::String(index) => Expr::Index(cur, loc_expr!(Expr::Str(index), s.loc_data, (s.file_name.clone(), location.1, location.2))),190 Suffix::Slice(desc) => Expr::Slice(cur, desc),191 Suffix::Expression(index) => Expr::Index(cur, index),192 Suffix::Apply(args) => Expr::Apply(cur, args),193 Suffix::Extend(body) => Expr::ObjExtend(cur, body),194 }), if s.loc_data { Some(Rc::new(location)) } else { None })195 }196 cur197 }198199 pub rule slice_desc(s: &ParserSettings) -> SliceDesc200 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {201 if let Some((end, step)) = pair {202 SliceDesc { start, end, step }203 }else{204 SliceDesc { start, end: None, step: None }205 }206 }207208 rule expr_suffix(s: &ParserSettings) -> Suffix209 = "." _ s:id() { Suffix::String(s) }210 / "[" _ s:slice_desc(s) _ "]" { Suffix::Slice(s) }211 / "[" _ s:expr(s) _ "]" { Suffix::Expression(s) }212 / "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }213 / "{" _ body:objinside(s) _ "}" { Suffix::Extend(body) }214 rule l_expr_suffix(s: &ParserSettings) -> LocSuffix215 = start:position!() suffix:expr_suffix(s) end:position!() {LocSuffix(suffix, ExprLocation(s.file_name.clone(), start, end))}216217 rule expr(s: &ParserSettings) -> LocExpr218 = start:position!() a:precedence! {219 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}220 --221 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}222 --223 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}224 --225 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}226 --227 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}228 --229 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Eq, b))}230 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Ne, b))}231 --232 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}233 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}234 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}235 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}236 --237 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}238 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}239 --240 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}241 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}242 --243 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}244 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}245 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mod, b))}246 --247 e:expr_basic_with_suffix(s) {e}248 "-" _ expr:expr_basic_with_suffix(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, expr)) }249 "!" _ expr:expr_basic_with_suffix(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, expr)) }250 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}251 } end:position!() {252 let LocExpr(e, _) = a;253 LocExpr(e, if s.loc_data {254 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))255 } else {256 None257 })258 }259 / e:expr_basic_with_suffix(s) {e}260261 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}262 }263}264265pub fn parse(266 str: &str,267 settings: &ParserSettings,268) -> Result<LocExpr, peg::error::ParseError<peg::str::LineCol>> {269 jsonnet_parser::jsonnet(str, settings)270}271272#[macro_export]273macro_rules! el {274 ($expr:expr) => {275 LocExpr(std::rc::Rc::new($expr), None)276 };277}278279#[cfg(test)]280pub mod tests {281 use super::{expr::*, parse};282 use crate::ParserSettings;283284 macro_rules! parse {285 ($s:expr) => {286 parse(287 $s,288 &ParserSettings {289 loc_data: false,290 file_name: "test.jsonnet".to_owned(),291 },292 )293 .unwrap()294 };295 }296297 mod expressions {298 use super::*;299300 pub fn basic_math() -> LocExpr {301 el!(Expr::BinaryOp(302 el!(Expr::Num(2.0)),303 BinaryOpType::Add,304 el!(Expr::BinaryOp(305 el!(Expr::Num(2.0)),306 BinaryOpType::Mul,307 el!(Expr::Num(2.0)),308 )),309 ))310 }311 }312313 #[test]314 fn empty_object() {315 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));316 }317318 #[test]319 fn basic_math() {320 assert_eq!(321 parse!("2+2*2"),322 el!(Expr::BinaryOp(323 el!(Expr::Num(2.0)),324 BinaryOpType::Add,325 el!(Expr::BinaryOp(326 el!(Expr::Num(2.0)),327 BinaryOpType::Mul,328 el!(Expr::Num(2.0))329 ))330 ))331 );332 }333334 #[test]335 fn basic_math_with_indents() {336 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());337 }338339 #[test]340 fn basic_math_parened() {341 assert_eq!(342 parse!("2+(2+2*2)"),343 el!(Expr::BinaryOp(344 el!(Expr::Num(2.0)),345 BinaryOpType::Add,346 el!(Expr::Parened(expressions::basic_math())),347 ))348 );349 }350351 /// Comments should not affect parsing352 #[test]353 fn comments() {354 assert_eq!(355 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),356 el!(Expr::BinaryOp(357 el!(Expr::Num(2.0)),358 BinaryOpType::Add,359 el!(Expr::BinaryOp(360 el!(Expr::Num(3.0)),361 BinaryOpType::Mul,362 el!(Expr::Num(4.0))363 ))364 ))365 );366 }367368 /// Comments should be able to be escaped369 #[test]370 fn comment_escaping() {371 assert_eq!(372 parse!("2/*\\*/+*/ - 22"),373 el!(Expr::BinaryOp(374 el!(Expr::Num(2.0)),375 BinaryOpType::Sub,376 el!(Expr::Num(22.0))377 ))378 );379 }380381 #[test]382 fn suffix_comparsion() {383 use Expr::*;384 assert_eq!(385 parse!("std.type(a) == \"string\""),386 el!(BinaryOp(387 el!(Apply(388 el!(Index(389 el!(Var("std".to_owned())),390 el!(Str("type".to_owned()))391 )),392 ArgsDesc(vec![Arg(None, el!(Var("a".to_owned())))])393 )),394 BinaryOpType::Eq,395 el!(Str("string".to_owned()))396 ))397 );398 }399400 #[test]401 fn array_comp() {402 use Expr::*;403 assert_eq!(404 parse!("[std.deepJoin(x) for x in arr]"),405 el!(ArrComp(406 el!(Apply(407 el!(Index(408 el!(Var("std".to_owned())),409 el!(Str("deepJoin".to_owned()))410 )),411 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])412 )),413 vec![CompSpec::ForSpec(ForSpecData(414 "x".to_owned(),415 el!(Var("arr".to_owned()))416 ))]417 )),418 )419 }420421 #[test]422 fn array_comp_with_ifs() {423 use Expr::*;424 assert_eq!(425 parse!("[k for k in std.objectFields(patch) if patch[k] == null]"),426 el!(ArrComp(427 el!(Var("k".to_owned())),428 vec![429 CompSpec::ForSpec(ForSpecData(430 "k".to_owned(),431 el!(Apply(432 el!(Index(433 el!(Var("std".to_owned())),434 el!(Str("objectFields".to_owned()))435 )),436 ArgsDesc(vec![Arg(None, el!(Var("patch".to_owned())))])437 ))438 )),439 CompSpec::IfSpec(IfSpecData(el!(BinaryOp(440 el!(Index(441 el!(Var("patch".to_owned())),442 el!(Var("k".to_owned()))443 )),444 BinaryOpType::Eq,445 el!(Literal(LiteralType::Null))446 ))))447 ]448 ))449 );450 }451452 #[test]453 fn reserved() {454 use Expr::*;455 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));456 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));457 }458459 #[test]460 fn multiple_args_buf() {461 parse!("a(b, null_fields)");462 }463464 #[test]465 fn infix_precedence() {466 use Expr::*;467 assert_eq!(468 parse!("!a && !b"),469 el!(BinaryOp(470 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),471 BinaryOpType::And,472 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))473 ))474 );475 }476477 #[test]478 fn can_parse_stdlib() {479 parse!(jsonnet_stdlib::STDLIB_STR);480 }481482 use test::Bencher;483484 // From source code485 #[bench]486 fn bench_parse_peg(b: &mut Bencher) {487 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))488 }489490 // From serialized blob491 #[bench]492 fn bench_parse_serde_bincode(b: &mut Bencher) {493 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();494 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))495 }496}1#![feature(box_syntax)]2#![feature(test)]34extern crate test;56use peg::parser;7use std::rc::Rc;8mod expr;9pub use expr::*;1011enum Suffix {12 String(String),13 Slice(SliceDesc),14 Expression(LocExpr),15 Apply(expr::ArgsDesc),16 Extend(expr::ObjBody),17}18struct LocSuffix(Suffix, ExprLocation);1920pub struct ParserSettings {21 pub loc_data: bool,22 pub file_name: String,23}2425parser! {26 grammar jsonnet_parser() for str {27 use peg::ParseLiteral;2829 /// Standard C-like comments30 rule comment()31 = "//" (!['\n'][_])* "\n"32 / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"33 / "#" (!['\n'][_])* "\n"3435 rule _() = ([' ' | '\n' | '\t'] / comment())*3637 /// For comma-delimited elements38 rule comma() = quiet!{_ "," _} / expected!("<comma>")39 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}40 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}41 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']42 /// Sequence of digits43 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }44 /// Number in scientific notation format45 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")4647 /// Reserved word followed by any non-alphanumberic48 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()49 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")5051 rule keyword(id: &'static str)52 = ##parse_string_literal(id) end_of_ident()53 // Adds location data information to existing expression54 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr55 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}5657 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }58 pub rule params(s: &ParserSettings) -> expr::ParamsDesc59 = params:(param(s) ** comma()) {60 let mut defaults_started = false;61 for param in ¶ms {62 defaults_started = defaults_started || param.1.is_some();63 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");64 }65 expr::ParamsDesc(params)66 }67 / { expr::ParamsDesc(Vec::new()) }6869 pub rule arg(s: &ParserSettings) -> expr::Arg70 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}71 / expr:expr(s) {expr::Arg(None, expr)}72 pub rule args(s: &ParserSettings) -> expr::ArgsDesc73 = args:arg(s) ** comma() comma()? {74 let mut named_started = false;75 for arg in &args {76 named_started = named_started || arg.0.is_some();77 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");78 }79 expr::ArgsDesc(args)80 }81 / { expr::ArgsDesc(Vec::new()) }8283 pub rule bind(s: &ParserSettings) -> expr::BindSpec84 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}85 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}86 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt87 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }88 pub rule string() -> String89 = v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}90 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}91 pub rule field_name(s: &ParserSettings) -> expr::FieldName92 = name:id() {expr::FieldName::Fixed(name)}93 / name:string() {expr::FieldName::Fixed(name)}94 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}95 pub rule visibility() -> expr::Visibility96 = ":::" {expr::Visibility::Unhide}97 / "::" {expr::Visibility::Hidden}98 / ":" {expr::Visibility::Normal}99 pub rule field(s: &ParserSettings) -> expr::FieldMember100 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{101 name,102 plus: plus.is_some(),103 params: None,104 visibility,105 value,106 }}107 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{108 name,109 plus: false,110 params: Some(params),111 visibility,112 value,113 }}114 pub rule obj_local(s: &ParserSettings) -> BindSpec115 = keyword("local") _ bind:bind(s) {bind}116 pub rule member(s: &ParserSettings) -> expr::Member117 = bind:obj_local(s) {expr::Member::BindStmt(bind)}118 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}119 / field:field(s) {expr::Member::Field(field)}120 pub rule objinside(s: &ParserSettings) -> expr::ObjBody121 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ first:forspec(s) rest:(_ rest:compspec(s) {rest})? {122 expr::ObjBody::ObjComp {123 pre_locals,124 key,125 value,126 post_locals,127 first,128 rest: rest.unwrap_or_default(),129 }130 }131 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}132 pub rule ifspec(s: &ParserSettings) -> IfSpecData133 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}134 pub rule forspec(s: &ParserSettings) -> ForSpecData135 = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}136 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>137 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}138 pub rule local_expr(s: &ParserSettings) -> LocExpr139 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)140 pub rule string_expr(s: &ParserSettings) -> LocExpr141 = l(s, <s:string() {Expr::Str(s)}>)142 pub rule obj_expr(s: &ParserSettings) -> LocExpr143 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)144 pub rule array_expr(s: &ParserSettings) -> LocExpr145 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)146 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr147 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)148 pub rule number_expr(s: &ParserSettings) -> LocExpr149 = l(s,<n:number() { expr::Expr::Num(n) }>)150 pub rule var_expr(s: &ParserSettings) -> LocExpr151 = l(s,<n:id() { expr::Expr::Var(n) }>)152 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr153 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{154 cond,155 cond_then,156 cond_else,157 }}>)158159 pub rule literal(s: &ParserSettings) -> LocExpr160 = l(s,<v:(161 keyword("null") {LiteralType::Null}162 / keyword("true") {LiteralType::True}163 / keyword("false") {LiteralType::False}164 / keyword("self") {LiteralType::This}165 / keyword("$") {LiteralType::Dollar}166 / keyword("super") {LiteralType::Super}167 ) {Expr::Literal(v)}>)168169 pub rule expr_basic(s: &ParserSettings) -> LocExpr170 = literal(s)171172 / string_expr(s) / number_expr(s)173 / array_expr(s)174 / obj_expr(s)175 / array_expr(s)176 / array_comp_expr(s)177178 / var_expr(s)179 / local_expr(s)180 / if_then_else_expr(s)181182 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)183 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)184185 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)186187 rule expr_basic_with_suffix(s: &ParserSettings) -> LocExpr188 = a:expr_basic(s) suffixes:(_ suffix:l_expr_suffix(s) {suffix})* {189 let mut cur = a;190 for suffix in suffixes {191 let LocSuffix(suffix, location) = suffix;192 cur = LocExpr(Rc::new(match suffix {193 Suffix::String(index) => Expr::Index(cur, loc_expr!(Expr::Str(index), s.loc_data, (s.file_name.clone(), location.1, location.2))),194 Suffix::Slice(desc) => Expr::Slice(cur, desc),195 Suffix::Expression(index) => Expr::Index(cur, index),196 Suffix::Apply(args) => Expr::Apply(cur, args),197 Suffix::Extend(body) => Expr::ObjExtend(cur, body),198 }), if s.loc_data { Some(Rc::new(location)) } else { None })199 }200 cur201 }202203 pub rule slice_desc(s: &ParserSettings) -> SliceDesc204 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {205 if let Some((end, step)) = pair {206 SliceDesc { start, end, step }207 }else{208 SliceDesc { start, end: None, step: None }209 }210 }211212 rule expr_suffix(s: &ParserSettings) -> Suffix213 = "." _ s:id() { Suffix::String(s) }214 / "[" _ s:slice_desc(s) _ "]" { Suffix::Slice(s) }215 / "[" _ s:expr(s) _ "]" { Suffix::Expression(s) }216 / "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }217 / "{" _ body:objinside(s) _ "}" { Suffix::Extend(body) }218 rule l_expr_suffix(s: &ParserSettings) -> LocSuffix219 = start:position!() suffix:expr_suffix(s) end:position!() {LocSuffix(suffix, ExprLocation(s.file_name.clone(), start, end))}220221 rule expr(s: &ParserSettings) -> LocExpr222 = start:position!() a:precedence! {223 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}224 --225 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}226 --227 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}228 --229 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}230 --231 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}232 --233 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Eq, b))}234 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Ne, b))}235 --236 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}237 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}238 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}239 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}240 --241 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}242 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}243 --244 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}245 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}246 --247 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}248 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}249 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mod, b))}250 --251 e:expr_basic_with_suffix(s) {e}252 "-" _ expr:expr(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, expr)) }253 "!" _ expr:expr(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, expr)) }254 "~" _ expr:expr(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::BitNot, expr)) }255 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}256 } end:position!() {257 let LocExpr(e, _) = a;258 LocExpr(e, if s.loc_data {259 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))260 } else {261 None262 })263 }264 / e:expr_basic_with_suffix(s) {e}265266 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}267 }268}269270pub fn parse(271 str: &str,272 settings: &ParserSettings,273) -> Result<LocExpr, peg::error::ParseError<peg::str::LineCol>> {274 jsonnet_parser::jsonnet(str, settings)275}276277#[macro_export]278macro_rules! el {279 ($expr:expr) => {280 LocExpr(std::rc::Rc::new($expr), None)281 };282}283284#[cfg(test)]285pub mod tests {286 use super::{expr::*, parse};287 use crate::ParserSettings;288289 macro_rules! parse {290 ($s:expr) => {291 parse(292 $s,293 &ParserSettings {294 loc_data: false,295 file_name: "test.jsonnet".to_owned(),296 },297 )298 .unwrap()299 };300 }301302 mod expressions {303 use super::*;304305 pub fn basic_math() -> LocExpr {306 el!(Expr::BinaryOp(307 el!(Expr::Num(2.0)),308 BinaryOpType::Add,309 el!(Expr::BinaryOp(310 el!(Expr::Num(2.0)),311 BinaryOpType::Mul,312 el!(Expr::Num(2.0)),313 )),314 ))315 }316 }317318 #[test]319 fn empty_object() {320 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));321 }322323 #[test]324 fn basic_math() {325 assert_eq!(326 parse!("2+2*2"),327 el!(Expr::BinaryOp(328 el!(Expr::Num(2.0)),329 BinaryOpType::Add,330 el!(Expr::BinaryOp(331 el!(Expr::Num(2.0)),332 BinaryOpType::Mul,333 el!(Expr::Num(2.0))334 ))335 ))336 );337 }338339 #[test]340 fn basic_math_with_indents() {341 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());342 }343344 #[test]345 fn basic_math_parened() {346 assert_eq!(347 parse!("2+(2+2*2)"),348 el!(Expr::BinaryOp(349 el!(Expr::Num(2.0)),350 BinaryOpType::Add,351 el!(Expr::Parened(expressions::basic_math())),352 ))353 );354 }355356 /// Comments should not affect parsing357 #[test]358 fn comments() {359 assert_eq!(360 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),361 el!(Expr::BinaryOp(362 el!(Expr::Num(2.0)),363 BinaryOpType::Add,364 el!(Expr::BinaryOp(365 el!(Expr::Num(3.0)),366 BinaryOpType::Mul,367 el!(Expr::Num(4.0))368 ))369 ))370 );371 }372373 /// Comments should be able to be escaped374 #[test]375 fn comment_escaping() {376 assert_eq!(377 parse!("2/*\\*/+*/ - 22"),378 el!(Expr::BinaryOp(379 el!(Expr::Num(2.0)),380 BinaryOpType::Sub,381 el!(Expr::Num(22.0))382 ))383 );384 }385386 #[test]387 fn suffix_comparsion() {388 use Expr::*;389 assert_eq!(390 parse!("std.type(a) == \"string\""),391 el!(BinaryOp(392 el!(Apply(393 el!(Index(394 el!(Var("std".to_owned())),395 el!(Str("type".to_owned()))396 )),397 ArgsDesc(vec![Arg(None, el!(Var("a".to_owned())))])398 )),399 BinaryOpType::Eq,400 el!(Str("string".to_owned()))401 ))402 );403 }404405 #[test]406 fn array_comp() {407 use Expr::*;408 assert_eq!(409 parse!("[std.deepJoin(x) for x in arr]"),410 el!(ArrComp(411 el!(Apply(412 el!(Index(413 el!(Var("std".to_owned())),414 el!(Str("deepJoin".to_owned()))415 )),416 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])417 )),418 vec![CompSpec::ForSpec(ForSpecData(419 "x".to_owned(),420 el!(Var("arr".to_owned()))421 ))]422 )),423 )424 }425426 #[test]427 fn array_comp_with_ifs() {428 use Expr::*;429 assert_eq!(430 parse!("[k for k in std.objectFields(patch) if patch[k] == null]"),431 el!(ArrComp(432 el!(Var("k".to_owned())),433 vec![434 CompSpec::ForSpec(ForSpecData(435 "k".to_owned(),436 el!(Apply(437 el!(Index(438 el!(Var("std".to_owned())),439 el!(Str("objectFields".to_owned()))440 )),441 ArgsDesc(vec![Arg(None, el!(Var("patch".to_owned())))])442 ))443 )),444 CompSpec::IfSpec(IfSpecData(el!(BinaryOp(445 el!(Index(446 el!(Var("patch".to_owned())),447 el!(Var("k".to_owned()))448 )),449 BinaryOpType::Eq,450 el!(Literal(LiteralType::Null))451 ))))452 ]453 ))454 );455 }456457 #[test]458 fn reserved() {459 use Expr::*;460 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));461 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));462 }463464 #[test]465 fn multiple_args_buf() {466 parse!("a(b, null_fields)");467 }468469 #[test]470 fn infix_precedence() {471 use Expr::*;472 assert_eq!(473 parse!("!a && !b"),474 el!(BinaryOp(475 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),476 BinaryOpType::And,477 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))478 ))479 );480 }481482 #[test]483 fn double_negation() {484 use Expr::*;485 assert_eq!(486 parse!("!!a"),487 el!(UnaryOp(488 UnaryOpType::Not,489 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned()))))490 ))491 )492 }493494 #[test]495 fn can_parse_stdlib() {496 parse!(jsonnet_stdlib::STDLIB_STR);497 }498499 use test::Bencher;500501 // From source code502 #[bench]503 fn bench_parse_peg(b: &mut Bencher) {504 b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))505 }506507 // From serialized blob508 #[bench]509 fn bench_parse_serde_bincode(b: &mut Bencher) {510 let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();511 b.iter(|| bincode::deserialize::<LocExpr>(&serialized))512 }513}