difftreelog
Add location info to all sub-expressions
in: master
2 files changed
crates/jrsonnet-parser/src/expr.rsdiffbeforeafterboth--- a/crates/jrsonnet-parser/src/expr.rs
+++ b/crates/jrsonnet-parser/src/expr.rs
@@ -403,11 +403,3 @@
)
};
}
-
-/// Creates LocExpr without location info
-#[macro_export]
-macro_rules! loc_expr_todo {
- ($expr:expr) => {
- LocExpr(Rc::new($expr), None)
- };
-}
crates/jrsonnet-parser/src/lib.rsdiffbeforeafterboth1#![allow(clippy::redundant_closure_call)]23use peg::parser;4use std::{5 path::{Path, PathBuf},6 rc::Rc,7};8mod expr;9pub use expr::*;10pub use jrsonnet_interner::IStr;11pub use peg;1213pub struct ParserSettings {14 pub loc_data: bool,15 pub file_name: Rc<Path>,16}1718macro_rules! expr_bin {19 ($a:ident $op:ident $b:ident) => {20 loc_expr_todo!(Expr::BinaryOp($a, $op, $b))21 };22}23macro_rules! expr_un {24 ($op:ident $a:ident) => {25 loc_expr_todo!(Expr::UnaryOp($op, $a))26 };27}2829parser! {30 grammar jsonnet_parser() for str {31 use peg::ParseLiteral;3233 /// Standard C-like comments34 rule comment()35 = "//" (!['\n'][_])* "\n"36 / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/"37 / "#" (!['\n'][_])* "\n"3839 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")40 rule _() = single_whitespace()*4142 /// For comma-delimited elements43 rule comma() = quiet!{_ "," _} / expected!("<comma>")44 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}45 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}46 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']47 /// Sequence of digits48 rule uint_str() -> &'input str = a:$(digit()+) { a }49 /// Number in scientific notation format50 rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "<number>") }} / expected!("<number>")5152 /// Reserved word followed by any non-alphanumberic53 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()54 rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")5556 rule keyword(id: &'static str) -> ()57 = ##parse_string_literal(id) end_of_ident()58 // Adds location data information to existing expression59 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr60 = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}6162 pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) }63 pub rule params(s: &ParserSettings) -> expr::ParamsDesc64 = params:param(s) ** comma() comma()? {65 let mut defaults_started = false;66 for param in ¶ms {67 defaults_started = defaults_started || param.1.is_some();68 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");69 }70 expr::ParamsDesc(Rc::new(params))71 }72 / { expr::ParamsDesc(Rc::new(Vec::new())) }7374 pub rule arg(s: &ParserSettings) -> (Option<IStr>, LocExpr)75 = quiet! { name:(s:$(id()) _ "=" _ {s})? expr:expr(s) {(name.map(Into::into), expr)} }76 / expected!("<argument>")7778 pub rule args(s: &ParserSettings) -> expr::ArgsDesc79 = args:arg(s)**comma() comma()? {?80 let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();81 let mut unnamed = Vec::with_capacity(unnamed_count);82 let mut named = Vec::with_capacity(args.len() - unnamed_count);83 let mut named_started = false;84 for (name, value) in args {85 if let Some(name) = name {86 named_started = true;87 named.push((name, value));88 } else {89 if named_started {90 return Err("<named argument>")91 }92 unnamed.push(value);93 }94 }95 Ok(expr::ArgsDesc::new(unnamed, named))96 }9798 pub rule bind(s: &ParserSettings) -> expr::BindSpec99 = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}}100 / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}}101 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt102 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }103104 pub rule whole_line() -> &'input str105 = str:$((!['\n'][_])* "\n") {str}106 pub rule string_block() -> String107 = "|||" (!['\n']single_whitespace())* "\n"108 empty_lines:$(['\n']*)109 prefix:[' ' | '\t']+ first_line:whole_line()110 lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*111 [' ' | '\t']*<, {prefix.len() - 1}> "|||"112 {let mut l = empty_lines.to_owned(); l.push_str(first_line); l.extend(lines); l}113 pub rule string() -> String114 = quiet!{ "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()}115 / "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()}116 / "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}117 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}118 / string_block() } / expected!("<string>")119120 pub rule field_name(s: &ParserSettings) -> expr::FieldName121 = name:$(id()) {expr::FieldName::Fixed(name.into())}122 / name:string() {expr::FieldName::Fixed(name.into())}123 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}124 pub rule visibility() -> expr::Visibility125 = ":::" {expr::Visibility::Unhide}126 / "::" {expr::Visibility::Hidden}127 / ":" {expr::Visibility::Normal}128 pub rule field(s: &ParserSettings) -> expr::FieldMember129 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{130 name,131 plus: plus.is_some(),132 params: None,133 visibility,134 value,135 }}136 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{137 name,138 plus: false,139 params: Some(params),140 visibility,141 value,142 }}143 pub rule obj_local(s: &ParserSettings) -> BindSpec144 = keyword("local") _ bind:bind(s) {bind}145 pub rule member(s: &ParserSettings) -> expr::Member146 = bind:obj_local(s) {expr::Member::BindStmt(bind)}147 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}148 / field:field(s) {expr::Member::Field(field)}149 pub rule objinside(s: &ParserSettings) -> expr::ObjBody150 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ plus:"+"? _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {151 let mut compspecs = vec![CompSpec::ForSpec(forspec)];152 compspecs.extend(others.unwrap_or_default());153 expr::ObjBody::ObjComp(expr::ObjComp{154 pre_locals,155 key,156 plus: plus.is_some(),157 value,158 post_locals,159 compspecs,160 })161 }162 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}163 pub rule ifspec(s: &ParserSettings) -> IfSpecData164 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}165 pub rule forspec(s: &ParserSettings) -> ForSpecData166 = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)}167 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>168 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}169 pub rule local_expr(s: &ParserSettings) -> LocExpr170 = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)171 pub rule string_expr(s: &ParserSettings) -> LocExpr172 = l(s, <s:string() {Expr::Str(s.into())}>)173 pub rule obj_expr(s: &ParserSettings) -> LocExpr174 = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)175 pub rule array_expr(s: &ParserSettings) -> LocExpr176 = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)177 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr178 = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {179 let mut specs = vec![CompSpec::ForSpec(forspec)];180 specs.extend(others.unwrap_or_default());181 Expr::ArrComp(expr, specs)182 }>)183 pub rule number_expr(s: &ParserSettings) -> LocExpr184 = l(s,<n:number() { expr::Expr::Num(n) }>)185 pub rule var_expr(s: &ParserSettings) -> LocExpr186 = l(s,<n:$(id()) { expr::Expr::Var(n.into()) }>)187 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr188 = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{189 cond,190 cond_then,191 cond_else,192 }}>)193194 pub rule literal(s: &ParserSettings) -> LocExpr195 = l(s,<v:(196 keyword("null") {LiteralType::Null}197 / keyword("true") {LiteralType::True}198 / keyword("false") {LiteralType::False}199 / keyword("self") {LiteralType::This}200 / keyword("$") {LiteralType::Dollar}201 / keyword("super") {LiteralType::Super}202 ) {Expr::Literal(v)}>)203204 pub rule expr_basic(s: &ParserSettings) -> LocExpr205 = literal(s)206207 / quiet!{l(s,<"$intrinsic(" name:$(id()) ")" {Expr::Intrinsic(name.into())}>)}208209 / string_expr(s) / number_expr(s)210 / array_expr(s)211 / obj_expr(s)212 / array_expr(s)213 / array_comp_expr(s)214215 / l(s,<keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}>)216 / l(s,<keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}>)217218 / var_expr(s)219 / local_expr(s)220 / if_then_else_expr(s)221222 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)223 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)224225 / l(s,<keyword("error") _ expr:expr(s) { Expr::ErrorStmt(expr) }>)226227 rule slice_part(s: &ParserSettings) -> Option<LocExpr>228 = e:(_ e:expr(s) _{e})? {e}229 pub rule slice_desc(s: &ParserSettings) -> SliceDesc230 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {231 let (end, step) = if let Some((end, step)) = pair {232 (end, step)233 }else{234 (None, None)235 };236237 SliceDesc { start, end, step }238 }239240 rule binop(x: rule<()>) -> ()241 = quiet!{ x() } / expected!("<binary op>")242 rule unaryop(x: rule<()>) -> ()243 = quiet!{ x() } / expected!("<unary op>")244245246 use BinaryOpType::*;247 use UnaryOpType::*;248 rule expr(s: &ParserSettings) -> LocExpr249 = start:position!() a:precedence! {250 a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}251 --252 a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}253 --254 a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}255 --256 a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}257 --258 a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}259 --260 a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}261 a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}262 --263 a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}264 a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}265 a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}266 a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}267 a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}268 --269 a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}270 a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}271 --272 a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}273 a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}274 --275 a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}276 a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}277 a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}278 --279 unaryop(<"-">) _ b:@ {expr_un!(Minus b)}280 unaryop(<"!">) _ b:@ {expr_un!(Not b)}281 unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}282 --283 a:(@) _ "[" _ s:slice_desc(s) _ "]" {loc_expr_todo!(Expr::Slice(a, s))}284 a:(@) _ "." _ s:$(id()) {loc_expr_todo!(Expr::Index(a, el!(Expr::Str(s.into()))))}285 a:(@) _ "[" _ s:expr(s) _ "]" {loc_expr_todo!(Expr::Index(a, s))}286 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {loc_expr_todo!(Expr::Apply(a, args, ts.is_some()))}287 a:(@) _ "{" _ body:objinside(s) _ "}" {loc_expr_todo!(Expr::ObjExtend(a, body))}288 --289 e:expr_basic(s) {e}290 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}291 } end:position!() {292 let LocExpr(e, _) = a;293 LocExpr(e, if s.loc_data {294 Some(ExprLocation(s.file_name.clone(), start, end))295 } else {296 None297 })298 }299 / e:expr_basic(s) {e}300301 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}302 }303}304305pub type ParseError = peg::error::ParseError<peg::str::LineCol>;306pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {307 jsonnet_parser::jsonnet(str, settings)308}309310#[macro_export]311macro_rules! el {312 ($expr:expr) => {313 LocExpr(std::rc::Rc::new($expr), None)314 };315}316317#[cfg(test)]318pub mod tests {319 use super::{expr::*, parse};320 use crate::ParserSettings;321 use std::path::PathBuf;322 use BinaryOpType::*;323324 macro_rules! parse {325 ($s:expr) => {326 parse(327 $s,328 &ParserSettings {329 loc_data: false,330 file_name: PathBuf::from("/test.jsonnet").into(),331 },332 )333 .unwrap()334 };335 }336337 mod expressions {338 use super::*;339340 pub fn basic_math() -> LocExpr {341 el!(Expr::BinaryOp(342 el!(Expr::Num(2.0)),343 Add,344 el!(Expr::BinaryOp(345 el!(Expr::Num(2.0)),346 Mul,347 el!(Expr::Num(2.0)),348 )),349 ))350 }351 }352353 #[test]354 fn multiline_string() {355 assert_eq!(356 parse!("|||\n Hello world!\n a\n|||"),357 el!(Expr::Str("Hello world!\n a\n".into())),358 );359 assert_eq!(360 parse!("|||\n Hello world!\n a\n|||"),361 el!(Expr::Str("Hello world!\n a\n".into())),362 );363 assert_eq!(364 parse!("|||\n\t\tHello world!\n\t\t\ta\n|||"),365 el!(Expr::Str("Hello world!\n\ta\n".into())),366 );367 assert_eq!(368 parse!("|||\n Hello world!\n a\n |||"),369 el!(Expr::Str("Hello world!\n a\n".into())),370 );371 }372373 #[test]374 fn slice() {375 parse!("a[1:]");376 parse!("a[1::]");377 parse!("a[:1:]");378 parse!("a[::1]");379 parse!("str[:len - 1]");380 }381382 #[test]383 fn string_escaping() {384 assert_eq!(385 parse!(r#""Hello, \"world\"!""#),386 el!(Expr::Str(r#"Hello, "world"!"#.into())),387 );388 assert_eq!(389 parse!(r#"'Hello \'world\'!'"#),390 el!(Expr::Str("Hello 'world'!".into())),391 );392 assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),);393 }394395 #[test]396 fn string_unescaping() {397 assert_eq!(398 parse!(r#""Hello\nWorld""#),399 el!(Expr::Str("Hello\nWorld".into())),400 );401 }402403 #[test]404 fn string_verbantim() {405 assert_eq!(406 parse!(r#"@"Hello\n""World""""#),407 el!(Expr::Str("Hello\\n\"World\"".into())),408 );409 }410411 #[test]412 fn imports() {413 assert_eq!(414 parse!("import \"hello\""),415 el!(Expr::Import(PathBuf::from("hello"))),416 );417 assert_eq!(418 parse!("importstr \"garnish.txt\""),419 el!(Expr::ImportStr(PathBuf::from("garnish.txt")))420 );421 }422423 #[test]424 fn empty_object() {425 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));426 }427428 #[test]429 fn basic_math() {430 assert_eq!(431 parse!("2+2*2"),432 el!(Expr::BinaryOp(433 el!(Expr::Num(2.0)),434 Add,435 el!(Expr::BinaryOp(436 el!(Expr::Num(2.0)),437 Mul,438 el!(Expr::Num(2.0))439 ))440 ))441 );442 }443444 #[test]445 fn basic_math_with_indents() {446 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());447 }448449 #[test]450 fn basic_math_parened() {451 assert_eq!(452 parse!("2+(2+2*2)"),453 el!(Expr::BinaryOp(454 el!(Expr::Num(2.0)),455 Add,456 el!(Expr::Parened(expressions::basic_math())),457 ))458 );459 }460461 /// Comments should not affect parsing462 #[test]463 fn comments() {464 assert_eq!(465 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),466 el!(Expr::BinaryOp(467 el!(Expr::Num(2.0)),468 Add,469 el!(Expr::BinaryOp(470 el!(Expr::Num(3.0)),471 Mul,472 el!(Expr::Num(4.0))473 ))474 ))475 );476 }477478 /// Comments should be able to be escaped479 #[test]480 fn comment_escaping() {481 assert_eq!(482 parse!("2/*\\*/+*/ - 22"),483 el!(Expr::BinaryOp(484 el!(Expr::Num(2.0)),485 Sub,486 el!(Expr::Num(22.0))487 ))488 );489 }490491 #[test]492 fn suffix() {493 // assert_eq!(parse!("std.test"), el!(Expr::Num(2.2)));494 // assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2)));495 // assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2)));496 // assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2)))497 }498499 #[test]500 fn array_comp() {501 use Expr::*;502 assert_eq!(503 parse!("[std.deepJoin(x) for x in arr]"),504 el!(ArrComp(505 el!(Apply(506 el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))),507 ArgsDesc::new(vec![el!(Var("x".into()))], vec![]),508 false,509 )),510 vec![CompSpec::ForSpec(ForSpecData(511 "x".into(),512 el!(Var("arr".into()))513 ))]514 )),515 )516 }517518 #[test]519 fn reserved() {520 use Expr::*;521 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));522 assert_eq!(parse!("nulla"), el!(Var("nulla".into())));523 }524525 #[test]526 fn multiple_args_buf() {527 parse!("a(b, null_fields)");528 }529530 #[test]531 fn infix_precedence() {532 use Expr::*;533 assert_eq!(534 parse!("!a && !b"),535 el!(BinaryOp(536 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),537 And,538 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))539 ))540 );541 }542543 #[test]544 fn infix_precedence_division() {545 use Expr::*;546 assert_eq!(547 parse!("!a / !b"),548 el!(BinaryOp(549 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),550 Div,551 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))552 ))553 );554 }555556 #[test]557 fn double_negation() {558 use Expr::*;559 assert_eq!(560 parse!("!!a"),561 el!(UnaryOp(562 UnaryOpType::Not,563 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()))))564 ))565 )566 }567568 #[test]569 fn array_test_error() {570 parse!("[a for a in b if c for e in f]");571 // ^^^^ failed code572 }573574 #[test]575 fn can_parse_stdlib() {576 parse!(jrsonnet_stdlib::STDLIB_STR);577 }578579 // From source code580 /*581 #[bench]582 fn bench_parse_peg(b: &mut Bencher) {583 b.iter(|| parse!(jrsonnet_stdlib::STDLIB_STR))584 }585 */586}1#![allow(clippy::redundant_closure_call)]23use peg::parser;4use std::{5 path::{Path, PathBuf},6 rc::Rc,7};8mod expr;9pub use expr::*;10pub use jrsonnet_interner::IStr;11pub use peg;1213pub struct ParserSettings {14 pub loc_data: bool,15 pub file_name: Rc<Path>,16}1718macro_rules! expr_bin {19 ($a:ident $op:ident $b:ident) => {20 Expr::BinaryOp($a, $op, $b)21 };22}23macro_rules! expr_un {24 ($op:ident $a:ident) => {25 Expr::UnaryOp($op, $a)26 };27}2829parser! {30 grammar jsonnet_parser() for str {31 use peg::ParseLiteral;3233 /// Standard C-like comments34 rule comment()35 = "//" (!['\n'][_])* "\n"36 / "/*" ("\\*/" / "\\\\" / (!("*/")[_]))* "*/"37 / "#" (!['\n'][_])* "\n"3839 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")40 rule _() = single_whitespace()*4142 /// For comma-delimited elements43 rule comma() = quiet!{_ "," _} / expected!("<comma>")44 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}45 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}46 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']47 /// Sequence of digits48 rule uint_str() -> &'input str = a:$(digit()+) { a }49 /// Number in scientific notation format50 rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.parse().map_err(|_| "<number>") }} / expected!("<number>")5152 /// Reserved word followed by any non-alphanumberic53 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()54 rule id() = quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")5556 rule keyword(id: &'static str) -> ()57 = ##parse_string_literal(id) end_of_ident()5859 pub rule param(s: &ParserSettings) -> expr::Param = name:$(id()) expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name.into(), expr) }60 pub rule params(s: &ParserSettings) -> expr::ParamsDesc61 = params:param(s) ** comma() comma()? {62 let mut defaults_started = false;63 for param in ¶ms {64 defaults_started = defaults_started || param.1.is_some();65 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");66 }67 expr::ParamsDesc(Rc::new(params))68 }69 / { expr::ParamsDesc(Rc::new(Vec::new())) }7071 pub rule arg(s: &ParserSettings) -> (Option<IStr>, LocExpr)72 = quiet! { name:(s:$(id()) _ "=" _ {s})? expr:expr(s) {(name.map(Into::into), expr)} }73 / expected!("<argument>")7475 pub rule args(s: &ParserSettings) -> expr::ArgsDesc76 = args:arg(s)**comma() comma()? {?77 let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();78 let mut unnamed = Vec::with_capacity(unnamed_count);79 let mut named = Vec::with_capacity(args.len() - unnamed_count);80 let mut named_started = false;81 for (name, value) in args {82 if let Some(name) = name {83 named_started = true;84 named.push((name, value));85 } else {86 if named_started {87 return Err("<named argument>")88 }89 unnamed.push(value);90 }91 }92 Ok(expr::ArgsDesc::new(unnamed, named))93 }9495 pub rule bind(s: &ParserSettings) -> expr::BindSpec96 = name:$(id()) _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: None, value: expr}}97 / name:$(id()) _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name:name.into(), params: Some(params), value: expr}}98 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt99 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }100101 pub rule whole_line() -> &'input str102 = str:$((!['\n'][_])* "\n") {str}103 pub rule string_block() -> String104 = "|||" (!['\n']single_whitespace())* "\n"105 empty_lines:$(['\n']*)106 prefix:[' ' | '\t']+ first_line:whole_line()107 lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*108 [' ' | '\t']*<, {prefix.len() - 1}> "|||"109 {let mut l = empty_lines.to_owned(); l.push_str(first_line); l.extend(lines); l}110 pub rule string() -> String111 = quiet!{ "\"" str:$(("\\\"" / "\\\\" / (!['"'][_]))*) "\"" {unescape::unescape(str).unwrap()}112 / "'" str:$(("\\'" / "\\\\" / (!['\''][_]))*) "'" {unescape::unescape(str).unwrap()}113 / "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}114 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}115 / string_block() } / expected!("<string>")116117 pub rule field_name(s: &ParserSettings) -> expr::FieldName118 = name:$(id()) {expr::FieldName::Fixed(name.into())}119 / name:string() {expr::FieldName::Fixed(name.into())}120 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}121 pub rule visibility() -> expr::Visibility122 = ":::" {expr::Visibility::Unhide}123 / "::" {expr::Visibility::Hidden}124 / ":" {expr::Visibility::Normal}125 pub rule field(s: &ParserSettings) -> expr::FieldMember126 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{127 name,128 plus: plus.is_some(),129 params: None,130 visibility,131 value,132 }}133 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{134 name,135 plus: false,136 params: Some(params),137 visibility,138 value,139 }}140 pub rule obj_local(s: &ParserSettings) -> BindSpec141 = keyword("local") _ bind:bind(s) {bind}142 pub rule member(s: &ParserSettings) -> expr::Member143 = bind:obj_local(s) {expr::Member::BindStmt(bind)}144 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}145 / field:field(s) {expr::Member::Field(field)}146 pub rule objinside(s: &ParserSettings) -> expr::ObjBody147 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ plus:"+"? _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {148 let mut compspecs = vec![CompSpec::ForSpec(forspec)];149 compspecs.extend(others.unwrap_or_default());150 expr::ObjBody::ObjComp(expr::ObjComp{151 pre_locals,152 key,153 plus: plus.is_some(),154 value,155 post_locals,156 compspecs,157 })158 }159 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}160 pub rule ifspec(s: &ParserSettings) -> IfSpecData161 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}162 pub rule forspec(s: &ParserSettings) -> ForSpecData163 = keyword("for") _ id:$(id()) _ keyword("in") _ cond:expr(s) {ForSpecData(id.into(), cond)}164 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>165 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}166 pub rule local_expr(s: &ParserSettings) -> Expr167 = keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }168 pub rule string_expr(s: &ParserSettings) -> Expr169 = s:string() {Expr::Str(s.into())}170 pub rule obj_expr(s: &ParserSettings) -> Expr171 = "{" _ body:objinside(s) _ "}" {Expr::Obj(body)}172 pub rule array_expr(s: &ParserSettings) -> Expr173 = "[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}174 pub rule array_comp_expr(s: &ParserSettings) -> Expr175 = "[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {176 let mut specs = vec![CompSpec::ForSpec(forspec)];177 specs.extend(others.unwrap_or_default());178 Expr::ArrComp(expr, specs)179 }180 pub rule number_expr(s: &ParserSettings) -> Expr181 = n:number() { expr::Expr::Num(n) }182 pub rule var_expr(s: &ParserSettings) -> Expr183 = n:$(id()) { expr::Expr::Var(n.into()) }184 pub rule if_then_else_expr(s: &ParserSettings) -> Expr185 = cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{186 cond,187 cond_then,188 cond_else,189 }}190191 pub rule literal(s: &ParserSettings) -> Expr192 = v:(193 keyword("null") {LiteralType::Null}194 / keyword("true") {LiteralType::True}195 / keyword("false") {LiteralType::False}196 / keyword("self") {LiteralType::This}197 / keyword("$") {LiteralType::Dollar}198 / keyword("super") {LiteralType::Super}199 ) {Expr::Literal(v)}200201 pub rule expr_basic(s: &ParserSettings) -> Expr202 = literal(s)203204 / quiet!{"$intrinsic(" name:$(id()) ")" {Expr::Intrinsic(name.into())}}205206 / string_expr(s) / number_expr(s)207 / array_expr(s)208 / obj_expr(s)209 / array_expr(s)210 / array_comp_expr(s)211212 / keyword("importstr") _ path:string() {Expr::ImportStr(PathBuf::from(path))}213 / keyword("import") _ path:string() {Expr::Import(PathBuf::from(path))}214215 / var_expr(s)216 / local_expr(s)217 / if_then_else_expr(s)218219 / keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}220 / assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }221222 / keyword("error") _ expr:expr(s) { Expr::ErrorStmt(expr) }223224 rule slice_part(s: &ParserSettings) -> Option<LocExpr>225 = e:(_ e:expr(s) _{e})? {e}226 pub rule slice_desc(s: &ParserSettings) -> SliceDesc227 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {228 let (end, step) = if let Some((end, step)) = pair {229 (end, step)230 }else{231 (None, None)232 };233234 SliceDesc { start, end, step }235 }236237 rule binop(x: rule<()>) -> ()238 = quiet!{ x() } / expected!("<binary op>")239 rule unaryop(x: rule<()>) -> ()240 = quiet!{ x() } / expected!("<unary op>")241242243 use BinaryOpType::*;244 use UnaryOpType::*;245 rule expr(s: &ParserSettings) -> LocExpr246 = precedence! {247 start:position!() v:@ end:position!() { loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end)) }248 --249 a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}250 --251 a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}252 --253 a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}254 --255 a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}256 --257 a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}258 --259 a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}260 a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}261 --262 a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}263 a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}264 a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}265 a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}266 a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}267 --268 a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}269 a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}270 --271 a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}272 a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}273 --274 a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}275 a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}276 a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}277 --278 unaryop(<"-">) _ b:@ {expr_un!(Minus b)}279 unaryop(<"!">) _ b:@ {expr_un!(Not b)}280 unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}281 --282 a:(@) _ "[" _ e:slice_desc(s) _ "]" {Expr::Slice(a, e)}283 a:(@) _ "." _ e:$(id()) {Expr::Index(a, el!(Expr::Str(e.into())))}284 a:(@) _ "[" _ e:expr(s) _ "]" {Expr::Index(a, e)}285 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {Expr::Apply(a, args, ts.is_some())}286 a:(@) _ "{" _ body:objinside(s) _ "}" {Expr::ObjExtend(a, body)}287 --288 e:expr_basic(s) {e}289 "(" _ e:expr(s) _ ")" {Expr::Parened(e)}290 }291292 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}293 }294}295296pub type ParseError = peg::error::ParseError<peg::str::LineCol>;297pub fn parse(str: &str, settings: &ParserSettings) -> Result<LocExpr, ParseError> {298 jsonnet_parser::jsonnet(str, settings)299}300301#[macro_export]302macro_rules! el {303 ($expr:expr) => {304 LocExpr(std::rc::Rc::new($expr), None)305 };306}307308#[cfg(test)]309pub mod tests {310 use super::{expr::*, parse};311 use crate::ParserSettings;312 use std::path::PathBuf;313 use BinaryOpType::*;314315 macro_rules! parse {316 ($s:expr) => {317 parse(318 $s,319 &ParserSettings {320 loc_data: false,321 file_name: PathBuf::from("/test.jsonnet").into(),322 },323 )324 .unwrap()325 };326 }327328 macro_rules! el_loc {329 ($expr:expr, $loc:expr$(,)?) => {330 LocExpr(std::rc::Rc::new($expr), Some($loc))331 };332 }333334 mod expressions {335 use super::*;336337 pub fn basic_math() -> LocExpr {338 el!(Expr::BinaryOp(339 el!(Expr::Num(2.0)),340 Add,341 el!(Expr::BinaryOp(342 el!(Expr::Num(2.0)),343 Mul,344 el!(Expr::Num(2.0)),345 )),346 ))347 }348 }349350 #[test]351 fn multiline_string() {352 assert_eq!(353 parse!("|||\n Hello world!\n a\n|||"),354 el!(Expr::Str("Hello world!\n a\n".into())),355 );356 assert_eq!(357 parse!("|||\n Hello world!\n a\n|||"),358 el!(Expr::Str("Hello world!\n a\n".into())),359 );360 assert_eq!(361 parse!("|||\n\t\tHello world!\n\t\t\ta\n|||"),362 el!(Expr::Str("Hello world!\n\ta\n".into())),363 );364 assert_eq!(365 parse!("|||\n Hello world!\n a\n |||"),366 el!(Expr::Str("Hello world!\n a\n".into())),367 );368 }369370 #[test]371 fn slice() {372 parse!("a[1:]");373 parse!("a[1::]");374 parse!("a[:1:]");375 parse!("a[::1]");376 parse!("str[:len - 1]");377 }378379 #[test]380 fn string_escaping() {381 assert_eq!(382 parse!(r#""Hello, \"world\"!""#),383 el!(Expr::Str(r#"Hello, "world"!"#.into())),384 );385 assert_eq!(386 parse!(r#"'Hello \'world\'!'"#),387 el!(Expr::Str("Hello 'world'!".into())),388 );389 assert_eq!(parse!(r#"'\\\\'"#), el!(Expr::Str("\\\\".into())),);390 }391392 #[test]393 fn string_unescaping() {394 assert_eq!(395 parse!(r#""Hello\nWorld""#),396 el!(Expr::Str("Hello\nWorld".into())),397 );398 }399400 #[test]401 fn string_verbantim() {402 assert_eq!(403 parse!(r#"@"Hello\n""World""""#),404 el!(Expr::Str("Hello\\n\"World\"".into())),405 );406 }407408 #[test]409 fn imports() {410 assert_eq!(411 parse!("import \"hello\""),412 el!(Expr::Import(PathBuf::from("hello"))),413 );414 assert_eq!(415 parse!("importstr \"garnish.txt\""),416 el!(Expr::ImportStr(PathBuf::from("garnish.txt")))417 );418 }419420 #[test]421 fn empty_object() {422 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));423 }424425 #[test]426 fn basic_math() {427 assert_eq!(428 parse!("2+2*2"),429 el!(Expr::BinaryOp(430 el!(Expr::Num(2.0)),431 Add,432 el!(Expr::BinaryOp(433 el!(Expr::Num(2.0)),434 Mul,435 el!(Expr::Num(2.0))436 ))437 ))438 );439 }440441 #[test]442 fn basic_math_with_indents() {443 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());444 }445446 #[test]447 fn basic_math_parened() {448 assert_eq!(449 parse!("2+(2+2*2)"),450 el!(Expr::BinaryOp(451 el!(Expr::Num(2.0)),452 Add,453 el!(Expr::Parened(expressions::basic_math())),454 ))455 );456 }457458 /// Comments should not affect parsing459 #[test]460 fn comments() {461 assert_eq!(462 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),463 el!(Expr::BinaryOp(464 el!(Expr::Num(2.0)),465 Add,466 el!(Expr::BinaryOp(467 el!(Expr::Num(3.0)),468 Mul,469 el!(Expr::Num(4.0))470 ))471 ))472 );473 }474475 /// Comments should be able to be escaped476 #[test]477 fn comment_escaping() {478 assert_eq!(479 parse!("2/*\\*/+*/ - 22"),480 el!(Expr::BinaryOp(481 el!(Expr::Num(2.0)),482 Sub,483 el!(Expr::Num(22.0))484 ))485 );486 }487488 #[test]489 fn suffix() {490 // assert_eq!(parse!("std.test"), el!(Expr::Num(2.2)));491 // assert_eq!(parse!("std(2)"), el!(Expr::Num(2.2)));492 // assert_eq!(parse!("std.test(2)"), el!(Expr::Num(2.2)));493 // assert_eq!(parse!("a[b]"), el!(Expr::Num(2.2)))494 }495496 #[test]497 fn array_comp() {498 use Expr::*;499 assert_eq!(500 parse!("[std.deepJoin(x) for x in arr]"),501 el!(ArrComp(502 el!(Apply(503 el!(Index(el!(Var("std".into())), el!(Str("deepJoin".into())))),504 ArgsDesc::new(vec![el!(Var("x".into()))], vec![]),505 false,506 )),507 vec![CompSpec::ForSpec(ForSpecData(508 "x".into(),509 el!(Var("arr".into()))510 ))]511 )),512 )513 }514515 #[test]516 fn reserved() {517 use Expr::*;518 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));519 assert_eq!(parse!("nulla"), el!(Var("nulla".into())));520 }521522 #[test]523 fn multiple_args_buf() {524 parse!("a(b, null_fields)");525 }526527 #[test]528 fn infix_precedence() {529 use Expr::*;530 assert_eq!(531 parse!("!a && !b"),532 el!(BinaryOp(533 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),534 And,535 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))536 ))537 );538 }539540 #[test]541 fn infix_precedence_division() {542 use Expr::*;543 assert_eq!(544 parse!("!a / !b"),545 el!(BinaryOp(546 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into())))),547 Div,548 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".into()))))549 ))550 );551 }552553 #[test]554 fn double_negation() {555 use Expr::*;556 assert_eq!(557 parse!("!!a"),558 el!(UnaryOp(559 UnaryOpType::Not,560 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".into()))))561 ))562 )563 }564565 #[test]566 fn array_test_error() {567 parse!("[a for a in b if c for e in f]");568 // ^^^^ failed code569 }570571 #[test]572 fn can_parse_stdlib() {573 parse!(jrsonnet_stdlib::STDLIB_STR);574 }575576 #[test]577 fn add_location_info_to_all_sub_expressions() {578 use Expr::*;579580 let file_name: std::rc::Rc<std::path::Path> = PathBuf::from("/test.jsonnet").into();581 let expr = parse(582 "{} { local x = 1, x: x } + {}",583 &ParserSettings {584 loc_data: true,585 file_name: file_name.clone(),586 },587 )588 .unwrap();589 assert_eq!(590 expr,591 el_loc!(592 BinaryOp(593 el_loc!(594 ObjExtend(595 el_loc!(596 Obj(ObjBody::MemberList(vec![])),597 ExprLocation(file_name.clone(), 0, 2)598 ),599 ObjBody::MemberList(vec![600 Member::BindStmt(BindSpec {601 name: "x".into(),602 params: None,603 value: el_loc!(604 Num(1.0),605 ExprLocation(file_name.clone(), 15, 16)606 )607 }),608 Member::Field(FieldMember {609 name: FieldName::Fixed("x".into()),610 plus: false,611 params: None,612 visibility: Visibility::Normal,613 value: el_loc!(614 Var("x".into()),615 ExprLocation(file_name.clone(), 21, 22)616 ),617 })618 ])619 ),620 ExprLocation(file_name.clone(), 0, 24)621 ),622 BinaryOpType::Add,623 el_loc!(624 Obj(ObjBody::MemberList(vec![])),625 ExprLocation(file_name.clone(), 27, 29)626 ),627 ),628 ExprLocation(file_name.clone(), 0, 29),629 ),630 );631 }632 // From source code633 /*634 #[bench]635 fn bench_parse_peg(b: &mut Bencher) {636 b.iter(|| parse!(jrsonnet_stdlib::STDLIB_STR))637 }638 */639}