1use jrsonnet_ir::{2 BinaryOp, Expr, ExprParams, IStr, IndexPart, Member, Slice, SliceDesc, Source, Span, Spanned,3 ExprParam, ArgsDesc, AssertExpr, ImportKind, LiteralType, IfElse, CompSpec, ForSpecData, IfSpecData, ObjMembers, ObjBody,4 ObjComp, FieldMember, Visibility, FieldName, unescape, AssertStmt, BindSpec, Destruct, DestructRest,5};6use peg::parser;7use std::rc::Rc;89pub struct ParserSettings {10 pub source: Source,11}1213macro_rules! expr_bin {14 ($a:ident $op:ident $b:ident) => {15 Expr::BinaryOp(Box::new(BinaryOp {16 lhs: $a,17 op: $op,18 rhs: $b,19 }))20 };21}22macro_rules! expr_un {23 ($op:ident $a:ident) => {24 Expr::UnaryOp($op, Box::new($a))25 };26}2728parser! {29 grammar jsonnet_parser() for str {30 use peg::ParseLiteral;3132 rule eof() = quiet!{![_]} / expected!("<eof>")33 rule eol() = "\n" / eof()3435 36 rule comment()37 = "//" (!eol()[_])* eol()38 / "/*" (!("*/")[_])* "*/"39 / "#" (!eol()[_])* eol()4041 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")42 rule _() = quiet!{([' ' | '\r' | '\n' | '\t']+) / comment()}* / expected!("<whitespace>")4344 45 rule comma() = quiet!{_ "," _} / expected!("<comma>")46 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}47 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}48 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']49 50 rule uint_str() -> &'input str = a:$(digit()+ ("_" digit()+)*) { a }51 52 rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.replace("_","").parse().map_err(|_| "<number>") }} / expected!("<number>")5354 55 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "importbin" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()56 rule id() -> IStr = v:$(quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")) { v.into() }5758 rule keyword(id: &'static str) -> ()59 = ##parse_string_literal(id) end_of_ident()6061 pub rule param(s: &ParserSettings) -> ExprParam = destruct:destruct(s) expr:(_ "=" _ expr:expr(s){expr})? { ExprParam { destruct, default: expr.map(Rc::new) } }62 pub rule params(s: &ParserSettings) -> ExprParams63 = params:param(s) ** comma() comma()? { ExprParams::new(params) }64 / { ExprParams::new(Vec::new()) }6566 pub rule arg(s: &ParserSettings) -> (Option<IStr>, Rc<Spanned<Expr>>)67 = name:(quiet! { (s:id() _ "=" !['='] _ {s})? } / expected!("<argument name>")) expr:expr(s) {(name, Rc::new(expr))}6869 pub rule args(s: &ParserSettings) -> ArgsDesc70 = args:arg(s)**comma() comma()? {?71 let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();72 let mut unnamed = Vec::with_capacity(unnamed_count);73 let mut named = Vec::with_capacity(args.len() - unnamed_count);74 let mut named_started = false;75 for (name, value) in args {76 if let Some(name) = name {77 named_started = true;78 named.push((name, value));79 } else {80 if named_started {81 return Err("<named argument>")82 }83 unnamed.push(value);84 }85 }86 Ok(ArgsDesc::new(unnamed, named))87 }8889 pub rule destruct_rest() -> DestructRest90 = "..." into:(_ into:id() {into})? {if let Some(into) = into {91 DestructRest::Keep(into)92 } else {DestructRest::Drop}}93 pub rule destruct_array(s: &ParserSettings) -> Destruct94 = "[" _ start:destruct(s)**comma() rest:(95 comma() _ rest:destruct_rest()? end:(96 comma() end:destruct(s)**comma() (_ comma())? {end}97 / comma()? {Vec::new()}98 ) {(rest, end)}99 / comma()? {(None, Vec::new())}100 ) _ "]" {?101 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Array {102 start,103 rest: rest.0,104 end: rest.1,105 });106 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")107 }108 pub rule destruct_object(s: &ParserSettings) -> Destruct109 = "{" _110 fields:(name:id() into:(_ ":" _ into:destruct(s) {into})? default:(_ "=" _ v:expr(s) {v})? {(name, into, default.map(Rc::new))})**comma()111 rest:(112 comma() rest:destruct_rest()? {rest}113 / comma()? {None}114 )115 _ "}" {?116 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Object {117 fields,118 rest,119 });120 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")121 }122 pub rule destruct(s: &ParserSettings) -> Destruct123 = v:id() {Destruct::Full(v)}124 / "?" {?125 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Skip);126 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")127 }128 / arr:destruct_array(s) {arr}129 / obj:destruct_object(s) {obj}130131 pub rule bind(s: &ParserSettings) -> BindSpec132 = into:destruct(s) _ "=" _ value:expr(s) {BindSpec::Field{into, value: Rc::new(value)}}133 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ value:expr(s) {BindSpec::Function{name, params, value: Rc::new(value)}}134135 pub rule assertion(s: &ParserSettings) -> AssertStmt136 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { AssertStmt(cond, msg) }137138 pub rule whole_line() -> &'input str139 = str:$((!['\n'][_])* "\n") {str}140 pub rule string_block() -> String141 = "|||" chomped:"-"? (!['\n']single_whitespace())* "\n"142 empty_lines:$(['\n']*)143 prefix:[' ' | '\t']+ first_line:whole_line()144 lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*145 [' ' | '\t']*<, {prefix.len() - 1}> "|||"146 {147 let mut l = empty_lines.to_owned();148 l.push_str(first_line);149 l.extend(lines);150 if chomped.is_some() {151 debug_assert!(l.ends_with('\n'));152 l.truncate(l.len() - 1);153 }154 l155 }156157 rule hex_char()158 = quiet! { ['0'..='9' | 'a'..='f' | 'A'..='F'] } / expected!("<hex char>")159160 rule string_char(c: rule<()>)161 = (!['\\']!c()[_])+162 / "\\\\"163 / "\\u" hex_char() hex_char() hex_char() hex_char()164 / "\\x" hex_char() hex_char()165 / ['\\'] (quiet! { ['b' | 'f' | 'n' | 'r' | 't' | '"' | '\''] } / expected!("<escape character>"))166 pub rule string() -> String167 = ['"'] str:$(string_char(<"\"">)*) ['"'] {? unescape::unescape(str).ok_or("<escaped string>")}168 / ['\''] str:$(string_char(<"\'">)*) ['\''] {? unescape::unescape(str).ok_or("<escaped string>")}169 / quiet!{ "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}170 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}171 / string_block() } / expected!("<string>")172173 pub rule field_name(s: &ParserSettings) -> FieldName174 = name:id() {FieldName::Fixed(name)}175 / name:string() {FieldName::Fixed(name.into())}176 / "[" _ expr:expr(s) _ "]" {FieldName::Dyn(expr)}177 pub rule visibility() -> Visibility178 = ":::" {Visibility::Unhide}179 / "::" {Visibility::Hidden}180 / ":" {Visibility::Normal}181 pub rule field(s: &ParserSettings) -> FieldMember182 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {FieldMember{183 name,184 plus: plus.is_some(),185 params: None,186 visibility,187 value: Rc::new(value),188 }}189 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {FieldMember{190 name,191 plus: false,192 params: Some(params),193 visibility,194 value: Rc::new(value),195 }}196 pub rule obj_local(s: &ParserSettings) -> BindSpec197 = keyword("local") _ bind:bind(s) {bind}198 pub rule member(s: &ParserSettings) -> Member199 = bind:obj_local(s) {Member::BindStmt(bind)}200 / assertion:assertion(s) {Member::AssertStmt(assertion)}201 / field:field(s) {Member::Field(field)}202 pub rule objinside(s: &ParserSettings) -> ObjBody203 = members:(member(s) ** comma()) comma()? _ compspecs:compspecs(s)? {?204 Ok(if let Some(compspecs) = compspecs {205 let mut locals = Vec::new();206 let mut field = None;207 for member in members {208 match member {209 Member::Field(field_member) => if field.replace(field_member).is_some() {210 return Err("<object comprehension can only contain one field>")211 },212 Member::BindStmt(bind_spec) => locals.push(bind_spec),213 Member::AssertStmt(assert_stmt) => return Err("<asserts are unsupported in object comprehension>"),214 }215 }216 ObjBody::ObjComp(ObjComp {217 locals: Rc::new(locals),218 field: field.map(Rc::new).ok_or("<missing object comprehension field>")?,219 compspecs220 })221 } else {222 let mut locals = Vec::new();223 let mut asserts = Vec::new();224 let mut fields = Vec::new();225 for member in members {226 match member {227 Member::Field(field_member) => fields.push(field_member),228 Member::BindStmt(bind_spec) => locals.push(bind_spec),229 Member::AssertStmt(assert_stmt) => asserts.push(assert_stmt),230 }231 }232 ObjBody::MemberList(ObjMembers {233 locals: Rc::new(locals),234 asserts: Rc::new(asserts),235 fields236 })237 })238 }239 pub rule ifspec(s: &ParserSettings) -> IfSpecData240 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}241 pub rule forspec(s: &ParserSettings) -> ForSpecData242 = keyword("for") _ id:destruct(s) _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}243 rule compspec(s: &ParserSettings) -> CompSpec244 = i:ifspec(s) { CompSpec::IfSpec(i) } / f:forspec(s) {CompSpec::ForSpec(f)}245 pub rule compspecs(s: &ParserSettings) -> Vec<CompSpec>246 = specs:compspec(s) ++ _ {?247 if !matches!(specs[0], CompSpec::ForSpec(_)) {248 return Err("<first compspec should be for>")249 }250 Ok(specs)251 }252 pub rule local_expr(s: &ParserSettings) -> Expr253 = keyword("local") _ binds:bind(s) ** comma() (_ ",")? _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, Box::new(expr)) }254 pub rule string_expr(s: &ParserSettings) -> Expr255 = s:string() {Expr::Str(s.into())}256 pub rule obj_expr(s: &ParserSettings) -> Expr257 = "{" _ body:objinside(s) _ "}" {Expr::Obj(body)}258 pub rule array_expr(s: &ParserSettings) -> Expr259 = "[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(Rc::new(elems))}260 pub rule array_comp_expr(s: &ParserSettings) -> Expr261 = "[" _ expr:expr(s) _ comma()? _ specs:(r: compspecs(s) _ {r}) "]" {262 Expr::ArrComp(Rc::new(expr), specs)263 }264 pub rule number_expr(s: &ParserSettings) -> Expr265 = n:number() {? if n.is_finite() {266 Ok(Expr::Num(n))267 } else {268 Err("!!!numbers are finite")269 }}270 pub rule var_expr(s: &ParserSettings) -> Expr271 = n:id() { Expr::Var(n) }272 pub rule id_loc(s: &ParserSettings) -> Spanned<Expr>273 = a:position!() n:id() b:position!() { Spanned::new(Expr::Str(n), Span(s.source.clone(), a as u32,b as u32)) }274 pub rule if_then_else_expr(s: &ParserSettings) -> Expr275 = cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse(Box::new(IfElse{276 cond,277 cond_then,278 cond_else,279 }))}280281 pub rule literal(s: &ParserSettings) -> Expr282 = v:(283 keyword("null") {LiteralType::Null}284 / keyword("true") {LiteralType::True}285 / keyword("false") {LiteralType::False}286 / keyword("self") {LiteralType::This}287 / keyword("$") {LiteralType::Dollar}288 / keyword("super") {LiteralType::Super}289 ) {Expr::Literal(v)}290291 rule import_kind() -> ImportKind292 = keyword("importstr") { ImportKind::Str }293 / keyword("importbin") { ImportKind::Bin }294 / keyword("import") { ImportKind::Normal }295296 pub rule expr_basic(s: &ParserSettings) -> Expr297 = literal(s)298299 / string_expr(s) / number_expr(s)300 / array_expr(s)301 / obj_expr(s)302 / array_expr(s)303 / array_comp_expr(s)304305 / kind:import_kind() _ path:expr(s) {Expr::Import(kind, Box::new(path))}306307 / var_expr(s)308 / local_expr(s)309 / if_then_else_expr(s)310311 / keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, Rc::new(expr))}312 / assert:assertion(s) _ ";" _ rest:expr(s) { Expr::AssertExpr(Rc::new(AssertExpr{313 assert, rest314 })) }315316 / keyword("error") _ expr:expr(s) { Expr::ErrorStmt(Box::new(expr)) }317318 rule slice_part(s: &ParserSettings) -> Option<Spanned<Expr>>319 = _ e:(e:expr(s) _{e})? {e}320 pub rule slice_desc(s: &ParserSettings) -> SliceDesc321 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {322 let (end, step) = if let Some((end, step)) = pair {323 (end, step)324 }else{325 (None, None)326 };327328 SliceDesc { start, end, step }329 }330331 rule binop(x: rule<()>) -> ()332 = quiet!{ x() } / expected!("<binary op>")333 rule unaryop(x: rule<()>) -> ()334 = quiet!{ x() } / expected!("<unary op>")335336 rule ensure_null_coaelse()337 = "" {?338 #[cfg(not(feature = "exp-null-coaelse"))] return Err("!!!experimental null coaelscing was not enabled");339 #[cfg(feature = "exp-null-coaelse")] Ok(())340 }341 use jrsonnet_ir::BinaryOpType::*;342 use jrsonnet_ir::UnaryOpType::*;343 rule expr(s: &ParserSettings) -> Spanned<Expr>344 = precedence! {345 "(" _ e:expr(s) _ ")" {e}346 start:position!() v:@ end:position!() { Spanned::new(v, Span(s.source.clone(), start as u32, end as u32)) }347 --348 a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}349 a:(@) _ binop(<"??">) _ ensure_null_coaelse() b:@ {350 #[cfg(feature = "exp-null-coaelse")] return expr_bin!(a NullCoaelse b);351 unreachable!("ensure_null_coaelse will fail if feature is not enabled")352 }353 --354 a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}355 --356 a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}357 --358 a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}359 --360 a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}361 --362 a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}363 a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}364 --365 a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}366 a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}367 a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}368 a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}369 a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}370 --371 a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}372 a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}373 --374 a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}375 a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}376 --377 a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}378 a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}379 a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}380 --381 unaryop(<"+">) _ b:@ {expr_un!(Plus b)}382 unaryop(<"-">) _ b:@ {expr_un!(Minus b)}383 unaryop(<"!">) _ b:@ {expr_un!(Not b)}384 unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}385 --386 value:(@) _ "[" _ slice:slice_desc(s) _ "]" {Expr::Slice(Box::new(Slice{value, slice}))}387 indexable:(@) _ parts:index_part(s)+ {Expr::Index{indexable: Box::new(indexable), parts}}388 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {Expr::Apply(Box::new(a), args, ts.is_some())}389 a:(@) _ "{" _ body:objinside(s) _ "}" {Expr::ObjExtend(Rc::new(a), body)}390 --391 e:expr_basic(s) {e}392 }393 pub rule index_part(s: &ParserSettings) -> IndexPart394 = n:("?" _ ensure_null_coaelse())? "." _ value:id_loc(s) {IndexPart {395 value,396 #[cfg(feature = "exp-null-coaelse")]397 null_coaelse: n.is_some(),398 }}399 / n:("?" _ "." _ ensure_null_coaelse())? "[" _ value:expr(s) _ "]" {IndexPart {400 value,401 #[cfg(feature = "exp-null-coaelse")]402 null_coaelse: n.is_some(),403 }}404405 pub rule jsonnet(s: &ParserSettings) -> Spanned<Expr> = _ e:expr(s) _ {e}406 }407}408409pub type ParseError = peg::error::ParseError<peg::str::LineCol>;410pub fn parse(str: &str, settings: &ParserSettings) -> Result<Spanned<Expr>, ParseError> {411 jsonnet_parser::jsonnet(str, settings)412}413414pub fn string_to_expr(str: IStr, settings: &ParserSettings) -> Spanned<Expr> {415 let len = str.len();416 Spanned::new(Expr::Str(str), Span(settings.source.clone(), 0, len as u32))417}418419#[cfg(test)]420pub mod tests {421 use insta::assert_snapshot;422 use jrsonnet_ir::{IStr, Source};423424 use super::parse;425 use crate::ParserSettings;426427 fn parsep(s: &str) -> String {428 let v = parse(429 s,430 &ParserSettings {431 source: Source::new_virtual("<test>".into(), IStr::empty()),432 },433 )434 .unwrap();435 format!("{v:#?}")436 }437438 macro_rules! parse {439 ($s:expr) => {440 assert_snapshot!(parsep($s));441 };442 }443444 #[test]445 fn multiline_string() {446 parse!("|||\n Hello world!\n a\n|||");447 parse!("|||\n Hello world!\n a\n|||");448 parse!("|||\n\t\tHello world!\n\t\t\ta\n|||");449 parse!("|||\n Hello world!\n a\n |||");450 }451452 #[test]453 fn slice() {454 parse!("a[1:]");455 parse!("a[1::]");456 parse!("a[:1:]");457 parse!("a[::1]");458 parse!("str[:len - 1]");459 }460461 #[test]462 fn string_escaping() {463 parse!(r#""Hello, \"world\"!""#);464 parse!(r#"'Hello \'world\'!'"#);465 parse!(r#"'\\\\'"#);466 }467468 #[test]469 fn string_unescaping() {470 parse!(r#""Hello\nWorld""#);471 }472473 #[test]474 fn string_verbantim() {475 parse!(r#"@"Hello\n""World""""#);476 }477478 #[test]479 fn imports() {480 parse!("import \"hello\"");481 parse!("importstr \"garnish.txt\"");482 parse!("importbin \"garnish.bin\"");483 }484485 #[test]486 fn empty_object() {487 parse!("{}");488 }489490 #[test]491 fn basic_math() {492 parse!("2+2*2");493 parse!("2 + 2 * 2 ");494 parse!("2+(2+2*2)");495 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4");496 }497498 #[test]499 fn suffix() {500 parse!("std.test");501 parse!("std(2)");502 parse!("std.test(2)");503 parse!("a[b]");504 }505506 #[test]507 fn array_comp() {508 parse!("[std.deepJoin(x) for x in arr]");509 }510511 #[test]512 fn reserved() {513 parse!("null");514 parse!("nulla");515 }516517 #[test]518 fn multiple_args_buf() {519 parse!("a(b, null_fields)");520 }521522 #[test]523 fn infix_precedence() {524 parse!("!a && !b");525 parse!("!a / !b");526 }527528 #[test]529 fn double_negation() {530 parse!("!!a");531 }532533 #[test]534 fn array_test_error() {535 parse!("[a for a in b if c for e in f]");536 }537538 #[test]539 fn missing_newline_between_comment_and_eof() {540 parse!(541 "{a:1}542543 //+213"544 );545 }546547 #[test]548 fn default_param_before_nondefault() {549 parse!("local x(foo = 'foo', bar) = null; null");550 }551552 #[test]553 fn add_location_info_to_all_sub_expressions() {554 parse!("{} { local x = 1, x: x } + {}");555 }556}