1#![allow(clippy::redundant_closure_call, clippy::derive_partial_eq_without_eq)]23use std::rc::Rc;45use peg::parser;6mod expr;7pub use expr::*;8pub use jrsonnet_interner::IStr;9pub use peg;10pub mod function;11mod location;12mod source;13mod unescape;1415pub use location::CodeLocation;16pub use source::{17 Source, SourceDefaultIgnoreJpath, SourceDirectory, SourceFifo, SourceFile, SourcePath,18 SourcePathT, SourceVirtual,19};2021pub struct ParserSettings {22 pub source: Source,23}2425macro_rules! expr_bin {26 ($a:ident $op:ident $b:ident) => {27 Expr::BinaryOp(Box::new(BinaryOp {28 lhs: $a,29 op: $op,30 rhs: $b,31 }))32 };33}34macro_rules! expr_un {35 ($op:ident $a:ident) => {36 Expr::UnaryOp($op, Box::new($a))37 };38}3940parser! {41 grammar jsonnet_parser() for str {42 use peg::ParseLiteral;4344 rule eof() = quiet!{![_]} / expected!("<eof>")45 rule eol() = "\n" / eof()4647 48 rule comment()49 = "//" (!eol()[_])* eol()50 / "/*" (!("*/")[_])* "*/"51 / "#" (!eol()[_])* eol()5253 rule single_whitespace() = quiet!{([' ' | '\r' | '\n' | '\t'] / comment())} / expected!("<whitespace>")54 rule _() = quiet!{([' ' | '\r' | '\n' | '\t']+) / comment()}* / expected!("<whitespace>")5556 57 rule comma() = quiet!{_ "," _} / expected!("<comma>")58 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}59 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}60 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']61 62 rule uint_str() -> &'input str = a:$(digit()+ ("_" digit()+)*) { a }63 64 rule number() -> f64 = quiet!{a:$(uint_str() ("." uint_str())? (['e'|'E'] (s:['+'|'-'])? uint_str())?) {? a.replace("_","").parse().map_err(|_| "<number>") }} / expected!("<number>")6566 67 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "importbin" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()68 rule id() -> IStr = v:$(quiet!{ !reserved() alpha() (alpha() / digit())*} / expected!("<identifier>")) { v.into() }6970 rule keyword(id: &'static str) -> ()71 = ##parse_string_literal(id) end_of_ident()7273 pub rule param(s: &ParserSettings) -> expr::ExprParam = destruct:destruct(s) expr:(_ "=" _ expr:expr(s){expr})? { expr::ExprParam { destruct, default: expr.map(Rc::new) } }74 pub rule params(s: &ParserSettings) -> expr::ExprParams75 = params:param(s) ** comma() comma()? { expr::ExprParams::new(params) }76 / { expr::ExprParams::new(Vec::new()) }7778 pub rule arg(s: &ParserSettings) -> (Option<IStr>, Rc<Spanned<Expr>>)79 = name:(quiet! { (s:id() _ "=" !['='] _ {s})? } / expected!("<argument name>")) expr:expr(s) {(name, Rc::new(expr))}8081 pub rule args(s: &ParserSettings) -> expr::ArgsDesc82 = args:arg(s)**comma() comma()? {?83 let unnamed_count = args.iter().take_while(|(n, _)| n.is_none()).count();84 let mut unnamed = Vec::with_capacity(unnamed_count);85 let mut named = Vec::with_capacity(args.len() - unnamed_count);86 let mut named_started = false;87 for (name, value) in args {88 if let Some(name) = name {89 named_started = true;90 named.push((name, value));91 } else {92 if named_started {93 return Err("<named argument>")94 }95 unnamed.push(value);96 }97 }98 Ok(expr::ArgsDesc::new(unnamed, named))99 }100101 pub rule destruct_rest() -> expr::DestructRest102 = "..." into:(_ into:id() {into})? {if let Some(into) = into {103 expr::DestructRest::Keep(into)104 } else {expr::DestructRest::Drop}}105 pub rule destruct_array(s: &ParserSettings) -> expr::Destruct106 = "[" _ start:destruct(s)**comma() rest:(107 comma() _ rest:destruct_rest()? end:(108 comma() end:destruct(s)**comma() (_ comma())? {end}109 / comma()? {Vec::new()}110 ) {(rest, end)}111 / comma()? {(None, Vec::new())}112 ) _ "]" {?113 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Array {114 start,115 rest: rest.0,116 end: rest.1,117 });118 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")119 }120 pub rule destruct_object(s: &ParserSettings) -> expr::Destruct121 = "{" _122 fields:(name:id() into:(_ ":" _ into:destruct(s) {into})? default:(_ "=" _ v:expr(s) {v})? {(name, into, default)})**comma()123 rest:(124 comma() rest:destruct_rest()? {rest}125 / comma()? {None}126 )127 _ "}" {?128 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Object {129 fields,130 rest,131 });132 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")133 }134 pub rule destruct(s: &ParserSettings) -> expr::Destruct135 = v:id() {expr::Destruct::Full(v)}136 / "?" {?137 #[cfg(feature = "exp-destruct")] return Ok(expr::Destruct::Skip);138 #[cfg(not(feature = "exp-destruct"))] Err("!!!experimental destructuring was not enabled")139 }140 / arr:destruct_array(s) {arr}141 / obj:destruct_object(s) {obj}142143 pub rule bind(s: &ParserSettings) -> expr::BindSpec144 = into:destruct(s) _ "=" _ value:expr(s) {expr::BindSpec::Field{into, value: Rc::new(value)}}145 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ value:expr(s) {expr::BindSpec::Function{name, params, value: Rc::new(value)}}146147 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt148 = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }149150 pub rule whole_line() -> &'input str151 = str:$((!['\n'][_])* "\n") {str}152 pub rule string_block() -> String153 = "|||" chomped:"-"? (!['\n']single_whitespace())* "\n"154 empty_lines:$(['\n']*)155 prefix:[' ' | '\t']+ first_line:whole_line()156 lines:("\n" {"\n"} / [' ' | '\t']*<{prefix.len()}> s:whole_line() {s})*157 [' ' | '\t']*<, {prefix.len() - 1}> "|||"158 {159 let mut l = empty_lines.to_owned();160 l.push_str(first_line);161 l.extend(lines);162 if chomped.is_some() {163 debug_assert!(l.ends_with('\n'));164 l.truncate(l.len() - 1);165 }166 l167 }168169 rule hex_char()170 = quiet! { ['0'..='9' | 'a'..='f' | 'A'..='F'] } / expected!("<hex char>")171172 rule string_char(c: rule<()>)173 = (!['\\']!c()[_])+174 / "\\\\"175 / "\\u" hex_char() hex_char() hex_char() hex_char()176 / "\\x" hex_char() hex_char()177 / ['\\'] (quiet! { ['b' | 'f' | 'n' | 'r' | 't' | '"' | '\''] } / expected!("<escape character>"))178 pub rule string() -> String179 = ['"'] str:$(string_char(<"\"">)*) ['"'] {? unescape::unescape(str).ok_or("<escaped string>")}180 / ['\''] str:$(string_char(<"\'">)*) ['\''] {? unescape::unescape(str).ok_or("<escaped string>")}181 / quiet!{ "@'" str:$(("''" / (!['\''][_]))*) "'" {str.replace("''", "'")}182 / "@\"" str:$(("\"\"" / (!['"'][_]))*) "\"" {str.replace("\"\"", "\"")}183 / string_block() } / expected!("<string>")184185 pub rule field_name(s: &ParserSettings) -> expr::FieldName186 = name:id() {expr::FieldName::Fixed(name)}187 / name:string() {expr::FieldName::Fixed(name.into())}188 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}189 pub rule visibility() -> expr::Visibility190 = ":::" {expr::Visibility::Unhide}191 / "::" {expr::Visibility::Hidden}192 / ":" {expr::Visibility::Normal}193 pub rule field(s: &ParserSettings) -> expr::FieldMember194 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{195 name,196 plus: plus.is_some(),197 params: None,198 visibility,199 value: Rc::new(value),200 }}201 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{202 name,203 plus: false,204 params: Some(params),205 visibility,206 value: Rc::new(value),207 }}208 pub rule obj_local(s: &ParserSettings) -> BindSpec209 = keyword("local") _ bind:bind(s) {bind}210 pub rule member(s: &ParserSettings) -> expr::Member211 = bind:obj_local(s) {expr::Member::BindStmt(bind)}212 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}213 / field:field(s) {expr::Member::Field(field)}214 pub rule objinside(s: &ParserSettings) -> expr::ObjBody215 = pre_locals:(b: obj_local(s) comma() {b})* &"[" field:field(s) post_locals:(comma() b:obj_local(s) {b})* _ ("," _)? forspec:forspec(s) others:(_ rest:compspec(s) {rest})? {216 let mut compspecs = vec![CompSpec::ForSpec(forspec)];217 compspecs.extend(others.unwrap_or_default());218 let mut locals = pre_locals;219 locals.extend(post_locals);220 expr::ObjBody::ObjComp(expr::ObjComp{221 locals: Rc::new(locals),222 field: Rc::new(field),223 compspecs,224 })225 }226 / members:(member(s) ** comma()) comma()? {227 let mut locals = Vec::new();228 let mut asserts = Vec::new();229 let mut fields = Vec::new();230 for member in members {231 match member {232 Member::Field(field_member) => fields.push(field_member),233 Member::BindStmt(bind_spec) => locals.push(bind_spec),234 Member::AssertStmt(assert_stmt) => asserts.push(assert_stmt),235 }236 }237 expr::ObjBody::MemberList(ObjMembers {238 locals: Rc::new(locals), asserts: Rc::new(asserts), fields239 })240 }241 pub rule ifspec(s: &ParserSettings) -> IfSpecData242 = keyword("if") _ expr:expr(s) {IfSpecData(expr)}243 pub rule forspec(s: &ParserSettings) -> ForSpecData244 = keyword("for") _ id:destruct(s) _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}245 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>246 = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} ) ** _ {s}247 pub rule local_expr(s: &ParserSettings) -> Expr248 = keyword("local") _ binds:bind(s) ** comma() (_ ",")? _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, Box::new(expr)) }249 pub rule string_expr(s: &ParserSettings) -> Expr250 = s:string() {Expr::Str(s.into())}251 pub rule obj_expr(s: &ParserSettings) -> Expr252 = "{" _ body:objinside(s) _ "}" {Expr::Obj(body)}253 pub rule array_expr(s: &ParserSettings) -> Expr254 = "[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(Rc::new(elems))}255 pub rule array_comp_expr(s: &ParserSettings) -> Expr256 = "[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {257 let mut specs = vec![CompSpec::ForSpec(forspec)];258 specs.extend(others.unwrap_or_default());259 Expr::ArrComp(Rc::new(expr), specs)260 }261 pub rule number_expr(s: &ParserSettings) -> Expr262 = n:number() {? if n.is_finite() {263 Ok(expr::Expr::Num(n))264 } else {265 Err("!!!numbers are finite")266 }}267 pub rule var_expr(s: &ParserSettings) -> Expr268 = n:id() { expr::Expr::Var(n) }269 pub rule id_loc(s: &ParserSettings) -> Spanned<Expr>270 = a:position!() n:id() b:position!() { Spanned::new(expr::Expr::Str(n), Span(s.source.clone(), a as u32,b as u32)) }271 pub rule if_then_else_expr(s: &ParserSettings) -> Expr272 = cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse(Box::new(IfElse{273 cond,274 cond_then,275 cond_else,276 }))}277278 pub rule literal(s: &ParserSettings) -> Expr279 = v:(280 keyword("null") {LiteralType::Null}281 / keyword("true") {LiteralType::True}282 / keyword("false") {LiteralType::False}283 / keyword("self") {LiteralType::This}284 / keyword("$") {LiteralType::Dollar}285 / keyword("super") {LiteralType::Super}286 ) {Expr::Literal(v)}287288 rule import_kind() -> ImportKind289 = keyword("importstr") { ImportKind::Str }290 / keyword("importbin") { ImportKind::Bin }291 / keyword("import") { ImportKind::Normal }292293 pub rule expr_basic(s: &ParserSettings) -> Expr294 = literal(s)295296 / string_expr(s) / number_expr(s)297 / array_expr(s)298 / obj_expr(s)299 / array_expr(s)300 / array_comp_expr(s)301302 / kind:import_kind() _ path:expr(s) {Expr::Import(kind, Box::new(path))}303304 / var_expr(s)305 / local_expr(s)306 / if_then_else_expr(s)307308 / keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, Rc::new(expr))}309 / assert:assertion(s) _ ";" _ rest:expr(s) { Expr::AssertExpr(Rc::new(AssertExpr{310 assert, rest311 })) }312313 / keyword("error") _ expr:expr(s) { Expr::ErrorStmt(Box::new(expr)) }314315 rule slice_part(s: &ParserSettings) -> Option<Spanned<Expr>>316 = _ e:(e:expr(s) _{e})? {e}317 pub rule slice_desc(s: &ParserSettings) -> SliceDesc318 = start:slice_part(s) ":" pair:(end:slice_part(s) step:(":" e:slice_part(s){e})? {(end, step.flatten())})? {319 let (end, step) = if let Some((end, step)) = pair {320 (end, step)321 }else{322 (None, None)323 };324325 SliceDesc { start, end, step }326 }327328 rule binop(x: rule<()>) -> ()329 = quiet!{ x() } / expected!("<binary op>")330 rule unaryop(x: rule<()>) -> ()331 = quiet!{ x() } / expected!("<unary op>")332333 rule ensure_null_coaelse()334 = "" {?335 #[cfg(not(feature = "exp-null-coaelse"))] return Err("!!!experimental null coaelscing was not enabled");336 #[cfg(feature = "exp-null-coaelse")] Ok(())337 }338 use BinaryOpType::*;339 use UnaryOpType::*;340 rule expr(s: &ParserSettings) -> Spanned<Expr>341 = precedence! {342 "(" _ e:expr(s) _ ")" {e}343 start:position!() v:@ end:position!() { Spanned::new(v, Span(s.source.clone(), start as u32, end as u32)) }344 --345 a:(@) _ binop(<"||">) _ b:@ {expr_bin!(a Or b)}346 a:(@) _ binop(<"??">) _ ensure_null_coaelse() b:@ {347 #[cfg(feature = "exp-null-coaelse")] return expr_bin!(a NullCoaelse b);348 unreachable!("ensure_null_coaelse will fail if feature is not enabled")349 }350 --351 a:(@) _ binop(<"&&">) _ b:@ {expr_bin!(a And b)}352 --353 a:(@) _ binop(<"|">) _ b:@ {expr_bin!(a BitOr b)}354 --355 a:@ _ binop(<"^">) _ b:(@) {expr_bin!(a BitXor b)}356 --357 a:(@) _ binop(<"&">) _ b:@ {expr_bin!(a BitAnd b)}358 --359 a:(@) _ binop(<"==">) _ b:@ {expr_bin!(a Eq b)}360 a:(@) _ binop(<"!=">) _ b:@ {expr_bin!(a Neq b)}361 --362 a:(@) _ binop(<"<">) _ b:@ {expr_bin!(a Lt b)}363 a:(@) _ binop(<">">) _ b:@ {expr_bin!(a Gt b)}364 a:(@) _ binop(<"<=">) _ b:@ {expr_bin!(a Lte b)}365 a:(@) _ binop(<">=">) _ b:@ {expr_bin!(a Gte b)}366 a:(@) _ binop(<keyword("in")>) _ b:@ {expr_bin!(a In b)}367 --368 a:(@) _ binop(<"<<">) _ b:@ {expr_bin!(a Lhs b)}369 a:(@) _ binop(<">>">) _ b:@ {expr_bin!(a Rhs b)}370 --371 a:(@) _ binop(<"+">) _ b:@ {expr_bin!(a Add b)}372 a:(@) _ binop(<"-">) _ b:@ {expr_bin!(a Sub b)}373 --374 a:(@) _ binop(<"*">) _ b:@ {expr_bin!(a Mul b)}375 a:(@) _ binop(<"/">) _ b:@ {expr_bin!(a Div b)}376 a:(@) _ binop(<"%">) _ b:@ {expr_bin!(a Mod b)}377 --378 unaryop(<"+">) _ b:@ {expr_un!(Plus b)}379 unaryop(<"-">) _ b:@ {expr_un!(Minus b)}380 unaryop(<"!">) _ b:@ {expr_un!(Not b)}381 unaryop(<"~">) _ b:@ {expr_un!(BitNot b)}382 --383 value:(@) _ "[" _ slice:slice_desc(s) _ "]" {Expr::Slice(Box::new(Slice{value, slice}))}384 indexable:(@) _ parts:index_part(s)+ {Expr::Index{indexable: Box::new(indexable), parts}}385 a:(@) _ "(" _ args:args(s) _ ")" ts:(_ keyword("tailstrict"))? {Expr::Apply(Box::new(a), args, ts.is_some())}386 a:(@) _ "{" _ body:objinside(s) _ "}" {Expr::ObjExtend(Rc::new(a), body)}387 --388 e:expr_basic(s) {e}389 }390 pub rule index_part(s: &ParserSettings) -> IndexPart391 = n:("?" _ ensure_null_coaelse())? "." _ value:id_loc(s) {IndexPart {392 value,393 #[cfg(feature = "exp-null-coaelse")]394 null_coaelse: n.is_some(),395 }}396 / n:("?" _ "." _ ensure_null_coaelse())? "[" _ value:expr(s) _ "]" {IndexPart {397 value,398 #[cfg(feature = "exp-null-coaelse")]399 null_coaelse: n.is_some(),400 }}401402 pub rule jsonnet(s: &ParserSettings) -> Spanned<Expr> = _ e:expr(s) _ {e}403 }404}405406pub type ParseError = peg::error::ParseError<peg::str::LineCol>;407pub fn parse(str: &str, settings: &ParserSettings) -> Result<Spanned<Expr>, ParseError> {408 jsonnet_parser::jsonnet(str, settings)409}410411pub fn string_to_expr(str: IStr, settings: &ParserSettings) -> Spanned<Expr> {412 let len = str.len();413 Spanned::new(Expr::Str(str), Span(settings.source.clone(), 0, len as u32))414}415416#[cfg(test)]417pub mod tests {418 use insta::assert_snapshot;419 use jrsonnet_interner::IStr;420421 use super::parse;422 use crate::{source::Source, ParserSettings};423424 fn parsep(s: &str) -> String {425 let v = parse(426 s,427 &ParserSettings {428 source: Source::new_virtual("<test>".into(), IStr::empty()),429 },430 )431 .unwrap();432 format!("{v:#?}")433 }434435 macro_rules! parse {436 ($s:expr) => {437 assert_snapshot!(parsep($s));438 };439 }440441 #[test]442 fn multiline_string() {443 parse!("|||\n Hello world!\n a\n|||");444 parse!("|||\n Hello world!\n a\n|||");445 parse!("|||\n\t\tHello world!\n\t\t\ta\n|||");446 parse!("|||\n Hello world!\n a\n |||");447 }448449 #[test]450 fn slice() {451 parse!("a[1:]");452 parse!("a[1::]");453 parse!("a[:1:]");454 parse!("a[::1]");455 parse!("str[:len - 1]");456 }457458 #[test]459 fn string_escaping() {460 parse!(r#""Hello, \"world\"!""#);461 parse!(r#"'Hello \'world\'!'"#);462 parse!(r#"'\\\\'"#);463 }464465 #[test]466 fn string_unescaping() {467 parse!(r#""Hello\nWorld""#);468 }469470 #[test]471 fn string_verbantim() {472 parse!(r#"@"Hello\n""World""""#);473 }474475 #[test]476 fn imports() {477 parse!("import \"hello\"");478 parse!("importstr \"garnish.txt\"");479 parse!("importbin \"garnish.bin\"");480 }481482 #[test]483 fn empty_object() {484 parse!("{}");485 }486487 #[test]488 fn basic_math() {489 parse!("2+2*2");490 parse!("2 + 2 * 2 ");491 parse!("2+(2+2*2)");492 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4");493 }494495 #[test]496 fn suffix() {497 parse!("std.test");498 parse!("std(2)");499 parse!("std.test(2)");500 parse!("a[b]");501 }502503 #[test]504 fn array_comp() {505 parse!("[std.deepJoin(x) for x in arr]");506 }507508 #[test]509 fn reserved() {510 parse!("null");511 parse!("nulla");512 }513514 #[test]515 fn multiple_args_buf() {516 parse!("a(b, null_fields)");517 }518519 #[test]520 fn infix_precedence() {521 parse!("!a && !b");522 parse!("!a / !b");523 }524525 #[test]526 fn double_negation() {527 parse!("!!a");528 }529530 #[test]531 fn array_test_error() {532 parse!("[a for a in b if c for e in f]");533 }534535 #[test]536 fn missing_newline_between_comment_and_eof() {537 parse!(538 "{a:1}539540 //+213"541 );542 }543544 #[test]545 fn default_param_before_nondefault() {546 parse!("local x(foo = 'foo', bar) = null; null");547 }548549 #[test]550 fn add_location_info_to_all_sub_expressions() {551 parse!("{} { local x = 1, x: x } + {}");552 }553}