difftreelog
feat(parser) save ast expression location
in: master
2 files changed
crates/jsonnet-parser/src/expr.rsdiffbeforeafterboth--- a/crates/jsonnet-parser/src/expr.rs
+++ b/crates/jsonnet-parser/src/expr.rs
@@ -1,11 +1,14 @@
-use std::fmt::Display;
+use std::{
+ fmt::{Debug, Display},
+ rc::Rc,
+};
#[derive(Debug, Clone, PartialEq)]
pub enum FieldName {
/// {fixed: 2}
Fixed(String),
/// {["dyn"+"amic"]: 3}
- Dyn(Box<Expr>),
+ Dyn(LocExpr),
}
#[derive(Debug, Clone, PartialEq)]
@@ -19,7 +22,7 @@
}
#[derive(Debug, Clone, PartialEq)]
-pub struct AssertStmt(pub Box<Expr>, pub Option<Box<Expr>>);
+pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);
#[derive(Debug, Clone, PartialEq)]
pub struct FieldMember {
@@ -27,7 +30,7 @@
pub plus: bool,
pub params: Option<ParamsDesc>,
pub visibility: Visibility,
- pub value: Expr,
+ pub value: LocExpr,
}
#[derive(Debug, Clone, PartialEq)]
@@ -77,7 +80,7 @@
/// name, default value
#[derive(Debug, Clone, PartialEq)]
-pub struct Param(pub String, pub Option<Box<Expr>>);
+pub struct Param(pub String, pub Option<LocExpr>);
/// Defined function parameters
#[derive(Debug, Clone, PartialEq)]
pub struct ParamsDesc(pub Vec<Param>);
@@ -88,7 +91,7 @@
}
#[derive(Debug, Clone, PartialEq)]
-pub struct Arg(pub Option<String>, pub Box<Expr>);
+pub struct Arg(pub Option<String>, pub LocExpr);
#[derive(Debug, Clone, PartialEq)]
pub struct ArgsDesc(pub Vec<Arg>);
@@ -96,13 +99,13 @@
pub struct BindSpec {
pub name: String,
pub params: Option<ParamsDesc>,
- pub value: Box<Expr>,
+ pub value: LocExpr,
}
#[derive(Debug, Clone, PartialEq)]
-pub struct IfSpecData(pub Box<Expr>);
+pub struct IfSpecData(pub LocExpr);
#[derive(Debug, Clone, PartialEq)]
-pub struct ForSpecData(pub String, pub Box<Expr>);
+pub struct ForSpecData(pub String, pub LocExpr);
#[derive(Debug, Clone, PartialEq)]
pub enum CompSpec {
@@ -115,8 +118,8 @@
MemberList(Vec<Member>),
ObjComp {
pre_locals: Vec<BindSpec>,
- key: Box<Expr>,
- value: Box<Expr>,
+ key: LocExpr,
+ value: LocExpr,
post_locals: Vec<BindSpec>,
first: ForSpecData,
rest: Vec<CompSpec>,
@@ -147,9 +150,9 @@
#[derive(Debug, Clone, PartialEq)]
pub struct SliceDesc {
- pub start: Option<Box<Expr>>,
- pub end: Option<Box<Expr>>,
- pub step: Option<Box<Expr>>,
+ pub start: Option<LocExpr>,
+ pub end: Option<LocExpr>,
+ pub step: Option<LocExpr>,
}
/// Syntax base
@@ -165,7 +168,7 @@
Var(String),
/// Array of expressions: [1, 2, "Hello"]
- Arr(Vec<Expr>),
+ Arr(Vec<LocExpr>),
/// Array comprehension:
/// ```jsonnet
/// ingredients: [
@@ -177,15 +180,15 @@
/// ]
/// ],
/// ```
- ArrComp(Box<Expr>, ForSpecData, Vec<CompSpec>),
+ ArrComp(LocExpr, ForSpecData, Vec<CompSpec>),
/// Object: {a: 2}
Obj(ObjBody),
/// Object extension: var1 {b: 2}
- ObjExtend(Box<Expr>, ObjBody),
+ ObjExtend(LocExpr, ObjBody),
/// (obj)
- Parened(Box<Expr>),
+ Parened(LocExpr),
/// Params in function definition
/// hello, world, test = 2
@@ -195,13 +198,13 @@
Args(ArgsDesc),
/// -2
- UnaryOp(UnaryOpType, Box<Expr>),
+ UnaryOp(UnaryOpType, LocExpr),
/// 2 - 2
- BinaryOp(Box<Expr>, BinaryOpType, Box<Expr>),
+ BinaryOp(LocExpr, BinaryOpType, LocExpr),
/// assert 2 == 2 : "Math is broken"
- AssertExpr(AssertStmt, Box<Expr>),
+ AssertExpr(AssertStmt, LocExpr),
/// local a = 2; { b: a }
- LocalExpr(Vec<BindSpec>, Box<Expr>),
+ LocalExpr(Vec<BindSpec>, LocExpr),
/// a = 3
Bind(BindSpec),
@@ -210,25 +213,66 @@
/// importStr "file.txt"
ImportStr(String),
/// error "I'm broken"
- Error(Box<Expr>),
+ Error(LocExpr),
/// a(b, c)
- Apply(Box<Expr>, ArgsDesc),
+ Apply(LocExpr, ArgsDesc),
///
- Select(Box<Expr>, String),
+ Select(LocExpr, String),
/// a[b]
- Index(Box<Expr>, Box<Expr>),
+ Index(LocExpr, LocExpr),
/// a[1::2]
- Slice(Box<Expr>, SliceDesc),
+ Slice(LocExpr, SliceDesc),
/// function(x) x
- Function(ParamsDesc, Box<Expr>),
+ Function(ParamsDesc, LocExpr),
/// if true == false then 1 else 2
IfElse {
cond: IfSpecData,
- cond_then: Box<Expr>,
- cond_else: Option<Box<Expr>>,
+ cond_then: LocExpr,
+ cond_else: Option<LocExpr>,
},
/// if 2 = 3
IfSpec(IfSpecData),
/// for elem in array
ForSpec(ForSpecData),
}
+
+/// file, begin offset, end offset
+#[derive(Clone, PartialEq)]
+pub struct ExprLocation(pub String, pub usize, pub usize);
+impl Debug for ExprLocation {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}:{:?}", self.0, self.1)
+ }
+}
+
+/// Holds AST expression and its location in source file+
+#[derive(Clone, PartialEq)]
+pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);
+impl Debug for LocExpr {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{:?} from {:?}", self.0, self.1)
+ }
+}
+
+/// Creates LocExpr from Expr and ExprLocation components
+#[macro_export]
+macro_rules! loc_expr {
+ ($expr:expr, $need_loc:expr, ($name:expr, $start:expr, $end:expr)) => {
+ LocExpr(
+ Rc::new($expr),
+ if $need_loc {
+ Some(Rc::new(ExprLocation($name.to_owned(), $start, $end)))
+ } else {
+ None
+ },
+ )
+ };
+}
+
+/// Creates LocExpr without location info
+#[macro_export]
+macro_rules! loc_expr_todo {
+ ($expr:expr) => {
+ LocExpr(Rc::new($expr), None)
+ };
+}
crates/jsonnet-parser/src/lib.rsdiffbeforeafterboth1#![feature(box_syntax)]23use peg::parser;45mod expr;6pub use expr::*;78enum Suffix {9 String(String),10 Slice(SliceDesc),11 Expression(Expr),12 Apply(expr::ArgsDesc),13 Extend(expr::ObjBody),14}1516parser! {17 grammar jsonnet_parser() for str {18 use peg::ParseLiteral;1920 /// Standard C-like comments21 rule comment() = "//" (!['\n'][_])* "\n" / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"22 rule _() = ([' ' | '\n' | '\t'] / comment())*2324 /// For comma-delimited elements25 rule comma() = quiet!{_ "," _} / expected!("<comma>")26 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}27 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}28 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']29 /// Sequence of digits30 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }31 /// Number in scientific notation format32 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")3334 /// Reserved word followed by any non-alphanumberic35 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()36 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")37 rule keyword(id: &'static str) = ##parse_string_literal(id) end_of_ident()3839 pub rule param() -> expr::Param = name:id() expr:(_ "=" _ expr:boxed_expr(){expr})? { expr::Param(name, expr) }40 pub rule params() -> expr::ParamsDesc41 = params:(param() ** comma()) {42 let mut defaults_started = false;43 for param in ¶ms {44 defaults_started = defaults_started || param.1.is_some();45 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");46 }47 expr::ParamsDesc(params)48 }49 / { expr::ParamsDesc(Vec::new()) }5051 pub rule arg() -> expr::Arg52 = name:id() _ "=" _ expr:boxed_expr() {expr::Arg(Some(name), expr)}53 / expr:boxed_expr() {expr::Arg(None, expr)}54 pub rule args() -> expr::ArgsDesc55 = args:arg() ** comma() comma()? {56 let mut named_started = false;57 for arg in &args {58 named_started = named_started || arg.0.is_some();59 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");60 }61 expr::ArgsDesc(args)62 }63 / { expr::ArgsDesc(Vec::new()) }6465 pub rule bind() -> expr::BindSpec66 = name:id() _ "=" _ expr:boxed_expr() {expr::BindSpec{name, params: None, value: expr}}67 / name:id() _ "(" _ params:params() _ ")" _ "=" _ expr:boxed_expr() {expr::BindSpec{name, params: Some(params), value: expr}}68 pub rule assertion() -> expr::AssertStmt = keyword("assert") _ cond:boxed_expr() msg:(_ ":" _ e:boxed_expr() {e})? { expr::AssertStmt(cond, msg) }69 pub rule string() -> String70 = "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}71 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}72 pub rule field_name() -> expr::FieldName73 = name:id() {expr::FieldName::Fixed(name)}74 / name:string() {expr::FieldName::Fixed(name)}75 / "[" _ expr:boxed_expr() _ "]" {expr::FieldName::Dyn(expr)}76 pub rule visibility() -> expr::Visibility77 = ":::" {expr::Visibility::Unhide}78 / "::" {expr::Visibility::Hidden}79 / ":" {expr::Visibility::Normal}80 pub rule field() -> expr::FieldMember81 = name:field_name() _ plus:"+"? _ visibility:visibility() _ value:expr() {expr::FieldMember{82 name,83 plus: plus.is_some(),84 params: None,85 visibility,86 value,87 }}88 / name:field_name() _ "(" _ params:params() _ ")" _ visibility:visibility() _ value:expr() {expr::FieldMember{89 name,90 plus: false,91 params: Some(params),92 visibility,93 value,94 }}95 pub rule obj_local() -> BindSpec96 = keyword("local") _ bind:bind() {bind}97 pub rule member() -> expr::Member98 = bind:obj_local() {expr::Member::BindStmt(bind)}99 / assertion:assertion() {expr::Member::AssertStmt(assertion)}100 / field:field() {expr::Member::Field(field)}101 pub rule objinside() -> expr::ObjBody102 = pre_locals:(b: obj_local() comma() {b})* "[" _ key:boxed_expr() _ "]" _ ":" _ value:boxed_expr() post_locals:(comma() b:obj_local() {b})* _ first:forspec() rest:(_ rest:compspec() {rest})? {103 expr::ObjBody::ObjComp {104 pre_locals,105 key,106 value,107 post_locals,108 first,109 rest: rest.unwrap_or_default(),110 }111 }112 / members:(member() ** comma()) comma()? {expr::ObjBody::MemberList(members)}113 pub rule ifspec() -> IfSpecData = keyword("if") _ expr:boxed_expr() {IfSpecData(expr)}114 pub rule forspec() -> ForSpecData = keyword("for") _ id:id() _ keyword("in") _ cond:boxed_expr() {ForSpecData(id, cond)}115 pub rule compspec() -> Vec<expr::CompSpec> = s:(i:ifspec() { expr::CompSpec::IfSpec(i) } / f:forspec() {expr::CompSpec::ForSpec(f)} )+ {s}116 pub rule bind_expr() -> Expr = bind:bind() {Expr::Bind(bind)}117 pub rule local_expr() -> Expr = keyword("local") _ binds:bind() ** comma() _ ";" _ expr:boxed_expr() { Expr::LocalExpr(binds, expr) }118 pub rule string_expr() -> Expr = s:string() {Expr::Str(s)}119 pub rule parened_expr() -> Expr = "(" e:boxed_expr() ")" {Expr::Parened(e)}120 pub rule obj_expr() -> Expr = "{" _ body:objinside() _ "}" {Expr::Obj(body)}121 pub rule array_expr() -> Expr = "[" _ elems:(expr() ** comma()) _ comma()? "]" {Expr::Arr(elems)}122 pub rule array_comp_expr() -> Expr = "[" _ expr:boxed_expr() _ comma()? _ forspec:forspec() _ others:(others: compspec() _ {others})? "]" {Expr::ArrComp(expr, forspec, others.unwrap_or_default())}123 pub rule index_expr() -> Expr124 = val:boxed_expr() "." idx:id() {Expr::Index(val, Box::new(Expr::Str(idx)))}125 / val:boxed_expr() "[" key:boxed_expr() "]" {Expr::Index(val, key)}126 pub rule number_expr() -> Expr = n:number() { expr::Expr::Num(n) }127 pub rule var_expr() -> Expr = n:id() { expr::Expr::Var(n) }128 pub rule if_then_else_expr() -> Expr = cond:ifspec() _ keyword("then") _ cond_then:boxed_expr() cond_else:(_ keyword("else") _ e:boxed_expr() {e})? {Expr::IfElse{129 cond,130 cond_then,131 cond_else,132 }}133134 pub rule literal() -> Expr135 = v:(136 keyword("null") {LiteralType::Null}137 / keyword("true") {LiteralType::True}138 / keyword("false") {LiteralType::False}139 / keyword("self") {LiteralType::This}140 / keyword("$") {LiteralType::Dollar}141 / keyword("super") {LiteralType::Super}142 ) {Expr::Literal(v)}143144 pub rule expr_basic() -> Expr145 = literal()146147 / string_expr() / number_expr()148 / array_expr()149 / obj_expr()150 / array_expr()151 / array_comp_expr()152153 / var_expr()154 / local_expr()155 / if_then_else_expr()156157 / keyword("function") _ "(" _ params:params() _ ")" _ expr:boxed_expr() {Expr::Function(params, expr)}158 / assertion:assertion() _ ";" _ expr:boxed_expr() { Expr::AssertExpr(assertion, expr) }159160 / keyword("error") _ expr:boxed_expr() { Expr::Error(expr) }161162 rule expr_basic_with_suffix() -> Expr163 = a:expr_basic() suffixes:(_ suffix:expr_suffix() {suffix})* {164 let mut cur = a;165 for suffix in suffixes {166 cur = match suffix {167 Suffix::String(index) => Expr::Index(Box::new(cur), Box::new(Expr::Str(index))),168 Suffix::Slice(desc) => Expr::Slice(Box::new(cur), desc),169 Suffix::Expression(index) => Expr::Index(Box::new(cur), Box::new(index)),170 Suffix::Apply(args) => Expr::Apply(Box::new(cur), args),171 Suffix::Extend(body) => Expr::ObjExtend(box cur, body),172 }173 }174 cur175 }176177 pub rule slice_desc() -> SliceDesc178 = start:boxed_expr()? _ ":" _ pair:(end:boxed_expr()? _ step:(":" _ e:boxed_expr() {e})? {(end, step)})? {179 if let Some((end, step)) = pair {180 SliceDesc { start, end, step }181 }else{182 SliceDesc { start, end: None, step: None }183 }184 }185186 rule expr_suffix() -> Suffix187 = "." _ s:id() { Suffix::String(s) }188 / "[" _ s:slice_desc() _ "]" { Suffix::Slice(s) }189 / "[" _ s:expr() _ "]" { Suffix::Expression(s) }190 / "(" _ args:args() _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }191 / "{" _ body:objinside() _ "}" { Suffix::Extend(body) }192193 rule expr() -> Expr194 = a:precedence! {195 a:(@) _ "||" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Or, Box::new(b))}196 --197 a:(@) _ "&&" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::And, Box::new(b))}198 --199 a:(@) _ "|" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::BitOr, Box::new(b))}200 --201 a:@ _ "^" _ b:(@) {Expr::BinaryOp(Box::new(a), BinaryOpType::BitXor, Box::new(b))}202 --203 a:(@) _ "&" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::BitAnd, Box::new(b))}204 --205 a:(@) _ "==" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Eq, Box::new(b))}206 a:(@) _ "!=" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Ne, Box::new(b))}207 --208 a:(@) _ "<" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Lt, Box::new(b))}209 a:(@) _ ">" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Gt, Box::new(b))}210 a:(@) _ "<=" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Lte, Box::new(b))}211 a:(@) _ ">=" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Gte, Box::new(b))}212 --213 a:(@) _ "<<" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Lhs, Box::new(b))}214 a:(@) _ ">>" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Rhs, Box::new(b))}215 --216 a:(@) _ "+" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Add, Box::new(b))}217 a:(@) _ "-" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Sub, Box::new(b))}218 --219 a:(@) _ "*" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Mul, Box::new(b))}220 a:(@) _ "/" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Div, Box::new(b))}221 a:(@) _ "%" _ b:@ {Expr::BinaryOp(Box::new(a), BinaryOpType::Mod, Box::new(b))}222 --223 e:expr_basic_with_suffix() {e}224 "-" _ expr:expr_basic_with_suffix() { Expr::UnaryOp(UnaryOpType::Minus, box expr) }225 "!" _ expr:expr_basic_with_suffix() { Expr::UnaryOp(UnaryOpType::Not, box expr) }226 "(" _ e:boxed_expr() _ ")" {Expr::Parened(e)}227 }228 / e:expr_basic_with_suffix() {e}229230 pub rule boxed_expr() -> Box<Expr> = e:expr() {Box::new(e)}231 pub rule jsonnet() -> Expr = _ e:expr() _ {e}232 }233}234235// TODO: impl FromStr from Expr236pub fn parse(str: &str) -> Result<Expr, peg::error::ParseError<peg::str::LineCol>> {237 jsonnet_parser::jsonnet(str)238}239240#[cfg(test)]241pub mod tests {242 use super::{expr::*, parse};243244 mod expressions {245 use super::*;246247 pub fn basic_math() -> Expr {248 Expr::BinaryOp(249 Box::new(Expr::Num(2.0)),250 BinaryOpType::Add,251 Box::new(Expr::BinaryOp(252 Box::new(Expr::Num(2.0)),253 BinaryOpType::Mul,254 Box::new(Expr::Num(2.0)),255 )),256 )257 }258 }259260 #[test]261 fn empty_object() {262 assert_eq!(parse("{}").unwrap(), Expr::Obj(ObjBody::MemberList(vec![])));263 }264265 #[test]266 fn basic_math() {267 assert_eq!(268 parse("2+2*2").unwrap(),269 Expr::BinaryOp(270 Box::new(Expr::Num(2.0)),271 BinaryOpType::Add,272 Box::new(Expr::BinaryOp(273 Box::new(Expr::Num(2.0)),274 BinaryOpType::Mul,275 Box::new(Expr::Num(2.0))276 ))277 )278 );279 }280281 #[test]282 fn basic_math_with_indents() {283 assert_eq!(parse("2 + 2 * 2 ").unwrap(), expressions::basic_math());284 }285286 #[test]287 fn basic_math_parened() {288 assert_eq!(289 parse("2+(2+2*2)").unwrap(),290 Expr::BinaryOp(291 Box::new(Expr::Num(2.0)),292 BinaryOpType::Add,293 Box::new(Expr::Parened(Box::new(expressions::basic_math()))),294 )295 );296 }297298 /// Comments should not affect parsing299 #[test]300 fn comments() {301 assert_eq!(302 parse("2//comment\n+//comment\n3/*test*/*/*test*/4").unwrap(),303 Expr::BinaryOp(304 box Expr::Num(2.0),305 BinaryOpType::Add,306 box Expr::BinaryOp(box Expr::Num(3.0), BinaryOpType::Mul, box Expr::Num(4.0))307 )308 );309 }310311 /// Comments should be able to be escaped312 #[test]313 fn comment_escaping() {314 assert_eq!(315 parse("2/*\\*/+*/ - 22").unwrap(),316 Expr::BinaryOp(box Expr::Num(2.0), BinaryOpType::Sub, box Expr::Num(22.0))317 );318 }319320 #[test]321 fn suffix_comparsion() {322 use Expr::*;323 assert_eq!(324 parse("std.type(a) == \"string\"").unwrap(),325 BinaryOp(326 box Apply(327 box Index(box Var("std".to_owned()), box Str("type".to_owned())),328 ArgsDesc(vec![Arg(None, box Var("a".to_owned()))])329 ),330 BinaryOpType::Eq,331 box Str("string".to_owned())332 )333 );334 }335336 #[test]337 fn array_comp() {338 use Expr::*;339 assert_eq!(340 parse("[std.deepJoin(x) for x in arr]").unwrap(),341 ArrComp(342 box Apply(343 box Index(box Var("std".to_owned()), box Str("deepJoin".to_owned())),344 ArgsDesc(vec![Arg(None, box Var("x".to_owned()))])345 ),346 ForSpecData("x".to_owned(), box Var("arr".to_owned())),347 vec![]348 ),349 )350 }351352 #[test]353 fn array_comp_with_ifs() {354 use Expr::*;355 assert_eq!(356 parse("[k for k in std.objectFields(patch) if patch[k] == null]").unwrap(),357 ArrComp(358 box Var("k".to_owned()),359 ForSpecData(360 "k".to_owned(),361 box Apply(362 box Index(363 box Var("std".to_owned()),364 box Str("objectFields".to_owned())365 ),366 ArgsDesc(vec![Arg(None, box Var("patch".to_owned()))])367 )368 ),369 vec![CompSpec::IfSpec(IfSpecData(box BinaryOp(370 box Index(box Var("patch".to_owned()), box Var("k".to_owned())),371 BinaryOpType::Eq,372 box Literal(LiteralType::Null)373 )))]374 ),375 );376 }377378 #[test]379 fn reserved() {380 use Expr::*;381 assert_eq!(parse("null").unwrap(), Literal(LiteralType::Null));382 assert_eq!(parse("nulla").unwrap(), Var("nulla".to_owned()));383 }384385 #[test]386 fn multiple_args_buf() {387 parse("a(b, null_fields)").unwrap();388 }389390 #[test]391 fn infix_precedence() {392 use Expr::*;393 assert_eq!(394 parse("!a && !b").unwrap(),395 BinaryOp(396 box UnaryOp(UnaryOpType::Not, box Var("a".to_owned())),397 BinaryOpType::And,398 box UnaryOp(UnaryOpType::Not, box Var("b".to_owned()))399 )400 );401 }402403 #[test]404 fn can_parse_stdlib() {405 parse(jsonnet_stdlib::STDLIB_STR).unwrap();406 }407}1#![feature(box_syntax)]23use peg::parser;4use std::rc::Rc;5mod expr;6pub use expr::*;78enum Suffix {9 String(String),10 Slice(SliceDesc),11 Expression(LocExpr),12 Apply(expr::ArgsDesc),13 Extend(expr::ObjBody),14}15struct LocSuffix(Suffix, ExprLocation);1617pub struct ParserSettings {18 pub loc_data: bool,19 pub file_name: String,20}2122parser! {23 grammar jsonnet_parser() for str {24 use peg::ParseLiteral;2526 /// Standard C-like comments27 rule comment() = "//" (!['\n'][_])* "\n" / "/*" ((!("*/")[_][_])/("\\" "*/"))* "*/"28 rule _() = ([' ' | '\n' | '\t'] / comment())*2930 /// For comma-delimited elements31 rule comma() = quiet!{_ "," _} / expected!("<comma>")32 rule alpha() -> char = c:$(['_' | 'a'..='z' | 'A'..='Z']) {c.chars().next().unwrap()}33 rule digit() -> char = d:$(['0'..='9']) {d.chars().next().unwrap()}34 rule end_of_ident() = !['0'..='9' | '_' | 'a'..='z' | 'A'..='Z']35 /// Sequence of digits36 rule uint() -> u32 = a:$(digit()+) { a.parse().unwrap() }37 /// Number in scientific notation format38 rule number() -> f64 = quiet!{a:$(uint() ("." uint())? (['e'|'E'] (s:['+'|'-'])? uint())?) { a.parse().unwrap() }} / expected!("<number>")3940 /// Reserved word followed by any non-alphanumberic41 rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()42 rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")4344 rule keyword(id: &'static str) = ##parse_string_literal(id) end_of_ident()45 rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}4647 pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }48 pub rule params(s: &ParserSettings) -> expr::ParamsDesc49 = params:(param(s) ** comma()) {50 let mut defaults_started = false;51 for param in ¶ms {52 defaults_started = defaults_started || param.1.is_some();53 assert_eq!(defaults_started, param.1.is_some(), "defauld parameters should be used after all positionals");54 }55 expr::ParamsDesc(params)56 }57 / { expr::ParamsDesc(Vec::new()) }5859 pub rule arg(s: &ParserSettings) -> expr::Arg60 = name:id() _ "=" _ expr:expr(s) {expr::Arg(Some(name), expr)}61 / expr:expr(s) {expr::Arg(None, expr)}62 pub rule args(s: &ParserSettings) -> expr::ArgsDesc63 = args:arg(s) ** comma() comma()? {64 let mut named_started = false;65 for arg in &args {66 named_started = named_started || arg.0.is_some();67 assert_eq!(named_started, arg.0.is_some(), "named args should be used after all positionals");68 }69 expr::ArgsDesc(args)70 }71 / { expr::ArgsDesc(Vec::new()) }7273 pub rule bind(s: &ParserSettings) -> expr::BindSpec74 = name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}75 / name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}76 pub rule assertion(s: &ParserSettings) -> expr::AssertStmt = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }77 pub rule string() -> String78 = "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}79 / "'" str:$((!['\''][_])*) "'" {str.to_owned()}80 pub rule field_name(s: &ParserSettings) -> expr::FieldName81 = name:id() {expr::FieldName::Fixed(name)}82 / name:string() {expr::FieldName::Fixed(name)}83 / "[" _ expr:expr(s) _ "]" {expr::FieldName::Dyn(expr)}84 pub rule visibility() -> expr::Visibility85 = ":::" {expr::Visibility::Unhide}86 / "::" {expr::Visibility::Hidden}87 / ":" {expr::Visibility::Normal}88 pub rule field(s: &ParserSettings) -> expr::FieldMember89 = name:field_name(s) _ plus:"+"? _ visibility:visibility() _ value:expr(s) {expr::FieldMember{90 name,91 plus: plus.is_some(),92 params: None,93 visibility,94 value,95 }}96 / name:field_name(s) _ "(" _ params:params(s) _ ")" _ visibility:visibility() _ value:expr(s) {expr::FieldMember{97 name,98 plus: false,99 params: Some(params),100 visibility,101 value,102 }}103 pub rule obj_local(s: &ParserSettings) -> BindSpec104 = keyword("local") _ bind:bind(s) {bind}105 pub rule member(s: &ParserSettings) -> expr::Member106 = bind:obj_local(s) {expr::Member::BindStmt(bind)}107 / assertion:assertion(s) {expr::Member::AssertStmt(assertion)}108 / field:field(s) {expr::Member::Field(field)}109 pub rule objinside(s: &ParserSettings) -> expr::ObjBody110 = pre_locals:(b: obj_local(s) comma() {b})* "[" _ key:expr(s) _ "]" _ ":" _ value:expr(s) post_locals:(comma() b:obj_local(s) {b})* _ first:forspec(s) rest:(_ rest:compspec(s) {rest})? {111 expr::ObjBody::ObjComp {112 pre_locals,113 key,114 value,115 post_locals,116 first,117 rest: rest.unwrap_or_default(),118 }119 }120 / members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}121 pub rule ifspec(s: &ParserSettings) -> IfSpecData = keyword("if") _ expr:expr(s) {IfSpecData(expr)}122 pub rule forspec(s: &ParserSettings) -> ForSpecData = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}123 pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec> = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}124 pub rule local_expr(s: &ParserSettings) -> LocExpr = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)125 pub rule string_expr(s: &ParserSettings) -> LocExpr = l(s, <s:string() {Expr::Str(s)}>)126 pub rule obj_expr(s: &ParserSettings) -> LocExpr = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)127 pub rule array_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)128 pub rule array_comp_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, forspec, others.unwrap_or_default())}>)129 pub rule number_expr(s: &ParserSettings) -> LocExpr = l(s,<n:number() { expr::Expr::Num(n) }>)130 pub rule var_expr(s: &ParserSettings) -> LocExpr = l(s,<n:id() { expr::Expr::Var(n) }>)131 pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{132 cond,133 cond_then,134 cond_else,135 }}>)136137 pub rule literal(s: &ParserSettings) -> LocExpr138 = l(s,<v:(139 keyword("null") {LiteralType::Null}140 / keyword("true") {LiteralType::True}141 / keyword("false") {LiteralType::False}142 / keyword("self") {LiteralType::This}143 / keyword("$") {LiteralType::Dollar}144 / keyword("super") {LiteralType::Super}145 ) {Expr::Literal(v)}>)146147 pub rule expr_basic(s: &ParserSettings) -> LocExpr148 = literal(s)149150 / string_expr(s) / number_expr(s)151 / array_expr(s)152 / obj_expr(s)153 / array_expr(s)154 / array_comp_expr(s)155156 / var_expr(s)157 / local_expr(s)158 / if_then_else_expr(s)159160 / l(s,<keyword("function") _ "(" _ params:params(s) _ ")" _ expr:expr(s) {Expr::Function(params, expr)}>)161 / l(s,<assertion:assertion(s) _ ";" _ expr:expr(s) { Expr::AssertExpr(assertion, expr) }>)162163 / l(s,<keyword("error") _ expr:expr(s) { Expr::Error(expr) }>)164165 rule expr_basic_with_suffix(s: &ParserSettings) -> LocExpr166 = a:expr_basic(s) suffixes:(_ suffix:l_expr_suffix(s) {suffix})* {167 let mut cur = a;168 for suffix in suffixes {169 let LocSuffix(suffix, location) = suffix;170 cur = LocExpr(Rc::new(match suffix {171 Suffix::String(index) => Expr::Index(cur, loc_expr!(Expr::Str(index), s.loc_data, (s.file_name.clone(), location.1, location.2))),172 Suffix::Slice(desc) => Expr::Slice(cur, desc),173 Suffix::Expression(index) => Expr::Index(cur, index),174 Suffix::Apply(args) => Expr::Apply(cur, args),175 Suffix::Extend(body) => Expr::ObjExtend(cur, body),176 }), if s.loc_data { Some(Rc::new(location)) } else { None })177 }178 cur179 }180181 pub rule slice_desc(s: &ParserSettings) -> SliceDesc182 = start:expr(s)? _ ":" _ pair:(end:expr(s)? _ step:(":" _ e:expr(s) {e})? {(end, step)})? {183 if let Some((end, step)) = pair {184 SliceDesc { start, end, step }185 }else{186 SliceDesc { start, end: None, step: None }187 }188 }189190 rule expr_suffix(s: &ParserSettings) -> Suffix191 = "." _ s:id() { Suffix::String(s) }192 / "[" _ s:slice_desc(s) _ "]" { Suffix::Slice(s) }193 / "[" _ s:expr(s) _ "]" { Suffix::Expression(s) }194 / "(" _ args:args(s) _ ")" (_ keyword("tailstrict"))? { Suffix::Apply(args) }195 / "{" _ body:objinside(s) _ "}" { Suffix::Extend(body) }196 rule l_expr_suffix(s: &ParserSettings) -> LocSuffix197 = start:position!() suffix:expr_suffix(s) end:position!() {LocSuffix(suffix, ExprLocation(s.file_name.clone(), start, end))}198199 rule expr(s: &ParserSettings) -> LocExpr200 = start:position!() a:precedence! {201 a:(@) _ "||" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Or, b))}202 --203 a:(@) _ "&&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::And, b))}204 --205 a:(@) _ "|" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitOr, b))}206 --207 a:@ _ "^" _ b:(@) {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitXor, b))}208 --209 a:(@) _ "&" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::BitAnd, b))}210 --211 a:(@) _ "==" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Eq, b))}212 a:(@) _ "!=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Ne, b))}213 --214 a:(@) _ "<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lt, b))}215 a:(@) _ ">" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gt, b))}216 a:(@) _ "<=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lte, b))}217 a:(@) _ ">=" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Gte, b))}218 --219 a:(@) _ "<<" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Lhs, b))}220 a:(@) _ ">>" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Rhs, b))}221 --222 a:(@) _ "+" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Add, b))}223 a:(@) _ "-" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Sub, b))}224 --225 a:(@) _ "*" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mul, b))}226 a:(@) _ "/" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Div, b))}227 a:(@) _ "%" _ b:@ {loc_expr_todo!(Expr::BinaryOp(a, BinaryOpType::Mod, b))}228 --229 e:expr_basic_with_suffix(s) {e}230 "-" _ expr:expr_basic_with_suffix(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Minus, expr)) }231 "!" _ expr:expr_basic_with_suffix(s) { loc_expr_todo!(Expr::UnaryOp(UnaryOpType::Not, expr)) }232 "(" _ e:expr(s) _ ")" {loc_expr_todo!(Expr::Parened(e))}233 } end:position!() {234 let LocExpr(e, _) = a;235 LocExpr(e, if s.loc_data {236 Some(Rc::new(ExprLocation(s.file_name.to_owned(), start, end)))237 } else {238 None239 })240 }241 / e:expr_basic_with_suffix(s) {e}242243 pub rule jsonnet(s: &ParserSettings) -> LocExpr = _ e:expr(s) _ {e}244 }245}246247// TODO: impl FromStr from Expr248pub fn parse(249 str: &str,250 settings: &ParserSettings,251) -> Result<LocExpr, peg::error::ParseError<peg::str::LineCol>> {252 jsonnet_parser::jsonnet(str, settings)253}254255#[cfg(test)]256pub mod tests {257 use super::{expr::*, parse};258 use crate::ParserSettings;259 macro_rules! el {260 ($expr:expr) => {261 LocExpr(std::rc::Rc::new($expr), None)262 };263 }264 macro_rules! parse {265 ($s:expr) => {266 parse(267 $s,268 &ParserSettings {269 loc_data: false,270 file_name: "test.jsonnet".to_owned(),271 },272 )273 .unwrap()274 };275 }276277 mod expressions {278 use super::*;279280 pub fn basic_math() -> LocExpr {281 el!(Expr::BinaryOp(282 el!(Expr::Num(2.0)),283 BinaryOpType::Add,284 el!(Expr::BinaryOp(285 el!(Expr::Num(2.0)),286 BinaryOpType::Mul,287 el!(Expr::Num(2.0)),288 )),289 ))290 }291 }292293 #[test]294 fn empty_object() {295 assert_eq!(parse!("{}"), el!(Expr::Obj(ObjBody::MemberList(vec![]))));296 }297298 #[test]299 fn basic_math() {300 assert_eq!(301 parse!("2+2*2"),302 el!(Expr::BinaryOp(303 el!(Expr::Num(2.0)),304 BinaryOpType::Add,305 el!(Expr::BinaryOp(306 el!(Expr::Num(2.0)),307 BinaryOpType::Mul,308 el!(Expr::Num(2.0))309 ))310 ))311 );312 }313314 #[test]315 fn basic_math_with_indents() {316 assert_eq!(parse!("2 + 2 * 2 "), expressions::basic_math());317 }318319 #[test]320 fn basic_math_parened() {321 assert_eq!(322 parse!("2+(2+2*2)"),323 el!(Expr::BinaryOp(324 el!(Expr::Num(2.0)),325 BinaryOpType::Add,326 el!(Expr::Parened(expressions::basic_math())),327 ))328 );329 }330331 /// Comments should not affect parsing332 #[test]333 fn comments() {334 assert_eq!(335 parse!("2//comment\n+//comment\n3/*test*/*/*test*/4"),336 el!(Expr::BinaryOp(337 el!(Expr::Num(2.0)),338 BinaryOpType::Add,339 el!(Expr::BinaryOp(340 el!(Expr::Num(3.0)),341 BinaryOpType::Mul,342 el!(Expr::Num(4.0))343 ))344 ))345 );346 }347348 /// Comments should be able to be escaped349 #[test]350 fn comment_escaping() {351 assert_eq!(352 parse!("2/*\\*/+*/ - 22"),353 el!(Expr::BinaryOp(354 el!(Expr::Num(2.0)),355 BinaryOpType::Sub,356 el!(Expr::Num(22.0))357 ))358 );359 }360361 #[test]362 fn suffix_comparsion() {363 use Expr::*;364 assert_eq!(365 parse!("std.type(a) == \"string\""),366 el!(BinaryOp(367 el!(Apply(368 el!(Index(369 el!(Var("std".to_owned())),370 el!(Str("type".to_owned()))371 )),372 ArgsDesc(vec![Arg(None, el!(Var("a".to_owned())))])373 )),374 BinaryOpType::Eq,375 el!(Str("string".to_owned()))376 ))377 );378 }379380 #[test]381 fn array_comp() {382 use Expr::*;383 assert_eq!(384 parse!("[std.deepJoin(x) for x in arr]"),385 el!(ArrComp(386 el!(Apply(387 el!(Index(388 el!(Var("std".to_owned())),389 el!(Str("deepJoin".to_owned()))390 )),391 ArgsDesc(vec![Arg(None, el!(Var("x".to_owned())))])392 )),393 ForSpecData("x".to_owned(), el!(Var("arr".to_owned()))),394 vec![]395 )),396 )397 }398399 #[test]400 fn array_comp_with_ifs() {401 use Expr::*;402 assert_eq!(403 parse!("[k for k in std.objectFields(patch) if patch[k] == null]"),404 el!(ArrComp(405 el!(Var("k".to_owned())),406 ForSpecData(407 "k".to_owned(),408 el!(Apply(409 el!(Index(410 el!(Var("std".to_owned())),411 el!(Str("objectFields".to_owned()))412 )),413 ArgsDesc(vec![Arg(None, el!(Var("patch".to_owned())))])414 ))415 ),416 vec![CompSpec::IfSpec(IfSpecData(el!(BinaryOp(417 el!(Index(418 el!(Var("patch".to_owned())),419 el!(Var("k".to_owned()))420 )),421 BinaryOpType::Eq,422 el!(Literal(LiteralType::Null))423 ))))]424 ))425 );426 }427428 #[test]429 fn reserved() {430 use Expr::*;431 assert_eq!(parse!("null"), el!(Literal(LiteralType::Null)));432 assert_eq!(parse!("nulla"), el!(Var("nulla".to_owned())));433 }434435 #[test]436 fn multiple_args_buf() {437 parse!("a(b, null_fields)");438 }439440 #[test]441 fn infix_precedence() {442 use Expr::*;443 assert_eq!(444 parse!("!a && !b"),445 el!(BinaryOp(446 el!(UnaryOp(UnaryOpType::Not, el!(Var("a".to_owned())))),447 BinaryOpType::And,448 el!(UnaryOp(UnaryOpType::Not, el!(Var("b".to_owned()))))449 ))450 );451 }452453 #[test]454 fn can_parse_stdlib() {455 parse!(jsonnet_stdlib::STDLIB_STR);456 }457}