git.delta.rocks / jrsonnet / refs/commits / 0444432f3e21

difftreelog

feat(parser) serializable AST

Лач2020-06-04parent: #3302a3d.patch.diff
in: master

3 files changed

modifiedcrates/jsonnet-parser/Cargo.tomldiffbeforeafterboth
--- a/crates/jsonnet-parser/Cargo.toml
+++ b/crates/jsonnet-parser/Cargo.toml
@@ -8,9 +8,13 @@
 default = []
 # Trace peg token parsing
 trace = ["peg/trace"]
+# TODO:
+# serialize = ["serde"]
 
 [dependencies]
 peg = "0.6.2"
+serde = { version = "1.0.111", features = ["derive", "rc"] }
 
 [dev-dependencies]
 jsonnet-stdlib = { version = "0.1.0", path = "../jsonnet-stdlib" }
+bincode = "1.2.1"
modifiedcrates/jsonnet-parser/src/expr.rsdiffbeforeafterboth
before · crates/jsonnet-parser/src/expr.rs
1use std::{fmt::Debug, rc::Rc};23#[derive(Debug, Clone, PartialEq)]4pub enum FieldName {5	/// {fixed: 2}6	Fixed(String),7	/// {["dyn"+"amic"]: 3}8	Dyn(LocExpr),9}1011#[derive(Debug, Clone, PartialEq)]12pub enum Visibility {13	/// :14	Normal,15	/// ::16	Hidden,17	/// :::18	Unhide,19}2021#[derive(Debug, Clone, PartialEq)]22pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);2324#[derive(Debug, Clone, PartialEq)]25pub struct FieldMember {26	pub name: FieldName,27	pub plus: bool,28	pub params: Option<ParamsDesc>,29	pub visibility: Visibility,30	pub value: LocExpr,31}3233#[derive(Debug, Clone, PartialEq)]34pub enum Member {35	Field(FieldMember),36	BindStmt(BindSpec),37	AssertStmt(AssertStmt),38}3940#[derive(Debug, Clone, Copy, PartialEq)]41pub enum UnaryOpType {42	Plus,43	Minus,44	BitNot,45	Not,46}4748#[derive(Debug, Clone, Copy, PartialEq)]49pub enum BinaryOpType {50	Mul,51	Div,52	Mod,5354	Add,55	Sub,5657	Lhs,58	Rhs,5960	Lt,61	Gt,62	Lte,63	Gte,6465	In,6667	Eq,68	Ne,6970	BitAnd,71	BitOr,72	BitXor,7374	And,75	Or,76}7778/// name, default value79#[derive(Debug, Clone, PartialEq)]80pub struct Param(pub String, pub Option<LocExpr>);81/// Defined function parameters82#[derive(Debug, Clone, PartialEq)]83pub struct ParamsDesc(pub Vec<Param>);84impl ParamsDesc {85	pub fn with_defaults(&self) -> Vec<Param> {86		self.0.iter().filter(|e| e.1.is_some()).cloned().collect()87	}88}8990#[derive(Debug, Clone, PartialEq)]91pub struct Arg(pub Option<String>, pub LocExpr);92#[derive(Debug, Clone, PartialEq)]93pub struct ArgsDesc(pub Vec<Arg>);9495#[derive(Debug, Clone, PartialEq)]96pub struct BindSpec {97	pub name: String,98	pub params: Option<ParamsDesc>,99	pub value: LocExpr,100}101102#[derive(Debug, Clone, PartialEq)]103pub struct IfSpecData(pub LocExpr);104#[derive(Debug, Clone, PartialEq)]105pub struct ForSpecData(pub String, pub LocExpr);106107#[derive(Debug, Clone, PartialEq)]108pub enum CompSpec {109	IfSpec(IfSpecData),110	ForSpec(ForSpecData),111}112113#[derive(Debug, Clone, PartialEq)]114pub enum ObjBody {115	MemberList(Vec<Member>),116	ObjComp {117		pre_locals: Vec<BindSpec>,118		key: LocExpr,119		value: LocExpr,120		post_locals: Vec<BindSpec>,121		first: ForSpecData,122		rest: Vec<CompSpec>,123	},124}125126#[derive(Debug, Clone, PartialEq)]127pub enum LiteralType {128	This,129	Super,130	Dollar,131	Null,132	True,133	False,134}135136#[derive(Debug, Clone, PartialEq)]137pub struct SliceDesc {138	pub start: Option<LocExpr>,139	pub end: Option<LocExpr>,140	pub step: Option<LocExpr>,141}142143/// Syntax base144#[derive(Debug, Clone, PartialEq)]145pub enum Expr {146	Literal(LiteralType),147148	/// String value: "hello"149	Str(String),150	/// Number: 1, 2.0, 2e+20151	Num(f64),152	/// Variable name: test153	Var(String),154155	/// Array of expressions: [1, 2, "Hello"]156	Arr(Vec<LocExpr>),157	/// Array comprehension:158	/// ```jsonnet159	///  ingredients: [160	///    { kind: kind, qty: 4 / 3 }161	///    for kind in [162	///      'Honey Syrup',163	///      'Lemon Juice',164	///      'Farmers Gin',165	///    ]166	///  ],167	/// ```168	ArrComp(LocExpr, Vec<CompSpec>),169170	/// Object: {a: 2}171	Obj(ObjBody),172	/// Object extension: var1 {b: 2}173	ObjExtend(LocExpr, ObjBody),174175	/// (obj)176	Parened(LocExpr),177178	/// Params in function definition179	/// hello, world, test = 2180	Params(ParamsDesc),181	/// Args in function call182	/// 2 + 2, 3, named = 6183	Args(ArgsDesc),184185	/// -2186	UnaryOp(UnaryOpType, LocExpr),187	/// 2 - 2188	BinaryOp(LocExpr, BinaryOpType, LocExpr),189	/// assert 2 == 2 : "Math is broken"190	AssertExpr(AssertStmt, LocExpr),191	/// local a = 2; { b: a }192	LocalExpr(Vec<BindSpec>, LocExpr),193194	/// a = 3195	Bind(BindSpec),196	/// import "hello"197	Import(String),198	/// importStr "file.txt"199	ImportStr(String),200	/// error "I'm broken"201	Error(LocExpr),202	/// a(b, c)203	Apply(LocExpr, ArgsDesc),204	///205	Select(LocExpr, String),206	/// a[b]207	Index(LocExpr, LocExpr),208	/// a[1::2]209	Slice(LocExpr, SliceDesc),210	/// function(x) x211	Function(ParamsDesc, LocExpr),212	/// if true == false then 1 else 2213	IfElse {214		cond: IfSpecData,215		cond_then: LocExpr,216		cond_else: Option<LocExpr>,217	},218	/// if 2 = 3219	IfSpec(IfSpecData),220	/// for elem in array221	ForSpec(ForSpecData),222}223224/// file, begin offset, end offset225#[derive(Clone, PartialEq)]226pub struct ExprLocation(pub String, pub usize, pub usize);227impl Debug for ExprLocation {228	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {229		write!(f, "{}:{:?}-{:?}", self.0, self.1, self.2)230	}231}232233/// Holds AST expression and its location in source file+234#[derive(Clone, PartialEq)]235pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);236impl Debug for LocExpr {237	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {238		write!(f, "{:?} from {:?}", self.0, self.1)239	}240}241242/// Creates LocExpr from Expr and ExprLocation components243#[macro_export]244macro_rules! loc_expr {245	($expr:expr, $need_loc:expr, ($name:expr, $start:expr, $end:expr)) => {246		LocExpr(247			std::rc::Rc::new($expr),248			if $need_loc {249				Some(std::rc::Rc::new(ExprLocation(250					$name.to_owned(),251					$start,252					$end,253				)))254			} else {255				None256				},257			)258	};259}260261/// Creates LocExpr without location info262#[macro_export]263macro_rules! loc_expr_todo {264	($expr:expr) => {265		LocExpr(Rc::new($expr), None)266	};267}
after · crates/jsonnet-parser/src/expr.rs
1use serde::{Deserialize, Serialize};2use std::{fmt::Debug, rc::Rc};34#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]5pub enum FieldName {6	/// {fixed: 2}7	Fixed(String),8	/// {["dyn"+"amic"]: 3}9	Dyn(LocExpr),10}1112#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]13pub enum Visibility {14	/// :15	Normal,16	/// ::17	Hidden,18	/// :::19	Unhide,20}2122#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]23pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);2425#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]26pub struct FieldMember {27	pub name: FieldName,28	pub plus: bool,29	pub params: Option<ParamsDesc>,30	pub visibility: Visibility,31	pub value: LocExpr,32}3334#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]35pub enum Member {36	Field(FieldMember),37	BindStmt(BindSpec),38	AssertStmt(AssertStmt),39}4041#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]42pub enum UnaryOpType {43	Plus,44	Minus,45	BitNot,46	Not,47}4849#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]50pub enum BinaryOpType {51	Mul,52	Div,53	Mod,5455	Add,56	Sub,5758	Lhs,59	Rhs,6061	Lt,62	Gt,63	Lte,64	Gte,6566	In,6768	Eq,69	Ne,7071	BitAnd,72	BitOr,73	BitXor,7475	And,76	Or,77}7879/// name, default value80#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]81pub struct Param(pub String, pub Option<LocExpr>);82/// Defined function parameters83#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]84pub struct ParamsDesc(pub Vec<Param>);85impl ParamsDesc {86	pub fn with_defaults(&self) -> Vec<Param> {87		self.0.iter().filter(|e| e.1.is_some()).cloned().collect()88	}89}9091#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]92pub struct Arg(pub Option<String>, pub LocExpr);93#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]94pub struct ArgsDesc(pub Vec<Arg>);9596#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]97pub struct BindSpec {98	pub name: String,99	pub params: Option<ParamsDesc>,100	pub value: LocExpr,101}102103#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]104pub struct IfSpecData(pub LocExpr);105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]106pub struct ForSpecData(pub String, pub LocExpr);107108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]109pub enum CompSpec {110	IfSpec(IfSpecData),111	ForSpec(ForSpecData),112}113114#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]115pub enum ObjBody {116	MemberList(Vec<Member>),117	ObjComp {118		pre_locals: Vec<BindSpec>,119		key: LocExpr,120		value: LocExpr,121		post_locals: Vec<BindSpec>,122		first: ForSpecData,123		rest: Vec<CompSpec>,124	},125}126127#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]128pub enum LiteralType {129	This,130	Super,131	Dollar,132	Null,133	True,134	False,135}136137#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]138pub struct SliceDesc {139	pub start: Option<LocExpr>,140	pub end: Option<LocExpr>,141	pub step: Option<LocExpr>,142}143144/// Syntax base145#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]146pub enum Expr {147	Literal(LiteralType),148149	/// String value: "hello"150	Str(String),151	/// Number: 1, 2.0, 2e+20152	Num(f64),153	/// Variable name: test154	Var(String),155156	/// Array of expressions: [1, 2, "Hello"]157	Arr(Vec<LocExpr>),158	/// Array comprehension:159	/// ```jsonnet160	///  ingredients: [161	///    { kind: kind, qty: 4 / 3 }162	///    for kind in [163	///      'Honey Syrup',164	///      'Lemon Juice',165	///      'Farmers Gin',166	///    ]167	///  ],168	/// ```169	ArrComp(LocExpr, Vec<CompSpec>),170171	/// Object: {a: 2}172	Obj(ObjBody),173	/// Object extension: var1 {b: 2}174	ObjExtend(LocExpr, ObjBody),175176	/// (obj)177	Parened(LocExpr),178179	/// Params in function definition180	/// hello, world, test = 2181	Params(ParamsDesc),182	/// Args in function call183	/// 2 + 2, 3, named = 6184	Args(ArgsDesc),185186	/// -2187	UnaryOp(UnaryOpType, LocExpr),188	/// 2 - 2189	BinaryOp(LocExpr, BinaryOpType, LocExpr),190	/// assert 2 == 2 : "Math is broken"191	AssertExpr(AssertStmt, LocExpr),192	/// local a = 2; { b: a }193	LocalExpr(Vec<BindSpec>, LocExpr),194195	/// a = 3196	Bind(BindSpec),197	/// import "hello"198	Import(String),199	/// importStr "file.txt"200	ImportStr(String),201	/// error "I'm broken"202	Error(LocExpr),203	/// a(b, c)204	Apply(LocExpr, ArgsDesc),205	///206	Select(LocExpr, String),207	/// a[b]208	Index(LocExpr, LocExpr),209	/// a[1::2]210	Slice(LocExpr, SliceDesc),211	/// function(x) x212	Function(ParamsDesc, LocExpr),213	/// if true == false then 1 else 2214	IfElse {215		cond: IfSpecData,216		cond_then: LocExpr,217		cond_else: Option<LocExpr>,218	},219	/// if 2 = 3220	IfSpec(IfSpecData),221	/// for elem in array222	ForSpec(ForSpecData),223}224225/// file, begin offset, end offset226#[derive(Clone, PartialEq, Serialize, Deserialize)]227pub struct ExprLocation(pub String, pub usize, pub usize);228impl Debug for ExprLocation {229	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {230		write!(f, "{}:{:?}-{:?}", self.0, self.1, self.2)231	}232}233234/// Holds AST expression and its location in source file+235#[derive(Clone, PartialEq, Serialize, Deserialize)]236pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);237impl Debug for LocExpr {238	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {239		write!(f, "{:?} from {:?}", self.0, self.1)240	}241}242243/// Creates LocExpr from Expr and ExprLocation components244#[macro_export]245macro_rules! loc_expr {246	($expr:expr, $need_loc:expr,($name:expr, $start:expr, $end:expr)) => {247		LocExpr(248			std::rc::Rc::new($expr),249			if $need_loc {250				Some(std::rc::Rc::new(ExprLocation(251					$name.to_owned(),252					$start,253					$end,254				)))255			} else {256				None257				},258			)259	};260}261262/// Creates LocExpr without location info263#[macro_export]264macro_rules! loc_expr_todo {265	($expr:expr) => {266		LocExpr(Rc::new($expr), None)267	};268}
modifiedcrates/jsonnet-parser/src/lib.rsdiffbeforeafterboth
--- a/crates/jsonnet-parser/src/lib.rs
+++ b/crates/jsonnet-parser/src/lib.rs
@@ -1,4 +1,7 @@
 #![feature(box_syntax)]
+#![feature(test)]
+
+extern crate test;
 
 use peg::parser;
 use std::rc::Rc;
@@ -41,8 +44,11 @@
 		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()
 		rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")
 
-		rule keyword(id: &'static str) = ##parse_string_literal(id) end_of_ident()
-		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
+		rule keyword(id: &'static str)
+			= ##parse_string_literal(id) end_of_ident()
+		// Adds location data information to existing expression
+		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr
+			= start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
 
 		pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }
 		pub rule params(s: &ParserSettings) -> expr::ParamsDesc
@@ -73,10 +79,11 @@
 		pub rule bind(s: &ParserSettings) -> expr::BindSpec
 			= name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}
 			/ name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}
-		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
+		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt
+			= keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
 		pub rule string() -> String
-			= "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
-			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}
+			= v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
+			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}
 		pub rule field_name(s: &ParserSettings) -> expr::FieldName
 			= name:id() {expr::FieldName::Fixed(name)}
 			/ name:string() {expr::FieldName::Fixed(name)}
@@ -118,21 +125,32 @@
 				}
 			}
 			/ members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}
-		pub rule ifspec(s: &ParserSettings) -> IfSpecData = keyword("if") _ expr:expr(s) {IfSpecData(expr)}
-		pub rule forspec(s: &ParserSettings) -> ForSpecData = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
-		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec> = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
-		pub rule local_expr(s: &ParserSettings) -> LocExpr = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
-		pub rule string_expr(s: &ParserSettings) -> LocExpr = l(s, <s:string() {Expr::Str(s)}>)
-		pub rule obj_expr(s: &ParserSettings) -> LocExpr = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
-		pub rule array_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
-		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
-		pub rule number_expr(s: &ParserSettings) -> LocExpr = l(s,<n:number() { expr::Expr::Num(n) }>)
-		pub rule var_expr(s: &ParserSettings) -> LocExpr = l(s,<n:id() { expr::Expr::Var(n) }>)
-		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
-			cond,
-			cond_then,
-			cond_else,
-		}}>)
+		pub rule ifspec(s: &ParserSettings) -> IfSpecData
+			= keyword("if") _ expr:expr(s) {IfSpecData(expr)}
+		pub rule forspec(s: &ParserSettings) -> ForSpecData
+			= keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
+		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>
+			= s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
+		pub rule local_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
+		pub rule string_expr(s: &ParserSettings) -> LocExpr
+			= l(s, <s:string() {Expr::Str(s)}>)
+		pub rule obj_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
+		pub rule array_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
+		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
+		pub rule number_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:number() { expr::Expr::Num(n) }>)
+		pub rule var_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:id() { expr::Expr::Var(n) }>)
+		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
+				cond,
+				cond_then,
+				cond_else,
+			}}>)
 
 		pub rule literal(s: &ParserSettings) -> LocExpr
 			= l(s,<v:(
@@ -244,7 +262,6 @@
 	}
 }
 
-// TODO: impl FromStr from Expr
 pub fn parse(
 	str: &str,
 	settings: &ParserSettings,
@@ -461,4 +478,19 @@
 	fn can_parse_stdlib() {
 		parse!(jsonnet_stdlib::STDLIB_STR);
 	}
+
+	use test::Bencher;
+
+	// From source code
+	#[bench]
+	fn bench_parse_peg(b: &mut Bencher) {
+		b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))
+	}
+
+	// From serialized blob
+	#[bench]
+	fn bench_parse_serde_bincode(b: &mut Bencher) {
+		let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();
+		b.iter(|| bincode::deserialize::<LocExpr>(&serialized))
+	}
 }