git.delta.rocks / jrsonnet / refs/commits / 0444432f3e21

difftreelog

feat(parser) serializable AST

Лач2020-06-04parent: #3302a3d.patch.diff
in: master

3 files changed

modifiedcrates/jsonnet-parser/Cargo.tomldiffbeforeafterboth
--- a/crates/jsonnet-parser/Cargo.toml
+++ b/crates/jsonnet-parser/Cargo.toml
@@ -8,9 +8,13 @@
 default = []
 # Trace peg token parsing
 trace = ["peg/trace"]
+# TODO:
+# serialize = ["serde"]
 
 [dependencies]
 peg = "0.6.2"
+serde = { version = "1.0.111", features = ["derive", "rc"] }
 
 [dev-dependencies]
 jsonnet-stdlib = { version = "0.1.0", path = "../jsonnet-stdlib" }
+bincode = "1.2.1"
modifiedcrates/jsonnet-parser/src/expr.rsdiffbeforeafterboth
1use serde::{Deserialize, Serialize};
1use std::{fmt::Debug, rc::Rc};2use std::{fmt::Debug, rc::Rc};
23
3#[derive(Debug, Clone, PartialEq)]4#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
4pub enum FieldName {5pub enum FieldName {
5	/// {fixed: 2}6	/// {fixed: 2}
6	Fixed(String),7	Fixed(String),
7	/// {["dyn"+"amic"]: 3}8	/// {["dyn"+"amic"]: 3}
8	Dyn(LocExpr),9	Dyn(LocExpr),
9}10}
1011
11#[derive(Debug, Clone, PartialEq)]12#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
12pub enum Visibility {13pub enum Visibility {
13	/// :14	/// :
14	Normal,15	Normal,
18	Unhide,19	Unhide,
19}20}
2021
21#[derive(Debug, Clone, PartialEq)]22#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
22pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);23pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);
2324
24#[derive(Debug, Clone, PartialEq)]25#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25pub struct FieldMember {26pub struct FieldMember {
26	pub name: FieldName,27	pub name: FieldName,
27	pub plus: bool,28	pub plus: bool,
30	pub value: LocExpr,31	pub value: LocExpr,
31}32}
3233
33#[derive(Debug, Clone, PartialEq)]34#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
34pub enum Member {35pub enum Member {
35	Field(FieldMember),36	Field(FieldMember),
36	BindStmt(BindSpec),37	BindStmt(BindSpec),
37	AssertStmt(AssertStmt),38	AssertStmt(AssertStmt),
38}39}
3940
40#[derive(Debug, Clone, Copy, PartialEq)]41#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
41pub enum UnaryOpType {42pub enum UnaryOpType {
42	Plus,43	Plus,
43	Minus,44	Minus,
44	BitNot,45	BitNot,
45	Not,46	Not,
46}47}
4748
48#[derive(Debug, Clone, Copy, PartialEq)]49#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
49pub enum BinaryOpType {50pub enum BinaryOpType {
50	Mul,51	Mul,
51	Div,52	Div,
76}77}
7778
78/// name, default value79/// name, default value
79#[derive(Debug, Clone, PartialEq)]80#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
80pub struct Param(pub String, pub Option<LocExpr>);81pub struct Param(pub String, pub Option<LocExpr>);
81/// Defined function parameters82/// Defined function parameters
82#[derive(Debug, Clone, PartialEq)]83#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
83pub struct ParamsDesc(pub Vec<Param>);84pub struct ParamsDesc(pub Vec<Param>);
84impl ParamsDesc {85impl ParamsDesc {
85	pub fn with_defaults(&self) -> Vec<Param> {86	pub fn with_defaults(&self) -> Vec<Param> {
86		self.0.iter().filter(|e| e.1.is_some()).cloned().collect()87		self.0.iter().filter(|e| e.1.is_some()).cloned().collect()
87	}88	}
88}89}
8990
90#[derive(Debug, Clone, PartialEq)]91#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
91pub struct Arg(pub Option<String>, pub LocExpr);92pub struct Arg(pub Option<String>, pub LocExpr);
92#[derive(Debug, Clone, PartialEq)]93#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
93pub struct ArgsDesc(pub Vec<Arg>);94pub struct ArgsDesc(pub Vec<Arg>);
9495
95#[derive(Debug, Clone, PartialEq)]96#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
96pub struct BindSpec {97pub struct BindSpec {
97	pub name: String,98	pub name: String,
98	pub params: Option<ParamsDesc>,99	pub params: Option<ParamsDesc>,
99	pub value: LocExpr,100	pub value: LocExpr,
100}101}
101102
102#[derive(Debug, Clone, PartialEq)]103#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
103pub struct IfSpecData(pub LocExpr);104pub struct IfSpecData(pub LocExpr);
104#[derive(Debug, Clone, PartialEq)]105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
105pub struct ForSpecData(pub String, pub LocExpr);106pub struct ForSpecData(pub String, pub LocExpr);
106107
107#[derive(Debug, Clone, PartialEq)]108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
108pub enum CompSpec {109pub enum CompSpec {
109	IfSpec(IfSpecData),110	IfSpec(IfSpecData),
110	ForSpec(ForSpecData),111	ForSpec(ForSpecData),
111}112}
112113
113#[derive(Debug, Clone, PartialEq)]114#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
114pub enum ObjBody {115pub enum ObjBody {
115	MemberList(Vec<Member>),116	MemberList(Vec<Member>),
116	ObjComp {117	ObjComp {
123	},124	},
124}125}
125126
126#[derive(Debug, Clone, PartialEq)]127#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
127pub enum LiteralType {128pub enum LiteralType {
128	This,129	This,
129	Super,130	Super,
133	False,134	False,
134}135}
135136
136#[derive(Debug, Clone, PartialEq)]137#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
137pub struct SliceDesc {138pub struct SliceDesc {
138	pub start: Option<LocExpr>,139	pub start: Option<LocExpr>,
139	pub end: Option<LocExpr>,140	pub end: Option<LocExpr>,
140	pub step: Option<LocExpr>,141	pub step: Option<LocExpr>,
141}142}
142143
143/// Syntax base144/// Syntax base
144#[derive(Debug, Clone, PartialEq)]145#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
145pub enum Expr {146pub enum Expr {
146	Literal(LiteralType),147	Literal(LiteralType),
147148
222}223}
223224
224/// file, begin offset, end offset225/// file, begin offset, end offset
225#[derive(Clone, PartialEq)]226#[derive(Clone, PartialEq, Serialize, Deserialize)]
226pub struct ExprLocation(pub String, pub usize, pub usize);227pub struct ExprLocation(pub String, pub usize, pub usize);
227impl Debug for ExprLocation {228impl Debug for ExprLocation {
228	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {229	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231}232}
232233
233/// Holds AST expression and its location in source file+234/// Holds AST expression and its location in source file+
234#[derive(Clone, PartialEq)]235#[derive(Clone, PartialEq, Serialize, Deserialize)]
235pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);236pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);
236impl Debug for LocExpr {237impl Debug for LocExpr {
237	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {238	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
modifiedcrates/jsonnet-parser/src/lib.rsdiffbeforeafterboth
--- a/crates/jsonnet-parser/src/lib.rs
+++ b/crates/jsonnet-parser/src/lib.rs
@@ -1,4 +1,7 @@
 #![feature(box_syntax)]
+#![feature(test)]
+
+extern crate test;
 
 use peg::parser;
 use std::rc::Rc;
@@ -41,8 +44,11 @@
 		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()
 		rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")
 
-		rule keyword(id: &'static str) = ##parse_string_literal(id) end_of_ident()
-		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
+		rule keyword(id: &'static str)
+			= ##parse_string_literal(id) end_of_ident()
+		// Adds location data information to existing expression
+		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr
+			= start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
 
 		pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }
 		pub rule params(s: &ParserSettings) -> expr::ParamsDesc
@@ -73,10 +79,11 @@
 		pub rule bind(s: &ParserSettings) -> expr::BindSpec
 			= name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}
 			/ name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}
-		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
+		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt
+			= keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
 		pub rule string() -> String
-			= "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
-			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}
+			= v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
+			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}
 		pub rule field_name(s: &ParserSettings) -> expr::FieldName
 			= name:id() {expr::FieldName::Fixed(name)}
 			/ name:string() {expr::FieldName::Fixed(name)}
@@ -118,21 +125,32 @@
 				}
 			}
 			/ members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}
-		pub rule ifspec(s: &ParserSettings) -> IfSpecData = keyword("if") _ expr:expr(s) {IfSpecData(expr)}
-		pub rule forspec(s: &ParserSettings) -> ForSpecData = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
-		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec> = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
-		pub rule local_expr(s: &ParserSettings) -> LocExpr = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
-		pub rule string_expr(s: &ParserSettings) -> LocExpr = l(s, <s:string() {Expr::Str(s)}>)
-		pub rule obj_expr(s: &ParserSettings) -> LocExpr = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
-		pub rule array_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
-		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
-		pub rule number_expr(s: &ParserSettings) -> LocExpr = l(s,<n:number() { expr::Expr::Num(n) }>)
-		pub rule var_expr(s: &ParserSettings) -> LocExpr = l(s,<n:id() { expr::Expr::Var(n) }>)
-		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
-			cond,
-			cond_then,
-			cond_else,
-		}}>)
+		pub rule ifspec(s: &ParserSettings) -> IfSpecData
+			= keyword("if") _ expr:expr(s) {IfSpecData(expr)}
+		pub rule forspec(s: &ParserSettings) -> ForSpecData
+			= keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
+		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>
+			= s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
+		pub rule local_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
+		pub rule string_expr(s: &ParserSettings) -> LocExpr
+			= l(s, <s:string() {Expr::Str(s)}>)
+		pub rule obj_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
+		pub rule array_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
+		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
+		pub rule number_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:number() { expr::Expr::Num(n) }>)
+		pub rule var_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:id() { expr::Expr::Var(n) }>)
+		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
+				cond,
+				cond_then,
+				cond_else,
+			}}>)
 
 		pub rule literal(s: &ParserSettings) -> LocExpr
 			= l(s,<v:(
@@ -244,7 +262,6 @@
 	}
 }
 
-// TODO: impl FromStr from Expr
 pub fn parse(
 	str: &str,
 	settings: &ParserSettings,
@@ -461,4 +478,19 @@
 	fn can_parse_stdlib() {
 		parse!(jsonnet_stdlib::STDLIB_STR);
 	}
+
+	use test::Bencher;
+
+	// From source code
+	#[bench]
+	fn bench_parse_peg(b: &mut Bencher) {
+		b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))
+	}
+
+	// From serialized blob
+	#[bench]
+	fn bench_parse_serde_bincode(b: &mut Bencher) {
+		let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();
+		b.iter(|| bincode::deserialize::<LocExpr>(&serialized))
+	}
 }