From 0444432f3e2120c8928575eda2192dec429e80d0 Mon Sep 17 00:00:00 2001
From: Лач <iam@lach.pw>
Date: Thu, 04 Jun 2020 10:27:30 +0000
Subject: [PATCH] feat(parser): serializable AST


---
--- a/crates/jsonnet-parser/Cargo.toml
+++ b/crates/jsonnet-parser/Cargo.toml
@@ -8,9 +8,13 @@
 default = []
 # Trace peg token parsing
 trace = ["peg/trace"]
+# TODO:
+# serialize = ["serde"]
 
 [dependencies]
 peg = "0.6.2"
+serde = { version = "1.0.111", features = ["derive", "rc"] }
 
 [dev-dependencies]
 jsonnet-stdlib = { version = "0.1.0", path = "../jsonnet-stdlib" }
+bincode = "1.2.1"
--- a/crates/jsonnet-parser/src/expr.rs
+++ b/crates/jsonnet-parser/src/expr.rs
@@ -1,6 +1,7 @@
+use serde::{Deserialize, Serialize};
 use std::{fmt::Debug, rc::Rc};
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum FieldName {
 	/// {fixed: 2}
 	Fixed(String),
@@ -8,7 +9,7 @@
 	Dyn(LocExpr),
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Visibility {
 	/// :
 	Normal,
@@ -18,10 +19,10 @@
 	Unhide,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct FieldMember {
 	pub name: FieldName,
 	pub plus: bool,
@@ -30,14 +31,14 @@
 	pub value: LocExpr,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Member {
 	Field(FieldMember),
 	BindStmt(BindSpec),
 	AssertStmt(AssertStmt),
 }
 
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 pub enum UnaryOpType {
 	Plus,
 	Minus,
@@ -45,7 +46,7 @@
 	Not,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 pub enum BinaryOpType {
 	Mul,
 	Div,
@@ -76,10 +77,10 @@
 }
 
 /// name, default value
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Param(pub String, pub Option<LocExpr>);
 /// Defined function parameters
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ParamsDesc(pub Vec<Param>);
 impl ParamsDesc {
 	pub fn with_defaults(&self) -> Vec<Param> {
@@ -87,30 +88,30 @@
 	}
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Arg(pub Option<String>, pub LocExpr);
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ArgsDesc(pub Vec<Arg>);
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct BindSpec {
 	pub name: String,
 	pub params: Option<ParamsDesc>,
 	pub value: LocExpr,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct IfSpecData(pub LocExpr);
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ForSpecData(pub String, pub LocExpr);
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum CompSpec {
 	IfSpec(IfSpecData),
 	ForSpec(ForSpecData),
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum ObjBody {
 	MemberList(Vec<Member>),
 	ObjComp {
@@ -123,7 +124,7 @@
 	},
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum LiteralType {
 	This,
 	Super,
@@ -133,7 +134,7 @@
 	False,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct SliceDesc {
 	pub start: Option<LocExpr>,
 	pub end: Option<LocExpr>,
@@ -141,7 +142,7 @@
 }
 
 /// Syntax base
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Expr {
 	Literal(LiteralType),
 
@@ -222,7 +223,7 @@
 }
 
 /// file, begin offset, end offset
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
 pub struct ExprLocation(pub String, pub usize, pub usize);
 impl Debug for ExprLocation {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -231,7 +232,7 @@
 }
 
 /// Holds AST expression and its location in source file+
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
 pub struct LocExpr(pub Rc<Expr>, pub Option<Rc<ExprLocation>>);
 impl Debug for LocExpr {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -242,7 +243,7 @@
 /// Creates LocExpr from Expr and ExprLocation components
 #[macro_export]
 macro_rules! loc_expr {
-	($expr:expr, $need_loc:expr, ($name:expr, $start:expr, $end:expr)) => {
+	($expr:expr, $need_loc:expr,($name:expr, $start:expr, $end:expr)) => {
 		LocExpr(
 			std::rc::Rc::new($expr),
 			if $need_loc {
--- a/crates/jsonnet-parser/src/lib.rs
+++ b/crates/jsonnet-parser/src/lib.rs
@@ -1,4 +1,7 @@
 #![feature(box_syntax)]
+#![feature(test)]
+
+extern crate test;
 
 use peg::parser;
 use std::rc::Rc;
@@ -41,8 +44,11 @@
 		rule reserved() = ("assert" / "else" / "error" / "false" / "for" / "function" / "if" / "import" / "importstr" / "in" / "local" / "null" / "tailstrict" / "then" / "self" / "super" / "true") end_of_ident()
 		rule id() -> String = quiet!{ !reserved() s:$(alpha() (alpha() / digit())*) {s.to_owned()}} / expected!("<identifier>")
 
-		rule keyword(id: &'static str) = ##parse_string_literal(id) end_of_ident()
-		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr = start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
+		rule keyword(id: &'static str)
+			= ##parse_string_literal(id) end_of_ident()
+		// Adds location data information to existing expression
+		rule l(s: &ParserSettings, x: rule<Expr>) -> LocExpr
+			= start:position!() v:x() end:position!() {loc_expr!(v, s.loc_data, (s.file_name.clone(), start, end))}
 
 		pub rule param(s: &ParserSettings) -> expr::Param = name:id() expr:(_ "=" _ expr:expr(s){expr})? { expr::Param(name, expr) }
 		pub rule params(s: &ParserSettings) -> expr::ParamsDesc
@@ -73,10 +79,11 @@
 		pub rule bind(s: &ParserSettings) -> expr::BindSpec
 			= name:id() _ "=" _ expr:expr(s) {expr::BindSpec{name, params: None, value: expr}}
 			/ name:id() _ "(" _ params:params(s) _ ")" _ "=" _ expr:expr(s) {expr::BindSpec{name, params: Some(params), value: expr}}
-		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt = keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
+		pub rule assertion(s: &ParserSettings) -> expr::AssertStmt
+			= keyword("assert") _ cond:expr(s) msg:(_ ":" _ e:expr(s) {e})? { expr::AssertStmt(cond, msg) }
 		pub rule string() -> String
-			= "\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
-			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}
+			= v:("\"" str:$(("\\\"" / !['"'][_])*) "\"" {str.to_owned()}
+			/ "'" str:$((!['\''][_])*) "'" {str.to_owned()}) {v.replace("\\n", "\n")}
 		pub rule field_name(s: &ParserSettings) -> expr::FieldName
 			= name:id() {expr::FieldName::Fixed(name)}
 			/ name:string() {expr::FieldName::Fixed(name)}
@@ -118,21 +125,32 @@
 				}
 			}
 			/ members:(member(s) ** comma()) comma()? {expr::ObjBody::MemberList(members)}
-		pub rule ifspec(s: &ParserSettings) -> IfSpecData = keyword("if") _ expr:expr(s) {IfSpecData(expr)}
-		pub rule forspec(s: &ParserSettings) -> ForSpecData = keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
-		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec> = s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
-		pub rule local_expr(s: &ParserSettings) -> LocExpr = l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
-		pub rule string_expr(s: &ParserSettings) -> LocExpr = l(s, <s:string() {Expr::Str(s)}>)
-		pub rule obj_expr(s: &ParserSettings) -> LocExpr = l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
-		pub rule array_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
-		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr = l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
-		pub rule number_expr(s: &ParserSettings) -> LocExpr = l(s,<n:number() { expr::Expr::Num(n) }>)
-		pub rule var_expr(s: &ParserSettings) -> LocExpr = l(s,<n:id() { expr::Expr::Var(n) }>)
-		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr = l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
-			cond,
-			cond_then,
-			cond_else,
-		}}>)
+		pub rule ifspec(s: &ParserSettings) -> IfSpecData
+			= keyword("if") _ expr:expr(s) {IfSpecData(expr)}
+		pub rule forspec(s: &ParserSettings) -> ForSpecData
+			= keyword("for") _ id:id() _ keyword("in") _ cond:expr(s) {ForSpecData(id, cond)}
+		pub rule compspec(s: &ParserSettings) -> Vec<expr::CompSpec>
+			= s:(i:ifspec(s) { expr::CompSpec::IfSpec(i) } / f:forspec(s) {expr::CompSpec::ForSpec(f)} )+ {s}
+		pub rule local_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<keyword("local") _ binds:bind(s) ** comma() _ ";" _ expr:expr(s) { Expr::LocalExpr(binds, expr) }>)
+		pub rule string_expr(s: &ParserSettings) -> LocExpr
+			= l(s, <s:string() {Expr::Str(s)}>)
+		pub rule obj_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"{" _ body:objinside(s) _ "}" {Expr::Obj(body)}>)
+		pub rule array_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ elems:(expr(s) ** comma()) _ comma()? "]" {Expr::Arr(elems)}>)
+		pub rule array_comp_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<"[" _ expr:expr(s) _ comma()? _ forspec:forspec(s) _ others:(others: compspec(s) _ {others})? "]" {Expr::ArrComp(expr, [vec![CompSpec::ForSpec(forspec)], others.unwrap_or_default()].concat())}>)
+		pub rule number_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:number() { expr::Expr::Num(n) }>)
+		pub rule var_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<n:id() { expr::Expr::Var(n) }>)
+		pub rule if_then_else_expr(s: &ParserSettings) -> LocExpr
+			= l(s,<cond:ifspec(s) _ keyword("then") _ cond_then:expr(s) cond_else:(_ keyword("else") _ e:expr(s) {e})? {Expr::IfElse{
+				cond,
+				cond_then,
+				cond_else,
+			}}>)
 
 		pub rule literal(s: &ParserSettings) -> LocExpr
 			= l(s,<v:(
@@ -244,7 +262,6 @@
 	}
 }
 
-// TODO: impl FromStr from Expr
 pub fn parse(
 	str: &str,
 	settings: &ParserSettings,
@@ -461,4 +478,19 @@
 	fn can_parse_stdlib() {
 		parse!(jsonnet_stdlib::STDLIB_STR);
 	}
+
+	use test::Bencher;
+
+	// From source code
+	#[bench]
+	fn bench_parse_peg(b: &mut Bencher) {
+		b.iter(|| parse!(jsonnet_stdlib::STDLIB_STR))
+	}
+
+	// From serialized blob
+	#[bench]
+	fn bench_parse_serde_bincode(b: &mut Bencher) {
+		let serialized = bincode::serialize(&parse!(jsonnet_stdlib::STDLIB_STR)).unwrap();
+		b.iter(|| bincode::deserialize::<LocExpr>(&serialized))
+	}
 }

-- 
gitstuff