git.delta.rocks / jrsonnet / refs/commits / 337d066dc72d

difftreelog

perf std ast codegen

Лач2020-06-27parent: #0ed995f.patch.diff
in: master

5 files changed

modifiedCargo.lockdiffbeforeafterboth
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,16 +142,18 @@
  "jsonnet-stdlib",
  "md5",
  "serde",
+ "structdump",
 ]
 
 [[package]]
 name = "jsonnet-parser"
 version = "0.1.0"
 dependencies = [
- "bincode",
  "jsonnet-stdlib",
  "peg",
  "serde",
+ "structdump",
+ "structdump-derive",
  "unescape",
 ]
 
@@ -247,9 +249,9 @@
 
 [[package]]
 name = "quote"
-version = "1.0.5"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42934bc9c8ab0d3b273a16d8551c8f0fcff46be73276ca083ec2414c15c4ba5e"
+checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
 dependencies = [
  "proc-macro2",
 ]
@@ -275,6 +277,23 @@
 ]
 
 [[package]]
+name = "structdump"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e16ec33a0342fdb67d13913b4ffae6527ebccfa04b5d7da174bdc7a31db29b8"
+
+[[package]]
+name = "structdump-derive"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06c337fdc077e02ccbfcc62af0090564a4af342975c3b7be09705efab90c1888"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "syn"
 version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
modifiedcrates/jsonnet-evaluator/Cargo.tomldiffbeforeafterboth
--- a/crates/jsonnet-evaluator/Cargo.toml
+++ b/crates/jsonnet-evaluator/Cargo.toml
@@ -9,8 +9,10 @@
 [features]
 default = ["serialized-stdlib", "faster"]
 # Serializes standard library AST instead of parsing them every run
-serialized-stdlib = ["serde", "bincode"]
-# Replace some standard library functions with faster implementations
+serialized-stdlib = ["serde", "bincode", "jsonnet-parser/deserialize"]
+# Same as above, but with generated code instead of serde. Reduces memory usage, but increases binary size and compilation time
+codegenerated-stdlib = []
+# Replace some standard library functions with faster implementations (I.e manifestJsonEx)
 # Library works fine without this feature, but requires more memory and time for std function calls
 faster = []
 
@@ -21,16 +23,12 @@
 indexmap = "1.4.0"
 md5 = "0.7.0"
 
-[dependencies.serde]
-version = "1.0.114"
-optional = true
+serde = { version = "1.0.114", optional = true }
+bincode = { version = "1.3.1", optional = true }
 
-[dependencies.bincode]
-version = "1.3.1"
-optional = true
-
 [build-dependencies]
-jsonnet-parser = { path = "../jsonnet-parser" }
+jsonnet-parser = { path = "../jsonnet-parser", features = ["dump", "serialize", "deserialize"] }
 jsonnet-stdlib = { path = "../jsonnet-stdlib" }
+structdump = "0.1.2"
 serde = "1.0.114"
 bincode = "1.3.1"
modifiedcrates/jsonnet-evaluator/README.mddiffbeforeafterboth
--- a/crates/jsonnet-evaluator/README.md
+++ b/crates/jsonnet-evaluator/README.md
@@ -2,6 +2,35 @@
 
 Interpreter for parsed jsonnet tree
 
+## Standard library
+
+jsonnet stdlib is embedded into evaluator, but there is different modes for this:
+
+- `codegenerated-stdlib`
+  - generates source code for reproducing stdlib AST ([Example](https://gist.githubusercontent.com/CertainLach/7b3149df556f3406f5e9368aaa9f32ec/raw/0c80d8ab9aa7b9288c6219a2779cb2ab37287669/a.rs))
+  - fastest on interpretation, slowest on compilation (it takes more than 5 minutes to optimize them by llvm)
+- `serialized-stdlib`
+  - serializes standard library AST using serde
+  - slower than `codegenerated-stdlib` at runtime, but have no compilation speed penality
+- none
+  - leaves only stdlib source code in binary, processing them same way as user supplied data
+  - slowest (as it involves parsing of standard library source code)
+
+Because of `codegenerated-stdlib` compilation slowdown, `serialized-stdlib` is used by default
+
+### Benchmark
+
+Can also be run via `cargo bench`
+
+```md
+# codegenerated-stdlib
+test tests::bench_codegen   ... bench:     401,696 ns/iter (+/- 38,521)
+# serialized-stdlib
+test tests::bench_serialize ... bench:   1,763,999 ns/iter (+/- 76,211)
+# none
+test tests::bench_parse     ... bench:   7,206,164 ns/iter (+/- 1,067,418)
+```
+
 ## Intristics
 
 Some functions from stdlib are implemented as intristics
modifiedcrates/jsonnet-evaluator/build.rsdiffbeforeafterboth
before · crates/jsonnet-evaluator/build.rs
1use bincode::serialize;2use jsonnet_parser::{3	parse, Expr, FieldMember, FieldName, LocExpr, Member, ObjBody, ParserSettings,4};5use jsonnet_stdlib::STDLIB_STR;6use std::{7	env,8	fs::File,9	io::Write,10	path::{Path, PathBuf},11	rc::Rc,12};1314fn main() {15	let parsed = parse(16		STDLIB_STR,17		&ParserSettings {18			file_name: Rc::new(PathBuf::from("std.jsonnet")),19			loc_data: true,20		},21	)22	.expect("parse");2324	let parsed = if cfg!(feature = "faster") {25		let LocExpr(expr, location) = parsed;26		LocExpr(27			Rc::new(match Rc::try_unwrap(expr).unwrap() {28				Expr::Obj(ObjBody::MemberList(members)) => Expr::Obj(ObjBody::MemberList(29					members30						.into_iter()31						.filter(|p| {32							!matches!(33								p,34								Member::Field(FieldMember {35									name: FieldName::Fixed(name),36									..37								}) if **name == *"join" || **name == *"manifestJsonEx" || **name == *"escapeStringJson"38							)39						})40						.collect(),41				)),42				_ => panic!("std value should be object"),43			}),44			location,45		)46	} else {47		parsed48	};4950	let out_dir = env::var("OUT_DIR").unwrap();51	let dest_path = Path::new(&out_dir).join("stdlib.bincode");52	let mut f = File::create(&dest_path).unwrap();53	f.write_all(&serialize(&parsed).expect("serialize"))54		.unwrap();55}
after · crates/jsonnet-evaluator/build.rs
1use bincode::serialize;2use jsonnet_parser::{3	parse, Expr, FieldMember, FieldName, LocExpr, Member, ObjBody, ParserSettings,4};5use jsonnet_stdlib::STDLIB_STR;6use std::{7	env,8	fs::File,9	io::Write,10	path::{Path, PathBuf},11	rc::Rc,12};13use structdump::CodegenResult;1415fn main() {16	let parsed = parse(17		STDLIB_STR,18		&ParserSettings {19			file_name: Rc::new(PathBuf::from("std.jsonnet")),20			loc_data: true,21		},22	)23	.expect("parse");2425	let parsed = if cfg!(feature = "faster") {26		let LocExpr(expr, location) = parsed;27		LocExpr(28			Rc::new(match Rc::try_unwrap(expr).unwrap() {29				Expr::Obj(ObjBody::MemberList(members)) => Expr::Obj(ObjBody::MemberList(30					members31						.into_iter()32						.filter(|p| {33							!matches!(34								p,35								Member::Field(FieldMember {36									name: FieldName::Fixed(name),37									..38								}) if **name == *"join" || **name == *"manifestJsonEx" || **name == *"escapeStringJson"39							)40						})41						.collect(),42				)),43				_ => panic!("std value should be object"),44			}),45			location,46		)47	} else {48		parsed49	};50	{51		let mut codegen = CodegenResult::default();52		let code = codegen.codegen(&parsed);5354		let out_dir = env::var("OUT_DIR").unwrap();55		let dest_path = Path::new(&out_dir).join("stdlib.rs");56		let mut f = File::create(&dest_path).unwrap();57		f.write_all(&code.as_bytes()).unwrap();58	}59	{60		let out_dir = env::var("OUT_DIR").unwrap();61		let dest_path = Path::new(&out_dir).join("stdlib.bincode");62		let mut f = File::create(&dest_path).unwrap();63		f.write_all(&serialize(&parsed).unwrap()).unwrap();64	}65}
modifiedcrates/jsonnet-evaluator/src/lib.rsdiffbeforeafterboth
--- a/crates/jsonnet-evaluator/src/lib.rs
+++ b/crates/jsonnet-evaluator/src/lib.rs
@@ -1,7 +1,11 @@
 #![feature(box_syntax, box_patterns)]
 #![feature(type_alias_impl_trait)]
 #![feature(debug_non_exhaustive)]
+#![feature(test)]
 #![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)]
+
+extern crate test;
+
 mod ctx;
 mod dynamic;
 mod error;
@@ -228,9 +232,24 @@
 		let std_path = Rc::new(PathBuf::from("std.jsonnet"));
 		self.run_in_state(|| {
 			use jsonnet_stdlib::STDLIB_STR;
-			if cfg!(feature = "serialized-stdlib") {
+			let mut parsed = false;
+			#[cfg(feature = "codegenerated-stdlib")]
+			if !parsed {
+				parsed = true;
+				#[allow(clippy::all)]
+				let stdlib = {
+					use jsonnet_parser::*;
+					include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))
+				};
+				self.add_parsed_file(std_path.clone(), STDLIB_STR.to_owned().into(), stdlib)
+					.unwrap();
+			}
+
+			#[cfg(feature = "serialized-stdlib")]
+			if !parsed {
+				parsed = true;
 				self.add_parsed_file(
-					std_path,
+					std_path.clone(),
 					STDLIB_STR.to_owned().into(),
 					bincode::deserialize(include_bytes!(concat!(
 						env!("OUT_DIR"),
@@ -239,7 +258,9 @@
 					.expect("deserialize stdlib"),
 				)
 				.unwrap();
-			} else {
+			}
+
+			if !parsed {
 				self.add_file(std_path, STDLIB_STR.to_owned().into())
 					.unwrap();
 			}
@@ -662,4 +683,43 @@
 		"#
 		);
 	}
+
+	use test::Bencher;
+
+	// This test is commented out by default, because of huge compilation slowdown
+	// #[bench]
+	// fn bench_codegen(b: &mut Bencher) {
+	// 	b.iter(|| {
+	// 		#[allow(clippy::all)]
+	// 		let stdlib = {
+	// 			use jsonnet_parser::*;
+	// 			include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))
+	// 		};
+	// 		stdlib
+	// 	})
+	// }
+
+	#[bench]
+	fn bench_serialize(b: &mut Bencher) {
+		b.iter(|| {
+			bincode::deserialize::<jsonnet_parser::LocExpr>(include_bytes!(concat!(
+				env!("OUT_DIR"),
+				"/stdlib.bincode"
+			)))
+			.expect("deserialize stdlib")
+		})
+	}
+
+	#[bench]
+	fn bench_parse(b: &mut Bencher) {
+		b.iter(|| {
+			jsonnet_parser::parse(
+				jsonnet_stdlib::STDLIB_STR,
+				&jsonnet_parser::ParserSettings {
+					loc_data: true,
+					file_name: Rc::new(PathBuf::from("std.jsonnet")),
+				},
+			)
+		})
+	}
 }