From 337d066dc72d7eb700e6d7a29c097f3af4fbf5f5 Mon Sep 17 00:00:00 2001 From: Лач Date: Sat, 27 Jun 2020 09:26:56 +0000 Subject: [PATCH] perf: std ast codegen --- --- a/Cargo.lock +++ b/Cargo.lock @@ -142,16 +142,18 @@ "jsonnet-stdlib", "md5", "serde", + "structdump", ] [[package]] name = "jsonnet-parser" version = "0.1.0" dependencies = [ - "bincode", "jsonnet-stdlib", "peg", "serde", + "structdump", + "structdump-derive", "unescape", ] @@ -247,9 +249,9 @@ [[package]] name = "quote" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42934bc9c8ab0d3b273a16d8551c8f0fcff46be73276ca083ec2414c15c4ba5e" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ "proc-macro2", ] @@ -275,6 +277,23 @@ ] [[package]] +name = "structdump" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e16ec33a0342fdb67d13913b4ffae6527ebccfa04b5d7da174bdc7a31db29b8" + +[[package]] +name = "structdump-derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c337fdc077e02ccbfcc62af0090564a4af342975c3b7be09705efab90c1888" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "syn" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" --- a/crates/jsonnet-evaluator/Cargo.toml +++ b/crates/jsonnet-evaluator/Cargo.toml @@ -9,8 +9,10 @@ [features] default = ["serialized-stdlib", "faster"] # Serializes standard library AST instead of parsing them every run -serialized-stdlib = ["serde", "bincode"] -# Replace some standard library functions with faster implementations +serialized-stdlib = ["serde", "bincode", "jsonnet-parser/deserialize"] +# Same as above, but with generated code instead of serde. Reduces memory usage, but increases binary size and compilation time +codegenerated-stdlib = [] +# Replace some standard library functions with faster implementations (I.e manifestJsonEx) # Library works fine without this feature, but requires more memory and time for std function calls faster = [] @@ -21,16 +23,12 @@ indexmap = "1.4.0" md5 = "0.7.0" -[dependencies.serde] -version = "1.0.114" -optional = true +serde = { version = "1.0.114", optional = true } +bincode = { version = "1.3.1", optional = true } -[dependencies.bincode] -version = "1.3.1" -optional = true - [build-dependencies] -jsonnet-parser = { path = "../jsonnet-parser" } +jsonnet-parser = { path = "../jsonnet-parser", features = ["dump", "serialize", "deserialize"] } jsonnet-stdlib = { path = "../jsonnet-stdlib" } +structdump = "0.1.2" serde = "1.0.114" bincode = "1.3.1" --- a/crates/jsonnet-evaluator/README.md +++ b/crates/jsonnet-evaluator/README.md @@ -2,6 +2,35 @@ Interpreter for parsed jsonnet tree +## Standard library + +jsonnet stdlib is embedded into evaluator, but there is different modes for this: + +- `codegenerated-stdlib` + - generates source code for reproducing stdlib AST ([Example](https://gist.githubusercontent.com/CertainLach/7b3149df556f3406f5e9368aaa9f32ec/raw/0c80d8ab9aa7b9288c6219a2779cb2ab37287669/a.rs)) + - fastest on interpretation, slowest on compilation (it takes more than 5 minutes to optimize them by llvm) +- `serialized-stdlib` + - serializes standard library AST using serde + - slower than `codegenerated-stdlib` at runtime, but have no compilation speed penality +- none + - leaves only stdlib source code in binary, processing them same way as user supplied data + - slowest (as it involves parsing of standard library source code) + +Because of `codegenerated-stdlib` compilation slowdown, `serialized-stdlib` is used by default + +### Benchmark + +Can also be run via `cargo bench` + +```md +# codegenerated-stdlib +test tests::bench_codegen ... bench: 401,696 ns/iter (+/- 38,521) +# serialized-stdlib +test tests::bench_serialize ... bench: 1,763,999 ns/iter (+/- 76,211) +# none +test tests::bench_parse ... bench: 7,206,164 ns/iter (+/- 1,067,418) +``` + ## Intristics Some functions from stdlib are implemented as intristics --- a/crates/jsonnet-evaluator/build.rs +++ b/crates/jsonnet-evaluator/build.rs @@ -10,6 +10,7 @@ path::{Path, PathBuf}, rc::Rc, }; +use structdump::CodegenResult; fn main() { let parsed = parse( @@ -46,10 +47,19 @@ } else { parsed }; + { + let mut codegen = CodegenResult::default(); + let code = codegen.codegen(&parsed); - let out_dir = env::var("OUT_DIR").unwrap(); - let dest_path = Path::new(&out_dir).join("stdlib.bincode"); - let mut f = File::create(&dest_path).unwrap(); - f.write_all(&serialize(&parsed).expect("serialize")) - .unwrap(); + let out_dir = env::var("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("stdlib.rs"); + let mut f = File::create(&dest_path).unwrap(); + f.write_all(&code.as_bytes()).unwrap(); + } + { + let out_dir = env::var("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("stdlib.bincode"); + let mut f = File::create(&dest_path).unwrap(); + f.write_all(&serialize(&parsed).unwrap()).unwrap(); + } } --- a/crates/jsonnet-evaluator/src/lib.rs +++ b/crates/jsonnet-evaluator/src/lib.rs @@ -1,7 +1,11 @@ #![feature(box_syntax, box_patterns)] #![feature(type_alias_impl_trait)] #![feature(debug_non_exhaustive)] +#![feature(test)] #![allow(macro_expanded_macro_exports_accessed_by_absolute_paths)] + +extern crate test; + mod ctx; mod dynamic; mod error; @@ -228,9 +232,24 @@ let std_path = Rc::new(PathBuf::from("std.jsonnet")); self.run_in_state(|| { use jsonnet_stdlib::STDLIB_STR; - if cfg!(feature = "serialized-stdlib") { + let mut parsed = false; + #[cfg(feature = "codegenerated-stdlib")] + if !parsed { + parsed = true; + #[allow(clippy::all)] + let stdlib = { + use jsonnet_parser::*; + include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) + }; + self.add_parsed_file(std_path.clone(), STDLIB_STR.to_owned().into(), stdlib) + .unwrap(); + } + + #[cfg(feature = "serialized-stdlib")] + if !parsed { + parsed = true; self.add_parsed_file( - std_path, + std_path.clone(), STDLIB_STR.to_owned().into(), bincode::deserialize(include_bytes!(concat!( env!("OUT_DIR"), @@ -239,7 +258,9 @@ .expect("deserialize stdlib"), ) .unwrap(); - } else { + } + + if !parsed { self.add_file(std_path, STDLIB_STR.to_owned().into()) .unwrap(); } @@ -662,4 +683,43 @@ "# ); } + + use test::Bencher; + + // This test is commented out by default, because of huge compilation slowdown + // #[bench] + // fn bench_codegen(b: &mut Bencher) { + // b.iter(|| { + // #[allow(clippy::all)] + // let stdlib = { + // use jsonnet_parser::*; + // include!(concat!(env!("OUT_DIR"), "/stdlib.rs")) + // }; + // stdlib + // }) + // } + + #[bench] + fn bench_serialize(b: &mut Bencher) { + b.iter(|| { + bincode::deserialize::(include_bytes!(concat!( + env!("OUT_DIR"), + "/stdlib.bincode" + ))) + .expect("deserialize stdlib") + }) + } + + #[bench] + fn bench_parse(b: &mut Bencher) { + b.iter(|| { + jsonnet_parser::parse( + jsonnet_stdlib::STDLIB_STR, + &jsonnet_parser::ParserSettings { + loc_data: true, + file_name: Rc::new(PathBuf::from("std.jsonnet")), + }, + ) + }) + } } -- gitstuff