difftreelog
test run cpp test suite with criterion
in: master
5 files changed
tests/.gitignorediffbeforeafterboth--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,2 +1,5 @@
/cpp_test_suite
/go_testdata
+/go_builtin_benchmarks
+/cpp_benchmarks
+/cpp_perf_tests
tests/Cargo.tomldiffbeforeafterboth--- a/tests/Cargo.toml
+++ b/tests/Cargo.toml
@@ -29,5 +29,5 @@
bench = false
[[bench]]
-name = "prim_comparsion"
+name = "cpp_test_suite"
harness = false
tests/benches/cpp_test_suite.rsdiffbeforeafterboth--- /dev/null
+++ b/tests/benches/cpp_test_suite.rs
@@ -0,0 +1,56 @@
+use std::{collections::HashMap, fs::read_dir, hint::black_box, path::Path};
+
+use criterion::{Criterion, criterion_group, criterion_main};
+use jrsonnet_evaluator::{
+ FileImportResolver, State, apply_tla, manifest::JsonFormat, trace::PathResolver,
+};
+
+fn bench_entry(c: &mut Criterion, path: &Path) {
+ c.bench_function(
+ path.file_name()
+ .expect("file path")
+ .to_str()
+ .expect("name is utf-8"),
+ |b| {
+ let mut s = State::builder();
+
+ s.context_initializer(jrsonnet_stdlib::ContextInitializer::new(
+ PathResolver::Absolute,
+ ))
+ .import_resolver(FileImportResolver::new(vec![]));
+
+ let s = s.build();
+ let _s = s.enter();
+
+ b.iter(|| {
+ let imported = s.import(path).expect("evaluated");
+ let res = apply_tla(&HashMap::new(), imported).expect("tla applied");
+ black_box(res.manifest(JsonFormat::cli(3)))
+ });
+ },
+ );
+}
+fn criterion_benchmark(c: &mut Criterion) {
+ for entry in read_dir("go_builtin_benchmarks").expect("dir exists") {
+ let entry = entry.expect("entry is valid");
+ assert!(entry.metadata().expect("entry is valid").is_file());
+ bench_entry(c, &entry.path());
+ }
+ for entry in read_dir("cpp_perf_tests").expect("dir exists") {
+ let entry = entry.expect("entry is valid");
+ assert!(entry.metadata().expect("entry is valid").is_file());
+ bench_entry(c, &entry.path());
+ }
+ for entry in read_dir("cpp_benchmarks").expect("dir exists") {
+ let entry = entry.expect("entry is valid");
+ // Skip .gitignore
+ if entry.path().extension().is_none() {
+ continue;
+ }
+ assert!(entry.metadata().expect("entry is valid").is_file());
+ bench_entry(c, &entry.path());
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
tests/benches/prim_comparsion.rsdiffbeforeafterboth--- a/tests/benches/prim_comparsion.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-use std::hint::black_box;
-
-use criterion::{Criterion, criterion_group, criterion_main};
-use jrsonnet_evaluator::{State, manifest::JsonFormat, trace::PathResolver};
-
-fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("prim_comparison", |b| {
- let mut s = State::builder();
-
- s.context_initializer(jrsonnet_stdlib::ContextInitializer::new(
- PathResolver::Absolute,
- ));
-
- let s = s.build();
- let _s = s.enter();
-
- b.iter(|| {
- black_box(
- s.evaluate_snippet(
- "snippet",
- "([ i < j for i in std.range(1, 1000) for j in std.range(1, 1000)])",
- )
- .expect("evaluated")
- .manifest(JsonFormat::cli(3))
- .expect("manifested"),
- )
- });
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
xtask/src/main.rsdiffbeforeafterboth1use anyhow::Result;2use clap::Parser;3use xshell::{Shell, cmd};45mod sourcegen;67#[derive(Parser)]8enum Opts {9 /// Generate files for rowan parser10 Sourcegen,11 /// Profile file execution12 Profile {13 #[arg(long, default_value = "true")]14 hyperfine: bool,15 #[arg(long)]16 callgrind: bool,17 #[arg(long)]18 cachegrind: bool,19 #[arg(long, default_value = env!("TARGET_PLATFORM"))]20 target: String,21 args: Vec<String>,22 },23 /// Run all lints enforced by this repo24 Lint {25 /// Also fix found issues when possible.26 #[arg(long)]27 fix: bool,28 },29 /// Build and run test file from `bindings/c`30 TestCBindings {31 #[arg(long, default_value = env!("TARGET_PLATFORM"))]32 target: String,33 /// Which bindings file to build and run34 #[arg(long, default_value = "libjsonnet_test_file")]35 test_file: String,36 args: Vec<String>,37 },38 /// Update C++/Golang golden testsuites from git39 UpdateTestsuites,40}4142fn main() -> Result<()> {43 let sh = Shell::new()?;44 match Opts::parse() {45 Opts::Sourcegen => sourcegen::generate_ungrammar(),46 Opts::Profile {47 hyperfine,48 callgrind,49 cachegrind,50 args,51 target,52 } => {53 let out = sh.create_temp_dir()?;5455 cmd!(sh, "cargo build --target={target} --profile releasedebug").run()?;56 let built = format!("./target/{target}/releasedebug/jrsonnet");57 let bench_cmd = format!("{built} {}", args.join(" "));58 if hyperfine {59 cmd!(sh, "hyperfine {bench_cmd}").run()?;60 }61 if callgrind {62 let args = args.clone();63 let mut callgrind_out = out.path().to_owned();64 callgrind_out.push("callgrind.out.1");65 cmd!(sh, "valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes --callgrind-out-file={callgrind_out} {built} {args...}").run()?;66 cmd!(sh, "kcachegrind {callgrind_out}").run()?;67 }68 if cachegrind {69 let mut cachegrind_out = out.path().to_owned();70 cachegrind_out.push("cachegrind.out.1");71 cmd!(72 sh,73 "valgrind --tool=cachegrind --cachegrind-out-file={cachegrind_out} {built} {args...}"74 )75 .run()?;76 cmd!(sh, "kcachegrind {cachegrind_out}").run()?;77 }7879 Ok(())80 }81 Opts::Lint { fix } => {82 let fmt_check = if fix { None } else { Some("--check") };83 cmd!(sh, "cargo fmt {fmt_check...}").run()?;84 Ok(())85 }86 Opts::TestCBindings {87 target,88 test_file,89 args,90 } => {91 cmd!(92 sh,93 "cargo build -p libjsonnet --target={target} --release --no-default-features --features=interop-common,interop-threading"94 )95 .run()?;96 let built = format!("./target/{target}/release/libjsonnet.a");97 let c_bindings = "./bindings/c/";98 cmd!(sh, "cp {built} {c_bindings}").run()?;99 sh.change_dir(c_bindings);100101 // TODO: Pass target to gcc?102 cmd!(sh, "gcc -c {test_file}.c").run()?;103 cmd!(sh, "gcc -o {test_file} -lc -lm {test_file}.o libjsonnet.a").run()?;104 let sh = Shell::new()?;105106 cmd!(sh, "{c_bindings}{test_file} {args...}").run()?;107108 Ok(())109 }110 Opts::UpdateTestsuites => {111 let _pushd = sh.push_dir("tests");112 let git_dir = sh.create_temp_dir()?;113 let git_dir_path = git_dir.path();114 cmd!(115 sh,116 "git clone https://github.com/google/jsonnet.git --depth=1 {git_dir_path}/jsonnet"117 )118 .run()?;119 cmd!(120 sh,121 "git clone https://github.com/google/go-jsonnet.git --depth=1 {git_dir_path}/go-jsonnet"122 )123 .run()?;124 sh.remove_path("cpp_test_suite")?;125 sh.remove_path("go_testdata")?;126 cmd!(sh, "mv {git_dir_path}/jsonnet/test_suite cpp_test_suite").run()?;127 cmd!(sh, "mv {git_dir_path}/go-jsonnet/testdata go_testdata").run()?;128129 Ok(())130 }131 }132}1use anyhow::Result;2use clap::Parser;3use xshell::{Shell, cmd};45mod sourcegen;67#[derive(Parser)]8enum Opts {9 /// Generate files for rowan parser10 Sourcegen,11 /// Profile file execution12 Profile {13 #[arg(long, default_value = "true")]14 hyperfine: bool,15 #[arg(long)]16 callgrind: bool,17 #[arg(long)]18 cachegrind: bool,19 #[arg(long, default_value = env!("TARGET_PLATFORM"))]20 target: String,21 args: Vec<String>,22 },23 /// Run all lints enforced by this repo24 Lint {25 /// Also fix found issues when possible.26 #[arg(long)]27 fix: bool,28 },29 /// Build and run test file from `bindings/c`30 TestCBindings {31 #[arg(long, default_value = env!("TARGET_PLATFORM"))]32 target: String,33 /// Which bindings file to build and run34 #[arg(long, default_value = "libjsonnet_test_file")]35 test_file: String,36 args: Vec<String>,37 },38 /// Update C++/Golang golden testsuites from git39 UpdateTestsuites,40}4142fn main() -> Result<()> {43 let sh = Shell::new()?;44 match Opts::parse() {45 Opts::Sourcegen => sourcegen::generate_ungrammar(),46 Opts::Profile {47 hyperfine,48 callgrind,49 cachegrind,50 args,51 target,52 } => {53 let out = sh.create_temp_dir()?;5455 cmd!(sh, "cargo build --target={target} --profile releasedebug").run()?;56 let built = format!("./target/{target}/releasedebug/jrsonnet");57 let bench_cmd = format!("{built} {}", args.join(" "));58 if hyperfine {59 cmd!(sh, "hyperfine {bench_cmd}").run()?;60 }61 if callgrind {62 let args = args.clone();63 let mut callgrind_out = out.path().to_owned();64 callgrind_out.push("callgrind.out.1");65 cmd!(sh, "valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes --callgrind-out-file={callgrind_out} {built} {args...}").run()?;66 cmd!(sh, "kcachegrind {callgrind_out}").run()?;67 }68 if cachegrind {69 let mut cachegrind_out = out.path().to_owned();70 cachegrind_out.push("cachegrind.out.1");71 cmd!(72 sh,73 "valgrind --tool=cachegrind --cachegrind-out-file={cachegrind_out} {built} {args...}"74 )75 .run()?;76 cmd!(sh, "kcachegrind {cachegrind_out}").run()?;77 }7879 Ok(())80 }81 Opts::Lint { fix } => {82 let fmt_check = if fix { None } else { Some("--check") };83 cmd!(sh, "cargo fmt {fmt_check...}").run()?;84 Ok(())85 }86 Opts::TestCBindings {87 target,88 test_file,89 args,90 } => {91 cmd!(92 sh,93 "cargo build -p libjsonnet --target={target} --release --no-default-features --features=interop-common,interop-threading"94 )95 .run()?;96 let built = format!("./target/{target}/release/libjsonnet.a");97 let c_bindings = "./bindings/c/";98 cmd!(sh, "cp {built} {c_bindings}").run()?;99 sh.change_dir(c_bindings);100101 // TODO: Pass target to gcc?102 cmd!(sh, "gcc -c {test_file}.c").run()?;103 cmd!(sh, "gcc -o {test_file} -lc -lm {test_file}.o libjsonnet.a").run()?;104 let sh = Shell::new()?;105106 cmd!(sh, "{c_bindings}{test_file} {args...}").run()?;107108 Ok(())109 }110 Opts::UpdateTestsuites => {111 let _pushd = sh.push_dir("tests");112 let git_dir = sh.create_temp_dir()?;113 let git_dir_path = git_dir.path();114 cmd!(115 sh,116 "git clone https://github.com/google/jsonnet.git --depth=1 {git_dir_path}/jsonnet"117 )118 .run()?;119 cmd!(120 sh,121 "git clone https://github.com/google/go-jsonnet.git --depth=1 {git_dir_path}/go-jsonnet"122 )123 .run()?;124 sh.remove_path("cpp_test_suite")?;125 sh.remove_path("go_testdata")?;126 sh.remove_path("cpp_perf_tests")?;127 sh.remove_path("cpp_benchmarks")?;128 sh.remove_path("go_builtin_benchmarks")?;129 cmd!(sh, "mv {git_dir_path}/jsonnet/test_suite cpp_test_suite").run()?;130 cmd!(sh, "mv {git_dir_path}/go-jsonnet/testdata go_testdata").run()?;131 cmd!(sh, "mv {git_dir_path}/jsonnet/perf_tests cpp_perf_tests").run()?;132 cmd!(sh, "mv {git_dir_path}/jsonnet/benchmarks cpp_benchmarks").run()?;133 cmd!(134 sh,135 "mv {git_dir_path}/go-jsonnet/builtin-benchmarks go_builtin_benchmarks"136 )137 .run()?;138139 Ok(())140 }141 }142}