git.delta.rocks / jrsonnet / refs/commits / d96a379e8bd8

--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,5 @@
 cache
 
 jsonnet-cpp
+jsonnet-sjsonnet
+benchmarks

--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,13 @@
 .PHONY: test build build-wasi
 jsonnet-cpp:
 	git clone https://github.com/google/jsonnet jsonnet-cpp
+.ONESHELL:
+jsonnet-sjsonnet:
+	mkdir jsonnet-sjsonnet && cd jsonnet-sjsonnet
+	wget https://github.com/databricks/sjsonnet/releases/download/0.2.4/sjsonnet.jar
+	echo "#!/bin/sh" > sjsonnet
+	echo "java -Xss400m -jar $(PWD)/jsonnet-sjsonnet/sjsonnet.jar $@" >> sjsonnet
+	chmod +x sjsonnet
 
 .ONESHELL:
 test-examples: jsonnet-cpp
@@ -21,3 +28,51 @@
 	RUSTFLAGS="-Zmutable-noalias=yes -C link-arg=-s" cargo build --release -p jrsonnet
 build-wasi:
 	cd ./bindings/ && cargo build --release -p jsonnet --target wasm32-wasi
+
+bench = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet $(1)" "jsonnet $(1)" "sjsonnet $(1)"
+bench-larger-stack = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet -s 200000 $(1)" "jsonnet -s 200000 $(1)" "sjsonnet $(1)"
+bench-no-scala = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet $(1)" "jsonnet $(1)"
+bench-no-go = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "jsonnet $(1)" "sjsonnet $(1)"
+
+.PHONY: benchmarks
+.ONESHELL:
+benchmarks: jsonnet-cpp jsonnet-sjsonnet
+	export PATH=$(PWD)/target/release/:$(PWD)/jsonnet-sjsonnet/:$(PATH)
+
+	mkdir -p $(PWD)/benchmarks
+
+	cd jsonnet-cpp/benchmarks/
+
+	jrsonnet -S gen_big_object.jsonnet > bench.05.gen.jsonnet
+
+	$(call bench,bench.01.jsonnet)
+	$(call bench,bench.02.jsonnet)
+	$(call bench,bench.03.jsonnet)
+	$(call bench,bench.04.jsonnet)
+	$(call bench,bench.05.gen.jsonnet)
+	# std.reverse not implemented in sjsonnet
+	$(call bench-no-scala,bench.06.jsonnet)
+	$(call bench-larger-stack,bench.07.jsonnet)
+	$(call bench,bench.08.jsonnet)
+
+	rm -f result.md
+	find . | /usr/bin/grep -oE "[a-z_0-9.]+.jsonnet$$" | grep -v gen_big_object | xargs -n1 -i sh -c 'printf "## {}\n\n" >> result.md && cat result.{}.md >> result.md && printf "\n" >> result.md'
+	cp result.md $(PWD)/benchmarks/benchmarks.md
+
+	cd ../perf_tests/
+
+	$(call bench,large_string_join.jsonnet)
+	golang overflows os stack on this benchmark
+	$(call bench-no-go,large_string_template.jsonnet)
+	$(call bench,realistic1.jsonnet)
+	$(call bench,realistic2.jsonnet)
+
+	rm -f result.md
+	find . | /usr/bin/grep -oE "[a-z_0-9.]+.jsonnet$$" | xargs -n1 -i sh -c 'printf "## {}\n\n" >> result.md && cat result.{}.md >> result.md && printf "\n" >> result.md'
+	cp result.md $(PWD)/benchmarks/perf_tests.md
+
+	cd $(PWD)/benchmarks/
+
+	rm -f result.md
+	printf "# Benchmark results\n\n" > result.md
+	cat benchmarks.md perf_tests.md >> result.md

2121
22Jrsonnet implements standard `libjsonnet.so` shared library, and should work as drop-in replacement for it22Jrsonnet implements standard `libjsonnet.so` shared library, and should work as drop-in replacement for it
2323
24WASM bindings are also available, Java bindings (Both JNI and WASM to .class compiled) are in progress
25
24See `./bindings/`26See `./bindings/`
2527
26## Benchmark28## Benchmark
27
28It is faster than golang implementation on mine kubernetes cluster configuration, which includes a lot of stuff, i.e prometheus operator
29
30Mine configuration contains two manifests, first one contains a lot of plain values, second one - a lot of computations
31
32- `gojsonnet` - golang impl
33- `../../../../jsonnet-rs/target/release/jrsonnet` - this repo impl
34
35```markdown
36# Manifest 1 (plain values)
37
38Benchmark #1: gojsonnet 02-prepare.jsonnet -J ../vendor/
39  Time (mean ± σ):     647.4 ms ±  12.8 ms    [User: 1.252 s, System: 0.069 s]
40  Range (min … max):   626.0 ms … 668.7 ms    10 runs
41
42Benchmark #2: ../../../../jsonnet-rs/target/release/jrsonnet 02-prepare.jsonnet -J ../vendor/
43  Time (mean ± σ):     163.7 ms ±   2.9 ms    [User: 138.5 ms, System: 24.6 ms]
44  Range (min … max):   161.4 ms … 174.0 ms    17 runs
4529
46Summary
47  '../../../../jsonnet-rs/target/release/jrsonnet 02-prepare.jsonnet -J ../vendor/' ran30This is fastest implementation of jsonnet, according to both official benchmarks
48    3.96 ± 0.10 times faster than 'gojsonnet 02-prepare.jsonnet -J ../vendor/'31and mine cluster configuration templating speed
4932
50# Manifest 2 (computations)33Official benchmark report are available [in this gist](https://gist.github.com/CertainLach/5770d7ad4836066f8e0bd91e823e451b), and updated sometimes. Here it tested against golang, C++, and scala impl. As you can see, it is a lot faster
5134
52Benchmark #1: gojsonnet 03-deploy.jsonnet -J ../vendor/35You can generate this report by calling `make benchmarks`, but it probally won't work in standard setup, you need to link golang jsonnet impl to gojsonnet, and c++ impl to jsonnet.
53  Time (mean ± σ):     14.387 s ±  0.473 s    [User: 27.657 s, System: 0.226 s]
54  Range (min … max):   13.865 s … 15.147 s    10 runs
5536
56Benchmark #2: ../../../../jsonnet-rs/target/release/jrsonnet 03-deploy.jsonnet -J ../vendor/37TODO: Create docker container for easier benchmarking and/or benchmark in CI
57  Time (mean ± σ):      2.373 s ±  0.083 s    [User: 2.304 s, System: 0.063 s]
58  Range (min … max):    2.286 s …  2.486 s    10 runs
5938
60Summary
61  '../../../../jsonnet-rs/target/release/jrsonnet 03-deploy.jsonnet -J ../vendor/' ran39Also, there is some ideas to improve performance even further, by i.e:
62    6.06 ± 0.29 times faster than 'gojsonnet 03-deploy.jsonnet -J ../vendor/'
63```
6440
65However, go impl still can be faster in some cases, because it curretly have more stdlib functions implemented in native, instead of stdlib (I.e `std.base64`)41- Mutating strings/arrays/objects instead of cloning on some operations (I.e concat), it should be possible by checking strong reference count, and mutating if there is only one reference
6642