git.delta.rocks / jrsonnet / refs/commits / d96a379e8bd8

difftreelog

chore add benchmark results

Лач2020-07-20parent: #14f17a7.patch.diff
in: master

3 files changed

modified.gitignorediffbeforeafterboth
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,5 @@
 cache
 
 jsonnet-cpp
+jsonnet-sjsonnet
+benchmarks
modifiedMakefilediffbeforeafterboth
before · Makefile
1.PHONY: test build build-wasi2jsonnet-cpp:3	git clone https://github.com/google/jsonnet jsonnet-cpp45.ONESHELL:6test-examples: jsonnet-cpp7	export JSONNET_BIN="$(PWD)/target/release/jrsonnet"8	export EXAMPLES_DIR="$(PWD)/jsonnet-cpp/examples/"9	cd ./jsonnet-cpp/examples/10	./check.sh1112.ONESHELL:13test-tests: jsonnet-cpp14	export JSONNET_BIN="$(PWD)/target/release/jrsonnet --trace-format=go --thread-stack-size=96"15	cd ./jsonnet-cpp/test_suite/16	./run_tests.sh1718test:19	cargo test20build:21	RUSTFLAGS="-Zmutable-noalias=yes -C link-arg=-s" cargo build --release -p jrsonnet22build-wasi:23	cd ./bindings/ && cargo build --release -p jsonnet --target wasm32-wasi
after · Makefile
1.PHONY: test build build-wasi2jsonnet-cpp:3	git clone https://github.com/google/jsonnet jsonnet-cpp4.ONESHELL:5jsonnet-sjsonnet:6	mkdir jsonnet-sjsonnet && cd jsonnet-sjsonnet7	wget https://github.com/databricks/sjsonnet/releases/download/0.2.4/sjsonnet.jar8	echo "#!/bin/sh" > sjsonnet9	echo "java -Xss400m -jar $(PWD)/jsonnet-sjsonnet/sjsonnet.jar $@" >> sjsonnet10	chmod +x sjsonnet1112.ONESHELL:13test-examples: jsonnet-cpp14	export JSONNET_BIN="$(PWD)/target/release/jrsonnet"15	export EXAMPLES_DIR="$(PWD)/jsonnet-cpp/examples/"16	cd ./jsonnet-cpp/examples/17	./check.sh1819.ONESHELL:20test-tests: jsonnet-cpp21	export JSONNET_BIN="$(PWD)/target/release/jrsonnet --trace-format=go --thread-stack-size=96"22	cd ./jsonnet-cpp/test_suite/23	./run_tests.sh2425test:26	cargo test27build:28	RUSTFLAGS="-Zmutable-noalias=yes -C link-arg=-s" cargo build --release -p jrsonnet29build-wasi:30	cd ./bindings/ && cargo build --release -p jsonnet --target wasm32-wasi3132bench = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet $(1)" "jsonnet $(1)" "sjsonnet $(1)"33bench-larger-stack = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet -s 200000 $(1)" "jsonnet -s 200000 $(1)" "sjsonnet $(1)"34bench-no-scala = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "gojsonnet $(1)" "jsonnet $(1)"35bench-no-go = hyperfine --export-markdown "result.$(1).md" "jrsonnet $(1)" "jsonnet $(1)" "sjsonnet $(1)"3637.PHONY: benchmarks38.ONESHELL:39benchmarks: jsonnet-cpp jsonnet-sjsonnet40	export PATH=$(PWD)/target/release/:$(PWD)/jsonnet-sjsonnet/:$(PATH)4142	mkdir -p $(PWD)/benchmarks4344	cd jsonnet-cpp/benchmarks/4546	jrsonnet -S gen_big_object.jsonnet > bench.05.gen.jsonnet4748	$(call bench,bench.01.jsonnet)49	$(call bench,bench.02.jsonnet)50	$(call bench,bench.03.jsonnet)51	$(call bench,bench.04.jsonnet)52	$(call bench,bench.05.gen.jsonnet)53	# std.reverse not implemented in sjsonnet54	$(call bench-no-scala,bench.06.jsonnet)55	$(call bench-larger-stack,bench.07.jsonnet)56	$(call bench,bench.08.jsonnet)5758	rm -f result.md59	find . | /usr/bin/grep -oE "[a-z_0-9.]+.jsonnet$$" | grep -v gen_big_object | xargs -n1 -i sh -c 'printf "## {}\n\n" >> result.md && cat result.{}.md >> result.md && printf "\n" >> result.md'60	cp result.md $(PWD)/benchmarks/benchmarks.md6162	cd ../perf_tests/6364	$(call bench,large_string_join.jsonnet)65	golang overflows os stack on this benchmark66	$(call bench-no-go,large_string_template.jsonnet)67	$(call bench,realistic1.jsonnet)68	$(call bench,realistic2.jsonnet)6970	rm -f result.md71	find . | /usr/bin/grep -oE "[a-z_0-9.]+.jsonnet$$" | xargs -n1 -i sh -c 'printf "## {}\n\n" >> result.md && cat result.{}.md >> result.md && printf "\n" >> result.md'72	cp result.md $(PWD)/benchmarks/perf_tests.md7374	cd $(PWD)/benchmarks/7576	rm -f result.md77	printf "# Benchmark results\n\n" > result.md78	cat benchmarks.md perf_tests.md >> result.md
modifiedREADME.mddiffbeforeafterboth
--- a/README.md
+++ b/README.md
@@ -21,45 +21,21 @@
 
 Jrsonnet implements standard `libjsonnet.so` shared library, and should work as drop-in replacement for it
 
+WASM bindings are also available, Java bindings (Both JNI and WASM to .class compiled) are in progress
+
 See `./bindings/`
 
 ## Benchmark
 
-It is faster than golang implementation on mine kubernetes cluster configuration, which includes a lot of stuff, i.e prometheus operator
+This is fastest implementation of jsonnet, according to both official benchmarks
+and mine cluster configuration templating speed
 
-Mine configuration contains two manifests, first one contains a lot of plain values, second one - a lot of computations
+Official benchmark report are available [in this gist](https://gist.github.com/CertainLach/5770d7ad4836066f8e0bd91e823e451b), and updated sometimes. Here it tested against golang, C++, and scala impl. As you can see, it is a lot faster
 
-- `gojsonnet` - golang impl
-- `../../../../jsonnet-rs/target/release/jrsonnet` - this repo impl
+You can generate this report by calling `make benchmarks`, but it probally won't work in standard setup, you need to link golang jsonnet impl to gojsonnet, and c++ impl to jsonnet.
 
-```markdown
-# Manifest 1 (plain values)
+TODO: Create docker container for easier benchmarking and/or benchmark in CI
 
-Benchmark #1: gojsonnet 02-prepare.jsonnet -J ../vendor/
-  Time (mean ± σ):     647.4 ms ±  12.8 ms    [User: 1.252 s, System: 0.069 s]
-  Range (min … max):   626.0 ms … 668.7 ms    10 runs
+Also, there is some ideas to improve performance even further, by i.e:
 
-Benchmark #2: ../../../../jsonnet-rs/target/release/jrsonnet 02-prepare.jsonnet -J ../vendor/
-  Time (mean ± σ):     163.7 ms ±   2.9 ms    [User: 138.5 ms, System: 24.6 ms]
-  Range (min … max):   161.4 ms … 174.0 ms    17 runs
-
-Summary
-  '../../../../jsonnet-rs/target/release/jrsonnet 02-prepare.jsonnet -J ../vendor/' ran
-    3.96 ± 0.10 times faster than 'gojsonnet 02-prepare.jsonnet -J ../vendor/'
-
-# Manifest 2 (computations)
-
-Benchmark #1: gojsonnet 03-deploy.jsonnet -J ../vendor/
-  Time (mean ± σ):     14.387 s ±  0.473 s    [User: 27.657 s, System: 0.226 s]
-  Range (min … max):   13.865 s … 15.147 s    10 runs
-
-Benchmark #2: ../../../../jsonnet-rs/target/release/jrsonnet 03-deploy.jsonnet -J ../vendor/
-  Time (mean ± σ):      2.373 s ±  0.083 s    [User: 2.304 s, System: 0.063 s]
-  Range (min … max):    2.286 s …  2.486 s    10 runs
-
-Summary
-  '../../../../jsonnet-rs/target/release/jrsonnet 03-deploy.jsonnet -J ../vendor/' ran
-    6.06 ± 0.29 times faster than 'gojsonnet 03-deploy.jsonnet -J ../vendor/'
-```
-
-However, go impl still can be faster in some cases, because it curretly have more stdlib functions implemented in native, instead of stdlib (I.e `std.base64`)
+- Mutating strings/arrays/objects instead of cloning on some operations (I.e concat), it should be possible by checking strong reference count, and mutating if there is only one reference