--- /dev/null +++ b/nix/benchmarks.md @@ -0,0 +1,7 @@ +# Benchmarks + +There are multiple implementations of jsonnet implemented in different languages: Rust (this repo), [Go](https://github.com/google/go-jsonnet/), [Scala](https://github.com/databricks/sjsonnet), [C++](https://github.com/google/jsonnet), [Haskell](https://github.com/moleike/haskell-jsonnet). + +For simplicity, I will call these implementations by the language of their implementation. + +Unfortunately, I haven't managed to measure performance of Haskell implementation, because I wasn't able to build it, and there is no binaries published anywhere, so this implementation is omitted from the following benchmarks --- a/nix/benchmarks.nix +++ b/nix/benchmarks.nix @@ -1,9 +1,189 @@ -{ stdenv, jrsonnet, go-jsonnet, sjsonnet, jsonnet, hyperfine }: +{ lib +, runCommand +, jsonnet-bundler +, cacert +, stdenv +, fetchFromGitHub +, jrsonnet +, go-jsonnet +, sjsonnet +, jsonnet +, hyperfine +, quick ? false +}: +let + jsonnetBench = fetchFromGitHub { + rev = "v0.19.1"; + owner = "google"; + repo = "jsonnet"; + hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU="; + }; + goJsonnetBench = (fetchFromGitHub { + owner = "google"; + repo = "go-jsonnet"; + rev = "v0.19.1"; + hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI="; + }) + "/builtin-benchmarks"; + graalvmBench = fetchFromGitHub { + owner = "oracle"; + repo = "graal"; + rev = "bc305df3fe587960f7635f0185571500e5988475"; + hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc="; + }; + kubePrometheusBench = + let + src = fetchFromGitHub { + owner = "prometheus-operator"; + repo = "kube-prometheus"; + rev = "6a570e3154eac26e32da61d784fbe626da4804df"; + hash = "sha256-s6BK8KQiSjql2w6R+8m5pvPPAcKW+SKjQwqxZHjimFw="; + }; + in + runCommand "kube-prometheus-vendor" + { + outputHash = "sha256-R60RI/7FQPOHisnzANb34m9WPj5D9FeMVoGOjB19zl8="; + outputHashMode = "recursive"; + buildInputs = [ cacert ]; + } + '' + mkdir -p $out + cp -r ${src}/* $out/ + cd $out + mkdir vendor + ${jsonnet-bundler}/bin/jb install + ''; + skipSlow = if quick then "slow benchmark" else ""; +in stdenv.mkDerivation { name = "benchmarks"; __impure = true; unpackPhase = "true"; - installPhase = "touch $out"; + buildInputs = [ + jrsonnet + go-jsonnet + sjsonnet + jsonnet + + hyperfine + ]; + + installPhase = + let + mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: '' + set -oux + + echo >> $out + echo "### ${name}" >> $out + echo >> $out + ${if skipGo != "" then '' + echo "> Note: No results for Go, ${skipGo}" >> $out + echo >> $out + '' else ""} + ${if skipScala != "" then '' + echo "> Note: No results for Scala, ${skipScala}" >> $out + echo >> $out + '' else ""} + ${if skipCpp != "" then '' + echo "> Note: No results for C++, ${skipCpp}" >> $out + echo >> $out + '' else ""} + echo "
" >> $out + echo "Source" >> $out + echo >> $out + echo "\`\`\`jsonnet" >> $out + ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""} + cat ${if omitSource then "// Omitted: too large" else path} >> $out + echo >> $out + echo "\`\`\`" >> $out + echo "
" >> $out + echo >> $out + path=${path} + ${if pathIsGenerator then '' + jrsonnet $path > generated.jsonnet + path=generated.jsonnet + '' else ""} + hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \ + "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \ + ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \ + ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \ + ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" } + cat result.md >> $out + ''; + in + '' + touch $out + cat ${./benchmarks.md} >> $out + echo >> $out + + echo "
" >> $out + echo "Tested versions" >> $out + echo >> $out + echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out + echo >> $out + echo "\`\`\`" >> $out + jrsonnet --help >> $out + echo "\`\`\`" >> $out + echo >> $out + echo Go: $(go-jsonnet --version) >> $out + echo >> $out + echo "\`\`\`" >> $out + go-jsonnet --help >> $out + echo "\`\`\`" >> $out + echo >> $out + echo C++: $(jsonnet --version) >> $out + echo >> $out + echo "\`\`\`" >> $out + jsonnet --help >> $out + echo "\`\`\`" >> $out + echo >> $out + echo Scala: >> $out + echo >> $out + echo "\`\`\`" >> $out + sjsonnet 2>> $out || true + echo "\`\`\`" >> $out + echo >> $out + echo "
" >> $out + echo >> $out + + echo >> $out + echo "## Real world" >> $out + ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}} + + echo >> $out + echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out + ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}} + ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}} + ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}} + ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}} + + echo >> $out + echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out + ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}} + ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}} + ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}} + ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}} + ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}} + ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}} + ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}} + + echo >> $out + echo "## Benchmarks from Go jsonnet (builtins)" >> $out + ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}} + ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}} + ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}} + ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}} + ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}} + ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}} + ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}} + ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}} + ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}} + ''; } --- a/nix/go-jsonnet.nix +++ b/nix/go-jsonnet.nix @@ -1,4 +1,4 @@ -{ lib, buildGo119Module, fetchFromGitHub }: +{ lib, buildGo119Module, fetchFromGitHub, makeWrapper }: buildGo119Module rec { pname = "go-jsonnet"; @@ -11,11 +11,13 @@ rev = "${version}"; hash = "sha256-J+bGdbYo2Ch3ORYD57yJA4jiPiS8IYASZ6kJHhyaqeU="; }; + vendorHash = "sha256-j1fTOUpLx34TgzW94A/BctLrg9XoTtb3cBizhVJoEEI="; - vendorHash = "sha256-j1fTOUpLx34TgzW94A/BctLrg9XoTtb3cBizhVJoEEI="; + buildInputs = [ makeWrapper ]; postInstall = '' mv $out/bin/jsonnet $out/bin/go-jsonnet + wrapProgram $out/bin/go-jsonnet --add-flags "--max-stack 200000" ''; doCheck = false; --- a/nix/jrsonnet.nix +++ b/nix/jrsonnet.nix @@ -1,22 +1,33 @@ -{ lib, fetchFromGitHub, rustPlatform }: +{ lib, fetchFromGitHub, rustPlatform, runCommand, makeWrapper }: let - jsonnet = fetchFromGitHub { - rev = "v${version}"; - owner = "google"; - repo = "jsonnet"; - hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU="; + filteredSrc = builtins.path { + name = "jrsonnet-src-filtered"; + filter = path: type: !(builtins.baseNameOf path == "flake.nix" || builtins.baseNameOf path == "nix"); + path = ../.; }; + + # for some reason, filteredSrc hash still depends on nix directory contents + # Moving it into a CA store drops leftover references + src = runCommand "jrsonnet-src" + { + __contentAddressed = true; + } "cp -r '${filteredSrc}' $out"; in rustPlatform.buildRustPackage rec { + inherit src; pname = "jrsonnet"; version = "git"; - src = ./..; + cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ]; + cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ]; + + buildInputs = [ makeWrapper ]; - cargoTestFlags = [ "--package=jrsonnet" ]; - cargoBuildFlags = [ "--package=jrsonnet" ]; + postInstall = '' + wrapProgram $out/bin/jrsonnet --add-flags "--max-stack=200000 --os-stack=200000" + ''; cargoLock = { lockFile = ../Cargo.lock; --- a/nix/jsonnet.nix +++ b/nix/jsonnet.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, jekyll, fetchFromGitHub }: +{ stdenv, lib, jekyll, fetchFromGitHub, makeWrapper }: stdenv.mkDerivation rec { pname = "jsonnet"; @@ -15,8 +15,11 @@ "jsonnet" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' mkdir -p $out/bin cp jsonnet $out/bin/jsonnet + wrapProgram $out/bin/jsonnet --add-flags "--max-stack 200000" ''; }