difftreelog
ci benchmark against stable release
in: master
4 files changed
flake.nixdiffbeforeafterboth29 cargo = rust;29 cargo = rust;30 };30 };31 };31 };32 jrsonnet-release = pkgs.callPackage ./nix/jrsonnet-release.nix {33 rustPlatform = pkgs.makeRustPlatform {34 rustc = rust;35 cargo = rust;36 };37 };323833 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {39 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {34 inherit go-jsonnet sjsonnet jsonnet jrsonnet;40 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;35 };41 };42 benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {43 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;44 quick = true;45 };46 benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {47 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;48 againstRelease = true;49 };50 benchmarks-quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {51 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;52 quick = true;53 againstRelease = true;54 };36 };55 };37 devShell = pkgs.mkShell {56 devShell = pkgs.mkShell {38 nativeBuildInputs = with pkgs;[57 nativeBuildInputs = with pkgs;[nix/benchmarks.nixdiffbeforeafterboth5, stdenv5, stdenv6, fetchFromGitHub6, fetchFromGitHub7, jrsonnet7, jrsonnet8, jrsonnet-release8, go-jsonnet9, go-jsonnet9, sjsonnet10, sjsonnet10, jsonnet11, jsonnet11, hyperfine12, hyperfine12, quick ? false13, quick ? false14, againstRelease ? false13}:15}:14let16let15 jsonnetBench = fetchFromGitHub {17 jsonnetBench = fetchFromGitHub {53 ${jsonnet-bundler}/bin/jb install55 ${jsonnet-bundler}/bin/jb install54 '';56 '';555758 # Removes outsiders from the output59 # Useful when comparing performance of different jrsonnet releases56 skipSlow = if quick then "slow benchmark" else "";60 skipSlow = if quick then "slow benchmark, but only quick requested" else "";57in61in58stdenv.mkDerivation {62stdenv.mkDerivation {59 name = "benchmarks";63 name = "benchmarks";67 jsonnet71 jsonnet687269 hyperfine73 hyperfine70 ];74 ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);717572 installPhase =76 installPhase =73 let77 let74 mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''78 mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''75 set -oux79 set -oux768077 echo >> $out81 echo >> $out78 echo "### ${name}" >> $out82 echo "### ${name}" >> $out79 echo >> $out83 echo >> $out80 ${if skipGo != "" then ''84 ${if skipGo != "" then ''81 echo "> Note: No results for Go, ${skipGo}" >> $out85 echo "> Note: No results for Go, ${skipGo}" >> $out82 echo >> $out86 echo >> $out83 '' else ""}87 '' else ""}84 ${if skipScala != "" then ''88 ${if skipScala != "" then ''85 echo "> Note: No results for Scala, ${skipScala}" >> $out89 echo "> Note: No results for Scala, ${skipScala}" >> $out86 echo >> $out90 echo >> $out87 '' else ""}91 '' else ""}88 ${if skipCpp != "" then ''92 ${if skipCpp != "" then ''89 echo "> Note: No results for C++, ${skipCpp}" >> $out93 echo "> Note: No results for C++, ${skipCpp}" >> $out90 echo >> $out94 echo >> $out91 '' else ""}95 '' else ""}92 echo "<details>" >> $out96 ${if !quick then ''97 echo "<details>" >> $out93 echo "<summary>Source</summary>" >> $out98 echo "<summary>Source</summary>" >> $out94 echo >> $out99 echo >> $out95 echo "\`\`\`jsonnet" >> $out100 echo "\`\`\`jsonnet" >> $out96 ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}101 ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}97 ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}102 ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}98 echo >> $out103 echo >> $out99 echo "\`\`\`" >> $out104 echo "\`\`\`" >> $out100 echo "</details>" >> $out105 echo "</details>" >> $out101 echo >> $out106 echo >> $out102 path=${path}107 '' else ""}108 path=${path}103 ${if pathIsGenerator then ''109 ${if pathIsGenerator then ''104 jrsonnet $path > generated.jsonnet110 jrsonnet $path > generated.jsonnet105 path=generated.jsonnet111 path=generated.jsonnet106 '' else ""}112 '' else ""}107 hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \113 hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \108 "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \114 "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \115 ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \109 ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \116 ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \110 ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \117 ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \111 ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }118 ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }112 cat result.md >> $out119 cat result.md >> $out113 '';120 '';114 in121 in115 ''122 ''116 touch $out123 touch $out117 cat ${./benchmarks.md} >> $out124 ${if !quick then ''125 cat ${./benchmarks.md} >> $out118 echo >> $out126 echo >> $out119127120 echo "<details>" >> $out128 echo "<details>" >> $out121 echo "<summary>Tested versions</summary>" >> $out129 echo "<summary>Tested versions</summary>" >> $out122 echo >> $out130 echo >> $out123 echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out131 echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out124 echo >> $out132 echo >> $out125 echo "\`\`\`" >> $out133 echo "\`\`\`" >> $out126 jrsonnet --help >> $out134 jrsonnet --help >> $out127 echo "\`\`\`" >> $out135 echo "\`\`\`" >> $out128 echo >> $out136 echo >> $out129 echo Go: $(go-jsonnet --version) >> $out137 echo Go: $(go-jsonnet --version) >> $out130 echo >> $out138 echo >> $out131 echo "\`\`\`" >> $out139 echo "\`\`\`" >> $out132 go-jsonnet --help >> $out140 go-jsonnet --help >> $out133 echo "\`\`\`" >> $out141 echo "\`\`\`" >> $out134 echo >> $out142 echo >> $out135 echo C++: $(jsonnet --version) >> $out143 echo C++: $(jsonnet --version) >> $out136 echo >> $out144 echo >> $out137 echo "\`\`\`" >> $out145 echo "\`\`\`" >> $out138 jsonnet --help >> $out146 jsonnet --help >> $out139 echo "\`\`\`" >> $out147 echo "\`\`\`" >> $out140 echo >> $out148 echo >> $out141 echo Scala: >> $out149 echo Scala: >> $out142 echo >> $out150 echo >> $out143 echo "\`\`\`" >> $out151 echo "\`\`\`" >> $out144 sjsonnet 2>> $out || true152 sjsonnet 2>> $out || true145 echo "\`\`\`" >> $out153 echo "\`\`\`" >> $out146 echo >> $out154 echo >> $out147 echo "</details>" >> $out155 echo "</details>" >> $out148 echo >> $out156 echo >> $out149157150 echo >> $out158 echo >> $out159 '' else ""}151 echo "## Real world" >> $out160 echo "## Real world" >> $out152 ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}161 ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}153 ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}162 ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}154163155 echo >> $out164 echo >> $out156 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out165 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out157 ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}}166 ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}}158 ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}}167 ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}}159 ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}168 ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}160 ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}169 ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}161170162 echo >> $out171 echo >> $out163 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out172 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out164 ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}}173 ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}}165 ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}}174 ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}}166 ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}}175 ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}}167 ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}}176 ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}}168 ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}177 ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}169 ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}}178 ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}}170 ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}}179 ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}}171 ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}}180 ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}}172 ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}}181 ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}}173182174 echo >> $out183 echo >> $out175 echo "## Benchmarks from Go jsonnet (builtins)" >> $out184 echo "## Benchmarks from Go jsonnet (builtins)" >> $out176 ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}}185 ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}}177 ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}}186 ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}}178 ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}}187 ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}}179 ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}}188 ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}}180 ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}}189 ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}}181 ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}}190 ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}}182 ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}}191 ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}}183 ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}}192 ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}}184 ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}}193 ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}}185 ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}}194 ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}}186 ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}195 ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}187 ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}}196 ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}}188 '';197 '';189}198}190199nix/jrsonnet-release.nixdiffbeforeafterboth334rustPlatform.buildRustPackage rec {4rustPlatform.buildRustPackage rec {5 pname = "jrsonnet";5 pname = "jrsonnet";6 version = "5f0f8de9f52f961e2ff162e0a3fd4ca20a275f1d";6 version = "d32fe45b8ed28fb39b5359a704922922368af1c0";778 src = fetchFromGitHub {8 src = fetchFromGitHub {9 owner = "CertainLach";9 owner = "CertainLach";10 repo = pname;10 repo = pname;11 rev = version;11 rev = version;12 hash = lib.fakeHash;12 hash = "sha256-R9Xt36bYS5upVDzt8hEifwmfocXpJbIKwvxkoJNEGVc=";13 };13 };14 cargoHash = "sha256-V+KGWeNlUnelofaGzufNPLGDyxazoFrjZ/n391VYYws=";141515 cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];16 cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];16 cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];17 cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];nix/jrsonnet.nixdiffbeforeafterboth20 pname = "jrsonnet";20 pname = "jrsonnet";21 version = "git";21 version = "git";222223 cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];23 cargoTestFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];24 cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];24 cargoBuildFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];252526 buildInputs = [ makeWrapper ];26 buildInputs = [ makeWrapper ];2727