1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 cpp-jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib;17let18 inherit (cpp-jsonnet) jsonnetBench;19 inherit (go-jsonnet) goJsonnetBench;20 graalvmBench = fetchFromGitHub {21 owner = "oracle";22 repo = "graal";23 rev = "bc305df3fe587960f7635f0185571500e5988475";24 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25 };26 kubePrometheusBench =27 let28 src = fetchFromGitHub {29 owner = "prometheus-operator";30 repo = "kube-prometheus";31 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33 };34 in35 runCommand "kube-prometheus-vendor"36 {37 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38 outputHashMode = "recursive";39 buildInputs = [ cacert ];40 }41 ''42 mkdir -p $out43 cp -r ${src}/* $out/44 cd $out45 chmod u+w jsonnetfile.lock.json46 mkdir vendor47 ${jsonnet-bundler}/bin/jb install48 '';4950 51 52 skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55 name = "benchmarks";56 __impure = true;57 unpackPhase = "true";5859 buildInputs = [60 sjsonnet61 cpp-jsonnet62 rsjsonnet63 go-jsonnet6465 hyperfine66 ];6768 installPhase =69 let70 mkBench =71 {72 name,73 path,74 omitSource ? false,75 pathIsGenerator ? false,76 skipRustAlternative ? "",77 skipScala ? "",78 skipCpp ? "",79 skipGo ? "",80 vendor ? "",81 }:82 ''83 echo >> $out84 echo "=== ${name}" >> $out85 echo >> $out86 ${optionalString (skipRustAlternative != "") ''87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88 echo >> $out89 ''}90 ${optionalString (skipGo != "") ''91 echo "> Note: No results for Go, ${skipGo}" >> $out92 echo >> $out93 ''}94 ${optionalString (skipScala != "") ''95 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96 echo >> $out97 ''}98 ${optionalString (skipCpp != "") ''99 echo "> Note: No results for C++, ${skipCpp}" >> $out100 echo >> $out101 ''}102 ${optionalString (!quick && !omitSource) ''103 echo ".Source" >> $out104 echo "[%collapsible]" >> $out105 echo "====" >> $out106 echo "[source,jsonnet]" >> $out107 echo "----" >> $out108 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109 cat ${path} >> $out110 echo >> $out111 echo "----" >> $out112 echo "====" >> $out113 echo >> $out114 ''}115 path=${path}116 ${optionalString pathIsGenerator ''117 go-jsonnet $path > generated.jsonnet118 path=generated.jsonnet119 ''}120 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121 ${122 concatStringsSep " " (123 forEach jrsonnetVariants (124 variant:125 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126 if variant.name != "" then " (${variant.name})" else ""127 }\""128 )129 )130 } \131 ${132 optionalString (skipRustAlternative == "")133 "\"rsjsonnet $path${134 optionalString (vendor != "") " -J ${vendor}"135 }\" -n \"Rust (alternative, rsjsonnet)\""136 } \137 ${138 optionalString (skipGo == "")139 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140 } \141 ${142 optionalString (skipScala == "")143 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144 } \145 ${146 optionalString (skipScala == "")147 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""148 } \149 ${optionalString (skipCpp == "")150 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""151 }152 cat result.adoc >> $out153 '';154 in155 ''156 set -oux157 ulimit -s unlimited158159 temp=$(mktemp -d)160 cd $temp161162 touch $out163 ${optionalString (!quick) ''164 cat ${./benchmarks.adoc} >> $out165 echo >> $out166167 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out168 echo >> $out169170 echo ".Tested versions" >> $out171 echo "[%collapsible]" >> $out172 echo "====" >> $out173 echo "* Go: $(go-jsonnet --version)" >> $out174 echo "* C++: $(jsonnet --version)" >> $out175 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out176 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out177 ${concatStringsSep "\n" (forEach jrsonnetVariants (variant:178 "echo \"* Rust${if variant.name != "" then " (${variant.name})" else ""}: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"179 ))}180 echo "====" >> $out181 echo >> $out182 ''}183 echo "== Real world" >> $out184 ${mkBench {185 name = "Graalvm CI";186 path = "${graalvmBench}/ci.jsonnet";187 omitSource = true;188 skipCpp = "takes longer than a hour";189 skipGo = skipSlow;190 skipScala = skipSlow;191 }}192 ${mkBench {193 name = "Kube-prometheus manifests";194 vendor = "${kubePrometheusBench}/vendor";195 path = "${kubePrometheusBench}/example.jsonnet";196 omitSource = true;197 skipCpp = "too slow, takes hours, skews results";198 skipGo = skipSlow;199 skipScala = skipSlow;200 }}201202 echo >> $out203 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out204 ${mkBench {205 name = "Large string join";206 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";207 skipScala = skipSlow;208 }}209 ${mkBench {210 name = "Large string template";211 omitSource = true;212 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";213 skipGo = "fails with os stack size exhausion";214 skipCpp = "too slow, takes hours, skews results";215 skipScala = skipSlow;216 }}217 ${mkBench {218 name = "Realistic 1";219 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";220 skipGo = skipSlow;221 skipCpp = "too slow, takes hours, skews results";222 skipScala = skipSlow;223 }}224 ${mkBench {225 name = "Realistic 2";226 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";227 skipGo = skipSlow;228 skipCpp = "too slow, takes hours, skews results";229 skipScala = skipSlow;230 }}231232 echo >> $out233 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out234 ${mkBench {235 name = "Tail call";236 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";237 skipScala = skipSlow;238 }}239 ${mkBench {240 name = "Inheritance recursion";241 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242 skipCpp = skipSlow;243 skipGo = skipSlow;244 }}245 ${mkBench {246 name = "Simple recursive call";247 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248 skipScala = skipSlow;249 skipGo = skipSlow;250 }}251 ${mkBench {252 name = "Foldl string concat";253 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";254 skipCpp = skipSlow;255 skipScala = skipSlow;256 }}257 ${mkBench {258 name = "Array sorts";259 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";260 skipCpp = skipSlow;261 }}262 ${mkBench {263 name = "Lazy array";264 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";265 skipGo = skipSlow;266 }}267 ${mkBench {268 name = "Inheritance function recursion";269 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";270 skipCpp = skipSlow;271 skipScala = skipSlow;272 }}273 ${mkBench {274 name = "String strips";275 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";276 skipCpp = "too slow, takes hours, skews results";277 skipScala = skipSlow;278 }}279 ${mkBench {280 name = "Big object";281 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";282 pathIsGenerator = true;283 skipScala = skipSlow;284 }}285286 echo >> $out287 echo "== Benchmarks from Go jsonnet (builtins)" >> $out288 ${mkBench {289 name = "std.base64";290 path = "${goJsonnetBench}/base64.jsonnet";291 skipRustAlternative = skipSlow;292 skipCpp = "too slow, takes hours, skews results";293 skipScala = skipSlow;294 }}295 ${mkBench {296 name = "std.base64Decode";297 path = "${goJsonnetBench}/base64Decode.jsonnet";298 skipRustAlternative = skipSlow;299 skipCpp = skipSlow;300 skipScala = skipSlow;301 }}302 ${mkBench {303 name = "std.base64DecodeBytes";304 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";305 skipRustAlternative = skipSlow;306 skipCpp = skipSlow;307 skipGo = skipSlow;308 skipScala = skipSlow;309 }}310 ${mkBench {311 name = "std.base64 (byte array)";312 path = "${goJsonnetBench}/base64_byte_array.jsonnet";313 skipRustAlternative = skipSlow;314 skipCpp = skipSlow;315 skipGo = skipSlow;316 skipScala = skipSlow;317 }}318 ${mkBench {319 name = "std.foldl";320 path = "${goJsonnetBench}/foldl.jsonnet";321 skipScala = skipSlow;322 }}323 ${mkBench {324 name = "std.manifestJsonEx";325 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";326 skipScala = skipSlow;327 skipCpp = skipSlow;328 }}329 ${mkBench {330 name = "std.manifestTomlEx";331 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";332 skipCpp = skipSlow;333 }}334 ${mkBench {335 name = "std.parseInt";336 path = "${goJsonnetBench}/parseInt.jsonnet";337 skipScala = skipSlow;338 skipCpp = skipSlow;339 }}340 ${mkBench {341 name = "std.reverse";342 path = "${goJsonnetBench}/reverse.jsonnet";343 skipCpp = skipSlow;344 skipGo = skipSlow;345 }}346 ${mkBench {347 name = "std.substr";348 path = "${goJsonnetBench}/substr.jsonnet";349 skipScala = skipSlow;350 }}351 ${mkBench {352 name = "Comparsion for array";353 path = "${goJsonnetBench}/comparison.jsonnet";354 skipCpp = "too slow, takes hours, skews results";355 }}356 ${mkBench {357 name = "Comparsion for primitives";358 path = "${goJsonnetBench}/comparison2.jsonnet";359 skipRustAlternative = skipSlow;360 skipCpp = "can't run: uses up to 192GB of RAM";361 skipGo = skipSlow;362 skipScala = skipSlow;363 }}364 '';365}