1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 cpp-jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib;17let18 inherit (cpp-jsonnet) jsonnetBench;19 inherit (go-jsonnet) goJsonnetBench;20 graalvmBench = fetchFromGitHub {21 owner = "oracle";22 repo = "graal";23 rev = "bc305df3fe587960f7635f0185571500e5988475";24 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25 };26 kubePrometheusBench =27 let28 src = fetchFromGitHub {29 owner = "prometheus-operator";30 repo = "kube-prometheus";31 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33 };34 in35 runCommand "kube-prometheus-vendor"36 {37 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38 outputHashMode = "recursive";39 buildInputs = [ cacert ];40 }41 ''42 mkdir -p $out43 cp -r ${src}/* $out/44 cd $out45 chmod u+w jsonnetfile.lock.json46 mkdir vendor47 ${jsonnet-bundler}/bin/jb install48 '';4950 51 52 skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55 name = "benchmarks";56 __impure = true;57 unpackPhase = "true";5859 buildInputs = [60 sjsonnet61 cpp-jsonnet62 rsjsonnet63 go-jsonnet6465 hyperfine66 ];6768 installPhase =69 let70 mkBench =71 {72 name,73 path,74 omitSource ? false,75 pathIsGenerator ? false,76 skipRustAlternative ? "",77 skipScala ? "",78 skipCpp ? "",79 skipGo ? "",80 vendor ? "",81 }:82 ''83 echo >> $out84 echo "### ${name}" >> $out85 echo >> $out86 ${optionalString (skipRustAlternative != "") ''87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88 echo >> $out89 ''}90 ${optionalString (skipGo != "") ''91 echo "> Note: No results for Go, ${skipGo}" >> $out92 echo >> $out93 ''}94 ${optionalString (skipScala != "") ''95 echo "> Note: No results for Scala/Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96 echo >> $out97 ''}98 ${optionalString (skipCpp != "") ''99 echo "> Note: No results for C++, ${skipCpp}" >> $out100 echo >> $out101 ''}102 ${optionalString (!quick && !omitSource) ''103 echo "<details>" >> $out104 echo "<summary>Source</summary>" >> $out105 echo >> $out106 echo "\`\`\`jsonnet" >> $out107 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}108 cat ${path} >> $out109 echo >> $out110 echo "\`\`\`" >> $out111 echo "</details>" >> $out112 echo >> $out113 ''}114 path=${path}115 ${optionalString pathIsGenerator ''116 go-jsonnet $path > generated.jsonnet117 path=generated.jsonnet118 ''}119 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \120 ${121 concatStringsSep " " (122 forEach jrsonnetVariants (123 variant:124 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${125 if variant.name != "" then " (${variant.name})" else ""126 }\""127 )128 )129 } \130 ${131 optionalString (skipRustAlternative == "")132 "\"rsjsonnet $path${133 optionalString (vendor != "") " -J ${vendor}"134 }\" -n \"Rust (alternative, rsjsonnet)\""135 } \136 ${137 optionalString (skipGo == "")138 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""139 } \140 ${141 optionalString (skipScala == "")142 "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""143 } \144 ${145 optionalString (skipScala == "")146 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""147 } \148 ${149 optionalString (skipScala == "")150 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""151 } \152 ${optionalString (skipCpp == "")153 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""154 }155 cat result.adoc >> $out156 '';157 in158 ''159 set -oux160161 temp=$(mktemp -d)162 cd $temp163164 touch $out165 ${optionalString (!quick) ''166 cat ${./benchmarks.md} >> $out167 echo >> $out168169 echo "<details>" >> $out170 echo "<summary>Tested versions</summary>" >> $out171 echo >> $out172 echo Go: $(go-jsonnet --version) >> $out173 echo >> $out174 echo "\`\`\`" >> $out175 go-jsonnet --help >> $out176 echo "\`\`\`" >> $out177 echo >> $out178 echo C++: $(jsonnet --version) >> $out179 echo >> $out180 echo "\`\`\`" >> $out181 jsonnet --help >> $out182 echo "\`\`\`" >> $out183 echo >> $out184 echo Scala: >> $out185 echo >> $out186 echo "\`\`\`" >> $out187 sjsonnet 2>> $out || true188 echo "\`\`\`" >> $out189 echo >> $out190 echo "Scala (native):" >> $out191 echo >> $out192 echo "\`\`\`" >> $out193 sjsonnet-native 2>> $out || true194 echo "\`\`\`" >> $out195 echo >> $out196 echo "Scala (GraalVM):" >> $out197 echo >> $out198 echo "\`\`\`" >> $out199 sjsonnet-graalvm 2>> $out || true200 echo "\`\`\`" >> $out201 echo >> $out202 echo "Rust (alternative):" >> $out203 echo >> $out204 echo "\`\`\`" >> $out205 rsjsonnet --help 2>> $out || true206 echo "\`\`\`" >> $out207 echo >> $out208 echo "</details>" >> $out209 echo >> $out210211 echo >> $out212 ''}213 echo "## Real world" >> $out214 ${mkBench {215 name = "Graalvm CI";216 path = "${graalvmBench}/ci.jsonnet";217 omitSource = true;218 skipCpp = "takes longer than a hour";219 skipGo = skipSlow;220 skipScala = skipSlow;221 }}222 ${mkBench {223 name = "Kube-prometheus manifests";224 vendor = "${kubePrometheusBench}/vendor";225 path = "${kubePrometheusBench}/example.jsonnet";226 omitSource = true;227 skipCpp = "too slow, takes hours, skews results";228 skipGo = skipSlow;229 skipScala = skipSlow;230 }}231232 echo >> $out233 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out234 ${mkBench {235 name = "Large string join";236 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";237 skipScala = skipSlow;238 }}239 ${mkBench {240 name = "Large string template";241 omitSource = true;242 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";243 skipGo = "fails with os stack size exhausion";244 skipCpp = "too slow, takes hours, skews results";245 skipScala = skipSlow;246 }}247 ${mkBench {248 name = "Realistic 1";249 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";250 skipGo = skipSlow;251 skipCpp = "too slow, takes hours, skews results";252 skipScala = skipSlow;253 }}254 ${mkBench {255 name = "Realistic 2";256 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";257 skipGo = skipSlow;258 skipCpp = "too slow, takes hours, skews results";259 skipScala = skipSlow;260 }}261262 echo >> $out263 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out264 ${mkBench {265 name = "Tail call";266 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";267 skipScala = skipSlow;268 }}269 ${mkBench {270 name = "Inheritance recursion";271 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";272 skipCpp = skipSlow;273 skipGo = skipSlow;274 }}275 ${mkBench {276 name = "Simple recursive call";277 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";278 skipScala = skipSlow;279 skipGo = skipSlow;280 }}281 ${mkBench {282 name = "Foldl string concat";283 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";284 skipCpp = skipSlow;285 skipScala = skipSlow;286 }}287 ${mkBench {288 name = "Array sorts";289 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";290 skipCpp = skipSlow;291 }}292 ${mkBench {293 name = "Lazy array";294 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";295 skipGo = skipSlow;296 skipScala = skipSlow;297 }}298 ${mkBench {299 name = "Inheritance function recursion";300 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";301 skipCpp = skipSlow;302 skipScala = skipSlow;303 }}304 ${mkBench {305 name = "String strips";306 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";307 skipCpp = "too slow, takes hours, skews results";308 skipScala = skipSlow;309 }}310 ${mkBench {311 name = "Big object";312 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";313 pathIsGenerator = true;314 skipScala = skipSlow;315 }}316317 echo >> $out318 echo "## Benchmarks from Go jsonnet (builtins)" >> $out319 ${mkBench {320 name = "std.base64";321 path = "${goJsonnetBench}/base64.jsonnet";322 skipRustAlternative = skipSlow;323 skipCpp = "too slow, takes hours, skews results";324 skipScala = skipSlow;325 }}326 ${mkBench {327 name = "std.base64Decode";328 path = "${goJsonnetBench}/base64Decode.jsonnet";329 skipRustAlternative = skipSlow;330 skipCpp = skipSlow;331 skipScala = skipSlow;332 }}333 ${mkBench {334 name = "std.base64DecodeBytes";335 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";336 skipRustAlternative = skipSlow;337 skipCpp = skipSlow;338 skipGo = skipSlow;339 skipScala = skipSlow;340 }}341 ${mkBench {342 name = "std.base64 (byte array)";343 path = "${goJsonnetBench}/base64_byte_array.jsonnet";344 skipRustAlternative = skipSlow;345 skipCpp = skipSlow;346 skipGo = skipSlow;347 skipScala = skipSlow;348 }}349 ${mkBench {350 name = "std.foldl";351 path = "${goJsonnetBench}/foldl.jsonnet";352 skipScala = skipSlow;353 }}354 ${mkBench {355 name = "std.manifestJsonEx";356 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";357 skipScala = skipSlow;358 skipCpp = skipSlow;359 }}360 ${mkBench {361 name = "std.manifestTomlEx";362 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";363 skipCpp = skipSlow;364 }}365 ${mkBench {366 name = "std.parseInt";367 path = "${goJsonnetBench}/parseInt.jsonnet";368 skipScala = skipSlow;369 skipCpp = skipSlow;370 }}371 ${mkBench {372 name = "std.reverse";373 path = "${goJsonnetBench}/reverse.jsonnet";374 skipCpp = skipSlow;375 skipGo = skipSlow;376 }}377 ${mkBench {378 name = "std.substr";379 path = "${goJsonnetBench}/substr.jsonnet";380 skipScala = skipSlow;381 }}382 ${mkBench {383 name = "Comparsion for array";384 path = "${goJsonnetBench}/comparison.jsonnet";385 skipCpp = "too slow, takes hours, skews results";386 }}387 ${mkBench {388 name = "Comparsion for primitives";389 path = "${goJsonnetBench}/comparison2.jsonnet";390 skipRustAlternative = skipSlow;391 skipCpp = "can't run: uses up to 192GB of RAM";392 skipGo = skipSlow;393 skipScala = skipSlow;394 }}395 '';396}