1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 cpp-jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib; let17 jsonnetBench = fetchFromGitHub {18 rev = "v0.19.1";19 owner = "google";20 repo = "jsonnet";21 hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22 };23 goJsonnetBench =24 (fetchFromGitHub {25 owner = "google";26 repo = "go-jsonnet";27 rev = "v0.19.1";28 hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29 })30 + "/builtin-benchmarks";31 graalvmBench = fetchFromGitHub {32 owner = "oracle";33 repo = "graal";34 rev = "bc305df3fe587960f7635f0185571500e5988475";35 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36 };37 kubePrometheusBench = let38 src = fetchFromGitHub {39 owner = "prometheus-operator";40 repo = "kube-prometheus";41 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43 };44 in45 runCommand "kube-prometheus-vendor"46 {47 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48 outputHashMode = "recursive";49 buildInputs = [cacert];50 }51 ''52 mkdir -p $out53 cp -r ${src}/* $out/54 cd $out55 chmod u+w jsonnetfile.lock.json56 mkdir vendor57 ${jsonnet-bundler}/bin/jb install58 '';5960 61 62 skipSlow =63 if quick64 then "slow benchmark, but only quick requested"65 else "";66in67 stdenv.mkDerivation {68 name = "benchmarks";69 __impure = true;70 unpackPhase = "true";7172 buildInputs = [73 sjsonnet74 cpp-jsonnet75 rsjsonnet76 go-jsonnet7778 hyperfine79 ];8081 installPhase = let82 mkBench = {83 name,84 path,85 omitSource ? false,86 pathIsGenerator ? false,87 skipRustAlternative ? "",88 skipScala ? "",89 skipCpp ? "",90 skipGo ? "",91 vendor ? "",92 }: ''93 echo >> $out94 echo "### ${name}" >> $out95 echo >> $out96 ${optionalString (skipRustAlternative != "") ''97 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out98 echo >> $out99 ''}100 ${optionalString (skipGo != "") ''101 echo "> Note: No results for Go, ${skipGo}" >> $out102 echo >> $out103 ''}104 ${optionalString (skipScala != "") ''105 echo "> Note: No results for Scala, ${skipScala}" >> $out106 echo >> $out107 ''}108 ${optionalString (skipCpp != "") ''109 echo "> Note: No results for C++, ${skipCpp}" >> $out110 echo >> $out111 ''}112 ${optionalString (!quick && !omitSource) ''113 echo "<details>" >> $out114 echo "<summary>Source</summary>" >> $out115 echo >> $out116 echo "\`\`\`jsonnet" >> $out117 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}118 cat ${path} >> $out119 echo >> $out120 echo "\`\`\`" >> $out121 echo "</details>" >> $out122 echo >> $out123 ''}124 path=${path}125 ${optionalString pathIsGenerator ''126 go-jsonnet $path > generated.jsonnet127 path=generated.jsonnet128 ''}129 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \130 ${concatStringsSep " " (forEach jrsonnetVariants (131 variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${132 if variant.name != ""133 then " (${variant.name})"134 else ""135 }\""136 ))} \137 ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \138 ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \139 ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \140 ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}141 cat result.adoc >> $out142 '';143 in ''144 set -oux145146 temp=$(mktemp -d)147 cd $temp148149 touch $out150 ${optionalString (!quick) ''151 cat ${./benchmarks.md} >> $out152 echo >> $out153154 echo "<details>" >> $out155 echo "<summary>Tested versions</summary>" >> $out156 echo >> $out157 echo Go: $(go-jsonnet --version) >> $out158 echo >> $out159 echo "\`\`\`" >> $out160 go-jsonnet --help >> $out161 echo "\`\`\`" >> $out162 echo >> $out163 echo C++: $(jsonnet --version) >> $out164 echo >> $out165 echo "\`\`\`" >> $out166 jsonnet --help >> $out167 echo "\`\`\`" >> $out168 echo >> $out169 echo Scala: >> $out170 echo >> $out171 echo "\`\`\`" >> $out172 sjsonnet 2>> $out || true173 echo "\`\`\`" >> $out174 echo >> $out175 echo "Rust (alternative):" >> $out176 echo >> $out177 echo "\`\`\`" >> $out178 rsjsonnet --help 2>> $out || true179 echo "\`\`\`" >> $out180 echo >> $out181 echo "</details>" >> $out182 echo >> $out183184 echo >> $out185 ''}186 echo "## Real world" >> $out187 ${mkBench {188 name = "Graalvm CI";189 path = "${graalvmBench}/ci.jsonnet";190 skipCpp = "takes longer than a hour";191 skipGo = skipSlow;192 skipScala = skipSlow;193 }}194 ${mkBench {195 name = "Kube-prometheus manifests";196 vendor = "${kubePrometheusBench}/vendor";197 path = "${kubePrometheusBench}/example.jsonnet";198 skipCpp = skipSlow;199 skipGo = skipSlow;200 skipScala = skipSlow;201 }}202203 echo >> $out204 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out205 ${mkBench {206 name = "Large string join";207 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";208 skipScala = skipSlow;209 }}210 ${mkBench {211 name = "Large string template";212 omitSource = true;213 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";214 skipGo = "fails with os stack size exhausion";215 skipCpp = skipSlow;216 skipScala = skipSlow;217 }}218 ${mkBench {219 name = "Realistic 1";220 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";221 skipGo = skipSlow;222 skipCpp = skipSlow;223 skipScala = skipSlow;224 }}225 ${mkBench {226 name = "Realistic 2";227 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";228 skipGo = skipSlow;229 skipCpp = skipSlow;230 skipScala = skipSlow;231 }}232233 echo >> $out234 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out235 ${mkBench {236 name = "Tail call";237 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";238 skipScala = skipSlow;239 }}240 ${mkBench {241 name = "Inheritance recursion";242 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";243 skipCpp = skipSlow;244 skipGo = skipSlow;245 }}246 ${mkBench {247 name = "Simple recursive call";248 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";249 skipScala = skipSlow;250 skipGo = skipSlow;251 }}252 ${mkBench {253 name = "Foldl string concat";254 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";255 skipCpp = skipSlow;256 skipScala = skipSlow;257 }}258 ${mkBench {259 name = "Array sorts";260 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";261 skipCpp = skipSlow;262 263 skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";264 }}265 ${mkBench {266 name = "Lazy array";267 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";268 skipGo = skipSlow;269 skipScala = skipSlow;270 }}271 ${mkBench {272 name = "Inheritance function recursion";273 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";274 skipCpp = skipSlow;275 skipScala = skipSlow;276 }}277 ${mkBench {278 name = "String strips";279 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";280 skipCpp = skipSlow;281 skipScala = skipSlow;282 }}283 ${mkBench {284 name = "Big object";285 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";286 pathIsGenerator = true;287 skipScala = skipSlow;288 }}289290 echo >> $out291 echo "## Benchmarks from Go jsonnet (builtins)" >> $out292 ${mkBench {293 name = "std.base64";294 path = "${goJsonnetBench}/base64.jsonnet";295 skipRustAlternative = skipSlow;296 skipCpp = skipSlow;297 skipScala = skipSlow;298 }}299 ${mkBench {300 name = "std.base64Decode";301 path = "${goJsonnetBench}/base64Decode.jsonnet";302 skipRustAlternative = skipSlow;303 skipCpp = skipSlow;304 skipScala = skipSlow;305 }}306 ${mkBench {307 name = "std.base64DecodeBytes";308 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";309 skipRustAlternative = skipSlow;310 skipCpp = skipSlow;311 skipGo = skipSlow;312 skipScala = skipSlow;313 }}314 ${mkBench {315 name = "std.base64 (byte array)";316 path = "${goJsonnetBench}/base64_byte_array.jsonnet";317 skipRustAlternative = skipSlow;318 skipCpp = skipSlow;319 skipGo = skipSlow;320 skipScala = skipSlow;321 }}322 ${mkBench {323 name = "std.foldl";324 path = "${goJsonnetBench}/foldl.jsonnet";325 skipScala = skipSlow;326 }}327 ${mkBench {328 name = "std.manifestJsonEx";329 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";330 skipScala = skipSlow;331 skipCpp = skipSlow;332 }}333 ${mkBench {334 name = "std.manifestTomlEx";335 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";336 skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";337 skipCpp = skipSlow;338 }}339 ${mkBench {340 name = "std.parseInt";341 path = "${goJsonnetBench}/parseInt.jsonnet";342 skipScala = skipSlow;343 skipCpp = skipSlow;344 }}345 ${mkBench {346 name = "std.reverse";347 path = "${goJsonnetBench}/reverse.jsonnet";348 skipCpp = skipSlow;349 skipGo = skipSlow;350 }}351 ${mkBench {352 name = "std.substr";353 path = "${goJsonnetBench}/substr.jsonnet";354 skipScala = skipSlow;355 }}356 ${mkBench {357 name = "Comparsion for array";358 path = "${goJsonnetBench}/comparison.jsonnet";359 skipScala = "array comparsion is not implemented";360 skipCpp = skipSlow;361 }}362 ${mkBench {363 name = "Comparsion for primitives";364 path = "${goJsonnetBench}/comparison2.jsonnet";365 skipRustAlternative = skipSlow;366 skipCpp = "can't run: uses up to 192GB of RAM";367 skipGo = skipSlow;368 skipScala = skipSlow;369 }}370 '';371 }