1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib; let17 jsonnetBench = fetchFromGitHub {18 rev = "v0.19.1";19 owner = "google";20 repo = "jsonnet";21 hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22 };23 goJsonnetBench =24 (fetchFromGitHub {25 owner = "google";26 repo = "go-jsonnet";27 rev = "v0.19.1";28 hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29 })30 + "/builtin-benchmarks";31 graalvmBench = fetchFromGitHub {32 owner = "oracle";33 repo = "graal";34 rev = "bc305df3fe587960f7635f0185571500e5988475";35 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36 };37 kubePrometheusBench = let38 src = fetchFromGitHub {39 owner = "prometheus-operator";40 repo = "kube-prometheus";41 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43 };44 in45 runCommand "kube-prometheus-vendor"46 {47 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48 outputHashMode = "recursive";49 buildInputs = [cacert];50 }51 ''52 mkdir -p $out53 cp -r ${src}/* $out/54 cd $out55 mkdir vendor56 ${jsonnet-bundler}/bin/jb install57 '';5859 60 61 skipSlow =62 if quick63 then "slow benchmark, but only quick requested"64 else "";65in66 stdenv.mkDerivation {67 name = "benchmarks";68 __impure = true;69 unpackPhase = "true";7071 buildInputs = [72 sjsonnet73 jsonnet74 rsjsonnet75 go-jsonnet7677 hyperfine78 ];7980 installPhase = let81 mkBench = {82 name,83 path,84 omitSource ? false,85 pathIsGenerator ? false,86 skipRustAlternative ? "",87 skipScala ? "",88 skipCpp ? "",89 skipGo ? "",90 vendor ? "",91 }: ''92 echo >> $out93 echo "### ${name}" >> $out94 echo >> $out95 ${optionalString (skipRustAlternative != "") ''96 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out97 echo >> $out98 ''}99 ${optionalString (skipGo != "") ''100 echo "> Note: No results for Go, ${skipGo}" >> $out101 echo >> $out102 ''}103 ${optionalString (skipScala != "") ''104 echo "> Note: No results for Scala, ${skipScala}" >> $out105 echo >> $out106 ''}107 ${optionalString (skipCpp != "") ''108 echo "> Note: No results for C++, ${skipCpp}" >> $out109 echo >> $out110 ''}111 ${optionalString (!quick && !omitSource) ''112 echo "<details>" >> $out113 echo "<summary>Source</summary>" >> $out114 echo >> $out115 echo "\`\`\`jsonnet" >> $out116 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}117 cat ${path} >> $out118 echo >> $out119 echo "\`\`\`" >> $out120 echo "</details>" >> $out121 echo >> $out122 ''}123 path=${path}124 ${optionalString pathIsGenerator ''125 go-jsonnet $path > generated.jsonnet126 path=generated.jsonnet127 ''}128 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \129 ${concatStringsSep " " (forEach jrsonnetVariants (130 variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${131 if variant.name != ""132 then " (${variant.name})"133 else ""134 }\""135 ))} \136 ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \137 ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \138 ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \139 ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}140 cat result.md >> $out141 '';142 in ''143 set -oux144145 temp=$(mktemp -d)146 cd $temp147148 touch $out149 ${optionalString (!quick) ''150 cat ${./benchmarks.md} >> $out151 echo >> $out152153 echo "<details>" >> $out154 echo "<summary>Tested versions</summary>" >> $out155 echo >> $out156 echo Go: $(go-jsonnet --version) >> $out157 echo >> $out158 echo "\`\`\`" >> $out159 go-jsonnet --help >> $out160 echo "\`\`\`" >> $out161 echo >> $out162 echo C++: $(jsonnet --version) >> $out163 echo >> $out164 echo "\`\`\`" >> $out165 jsonnet --help >> $out166 echo "\`\`\`" >> $out167 echo >> $out168 echo Scala: >> $out169 echo >> $out170 echo "\`\`\`" >> $out171 sjsonnet 2>> $out || true172 echo "\`\`\`" >> $out173 echo >> $out174 echo "</details>" >> $out175 echo >> $out176 echo "Rust (alternative):" >> $out177 echo >> $out178 echo "\`\`\`" >> $out179 rsjsonnet --help 2>> $out || true180 echo "\`\`\`" >> $out181 echo >> $out182 echo "</details>" >> $out183 echo >> $out184185 echo >> $out186 ''}187 echo "## Real world" >> $out188 ${mkBench {189 name = "Graalvm CI";190 path = "${graalvmBench}/ci.jsonnet";191 skipCpp = "takes longer than a hour";192 skipGo = skipSlow;193 skipScala = skipSlow;194 }}195 ${mkBench {196 name = "Kube-prometheus manifests";197 vendor = "${kubePrometheusBench}/vendor";198 path = "${kubePrometheusBench}/example.jsonnet";199 skipCpp = skipSlow;200 skipGo = skipSlow;201 skipScala = skipSlow;202 }}203204 echo >> $out205 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out206 ${mkBench {207 name = "Large string join";208 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";209 skipScala = skipSlow;210 }}211 ${mkBench {212 name = "Large string template";213 omitSource = true;214 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";215 skipGo = "fails with os stack size exhausion";216 skipCpp = skipSlow;217 skipScala = skipSlow;218 }}219 ${mkBench {220 name = "Realistic 1";221 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";222 skipGo = skipSlow;223 skipCpp = skipSlow;224 skipScala = skipSlow;225 }}226 ${mkBench {227 name = "Realistic 2";228 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229 skipGo = skipSlow;230 skipCpp = skipSlow;231 skipScala = skipSlow;232 }}233234 echo >> $out235 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out236 ${mkBench {237 name = "Tail call";238 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";239 skipScala = skipSlow;240 }}241 ${mkBench {242 name = "Inheritance recursion";243 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";244 skipCpp = skipSlow;245 skipGo = skipSlow;246 }}247 ${mkBench {248 name = "Simple recursive call";249 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";250 skipScala = skipSlow;251 skipGo = skipSlow;252 }}253 ${mkBench {254 name = "Foldl string concat";255 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";256 skipCpp = skipSlow;257 skipScala = skipSlow;258 }}259 ${mkBench {260 name = "Array sorts";261 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";262 skipCpp = skipSlow;263 264 skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";265 }}266 ${mkBench {267 name = "Lazy array";268 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";269 skipGo = skipSlow;270 skipScala = skipSlow;271 }}272 ${mkBench {273 name = "Inheritance function recursion";274 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";275 skipCpp = skipSlow;276 skipScala = skipSlow;277 }}278 ${mkBench {279 name = "String strips";280 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";281 skipCpp = skipSlow;282 skipScala = skipSlow;283 }}284 ${mkBench {285 name = "Big object";286 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287 pathIsGenerator = true;288 skipScala = skipSlow;289 }}290291 echo >> $out292 echo "## Benchmarks from Go jsonnet (builtins)" >> $out293 ${mkBench {294 name = "std.base64";295 path = "${goJsonnetBench}/base64.jsonnet";296 skipRustAlternative = skipSlow;297 skipCpp = skipSlow;298 skipScala = skipSlow;299 }}300 ${mkBench {301 name = "std.base64Decode";302 path = "${goJsonnetBench}/base64Decode.jsonnet";303 skipRustAlternative = skipSlow;304 skipCpp = skipSlow;305 skipScala = skipSlow;306 }}307 ${mkBench {308 name = "std.base64DecodeBytes";309 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";310 skipRustAlternative = skipSlow;311 skipCpp = skipSlow;312 skipGo = skipSlow;313 skipScala = skipSlow;314 }}315 ${mkBench {316 name = "std.base64 (byte array)";317 path = "${goJsonnetBench}/base64_byte_array.jsonnet";318 skipRustAlternative = skipSlow;319 skipCpp = skipSlow;320 skipGo = skipSlow;321 skipScala = skipSlow;322 }}323 ${mkBench {324 name = "std.foldl";325 path = "${goJsonnetBench}/foldl.jsonnet";326 skipScala = skipSlow;327 }}328 ${mkBench {329 name = "std.manifestJsonEx";330 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";331 skipScala = skipSlow;332 skipCpp = skipSlow;333 }}334 ${mkBench {335 name = "std.manifestTomlEx";336 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";337 skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";338 skipCpp = skipSlow;339 }}340 ${mkBench {341 name = "std.parseInt";342 path = "${goJsonnetBench}/parseInt.jsonnet";343 skipScala = skipSlow;344 skipCpp = skipSlow;345 }}346 ${mkBench {347 name = "std.reverse";348 path = "${goJsonnetBench}/reverse.jsonnet";349 skipCpp = skipSlow;350 skipGo = skipSlow;351 }}352 ${mkBench {353 name = "std.substr";354 path = "${goJsonnetBench}/substr.jsonnet";355 skipScala = skipSlow;356 }}357 ${mkBench {358 name = "Comparsion for array";359 path = "${goJsonnetBench}/comparison.jsonnet";360 skipScala = "array comparsion is not implemented";361 skipCpp = skipSlow;362 }}363 ${mkBench {364 name = "Comparsion for primitives";365 path = "${goJsonnetBench}/comparison2.jsonnet";366 skipRustAlternative = skipSlow;367 skipCpp = "can't run: uses up to 192GB of RAM";368 skipGo = skipSlow;369 skipScala = skipSlow;370 }}371 '';372 }