1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib; let17 jsonnetBench = fetchFromGitHub {18 rev = "v0.19.1";19 owner = "google";20 repo = "jsonnet";21 hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22 };23 goJsonnetBench =24 (fetchFromGitHub {25 owner = "google";26 repo = "go-jsonnet";27 rev = "v0.19.1";28 hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29 })30 + "/builtin-benchmarks";31 graalvmBench = fetchFromGitHub {32 owner = "oracle";33 repo = "graal";34 rev = "bc305df3fe587960f7635f0185571500e5988475";35 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36 };37 kubePrometheusBench = let38 src = fetchFromGitHub {39 owner = "prometheus-operator";40 repo = "kube-prometheus";41 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43 };44 in45 runCommand "kube-prometheus-vendor"46 {47 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48 outputHashMode = "recursive";49 buildInputs = [cacert];50 }51 ''52 mkdir -p $out53 cp -r ${src}/* $out/54 cd $out55 mkdir vendor56 ${jsonnet-bundler}/bin/jb install57 '';5859 60 61 skipSlow =62 if quick63 then "slow benchmark, but only quick requested"64 else "";65in66 stdenv.mkDerivation {67 name = "benchmarks";68 __impure = true;69 unpackPhase = "true";7071 buildInputs = [72 sjsonnet73 jsonnet74 rsjsonnet75 go-jsonnet7677 hyperfine78 ];7980 installPhase = let81 mkBench = {82 name,83 path,84 omitSource ? false,85 pathIsGenerator ? false,86 skipRustAlternative ? "",87 skipScala ? "",88 skipCpp ? "",89 skipGo ? "",90 vendor ? "",91 }: ''92 echo >> $out93 echo "### ${name}" >> $out94 echo >> $out95 ${optionalString (skipRustAlternative != "") ''96 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out97 echo >> $out98 ''}99 ${optionalString (skipGo != "") ''100 echo "> Note: No results for Go, ${skipGo}" >> $out101 echo >> $out102 ''}103 ${optionalString (skipScala != "") ''104 echo "> Note: No results for Scala, ${skipScala}" >> $out105 echo >> $out106 ''}107 ${optionalString (skipCpp != "") ''108 echo "> Note: No results for C++, ${skipCpp}" >> $out109 echo >> $out110 ''}111 ${optionalString (!quick && !omitSource) ''112 echo "<details>" >> $out113 echo "<summary>Source</summary>" >> $out114 echo >> $out115 echo "\`\`\`jsonnet" >> $out116 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}117 cat ${path} >> $out118 echo >> $out119 echo "\`\`\`" >> $out120 echo "</details>" >> $out121 echo >> $out122 ''}123 path=${path}124 ${optionalString pathIsGenerator ''125 go-jsonnet $path > generated.jsonnet126 path=generated.jsonnet127 ''}128 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \129 ${concatStringsSep " " (forEach jrsonnetVariants (130 variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${131 if variant.name != ""132 then " (${variant.name})"133 else ""134 }\""135 ))} \136 ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \137 ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \138 ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \139 ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}140 cat result.md >> $out141 '';142 in ''143 set -oux144145 temp=$(mktemp -d)146 cd $temp147148 touch $out149 ${optionalString (!quick) ''150 cat ${./benchmarks.md} >> $out151 echo >> $out152153 echo "<details>" >> $out154 echo "<summary>Tested versions</summary>" >> $out155 echo >> $out156 echo Go: $(go-jsonnet --version) >> $out157 echo >> $out158 echo "\`\`\`" >> $out159 go-jsonnet --help >> $out160 echo "\`\`\`" >> $out161 echo >> $out162 echo C++: $(jsonnet --version) >> $out163 echo >> $out164 echo "\`\`\`" >> $out165 jsonnet --help >> $out166 echo "\`\`\`" >> $out167 echo >> $out168 echo Scala: >> $out169 echo >> $out170 echo "\`\`\`" >> $out171 sjsonnet 2>> $out || true172 echo "\`\`\`" >> $out173 echo >> $out174 echo "Rust (alternative):" >> $out175 echo >> $out176 echo "\`\`\`" >> $out177 rsjsonnet --help 2>> $out || true178 echo "\`\`\`" >> $out179 echo >> $out180 echo "</details>" >> $out181 echo >> $out182183 echo >> $out184 ''}185 echo "## Real world" >> $out186 ${mkBench {187 name = "Graalvm CI";188 path = "${graalvmBench}/ci.jsonnet";189 skipCpp = "takes longer than a hour";190 skipGo = skipSlow;191 skipScala = skipSlow;192 }}193 ${mkBench {194 name = "Kube-prometheus manifests";195 vendor = "${kubePrometheusBench}/vendor";196 path = "${kubePrometheusBench}/example.jsonnet";197 skipCpp = skipSlow;198 skipGo = skipSlow;199 skipScala = skipSlow;200 }}201202 echo >> $out203 echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out204 ${mkBench {205 name = "Large string join";206 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";207 skipScala = skipSlow;208 }}209 ${mkBench {210 name = "Large string template";211 omitSource = true;212 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";213 skipGo = "fails with os stack size exhausion";214 skipCpp = skipSlow;215 skipScala = skipSlow;216 }}217 ${mkBench {218 name = "Realistic 1";219 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";220 skipGo = skipSlow;221 skipCpp = skipSlow;222 skipScala = skipSlow;223 }}224 ${mkBench {225 name = "Realistic 2";226 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";227 skipGo = skipSlow;228 skipCpp = skipSlow;229 skipScala = skipSlow;230 }}231232 echo >> $out233 echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out234 ${mkBench {235 name = "Tail call";236 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";237 skipScala = skipSlow;238 }}239 ${mkBench {240 name = "Inheritance recursion";241 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242 skipCpp = skipSlow;243 skipGo = skipSlow;244 }}245 ${mkBench {246 name = "Simple recursive call";247 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248 skipScala = skipSlow;249 skipGo = skipSlow;250 }}251 ${mkBench {252 name = "Foldl string concat";253 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";254 skipCpp = skipSlow;255 skipScala = skipSlow;256 }}257 ${mkBench {258 name = "Array sorts";259 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";260 skipCpp = skipSlow;261 262 skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";263 }}264 ${mkBench {265 name = "Lazy array";266 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";267 skipGo = skipSlow;268 skipScala = skipSlow;269 }}270 ${mkBench {271 name = "Inheritance function recursion";272 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";273 skipCpp = skipSlow;274 skipScala = skipSlow;275 }}276 ${mkBench {277 name = "String strips";278 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";279 skipCpp = skipSlow;280 skipScala = skipSlow;281 }}282 ${mkBench {283 name = "Big object";284 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";285 pathIsGenerator = true;286 skipScala = skipSlow;287 }}288289 echo >> $out290 echo "## Benchmarks from Go jsonnet (builtins)" >> $out291 ${mkBench {292 name = "std.base64";293 path = "${goJsonnetBench}/base64.jsonnet";294 skipRustAlternative = skipSlow;295 skipCpp = skipSlow;296 skipScala = skipSlow;297 }}298 ${mkBench {299 name = "std.base64Decode";300 path = "${goJsonnetBench}/base64Decode.jsonnet";301 skipRustAlternative = skipSlow;302 skipCpp = skipSlow;303 skipScala = skipSlow;304 }}305 ${mkBench {306 name = "std.base64DecodeBytes";307 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";308 skipRustAlternative = skipSlow;309 skipCpp = skipSlow;310 skipGo = skipSlow;311 skipScala = skipSlow;312 }}313 ${mkBench {314 name = "std.base64 (byte array)";315 path = "${goJsonnetBench}/base64_byte_array.jsonnet";316 skipRustAlternative = skipSlow;317 skipCpp = skipSlow;318 skipGo = skipSlow;319 skipScala = skipSlow;320 }}321 ${mkBench {322 name = "std.foldl";323 path = "${goJsonnetBench}/foldl.jsonnet";324 skipScala = skipSlow;325 }}326 ${mkBench {327 name = "std.manifestJsonEx";328 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";329 skipScala = skipSlow;330 skipCpp = skipSlow;331 }}332 ${mkBench {333 name = "std.manifestTomlEx";334 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";335 skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";336 skipCpp = skipSlow;337 }}338 ${mkBench {339 name = "std.parseInt";340 path = "${goJsonnetBench}/parseInt.jsonnet";341 skipScala = skipSlow;342 skipCpp = skipSlow;343 }}344 ${mkBench {345 name = "std.reverse";346 path = "${goJsonnetBench}/reverse.jsonnet";347 skipCpp = skipSlow;348 skipGo = skipSlow;349 }}350 ${mkBench {351 name = "std.substr";352 path = "${goJsonnetBench}/substr.jsonnet";353 skipScala = skipSlow;354 }}355 ${mkBench {356 name = "Comparsion for array";357 path = "${goJsonnetBench}/comparison.jsonnet";358 skipScala = "array comparsion is not implemented";359 skipCpp = skipSlow;360 }}361 ${mkBench {362 name = "Comparsion for primitives";363 path = "${goJsonnetBench}/comparison2.jsonnet";364 skipRustAlternative = skipSlow;365 skipCpp = "can't run: uses up to 192GB of RAM";366 skipGo = skipSlow;367 skipScala = skipSlow;368 }}369 '';370 }