1{2 lib,3 runCommand,4 stdenv,5 fetchJrq,6 go-jsonnet,7 sjsonnet,8 cpp-jsonnet,9 rsjsonnet,10 hyperfine,11 quick ? false,12 jrsonnetVariants,13}:14with lib;15let16 inherit (cpp-jsonnet) jsonnetBench;17 inherit (go-jsonnet) goJsonnetBench;18 realworldVendor = fetchJrq {19 name = "realworld-vendor";20 lockfile = ../tests/realworld/jsonnetfile.lock.json;21 vendorHash = "sha256-oEUzM6Bhu8ZT8vCtYDbBEjG5BFHYpID+1/2pgXvIAgo=";22 };23 realworldBench = runCommand "realworld-bench" { } ''24 mkdir -p $out25 cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/26 cp -r ${realworldVendor} $out/vendor27 '';2829 30 31 skipSlow = if quick then "slow benchmark, but only quick requested" else "";32in33stdenv.mkDerivation {34 name = "benchmarks";35 36 unpackPhase = "true";3738 buildInputs = [39 sjsonnet40 cpp-jsonnet41 rsjsonnet42 go-jsonnet4344 hyperfine45 ];4647 installPhase =48 let49 mkBench =50 {51 name,52 path,53 omitSource ? false,54 pathIsGenerator ? false,55 skipRustAlternative ? "",56 skipScala ? "",57 skipCpp ? "",58 skipGo ? "",59 jpaths ? [ ],60 }:61 let62 jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;63 in64 ''65 echo >> $out66 echo "=== ${name}" >> $out67 echo >> $out68 ${optionalString (skipRustAlternative != "") ''69 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out70 echo >> $out71 ''}72 ${optionalString (skipGo != "") ''73 echo "> Note: No results for Go, ${skipGo}" >> $out74 echo >> $out75 ''}76 ${optionalString (skipScala != "") ''77 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out78 echo >> $out79 ''}80 ${optionalString (skipCpp != "") ''81 echo "> Note: No results for C++, ${skipCpp}" >> $out82 echo >> $out83 ''}84 ${optionalString (!omitSource) ''85 echo ".Source" >> $out86 echo "[%collapsible]" >> $out87 echo "====" >> $out88 echo "[source,jsonnet]" >> $out89 echo "----" >> $out90 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}91 cat ${path} >> $out92 echo >> $out93 echo "----" >> $out94 echo "====" >> $out95 echo >> $out96 ''}97 path=${path}98 ${optionalString pathIsGenerator ''99 go-jsonnet $path > generated.jsonnet100 path=generated.jsonnet101 ''}102 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \103 ${104 concatStringsSep " " (105 forEach jrsonnetVariants (106 variant:107 "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${108 if variant.name != "" then " (${variant.name})" else ""109 }\""110 )111 )112 } \113 ${114 optionalString (115 skipRustAlternative == ""116 ) "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""117 } \118 ${optionalString (skipGo == "") "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""} \119 ${120 optionalString (skipScala == "") "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""121 } \122 ${123 # My aarch64-linux machine can't run graalvm image:124 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].125 optionalString (126 skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux"127 ) "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""128 } \129 ${optionalString (skipCpp == "") "\"jsonnet $path${jpathArgs}\" -n \"C++\""}130 cat result.adoc >> $out131 '';132 in133 ''134 set -oux135 ulimit -s unlimited136137 temp=$(mktemp -d)138 cd $temp139140 touch $out141 ${optionalString (true) ''142 cat ${./benchmarks.adoc} >> $out143 echo >> $out144145 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out146 echo >> $out147148 echo ".Tested versions" >> $out149 echo "[%collapsible]" >> $out150 echo "====" >> $out151 echo "* Go: $(go-jsonnet --version)" >> $out152 echo "* C++: $(jsonnet --version)" >> $out153 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out154 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out155 ${concatStringsSep "\n" (156 forEach jrsonnetVariants (157 variant:158 "echo \"* Rust${159 if variant.name != "" then " (${variant.name})" else ""160 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"161 )162 )}163 echo "====" >> $out164 echo >> $out165 ''}166 echo "== Real world" >> $out167 ${mkBench {168 name = "GitLab runbooks dashboards";169 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";170 jpaths = [171 "${realworldBench}/vendor"172 "${realworldBench}/vendor/runbooks/libsonnet"173 "${realworldBench}/vendor/runbooks/dashboards"174 "${realworldBench}/vendor/runbooks/services"175 "${realworldBench}/vendor/runbooks/metrics-catalog"176 ];177 skipCpp = "too slow, takes hours, skews results";178 skipGo = skipSlow;179 }}180 ${mkBench {181 name = "GraalVM CI";182 path = "${realworldBench}/entry-graalvm.jsonnet";183 jpaths = [184 "${realworldBench}/vendor/graal"185 ];186 skipCpp = "too slow, takes hours, skews results";187 skipGo = skipSlow;188 }}189 ${mkBench {190 name = "Kube-prometheus";191 path = "${realworldBench}/entry-kube-prometheus.jsonnet";192 jpaths = [193 "${realworldBench}/vendor"194 ];195 skipCpp = "too slow, takes hours, skews results";196 skipGo = skipSlow;197 }}198 ${mkBench {199 name = "Loki manifests";200 path = "${realworldBench}/entry-loki.jsonnet";201 jpaths = [202 "${realworldBench}/vendor"203 "${realworldBench}"204 ];205 skipCpp = "too slow, takes hours, skews results";206 skipGo = skipSlow;207 }}208 ${mkBench {209 name = "Mimir manifests";210 path = "${realworldBench}/entry-mimir.jsonnet";211 jpaths = [212 "${realworldBench}/vendor"213 "${realworldBench}"214 ];215 skipCpp = "too slow, takes hours, skews results";216 skipGo = skipSlow;217 skipScala = "https://github.com/databricks/sjsonnet/issues/829";218 }}219 ${mkBench {220 name = "Tempo manifests";221 path = "${realworldBench}/entry-tempo.jsonnet";222 jpaths = [223 "${realworldBench}/vendor"224 "${realworldBench}"225 ];226 skipCpp = "too slow, takes hours, skews results";227 skipGo = skipSlow;228 }}229230 echo >> $out231 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out232 ${mkBench {233 name = "Large string join";234 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";235 }}236 ${mkBench {237 name = "Large string template";238 omitSource = true;239 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";240 skipGo = "fails with os stack size exhausion";241 skipCpp = "too slow, takes hours, skews results";242 }}243 ${mkBench {244 name = "Realistic 1";245 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";246 skipGo = skipSlow;247 skipCpp = "too slow, takes hours, skews results";248 }}249 ${mkBench {250 name = "Realistic 2";251 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";252 skipGo = skipSlow;253 skipCpp = "too slow, takes hours, skews results";254 }}255256 echo >> $out257 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out258 ${mkBench {259 name = "Tail call";260 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";261 }}262 ${mkBench {263 name = "Inheritance recursion";264 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";265 skipCpp = skipSlow;266 skipGo = skipSlow;267 }}268 ${mkBench {269 name = "Simple recursive call";270 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";271 skipGo = skipSlow;272 }}273 ${mkBench {274 name = "Foldl string concat";275 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";276 skipCpp = skipSlow;277 }}278 ${mkBench {279 name = "Array sorts";280 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";281 skipCpp = skipSlow;282 }}283 ${mkBench {284 name = "Lazy array";285 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";286 skipGo = skipSlow;287 }}288 ${mkBench {289 name = "Inheritance function recursion";290 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";291 skipCpp = skipSlow;292 }}293 ${mkBench {294 name = "String strips";295 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";296 skipCpp = "too slow, takes hours, skews results";297 }}298 ${mkBench {299 name = "Big object";300 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";301 pathIsGenerator = true;302 }}303304 echo >> $out305 echo "== Benchmarks from Go jsonnet (builtins)" >> $out306 ${mkBench {307 name = "std.base64";308 path = "${goJsonnetBench}/base64.jsonnet";309 skipRustAlternative = skipSlow;310 skipCpp = "too slow, takes hours, skews results";311 }}312 ${mkBench {313 name = "std.base64Decode";314 path = "${goJsonnetBench}/base64Decode.jsonnet";315 skipRustAlternative = skipSlow;316 skipCpp = skipSlow;317 }}318 ${mkBench {319 name = "std.base64DecodeBytes";320 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";321 skipRustAlternative = skipSlow;322 skipCpp = skipSlow;323 skipGo = skipSlow;324 }}325 ${mkBench {326 name = "std.base64 (byte array)";327 path = "${goJsonnetBench}/base64_byte_array.jsonnet";328 skipRustAlternative = skipSlow;329 skipCpp = skipSlow;330 skipGo = skipSlow;331 }}332 ${mkBench {333 name = "std.foldl";334 path = "${goJsonnetBench}/foldl.jsonnet";335 }}336 ${mkBench {337 name = "std.manifestJsonEx";338 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";339 skipCpp = skipSlow;340 }}341 ${mkBench {342 name = "std.manifestTomlEx";343 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";344 skipCpp = skipSlow;345 }}346 ${mkBench {347 name = "std.parseInt";348 path = "${goJsonnetBench}/parseInt.jsonnet";349 skipCpp = skipSlow;350 }}351 ${mkBench {352 name = "std.reverse";353 path = "${goJsonnetBench}/reverse.jsonnet";354 skipCpp = skipSlow;355 skipGo = skipSlow;356 }}357 ${mkBench {358 name = "std.substr";359 path = "${goJsonnetBench}/substr.jsonnet";360 }}361 ${mkBench {362 name = "Comparsion for array";363 path = "${goJsonnetBench}/comparison.jsonnet";364 skipCpp = "too slow, takes hours, skews results";365 }}366 ${mkBench {367 name = "Comparsion for primitives";368 path = "${goJsonnetBench}/comparison2.jsonnet";369 skipRustAlternative = skipSlow;370 skipCpp = "can't run: uses up to 192GB of RAM";371 skipGo = skipSlow;372 }}373 '';374}