1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 cpp-jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib;17let18 inherit (cpp-jsonnet) jsonnetBench;19 inherit (go-jsonnet) goJsonnetBench;20 graalvmBench = fetchFromGitHub {21 owner = "oracle";22 repo = "graal";23 rev = "bc305df3fe587960f7635f0185571500e5988475";24 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25 };26 kubePrometheusBench =27 let28 src = fetchFromGitHub {29 owner = "prometheus-operator";30 repo = "kube-prometheus";31 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33 };34 in35 runCommand "kube-prometheus-vendor"36 {37 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38 outputHashMode = "recursive";39 buildInputs = [ cacert ];40 }41 ''42 mkdir -p $out43 cp -r ${src}/* $out/44 cd $out45 chmod u+w jsonnetfile.lock.json46 mkdir vendor47 ${jsonnet-bundler}/bin/jb install48 '';4950 51 52 skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55 name = "benchmarks";56 57 unpackPhase = "true";5859 buildInputs = [60 sjsonnet61 cpp-jsonnet62 rsjsonnet63 go-jsonnet6465 hyperfine66 ];6768 installPhase =69 let70 mkBench =71 {72 name,73 path,74 omitSource ? false,75 pathIsGenerator ? false,76 skipRustAlternative ? "",77 skipScala ? "",78 skipCpp ? "",79 skipGo ? "",80 vendor ? "",81 }:82 ''83 echo >> $out84 echo "=== ${name}" >> $out85 echo >> $out86 ${optionalString (skipRustAlternative != "") ''87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88 echo >> $out89 ''}90 ${optionalString (skipGo != "") ''91 echo "> Note: No results for Go, ${skipGo}" >> $out92 echo >> $out93 ''}94 ${optionalString (skipScala != "") ''95 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96 echo >> $out97 ''}98 ${optionalString (skipCpp != "") ''99 echo "> Note: No results for C++, ${skipCpp}" >> $out100 echo >> $out101 ''}102 ${optionalString (!quick && !omitSource) ''103 echo ".Source" >> $out104 echo "[%collapsible]" >> $out105 echo "====" >> $out106 echo "[source,jsonnet]" >> $out107 echo "----" >> $out108 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109 cat ${path} >> $out110 echo >> $out111 echo "----" >> $out112 echo "====" >> $out113 echo >> $out114 ''}115 path=${path}116 ${optionalString pathIsGenerator ''117 go-jsonnet $path > generated.jsonnet118 path=generated.jsonnet119 ''}120 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121 ${122 concatStringsSep " " (123 forEach jrsonnetVariants (124 variant:125 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126 if variant.name != "" then " (${variant.name})" else ""127 }\""128 )129 )130 } \131 ${132 optionalString (skipRustAlternative == "")133 "\"rsjsonnet $path${134 optionalString (vendor != "") " -J ${vendor}"135 }\" -n \"Rust (alternative, rsjsonnet)\""136 } \137 ${138 optionalString (skipGo == "")139 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140 } \141 ${142 optionalString (skipScala == "")143 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144 } \145 ${146 # My aarch64-linux machine can't run graalvm image:147 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].148 optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")149 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""150 } \151 ${optionalString (skipCpp == "")152 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""153 }154 cat result.adoc >> $out155 '';156 in157 ''158 set -oux159 ulimit -s unlimited160161 temp=$(mktemp -d)162 cd $temp163164 touch $out165 ${optionalString (!quick) ''166 cat ${./benchmarks.adoc} >> $out167 echo >> $out168169 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out170 echo >> $out171172 echo ".Tested versions" >> $out173 echo "[%collapsible]" >> $out174 echo "====" >> $out175 echo "* Go: $(go-jsonnet --version)" >> $out176 echo "* C++: $(jsonnet --version)" >> $out177 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out178 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out179 ${concatStringsSep "\n" (180 forEach jrsonnetVariants (181 variant:182 "echo \"* Rust${183 if variant.name != "" then " (${variant.name})" else ""184 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"185 )186 )}187 echo "====" >> $out188 echo >> $out189 ''}190 echo "== Real world" >> $out191 ${mkBench {192 name = "Graalvm CI";193 path = "${graalvmBench}/ci.jsonnet";194 omitSource = true;195 skipCpp = "takes longer than a hour";196 skipGo = skipSlow;197 skipScala = skipSlow;198 }}199 ${mkBench {200 name = "Kube-prometheus manifests";201 vendor = "${kubePrometheusBench}/vendor";202 path = "${kubePrometheusBench}/example.jsonnet";203 omitSource = true;204 skipCpp = "too slow, takes hours, skews results";205 skipGo = skipSlow;206 skipScala = skipSlow;207 }}208209 echo >> $out210 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out211 ${mkBench {212 name = "Large string join";213 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";214 skipScala = skipSlow;215 }}216 ${mkBench {217 name = "Large string template";218 omitSource = true;219 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";220 skipGo = "fails with os stack size exhausion";221 skipCpp = "too slow, takes hours, skews results";222 skipScala = skipSlow;223 }}224 ${mkBench {225 name = "Realistic 1";226 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";227 skipGo = skipSlow;228 skipCpp = "too slow, takes hours, skews results";229 skipScala = skipSlow;230 }}231 ${mkBench {232 name = "Realistic 2";233 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";234 skipGo = skipSlow;235 skipCpp = "too slow, takes hours, skews results";236 skipScala = skipSlow;237 }}238239 echo >> $out240 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out241 ${mkBench {242 name = "Tail call";243 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";244 skipScala = skipSlow;245 }}246 ${mkBench {247 name = "Inheritance recursion";248 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";249 skipCpp = skipSlow;250 skipGo = skipSlow;251 }}252 ${mkBench {253 name = "Simple recursive call";254 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";255 skipScala = skipSlow;256 skipGo = skipSlow;257 }}258 ${mkBench {259 name = "Foldl string concat";260 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";261 skipCpp = skipSlow;262 skipScala = skipSlow;263 }}264 ${mkBench {265 name = "Array sorts";266 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";267 skipCpp = skipSlow;268 }}269 ${mkBench {270 name = "Lazy array";271 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";272 skipGo = skipSlow;273 }}274 ${mkBench {275 name = "Inheritance function recursion";276 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";277 skipCpp = skipSlow;278 skipScala = skipSlow;279 }}280 ${mkBench {281 name = "String strips";282 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";283 skipCpp = "too slow, takes hours, skews results";284 skipScala = skipSlow;285 }}286 ${mkBench {287 name = "Big object";288 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";289 pathIsGenerator = true;290 skipScala = skipSlow;291 }}292293 echo >> $out294 echo "== Benchmarks from Go jsonnet (builtins)" >> $out295 ${mkBench {296 name = "std.base64";297 path = "${goJsonnetBench}/base64.jsonnet";298 skipRustAlternative = skipSlow;299 skipCpp = "too slow, takes hours, skews results";300 skipScala = skipSlow;301 }}302 ${mkBench {303 name = "std.base64Decode";304 path = "${goJsonnetBench}/base64Decode.jsonnet";305 skipRustAlternative = skipSlow;306 skipCpp = skipSlow;307 skipScala = skipSlow;308 }}309 ${mkBench {310 name = "std.base64DecodeBytes";311 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";312 skipRustAlternative = skipSlow;313 skipCpp = skipSlow;314 skipGo = skipSlow;315 skipScala = skipSlow;316 }}317 ${mkBench {318 name = "std.base64 (byte array)";319 path = "${goJsonnetBench}/base64_byte_array.jsonnet";320 skipRustAlternative = skipSlow;321 skipCpp = skipSlow;322 skipGo = skipSlow;323 skipScala = skipSlow;324 }}325 ${mkBench {326 name = "std.foldl";327 path = "${goJsonnetBench}/foldl.jsonnet";328 skipScala = skipSlow;329 }}330 ${mkBench {331 name = "std.manifestJsonEx";332 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";333 skipScala = skipSlow;334 skipCpp = skipSlow;335 }}336 ${mkBench {337 name = "std.manifestTomlEx";338 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";339 skipCpp = skipSlow;340 }}341 ${mkBench {342 name = "std.parseInt";343 path = "${goJsonnetBench}/parseInt.jsonnet";344 skipScala = skipSlow;345 skipCpp = skipSlow;346 }}347 ${mkBench {348 name = "std.reverse";349 path = "${goJsonnetBench}/reverse.jsonnet";350 skipCpp = skipSlow;351 skipGo = skipSlow;352 }}353 ${mkBench {354 name = "std.substr";355 path = "${goJsonnetBench}/substr.jsonnet";356 skipScala = skipSlow;357 }}358 ${mkBench {359 name = "Comparsion for array";360 path = "${goJsonnetBench}/comparison.jsonnet";361 skipCpp = "too slow, takes hours, skews results";362 }}363 ${mkBench {364 name = "Comparsion for primitives";365 path = "${goJsonnetBench}/comparison2.jsonnet";366 skipRustAlternative = skipSlow;367 skipCpp = "can't run: uses up to 192GB of RAM";368 skipGo = skipSlow;369 skipScala = skipSlow;370 }}371 '';372}