1{2 lib,3 runCommand,4 jsonnet-bundler,5 cacert,6 stdenv,7 fetchFromGitHub,8 go-jsonnet,9 sjsonnet,10 cpp-jsonnet,11 rsjsonnet,12 hyperfine,13 quick ? false,14 jrsonnetVariants,15}:16with lib;17let18 inherit (cpp-jsonnet) jsonnetBench;19 inherit (go-jsonnet) goJsonnetBench;20 graalvmBench = fetchFromGitHub {21 owner = "oracle";22 repo = "graal";23 rev = "bc305df3fe587960f7635f0185571500e5988475";24 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25 };26 kubePrometheusBench =27 let28 src = fetchFromGitHub {29 owner = "prometheus-operator";30 repo = "kube-prometheus";31 rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32 hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33 };34 in35 runCommand "kube-prometheus-vendor"36 {37 outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38 outputHashMode = "recursive";39 buildInputs = [ cacert ];40 }41 ''42 mkdir -p $out43 cp -r ${src}/* $out/44 cd $out45 chmod u+w jsonnetfile.lock.json46 mkdir vendor47 ${jsonnet-bundler}/bin/jb install48 '';4950 51 52 skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55 name = "benchmarks";56 57 unpackPhase = "true";5859 buildInputs = [60 sjsonnet61 cpp-jsonnet62 rsjsonnet63 go-jsonnet6465 hyperfine66 ];6768 installPhase =69 let70 mkBench =71 {72 name,73 path,74 omitSource ? false,75 pathIsGenerator ? false,76 skipRustAlternative ? "",77 skipScala ? "",78 skipCpp ? "",79 skipGo ? "",80 vendor ? "",81 }:82 ''83 echo >> $out84 echo "=== ${name}" >> $out85 echo >> $out86 ${optionalString (skipRustAlternative != "") ''87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88 echo >> $out89 ''}90 ${optionalString (skipGo != "") ''91 echo "> Note: No results for Go, ${skipGo}" >> $out92 echo >> $out93 ''}94 ${optionalString (skipScala != "") ''95 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96 echo >> $out97 ''}98 ${optionalString (skipCpp != "") ''99 echo "> Note: No results for C++, ${skipCpp}" >> $out100 echo >> $out101 ''}102 ${optionalString (!omitSource) ''103 echo ".Source" >> $out104 echo "[%collapsible]" >> $out105 echo "====" >> $out106 echo "[source,jsonnet]" >> $out107 echo "----" >> $out108 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109 cat ${path} >> $out110 echo >> $out111 echo "----" >> $out112 echo "====" >> $out113 echo >> $out114 ''}115 path=${path}116 ${optionalString pathIsGenerator ''117 go-jsonnet $path > generated.jsonnet118 path=generated.jsonnet119 ''}120 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121 ${122 concatStringsSep " " (123 forEach jrsonnetVariants (124 variant:125 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126 if variant.name != "" then " (${variant.name})" else ""127 }\""128 )129 )130 } \131 ${132 optionalString (skipRustAlternative == "")133 "\"rsjsonnet $path${134 optionalString (vendor != "") " -J ${vendor}"135 }\" -n \"Rust (alternative, rsjsonnet)\""136 } \137 ${138 optionalString (skipGo == "")139 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140 } \141 ${142 optionalString (skipScala == "")143 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144 } \145 ${146 # My aarch64-linux machine can't run graalvm image:147 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].148 optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")149 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""150 } \151 ${optionalString (skipCpp == "")152 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""153 }154 cat result.adoc >> $out155 '';156 in157 ''158 set -oux159 ulimit -s unlimited160161 temp=$(mktemp -d)162 cd $temp163164 touch $out165 ${optionalString (true) ''166 cat ${./benchmarks.adoc} >> $out167 echo >> $out168169 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out170 echo >> $out171172 echo ".Tested versions" >> $out173 echo "[%collapsible]" >> $out174 echo "====" >> $out175 echo "* Go: $(go-jsonnet --version)" >> $out176 echo "* C++: $(jsonnet --version)" >> $out177 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out178 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out179 ${concatStringsSep "\n" (180 forEach jrsonnetVariants (181 variant:182 "echo \"* Rust${183 if variant.name != "" then " (${variant.name})" else ""184 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"185 )186 )}187 echo "====" >> $out188 echo >> $out189 ''}190 echo "== Real world" >> $out191 ${mkBench {192 name = "Graalvm CI";193 path = "${graalvmBench}/ci.jsonnet";194 omitSource = true;195 skipCpp = "takes longer than a hour";196 skipGo = skipSlow;197 }}198 ${mkBench {199 name = "Kube-prometheus manifests";200 vendor = "${kubePrometheusBench}/vendor";201 path = "${kubePrometheusBench}/example.jsonnet";202 omitSource = true;203 skipCpp = "too slow, takes hours, skews results";204 skipGo = skipSlow;205 }}206207 echo >> $out208 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out209 ${mkBench {210 name = "Large string join";211 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";212 }}213 ${mkBench {214 name = "Large string template";215 omitSource = true;216 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";217 skipGo = "fails with os stack size exhausion";218 skipCpp = "too slow, takes hours, skews results";219 }}220 ${mkBench {221 name = "Realistic 1";222 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";223 skipGo = skipSlow;224 skipCpp = "too slow, takes hours, skews results";225 }}226 ${mkBench {227 name = "Realistic 2";228 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229 skipGo = skipSlow;230 skipCpp = "too slow, takes hours, skews results";231 }}232233 echo >> $out234 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out235 ${mkBench {236 name = "Tail call";237 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";238 }}239 ${mkBench {240 name = "Inheritance recursion";241 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242 skipCpp = skipSlow;243 skipGo = skipSlow;244 }}245 ${mkBench {246 name = "Simple recursive call";247 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248 skipGo = skipSlow;249 }}250 ${mkBench {251 name = "Foldl string concat";252 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";253 skipCpp = skipSlow;254 }}255 ${mkBench {256 name = "Array sorts";257 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";258 skipCpp = skipSlow;259 }}260 ${mkBench {261 name = "Lazy array";262 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";263 skipGo = skipSlow;264 }}265 ${mkBench {266 name = "Inheritance function recursion";267 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";268 skipCpp = skipSlow;269 }}270 ${mkBench {271 name = "String strips";272 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";273 skipCpp = "too slow, takes hours, skews results";274 }}275 ${mkBench {276 name = "Big object";277 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";278 pathIsGenerator = true;279 }}280281 echo >> $out282 echo "== Benchmarks from Go jsonnet (builtins)" >> $out283 ${mkBench {284 name = "std.base64";285 path = "${goJsonnetBench}/base64.jsonnet";286 skipRustAlternative = skipSlow;287 skipCpp = "too slow, takes hours, skews results";288 }}289 ${mkBench {290 name = "std.base64Decode";291 path = "${goJsonnetBench}/base64Decode.jsonnet";292 skipRustAlternative = skipSlow;293 skipCpp = skipSlow;294 }}295 ${mkBench {296 name = "std.base64DecodeBytes";297 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";298 skipRustAlternative = skipSlow;299 skipCpp = skipSlow;300 skipGo = skipSlow;301 }}302 ${mkBench {303 name = "std.base64 (byte array)";304 path = "${goJsonnetBench}/base64_byte_array.jsonnet";305 skipRustAlternative = skipSlow;306 skipCpp = skipSlow;307 skipGo = skipSlow;308 }}309 ${mkBench {310 name = "std.foldl";311 path = "${goJsonnetBench}/foldl.jsonnet";312 }}313 ${mkBench {314 name = "std.manifestJsonEx";315 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";316 skipCpp = skipSlow;317 }}318 ${mkBench {319 name = "std.manifestTomlEx";320 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";321 skipCpp = skipSlow;322 }}323 ${mkBench {324 name = "std.parseInt";325 path = "${goJsonnetBench}/parseInt.jsonnet";326 skipCpp = skipSlow;327 }}328 ${mkBench {329 name = "std.reverse";330 path = "${goJsonnetBench}/reverse.jsonnet";331 skipCpp = skipSlow;332 skipGo = skipSlow;333 }}334 ${mkBench {335 name = "std.substr";336 path = "${goJsonnetBench}/substr.jsonnet";337 }}338 ${mkBench {339 name = "Comparsion for array";340 path = "${goJsonnetBench}/comparison.jsonnet";341 skipCpp = "too slow, takes hours, skews results";342 }}343 ${mkBench {344 name = "Comparsion for primitives";345 path = "${goJsonnetBench}/comparison2.jsonnet";346 skipRustAlternative = skipSlow;347 skipCpp = "can't run: uses up to 192GB of RAM";348 skipGo = skipSlow;349 }}350 '';351}