1{2 lib,3 runCommand,4 stdenv,5 fetchFromGitHub,6 fetchJrq,7 go-jsonnet,8 sjsonnet,9 cpp-jsonnet,10 rsjsonnet,11 hyperfine,12 quick ? false,13 jrsonnetVariants,14}:15with lib;16let17 inherit (cpp-jsonnet) jsonnetBench;18 inherit (go-jsonnet) goJsonnetBench;19 graalvmBench = fetchFromGitHub {20 owner = "oracle";21 repo = "graal";22 rev = "bc305df3fe587960f7635f0185571500e5988475";23 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";24 };25 realworldVendor = fetchJrq {26 name = "realworld-vendor";27 lockfile = ../tests/realworld/jsonnetfile.lock.json;28 vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";29 };30 realworldBench = runCommand "realworld-bench" { } ''31 mkdir -p $out32 cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/33 cp -r ${realworldVendor} $out/vendor34 '';3536 37 38 skipSlow = if quick then "slow benchmark, but only quick requested" else "";39in40stdenv.mkDerivation {41 name = "benchmarks";42 43 unpackPhase = "true";4445 buildInputs = [46 sjsonnet47 cpp-jsonnet48 rsjsonnet49 go-jsonnet5051 hyperfine52 ];5354 installPhase =55 let56 mkBench =57 {58 name,59 path,60 omitSource ? false,61 pathIsGenerator ? false,62 skipRustAlternative ? "",63 skipScala ? "",64 skipCpp ? "",65 skipGo ? "",66 jpaths ? [ ],67 }:68 let69 jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;70 in71 ''72 echo >> $out73 echo "=== ${name}" >> $out74 echo >> $out75 ${optionalString (skipRustAlternative != "") ''76 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out77 echo >> $out78 ''}79 ${optionalString (skipGo != "") ''80 echo "> Note: No results for Go, ${skipGo}" >> $out81 echo >> $out82 ''}83 ${optionalString (skipScala != "") ''84 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out85 echo >> $out86 ''}87 ${optionalString (skipCpp != "") ''88 echo "> Note: No results for C++, ${skipCpp}" >> $out89 echo >> $out90 ''}91 ${optionalString (!omitSource) ''92 echo ".Source" >> $out93 echo "[%collapsible]" >> $out94 echo "====" >> $out95 echo "[source,jsonnet]" >> $out96 echo "----" >> $out97 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}98 cat ${path} >> $out99 echo >> $out100 echo "----" >> $out101 echo "====" >> $out102 echo >> $out103 ''}104 path=${path}105 ${optionalString pathIsGenerator ''106 go-jsonnet $path > generated.jsonnet107 path=generated.jsonnet108 ''}109 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \110 ${111 concatStringsSep " " (112 forEach jrsonnetVariants (113 variant:114 "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${115 if variant.name != "" then " (${variant.name})" else ""116 }\""117 )118 )119 } \120 ${121 optionalString (122 skipRustAlternative == ""123 ) "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""124 } \125 ${optionalString (skipGo == "") "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""} \126 ${127 optionalString (skipScala == "") "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""128 } \129 ${130 # My aarch64-linux machine can't run graalvm image:131 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].132 optionalString (133 skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux"134 ) "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""135 } \136 ${optionalString (skipCpp == "") "\"jsonnet $path${jpathArgs}\" -n \"C++\""}137 cat result.adoc >> $out138 '';139 in140 ''141 set -oux142 ulimit -s unlimited143144 temp=$(mktemp -d)145 cd $temp146147 touch $out148 ${optionalString (true) ''149 cat ${./benchmarks.adoc} >> $out150 echo >> $out151152 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out153 echo >> $out154155 echo ".Tested versions" >> $out156 echo "[%collapsible]" >> $out157 echo "====" >> $out158 echo "* Go: $(go-jsonnet --version)" >> $out159 echo "* C++: $(jsonnet --version)" >> $out160 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out161 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out162 ${concatStringsSep "\n" (163 forEach jrsonnetVariants (164 variant:165 "echo \"* Rust${166 if variant.name != "" then " (${variant.name})" else ""167 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"168 )169 )}170 echo "====" >> $out171 echo >> $out172 ''}173 echo "== Real world" >> $out174 ${mkBench {175 name = "Graalvm CI";176 path = "${graalvmBench}/ci.jsonnet";177 omitSource = true;178 skipCpp = "takes longer than a hour";179 skipGo = skipSlow;180 }}181 ${mkBench {182 name = "Loki manifests";183 path = "${realworldBench}/entry-loki.jsonnet";184 jpaths = [ "${realworldBench}/vendor" ];185 skipCpp = "too slow, takes hours, skews results";186 skipGo = skipSlow;187 }}188 ${mkBench {189 name = "Mimir manifests";190 path = "${realworldBench}/entry-mimir.jsonnet";191 jpaths = [ "${realworldBench}/vendor" ];192 skipCpp = "too slow, takes hours, skews results";193 skipGo = skipSlow;194 }}195 ${mkBench {196 name = "Tempo manifests";197 path = "${realworldBench}/entry-tempo.jsonnet";198 jpaths = [ "${realworldBench}/vendor" ];199 skipCpp = "too slow, takes hours, skews results";200 skipGo = skipSlow;201 }}202 ${mkBench {203 name = "GitLab runbooks dashboards";204 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";205 jpaths = [206 "${realworldBench}/vendor"207 "${realworldBench}/vendor/runbooks/libsonnet"208 "${realworldBench}/vendor/runbooks/dashboards"209 "${realworldBench}/vendor/runbooks/services"210 "${realworldBench}/vendor/runbooks/metrics-catalog"211 ];212 skipCpp = "too slow, takes hours, skews results";213 skipGo = skipSlow;214 }}215216 echo >> $out217 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out218 ${mkBench {219 name = "Large string join";220 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";221 }}222 ${mkBench {223 name = "Large string template";224 omitSource = true;225 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";226 skipGo = "fails with os stack size exhausion";227 skipCpp = "too slow, takes hours, skews results";228 }}229 ${mkBench {230 name = "Realistic 1";231 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";232 skipGo = skipSlow;233 skipCpp = "too slow, takes hours, skews results";234 }}235 ${mkBench {236 name = "Realistic 2";237 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";238 skipGo = skipSlow;239 skipCpp = "too slow, takes hours, skews results";240 }}241242 echo >> $out243 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out244 ${mkBench {245 name = "Tail call";246 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";247 }}248 ${mkBench {249 name = "Inheritance recursion";250 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";251 skipCpp = skipSlow;252 skipGo = skipSlow;253 }}254 ${mkBench {255 name = "Simple recursive call";256 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";257 skipGo = skipSlow;258 }}259 ${mkBench {260 name = "Foldl string concat";261 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";262 skipCpp = skipSlow;263 }}264 ${mkBench {265 name = "Array sorts";266 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";267 skipCpp = skipSlow;268 }}269 ${mkBench {270 name = "Lazy array";271 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";272 skipGo = skipSlow;273 }}274 ${mkBench {275 name = "Inheritance function recursion";276 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";277 skipCpp = skipSlow;278 }}279 ${mkBench {280 name = "String strips";281 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";282 skipCpp = "too slow, takes hours, skews results";283 }}284 ${mkBench {285 name = "Big object";286 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287 pathIsGenerator = true;288 }}289290 echo >> $out291 echo "== Benchmarks from Go jsonnet (builtins)" >> $out292 ${mkBench {293 name = "std.base64";294 path = "${goJsonnetBench}/base64.jsonnet";295 skipRustAlternative = skipSlow;296 skipCpp = "too slow, takes hours, skews results";297 }}298 ${mkBench {299 name = "std.base64Decode";300 path = "${goJsonnetBench}/base64Decode.jsonnet";301 skipRustAlternative = skipSlow;302 skipCpp = skipSlow;303 }}304 ${mkBench {305 name = "std.base64DecodeBytes";306 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";307 skipRustAlternative = skipSlow;308 skipCpp = skipSlow;309 skipGo = skipSlow;310 }}311 ${mkBench {312 name = "std.base64 (byte array)";313 path = "${goJsonnetBench}/base64_byte_array.jsonnet";314 skipRustAlternative = skipSlow;315 skipCpp = skipSlow;316 skipGo = skipSlow;317 }}318 ${mkBench {319 name = "std.foldl";320 path = "${goJsonnetBench}/foldl.jsonnet";321 }}322 ${mkBench {323 name = "std.manifestJsonEx";324 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";325 skipCpp = skipSlow;326 }}327 ${mkBench {328 name = "std.manifestTomlEx";329 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";330 skipCpp = skipSlow;331 }}332 ${mkBench {333 name = "std.parseInt";334 path = "${goJsonnetBench}/parseInt.jsonnet";335 skipCpp = skipSlow;336 }}337 ${mkBench {338 name = "std.reverse";339 path = "${goJsonnetBench}/reverse.jsonnet";340 skipCpp = skipSlow;341 skipGo = skipSlow;342 }}343 ${mkBench {344 name = "std.substr";345 path = "${goJsonnetBench}/substr.jsonnet";346 }}347 ${mkBench {348 name = "Comparsion for array";349 path = "${goJsonnetBench}/comparison.jsonnet";350 skipCpp = "too slow, takes hours, skews results";351 }}352 ${mkBench {353 name = "Comparsion for primitives";354 path = "${goJsonnetBench}/comparison2.jsonnet";355 skipRustAlternative = skipSlow;356 skipCpp = "can't run: uses up to 192GB of RAM";357 skipGo = skipSlow;358 }}359 '';360}