1{2 lib,3 runCommand,4 stdenv,5 fetchFromGitHub,6 fetchJrq,7 go-jsonnet,8 sjsonnet,9 cpp-jsonnet,10 rsjsonnet,11 hyperfine,12 quick ? false,13 jrsonnetVariants,14}:15with lib;16let17 inherit (cpp-jsonnet) jsonnetBench;18 inherit (go-jsonnet) goJsonnetBench;19 graalvmBench = fetchFromGitHub {20 owner = "oracle";21 repo = "graal";22 rev = "bc305df3fe587960f7635f0185571500e5988475";23 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";24 };25 realworldVendor = fetchJrq {26 name = "realworld-vendor";27 lockfile = ../tests/realworld/jsonnetfile.lock.json;28 vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";29 };30 realworldBench = runCommand "realworld-bench" { } ''31 mkdir -p $out32 cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/33 cp -r ${realworldVendor} $out/vendor34 '';3536 37 38 skipSlow = if quick then "slow benchmark, but only quick requested" else "";39in40stdenv.mkDerivation {41 name = "benchmarks";42 43 unpackPhase = "true";4445 buildInputs = [46 sjsonnet47 cpp-jsonnet48 rsjsonnet49 go-jsonnet5051 hyperfine52 ];5354 installPhase =55 let56 mkBench =57 {58 name,59 path,60 omitSource ? false,61 pathIsGenerator ? false,62 skipRustAlternative ? "",63 skipScala ? "",64 skipCpp ? "",65 skipGo ? "",66 jpaths ? [ ],67 }:68 let69 jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;70 in71 ''72 echo >> $out73 echo "=== ${name}" >> $out74 echo >> $out75 ${optionalString (skipRustAlternative != "") ''76 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out77 echo >> $out78 ''}79 ${optionalString (skipGo != "") ''80 echo "> Note: No results for Go, ${skipGo}" >> $out81 echo >> $out82 ''}83 ${optionalString (skipScala != "") ''84 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out85 echo >> $out86 ''}87 ${optionalString (skipCpp != "") ''88 echo "> Note: No results for C++, ${skipCpp}" >> $out89 echo >> $out90 ''}91 ${optionalString (!omitSource) ''92 echo ".Source" >> $out93 echo "[%collapsible]" >> $out94 echo "====" >> $out95 echo "[source,jsonnet]" >> $out96 echo "----" >> $out97 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}98 cat ${path} >> $out99 echo >> $out100 echo "----" >> $out101 echo "====" >> $out102 echo >> $out103 ''}104 path=${path}105 ${optionalString pathIsGenerator ''106 go-jsonnet $path > generated.jsonnet107 path=generated.jsonnet108 ''}109 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \110 ${111 concatStringsSep " " (112 forEach jrsonnetVariants (113 variant:114 "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${115 if variant.name != "" then " (${variant.name})" else ""116 }\""117 )118 )119 } \120 ${121 optionalString (skipRustAlternative == "")122 "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""123 } \124 ${125 optionalString (skipGo == "")126 "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""127 } \128 ${129 optionalString (skipScala == "")130 "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""131 } \132 ${133 # My aarch64-linux machine can't run graalvm image:134 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].135 optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")136 "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""137 } \138 ${optionalString (skipCpp == "")139 "\"jsonnet $path${jpathArgs}\" -n \"C++\""140 }141 cat result.adoc >> $out142 '';143 in144 ''145 set -oux146 ulimit -s unlimited147148 temp=$(mktemp -d)149 cd $temp150151 touch $out152 ${optionalString (true) ''153 cat ${./benchmarks.adoc} >> $out154 echo >> $out155156 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out157 echo >> $out158159 echo ".Tested versions" >> $out160 echo "[%collapsible]" >> $out161 echo "====" >> $out162 echo "* Go: $(go-jsonnet --version)" >> $out163 echo "* C++: $(jsonnet --version)" >> $out164 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out165 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out166 ${concatStringsSep "\n" (167 forEach jrsonnetVariants (168 variant:169 "echo \"* Rust${170 if variant.name != "" then " (${variant.name})" else ""171 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"172 )173 )}174 echo "====" >> $out175 echo >> $out176 ''}177 echo "== Real world" >> $out178 ${mkBench {179 name = "Graalvm CI";180 path = "${graalvmBench}/ci.jsonnet";181 omitSource = true;182 skipCpp = "takes longer than a hour";183 skipGo = skipSlow;184 }}185 ${mkBench {186 name = "Loki manifests";187 path = "${realworldBench}/entry-loki.jsonnet";188 jpaths = [ "${realworldBench}/vendor" ];189 skipCpp = "too slow, takes hours, skews results";190 skipGo = skipSlow;191 }}192 ${mkBench {193 name = "Mimir manifests";194 path = "${realworldBench}/entry-mimir.jsonnet";195 jpaths = [ "${realworldBench}/vendor" ];196 skipCpp = "too slow, takes hours, skews results";197 skipGo = skipSlow;198 }}199 ${mkBench {200 name = "Tempo manifests";201 path = "${realworldBench}/entry-tempo.jsonnet";202 jpaths = [ "${realworldBench}/vendor" ];203 skipCpp = "too slow, takes hours, skews results";204 skipGo = skipSlow;205 }}206 ${mkBench {207 name = "GitLab runbooks dashboards";208 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";209 jpaths = [210 "${realworldBench}/vendor"211 "${realworldBench}/vendor/runbooks/libsonnet"212 "${realworldBench}/vendor/runbooks/dashboards"213 "${realworldBench}/vendor/runbooks/services"214 "${realworldBench}/vendor/runbooks/metrics-catalog"215 ];216 skipCpp = "too slow, takes hours, skews results";217 skipGo = skipSlow;218 }}219220 echo >> $out221 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out222 ${mkBench {223 name = "Large string join";224 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";225 }}226 ${mkBench {227 name = "Large string template";228 omitSource = true;229 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";230 skipGo = "fails with os stack size exhausion";231 skipCpp = "too slow, takes hours, skews results";232 }}233 ${mkBench {234 name = "Realistic 1";235 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";236 skipGo = skipSlow;237 skipCpp = "too slow, takes hours, skews results";238 }}239 ${mkBench {240 name = "Realistic 2";241 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";242 skipGo = skipSlow;243 skipCpp = "too slow, takes hours, skews results";244 }}245246 echo >> $out247 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out248 ${mkBench {249 name = "Tail call";250 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";251 }}252 ${mkBench {253 name = "Inheritance recursion";254 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";255 skipCpp = skipSlow;256 skipGo = skipSlow;257 }}258 ${mkBench {259 name = "Simple recursive call";260 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";261 skipGo = skipSlow;262 }}263 ${mkBench {264 name = "Foldl string concat";265 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";266 skipCpp = skipSlow;267 }}268 ${mkBench {269 name = "Array sorts";270 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";271 skipCpp = skipSlow;272 }}273 ${mkBench {274 name = "Lazy array";275 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";276 skipGo = skipSlow;277 }}278 ${mkBench {279 name = "Inheritance function recursion";280 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";281 skipCpp = skipSlow;282 }}283 ${mkBench {284 name = "String strips";285 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";286 skipCpp = "too slow, takes hours, skews results";287 }}288 ${mkBench {289 name = "Big object";290 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";291 pathIsGenerator = true;292 }}293294 echo >> $out295 echo "== Benchmarks from Go jsonnet (builtins)" >> $out296 ${mkBench {297 name = "std.base64";298 path = "${goJsonnetBench}/base64.jsonnet";299 skipRustAlternative = skipSlow;300 skipCpp = "too slow, takes hours, skews results";301 }}302 ${mkBench {303 name = "std.base64Decode";304 path = "${goJsonnetBench}/base64Decode.jsonnet";305 skipRustAlternative = skipSlow;306 skipCpp = skipSlow;307 }}308 ${mkBench {309 name = "std.base64DecodeBytes";310 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";311 skipRustAlternative = skipSlow;312 skipCpp = skipSlow;313 skipGo = skipSlow;314 }}315 ${mkBench {316 name = "std.base64 (byte array)";317 path = "${goJsonnetBench}/base64_byte_array.jsonnet";318 skipRustAlternative = skipSlow;319 skipCpp = skipSlow;320 skipGo = skipSlow;321 }}322 ${mkBench {323 name = "std.foldl";324 path = "${goJsonnetBench}/foldl.jsonnet";325 }}326 ${mkBench {327 name = "std.manifestJsonEx";328 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";329 skipCpp = skipSlow;330 }}331 ${mkBench {332 name = "std.manifestTomlEx";333 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";334 skipCpp = skipSlow;335 }}336 ${mkBench {337 name = "std.parseInt";338 path = "${goJsonnetBench}/parseInt.jsonnet";339 skipCpp = skipSlow;340 }}341 ${mkBench {342 name = "std.reverse";343 path = "${goJsonnetBench}/reverse.jsonnet";344 skipCpp = skipSlow;345 skipGo = skipSlow;346 }}347 ${mkBench {348 name = "std.substr";349 path = "${goJsonnetBench}/substr.jsonnet";350 }}351 ${mkBench {352 name = "Comparsion for array";353 path = "${goJsonnetBench}/comparison.jsonnet";354 skipCpp = "too slow, takes hours, skews results";355 }}356 ${mkBench {357 name = "Comparsion for primitives";358 path = "${goJsonnetBench}/comparison2.jsonnet";359 skipRustAlternative = skipSlow;360 skipCpp = "can't run: uses up to 192GB of RAM";361 skipGo = skipSlow;362 }}363 '';364}