git.delta.rocks / jrsonnet / refs/commits / 341222d06daf

1{2  lib,3  runCommand,4  stdenv,5  fetchJrq,6  go-jsonnet,7  sjsonnet,8  cpp-jsonnet,9  rsjsonnet,10  hyperfine,11  quick ? false,12  jrsonnetVariants,13}:14with lib;15let16  inherit (cpp-jsonnet) jsonnetBench;17  inherit (go-jsonnet) goJsonnetBench;18  realworldVendor = fetchJrq {19    name = "realworld-vendor";20    lockfile = ../tests/realworld/jsonnetfile.lock.json;21    vendorHash = "sha256-oEUzM6Bhu8ZT8vCtYDbBEjG5BFHYpID+1/2pgXvIAgo=";22  };23  realworldBench = runCommand "realworld-bench" { } ''24    mkdir -p $out25    cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/26    cp -r ${realworldVendor} $out/vendor27  '';2829  # Removes outsiders from the output30  # Useful when comparing performance of different jrsonnet releases31  skipSlow = if quick then "slow benchmark, but only quick requested" else "";32in33stdenv.mkDerivation {34  name = "benchmarks";35  # __impure = true; # not supported by hercules-ci36  unpackPhase = "true";3738  buildInputs = [39    sjsonnet40    cpp-jsonnet41    rsjsonnet42    go-jsonnet4344    hyperfine45  ];4647  installPhase =48    let49      mkBench =50        {51          name,52          path,53          omitSource ? false,54          pathIsGenerator ? false,55          skipRustAlternative ? "",56          skipScala ? "",57          skipCpp ? "",58          skipGo ? "",59          jpaths ? [ ],60        }:61        let62          jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;63        in64        ''65          echo >> $out66          echo "=== ${name}" >> $out67          echo >> $out68          ${optionalString (skipRustAlternative != "") ''69            echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out70            echo >> $out71          ''}72          ${optionalString (skipGo != "") ''73            echo "> Note: No results for Go, ${skipGo}" >> $out74            echo >> $out75          ''}76          ${optionalString (skipScala != "") ''77            echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out78            echo >> $out79          ''}80          ${optionalString (skipCpp != "") ''81            echo "> Note: No results for C++, ${skipCpp}" >> $out82            echo >> $out83          ''}84          ${optionalString (!omitSource) ''85            echo ".Source" >> $out86            echo "[%collapsible]" >> $out87            echo "====" >> $out88            echo "[source,jsonnet]" >> $out89            echo "----" >> $out90            ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}91            cat ${path} >> $out92            echo >> $out93            echo "----" >> $out94            echo "====" >> $out95            echo >> $out96          ''}97          path=${path}98          ${optionalString pathIsGenerator ''99            go-jsonnet $path > generated.jsonnet100            path=generated.jsonnet101          ''}102          hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \103            ${104              concatStringsSep " " (105                forEach jrsonnetVariants (106                  variant:107                  "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${108                    if variant.name != "" then " (${variant.name})" else ""109                  }\""110                )111              )112            } \113            ${114              optionalString (115                skipRustAlternative == ""116              ) "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""117            } \118            ${optionalString (skipGo == "") "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""} \119            ${120              optionalString (skipScala == "") "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""121            } \122            ${123              # My aarch64-linux machine can't run graalvm image:124              # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].125              optionalString (126                skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux"127              ) "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""128            } \129            ${optionalString (skipCpp == "") "\"jsonnet $path${jpathArgs}\" -n \"C++\""}130          cat result.adoc >> $out131        '';132    in133    ''134      set -oux135      ulimit -s unlimited136137      temp=$(mktemp -d)138      cd $temp139140      touch $out141      ${optionalString (true) ''142        cat ${./benchmarks.adoc} >> $out143        echo >> $out144145        echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out146        echo >> $out147148        echo ".Tested versions" >> $out149        echo "[%collapsible]" >> $out150        echo "====" >> $out151        echo "* Go: $(go-jsonnet --version)" >> $out152        echo "* C++: $(jsonnet --version)" >> $out153        echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out154        echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out155        ${concatStringsSep "\n" (156          forEach jrsonnetVariants (157            variant:158            "echo \"* Rust${159              if variant.name != "" then " (${variant.name})" else ""160            }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"161          )162        )}163        echo "====" >> $out164        echo >> $out165      ''}166      echo "== Real world" >> $out167      ${mkBench {168        name = "GitLab runbooks dashboards";169        path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";170        jpaths = [171          "${realworldBench}/vendor"172          "${realworldBench}/vendor/runbooks/libsonnet"173          "${realworldBench}/vendor/runbooks/dashboards"174          "${realworldBench}/vendor/runbooks/services"175          "${realworldBench}/vendor/runbooks/metrics-catalog"176        ];177        skipCpp = "too slow, takes hours, skews results";178        skipGo = skipSlow;179      }}180      ${mkBench {181        name = "GraalVM CI";182        path = "${realworldBench}/entry-graalvm.jsonnet";183        jpaths = [184          "${realworldBench}/vendor/graal"185        ];186        skipCpp = "too slow, takes hours, skews results";187        skipGo = skipSlow;188      }}189      ${mkBench {190        name = "Kube-prometheus";191        path = "${realworldBench}/entry-kube-prometheus.jsonnet";192        jpaths = [193          "${realworldBench}/vendor"194        ];195        skipCpp = "too slow, takes hours, skews results";196        skipGo = skipSlow;197      }}198      ${mkBench {199        name = "Loki manifests";200        path = "${realworldBench}/entry-loki.jsonnet";201        jpaths = [202          "${realworldBench}/vendor"203          "${realworldBench}"204        ];205        skipCpp = "too slow, takes hours, skews results";206        skipGo = skipSlow;207      }}208      ${mkBench {209        name = "Mimir manifests";210        path = "${realworldBench}/entry-mimir.jsonnet";211        jpaths = [212          "${realworldBench}/vendor"213          "${realworldBench}"214        ];215        skipCpp = "too slow, takes hours, skews results";216        skipGo = skipSlow;217        skipScala = "https://github.com/databricks/sjsonnet/issues/829";218      }}219      ${mkBench {220        name = "Tempo manifests";221        path = "${realworldBench}/entry-tempo.jsonnet";222        jpaths = [223          "${realworldBench}/vendor"224          "${realworldBench}"225        ];226        skipCpp = "too slow, takes hours, skews results";227        skipGo = skipSlow;228      }}229230      echo >> $out231      echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out232      ${mkBench {233        name = "Large string join";234        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";235      }}236      ${mkBench {237        name = "Large string template";238        omitSource = true;239        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";240        skipGo = "fails with os stack size exhausion";241        skipCpp = "too slow, takes hours, skews results";242      }}243      ${mkBench {244        name = "Realistic 1";245        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";246        skipGo = skipSlow;247        skipCpp = "too slow, takes hours, skews results";248      }}249      ${mkBench {250        name = "Realistic 2";251        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";252        skipGo = skipSlow;253        skipCpp = "too slow, takes hours, skews results";254      }}255256      echo >> $out257      echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out258      ${mkBench {259        name = "Tail call";260        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";261      }}262      ${mkBench {263        name = "Inheritance recursion";264        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";265        skipCpp = skipSlow;266        skipGo = skipSlow;267      }}268      ${mkBench {269        name = "Simple recursive call";270        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";271        skipGo = skipSlow;272      }}273      ${mkBench {274        name = "Foldl string concat";275        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";276        skipCpp = skipSlow;277      }}278      ${mkBench {279        name = "Array sorts";280        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";281        skipCpp = skipSlow;282      }}283      ${mkBench {284        name = "Lazy array";285        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";286        skipGo = skipSlow;287      }}288      ${mkBench {289        name = "Inheritance function recursion";290        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";291        skipCpp = skipSlow;292      }}293      ${mkBench {294        name = "String strips";295        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";296        skipCpp = "too slow, takes hours, skews results";297      }}298      ${mkBench {299        name = "Big object";300        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";301        pathIsGenerator = true;302      }}303304      echo >> $out305      echo "== Benchmarks from Go jsonnet (builtins)" >> $out306      ${mkBench {307        name = "std.base64";308        path = "${goJsonnetBench}/base64.jsonnet";309        skipRustAlternative = skipSlow;310        skipCpp = "too slow, takes hours, skews results";311      }}312      ${mkBench {313        name = "std.base64Decode";314        path = "${goJsonnetBench}/base64Decode.jsonnet";315        skipRustAlternative = skipSlow;316        skipCpp = skipSlow;317      }}318      ${mkBench {319        name = "std.base64DecodeBytes";320        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";321        skipRustAlternative = skipSlow;322        skipCpp = skipSlow;323        skipGo = skipSlow;324      }}325      ${mkBench {326        name = "std.base64 (byte array)";327        path = "${goJsonnetBench}/base64_byte_array.jsonnet";328        skipRustAlternative = skipSlow;329        skipCpp = skipSlow;330        skipGo = skipSlow;331      }}332      ${mkBench {333        name = "std.foldl";334        path = "${goJsonnetBench}/foldl.jsonnet";335      }}336      ${mkBench {337        name = "std.manifestJsonEx";338        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";339        skipCpp = skipSlow;340      }}341      ${mkBench {342        name = "std.manifestTomlEx";343        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";344        skipCpp = skipSlow;345      }}346      ${mkBench {347        name = "std.parseInt";348        path = "${goJsonnetBench}/parseInt.jsonnet";349        skipCpp = skipSlow;350      }}351      ${mkBench {352        name = "std.reverse";353        path = "${goJsonnetBench}/reverse.jsonnet";354        skipCpp = skipSlow;355        skipGo = skipSlow;356      }}357      ${mkBench {358        name = "std.substr";359        path = "${goJsonnetBench}/substr.jsonnet";360      }}361      ${mkBench {362        name = "Comparsion for array";363        path = "${goJsonnetBench}/comparison.jsonnet";364        skipCpp = "too slow, takes hours, skews results";365      }}366      ${mkBench {367        name = "Comparsion for primitives";368        path = "${goJsonnetBench}/comparison2.jsonnet";369        skipRustAlternative = skipSlow;370        skipCpp = "can't run: uses up to 192GB of RAM";371        skipGo = skipSlow;372      }}373    '';374}