git.delta.rocks / jrsonnet / refs/commits / 341222d06daf

1{2  lib,3  runCommand,4  stdenv,5  fetchFromGitHub,6  fetchJrq,7  go-jsonnet,8  sjsonnet,9  cpp-jsonnet,10  rsjsonnet,11  hyperfine,12  quick ? false,13  jrsonnetVariants,14}:15with lib;16let17  inherit (cpp-jsonnet) jsonnetBench;18  inherit (go-jsonnet) goJsonnetBench;19  graalvmBench = fetchFromGitHub {20    owner = "oracle";21    repo = "graal";22    rev = "bc305df3fe587960f7635f0185571500e5988475";23    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";24  };25  realworldVendor = fetchJrq {26    name = "realworld-vendor";27    lockfile = ../tests/realworld/jsonnetfile.lock.json;28    vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";29  };30  realworldBench = runCommand "realworld-bench" { } ''31    mkdir -p $out32    cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/33    cp -r ${realworldVendor} $out/vendor34  '';3536  # Removes outsiders from the output37  # Useful when comparing performance of different jrsonnet releases38  skipSlow = if quick then "slow benchmark, but only quick requested" else "";39in40stdenv.mkDerivation {41  name = "benchmarks";42  # __impure = true; # not supported by hercules-ci43  unpackPhase = "true";4445  buildInputs = [46    sjsonnet47    cpp-jsonnet48    rsjsonnet49    go-jsonnet5051    hyperfine52  ];5354  installPhase =55    let56      mkBench =57        {58          name,59          path,60          omitSource ? false,61          pathIsGenerator ? false,62          skipRustAlternative ? "",63          skipScala ? "",64          skipCpp ? "",65          skipGo ? "",66          jpaths ? [ ],67        }:68        let69          jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;70        in71        ''72          echo >> $out73          echo "=== ${name}" >> $out74          echo >> $out75          ${optionalString (skipRustAlternative != "") ''76            echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out77            echo >> $out78          ''}79          ${optionalString (skipGo != "") ''80            echo "> Note: No results for Go, ${skipGo}" >> $out81            echo >> $out82          ''}83          ${optionalString (skipScala != "") ''84            echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out85            echo >> $out86          ''}87          ${optionalString (skipCpp != "") ''88            echo "> Note: No results for C++, ${skipCpp}" >> $out89            echo >> $out90          ''}91          ${optionalString (!omitSource) ''92            echo ".Source" >> $out93            echo "[%collapsible]" >> $out94            echo "====" >> $out95            echo "[source,jsonnet]" >> $out96            echo "----" >> $out97            ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}98            cat ${path} >> $out99            echo >> $out100            echo "----" >> $out101            echo "====" >> $out102            echo >> $out103          ''}104          path=${path}105          ${optionalString pathIsGenerator ''106            go-jsonnet $path > generated.jsonnet107            path=generated.jsonnet108          ''}109          hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \110            ${111              concatStringsSep " " (112                forEach jrsonnetVariants (113                  variant:114                  "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${115                    if variant.name != "" then " (${variant.name})" else ""116                  }\""117                )118              )119            } \120            ${121              optionalString (122                skipRustAlternative == ""123              ) "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""124            } \125            ${optionalString (skipGo == "") "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""} \126            ${127              optionalString (skipScala == "") "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""128            } \129            ${130              # My aarch64-linux machine can't run graalvm image:131              # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].132              optionalString (133                skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux"134              ) "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""135            } \136            ${optionalString (skipCpp == "") "\"jsonnet $path${jpathArgs}\" -n \"C++\""}137          cat result.adoc >> $out138        '';139    in140    ''141      set -oux142      ulimit -s unlimited143144      temp=$(mktemp -d)145      cd $temp146147      touch $out148      ${optionalString (true) ''149        cat ${./benchmarks.adoc} >> $out150        echo >> $out151152        echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out153        echo >> $out154155        echo ".Tested versions" >> $out156        echo "[%collapsible]" >> $out157        echo "====" >> $out158        echo "* Go: $(go-jsonnet --version)" >> $out159        echo "* C++: $(jsonnet --version)" >> $out160        echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out161        echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out162        ${concatStringsSep "\n" (163          forEach jrsonnetVariants (164            variant:165            "echo \"* Rust${166              if variant.name != "" then " (${variant.name})" else ""167            }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"168          )169        )}170        echo "====" >> $out171        echo >> $out172      ''}173      echo "== Real world" >> $out174      ${mkBench {175        name = "Graalvm CI";176        path = "${graalvmBench}/ci.jsonnet";177        omitSource = true;178        skipCpp = "takes longer than a hour";179        skipGo = skipSlow;180      }}181      ${mkBench {182        name = "Loki manifests";183        path = "${realworldBench}/entry-loki.jsonnet";184        jpaths = [ "${realworldBench}/vendor" ];185        skipCpp = "too slow, takes hours, skews results";186        skipGo = skipSlow;187      }}188      ${mkBench {189        name = "Mimir manifests";190        path = "${realworldBench}/entry-mimir.jsonnet";191        jpaths = [ "${realworldBench}/vendor" ];192        skipCpp = "too slow, takes hours, skews results";193        skipGo = skipSlow;194      }}195      ${mkBench {196        name = "Tempo manifests";197        path = "${realworldBench}/entry-tempo.jsonnet";198        jpaths = [ "${realworldBench}/vendor" ];199        skipCpp = "too slow, takes hours, skews results";200        skipGo = skipSlow;201      }}202      ${mkBench {203        name = "GitLab runbooks dashboards";204        path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";205        jpaths = [206          "${realworldBench}/vendor"207          "${realworldBench}/vendor/runbooks/libsonnet"208          "${realworldBench}/vendor/runbooks/dashboards"209          "${realworldBench}/vendor/runbooks/services"210          "${realworldBench}/vendor/runbooks/metrics-catalog"211        ];212        skipCpp = "too slow, takes hours, skews results";213        skipGo = skipSlow;214      }}215216      echo >> $out217      echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out218      ${mkBench {219        name = "Large string join";220        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";221      }}222      ${mkBench {223        name = "Large string template";224        omitSource = true;225        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";226        skipGo = "fails with os stack size exhausion";227        skipCpp = "too slow, takes hours, skews results";228      }}229      ${mkBench {230        name = "Realistic 1";231        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";232        skipGo = skipSlow;233        skipCpp = "too slow, takes hours, skews results";234      }}235      ${mkBench {236        name = "Realistic 2";237        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";238        skipGo = skipSlow;239        skipCpp = "too slow, takes hours, skews results";240      }}241242      echo >> $out243      echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out244      ${mkBench {245        name = "Tail call";246        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";247      }}248      ${mkBench {249        name = "Inheritance recursion";250        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";251        skipCpp = skipSlow;252        skipGo = skipSlow;253      }}254      ${mkBench {255        name = "Simple recursive call";256        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";257        skipGo = skipSlow;258      }}259      ${mkBench {260        name = "Foldl string concat";261        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";262        skipCpp = skipSlow;263      }}264      ${mkBench {265        name = "Array sorts";266        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";267        skipCpp = skipSlow;268      }}269      ${mkBench {270        name = "Lazy array";271        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";272        skipGo = skipSlow;273      }}274      ${mkBench {275        name = "Inheritance function recursion";276        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";277        skipCpp = skipSlow;278      }}279      ${mkBench {280        name = "String strips";281        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";282        skipCpp = "too slow, takes hours, skews results";283      }}284      ${mkBench {285        name = "Big object";286        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287        pathIsGenerator = true;288      }}289290      echo >> $out291      echo "== Benchmarks from Go jsonnet (builtins)" >> $out292      ${mkBench {293        name = "std.base64";294        path = "${goJsonnetBench}/base64.jsonnet";295        skipRustAlternative = skipSlow;296        skipCpp = "too slow, takes hours, skews results";297      }}298      ${mkBench {299        name = "std.base64Decode";300        path = "${goJsonnetBench}/base64Decode.jsonnet";301        skipRustAlternative = skipSlow;302        skipCpp = skipSlow;303      }}304      ${mkBench {305        name = "std.base64DecodeBytes";306        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";307        skipRustAlternative = skipSlow;308        skipCpp = skipSlow;309        skipGo = skipSlow;310      }}311      ${mkBench {312        name = "std.base64 (byte array)";313        path = "${goJsonnetBench}/base64_byte_array.jsonnet";314        skipRustAlternative = skipSlow;315        skipCpp = skipSlow;316        skipGo = skipSlow;317      }}318      ${mkBench {319        name = "std.foldl";320        path = "${goJsonnetBench}/foldl.jsonnet";321      }}322      ${mkBench {323        name = "std.manifestJsonEx";324        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";325        skipCpp = skipSlow;326      }}327      ${mkBench {328        name = "std.manifestTomlEx";329        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";330        skipCpp = skipSlow;331      }}332      ${mkBench {333        name = "std.parseInt";334        path = "${goJsonnetBench}/parseInt.jsonnet";335        skipCpp = skipSlow;336      }}337      ${mkBench {338        name = "std.reverse";339        path = "${goJsonnetBench}/reverse.jsonnet";340        skipCpp = skipSlow;341        skipGo = skipSlow;342      }}343      ${mkBench {344        name = "std.substr";345        path = "${goJsonnetBench}/substr.jsonnet";346      }}347      ${mkBench {348        name = "Comparsion for array";349        path = "${goJsonnetBench}/comparison.jsonnet";350        skipCpp = "too slow, takes hours, skews results";351      }}352      ${mkBench {353        name = "Comparsion for primitives";354        path = "${goJsonnetBench}/comparison2.jsonnet";355        skipRustAlternative = skipSlow;356        skipCpp = "can't run: uses up to 192GB of RAM";357        skipGo = skipSlow;358      }}359    '';360}