git.delta.rocks / jrsonnet / refs/commits / 7d0b3bd3121d

1{2  lib,3  runCommand,4  stdenv,5  fetchFromGitHub,6  fetchJrq,7  go-jsonnet,8  sjsonnet,9  cpp-jsonnet,10  rsjsonnet,11  hyperfine,12  quick ? false,13  jrsonnetVariants,14}:15with lib;16let17  inherit (cpp-jsonnet) jsonnetBench;18  inherit (go-jsonnet) goJsonnetBench;19  graalvmBench = fetchFromGitHub {20    owner = "oracle";21    repo = "graal";22    rev = "bc305df3fe587960f7635f0185571500e5988475";23    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";24  };25  realworldVendor = fetchJrq {26    name = "realworld-vendor";27    lockfile = ../tests/realworld/jsonnetfile.lock.json;28    vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";29  };30  realworldBench = runCommand "realworld-bench" { } ''31    mkdir -p $out32    cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/33    cp -r ${realworldVendor} $out/vendor34  '';3536  # Removes outsiders from the output37  # Useful when comparing performance of different jrsonnet releases38  skipSlow = if quick then "slow benchmark, but only quick requested" else "";39in40stdenv.mkDerivation {41  name = "benchmarks";42  # __impure = true; # not supported by hercules-ci43  unpackPhase = "true";4445  buildInputs = [46    sjsonnet47    cpp-jsonnet48    rsjsonnet49    go-jsonnet5051    hyperfine52  ];5354  installPhase =55    let56      mkBench =57        {58          name,59          path,60          omitSource ? false,61          pathIsGenerator ? false,62          skipRustAlternative ? "",63          skipScala ? "",64          skipCpp ? "",65          skipGo ? "",66          jpaths ? [ ],67        }:68        let69          jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;70        in71        ''72          echo >> $out73          echo "=== ${name}" >> $out74          echo >> $out75          ${optionalString (skipRustAlternative != "") ''76            echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out77            echo >> $out78          ''}79          ${optionalString (skipGo != "") ''80            echo "> Note: No results for Go, ${skipGo}" >> $out81            echo >> $out82          ''}83          ${optionalString (skipScala != "") ''84            echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out85            echo >> $out86          ''}87          ${optionalString (skipCpp != "") ''88            echo "> Note: No results for C++, ${skipCpp}" >> $out89            echo >> $out90          ''}91          ${optionalString (!omitSource) ''92            echo ".Source" >> $out93            echo "[%collapsible]" >> $out94            echo "====" >> $out95            echo "[source,jsonnet]" >> $out96            echo "----" >> $out97            ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}98            cat ${path} >> $out99            echo >> $out100            echo "----" >> $out101            echo "====" >> $out102            echo >> $out103          ''}104          path=${path}105          ${optionalString pathIsGenerator ''106            go-jsonnet $path > generated.jsonnet107            path=generated.jsonnet108          ''}109          hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \110            ${111              concatStringsSep " " (112                forEach jrsonnetVariants (113                  variant:114                  "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${115                    if variant.name != "" then " (${variant.name})" else ""116                  }\""117                )118              )119            } \120            ${121              optionalString (skipRustAlternative == "")122                "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""123            } \124            ${125              optionalString (skipGo == "")126                "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""127            } \128            ${129              optionalString (skipScala == "")130                "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""131            } \132            ${133              # My aarch64-linux machine can't run graalvm image:134              # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].135              optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")136                "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""137            } \138            ${optionalString (skipCpp == "")139              "\"jsonnet $path${jpathArgs}\" -n \"C++\""140            }141          cat result.adoc >> $out142        '';143    in144    ''145      set -oux146      ulimit -s unlimited147148      temp=$(mktemp -d)149      cd $temp150151      touch $out152      ${optionalString (true) ''153        cat ${./benchmarks.adoc} >> $out154        echo >> $out155156        echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out157        echo >> $out158159        echo ".Tested versions" >> $out160        echo "[%collapsible]" >> $out161        echo "====" >> $out162        echo "* Go: $(go-jsonnet --version)" >> $out163        echo "* C++: $(jsonnet --version)" >> $out164        echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out165        echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out166        ${concatStringsSep "\n" (167          forEach jrsonnetVariants (168            variant:169            "echo \"* Rust${170              if variant.name != "" then " (${variant.name})" else ""171            }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"172          )173        )}174        echo "====" >> $out175        echo >> $out176      ''}177      echo "== Real world" >> $out178      ${mkBench {179        name = "Graalvm CI";180        path = "${graalvmBench}/ci.jsonnet";181        omitSource = true;182        skipCpp = "takes longer than a hour";183        skipGo = skipSlow;184      }}185      ${mkBench {186        name = "Loki manifests";187        path = "${realworldBench}/entry-loki.jsonnet";188        jpaths = [ "${realworldBench}/vendor" ];189        skipCpp = "too slow, takes hours, skews results";190        skipGo = skipSlow;191      }}192      ${mkBench {193        name = "Mimir manifests";194        path = "${realworldBench}/entry-mimir.jsonnet";195        jpaths = [ "${realworldBench}/vendor" ];196        skipCpp = "too slow, takes hours, skews results";197        skipGo = skipSlow;198      }}199      ${mkBench {200        name = "Tempo manifests";201        path = "${realworldBench}/entry-tempo.jsonnet";202        jpaths = [ "${realworldBench}/vendor" ];203        skipCpp = "too slow, takes hours, skews results";204        skipGo = skipSlow;205      }}206      ${mkBench {207        name = "GitLab runbooks dashboards";208        path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";209        jpaths = [210          "${realworldBench}/vendor"211          "${realworldBench}/vendor/runbooks/libsonnet"212          "${realworldBench}/vendor/runbooks/dashboards"213          "${realworldBench}/vendor/runbooks/services"214          "${realworldBench}/vendor/runbooks/metrics-catalog"215        ];216        skipCpp = "too slow, takes hours, skews results";217        skipGo = skipSlow;218      }}219220      echo >> $out221      echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out222      ${mkBench {223        name = "Large string join";224        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";225      }}226      ${mkBench {227        name = "Large string template";228        omitSource = true;229        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";230        skipGo = "fails with os stack size exhausion";231        skipCpp = "too slow, takes hours, skews results";232      }}233      ${mkBench {234        name = "Realistic 1";235        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";236        skipGo = skipSlow;237        skipCpp = "too slow, takes hours, skews results";238      }}239      ${mkBench {240        name = "Realistic 2";241        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";242        skipGo = skipSlow;243        skipCpp = "too slow, takes hours, skews results";244      }}245246      echo >> $out247      echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out248      ${mkBench {249        name = "Tail call";250        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";251      }}252      ${mkBench {253        name = "Inheritance recursion";254        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";255        skipCpp = skipSlow;256        skipGo = skipSlow;257      }}258      ${mkBench {259        name = "Simple recursive call";260        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";261        skipGo = skipSlow;262      }}263      ${mkBench {264        name = "Foldl string concat";265        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";266        skipCpp = skipSlow;267      }}268      ${mkBench {269        name = "Array sorts";270        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";271        skipCpp = skipSlow;272      }}273      ${mkBench {274        name = "Lazy array";275        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";276        skipGo = skipSlow;277      }}278      ${mkBench {279        name = "Inheritance function recursion";280        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";281        skipCpp = skipSlow;282      }}283      ${mkBench {284        name = "String strips";285        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";286        skipCpp = "too slow, takes hours, skews results";287      }}288      ${mkBench {289        name = "Big object";290        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";291        pathIsGenerator = true;292      }}293294      echo >> $out295      echo "== Benchmarks from Go jsonnet (builtins)" >> $out296      ${mkBench {297        name = "std.base64";298        path = "${goJsonnetBench}/base64.jsonnet";299        skipRustAlternative = skipSlow;300        skipCpp = "too slow, takes hours, skews results";301      }}302      ${mkBench {303        name = "std.base64Decode";304        path = "${goJsonnetBench}/base64Decode.jsonnet";305        skipRustAlternative = skipSlow;306        skipCpp = skipSlow;307      }}308      ${mkBench {309        name = "std.base64DecodeBytes";310        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";311        skipRustAlternative = skipSlow;312        skipCpp = skipSlow;313        skipGo = skipSlow;314      }}315      ${mkBench {316        name = "std.base64 (byte array)";317        path = "${goJsonnetBench}/base64_byte_array.jsonnet";318        skipRustAlternative = skipSlow;319        skipCpp = skipSlow;320        skipGo = skipSlow;321      }}322      ${mkBench {323        name = "std.foldl";324        path = "${goJsonnetBench}/foldl.jsonnet";325      }}326      ${mkBench {327        name = "std.manifestJsonEx";328        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";329        skipCpp = skipSlow;330      }}331      ${mkBench {332        name = "std.manifestTomlEx";333        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";334        skipCpp = skipSlow;335      }}336      ${mkBench {337        name = "std.parseInt";338        path = "${goJsonnetBench}/parseInt.jsonnet";339        skipCpp = skipSlow;340      }}341      ${mkBench {342        name = "std.reverse";343        path = "${goJsonnetBench}/reverse.jsonnet";344        skipCpp = skipSlow;345        skipGo = skipSlow;346      }}347      ${mkBench {348        name = "std.substr";349        path = "${goJsonnetBench}/substr.jsonnet";350      }}351      ${mkBench {352        name = "Comparsion for array";353        path = "${goJsonnetBench}/comparison.jsonnet";354        skipCpp = "too slow, takes hours, skews results";355      }}356      ${mkBench {357        name = "Comparsion for primitives";358        path = "${goJsonnetBench}/comparison2.jsonnet";359        skipRustAlternative = skipSlow;360        skipCpp = "can't run: uses up to 192GB of RAM";361        skipGo = skipSlow;362      }}363    '';364}