`644f9f3b456d`

1 file changed

modifiednix/benchmarks.nixdiff before after both
1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  cpp-jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib;17let18  inherit (cpp-jsonnet) jsonnetBench;19  inherit (go-jsonnet) goJsonnetBench;20  graalvmBench = fetchFromGitHub {21    owner = "oracle";22    repo = "graal";23    rev = "bc305df3fe587960f7635f0185571500e5988475";24    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25  };26  kubePrometheusBench =27    let28      src = fetchFromGitHub {29        owner = "prometheus-operator";30        repo = "kube-prometheus";31        rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32        hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33      };34    in35    runCommand "kube-prometheus-vendor"36      {37        outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38        outputHashMode = "recursive";39        buildInputs = [ cacert ];40      }41      ''42        mkdir -p $out43        cp -r ${src}/* $out/44        cd $out45        chmod u+w jsonnetfile.lock.json46        mkdir vendor47        ${jsonnet-bundler}/bin/jb install48      '';4950  # Removes outsiders from the output51  # Useful when comparing performance of different jrsonnet releases52  skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55  name = "benchmarks";56  # __impure = true; # not supported by hercules-ci57  unpackPhase = "true";5859  buildInputs = [60    sjsonnet61    cpp-jsonnet62    rsjsonnet63    go-jsonnet6465    hyperfine66  ];6768  installPhase =69    let70      mkBench =71        {72          name,73          path,74          omitSource ? false,75          pathIsGenerator ? false,76          skipRustAlternative ? "",77          skipScala ? "",78          skipCpp ? "",79          skipGo ? "",80          vendor ? "",81        }:82        ''83          echo >> $out84          echo "=== ${name}" >> $out85          echo >> $out86          ${optionalString (skipRustAlternative != "") ''87            echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88            echo >> $out89          ''}90          ${optionalString (skipGo != "") ''91            echo "> Note: No results for Go, ${skipGo}" >> $out92            echo >> $out93          ''}94          ${optionalString (skipScala != "") ''95            echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96            echo >> $out97          ''}98          ${optionalString (skipCpp != "") ''99            echo "> Note: No results for C++, ${skipCpp}" >> $out100            echo >> $out101          ''}102          ${optionalString (!omitSource) ''103            echo ".Source" >> $out104            echo "[%collapsible]" >> $out105            echo "====" >> $out106            echo "[source,jsonnet]" >> $out107            echo "----" >> $out108            ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109            cat ${path} >> $out110            echo >> $out111            echo "----" >> $out112            echo "====" >> $out113            echo >> $out114          ''}115          path=${path}116          ${optionalString pathIsGenerator ''117            go-jsonnet $path > generated.jsonnet118            path=generated.jsonnet119          ''}120          hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121            ${122              concatStringsSep " " (123                forEach jrsonnetVariants (124                  variant:125                  "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126                    if variant.name != "" then " (${variant.name})" else ""127                  }\""128                )129              )130            } \131            ${132              optionalString (skipRustAlternative == "")133                "\"rsjsonnet $path${134                  optionalString (vendor != "") " -J ${vendor}"135                }\" -n \"Rust (alternative, rsjsonnet)\""136            } \137            ${138              optionalString (skipGo == "")139                "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140            } \141            ${142              optionalString (skipScala == "")143                "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144            } \145            ${146              # My aarch64-linux machine can't run graalvm image:147              # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].148              optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")149                "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""150            } \151            ${optionalString (skipCpp == "")152              "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""153            }154          cat result.adoc >> $out155        '';156    in157    ''158      set -oux159      ulimit -s unlimited160161      temp=$(mktemp -d)162      cd $temp163164      touch $out165      ${optionalString (true) ''166        cat ${./benchmarks.adoc} >> $out167        echo >> $out168169        echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out170        echo >> $out171172        echo ".Tested versions" >> $out173        echo "[%collapsible]" >> $out174        echo "====" >> $out175        echo "* Go: $(go-jsonnet --version)" >> $out176        echo "* C++: $(jsonnet --version)" >> $out177        echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out178        echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out179        ${concatStringsSep "\n" (180          forEach jrsonnetVariants (181            variant:182            "echo \"* Rust${183              if variant.name != "" then " (${variant.name})" else ""184            }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"185          )186        )}187        echo "====" >> $out188        echo >> $out189      ''}190      echo "== Real world" >> $out191      ${mkBench {192        name = "Graalvm CI";193        path = "${graalvmBench}/ci.jsonnet";194        omitSource = true;195        skipCpp = "takes longer than a hour";196        skipGo = skipSlow;197      }}198      ${mkBench {199        name = "Kube-prometheus manifests";200        vendor = "${kubePrometheusBench}/vendor";201        path = "${kubePrometheusBench}/example.jsonnet";202        omitSource = true;203        skipCpp = "too slow, takes hours, skews results";204        skipGo = skipSlow;205      }}206207      echo >> $out208      echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out209      ${mkBench {210        name = "Large string join";211        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";212      }}213      ${mkBench {214        name = "Large string template";215        omitSource = true;216        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";217        skipGo = "fails with os stack size exhausion";218        skipCpp = "too slow, takes hours, skews results";219      }}220      ${mkBench {221        name = "Realistic 1";222        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";223        skipGo = skipSlow;224        skipCpp = "too slow, takes hours, skews results";225      }}226      ${mkBench {227        name = "Realistic 2";228        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229        skipGo = skipSlow;230        skipCpp = "too slow, takes hours, skews results";231      }}232233      echo >> $out234      echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out235      ${mkBench {236        name = "Tail call";237        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";238      }}239      ${mkBench {240        name = "Inheritance recursion";241        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242        skipCpp = skipSlow;243        skipGo = skipSlow;244      }}245      ${mkBench {246        name = "Simple recursive call";247        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248        skipGo = skipSlow;249      }}250      ${mkBench {251        name = "Foldl string concat";252        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";253        skipCpp = skipSlow;254      }}255      ${mkBench {256        name = "Array sorts";257        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";258        skipCpp = skipSlow;259      }}260      ${mkBench {261        name = "Lazy array";262        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";263        skipGo = skipSlow;264      }}265      ${mkBench {266        name = "Inheritance function recursion";267        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";268        skipCpp = skipSlow;269      }}270      ${mkBench {271        name = "String strips";272        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";273        skipCpp = "too slow, takes hours, skews results";274      }}275      ${mkBench {276        name = "Big object";277        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";278        pathIsGenerator = true;279      }}280281      echo >> $out282      echo "== Benchmarks from Go jsonnet (builtins)" >> $out283      ${mkBench {284        name = "std.base64";285        path = "${goJsonnetBench}/base64.jsonnet";286        skipRustAlternative = skipSlow;287        skipCpp = "too slow, takes hours, skews results";288      }}289      ${mkBench {290        name = "std.base64Decode";291        path = "${goJsonnetBench}/base64Decode.jsonnet";292        skipRustAlternative = skipSlow;293        skipCpp = skipSlow;294      }}295      ${mkBench {296        name = "std.base64DecodeBytes";297        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";298        skipRustAlternative = skipSlow;299        skipCpp = skipSlow;300        skipGo = skipSlow;301      }}302      ${mkBench {303        name = "std.base64 (byte array)";304        path = "${goJsonnetBench}/base64_byte_array.jsonnet";305        skipRustAlternative = skipSlow;306        skipCpp = skipSlow;307        skipGo = skipSlow;308      }}309      ${mkBench {310        name = "std.foldl";311        path = "${goJsonnetBench}/foldl.jsonnet";312      }}313      ${mkBench {314        name = "std.manifestJsonEx";315        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";316        skipCpp = skipSlow;317      }}318      ${mkBench {319        name = "std.manifestTomlEx";320        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";321        skipCpp = skipSlow;322      }}323      ${mkBench {324        name = "std.parseInt";325        path = "${goJsonnetBench}/parseInt.jsonnet";326        skipCpp = skipSlow;327      }}328      ${mkBench {329        name = "std.reverse";330        path = "${goJsonnetBench}/reverse.jsonnet";331        skipCpp = skipSlow;332        skipGo = skipSlow;333      }}334      ${mkBench {335        name = "std.substr";336        path = "${goJsonnetBench}/substr.jsonnet";337      }}338      ${mkBench {339        name = "Comparsion for array";340        path = "${goJsonnetBench}/comparison.jsonnet";341        skipCpp = "too slow, takes hours, skews results";342      }}343      ${mkBench {344        name = "Comparsion for primitives";345        path = "${goJsonnetBench}/comparison2.jsonnet";346        skipRustAlternative = skipSlow;347        skipCpp = "can't run: uses up to 192GB of RAM";348        skipGo = skipSlow;349      }}350    '';351}