`644f9f3b456d`

1 file changed

modifiednix/benchmarks.nixdiff before after both
1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  cpp-jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib;17let18  inherit (cpp-jsonnet) jsonnetBench;19  inherit (go-jsonnet) goJsonnetBench;20  graalvmBench = fetchFromGitHub {21    owner = "oracle";22    repo = "graal";23    rev = "bc305df3fe587960f7635f0185571500e5988475";24    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";25  };26  kubePrometheusBench =27    let28      src = fetchFromGitHub {29        owner = "prometheus-operator";30        repo = "kube-prometheus";31        rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";32        hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";33      };34    in35    runCommand "kube-prometheus-vendor"36      {37        outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";38        outputHashMode = "recursive";39        buildInputs = [ cacert ];40      }41      ''42        mkdir -p $out43        cp -r ${src}/* $out/44        cd $out45        chmod u+w jsonnetfile.lock.json46        mkdir vendor47        ${jsonnet-bundler}/bin/jb install48      '';4950  # Removes outsiders from the output51  # Useful when comparing performance of different jrsonnet releases52  skipSlow = if quick then "slow benchmark, but only quick requested" else "";53in54stdenv.mkDerivation {55  name = "benchmarks";56  # __impure = true; # not supported by hercules-ci57  unpackPhase = "true";5859  buildInputs = [60    sjsonnet61    cpp-jsonnet62    rsjsonnet63    go-jsonnet6465    hyperfine66  ];6768  installPhase =69    let70      mkBench =71        {72          name,73          path,74          omitSource ? false,75          pathIsGenerator ? false,76          skipRustAlternative ? "",77          skipScala ? "",78          skipCpp ? "",79          skipGo ? "",80          vendor ? "",81        }:82        ''83          echo >> $out84          echo "=== ${name}" >> $out85          echo >> $out86          ${optionalString (skipRustAlternative != "") ''87            echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88            echo >> $out89          ''}90          ${optionalString (skipGo != "") ''91            echo "> Note: No results for Go, ${skipGo}" >> $out92            echo >> $out93          ''}94          ${optionalString (skipScala != "") ''95            echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96            echo >> $out97          ''}98          ${optionalString (skipCpp != "") ''99            echo "> Note: No results for C++, ${skipCpp}" >> $out100            echo >> $out101          ''}102          ${optionalString (!quick && !omitSource) ''103            echo ".Source" >> $out104            echo "[%collapsible]" >> $out105            echo "====" >> $out106            echo "[source,jsonnet]" >> $out107            echo "----" >> $out108            ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109            cat ${path} >> $out110            echo >> $out111            echo "----" >> $out112            echo "====" >> $out113            echo >> $out114          ''}115          path=${path}116          ${optionalString pathIsGenerator ''117            go-jsonnet $path > generated.jsonnet118            path=generated.jsonnet119          ''}120          hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121            ${122              concatStringsSep " " (123                forEach jrsonnetVariants (124                  variant:125                  "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126                    if variant.name != "" then " (${variant.name})" else ""127                  }\""128                )129              )130            } \131            ${132              optionalString (skipRustAlternative == "")133                "\"rsjsonnet $path${134                  optionalString (vendor != "") " -J ${vendor}"135                }\" -n \"Rust (alternative, rsjsonnet)\""136            } \137            ${138              optionalString (skipGo == "")139                "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140            } \141            ${142              optionalString (skipScala == "")143                "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144            } \145            ${146              # My aarch64-linux machine can't run graalvm image:147              # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].148              optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")149                "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""150            } \151            ${optionalString (skipCpp == "")152              "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""153            }154          cat result.adoc >> $out155        '';156    in157    ''158      set -oux159      ulimit -s unlimited160161      temp=$(mktemp -d)162      cd $temp163164      touch $out165      ${optionalString (!quick) ''166        cat ${./benchmarks.adoc} >> $out167        echo >> $out168169        echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out170        echo >> $out171172        echo ".Tested versions" >> $out173        echo "[%collapsible]" >> $out174        echo "====" >> $out175        echo "* Go: $(go-jsonnet --version)" >> $out176        echo "* C++: $(jsonnet --version)" >> $out177        echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out178        echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out179        ${concatStringsSep "\n" (180          forEach jrsonnetVariants (181            variant:182            "echo \"* Rust${183              if variant.name != "" then " (${variant.name})" else ""184            }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"185          )186        )}187        echo "====" >> $out188        echo >> $out189      ''}190      echo "== Real world" >> $out191      ${mkBench {192        name = "Graalvm CI";193        path = "${graalvmBench}/ci.jsonnet";194        omitSource = true;195        skipCpp = "takes longer than a hour";196        skipGo = skipSlow;197        skipScala = skipSlow;198      }}199      ${mkBench {200        name = "Kube-prometheus manifests";201        vendor = "${kubePrometheusBench}/vendor";202        path = "${kubePrometheusBench}/example.jsonnet";203        omitSource = true;204        skipCpp = "too slow, takes hours, skews results";205        skipGo = skipSlow;206        skipScala = skipSlow;207      }}208209      echo >> $out210      echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out211      ${mkBench {212        name = "Large string join";213        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";214        skipScala = skipSlow;215      }}216      ${mkBench {217        name = "Large string template";218        omitSource = true;219        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";220        skipGo = "fails with os stack size exhausion";221        skipCpp = "too slow, takes hours, skews results";222        skipScala = skipSlow;223      }}224      ${mkBench {225        name = "Realistic 1";226        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";227        skipGo = skipSlow;228        skipCpp = "too slow, takes hours, skews results";229        skipScala = skipSlow;230      }}231      ${mkBench {232        name = "Realistic 2";233        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";234        skipGo = skipSlow;235        skipCpp = "too slow, takes hours, skews results";236        skipScala = skipSlow;237      }}238239      echo >> $out240      echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out241      ${mkBench {242        name = "Tail call";243        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";244        skipScala = skipSlow;245      }}246      ${mkBench {247        name = "Inheritance recursion";248        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";249        skipCpp = skipSlow;250        skipGo = skipSlow;251      }}252      ${mkBench {253        name = "Simple recursive call";254        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";255        skipScala = skipSlow;256        skipGo = skipSlow;257      }}258      ${mkBench {259        name = "Foldl string concat";260        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";261        skipCpp = skipSlow;262        skipScala = skipSlow;263      }}264      ${mkBench {265        name = "Array sorts";266        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";267        skipCpp = skipSlow;268      }}269      ${mkBench {270        name = "Lazy array";271        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";272        skipGo = skipSlow;273      }}274      ${mkBench {275        name = "Inheritance function recursion";276        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";277        skipCpp = skipSlow;278        skipScala = skipSlow;279      }}280      ${mkBench {281        name = "String strips";282        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";283        skipCpp = "too slow, takes hours, skews results";284        skipScala = skipSlow;285      }}286      ${mkBench {287        name = "Big object";288        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";289        pathIsGenerator = true;290        skipScala = skipSlow;291      }}292293      echo >> $out294      echo "== Benchmarks from Go jsonnet (builtins)" >> $out295      ${mkBench {296        name = "std.base64";297        path = "${goJsonnetBench}/base64.jsonnet";298        skipRustAlternative = skipSlow;299        skipCpp = "too slow, takes hours, skews results";300        skipScala = skipSlow;301      }}302      ${mkBench {303        name = "std.base64Decode";304        path = "${goJsonnetBench}/base64Decode.jsonnet";305        skipRustAlternative = skipSlow;306        skipCpp = skipSlow;307        skipScala = skipSlow;308      }}309      ${mkBench {310        name = "std.base64DecodeBytes";311        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";312        skipRustAlternative = skipSlow;313        skipCpp = skipSlow;314        skipGo = skipSlow;315        skipScala = skipSlow;316      }}317      ${mkBench {318        name = "std.base64 (byte array)";319        path = "${goJsonnetBench}/base64_byte_array.jsonnet";320        skipRustAlternative = skipSlow;321        skipCpp = skipSlow;322        skipGo = skipSlow;323        skipScala = skipSlow;324      }}325      ${mkBench {326        name = "std.foldl";327        path = "${goJsonnetBench}/foldl.jsonnet";328        skipScala = skipSlow;329      }}330      ${mkBench {331        name = "std.manifestJsonEx";332        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";333        skipScala = skipSlow;334        skipCpp = skipSlow;335      }}336      ${mkBench {337        name = "std.manifestTomlEx";338        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";339        skipCpp = skipSlow;340      }}341      ${mkBench {342        name = "std.parseInt";343        path = "${goJsonnetBench}/parseInt.jsonnet";344        skipScala = skipSlow;345        skipCpp = skipSlow;346      }}347      ${mkBench {348        name = "std.reverse";349        path = "${goJsonnetBench}/reverse.jsonnet";350        skipCpp = skipSlow;351        skipGo = skipSlow;352      }}353      ${mkBench {354        name = "std.substr";355        path = "${goJsonnetBench}/substr.jsonnet";356        skipScala = skipSlow;357      }}358      ${mkBench {359        name = "Comparsion for array";360        path = "${goJsonnetBench}/comparison.jsonnet";361        skipCpp = "too slow, takes hours, skews results";362      }}363      ${mkBench {364        name = "Comparsion for primitives";365        path = "${goJsonnetBench}/comparison2.jsonnet";366        skipRustAlternative = skipSlow;367        skipCpp = "can't run: uses up to 192GB of RAM";368        skipGo = skipSlow;369        skipScala = skipSlow;370      }}371    '';372}