git.delta.rocks / jrsonnet / refs/commits / 31c77c8e98ed

1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  cpp-jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib; let17  jsonnetBench = fetchFromGitHub {18    rev = "v0.19.1";19    owner = "google";20    repo = "jsonnet";21    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22  };23  goJsonnetBench =24    (fetchFromGitHub {25      owner = "google";26      repo = "go-jsonnet";27      rev = "v0.19.1";28      hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29    })30    + "/builtin-benchmarks";31  graalvmBench = fetchFromGitHub {32    owner = "oracle";33    repo = "graal";34    rev = "bc305df3fe587960f7635f0185571500e5988475";35    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36  };37  kubePrometheusBench = let38    src = fetchFromGitHub {39      owner = "prometheus-operator";40      repo = "kube-prometheus";41      rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42      hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43    };44  in45    runCommand "kube-prometheus-vendor"46    {47      outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48      outputHashMode = "recursive";49      buildInputs = [cacert];50    }51    ''52      mkdir -p $out53      cp -r ${src}/* $out/54      cd $out55      chmod u+w jsonnetfile.lock.json56      mkdir vendor57      ${jsonnet-bundler}/bin/jb install58    '';5960  # Removes outsiders from the output61  # Useful when comparing performance of different jrsonnet releases62  skipSlow =63    if quick64    then "slow benchmark, but only quick requested"65    else "";66in67  stdenv.mkDerivation {68    name = "benchmarks";69    __impure = true;70    unpackPhase = "true";7172    buildInputs = [73      sjsonnet74      cpp-jsonnet75      rsjsonnet76      go-jsonnet7778      hyperfine79    ];8081    installPhase = let82      mkBench = {83        name,84        path,85        omitSource ? false,86        pathIsGenerator ? false,87        skipRustAlternative ? "",88        skipScala ? "",89        skipCpp ? "",90        skipGo ? "",91        vendor ? "",92      }: ''93        echo >> $out94        echo "### ${name}" >> $out95        echo >> $out96        ${optionalString (skipRustAlternative != "") ''97          echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out98          echo >> $out99        ''}100        ${optionalString (skipGo != "") ''101          echo "> Note: No results for Go, ${skipGo}" >> $out102          echo >> $out103        ''}104        ${optionalString (skipScala != "") ''105          echo "> Note: No results for Scala, ${skipScala}" >> $out106          echo >> $out107        ''}108        ${optionalString (skipCpp != "") ''109          echo "> Note: No results for C++, ${skipCpp}" >> $out110          echo >> $out111        ''}112        ${optionalString (!quick && !omitSource) ''113          echo "<details>" >> $out114          echo "<summary>Source</summary>" >> $out115          echo >> $out116          echo "\`\`\`jsonnet" >> $out117          ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}118          cat ${path} >> $out119          echo >> $out120          echo "\`\`\`" >> $out121          echo "</details>" >> $out122          echo >> $out123        ''}124        path=${path}125        ${optionalString pathIsGenerator ''126          go-jsonnet $path > generated.jsonnet127          path=generated.jsonnet128        ''}129        hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \130          ${concatStringsSep " " (forEach jrsonnetVariants (131          variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${132            if variant.name != ""133            then " (${variant.name})"134            else ""135          }\""136        ))} \137          ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \138          ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \139          ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \140          ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}141        cat result.adoc >> $out142      '';143    in ''144      set -oux145146      temp=$(mktemp -d)147      cd $temp148149      touch $out150      ${optionalString (!quick) ''151        cat ${./benchmarks.md} >> $out152        echo >> $out153154        echo "<details>" >> $out155        echo "<summary>Tested versions</summary>" >> $out156        echo >> $out157        echo Go: $(go-jsonnet --version) >> $out158        echo >> $out159        echo "\`\`\`" >> $out160        go-jsonnet --help >> $out161        echo "\`\`\`" >> $out162        echo >> $out163        echo C++: $(jsonnet --version) >> $out164        echo >> $out165        echo "\`\`\`" >> $out166        jsonnet --help >> $out167        echo "\`\`\`" >> $out168        echo >> $out169        echo Scala: >> $out170        echo >> $out171        echo "\`\`\`" >> $out172        sjsonnet 2>> $out || true173        echo "\`\`\`" >> $out174        echo >> $out175        echo "Rust (alternative):" >> $out176        echo >> $out177        echo "\`\`\`" >> $out178        rsjsonnet --help 2>> $out || true179        echo "\`\`\`" >> $out180        echo >> $out181        echo "</details>" >> $out182        echo >> $out183184        echo >> $out185      ''}186      echo "## Real world" >> $out187      ${mkBench {188        name = "Graalvm CI";189        path = "${graalvmBench}/ci.jsonnet";190        skipCpp = "takes longer than a hour";191        skipGo = skipSlow;192        skipScala = skipSlow;193      }}194      ${mkBench {195        name = "Kube-prometheus manifests";196        vendor = "${kubePrometheusBench}/vendor";197        path = "${kubePrometheusBench}/example.jsonnet";198        skipCpp = skipSlow;199        skipGo = skipSlow;200        skipScala = skipSlow;201      }}202203      echo >> $out204      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out205      ${mkBench {206        name = "Large string join";207        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";208        skipScala = skipSlow;209      }}210      ${mkBench {211        name = "Large string template";212        omitSource = true;213        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";214        skipGo = "fails with os stack size exhausion";215        skipCpp = skipSlow;216        skipScala = skipSlow;217      }}218      ${mkBench {219        name = "Realistic 1";220        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";221        skipGo = skipSlow;222        skipCpp = skipSlow;223        skipScala = skipSlow;224      }}225      ${mkBench {226        name = "Realistic 2";227        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";228        skipGo = skipSlow;229        skipCpp = skipSlow;230        skipScala = skipSlow;231      }}232233      echo >> $out234      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out235      ${mkBench {236        name = "Tail call";237        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";238        skipScala = skipSlow;239      }}240      ${mkBench {241        name = "Inheritance recursion";242        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";243        skipCpp = skipSlow;244        skipGo = skipSlow;245      }}246      ${mkBench {247        name = "Simple recursive call";248        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";249        skipScala = skipSlow;250        skipGo = skipSlow;251      }}252      ${mkBench {253        name = "Foldl string concat";254        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";255        skipCpp = skipSlow;256        skipScala = skipSlow;257      }}258      ${mkBench {259        name = "Array sorts";260        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";261        skipCpp = skipSlow;262        # std.assertEqual(reverse(std.range(1, 1000)), sort(std.range(1, 1000), keyF=function(x) -x))263        skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";264      }}265      ${mkBench {266        name = "Lazy array";267        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";268        skipGo = skipSlow;269        skipScala = skipSlow;270      }}271      ${mkBench {272        name = "Inheritance function recursion";273        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";274        skipCpp = skipSlow;275        skipScala = skipSlow;276      }}277      ${mkBench {278        name = "String strips";279        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";280        skipCpp = skipSlow;281        skipScala = skipSlow;282      }}283      ${mkBench {284        name = "Big object";285        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";286        pathIsGenerator = true;287        skipScala = skipSlow;288      }}289290      echo >> $out291      echo "## Benchmarks from Go jsonnet (builtins)" >> $out292      ${mkBench {293        name = "std.base64";294        path = "${goJsonnetBench}/base64.jsonnet";295        skipRustAlternative = skipSlow;296        skipCpp = skipSlow;297        skipScala = skipSlow;298      }}299      ${mkBench {300        name = "std.base64Decode";301        path = "${goJsonnetBench}/base64Decode.jsonnet";302        skipRustAlternative = skipSlow;303        skipCpp = skipSlow;304        skipScala = skipSlow;305      }}306      ${mkBench {307        name = "std.base64DecodeBytes";308        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";309        skipRustAlternative = skipSlow;310        skipCpp = skipSlow;311        skipGo = skipSlow;312        skipScala = skipSlow;313      }}314      ${mkBench {315        name = "std.base64 (byte array)";316        path = "${goJsonnetBench}/base64_byte_array.jsonnet";317        skipRustAlternative = skipSlow;318        skipCpp = skipSlow;319        skipGo = skipSlow;320        skipScala = skipSlow;321      }}322      ${mkBench {323        name = "std.foldl";324        path = "${goJsonnetBench}/foldl.jsonnet";325        skipScala = skipSlow;326      }}327      ${mkBench {328        name = "std.manifestJsonEx";329        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";330        skipScala = skipSlow;331        skipCpp = skipSlow;332      }}333      ${mkBench {334        name = "std.manifestTomlEx";335        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";336        skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";337        skipCpp = skipSlow;338      }}339      ${mkBench {340        name = "std.parseInt";341        path = "${goJsonnetBench}/parseInt.jsonnet";342        skipScala = skipSlow;343        skipCpp = skipSlow;344      }}345      ${mkBench {346        name = "std.reverse";347        path = "${goJsonnetBench}/reverse.jsonnet";348        skipCpp = skipSlow;349        skipGo = skipSlow;350      }}351      ${mkBench {352        name = "std.substr";353        path = "${goJsonnetBench}/substr.jsonnet";354        skipScala = skipSlow;355      }}356      ${mkBench {357        name = "Comparsion for array";358        path = "${goJsonnetBench}/comparison.jsonnet";359        skipScala = "array comparsion is not implemented";360        skipCpp = skipSlow;361      }}362      ${mkBench {363        name = "Comparsion for primitives";364        path = "${goJsonnetBench}/comparison2.jsonnet";365        skipRustAlternative = skipSlow;366        skipCpp = "can't run: uses up to 192GB of RAM";367        skipGo = skipSlow;368        skipScala = skipSlow;369      }}370    '';371  }