git.delta.rocks / jrsonnet / refs/commits / ad68a2495da3

1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib; let17  jsonnetBench = fetchFromGitHub {18    rev = "v0.19.1";19    owner = "google";20    repo = "jsonnet";21    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22  };23  goJsonnetBench =24    (fetchFromGitHub {25      owner = "google";26      repo = "go-jsonnet";27      rev = "v0.19.1";28      hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29    })30    + "/builtin-benchmarks";31  graalvmBench = fetchFromGitHub {32    owner = "oracle";33    repo = "graal";34    rev = "bc305df3fe587960f7635f0185571500e5988475";35    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36  };37  kubePrometheusBench = let38    src = fetchFromGitHub {39      owner = "prometheus-operator";40      repo = "kube-prometheus";41      rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42      hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43    };44  in45    runCommand "kube-prometheus-vendor"46    {47      outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48      outputHashMode = "recursive";49      buildInputs = [cacert];50    }51    ''52      mkdir -p $out53      cp -r ${src}/* $out/54      cd $out55      mkdir vendor56      ${jsonnet-bundler}/bin/jb install57    '';5859  # Removes outsiders from the output60  # Useful when comparing performance of different jrsonnet releases61  skipSlow =62    if quick63    then "slow benchmark, but only quick requested"64    else "";65in66  stdenv.mkDerivation {67    name = "benchmarks";68    __impure = true;69    unpackPhase = "true";7071    buildInputs = [72      sjsonnet73      jsonnet74      rsjsonnet75      go-jsonnet7677      hyperfine78    ];7980    installPhase = let81      mkBench = {82        name,83        path,84        omitSource ? false,85        pathIsGenerator ? false,86        skipRustAlternative ? "",87        skipScala ? "",88        skipCpp ? "",89        skipGo ? "",90        vendor ? "",91      }: ''92        echo >> $out93        echo "### ${name}" >> $out94        echo >> $out95        ${optionalString (skipRustAlternative != "") ''96          echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out97          echo >> $out98        ''}99        ${optionalString (skipGo != "") ''100          echo "> Note: No results for Go, ${skipGo}" >> $out101          echo >> $out102        ''}103        ${optionalString (skipScala != "") ''104          echo "> Note: No results for Scala, ${skipScala}" >> $out105          echo >> $out106        ''}107        ${optionalString (skipCpp != "") ''108          echo "> Note: No results for C++, ${skipCpp}" >> $out109          echo >> $out110        ''}111        ${optionalString (!quick && !omitSource) ''112          echo "<details>" >> $out113          echo "<summary>Source</summary>" >> $out114          echo >> $out115          echo "\`\`\`jsonnet" >> $out116          ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}117          cat ${path} >> $out118          echo >> $out119          echo "\`\`\`" >> $out120          echo "</details>" >> $out121          echo >> $out122        ''}123        path=${path}124        ${optionalString pathIsGenerator ''125          go-jsonnet $path > generated.jsonnet126          path=generated.jsonnet127        ''}128        hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \129          ${concatStringsSep " " (forEach jrsonnetVariants (130          variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${131            if variant.name != ""132            then " (${variant.name})"133            else ""134          }\""135        ))} \136          ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \137          ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \138          ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \139          ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}140        cat result.md >> $out141      '';142    in ''143      set -oux144145      temp=$(mktemp -d)146      cd $temp147148      touch $out149      ${optionalString (!quick) ''150        cat ${./benchmarks.md} >> $out151        echo >> $out152153        echo "<details>" >> $out154        echo "<summary>Tested versions</summary>" >> $out155        echo >> $out156        echo Go: $(go-jsonnet --version) >> $out157        echo >> $out158        echo "\`\`\`" >> $out159        go-jsonnet --help >> $out160        echo "\`\`\`" >> $out161        echo >> $out162        echo C++: $(jsonnet --version) >> $out163        echo >> $out164        echo "\`\`\`" >> $out165        jsonnet --help >> $out166        echo "\`\`\`" >> $out167        echo >> $out168        echo Scala: >> $out169        echo >> $out170        echo "\`\`\`" >> $out171        sjsonnet 2>> $out || true172        echo "\`\`\`" >> $out173        echo >> $out174        echo "</details>" >> $out175        echo >> $out176        echo "Rust (alternative):" >> $out177        echo >> $out178        echo "\`\`\`" >> $out179        rsjsonnet --help 2>> $out || true180        echo "\`\`\`" >> $out181        echo >> $out182        echo "</details>" >> $out183        echo >> $out184185        echo >> $out186      ''}187      echo "## Real world" >> $out188      ${mkBench {189        name = "Graalvm CI";190        path = "${graalvmBench}/ci.jsonnet";191        skipCpp = "takes longer than a hour";192        skipGo = skipSlow;193        skipScala = skipSlow;194      }}195      ${mkBench {196        name = "Kube-prometheus manifests";197        vendor = "${kubePrometheusBench}/vendor";198        path = "${kubePrometheusBench}/example.jsonnet";199        skipCpp = skipSlow;200        skipGo = skipSlow;201        skipScala = skipSlow;202      }}203204      echo >> $out205      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out206      ${mkBench {207        name = "Large string join";208        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";209        skipScala = skipSlow;210      }}211      ${mkBench {212        name = "Large string template";213        omitSource = true;214        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";215        skipGo = "fails with os stack size exhausion";216        skipCpp = skipSlow;217        skipScala = skipSlow;218      }}219      ${mkBench {220        name = "Realistic 1";221        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";222        skipGo = skipSlow;223        skipCpp = skipSlow;224        skipScala = skipSlow;225      }}226      ${mkBench {227        name = "Realistic 2";228        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229        skipGo = skipSlow;230        skipCpp = skipSlow;231        skipScala = skipSlow;232      }}233234      echo >> $out235      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out236      ${mkBench {237        name = "Tail call";238        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";239        skipScala = skipSlow;240      }}241      ${mkBench {242        name = "Inheritance recursion";243        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";244        skipCpp = skipSlow;245        skipGo = skipSlow;246      }}247      ${mkBench {248        name = "Simple recursive call";249        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";250        skipScala = skipSlow;251        skipGo = skipSlow;252      }}253      ${mkBench {254        name = "Foldl string concat";255        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";256        skipCpp = skipSlow;257        skipScala = skipSlow;258      }}259      ${mkBench {260        name = "Array sorts";261        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";262        skipCpp = skipSlow;263        # std.assertEqual(reverse(std.range(1, 1000)), sort(std.range(1, 1000), keyF=function(x) -x))264        skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";265      }}266      ${mkBench {267        name = "Lazy array";268        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";269        skipGo = skipSlow;270        skipScala = skipSlow;271      }}272      ${mkBench {273        name = "Inheritance function recursion";274        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";275        skipCpp = skipSlow;276        skipScala = skipSlow;277      }}278      ${mkBench {279        name = "String strips";280        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";281        skipCpp = skipSlow;282        skipScala = skipSlow;283      }}284      ${mkBench {285        name = "Big object";286        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287        pathIsGenerator = true;288        skipScala = skipSlow;289      }}290291      echo >> $out292      echo "## Benchmarks from Go jsonnet (builtins)" >> $out293      ${mkBench {294        name = "std.base64";295        path = "${goJsonnetBench}/base64.jsonnet";296        skipRustAlternative = skipSlow;297        skipCpp = skipSlow;298        skipScala = skipSlow;299      }}300      ${mkBench {301        name = "std.base64Decode";302        path = "${goJsonnetBench}/base64Decode.jsonnet";303        skipRustAlternative = skipSlow;304        skipCpp = skipSlow;305        skipScala = skipSlow;306      }}307      ${mkBench {308        name = "std.base64DecodeBytes";309        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";310        skipRustAlternative = skipSlow;311        skipCpp = skipSlow;312        skipGo = skipSlow;313        skipScala = skipSlow;314      }}315      ${mkBench {316        name = "std.base64 (byte array)";317        path = "${goJsonnetBench}/base64_byte_array.jsonnet";318        skipRustAlternative = skipSlow;319        skipCpp = skipSlow;320        skipGo = skipSlow;321        skipScala = skipSlow;322      }}323      ${mkBench {324        name = "std.foldl";325        path = "${goJsonnetBench}/foldl.jsonnet";326        skipScala = skipSlow;327      }}328      ${mkBench {329        name = "std.manifestJsonEx";330        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";331        skipScala = skipSlow;332        skipCpp = skipSlow;333      }}334      ${mkBench {335        name = "std.manifestTomlEx";336        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";337        skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";338        skipCpp = skipSlow;339      }}340      ${mkBench {341        name = "std.parseInt";342        path = "${goJsonnetBench}/parseInt.jsonnet";343        skipScala = skipSlow;344        skipCpp = skipSlow;345      }}346      ${mkBench {347        name = "std.reverse";348        path = "${goJsonnetBench}/reverse.jsonnet";349        skipCpp = skipSlow;350        skipGo = skipSlow;351      }}352      ${mkBench {353        name = "std.substr";354        path = "${goJsonnetBench}/substr.jsonnet";355        skipScala = skipSlow;356      }}357      ${mkBench {358        name = "Comparsion for array";359        path = "${goJsonnetBench}/comparison.jsonnet";360        skipScala = "array comparsion is not implemented";361        skipCpp = skipSlow;362      }}363      ${mkBench {364        name = "Comparsion for primitives";365        path = "${goJsonnetBench}/comparison2.jsonnet";366        skipRustAlternative = skipSlow;367        skipCpp = "can't run: uses up to 192GB of RAM";368        skipGo = skipSlow;369        skipScala = skipSlow;370      }}371    '';372  }