git.delta.rocks / jrsonnet / refs/commits / ad68a2495da3

1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib; let17  jsonnetBench = fetchFromGitHub {18    rev = "v0.19.1";19    owner = "google";20    repo = "jsonnet";21    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22  };23  goJsonnetBench =24    (fetchFromGitHub {25      owner = "google";26      repo = "go-jsonnet";27      rev = "v0.19.1";28      hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29    })30    + "/builtin-benchmarks";31  graalvmBench = fetchFromGitHub {32    owner = "oracle";33    repo = "graal";34    rev = "bc305df3fe587960f7635f0185571500e5988475";35    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36  };37  kubePrometheusBench = let38    src = fetchFromGitHub {39      owner = "prometheus-operator";40      repo = "kube-prometheus";41      rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42      hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43    };44  in45    runCommand "kube-prometheus-vendor"46    {47      outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48      outputHashMode = "recursive";49      buildInputs = [cacert];50    }51    ''52      mkdir -p $out53      cp -r ${src}/* $out/54      cd $out55      mkdir vendor56      ${jsonnet-bundler}/bin/jb install57    '';5859  # Removes outsiders from the output60  # Useful when comparing performance of different jrsonnet releases61  skipSlow =62    if quick63    then "slow benchmark, but only quick requested"64    else "";65in66  stdenv.mkDerivation {67    name = "benchmarks";68    __impure = true;69    unpackPhase = "true";7071    buildInputs = [72      sjsonnet73      jsonnet74      rsjsonnet75      go-jsonnet7677      hyperfine78    ];7980    installPhase = let81      mkBench = {82        name,83        path,84        omitSource ? false,85        pathIsGenerator ? false,86        skipRustAlternative ? "",87        skipScala ? "",88        skipCpp ? "",89        skipGo ? "",90        vendor ? "",91      }: ''92        echo >> $out93        echo "### ${name}" >> $out94        echo >> $out95        ${optionalString (skipRustAlternative != "") ''96          echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out97          echo >> $out98        ''}99        ${optionalString (skipGo != "") ''100          echo "> Note: No results for Go, ${skipGo}" >> $out101          echo >> $out102        ''}103        ${optionalString (skipScala != "") ''104          echo "> Note: No results for Scala, ${skipScala}" >> $out105          echo >> $out106        ''}107        ${optionalString (skipCpp != "") ''108          echo "> Note: No results for C++, ${skipCpp}" >> $out109          echo >> $out110        ''}111        ${optionalString (!quick && !omitSource) ''112          echo "<details>" >> $out113          echo "<summary>Source</summary>" >> $out114          echo >> $out115          echo "\`\`\`jsonnet" >> $out116          ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}117          cat ${path} >> $out118          echo >> $out119          echo "\`\`\`" >> $out120          echo "</details>" >> $out121          echo >> $out122        ''}123        path=${path}124        ${optionalString pathIsGenerator ''125          go-jsonnet $path > generated.jsonnet126          path=generated.jsonnet127        ''}128        hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \129          ${concatStringsSep " " (forEach jrsonnetVariants (130          variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${131            if variant.name != ""132            then " (${variant.name})"133            else ""134          }\""135        ))} \136          ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \137          ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \138          ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \139          ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}140        cat result.md >> $out141      '';142    in ''143      set -oux144145      temp=$(mktemp -d)146      cd $temp147148      touch $out149      ${optionalString (!quick) ''150        cat ${./benchmarks.md} >> $out151        echo >> $out152153        echo "<details>" >> $out154        echo "<summary>Tested versions</summary>" >> $out155        echo >> $out156        echo Go: $(go-jsonnet --version) >> $out157        echo >> $out158        echo "\`\`\`" >> $out159        go-jsonnet --help >> $out160        echo "\`\`\`" >> $out161        echo >> $out162        echo C++: $(jsonnet --version) >> $out163        echo >> $out164        echo "\`\`\`" >> $out165        jsonnet --help >> $out166        echo "\`\`\`" >> $out167        echo >> $out168        echo Scala: >> $out169        echo >> $out170        echo "\`\`\`" >> $out171        sjsonnet 2>> $out || true172        echo "\`\`\`" >> $out173        echo >> $out174        echo "Rust (alternative):" >> $out175        echo >> $out176        echo "\`\`\`" >> $out177        rsjsonnet --help 2>> $out || true178        echo "\`\`\`" >> $out179        echo >> $out180        echo "</details>" >> $out181        echo >> $out182183        echo >> $out184      ''}185      echo "## Real world" >> $out186      ${mkBench {187        name = "Graalvm CI";188        path = "${graalvmBench}/ci.jsonnet";189        skipCpp = "takes longer than a hour";190        skipGo = skipSlow;191        skipScala = skipSlow;192      }}193      ${mkBench {194        name = "Kube-prometheus manifests";195        vendor = "${kubePrometheusBench}/vendor";196        path = "${kubePrometheusBench}/example.jsonnet";197        skipCpp = skipSlow;198        skipGo = skipSlow;199        skipScala = skipSlow;200      }}201202      echo >> $out203      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out204      ${mkBench {205        name = "Large string join";206        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";207        skipScala = skipSlow;208      }}209      ${mkBench {210        name = "Large string template";211        omitSource = true;212        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";213        skipGo = "fails with os stack size exhausion";214        skipCpp = skipSlow;215        skipScala = skipSlow;216      }}217      ${mkBench {218        name = "Realistic 1";219        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";220        skipGo = skipSlow;221        skipCpp = skipSlow;222        skipScala = skipSlow;223      }}224      ${mkBench {225        name = "Realistic 2";226        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";227        skipGo = skipSlow;228        skipCpp = skipSlow;229        skipScala = skipSlow;230      }}231232      echo >> $out233      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out234      ${mkBench {235        name = "Tail call";236        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";237        skipScala = skipSlow;238      }}239      ${mkBench {240        name = "Inheritance recursion";241        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242        skipCpp = skipSlow;243        skipGo = skipSlow;244      }}245      ${mkBench {246        name = "Simple recursive call";247        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248        skipScala = skipSlow;249        skipGo = skipSlow;250      }}251      ${mkBench {252        name = "Foldl string concat";253        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";254        skipCpp = skipSlow;255        skipScala = skipSlow;256      }}257      ${mkBench {258        name = "Array sorts";259        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";260        skipCpp = skipSlow;261        # std.assertEqual(reverse(std.range(1, 1000)), sort(std.range(1, 1000), keyF=function(x) -x))262        skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";263      }}264      ${mkBench {265        name = "Lazy array";266        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";267        skipGo = skipSlow;268        skipScala = skipSlow;269      }}270      ${mkBench {271        name = "Inheritance function recursion";272        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";273        skipCpp = skipSlow;274        skipScala = skipSlow;275      }}276      ${mkBench {277        name = "String strips";278        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";279        skipCpp = skipSlow;280        skipScala = skipSlow;281      }}282      ${mkBench {283        name = "Big object";284        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";285        pathIsGenerator = true;286        skipScala = skipSlow;287      }}288289      echo >> $out290      echo "## Benchmarks from Go jsonnet (builtins)" >> $out291      ${mkBench {292        name = "std.base64";293        path = "${goJsonnetBench}/base64.jsonnet";294        skipRustAlternative = skipSlow;295        skipCpp = skipSlow;296        skipScala = skipSlow;297      }}298      ${mkBench {299        name = "std.base64Decode";300        path = "${goJsonnetBench}/base64Decode.jsonnet";301        skipRustAlternative = skipSlow;302        skipCpp = skipSlow;303        skipScala = skipSlow;304      }}305      ${mkBench {306        name = "std.base64DecodeBytes";307        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";308        skipRustAlternative = skipSlow;309        skipCpp = skipSlow;310        skipGo = skipSlow;311        skipScala = skipSlow;312      }}313      ${mkBench {314        name = "std.base64 (byte array)";315        path = "${goJsonnetBench}/base64_byte_array.jsonnet";316        skipRustAlternative = skipSlow;317        skipCpp = skipSlow;318        skipGo = skipSlow;319        skipScala = skipSlow;320      }}321      ${mkBench {322        name = "std.foldl";323        path = "${goJsonnetBench}/foldl.jsonnet";324        skipScala = skipSlow;325      }}326      ${mkBench {327        name = "std.manifestJsonEx";328        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";329        skipScala = skipSlow;330        skipCpp = skipSlow;331      }}332      ${mkBench {333        name = "std.manifestTomlEx";334        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";335        skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";336        skipCpp = skipSlow;337      }}338      ${mkBench {339        name = "std.parseInt";340        path = "${goJsonnetBench}/parseInt.jsonnet";341        skipScala = skipSlow;342        skipCpp = skipSlow;343      }}344      ${mkBench {345        name = "std.reverse";346        path = "${goJsonnetBench}/reverse.jsonnet";347        skipCpp = skipSlow;348        skipGo = skipSlow;349      }}350      ${mkBench {351        name = "std.substr";352        path = "${goJsonnetBench}/substr.jsonnet";353        skipScala = skipSlow;354      }}355      ${mkBench {356        name = "Comparsion for array";357        path = "${goJsonnetBench}/comparison.jsonnet";358        skipScala = "array comparsion is not implemented";359        skipCpp = skipSlow;360      }}361      ${mkBench {362        name = "Comparsion for primitives";363        path = "${goJsonnetBench}/comparison2.jsonnet";364        skipRustAlternative = skipSlow;365        skipCpp = "can't run: uses up to 192GB of RAM";366        skipGo = skipSlow;367        skipScala = skipSlow;368      }}369    '';370  }