git.delta.rocks / jrsonnet / refs/commits / ec8e2e1ce158

1{ lib2, runCommand3, jsonnet-bundler4, cacert5, stdenv6, fetchFromGitHub7, jrsonnet8, jrsonnet-release9, go-jsonnet10, sjsonnet11, jsonnet12, hyperfine13, quick ? false14, againstRelease ? false15}:16let17  jsonnetBench = fetchFromGitHub {18    rev = "v0.19.1";19    owner = "google";20    repo = "jsonnet";21    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22  };23  goJsonnetBench = (fetchFromGitHub {24    owner = "google";25    repo = "go-jsonnet";26    rev = "v0.19.1";27    hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";28  }) + "/builtin-benchmarks";29  graalvmBench = fetchFromGitHub {30    owner = "oracle";31    repo = "graal";32    rev = "bc305df3fe587960f7635f0185571500e5988475";33    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";34  };35  kubePrometheusBench =36    let37      src = fetchFromGitHub {38        owner = "prometheus-operator";39        repo = "kube-prometheus";40        rev = "6a570e3154eac26e32da61d784fbe626da4804df";41        hash = "sha256-s6BK8KQiSjql2w6R+8m5pvPPAcKW+SKjQwqxZHjimFw=";42      };43    in44    runCommand "kube-prometheus-vendor"45      {46        outputHash = "sha256-R60RI/7FQPOHisnzANb34m9WPj5D9FeMVoGOjB19zl8=";47        outputHashMode = "recursive";48        buildInputs = [ cacert ];49      }50      ''51        mkdir -p $out52        cp -r ${src}/* $out/53        cd $out54        mkdir vendor55        ${jsonnet-bundler}/bin/jb install56      '';5758  # Removes outsiders from the output59  # Useful when comparing performance of different jrsonnet releases60  skipSlow = if quick then "slow benchmark, but only quick requested" else "";61in62stdenv.mkDerivation {63  name = "benchmarks";64  __impure = true;65  unpackPhase = "true";6667  buildInputs = [68    jrsonnet69    go-jsonnet70    sjsonnet71    jsonnet7273    hyperfine74  ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);7576  installPhase =77    let78      mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''79        set -oux8081        echo >> $out82        echo "### ${name}" >> $out83        echo >> $out84        ${if skipGo != "" then ''85          echo "> Note: No results for Go, ${skipGo}" >> $out86          echo >> $out87        '' else ""}88        ${if skipScala != "" then ''89          echo "> Note: No results for Scala, ${skipScala}" >> $out90          echo >> $out91        '' else ""}92        ${if skipCpp != "" then ''93          echo "> Note: No results for C++, ${skipCpp}" >> $out94          echo >> $out95        '' else ""}96        ${if !quick then ''97          echo "<details>" >> $out98          echo "<summary>Source</summary>" >> $out99          echo >> $out100          echo "\`\`\`jsonnet" >> $out101          ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}102          ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}103          echo >> $out104          echo "\`\`\`" >> $out105          echo "</details>" >> $out106          echo >> $out107        '' else ""}108        path=${path}109        ${if pathIsGenerator then ''110          jrsonnet $path > generated.jsonnet111          path=generated.jsonnet112        '' else ""}113        hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \114          "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \115          ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \116          ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \117          ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \118          ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }119        cat result.md >> $out120      '';121    in122    ''123      touch $out124      ${if !quick then ''125        cat ${./benchmarks.md} >> $out126        echo >> $out127128        echo "<details>" >> $out129        echo "<summary>Tested versions</summary>" >> $out130        echo >> $out131        echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out132        echo >> $out133        echo "\`\`\`" >> $out134        jrsonnet --help >> $out135        echo "\`\`\`" >> $out136        echo >> $out137        echo Go: $(go-jsonnet --version) >> $out138        echo >> $out139        echo "\`\`\`" >> $out140        go-jsonnet --help >> $out141        echo "\`\`\`" >> $out142        echo >> $out143        echo C++: $(jsonnet --version) >> $out144        echo >> $out145        echo "\`\`\`" >> $out146        jsonnet --help >> $out147        echo "\`\`\`" >> $out148        echo >> $out149        echo Scala: >> $out150        echo >> $out151        echo "\`\`\`" >> $out152        sjsonnet 2>> $out || true153        echo "\`\`\`" >> $out154        echo >> $out155        echo "</details>" >> $out156        echo >> $out157158        echo >> $out159      '' else ""}160      echo "## Real world" >> $out161      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}162      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}163164      echo >> $out165      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out166      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}}167      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}}168      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}169      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}170171      echo >> $out172      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out173      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}}174      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}}175      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}}176      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}}177      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}178      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}}179      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}}180      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}}181      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}}182183      echo >> $out184      echo "## Benchmarks from Go jsonnet (builtins)" >> $out185      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}}186      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}}187      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}}188      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}}189      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}}190      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}}191      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}}192      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}}193      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}}194      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}}195      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}196      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}}197    '';198}