git.delta.rocks / jrsonnet / refs/commits / ec8e2e1ce158

difftreelog

ci benchmark against stable release

Yaroslav Bolyukin2022-12-03parent: #fdd9d5d.patch.diff
in: master

4 files changed

modifiedflake.nixdiffbeforeafterboth
--- a/flake.nix
+++ b/flake.nix
@@ -29,9 +29,28 @@
               cargo = rust;
             };
           };
+          jrsonnet-release = pkgs.callPackage ./nix/jrsonnet-release.nix {
+            rustPlatform = pkgs.makeRustPlatform {
+              rustc = rust;
+              cargo = rust;
+            };
+          };
 
           benchmarks = pkgs.callPackage ./nix/benchmarks.nix {
-            inherit go-jsonnet sjsonnet jsonnet jrsonnet;
+            inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
+          };
+          benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {
+            inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
+            quick = true;
+          };
+          benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {
+            inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
+            againstRelease = true;
+          };
+          benchmarks-quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {
+            inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
+            quick = true;
+            againstRelease = true;
           };
         };
         devShell = pkgs.mkShell {
modifiednix/benchmarks.nixdiffbeforeafterboth
5, stdenv5, stdenv
6, fetchFromGitHub6, fetchFromGitHub
7, jrsonnet7, jrsonnet
8, jrsonnet-release
8, go-jsonnet9, go-jsonnet
9, sjsonnet10, sjsonnet
10, jsonnet11, jsonnet
11, hyperfine12, hyperfine
12, quick ? false13, quick ? false
14, againstRelease ? false
13}:15}:
14let16let
15  jsonnetBench = fetchFromGitHub {17  jsonnetBench = fetchFromGitHub {
53        ${jsonnet-bundler}/bin/jb install55        ${jsonnet-bundler}/bin/jb install
54      '';56      '';
5557
58  # Removes outsiders from the output
59  # Useful when comparing performance of different jrsonnet releases
56  skipSlow = if quick then "slow benchmark" else "";60  skipSlow = if quick then "slow benchmark, but only quick requested" else "";
57in61in
58stdenv.mkDerivation {62stdenv.mkDerivation {
59  name = "benchmarks";63  name = "benchmarks";
67    jsonnet71    jsonnet
6872
69    hyperfine73    hyperfine
70  ];74  ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);
7175
72  installPhase =76  installPhase =
73    let77    let
74      mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''78      mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''
75        set -oux79        set -oux
7680
77        echo >> $out81        echo >> $out
78        echo "### ${name}" >> $out82        echo "### ${name}" >> $out
79        echo >> $out83        echo >> $out
80        ${if skipGo != "" then ''84        ${if skipGo != "" then ''
81          echo "> Note: No results for Go, ${skipGo}" >> $out85          echo "> Note: No results for Go, ${skipGo}" >> $out
82          echo >> $out86          echo >> $out
83        '' else ""}87        '' else ""}
84        ${if skipScala != "" then ''88        ${if skipScala != "" then ''
85          echo "> Note: No results for Scala, ${skipScala}" >> $out89          echo "> Note: No results for Scala, ${skipScala}" >> $out
86          echo >> $out90          echo >> $out
87        '' else ""}91        '' else ""}
88        ${if skipCpp != "" then ''92        ${if skipCpp != "" then ''
89          echo "> Note: No results for C++, ${skipCpp}" >> $out93          echo "> Note: No results for C++, ${skipCpp}" >> $out
90          echo >> $out94          echo >> $out
91        '' else ""}95        '' else ""}
92        echo "<details>" >> $out96        ${if !quick then ''
97          echo "<details>" >> $out
93        echo "<summary>Source</summary>" >> $out98          echo "<summary>Source</summary>" >> $out
94        echo >> $out99          echo >> $out
95        echo "\`\`\`jsonnet" >> $out100          echo "\`\`\`jsonnet" >> $out
96        ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}101          ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
97        ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}102          ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
98        echo >> $out103          echo >> $out
99        echo "\`\`\`" >> $out104          echo "\`\`\`" >> $out
100        echo "</details>" >> $out105          echo "</details>" >> $out
101        echo >> $out106          echo >> $out
102        path=${path}107        '' else ""}
108        path=${path}
103        ${if pathIsGenerator then ''109        ${if pathIsGenerator then ''
104          jrsonnet $path > generated.jsonnet110          jrsonnet $path > generated.jsonnet
105          path=generated.jsonnet111          path=generated.jsonnet
106        '' else ""}112        '' else ""}
107        hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \113        hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \
108          "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \114          "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \
115          ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \
109          ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \116          ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \
110          ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \117          ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \
111          ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }118          ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }
112        cat result.md >> $out119        cat result.md >> $out
113      '';120      '';
114    in121    in
115    ''122    ''
116      touch $out123      touch $out
117      cat ${./benchmarks.md} >> $out124      ${if !quick then ''
125        cat ${./benchmarks.md} >> $out
118      echo >> $out126        echo >> $out
119127
120      echo "<details>" >> $out128        echo "<details>" >> $out
121      echo "<summary>Tested versions</summary>" >> $out129        echo "<summary>Tested versions</summary>" >> $out
122      echo >> $out130        echo >> $out
123      echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out131        echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
124      echo >> $out132        echo >> $out
125      echo "\`\`\`" >> $out133        echo "\`\`\`" >> $out
126      jrsonnet --help >> $out134        jrsonnet --help >> $out
127      echo "\`\`\`" >> $out135        echo "\`\`\`" >> $out
128      echo >> $out136        echo >> $out
129      echo Go: $(go-jsonnet --version) >> $out137        echo Go: $(go-jsonnet --version) >> $out
130      echo >> $out138        echo >> $out
131      echo "\`\`\`" >> $out139        echo "\`\`\`" >> $out
132      go-jsonnet --help >> $out140        go-jsonnet --help >> $out
133      echo "\`\`\`" >> $out141        echo "\`\`\`" >> $out
134      echo >> $out142        echo >> $out
135      echo C++: $(jsonnet --version) >> $out143        echo C++: $(jsonnet --version) >> $out
136      echo >> $out144        echo >> $out
137      echo "\`\`\`" >> $out145        echo "\`\`\`" >> $out
138      jsonnet --help >> $out146        jsonnet --help >> $out
139      echo "\`\`\`" >> $out147        echo "\`\`\`" >> $out
140      echo >> $out148        echo >> $out
141      echo Scala: >> $out149        echo Scala: >> $out
142      echo >> $out150        echo >> $out
143      echo "\`\`\`" >> $out151        echo "\`\`\`" >> $out
144      sjsonnet 2>> $out || true152        sjsonnet 2>> $out || true
145      echo "\`\`\`" >> $out153        echo "\`\`\`" >> $out
146      echo >> $out154        echo >> $out
147      echo "</details>" >> $out155        echo "</details>" >> $out
148      echo >> $out156        echo >> $out
149157
150      echo >> $out158        echo >> $out
159      '' else ""}
151      echo "## Real world" >> $out160      echo "## Real world" >> $out
152      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}161      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}
153      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}162      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}
154163
155      echo >> $out164      echo >> $out
156      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out165      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out
157      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}}166      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";}}
158      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}}167      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow;}}
159      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}168      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}
160      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}169      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow;}}
161170
162      echo >> $out171      echo >> $out
163      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out172      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out
164      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}}173      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";}}
165      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}}174      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow;}}
166      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}}175      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";}}
167      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}}176      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";}}
168      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}177      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}
169      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}}178      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";}}
170      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}}179      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";}}
171      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}}180      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow;}}
172      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}}181      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true;}}
173182
174      echo >> $out183      echo >> $out
175      echo "## Benchmarks from Go jsonnet (builtins)" >> $out184      echo "## Benchmarks from Go jsonnet (builtins)" >> $out
176      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}}185      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow;}}
177      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}}186      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow;}}
178      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}}187      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow;}}
179      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}}188      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow;}}
180      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}}189      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet";}}
181      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}}190      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet";}}
182      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}}191      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented";}}
183      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}}192      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet";}}
184      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}}193      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented";}}
185      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}}194      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet";}}
186      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}195      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}
187      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}}196      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM";}}
188    '';197    '';
189}198}
190199
modifiednix/jrsonnet-release.nixdiffbeforeafterboth
--- a/nix/jrsonnet-release.nix
+++ b/nix/jrsonnet-release.nix
@@ -3,14 +3,15 @@
 
 rustPlatform.buildRustPackage rec {
   pname = "jrsonnet";
-  version = "5f0f8de9f52f961e2ff162e0a3fd4ca20a275f1d";
+  version = "d32fe45b8ed28fb39b5359a704922922368af1c0";
 
   src = fetchFromGitHub {
     owner = "CertainLach";
     repo = pname;
     rev = version;
-    hash = lib.fakeHash;
+    hash = "sha256-R9Xt36bYS5upVDzt8hEifwmfocXpJbIKwvxkoJNEGVc=";
   };
+  cargoHash = "sha256-V+KGWeNlUnelofaGzufNPLGDyxazoFrjZ/n391VYYws=";
 
   cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
   cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
modifiednix/jrsonnet.nixdiffbeforeafterboth
--- a/nix/jrsonnet.nix
+++ b/nix/jrsonnet.nix
@@ -20,8 +20,8 @@
   pname = "jrsonnet";
   version = "git";
 
-  cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
-  cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
+  cargoTestFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
+  cargoBuildFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
 
   buildInputs = [ makeWrapper ];