git.delta.rocks / jrsonnet / refs/commits / ccafbf79faf6

difftreelog

ci fix benchmarking

Yaroslav Bolyukin2022-12-07parent: #f9767f0.patch.diff
in: master

2 files changed

modifiedflake.nixdiffbeforeafterboth
51            inherit go-jsonnet sjsonnet jsonnet;51            inherit go-jsonnet sjsonnet jsonnet;
52            jrsonnetVariants = [52            jrsonnetVariants = [
53              { drv = jrsonnet; name = "current"; }53              { drv = jrsonnet; name = "current"; }
54              { drv = jrsonnet-nightly; name = "current-nightly"; }
55            ];54            ];
56          };55          };
57          benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {56          benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {
58            inherit go-jsonnet sjsonnet jsonnet;57            inherit go-jsonnet sjsonnet jsonnet;
59            quick = true;58            quick = true;
60            jrsonnetVariants = [59            jrsonnetVariants = [
61              { drv = jrsonnet; name = "current"; }60              { drv = jrsonnet; name = "current"; }
62              { drv = jrsonnet-nightly; name = "current-nightly"; }
63            ];61            ];
64          };62          };
65          benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {63          benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {
modifiednix/benchmarks.nixdiffbeforeafterboth
120        cat result.md >> $out120        cat result.md >> $out
121      '';121      '';
122    in122    in
123    ''123    ''
124      touch $out124      touch $out
125      ${optionalString (!quick) ''125      ${optionalString (!quick) ''
126        cat ${./benchmarks.md} >> $out126        cat ${./benchmarks.md} >> $out
127        echo >> $out127        echo >> $out
128128
129        echo "<details>" >> $out129        echo "<details>" >> $out
130        echo "<summary>Tested versions</summary>" >> $out130        echo "<summary>Tested versions</summary>" >> $out
131        echo >> $out131        echo >> $out
132        echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out132        echo Go: $(go-jsonnet --version) >> $out
133        echo >> $out
134        echo "\`\`\`" >> $out
135        jrsonnet --help >> $out
136        echo "\`\`\`" >> $out
137        echo >> $out
138        echo Go: $(go-jsonnet --version) >> $out133        echo >> $out
139        echo >> $out134        echo "\`\`\`" >> $out
140        echo "\`\`\`" >> $out135        go-jsonnet --help >> $out
141        go-jsonnet --help >> $out136        echo "\`\`\`" >> $out
142        echo "\`\`\`" >> $out137        echo >> $out
143        echo >> $out138        echo C++: $(jsonnet --version) >> $out
144        echo C++: $(jsonnet --version) >> $out139        echo >> $out
145        echo >> $out140        echo "\`\`\`" >> $out
146        echo "\`\`\`" >> $out141        jsonnet --help >> $out
147        jsonnet --help >> $out142        echo "\`\`\`" >> $out
148        echo "\`\`\`" >> $out143        echo >> $out
149        echo >> $out144        echo Scala: >> $out
150        echo Scala: >> $out145        echo >> $out
151        echo >> $out146        echo "\`\`\`" >> $out
152        echo "\`\`\`" >> $out147        sjsonnet 2>> $out || true
153        sjsonnet 2>> $out || true148        echo "\`\`\`" >> $out
154        echo "\`\`\`" >> $out149        echo >> $out
155        echo >> $out150        echo "</details>" >> $out
156        echo "</details>" >> $out151        echo >> $out
157        echo >> $out152
158153        echo >> $out
159        echo >> $out154      ''}
160      ''}155      echo "## Real world" >> $out
161      echo "## Real world" >> $out156      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour"; skipGo = skipSlow; skipScala = skipSlow;}}
162      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour"; skipGo = skipSlow; skipScala = skipSlow;}}157      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}
163      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}158
164159      echo >> $out
165      echo >> $out160      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out
166      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out161      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet"; skipScala = skipSlow;}}
167      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet"; skipScala = skipSlow;}}162      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow; skipScala = skipSlow;}}
168      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow; skipScala = skipSlow;}}163      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}
169      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}164      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}
170      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}165
171166      echo >> $out
172      echo >> $out167      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out
173      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out168      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet"; skipScala = skipSlow;}}
174      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet"; skipScala = skipSlow;}}169      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow;}}
175      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow;}}170      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet"; skipScala = skipSlow; skipGo = skipSlow;}}
176      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet"; skipScala = skipSlow; skipGo = skipSlow;}}171      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}
177      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}172      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}
178      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}173      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet"; skipGo = skipSlow; skipScala = skipSlow;}}
179      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet"; skipGo = skipSlow; skipScala = skipSlow;}}174      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}
180      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}175      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}
181      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}176      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true; skipScala = skipSlow;}}
182      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true; skipScala = skipSlow;}}177
183178      echo >> $out
184      echo >> $out179      echo "## Benchmarks from Go jsonnet (builtins)" >> $out
185      echo "## Benchmarks from Go jsonnet (builtins)" >> $out180      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}
186      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}181      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}
187      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}182      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}
188      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}183      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}
189      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}184      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet"; skipScala = skipSlow;}}
190      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet"; skipScala = skipSlow;}}185      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}
191      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}186      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented"; skipCpp=skipSlow;}}
192      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented"; skipCpp=skipSlow;}}187      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}
193      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}188      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow; skipGo = skipSlow;}}
194      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow; skipGo = skipSlow;}}189      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet"; skipScala = skipSlow;}}
195      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet"; skipScala = skipSlow;}}190      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}
196      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}191      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM"; skipGo = skipSlow; skipScala = skipSlow;}}
197      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM"; skipGo = skipSlow; skipScala = skipSlow;}}192    '';
198    '';
199}193}
200194