difftreelog
ci benchmark stack size
in: master
1 file changed
nix/benchmarks.nixdiffbeforeafterboth130 cat result.adoc >> $out130 cat result.adoc >> $out131 '';131 '';132 in132 in133 ''133 ''134 set -oux134 set -oux135 # ulimit -s unlimited135 ulimit -s 65535136136137 temp=$(mktemp -d)137 temp=$(mktemp -d)138 cd $temp138 cd $temp139139140 touch $out140 touch $out141 ${optionalString (true) ''141 ${optionalString (true) ''142 cat ${./benchmarks.adoc} >> $out142 cat ${./benchmarks.adoc} >> $out143 echo >> $out143 echo >> $out144144145 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out145 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out146 echo >> $out146 echo >> $out147147148 echo ".Tested versions" >> $out148 echo ".Tested versions" >> $out149 echo "[%collapsible]" >> $out149 echo "[%collapsible]" >> $out150 echo "====" >> $out150 echo "====" >> $out151 echo "* Go: $(go-jsonnet --version)" >> $out151 echo "* Go: $(go-jsonnet --version)" >> $out152 echo "* C++: $(jsonnet --version)" >> $out152 echo "* C++: $(jsonnet --version)" >> $out153 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out153 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out154 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out154 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out155 ${concatStringsSep "\n" (155 ${concatStringsSep "\n" (156 forEach jrsonnetVariants (156 forEach jrsonnetVariants (157 variant:157 variant:158 "echo \"* Rust${158 "echo \"* Rust${159 if variant.name != "" then " (${variant.name})" else ""159 if variant.name != "" then " (${variant.name})" else ""160 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"160 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"161 )161 )162 )}162 )}163 echo "====" >> $out163 echo "====" >> $out164 echo >> $out164 echo >> $out165 ''}165 ''}166 echo "== Real world" >> $out166 echo "== Real world" >> $out167 ${mkBench {167 ${mkBench {168 name = "GitLab runbooks dashboards";168 name = "GitLab runbooks dashboards";169 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";169 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";170 jpaths = [170 jpaths = [171 "${realworldBench}/vendor"171 "${realworldBench}/vendor"172 "${realworldBench}/vendor/runbooks/libsonnet"172 "${realworldBench}/vendor/runbooks/libsonnet"173 "${realworldBench}/vendor/runbooks/dashboards"173 "${realworldBench}/vendor/runbooks/dashboards"174 "${realworldBench}/vendor/runbooks/services"174 "${realworldBench}/vendor/runbooks/services"175 "${realworldBench}/vendor/runbooks/metrics-catalog"175 "${realworldBench}/vendor/runbooks/metrics-catalog"176 ];176 ];177 skipCpp = "too slow, takes hours, skews results";177 skipCpp = "too slow, takes hours, skews results";178 skipGo = skipSlow;178 skipGo = skipSlow;179 }}179 }}180 ${mkBench {180 ${mkBench {181 name = "GraalVM CI";181 name = "GraalVM CI";182 path = "${realworldBench}/entry-graalvm.jsonnet";182 path = "${realworldBench}/entry-graalvm.jsonnet";183 jpaths = [183 jpaths = [184 "${realworldBench}/vendor/graal"184 "${realworldBench}/vendor/graal"185 ];185 ];186 skipCpp = "too slow, takes hours, skews results";186 skipCpp = "too slow, takes hours, skews results";187 skipGo = skipSlow;187 skipGo = skipSlow;188 }}188 }}189 ${mkBench {189 ${mkBench {190 name = "Kube-prometheus";190 name = "Kube-prometheus";191 path = "${realworldBench}/entry-kube-prometheus.jsonnet";191 path = "${realworldBench}/entry-kube-prometheus.jsonnet";192 jpaths = [192 jpaths = [193 "${realworldBench}/vendor"193 "${realworldBench}/vendor"194 ];194 ];195 skipCpp = "too slow, takes hours, skews results";195 skipCpp = "too slow, takes hours, skews results";196 skipGo = skipSlow;196 skipGo = skipSlow;197 }}197 }}198 ${mkBench {198 ${mkBench {199 name = "Loki manifests";199 name = "Loki manifests";200 path = "${realworldBench}/entry-loki.jsonnet";200 path = "${realworldBench}/entry-loki.jsonnet";201 jpaths = [201 jpaths = [202 "${realworldBench}/vendor"202 "${realworldBench}/vendor"203 "${realworldBench}"203 "${realworldBench}"204 ];204 ];205 skipCpp = "too slow, takes hours, skews results";205 skipCpp = "too slow, takes hours, skews results";206 skipGo = skipSlow;206 skipGo = skipSlow;207 }}207 }}208 ${mkBench {208 ${mkBench {209 name = "Mimir manifests";209 name = "Mimir manifests";210 path = "${realworldBench}/entry-mimir.jsonnet";210 path = "${realworldBench}/entry-mimir.jsonnet";211 jpaths = [211 jpaths = [212 "${realworldBench}/vendor"212 "${realworldBench}/vendor"213 "${realworldBench}"213 "${realworldBench}"214 ];214 ];215 skipCpp = "too slow, takes hours, skews results";215 skipCpp = "too slow, takes hours, skews results";216 skipGo = skipSlow;216 skipGo = skipSlow;217 skipScala = "https://github.com/databricks/sjsonnet/issues/829";217 skipScala = "https://github.com/databricks/sjsonnet/issues/829";218 }}218 }}219 ${mkBench {219 ${mkBench {220 name = "Tempo manifests";220 name = "Tempo manifests";221 path = "${realworldBench}/entry-tempo.jsonnet";221 path = "${realworldBench}/entry-tempo.jsonnet";222 jpaths = [222 jpaths = [223 "${realworldBench}/vendor"223 "${realworldBench}/vendor"224 "${realworldBench}"224 "${realworldBench}"225 ];225 ];226 skipCpp = "too slow, takes hours, skews results";226 skipCpp = "too slow, takes hours, skews results";227 skipGo = skipSlow;227 skipGo = skipSlow;228 }}228 }}229229230 echo >> $out230 echo >> $out231 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out231 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out232 ${mkBench {232 ${mkBench {233 name = "Large string join";233 name = "Large string join";234 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";234 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";235 }}235 }}236 ${mkBench {236 ${mkBench {237 name = "Large string template";237 name = "Large string template";238 omitSource = true;238 omitSource = true;239 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";239 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";240 skipGo = "fails with os stack size exhausion";240 skipGo = "fails with os stack size exhausion";241 skipCpp = "too slow, takes hours, skews results";241 skipCpp = "too slow, takes hours, skews results";242 }}242 }}243 ${mkBench {243 ${mkBench {244 name = "Realistic 1";244 name = "Realistic 1";245 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";245 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";246 skipGo = skipSlow;246 skipGo = skipSlow;247 skipCpp = "too slow, takes hours, skews results";247 skipCpp = "too slow, takes hours, skews results";248 }}248 }}249 ${mkBench {249 ${mkBench {250 name = "Realistic 2";250 name = "Realistic 2";251 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";251 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";252 skipGo = skipSlow;252 skipGo = skipSlow;253 skipCpp = "too slow, takes hours, skews results";253 skipCpp = "too slow, takes hours, skews results";254 }}254 }}255255256 echo >> $out256 echo >> $out257 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out257 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out258 ${mkBench {258 ${mkBench {259 name = "Tail call";259 name = "Tail call";260 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";260 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";261 }}261 }}262 ${mkBench {262 ${mkBench {263 name = "Inheritance recursion";263 name = "Inheritance recursion";264 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";264 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";265 skipCpp = skipSlow;265 skipCpp = skipSlow;266 skipGo = skipSlow;266 skipGo = skipSlow;267 }}267 }}268 ${mkBench {268 ${mkBench {269 name = "Simple recursive call";269 name = "Simple recursive call";270 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";270 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";271 skipGo = skipSlow;271 skipGo = skipSlow;272 }}272 }}273 ${mkBench {273 ${mkBench {274 name = "Foldl string concat";274 name = "Foldl string concat";275 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";275 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";276 skipCpp = skipSlow;276 skipCpp = skipSlow;277 }}277 }}278 ${mkBench {278 ${mkBench {279 name = "Array sorts";279 name = "Array sorts";280 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";280 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";281 skipCpp = skipSlow;281 skipCpp = skipSlow;282 }}282 }}283 ${mkBench {283 ${mkBench {284 name = "Lazy array";284 name = "Lazy array";285 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";285 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";286 skipGo = skipSlow;286 skipGo = skipSlow;287 }}287 }}288 ${mkBench {288 ${mkBench {289 name = "Inheritance function recursion";289 name = "Inheritance function recursion";290 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";290 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";291 skipCpp = skipSlow;291 skipCpp = skipSlow;292 }}292 }}293 ${mkBench {293 ${mkBench {294 name = "String strips";294 name = "String strips";295 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";295 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";296 skipCpp = "too slow, takes hours, skews results";296 skipCpp = "too slow, takes hours, skews results";297 }}297 }}298 ${mkBench {298 ${mkBench {299 name = "Big object";299 name = "Big object";300 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";300 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";301 pathIsGenerator = true;301 pathIsGenerator = true;302 }}302 }}303303304 echo >> $out304 echo >> $out305 echo "== Benchmarks from Go jsonnet (builtins)" >> $out305 echo "== Benchmarks from Go jsonnet (builtins)" >> $out306 ${mkBench {306 ${mkBench {307 name = "std.base64";307 name = "std.base64";308 path = "${goJsonnetBench}/base64.jsonnet";308 path = "${goJsonnetBench}/base64.jsonnet";309 skipRustAlternative = skipSlow;309 skipRustAlternative = skipSlow;310 skipCpp = "too slow, takes hours, skews results";310 skipCpp = "too slow, takes hours, skews results";311 }}311 }}312 ${mkBench {312 ${mkBench {313 name = "std.base64Decode";313 name = "std.base64Decode";314 path = "${goJsonnetBench}/base64Decode.jsonnet";314 path = "${goJsonnetBench}/base64Decode.jsonnet";315 skipRustAlternative = skipSlow;315 skipRustAlternative = skipSlow;316 skipCpp = skipSlow;316 skipCpp = skipSlow;317 }}317 }}318 ${mkBench {318 ${mkBench {319 name = "std.base64DecodeBytes";319 name = "std.base64DecodeBytes";320 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";320 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";321 skipRustAlternative = skipSlow;321 skipRustAlternative = skipSlow;322 skipCpp = skipSlow;322 skipCpp = skipSlow;323 skipGo = skipSlow;323 skipGo = skipSlow;324 }}324 }}325 ${mkBench {325 ${mkBench {326 name = "std.base64 (byte array)";326 name = "std.base64 (byte array)";327 path = "${goJsonnetBench}/base64_byte_array.jsonnet";327 path = "${goJsonnetBench}/base64_byte_array.jsonnet";328 skipRustAlternative = skipSlow;328 skipRustAlternative = skipSlow;329 skipCpp = skipSlow;329 skipCpp = skipSlow;330 skipGo = skipSlow;330 skipGo = skipSlow;331 }}331 }}332 ${mkBench {332 ${mkBench {333 name = "std.foldl";333 name = "std.foldl";334 path = "${goJsonnetBench}/foldl.jsonnet";334 path = "${goJsonnetBench}/foldl.jsonnet";335 }}335 }}336 ${mkBench {336 ${mkBench {337 name = "std.manifestJsonEx";337 name = "std.manifestJsonEx";338 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";338 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";339 skipCpp = skipSlow;339 skipCpp = skipSlow;340 }}340 }}341 ${mkBench {341 ${mkBench {342 name = "std.manifestTomlEx";342 name = "std.manifestTomlEx";343 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";343 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";344 skipCpp = skipSlow;344 skipCpp = skipSlow;345 }}345 }}346 ${mkBench {346 ${mkBench {347 name = "std.parseInt";347 name = "std.parseInt";348 path = "${goJsonnetBench}/parseInt.jsonnet";348 path = "${goJsonnetBench}/parseInt.jsonnet";349 skipCpp = skipSlow;349 skipCpp = skipSlow;350 }}350 }}351 ${mkBench {351 ${mkBench {352 name = "std.reverse";352 name = "std.reverse";353 path = "${goJsonnetBench}/reverse.jsonnet";353 path = "${goJsonnetBench}/reverse.jsonnet";354 skipCpp = skipSlow;354 skipCpp = skipSlow;355 skipGo = skipSlow;355 skipGo = skipSlow;356 }}356 }}357 ${mkBench {357 ${mkBench {358 name = "std.substr";358 name = "std.substr";359 path = "${goJsonnetBench}/substr.jsonnet";359 path = "${goJsonnetBench}/substr.jsonnet";360 }}360 }}361 ${mkBench {361 ${mkBench {362 name = "Comparsion for array";362 name = "Comparsion for array";363 path = "${goJsonnetBench}/comparison.jsonnet";363 path = "${goJsonnetBench}/comparison.jsonnet";364 skipCpp = "too slow, takes hours, skews results";364 skipCpp = "too slow, takes hours, skews results";365 }}365 }}366 ${mkBench {366 ${mkBench {367 name = "Comparsion for primitives";367 name = "Comparsion for primitives";368 path = "${goJsonnetBench}/comparison2.jsonnet";368 path = "${goJsonnetBench}/comparison2.jsonnet";369 skipRustAlternative = skipSlow;369 skipRustAlternative = skipSlow;370 skipCpp = "can't run: uses up to 192GB of RAM";370 skipCpp = "can't run: uses up to 192GB of RAM";371 skipGo = skipSlow;371 skipGo = skipSlow;372 }}372 }}373 '';373 '';374}374}375375