difftreelog
ci reenable scala for quick benchmarking
in: master
1 file changed
nix/benchmarks.nixdiffbeforeafterboth79 skipGo ? "",79 skipGo ? "",80 vendor ? "",80 vendor ? "",81 }:81 }:82 ''82 ''83 echo >> $out83 echo >> $out84 echo "=== ${name}" >> $out84 echo "=== ${name}" >> $out85 echo >> $out85 echo >> $out86 ${optionalString (skipRustAlternative != "") ''86 ${optionalString (skipRustAlternative != "") ''87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out87 echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out88 echo >> $out88 echo >> $out89 ''}89 ''}90 ${optionalString (skipGo != "") ''90 ${optionalString (skipGo != "") ''91 echo "> Note: No results for Go, ${skipGo}" >> $out91 echo "> Note: No results for Go, ${skipGo}" >> $out92 echo >> $out92 echo >> $out93 ''}93 ''}94 ${optionalString (skipScala != "") ''94 ${optionalString (skipScala != "") ''95 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out95 echo "> Note: No results for Scala (native)/Scala (GraalVM), ${skipScala}" >> $out96 echo >> $out96 echo >> $out97 ''}97 ''}98 ${optionalString (skipCpp != "") ''98 ${optionalString (skipCpp != "") ''99 echo "> Note: No results for C++, ${skipCpp}" >> $out99 echo "> Note: No results for C++, ${skipCpp}" >> $out100 echo >> $out100 echo >> $out101 ''}101 ''}102 ${optionalString (!quick && !omitSource) ''102 ${optionalString (!omitSource) ''103 echo ".Source" >> $out103 echo ".Source" >> $out104 echo "[%collapsible]" >> $out104 echo "[%collapsible]" >> $out105 echo "====" >> $out105 echo "====" >> $out106 echo "[source,jsonnet]" >> $out106 echo "[source,jsonnet]" >> $out107 echo "----" >> $out107 echo "----" >> $out108 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}108 ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}109 cat ${path} >> $out109 cat ${path} >> $out110 echo >> $out110 echo >> $out111 echo "----" >> $out111 echo "----" >> $out112 echo "====" >> $out112 echo "====" >> $out113 echo >> $out113 echo >> $out114 ''}114 ''}115 path=${path}115 path=${path}116 ${optionalString pathIsGenerator ''116 ${optionalString pathIsGenerator ''117 go-jsonnet $path > generated.jsonnet117 go-jsonnet $path > generated.jsonnet118 path=generated.jsonnet118 path=generated.jsonnet119 ''}119 ''}120 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \120 hyperfine -N -w4 -m20 --output=pipe --style=basic --export-asciidoc result.adoc \121 ${121 ${122 concatStringsSep " " (122 concatStringsSep " " (123 forEach jrsonnetVariants (123 forEach jrsonnetVariants (124 variant:124 variant:125 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${125 "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${126 if variant.name != "" then " (${variant.name})" else ""126 if variant.name != "" then " (${variant.name})" else ""127 }\""127 }\""128 )128 )129 )129 )130 } \130 } \131 ${131 ${132 optionalString (skipRustAlternative == "")132 optionalString (skipRustAlternative == "")133 "\"rsjsonnet $path${133 "\"rsjsonnet $path${134 optionalString (vendor != "") " -J ${vendor}"134 optionalString (vendor != "") " -J ${vendor}"135 }\" -n \"Rust (alternative, rsjsonnet)\""135 }\" -n \"Rust (alternative, rsjsonnet)\""136 } \136 } \137 ${137 ${138 optionalString (skipGo == "")138 optionalString (skipGo == "")139 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""139 "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""140 } \140 } \141 ${141 ${142 optionalString (skipScala == "")142 optionalString (skipScala == "")143 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""143 "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""144 } \144 } \145 ${145 ${146 # My aarch64-linux machine can't run graalvm image:146 # My aarch64-linux machine can't run graalvm image:147 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].147 # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].148 optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")148 optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")149 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""149 "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""150 } \150 } \151 ${optionalString (skipCpp == "")151 ${optionalString (skipCpp == "")152 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""152 "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""153 }153 }154 cat result.adoc >> $out154 cat result.adoc >> $out155 '';155 '';156 in156 in157 ''157 ''158 set -oux158 set -oux159 ulimit -s unlimited159 ulimit -s unlimited160160161 temp=$(mktemp -d)161 temp=$(mktemp -d)162 cd $temp162 cd $temp163163164 touch $out164 touch $out165 ${optionalString (!quick) ''165 ${optionalString (true) ''166 cat ${./benchmarks.adoc} >> $out166 cat ${./benchmarks.adoc} >> $out167 echo >> $out167 echo >> $out168168169 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out169 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out170 echo >> $out170 echo >> $out171171172 echo ".Tested versions" >> $out172 echo ".Tested versions" >> $out173 echo "[%collapsible]" >> $out173 echo "[%collapsible]" >> $out174 echo "====" >> $out174 echo "====" >> $out175 echo "* Go: $(go-jsonnet --version)" >> $out175 echo "* Go: $(go-jsonnet --version)" >> $out176 echo "* C++: $(jsonnet --version)" >> $out176 echo "* C++: $(jsonnet --version)" >> $out177 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out177 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out178 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out178 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out179 ${concatStringsSep "\n" (179 ${concatStringsSep "\n" (180 forEach jrsonnetVariants (180 forEach jrsonnetVariants (181 variant:181 variant:182 "echo \"* Rust${182 "echo \"* Rust${183 if variant.name != "" then " (${variant.name})" else ""183 if variant.name != "" then " (${variant.name})" else ""184 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"184 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"185 )185 )186 )}186 )}187 echo "====" >> $out187 echo "====" >> $out188 echo >> $out188 echo >> $out189 ''}189 ''}190 echo "== Real world" >> $out190 echo "== Real world" >> $out191 ${mkBench {191 ${mkBench {192 name = "Graalvm CI";192 name = "Graalvm CI";193 path = "${graalvmBench}/ci.jsonnet";193 path = "${graalvmBench}/ci.jsonnet";194 omitSource = true;194 omitSource = true;195 skipCpp = "takes longer than a hour";195 skipCpp = "takes longer than a hour";196 skipGo = skipSlow;196 skipGo = skipSlow;197 skipScala = skipSlow;197 }}198 }}198 ${mkBench {199 ${mkBench {199 name = "Kube-prometheus manifests";200 name = "Kube-prometheus manifests";200 vendor = "${kubePrometheusBench}/vendor";201 vendor = "${kubePrometheusBench}/vendor";201 path = "${kubePrometheusBench}/example.jsonnet";202 path = "${kubePrometheusBench}/example.jsonnet";202 omitSource = true;203 omitSource = true;203 skipCpp = "too slow, takes hours, skews results";204 skipCpp = "too slow, takes hours, skews results";204 skipGo = skipSlow;205 skipGo = skipSlow;205 }}206 skipScala = skipSlow;207 }}206208207 echo >> $out209 echo >> $out208 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out210 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out209 ${mkBench {211 ${mkBench {210 name = "Large string join";212 name = "Large string join";211 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";213 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";212 }}214 skipScala = skipSlow;215 }}213 ${mkBench {216 ${mkBench {214 name = "Large string template";217 name = "Large string template";215 omitSource = true;218 omitSource = true;216 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";219 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";217 skipGo = "fails with os stack size exhausion";220 skipGo = "fails with os stack size exhausion";218 skipCpp = "too slow, takes hours, skews results";221 skipCpp = "too slow, takes hours, skews results";219 }}222 skipScala = skipSlow;223 }}220 ${mkBench {224 ${mkBench {221 name = "Realistic 1";225 name = "Realistic 1";222 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";226 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";223 skipGo = skipSlow;227 skipGo = skipSlow;224 skipCpp = "too slow, takes hours, skews results";228 skipCpp = "too slow, takes hours, skews results";225 }}229 skipScala = skipSlow;230 }}226 ${mkBench {231 ${mkBench {227 name = "Realistic 2";232 name = "Realistic 2";228 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";233 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229 skipGo = skipSlow;234 skipGo = skipSlow;230 skipCpp = "too slow, takes hours, skews results";235 skipCpp = "too slow, takes hours, skews results";231 }}236 skipScala = skipSlow;237 }}232238233 echo >> $out239 echo >> $out234 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out240 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out235 ${mkBench {241 ${mkBench {236 name = "Tail call";242 name = "Tail call";237 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";243 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";238 }}244 skipScala = skipSlow;245 }}239 ${mkBench {246 ${mkBench {240 name = "Inheritance recursion";247 name = "Inheritance recursion";241 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";248 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";242 skipCpp = skipSlow;249 skipCpp = skipSlow;243 skipGo = skipSlow;250 skipGo = skipSlow;244 }}251 }}245 ${mkBench {252 ${mkBench {246 name = "Simple recursive call";253 name = "Simple recursive call";247 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";254 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";248 skipGo = skipSlow;255 skipScala = skipSlow;256 skipGo = skipSlow;249 }}257 }}250 ${mkBench {258 ${mkBench {251 name = "Foldl string concat";259 name = "Foldl string concat";252 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";260 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";253 skipCpp = skipSlow;261 skipCpp = skipSlow;254 }}262 skipScala = skipSlow;263 }}255 ${mkBench {264 ${mkBench {256 name = "Array sorts";265 name = "Array sorts";257 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";266 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";258 skipCpp = skipSlow;267 skipCpp = skipSlow;259 }}268 }}260 ${mkBench {269 ${mkBench {261 name = "Lazy array";270 name = "Lazy array";262 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";271 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";263 skipGo = skipSlow;272 skipGo = skipSlow;264 }}273 }}265 ${mkBench {274 ${mkBench {266 name = "Inheritance function recursion";275 name = "Inheritance function recursion";267 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";276 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";268 skipCpp = skipSlow;277 skipCpp = skipSlow;269 }}278 skipScala = skipSlow;279 }}270 ${mkBench {280 ${mkBench {271 name = "String strips";281 name = "String strips";272 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";282 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";273 skipCpp = "too slow, takes hours, skews results";283 skipCpp = "too slow, takes hours, skews results";274 }}284 skipScala = skipSlow;285 }}275 ${mkBench {286 ${mkBench {276 name = "Big object";287 name = "Big object";277 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";288 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";278 pathIsGenerator = true;289 pathIsGenerator = true;279 }}290 skipScala = skipSlow;291 }}280292281 echo >> $out293 echo >> $out282 echo "== Benchmarks from Go jsonnet (builtins)" >> $out294 echo "== Benchmarks from Go jsonnet (builtins)" >> $out283 ${mkBench {295 ${mkBench {284 name = "std.base64";296 name = "std.base64";285 path = "${goJsonnetBench}/base64.jsonnet";297 path = "${goJsonnetBench}/base64.jsonnet";286 skipRustAlternative = skipSlow;298 skipRustAlternative = skipSlow;287 skipCpp = "too slow, takes hours, skews results";299 skipCpp = "too slow, takes hours, skews results";288 }}300 skipScala = skipSlow;301 }}289 ${mkBench {302 ${mkBench {290 name = "std.base64Decode";303 name = "std.base64Decode";291 path = "${goJsonnetBench}/base64Decode.jsonnet";304 path = "${goJsonnetBench}/base64Decode.jsonnet";292 skipRustAlternative = skipSlow;305 skipRustAlternative = skipSlow;293 skipCpp = skipSlow;306 skipCpp = skipSlow;294 }}307 skipScala = skipSlow;308 }}295 ${mkBench {309 ${mkBench {296 name = "std.base64DecodeBytes";310 name = "std.base64DecodeBytes";297 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";311 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";298 skipRustAlternative = skipSlow;312 skipRustAlternative = skipSlow;299 skipCpp = skipSlow;313 skipCpp = skipSlow;300 skipGo = skipSlow;314 skipGo = skipSlow;301 }}315 skipScala = skipSlow;316 }}302 ${mkBench {317 ${mkBench {303 name = "std.base64 (byte array)";318 name = "std.base64 (byte array)";304 path = "${goJsonnetBench}/base64_byte_array.jsonnet";319 path = "${goJsonnetBench}/base64_byte_array.jsonnet";305 skipRustAlternative = skipSlow;320 skipRustAlternative = skipSlow;306 skipCpp = skipSlow;321 skipCpp = skipSlow;307 skipGo = skipSlow;322 skipGo = skipSlow;308 }}323 skipScala = skipSlow;324 }}309 ${mkBench {325 ${mkBench {310 name = "std.foldl";326 name = "std.foldl";311 path = "${goJsonnetBench}/foldl.jsonnet";327 path = "${goJsonnetBench}/foldl.jsonnet";312 }}328 skipScala = skipSlow;329 }}313 ${mkBench {330 ${mkBench {314 name = "std.manifestJsonEx";331 name = "std.manifestJsonEx";315 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";332 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";316 skipCpp = skipSlow;333 skipScala = skipSlow;334 skipCpp = skipSlow;317 }}335 }}318 ${mkBench {336 ${mkBench {319 name = "std.manifestTomlEx";337 name = "std.manifestTomlEx";320 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";338 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";321 skipCpp = skipSlow;339 skipCpp = skipSlow;322 }}340 }}323 ${mkBench {341 ${mkBench {324 name = "std.parseInt";342 name = "std.parseInt";325 path = "${goJsonnetBench}/parseInt.jsonnet";343 path = "${goJsonnetBench}/parseInt.jsonnet";326 skipCpp = skipSlow;344 skipScala = skipSlow;345 skipCpp = skipSlow;327 }}346 }}328 ${mkBench {347 ${mkBench {329 name = "std.reverse";348 name = "std.reverse";330 path = "${goJsonnetBench}/reverse.jsonnet";349 path = "${goJsonnetBench}/reverse.jsonnet";331 skipCpp = skipSlow;350 skipCpp = skipSlow;332 skipGo = skipSlow;351 skipGo = skipSlow;333 }}352 }}334 ${mkBench {353 ${mkBench {335 name = "std.substr";354 name = "std.substr";336 path = "${goJsonnetBench}/substr.jsonnet";355 path = "${goJsonnetBench}/substr.jsonnet";337 }}356 skipScala = skipSlow;357 }}338 ${mkBench {358 ${mkBench {339 name = "Comparsion for array";359 name = "Comparsion for array";340 path = "${goJsonnetBench}/comparison.jsonnet";360 path = "${goJsonnetBench}/comparison.jsonnet";341 skipCpp = "too slow, takes hours, skews results";361 skipCpp = "too slow, takes hours, skews results";342 }}362 }}343 ${mkBench {363 ${mkBench {344 name = "Comparsion for primitives";364 name = "Comparsion for primitives";345 path = "${goJsonnetBench}/comparison2.jsonnet";365 path = "${goJsonnetBench}/comparison2.jsonnet";346 skipRustAlternative = skipSlow;366 skipRustAlternative = skipSlow;347 skipCpp = "can't run: uses up to 192GB of RAM";367 skipCpp = "can't run: uses up to 192GB of RAM";348 skipGo = skipSlow;368 skipGo = skipSlow;369 skipScala = skipSlow;349 }}370 }}350 '';371 '';372}351}373352