difftreelog
ci switch all benchmarks to realworld tests dir
in: master
3 files changed
nix/benchmarks.nixdiffbeforeafterboth2 lib,2 lib,3 runCommand,3 runCommand,4 stdenv,4 stdenv,5 fetchFromGitHub,6 fetchJrq,5 fetchJrq,7 go-jsonnet,6 go-jsonnet,8 sjsonnet,7 sjsonnet,16let15let17 inherit (cpp-jsonnet) jsonnetBench;16 inherit (cpp-jsonnet) jsonnetBench;18 inherit (go-jsonnet) goJsonnetBench;17 inherit (go-jsonnet) goJsonnetBench;19 graalvmBench = fetchFromGitHub {20 owner = "oracle";21 repo = "graal";22 rev = "bc305df3fe587960f7635f0185571500e5988475";23 hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";24 };25 realworldVendor = fetchJrq {18 realworldVendor = fetchJrq {26 name = "realworld-vendor";19 name = "realworld-vendor";27 lockfile = ../tests/realworld/jsonnetfile.lock.json;20 lockfile = ../tests/realworld/jsonnetfile.lock.json;28 vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";21 vendorHash = "sha256-oEUzM6Bhu8ZT8vCtYDbBEjG5BFHYpID+1/2pgXvIAgo=";29 };22 };30 realworldBench = runCommand "realworld-bench" { } ''23 realworldBench = runCommand "realworld-bench" { } ''31 mkdir -p $out24 mkdir -p $out137 cat result.adoc >> $out130 cat result.adoc >> $out138 '';131 '';139 in132 in140 ''133 ''141 set -oux134 set -oux142 ulimit -s unlimited135 ulimit -s unlimited143136144 temp=$(mktemp -d)137 temp=$(mktemp -d)145 cd $temp138 cd $temp146139147 touch $out140 touch $out148 ${optionalString (true) ''141 ${optionalString (true) ''149 cat ${./benchmarks.adoc} >> $out142 cat ${./benchmarks.adoc} >> $out150 echo >> $out143 echo >> $out151144152 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out145 echo "CPU: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs), $(grep -c '^processor' /proc/cpuinfo) threads" >> $out153 echo >> $out146 echo >> $out154147155 echo ".Tested versions" >> $out148 echo ".Tested versions" >> $out156 echo "[%collapsible]" >> $out149 echo "[%collapsible]" >> $out157 echo "====" >> $out150 echo "====" >> $out158 echo "* Go: $(go-jsonnet --version)" >> $out151 echo "* Go: $(go-jsonnet --version)" >> $out159 echo "* C++: $(jsonnet --version)" >> $out152 echo "* C++: $(jsonnet --version)" >> $out160 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out153 echo "* Scala (native/GraalVM): $(sjsonnet-native 2>&1 | grep -oP 'Sjsonnet \S+')" >> $out161 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out154 echo "* Rust (alternative): rsjsonnet ${rsjsonnet.version} (${rsjsonnet.src.rev})" >> $out162 ${concatStringsSep "\n" (155 ${concatStringsSep "\n" (163 forEach jrsonnetVariants (156 forEach jrsonnetVariants (164 variant:157 variant:165 "echo \"* Rust${158 "echo \"* Rust${166 if variant.name != "" then " (${variant.name})" else ""159 if variant.name != "" then " (${variant.name})" else ""167 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"160 }: $(${variant.drv}/bin/jrsonnet --version 2>&1)\" >> $out"168 )161 )169 )}162 )}170 echo "====" >> $out163 echo "====" >> $out171 echo >> $out164 echo >> $out172 ''}165 ''}173 echo "== Real world" >> $out166 echo "== Real world" >> $out174 ${mkBench {167 ${mkBench {175 name = "Graalvm CI";168 name = "GitLab runbooks dashboards";176 path = "${graalvmBench}/ci.jsonnet";169 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";177 omitSource = true;170 jpaths = [171 "${realworldBench}/vendor"172 "${realworldBench}/vendor/runbooks/libsonnet"173 "${realworldBench}/vendor/runbooks/dashboards"174 "${realworldBench}/vendor/runbooks/services"175 "${realworldBench}/vendor/runbooks/metrics-catalog"176 ];178 skipCpp = "takes longer than a hour";177 skipCpp = "too slow, takes hours, skews results";179 skipGo = skipSlow;178 skipGo = skipSlow;180 }}179 }}181 ${mkBench {180 ${mkBench {182 name = "Loki manifests";181 name = "GraalVM CI";183 path = "${realworldBench}/entry-loki.jsonnet";182 path = "${realworldBench}/entry-graalvm.jsonnet";184 jpaths = [ "${realworldBench}/vendor" ];183 jpaths = [184 "${realworldBench}/vendor/graal"185 ];185 skipCpp = "too slow, takes hours, skews results";186 skipCpp = "too slow, takes hours, skews results";186 skipGo = skipSlow;187 skipGo = skipSlow;187 }}188 }}188 ${mkBench {189 ${mkBench {189 name = "Mimir manifests";190 name = "Kube-prometheus";190 path = "${realworldBench}/entry-mimir.jsonnet";191 path = "${realworldBench}/entry-kube-prometheus.jsonnet";191 jpaths = [ "${realworldBench}/vendor" ];192 jpaths = [193 "${realworldBench}/vendor"194 ];192 skipCpp = "too slow, takes hours, skews results";195 skipCpp = "too slow, takes hours, skews results";193 skipGo = skipSlow;196 skipGo = skipSlow;194 }}197 }}195 ${mkBench {198 ${mkBench {196 name = "Tempo manifests";199 name = "Loki manifests";197 path = "${realworldBench}/entry-tempo.jsonnet";200 path = "${realworldBench}/entry-loki.jsonnet";198 jpaths = [ "${realworldBench}/vendor" ];201 jpaths = [202 "${realworldBench}/vendor"203 "${realworldBench}"204 ];199 skipCpp = "too slow, takes hours, skews results";205 skipCpp = "too slow, takes hours, skews results";200 skipGo = skipSlow;206 skipGo = skipSlow;201 }}207 }}202 ${mkBench {208 ${mkBench {203 name = "GitLab runbooks dashboards";209 name = "Mimir manifests";204 path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";210 path = "${realworldBench}/entry-mimir.jsonnet";205 jpaths = [211 jpaths = [206 "${realworldBench}/vendor"212 "${realworldBench}/vendor"207 "${realworldBench}/vendor/runbooks/libsonnet"213 "${realworldBench}"214 ];215 skipCpp = "too slow, takes hours, skews results";216 skipGo = skipSlow;217 skipScala = "https://github.com/databricks/sjsonnet/issues/829";208 "${realworldBench}/vendor/runbooks/dashboards"218 }}219 ${mkBench {220 name = "Tempo manifests";221 path = "${realworldBench}/entry-tempo.jsonnet";209 "${realworldBench}/vendor/runbooks/services"222 jpaths = [223 "${realworldBench}/vendor"210 "${realworldBench}/vendor/runbooks/metrics-catalog"224 "${realworldBench}"211 ];225 ];212 skipCpp = "too slow, takes hours, skews results";226 skipCpp = "too slow, takes hours, skews results";213 skipGo = skipSlow;227 skipGo = skipSlow;214 }}228 }}215229216 echo >> $out230 echo >> $out217 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out231 echo "== Benchmarks from C++ jsonnet (/perf_tests)" >> $out218 ${mkBench {232 ${mkBench {219 name = "Large string join";233 name = "Large string join";220 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";234 path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";221 }}235 }}222 ${mkBench {236 ${mkBench {223 name = "Large string template";237 name = "Large string template";224 omitSource = true;238 omitSource = true;225 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";239 path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";226 skipGo = "fails with os stack size exhausion";240 skipGo = "fails with os stack size exhausion";227 skipCpp = "too slow, takes hours, skews results";241 skipCpp = "too slow, takes hours, skews results";228 }}242 }}229 ${mkBench {243 ${mkBench {230 name = "Realistic 1";244 name = "Realistic 1";231 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";245 path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";232 skipGo = skipSlow;246 skipGo = skipSlow;233 skipCpp = "too slow, takes hours, skews results";247 skipCpp = "too slow, takes hours, skews results";234 }}248 }}235 ${mkBench {249 ${mkBench {236 name = "Realistic 2";250 name = "Realistic 2";237 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";251 path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";238 skipGo = skipSlow;252 skipGo = skipSlow;239 skipCpp = "too slow, takes hours, skews results";253 skipCpp = "too slow, takes hours, skews results";240 }}254 }}241255242 echo >> $out256 echo >> $out243 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out257 echo "== Benchmarks from C++ jsonnet (/benchmarks)" >> $out244 ${mkBench {258 ${mkBench {245 name = "Tail call";259 name = "Tail call";246 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";260 path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";247 }}261 }}248 ${mkBench {262 ${mkBench {249 name = "Inheritance recursion";263 name = "Inheritance recursion";250 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";264 path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";251 skipCpp = skipSlow;265 skipCpp = skipSlow;252 skipGo = skipSlow;266 skipGo = skipSlow;253 }}267 }}254 ${mkBench {268 ${mkBench {255 name = "Simple recursive call";269 name = "Simple recursive call";256 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";270 path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";257 skipGo = skipSlow;271 skipGo = skipSlow;258 }}272 }}259 ${mkBench {273 ${mkBench {260 name = "Foldl string concat";274 name = "Foldl string concat";261 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";275 path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";262 skipCpp = skipSlow;276 skipCpp = skipSlow;263 }}277 }}264 ${mkBench {278 ${mkBench {265 name = "Array sorts";279 name = "Array sorts";266 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";280 path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";267 skipCpp = skipSlow;281 skipCpp = skipSlow;268 }}282 }}269 ${mkBench {283 ${mkBench {270 name = "Lazy array";284 name = "Lazy array";271 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";285 path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";272 skipGo = skipSlow;286 skipGo = skipSlow;273 }}287 }}274 ${mkBench {288 ${mkBench {275 name = "Inheritance function recursion";289 name = "Inheritance function recursion";276 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";290 path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";277 skipCpp = skipSlow;291 skipCpp = skipSlow;278 }}292 }}279 ${mkBench {293 ${mkBench {280 name = "String strips";294 name = "String strips";281 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";295 path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";282 skipCpp = "too slow, takes hours, skews results";296 skipCpp = "too slow, takes hours, skews results";283 }}297 }}284 ${mkBench {298 ${mkBench {285 name = "Big object";299 name = "Big object";286 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";300 path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287 pathIsGenerator = true;301 pathIsGenerator = true;288 }}302 }}289303290 echo >> $out304 echo >> $out291 echo "== Benchmarks from Go jsonnet (builtins)" >> $out305 echo "== Benchmarks from Go jsonnet (builtins)" >> $out292 ${mkBench {306 ${mkBench {293 name = "std.base64";307 name = "std.base64";294 path = "${goJsonnetBench}/base64.jsonnet";308 path = "${goJsonnetBench}/base64.jsonnet";295 skipRustAlternative = skipSlow;309 skipRustAlternative = skipSlow;296 skipCpp = "too slow, takes hours, skews results";310 skipCpp = "too slow, takes hours, skews results";297 }}311 }}298 ${mkBench {312 ${mkBench {299 name = "std.base64Decode";313 name = "std.base64Decode";300 path = "${goJsonnetBench}/base64Decode.jsonnet";314 path = "${goJsonnetBench}/base64Decode.jsonnet";301 skipRustAlternative = skipSlow;315 skipRustAlternative = skipSlow;302 skipCpp = skipSlow;316 skipCpp = skipSlow;303 }}317 }}304 ${mkBench {318 ${mkBench {305 name = "std.base64DecodeBytes";319 name = "std.base64DecodeBytes";306 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";320 path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";307 skipRustAlternative = skipSlow;321 skipRustAlternative = skipSlow;308 skipCpp = skipSlow;322 skipCpp = skipSlow;309 skipGo = skipSlow;323 skipGo = skipSlow;310 }}324 }}311 ${mkBench {325 ${mkBench {312 name = "std.base64 (byte array)";326 name = "std.base64 (byte array)";313 path = "${goJsonnetBench}/base64_byte_array.jsonnet";327 path = "${goJsonnetBench}/base64_byte_array.jsonnet";314 skipRustAlternative = skipSlow;328 skipRustAlternative = skipSlow;315 skipCpp = skipSlow;329 skipCpp = skipSlow;316 skipGo = skipSlow;330 skipGo = skipSlow;317 }}331 }}318 ${mkBench {332 ${mkBench {319 name = "std.foldl";333 name = "std.foldl";320 path = "${goJsonnetBench}/foldl.jsonnet";334 path = "${goJsonnetBench}/foldl.jsonnet";321 }}335 }}322 ${mkBench {336 ${mkBench {323 name = "std.manifestJsonEx";337 name = "std.manifestJsonEx";324 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";338 path = "${goJsonnetBench}/manifestJsonEx.jsonnet";325 skipCpp = skipSlow;339 skipCpp = skipSlow;326 }}340 }}327 ${mkBench {341 ${mkBench {328 name = "std.manifestTomlEx";342 name = "std.manifestTomlEx";329 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";343 path = "${goJsonnetBench}/manifestTomlEx.jsonnet";330 skipCpp = skipSlow;344 skipCpp = skipSlow;331 }}345 }}332 ${mkBench {346 ${mkBench {333 name = "std.parseInt";347 name = "std.parseInt";334 path = "${goJsonnetBench}/parseInt.jsonnet";348 path = "${goJsonnetBench}/parseInt.jsonnet";335 skipCpp = skipSlow;349 skipCpp = skipSlow;336 }}350 }}337 ${mkBench {351 ${mkBench {338 name = "std.reverse";352 name = "std.reverse";339 path = "${goJsonnetBench}/reverse.jsonnet";353 path = "${goJsonnetBench}/reverse.jsonnet";340 skipCpp = skipSlow;354 skipCpp = skipSlow;341 skipGo = skipSlow;355 skipGo = skipSlow;342 }}356 }}343 ${mkBench {357 ${mkBench {344 name = "std.substr";358 name = "std.substr";345 path = "${goJsonnetBench}/substr.jsonnet";359 path = "${goJsonnetBench}/substr.jsonnet";346 }}360 }}347 ${mkBench {361 ${mkBench {348 name = "Comparsion for array";362 name = "Comparsion for array";349 path = "${goJsonnetBench}/comparison.jsonnet";363 path = "${goJsonnetBench}/comparison.jsonnet";350 skipCpp = "too slow, takes hours, skews results";364 skipCpp = "too slow, takes hours, skews results";351 }}365 }}352 ${mkBench {366 ${mkBench {353 name = "Comparsion for primitives";367 name = "Comparsion for primitives";354 path = "${goJsonnetBench}/comparison2.jsonnet";368 path = "${goJsonnetBench}/comparison2.jsonnet";355 skipRustAlternative = skipSlow;369 skipRustAlternative = skipSlow;356 skipCpp = "can't run: uses up to 192GB of RAM";370 skipCpp = "can't run: uses up to 192GB of RAM";357 skipGo = skipSlow;371 skipGo = skipSlow;358 }}372 }}359 '';373 '';360}374}361375nix/fetch-jrq.nixdiffbeforeafterboth10 name ? "jrq-vendor",10 name ? "jrq-vendor",11}:11}:12stdenvNoCC.mkDerivation (finalAttrs: {12stdenvNoCC.mkDerivation (finalAttrs: {13 inherit name;13 name = "${name}-${builtins.substring 0 8 (builtins.hashFile "sha256" lockfile)}";141415 outputHashMode = "recursive";15 outputHashMode = "recursive";16 outputHashAlgo = "sha256";16 outputHashAlgo = "sha256";tests/realworld/entry-kube-prometheus.jsonnetdiffbeforeafterbothno changes