From 3ab5bdb7f60f412e356794686502f1a8f42cdcc6 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Thu, 07 May 2026 21:10:06 +0000 Subject: [PATCH] ci: more realworld benchmarks --- --- a/flake.nix +++ b/flake.nix @@ -218,6 +218,9 @@ in { legacyPackages = { + fetchJrq = pkgs.callPackage ./nix/fetch-jrq.nix { + inherit (self'.packages) jrsonnet; + }; release = optionalAttrs pkgs.stdenv.hostPlatform.isLinux ( { jrsonnet-linux-glibc = self'.packages.jrsonnet; @@ -254,6 +257,7 @@ ); benchmarks = optionalAttrs (system == "x86_64-linux" || system == "aarch64-linux") { default = pkgs.callPackage ./nix/benchmarks.nix { + inherit (config.legacyPackages) fetchJrq; inherit (config.legacyPackages.jsonnetImpls) go-jsonnet sjsonnet @@ -268,6 +272,7 @@ ]; }; quick = pkgs.callPackage ./nix/benchmarks.nix { + inherit (config.legacyPackages) fetchJrq; inherit (config.legacyPackages.jsonnetImpls) go-jsonnet sjsonnet @@ -283,6 +288,7 @@ ]; }; against-release = pkgs.callPackage ./nix/benchmarks.nix { + inherit (config.legacyPackages) fetchJrq; inherit (config.legacyPackages.jsonnetImpls) go-jsonnet sjsonnet @@ -305,6 +311,7 @@ ]; }; quick-against-release = pkgs.callPackage ./nix/benchmarks.nix { + inherit (config.legacyPackages) fetchJrq; inherit (config.legacyPackages.jsonnetImpls) go-jsonnet sjsonnet --- a/nix/benchmarks.nix +++ b/nix/benchmarks.nix @@ -1,10 +1,9 @@ { lib, runCommand, - jsonnet-bundler, - cacert, stdenv, fetchFromGitHub, + fetchJrq, go-jsonnet, sjsonnet, cpp-jsonnet, @@ -23,29 +22,16 @@ rev = "bc305df3fe587960f7635f0185571500e5988475"; hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc="; }; - kubePrometheusBench = - let - src = fetchFromGitHub { - owner = "prometheus-operator"; - repo = "kube-prometheus"; - rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd"; - hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM="; - }; - in - runCommand "kube-prometheus-vendor" - { - outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw="; - outputHashMode = "recursive"; - buildInputs = [ cacert ]; - } - '' - mkdir -p $out - cp -r ${src}/* $out/ - cd $out - chmod u+w jsonnetfile.lock.json - mkdir vendor - ${jsonnet-bundler}/bin/jb install - ''; + realworldVendor = fetchJrq { + name = "realworld-vendor"; + lockfile = ../tests/realworld/jsonnetfile.lock.json; + vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc="; + }; + realworldBench = runCommand "realworld-bench" { } '' + mkdir -p $out + cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/ + cp -r ${realworldVendor} $out/vendor + ''; # Removes outsiders from the output # Useful when comparing performance of different jrsonnet releases @@ -77,8 +63,11 @@ skipScala ? "", skipCpp ? "", skipGo ? "", - vendor ? "", + jpaths ? [ ], }: + let + jpathArgs = concatMapStrings (p: " -J ${p}") jpaths; + in '' echo >> $out echo "=== ${name}" >> $out @@ -122,7 +111,7 @@ concatStringsSep " " ( forEach jrsonnetVariants ( variant: - "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${ + "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${ if variant.name != "" then " (${variant.name})" else "" }\"" ) @@ -130,26 +119,24 @@ } \ ${ optionalString (skipRustAlternative == "") - "\"rsjsonnet $path${ - optionalString (vendor != "") " -J ${vendor}" - }\" -n \"Rust (alternative, rsjsonnet)\"" + "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\"" } \ ${ optionalString (skipGo == "") - "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\"" + "\"go-jsonnet $path${jpathArgs}\" -n \"Go\"" } \ ${ optionalString (skipScala == "") - "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\"" + "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\"" } \ ${ # My aarch64-linux machine can't run graalvm image: # The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE]. optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux") - "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\"" + "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\"" } \ ${optionalString (skipCpp == "") - "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\"" + "\"jsonnet $path${jpathArgs}\" -n \"C++\"" } cat result.adoc >> $out ''; @@ -196,10 +183,36 @@ skipGo = skipSlow; }} ${mkBench { - name = "Kube-prometheus manifests"; - vendor = "${kubePrometheusBench}/vendor"; - path = "${kubePrometheusBench}/example.jsonnet"; - omitSource = true; + name = "Loki manifests"; + path = "${realworldBench}/entry-loki.jsonnet"; + jpaths = [ "${realworldBench}/vendor" ]; + skipCpp = "too slow, takes hours, skews results"; + skipGo = skipSlow; + }} + ${mkBench { + name = "Mimir manifests"; + path = "${realworldBench}/entry-mimir.jsonnet"; + jpaths = [ "${realworldBench}/vendor" ]; + skipCpp = "too slow, takes hours, skews results"; + skipGo = skipSlow; + }} + ${mkBench { + name = "Tempo manifests"; + path = "${realworldBench}/entry-tempo.jsonnet"; + jpaths = [ "${realworldBench}/vendor" ]; + skipCpp = "too slow, takes hours, skews results"; + skipGo = skipSlow; + }} + ${mkBench { + name = "GitLab runbooks dashboards"; + path = "${realworldBench}/entry-gitlab-runbooks.jsonnet"; + jpaths = [ + "${realworldBench}/vendor" + "${realworldBench}/vendor/runbooks/libsonnet" + "${realworldBench}/vendor/runbooks/dashboards" + "${realworldBench}/vendor/runbooks/services" + "${realworldBench}/vendor/runbooks/metrics-catalog" + ]; skipCpp = "too slow, takes hours, skews results"; skipGo = skipSlow; }} --- /dev/null +++ b/nix/fetch-jrq.nix @@ -0,0 +1,49 @@ +{ + lib, + stdenvNoCC, + cacert, + jrsonnet, +}: +{ + lockfile, + vendorHash, + name ? "jrq-vendor", +}: +stdenvNoCC.mkDerivation (finalAttrs: { + inherit name; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = vendorHash; + + nativeBuildInputs = [ + jrsonnet + cacert + ]; + + dontUnpack = true; + dontConfigure = true; + dontInstall = true; + dontFixup = true; + + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + buildPhase = '' + runHook preBuild + + export HOME=$TMPDIR + + install -m644 ${lockfile} jsonnetfile.json + install -m644 ${lockfile} jsonnetfile.lock.json + + mkdir -p "$out" + jrb --jsonnetpkg-home "$out" install + + runHook postBuild + ''; + + passthru = { + inherit lockfile vendorHash; + }; +}) --- /dev/null +++ b/tests/realworld/.gitignore @@ -0,0 +1 @@ +/vendor --- /dev/null +++ b/tests/realworld/entry-gitlab-runbooks.jsonnet @@ -0,0 +1 @@ +import 'api/main.dashboard.jsonnet' --- /dev/null +++ b/tests/realworld/entry-graalvm.jsonnet @@ -0,0 +1,42 @@ +local common = import 'ci/ci_common/common.jsonnet'; +local graal_common = import 'graal-common.json'; + +local compiler = import 'compiler/ci/ci.jsonnet'; +local wasm = import 'wasm/ci/ci.jsonnet'; +local espresso = import 'espresso/ci/ci.jsonnet'; +local regex = import 'regex/ci/ci.jsonnet'; +local sdk = import 'sdk/ci/ci.jsonnet'; +local substratevm = import 'substratevm/ci/ci.jsonnet'; +local sulong = import 'sulong/ci/ci.jsonnet'; +local tools = import 'tools/ci/ci.jsonnet'; +local truffle = import 'truffle/ci/ci.jsonnet'; +local javadoc = import 'ci_includes/publish-javadoc.jsonnet'; +local visualizer = import 'visualizer/ci/ci.jsonnet'; +local web_image = import 'web-image/ci/ci.jsonnet'; + +{ + ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'), + overlay: graal_common.ci.overlay, + specVersion: '7', + tierConfig: { + tier1: 'gate', + tier2: 'gate', + tier3: 'gate', + tier4: 'post-merge', + }, + builds: [common.add_excludes_guard(common.with_style_component(b)) for b in ( + common.with_components(compiler.builds, ['compiler']) + + common.with_components(wasm.builds, ['wasm']) + + common.with_components(espresso.builds, ['espresso']) + + common.with_components(regex.builds, ['regex']) + + common.with_components(sdk.builds, ['sdk']) + + common.with_components(substratevm.builds, ['svm']) + + common.with_components(sulong.builds, ['sulong']) + + common.with_components(tools.builds, ['tools']) + + common.with_components(truffle.builds, ['truffle']) + + common.with_components(javadoc.builds, ['javadoc']) + + common.with_components(visualizer.builds, ['visualizer']) + + common.with_components(web_image.builds, ['webimage']) + )], + assert (import 'ci/ci_common/run-spec-demo.jsonnet').check(), +} --- /dev/null +++ b/tests/realworld/entry-loki.jsonnet @@ -0,0 +1,30 @@ +local loki = import 'loki/loki.libsonnet'; + +loki { + _config+:: { + namespace: 'loki', + cluster: 'loki-bench', + storage_backend: 's3', + s3_address: 's3.example.com', + s3_bucket_name: 'loki-bench', + s3_access_key: 'AKIA', + s3_secret_access_key: 'SECRET', + boltdb_shipper_shared_store: 's3', + + using_boltdb_shipper: false, + using_tsdb_shipper: true, + use_index_gateway: true, + + loki+: { + schema_config+: { + configs: [{ + from: '2024-01-01', + store: 'tsdb', + object_store: 's3', + schema: 'v13', + index: { prefix: 'loki_index_', period: '24h' }, + }], + }, + }, + }, +} --- /dev/null +++ b/tests/realworld/entry-mimir.jsonnet @@ -0,0 +1,14 @@ +local mimir = import 'mimir/mimir.libsonnet'; + +mimir { + _config+:: { + namespace: 'mimir', + cluster: 'mimir-bench', + external_url: 'http://mimir.example.com', + + storage_backend: 'gcs', + blocks_storage_bucket_name: 'mimir-blocks', + ruler_storage_bucket_name: 'mimir-ruler', + alertmanager_storage_bucket_name: 'mimir-alertmanager', + }, +} --- /dev/null +++ b/tests/realworld/entry-tempo.jsonnet @@ -0,0 +1,36 @@ +local tempo = import 'microservices/tempo.libsonnet'; + +tempo { + _images+:: { + tempo: 'grafana/tempo:latest', + tempo_vulture: 'grafana/tempo-vulture:latest', + tempo_query: 'grafana/tempo-query:latest', + }, + + _config+:: { + namespace: 'tracing', + distributor+: { + receivers: { + otlp: { protocols: { grpc: { endpoint: '0.0.0.0:4317' } } }, + }, + }, + metrics_generator+: { + pvc_size: '10Gi', + pvc_storage_class: 'fast', + ephemeral_storage_request_size: '10Gi', + ephemeral_storage_limit_size: '11Gi', + }, + live_store+: { + pvc_size: '10Gi', + pvc_storage_class: 'fast', + }, + backend_scheduler+: { + pvc_size: '200Mi', + pvc_storage_class: 'fast', + }, + backend: 'gcs', + bucket: 'tempo-bench', + kafka_address: 'kafka:9092', + kafka_topic: 'tempo', + }, +} --- /dev/null +++ b/tests/realworld/jsonnetfile.json @@ -0,0 +1,52 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/kube-prometheus" + } + } + }, + { + "source": { + "git": { + "remote": "https://github.com/oracle/graal" + } + } + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki", + "subdir": "production/ksonnet/loki" + } + } + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/mimir", + "subdir": "operations/mimir" + } + } + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/tempo", + "subdir": "operations/jsonnet/microservices" + } + } + }, + { + "source": { + "git": { + "remote": "https://gitlab.com/gitlab-com/runbooks" + } + }, + "version": "c4082c94065b1b6a521eed59ed624478131cd9c0" + } + ], + "legacyImports": true +} --- /dev/null +++ b/tests/realworld/jsonnetfile.lock.json @@ -0,0 +1,341 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/kube-prometheus" + } + }, + "version": "392c6c5e8e83c9a0f76355d4b0837dccf76ad7ea" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "mixin-utils" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/brancz/kubernetes-grafana", + "subdir": "grafana" + } + }, + "version": "5698c8940b6dadca3f42107b7839557bc041761f" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib.git", + "subdir": "grafonnet" + } + }, + "version": "a1d61cce1da59c71409b99b5c7568511fec661ea" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafana", + "subdir": "grafana-mixin" + } + }, + "version": "278a9c4cb758a546229e2b408def7e3dcb5f4ec0", + "name": "grafana-mixin" + }, + { + "source": { + "git": { + "remote": "https://github.com/etcd-io/etcd", + "subdir": "contrib/mixin" + } + }, + "version": "0cbc031e7f224102bd1ec42a81f659a2b8c3350c" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-v10.0.0" + } + }, + "version": "7380c9c64fb973f34c3ec46265621a2b0dee0058" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/docsonnet.git", + "subdir": "doc-util" + } + }, + "version": "bf6f08ae02a51c48bdcec4629b1c1a5a62c6f803" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/xtd.git" + } + }, + "version": "4d7f8cb24d613430799f9d56809cc6964f35cea9" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/prometheus-operator", + "subdir": "jsonnet/prometheus-operator" + } + }, + "version": "288f0ae3f4ef14e3c75078cbd7dfe1857c3fb066" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/prometheus-operator", + "subdir": "jsonnet/mixin" + } + }, + "version": "288f0ae3f4ef14e3c75078cbd7dfe1857c3fb066", + "name": "prometheus-operator-mixin" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin" + } + }, + "version": "acd544dbd02ebce1e000abe006df6d2e4ac1438e" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-latest" + } + }, + "version": "7380c9c64fb973f34c3ec46265621a2b0dee0058" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-v11.4.0" + } + }, + "version": "7380c9c64fb973f34c3ec46265621a2b0dee0058" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics", + "subdir": "jsonnet/kube-state-metrics" + } + }, + "version": "cd5430fd1834f43ffd093e3e22c2af1a625dbe2e" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics", + "subdir": "jsonnet/kube-state-metrics-mixin" + } + }, + "version": "cd5430fd1834f43ffd093e3e22c2af1a625dbe2e" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/node_exporter", + "subdir": "docs/node-mixin" + } + }, + "version": "d6d0e710bb7daf07a2743fde060f0d5f32c565f3" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/prometheus", + "subdir": "documentation/prometheus-mixin" + } + }, + "version": "91c184a899b8e8237cdd08876fba54aa5f9feb6c", + "name": "prometheus" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/alertmanager", + "subdir": "doc/alertmanager-mixin" + } + }, + "version": "c85e06d3cd4ba24d003454ee11834404224d7763", + "name": "alertmanager" + }, + { + "source": { + "git": { + "remote": "https://github.com/pyrra-dev/pyrra.git", + "subdir": "jsonnet/controller-gen" + } + }, + "version": "25bac00a6c478211326eed4aa46583d1b9466ca8", + "name": "pyrra" + }, + { + "source": { + "git": { + "remote": "https://github.com/thanos-io/thanos", + "subdir": "mixin" + } + }, + "version": "cdca5483752e1635d249fbb1271978262775dc1b", + "name": "thanos-mixin" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "grafana-builder" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/oracle/graal" + } + }, + "version": "d24e4efdac60ee798d7a795cf015f084556c912e" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki", + "subdir": "production/ksonnet/loki" + } + }, + "version": "fe7f43b7f36d8f90931a3bfd35fadc83e1c23d5f" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "consul" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "jaeger-agent-mixin" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "ksonnet-util" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "memcached" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/mimir", + "subdir": "operations/mimir" + } + }, + "version": "39dd4ffb859d0abb3951d406495bc4bd422600dd" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/rollout-operator.git", + "subdir": "operations/rollout-operator" + } + }, + "version": "2afe062893496f0554b4764bbe1638f8019832fb" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/keda-libsonnet.git", + "subdir": "2.15" + } + }, + "version": "dbc8cf1a9847f123d8325378111155fb135983ab", + "name": "keda-libsonnet" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/tempo", + "subdir": "operations/jsonnet/microservices" + } + }, + "version": "f6124c0dea954763dbaa461dcc7b8a89bc8d2635" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git", + "subdir": "1.32" + } + }, + "version": "55380470fb7979e6ce0c4316cb9c27a266caf298" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/vertical-pod-autoscaler-libsonnet.git", + "subdir": "1.0.0" + } + }, + "version": "5d16cd466d12489e6ffba5b17813a0203016e68d" + }, + { + "source": { + "git": { + "remote": "https://gitlab.com/gitlab-com/runbooks" + } + }, + "version": "c4082c94065b1b6a521eed59ed624478131cd9c0" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "memcached-mixin" + } + }, + "version": "3b8056670048416f5fb660dc039e473eb20a40b2" + }, + { + "source": { + "git": { + "remote": "https://github.com/yugui/jsonnetunit.git", + "subdir": "jsonnetunit" + } + }, + "version": "6927c58cae7624a00f368b977ccc477d4f74071f" + } + ], + "legacyImports": true +} --- /dev/null +++ b/tests/realworld/k.libsonnet @@ -0,0 +1 @@ +import '1.32/main.libsonnet' -- gitstuff