difftreelog
ci more realworld benchmarks
in: master
12 files changed
flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -218,6 +218,9 @@
in
{
legacyPackages = {
+ fetchJrq = pkgs.callPackage ./nix/fetch-jrq.nix {
+ inherit (self'.packages) jrsonnet;
+ };
release = optionalAttrs pkgs.stdenv.hostPlatform.isLinux (
{
jrsonnet-linux-glibc = self'.packages.jrsonnet;
@@ -254,6 +257,7 @@
);
benchmarks = optionalAttrs (system == "x86_64-linux" || system == "aarch64-linux") {
default = pkgs.callPackage ./nix/benchmarks.nix {
+ inherit (config.legacyPackages) fetchJrq;
inherit (config.legacyPackages.jsonnetImpls)
go-jsonnet
sjsonnet
@@ -268,6 +272,7 @@
];
};
quick = pkgs.callPackage ./nix/benchmarks.nix {
+ inherit (config.legacyPackages) fetchJrq;
inherit (config.legacyPackages.jsonnetImpls)
go-jsonnet
sjsonnet
@@ -283,6 +288,7 @@
];
};
against-release = pkgs.callPackage ./nix/benchmarks.nix {
+ inherit (config.legacyPackages) fetchJrq;
inherit (config.legacyPackages.jsonnetImpls)
go-jsonnet
sjsonnet
@@ -305,6 +311,7 @@
];
};
quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {
+ inherit (config.legacyPackages) fetchJrq;
inherit (config.legacyPackages.jsonnetImpls)
go-jsonnet
sjsonnet
nix/benchmarks.nixdiffbeforeafterboth--- a/nix/benchmarks.nix
+++ b/nix/benchmarks.nix
@@ -1,10 +1,9 @@
{
lib,
runCommand,
- jsonnet-bundler,
- cacert,
stdenv,
fetchFromGitHub,
+ fetchJrq,
go-jsonnet,
sjsonnet,
cpp-jsonnet,
@@ -23,29 +22,16 @@
rev = "bc305df3fe587960f7635f0185571500e5988475";
hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";
};
- kubePrometheusBench =
- let
- src = fetchFromGitHub {
- owner = "prometheus-operator";
- repo = "kube-prometheus";
- rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";
- hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";
- };
- in
- runCommand "kube-prometheus-vendor"
- {
- outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";
- outputHashMode = "recursive";
- buildInputs = [ cacert ];
- }
- ''
- mkdir -p $out
- cp -r ${src}/* $out/
- cd $out
- chmod u+w jsonnetfile.lock.json
- mkdir vendor
- ${jsonnet-bundler}/bin/jb install
- '';
+ realworldVendor = fetchJrq {
+ name = "realworld-vendor";
+ lockfile = ../tests/realworld/jsonnetfile.lock.json;
+ vendorHash = "sha256-6tXi6bRw77YKB17PhSpJnVYQcsGOvB8sgjKIrFtrwfc=";
+ };
+ realworldBench = runCommand "realworld-bench" { } ''
+ mkdir -p $out
+ cp ${../tests/realworld}/*.jsonnet ${../tests/realworld}/*.libsonnet $out/
+ cp -r ${realworldVendor} $out/vendor
+ '';
# Removes outsiders from the output
# Useful when comparing performance of different jrsonnet releases
@@ -77,8 +63,11 @@
skipScala ? "",
skipCpp ? "",
skipGo ? "",
- vendor ? "",
+ jpaths ? [ ],
}:
+ let
+ jpathArgs = concatMapStrings (p: " -J ${p}") jpaths;
+ in
''
echo >> $out
echo "=== ${name}" >> $out
@@ -122,7 +111,7 @@
concatStringsSep " " (
forEach jrsonnetVariants (
variant:
- "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${
+ "\"${variant.drv}/bin/jrsonnet $path${jpathArgs}\" -n \"Rust${
if variant.name != "" then " (${variant.name})" else ""
}\""
)
@@ -130,26 +119,24 @@
} \
${
optionalString (skipRustAlternative == "")
- "\"rsjsonnet $path${
- optionalString (vendor != "") " -J ${vendor}"
- }\" -n \"Rust (alternative, rsjsonnet)\""
+ "\"rsjsonnet $path${jpathArgs}\" -n \"Rust (alternative, rsjsonnet)\""
} \
${
optionalString (skipGo == "")
- "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""
+ "\"go-jsonnet $path${jpathArgs}\" -n \"Go\""
} \
${
optionalString (skipScala == "")
- "\"sjsonnet-native $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (native)\""
+ "\"sjsonnet-native $path${jpathArgs}\" -n \"Scala (native)\""
} \
${
# My aarch64-linux machine can't run graalvm image:
# The current machine does not support all of the following CPU features that are required by the image: [FP, ASIMD, CRC32, LSE].
optionalString (skipScala == "" && stdenv.hostPlatform.system != "aarch64-linux")
- "\"sjsonnet-graalvm $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala (GraalVM)\""
+ "\"sjsonnet-graalvm $path${jpathArgs}\" -n \"Scala (GraalVM)\""
} \
${optionalString (skipCpp == "")
- "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""
+ "\"jsonnet $path${jpathArgs}\" -n \"C++\""
}
cat result.adoc >> $out
'';
@@ -196,10 +183,36 @@
skipGo = skipSlow;
}}
${mkBench {
- name = "Kube-prometheus manifests";
- vendor = "${kubePrometheusBench}/vendor";
- path = "${kubePrometheusBench}/example.jsonnet";
- omitSource = true;
+ name = "Loki manifests";
+ path = "${realworldBench}/entry-loki.jsonnet";
+ jpaths = [ "${realworldBench}/vendor" ];
+ skipCpp = "too slow, takes hours, skews results";
+ skipGo = skipSlow;
+ }}
+ ${mkBench {
+ name = "Mimir manifests";
+ path = "${realworldBench}/entry-mimir.jsonnet";
+ jpaths = [ "${realworldBench}/vendor" ];
+ skipCpp = "too slow, takes hours, skews results";
+ skipGo = skipSlow;
+ }}
+ ${mkBench {
+ name = "Tempo manifests";
+ path = "${realworldBench}/entry-tempo.jsonnet";
+ jpaths = [ "${realworldBench}/vendor" ];
+ skipCpp = "too slow, takes hours, skews results";
+ skipGo = skipSlow;
+ }}
+ ${mkBench {
+ name = "GitLab runbooks dashboards";
+ path = "${realworldBench}/entry-gitlab-runbooks.jsonnet";
+ jpaths = [
+ "${realworldBench}/vendor"
+ "${realworldBench}/vendor/runbooks/libsonnet"
+ "${realworldBench}/vendor/runbooks/dashboards"
+ "${realworldBench}/vendor/runbooks/services"
+ "${realworldBench}/vendor/runbooks/metrics-catalog"
+ ];
skipCpp = "too slow, takes hours, skews results";
skipGo = skipSlow;
}}
nix/fetch-jrq.nixdiffbeforeafterboth--- /dev/null
+++ b/nix/fetch-jrq.nix
@@ -0,0 +1,49 @@
+{
+ lib,
+ stdenvNoCC,
+ cacert,
+ jrsonnet,
+}:
+{
+ lockfile,
+ vendorHash,
+ name ? "jrq-vendor",
+}:
+stdenvNoCC.mkDerivation (finalAttrs: {
+ inherit name;
+
+ outputHashMode = "recursive";
+ outputHashAlgo = "sha256";
+ outputHash = vendorHash;
+
+ nativeBuildInputs = [
+ jrsonnet
+ cacert
+ ];
+
+ dontUnpack = true;
+ dontConfigure = true;
+ dontInstall = true;
+ dontFixup = true;
+
+ SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+ GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+ buildPhase = ''
+ runHook preBuild
+
+ export HOME=$TMPDIR
+
+ install -m644 ${lockfile} jsonnetfile.json
+ install -m644 ${lockfile} jsonnetfile.lock.json
+
+ mkdir -p "$out"
+ jrb --jsonnetpkg-home "$out" install
+
+ runHook postBuild
+ '';
+
+ passthru = {
+ inherit lockfile vendorHash;
+ };
+})
tests/realworld/.gitignorediffbeforeafterboth--- /dev/null
+++ b/tests/realworld/.gitignore
@@ -0,0 +1 @@
+/vendor
tests/realworld/entry-gitlab-runbooks.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/entry-gitlab-runbooks.jsonnet
@@ -0,0 +1 @@
+import 'api/main.dashboard.jsonnet'
tests/realworld/entry-graalvm.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/entry-graalvm.jsonnet
@@ -0,0 +1,42 @@
+local common = import 'ci/ci_common/common.jsonnet';
+local graal_common = import 'graal-common.json';
+
+local compiler = import 'compiler/ci/ci.jsonnet';
+local wasm = import 'wasm/ci/ci.jsonnet';
+local espresso = import 'espresso/ci/ci.jsonnet';
+local regex = import 'regex/ci/ci.jsonnet';
+local sdk = import 'sdk/ci/ci.jsonnet';
+local substratevm = import 'substratevm/ci/ci.jsonnet';
+local sulong = import 'sulong/ci/ci.jsonnet';
+local tools = import 'tools/ci/ci.jsonnet';
+local truffle = import 'truffle/ci/ci.jsonnet';
+local javadoc = import 'ci_includes/publish-javadoc.jsonnet';
+local visualizer = import 'visualizer/ci/ci.jsonnet';
+local web_image = import 'web-image/ci/ci.jsonnet';
+
+{
+ ci_resources:: (import 'ci/ci_common/ci-resources.libsonnet'),
+ overlay: graal_common.ci.overlay,
+ specVersion: '7',
+ tierConfig: {
+ tier1: 'gate',
+ tier2: 'gate',
+ tier3: 'gate',
+ tier4: 'post-merge',
+ },
+ builds: [common.add_excludes_guard(common.with_style_component(b)) for b in (
+ common.with_components(compiler.builds, ['compiler']) +
+ common.with_components(wasm.builds, ['wasm']) +
+ common.with_components(espresso.builds, ['espresso']) +
+ common.with_components(regex.builds, ['regex']) +
+ common.with_components(sdk.builds, ['sdk']) +
+ common.with_components(substratevm.builds, ['svm']) +
+ common.with_components(sulong.builds, ['sulong']) +
+ common.with_components(tools.builds, ['tools']) +
+ common.with_components(truffle.builds, ['truffle']) +
+ common.with_components(javadoc.builds, ['javadoc']) +
+ common.with_components(visualizer.builds, ['visualizer']) +
+ common.with_components(web_image.builds, ['webimage'])
+ )],
+ assert (import 'ci/ci_common/run-spec-demo.jsonnet').check(),
+}
tests/realworld/entry-loki.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/entry-loki.jsonnet
@@ -0,0 +1,30 @@
+local loki = import 'loki/loki.libsonnet';
+
+loki {
+ _config+:: {
+ namespace: 'loki',
+ cluster: 'loki-bench',
+ storage_backend: 's3',
+ s3_address: 's3.example.com',
+ s3_bucket_name: 'loki-bench',
+ s3_access_key: 'AKIA',
+ s3_secret_access_key: 'SECRET',
+ boltdb_shipper_shared_store: 's3',
+
+ using_boltdb_shipper: false,
+ using_tsdb_shipper: true,
+ use_index_gateway: true,
+
+ loki+: {
+ schema_config+: {
+ configs: [{
+ from: '2024-01-01',
+ store: 'tsdb',
+ object_store: 's3',
+ schema: 'v13',
+ index: { prefix: 'loki_index_', period: '24h' },
+ }],
+ },
+ },
+ },
+}
tests/realworld/entry-mimir.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/entry-mimir.jsonnet
@@ -0,0 +1,14 @@
+local mimir = import 'mimir/mimir.libsonnet';
+
+mimir {
+ _config+:: {
+ namespace: 'mimir',
+ cluster: 'mimir-bench',
+ external_url: 'http://mimir.example.com',
+
+ storage_backend: 'gcs',
+ blocks_storage_bucket_name: 'mimir-blocks',
+ ruler_storage_bucket_name: 'mimir-ruler',
+ alertmanager_storage_bucket_name: 'mimir-alertmanager',
+ },
+}
tests/realworld/entry-tempo.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/entry-tempo.jsonnet
@@ -0,0 +1,36 @@
+local tempo = import 'microservices/tempo.libsonnet';
+
+tempo {
+ _images+:: {
+ tempo: 'grafana/tempo:latest',
+ tempo_vulture: 'grafana/tempo-vulture:latest',
+ tempo_query: 'grafana/tempo-query:latest',
+ },
+
+ _config+:: {
+ namespace: 'tracing',
+ distributor+: {
+ receivers: {
+ otlp: { protocols: { grpc: { endpoint: '0.0.0.0:4317' } } },
+ },
+ },
+ metrics_generator+: {
+ pvc_size: '10Gi',
+ pvc_storage_class: 'fast',
+ ephemeral_storage_request_size: '10Gi',
+ ephemeral_storage_limit_size: '11Gi',
+ },
+ live_store+: {
+ pvc_size: '10Gi',
+ pvc_storage_class: 'fast',
+ },
+ backend_scheduler+: {
+ pvc_size: '200Mi',
+ pvc_storage_class: 'fast',
+ },
+ backend: 'gcs',
+ bucket: 'tempo-bench',
+ kafka_address: 'kafka:9092',
+ kafka_topic: 'tempo',
+ },
+}
tests/realworld/jsonnetfile.jsondiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/jsonnetfile.json
@@ -0,0 +1,52 @@
+{
+ "version": 1,
+ "dependencies": [
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/prometheus-operator/kube-prometheus"
+ }
+ }
+ },
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/oracle/graal"
+ }
+ }
+ },
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/grafana/loki",
+ "subdir": "production/ksonnet/loki"
+ }
+ }
+ },
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/grafana/mimir",
+ "subdir": "operations/mimir"
+ }
+ }
+ },
+ {
+ "source": {
+ "git": {
+ "remote": "https://github.com/grafana/tempo",
+ "subdir": "operations/jsonnet/microservices"
+ }
+ }
+ },
+ {
+ "source": {
+ "git": {
+ "remote": "https://gitlab.com/gitlab-com/runbooks"
+ }
+ },
+ "version": "c4082c94065b1b6a521eed59ed624478131cd9c0"
+ }
+ ],
+ "legacyImports": true
+}
tests/realworld/jsonnetfile.lock.jsondiffbeforeafterbothno changes
tests/realworld/k.libsonnetdiffbeforeafterboth--- /dev/null
+++ b/tests/realworld/k.libsonnet
@@ -0,0 +1 @@
+import '1.32/main.libsonnet'