difftreelog
ci benchmark against stable release
in: master
4 files changed
flake.nixdiffbeforeafterboth29 cargo = rust;29 cargo = rust;30 };30 };31 };31 };32 jrsonnet-release = pkgs.callPackage ./nix/jrsonnet-release.nix {33 rustPlatform = pkgs.makeRustPlatform {34 rustc = rust;35 cargo = rust;36 };37 };323833 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {39 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {34 inherit go-jsonnet sjsonnet jsonnet jrsonnet;40 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;35 };41 };42 benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {43 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;44 quick = true;45 };46 benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {47 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;48 againstRelease = true;49 };50 benchmarks-quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {51 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;52 quick = true;53 againstRelease = true;54 };36 };55 };37 devShell = pkgs.mkShell {56 devShell = pkgs.mkShell {38 nativeBuildInputs = with pkgs;[57 nativeBuildInputs = with pkgs;[nix/benchmarks.nixdiffbeforeafterboth--- a/nix/benchmarks.nix
+++ b/nix/benchmarks.nix
@@ -5,11 +5,13 @@
, stdenv
, fetchFromGitHub
, jrsonnet
+, jrsonnet-release
, go-jsonnet
, sjsonnet
, jsonnet
, hyperfine
, quick ? false
+, againstRelease ? false
}:
let
jsonnetBench = fetchFromGitHub {
@@ -53,7 +55,9 @@
${jsonnet-bundler}/bin/jb install
'';
- skipSlow = if quick then "slow benchmark" else "";
+ # Removes outsiders from the output
+ # Useful when comparing performance of different jrsonnet releases
+ skipSlow = if quick then "slow benchmark, but only quick requested" else "";
in
stdenv.mkDerivation {
name = "benchmarks";
@@ -67,7 +71,7 @@
jsonnet
hyperfine
- ];
+ ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);
installPhase =
let
@@ -89,23 +93,26 @@
echo "> Note: No results for C++, ${skipCpp}" >> $out
echo >> $out
'' else ""}
- echo "<details>" >> $out
- echo "<summary>Source</summary>" >> $out
- echo >> $out
- echo "\`\`\`jsonnet" >> $out
- ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
- ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
- echo >> $out
- echo "\`\`\`" >> $out
- echo "</details>" >> $out
- echo >> $out
+ ${if !quick then ''
+ echo "<details>" >> $out
+ echo "<summary>Source</summary>" >> $out
+ echo >> $out
+ echo "\`\`\`jsonnet" >> $out
+ ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
+ ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
+ echo >> $out
+ echo "\`\`\`" >> $out
+ echo "</details>" >> $out
+ echo >> $out
+ '' else ""}
path=${path}
${if pathIsGenerator then ''
jrsonnet $path > generated.jsonnet
path=generated.jsonnet
'' else ""}
- hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \
+ hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \
"jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \
+ ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \
${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \
${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \
${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }
@@ -114,40 +121,42 @@
in
''
touch $out
- cat ${./benchmarks.md} >> $out
- echo >> $out
+ ${if !quick then ''
+ cat ${./benchmarks.md} >> $out
+ echo >> $out
- echo "<details>" >> $out
- echo "<summary>Tested versions</summary>" >> $out
- echo >> $out
- echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- jrsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo Go: $(go-jsonnet --version) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- go-jsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo C++: $(jsonnet --version) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- jsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo Scala: >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- sjsonnet 2>> $out || true
- echo "\`\`\`" >> $out
- echo >> $out
- echo "</details>" >> $out
- echo >> $out
+ echo "<details>" >> $out
+ echo "<summary>Tested versions</summary>" >> $out
+ echo >> $out
+ echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ jrsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo Go: $(go-jsonnet --version) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ go-jsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo C++: $(jsonnet --version) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ jsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo Scala: >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ sjsonnet 2>> $out || true
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo "</details>" >> $out
+ echo >> $out
- echo >> $out
+ echo >> $out
+ '' else ""}
echo "## Real world" >> $out
${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}
${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}
nix/jrsonnet-release.nixdiffbeforeafterboth--- a/nix/jrsonnet-release.nix
+++ b/nix/jrsonnet-release.nix
@@ -3,14 +3,15 @@
rustPlatform.buildRustPackage rec {
pname = "jrsonnet";
- version = "5f0f8de9f52f961e2ff162e0a3fd4ca20a275f1d";
+ version = "d32fe45b8ed28fb39b5359a704922922368af1c0";
src = fetchFromGitHub {
owner = "CertainLach";
repo = pname;
rev = version;
- hash = lib.fakeHash;
+ hash = "sha256-R9Xt36bYS5upVDzt8hEifwmfocXpJbIKwvxkoJNEGVc=";
};
+ cargoHash = "sha256-V+KGWeNlUnelofaGzufNPLGDyxazoFrjZ/n391VYYws=";
cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
nix/jrsonnet.nixdiffbeforeafterboth--- a/nix/jrsonnet.nix
+++ b/nix/jrsonnet.nix
@@ -20,8 +20,8 @@
pname = "jrsonnet";
version = "git";
- cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
- cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
+ cargoTestFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
+ cargoBuildFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
buildInputs = [ makeWrapper ];