difftreelog
ci benchmark against stable release
in: master
4 files changed
flake.nixdiffbeforeafterboth1{2 description = "Jrsonnet";3 inputs = {4 nixpkgs.url = "github:nixos/nixpkgs";5 flake-utils.url = "github:numtide/flake-utils";6 rust-overlay.url = "github:oxalica/rust-overlay";7 };8 outputs = { nixpkgs, flake-utils, rust-overlay, ... }:9 flake-utils.lib.eachDefaultSystem (system:10 let11 pkgs = import nixpkgs {12 inherit system;13 overlays = [ rust-overlay.overlays.default ];14 };15 rust = ((pkgs.rustChannelOf { date = "2022-11-10"; channel = "nightly"; }).default.override {16 extensions = [ "rust-src" "miri" ];17 });18 in19 rec {20 packages = rec {21 go-jsonnet = pkgs.callPackage ./nix/go-jsonnet.nix { };22 sjsonnet = pkgs.callPackage ./nix/sjsonnet.nix { };23 jsonnet = pkgs.callPackage ./nix/jsonnet.nix { };24 # I didn't managed to build it, and nixpkgs version is marked as broken25 # haskell-jsonnet = pkgs.callPackage ./nix/haskell-jsonnet.nix { };26 jrsonnet = pkgs.callPackage ./nix/jrsonnet.nix {27 rustPlatform = pkgs.makeRustPlatform {28 rustc = rust;29 cargo = rust;30 };31 };3233 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {34 inherit go-jsonnet sjsonnet jsonnet jrsonnet;35 };36 };37 devShell = pkgs.mkShell {38 nativeBuildInputs = with pkgs;[39 rust40 cargo-edit41 lld42 hyperfine43 valgrind44 ];45 };46 }47 );48}nix/benchmarks.nixdiffbeforeafterboth--- a/nix/benchmarks.nix
+++ b/nix/benchmarks.nix
@@ -5,11 +5,13 @@
, stdenv
, fetchFromGitHub
, jrsonnet
+, jrsonnet-release
, go-jsonnet
, sjsonnet
, jsonnet
, hyperfine
, quick ? false
+, againstRelease ? false
}:
let
jsonnetBench = fetchFromGitHub {
@@ -53,7 +55,9 @@
${jsonnet-bundler}/bin/jb install
'';
- skipSlow = if quick then "slow benchmark" else "";
+ # Removes outsiders from the output
+ # Useful when comparing performance of different jrsonnet releases
+ skipSlow = if quick then "slow benchmark, but only quick requested" else "";
in
stdenv.mkDerivation {
name = "benchmarks";
@@ -67,7 +71,7 @@
jsonnet
hyperfine
- ];
+ ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);
installPhase =
let
@@ -89,23 +93,26 @@
echo "> Note: No results for C++, ${skipCpp}" >> $out
echo >> $out
'' else ""}
- echo "<details>" >> $out
- echo "<summary>Source</summary>" >> $out
- echo >> $out
- echo "\`\`\`jsonnet" >> $out
- ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
- ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
- echo >> $out
- echo "\`\`\`" >> $out
- echo "</details>" >> $out
- echo >> $out
+ ${if !quick then ''
+ echo "<details>" >> $out
+ echo "<summary>Source</summary>" >> $out
+ echo >> $out
+ echo "\`\`\`jsonnet" >> $out
+ ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
+ ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
+ echo >> $out
+ echo "\`\`\`" >> $out
+ echo "</details>" >> $out
+ echo >> $out
+ '' else ""}
path=${path}
${if pathIsGenerator then ''
jrsonnet $path > generated.jsonnet
path=generated.jsonnet
'' else ""}
- hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \
+ hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \
"jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \
+ ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \
${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \
${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \
${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }
@@ -114,40 +121,42 @@
in
''
touch $out
- cat ${./benchmarks.md} >> $out
- echo >> $out
+ ${if !quick then ''
+ cat ${./benchmarks.md} >> $out
+ echo >> $out
- echo "<details>" >> $out
- echo "<summary>Tested versions</summary>" >> $out
- echo >> $out
- echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- jrsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo Go: $(go-jsonnet --version) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- go-jsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo C++: $(jsonnet --version) >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- jsonnet --help >> $out
- echo "\`\`\`" >> $out
- echo >> $out
- echo Scala: >> $out
- echo >> $out
- echo "\`\`\`" >> $out
- sjsonnet 2>> $out || true
- echo "\`\`\`" >> $out
- echo >> $out
- echo "</details>" >> $out
- echo >> $out
+ echo "<details>" >> $out
+ echo "<summary>Tested versions</summary>" >> $out
+ echo >> $out
+ echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ jrsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo Go: $(go-jsonnet --version) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ go-jsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo C++: $(jsonnet --version) >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ jsonnet --help >> $out
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo Scala: >> $out
+ echo >> $out
+ echo "\`\`\`" >> $out
+ sjsonnet 2>> $out || true
+ echo "\`\`\`" >> $out
+ echo >> $out
+ echo "</details>" >> $out
+ echo >> $out
- echo >> $out
+ echo >> $out
+ '' else ""}
echo "## Real world" >> $out
${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}
${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}
nix/jrsonnet-release.nixdiffbeforeafterboth--- a/nix/jrsonnet-release.nix
+++ b/nix/jrsonnet-release.nix
@@ -3,14 +3,15 @@
rustPlatform.buildRustPackage rec {
pname = "jrsonnet";
- version = "5f0f8de9f52f961e2ff162e0a3fd4ca20a275f1d";
+ version = "d32fe45b8ed28fb39b5359a704922922368af1c0";
src = fetchFromGitHub {
owner = "CertainLach";
repo = pname;
rev = version;
- hash = lib.fakeHash;
+ hash = "sha256-R9Xt36bYS5upVDzt8hEifwmfocXpJbIKwvxkoJNEGVc=";
};
+ cargoHash = "sha256-V+KGWeNlUnelofaGzufNPLGDyxazoFrjZ/n391VYYws=";
cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
nix/jrsonnet.nixdiffbeforeafterboth--- a/nix/jrsonnet.nix
+++ b/nix/jrsonnet.nix
@@ -20,8 +20,8 @@
pname = "jrsonnet";
version = "git";
- cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
- cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
+ cargoTestFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
+ cargoBuildFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
buildInputs = [ makeWrapper ];