git.delta.rocks / jrsonnet / refs/commits / ec8e2e1ce158

difftreelog

ci benchmark against stable release

Yaroslav Bolyukin2022-12-03parent: #fdd9d5d.patch.diff
in: master

4 files changed

modifiedflake.nixdiffbeforeafterboth
29 cargo = rust;29 cargo = rust;
30 };30 };
31 };31 };
32 jrsonnet-release = pkgs.callPackage ./nix/jrsonnet-release.nix {
33 rustPlatform = pkgs.makeRustPlatform {
34 rustc = rust;
35 cargo = rust;
36 };
37 };
3238
33 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {39 benchmarks = pkgs.callPackage ./nix/benchmarks.nix {
34 inherit go-jsonnet sjsonnet jsonnet jrsonnet;40 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
35 };41 };
42 benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {
43 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
44 quick = true;
45 };
46 benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {
47 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
48 againstRelease = true;
49 };
50 benchmarks-quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {
51 inherit go-jsonnet sjsonnet jsonnet jrsonnet jrsonnet-release;
52 quick = true;
53 againstRelease = true;
54 };
36 };55 };
37 devShell = pkgs.mkShell {56 devShell = pkgs.mkShell {
38 nativeBuildInputs = with pkgs;[57 nativeBuildInputs = with pkgs;[
modifiednix/benchmarks.nixdiffbeforeafterboth
--- a/nix/benchmarks.nix
+++ b/nix/benchmarks.nix
@@ -5,11 +5,13 @@
 , stdenv
 , fetchFromGitHub
 , jrsonnet
+, jrsonnet-release
 , go-jsonnet
 , sjsonnet
 , jsonnet
 , hyperfine
 , quick ? false
+, againstRelease ? false
 }:
 let
   jsonnetBench = fetchFromGitHub {
@@ -53,7 +55,9 @@
         ${jsonnet-bundler}/bin/jb install
       '';
 
-  skipSlow = if quick then "slow benchmark" else "";
+  # Removes outsiders from the output
+  # Useful when comparing performance of different jrsonnet releases
+  skipSlow = if quick then "slow benchmark, but only quick requested" else "";
 in
 stdenv.mkDerivation {
   name = "benchmarks";
@@ -67,7 +71,7 @@
     jsonnet
 
     hyperfine
-  ];
+  ] ++ (if againstRelease then [ jrsonnet-release ] else [ ]);
 
   installPhase =
     let
@@ -89,23 +93,26 @@
           echo "> Note: No results for C++, ${skipCpp}" >> $out
           echo >> $out
         '' else ""}
-        echo "<details>" >> $out
-        echo "<summary>Source</summary>" >> $out
-        echo >> $out
-        echo "\`\`\`jsonnet" >> $out
-        ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
-        ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
-        echo >> $out
-        echo "\`\`\`" >> $out
-        echo "</details>" >> $out
-        echo >> $out
+        ${if !quick then ''
+          echo "<details>" >> $out
+          echo "<summary>Source</summary>" >> $out
+          echo >> $out
+          echo "\`\`\`jsonnet" >> $out
+          ${if pathIsGenerator then "echo \"// Generator source\" >> $out" else ""}
+          ${if omitSource then "echo \"// Omitted: too large\" >> $out" else "cat ${path} >> $out"}
+          echo >> $out
+          echo "\`\`\`" >> $out
+          echo "</details>" >> $out
+          echo >> $out
+        '' else ""}
         path=${path}
         ${if pathIsGenerator then ''
           jrsonnet $path > generated.jsonnet
           path=generated.jsonnet
         '' else ""}
-        hyperfine -N ${if quick then "-r1" else ""} --output=pipe --style=basic --export-markdown result.md \
+        hyperfine -N -w4 --output=pipe --style=basic --export-markdown result.md \
           "jrsonnet $path ${if vendor != "" then "-J${vendor}" else ""}" -n "Rust" \
+          ${if againstRelease then "\"jrsonnet-release $path ${if vendor != "" then "-J${vendor}" else ""}\" -n \"Rust (released)\"" else "" } \
           ${if skipGo == "" then "\"go-jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Go\"" else "" } \
           ${if skipScala == "" then "\"sjsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"Scala\"" else "" } \
           ${if skipCpp == "" then "\"jsonnet $path ${if vendor != "" then "-J ${vendor}" else ""}\" -n \"C++\"" else "" }
@@ -114,40 +121,42 @@
     in
     ''
       touch $out
-      cat ${./benchmarks.md} >> $out
-      echo >> $out
+      ${if !quick then ''
+        cat ${./benchmarks.md} >> $out
+        echo >> $out
 
-      echo "<details>" >> $out
-      echo "<summary>Tested versions</summary>" >> $out
-      echo >> $out
-      echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
-      echo >> $out
-      echo "\`\`\`" >> $out
-      jrsonnet --help >> $out
-      echo "\`\`\`" >> $out
-      echo >> $out
-      echo Go: $(go-jsonnet --version) >> $out
-      echo >> $out
-      echo "\`\`\`" >> $out
-      go-jsonnet --help >> $out
-      echo "\`\`\`" >> $out
-      echo >> $out
-      echo C++: $(jsonnet --version) >> $out
-      echo >> $out
-      echo "\`\`\`" >> $out
-      jsonnet --help >> $out
-      echo "\`\`\`" >> $out
-      echo >> $out
-      echo Scala: >> $out
-      echo >> $out
-      echo "\`\`\`" >> $out
-      sjsonnet 2>> $out || true
-      echo "\`\`\`" >> $out
-      echo >> $out
-      echo "</details>" >> $out
-      echo >> $out
+        echo "<details>" >> $out
+        echo "<summary>Tested versions</summary>" >> $out
+        echo >> $out
+        echo Rust: git as $(date +'%d.%m.%Y' -u) >> $out
+        echo >> $out
+        echo "\`\`\`" >> $out
+        jrsonnet --help >> $out
+        echo "\`\`\`" >> $out
+        echo >> $out
+        echo Go: $(go-jsonnet --version) >> $out
+        echo >> $out
+        echo "\`\`\`" >> $out
+        go-jsonnet --help >> $out
+        echo "\`\`\`" >> $out
+        echo >> $out
+        echo C++: $(jsonnet --version) >> $out
+        echo >> $out
+        echo "\`\`\`" >> $out
+        jsonnet --help >> $out
+        echo "\`\`\`" >> $out
+        echo >> $out
+        echo Scala: >> $out
+        echo >> $out
+        echo "\`\`\`" >> $out
+        sjsonnet 2>> $out || true
+        echo "\`\`\`" >> $out
+        echo >> $out
+        echo "</details>" >> $out
+        echo >> $out
 
-      echo >> $out
+        echo >> $out
+      '' else ""}
       echo "## Real world" >> $out
       ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour";}}
       ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow;}}
modifiednix/jrsonnet-release.nixdiffbeforeafterboth
--- a/nix/jrsonnet-release.nix
+++ b/nix/jrsonnet-release.nix
@@ -3,14 +3,15 @@
 
 rustPlatform.buildRustPackage rec {
   pname = "jrsonnet";
-  version = "5f0f8de9f52f961e2ff162e0a3fd4ca20a275f1d";
+  version = "d32fe45b8ed28fb39b5359a704922922368af1c0";
 
   src = fetchFromGitHub {
     owner = "CertainLach";
     repo = pname;
     rev = version;
-    hash = lib.fakeHash;
+    hash = "sha256-R9Xt36bYS5upVDzt8hEifwmfocXpJbIKwvxkoJNEGVc=";
   };
+  cargoHash = "sha256-V+KGWeNlUnelofaGzufNPLGDyxazoFrjZ/n391VYYws=";
 
   cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
   cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
modifiednix/jrsonnet.nixdiffbeforeafterboth
--- a/nix/jrsonnet.nix
+++ b/nix/jrsonnet.nix
@@ -20,8 +20,8 @@
   pname = "jrsonnet";
   version = "git";
 
-  cargoTestFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
-  cargoBuildFlags = [ "--package=jrsonnet --features=mimalloc,legacy-this-file" ];
+  cargoTestFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
+  cargoBuildFlags = [ "--features=mimalloc,legacy-this-file,nightly" ];
 
   buildInputs = [ makeWrapper ];