git.delta.rocks / jrsonnet / refs/commits / 8d2d63334ec3

difftreelog

doc: rerun benchmarks (#157)

Yaroslav Bolyukin2024-04-27parent: #0e1ae58.patch.diff
in: master
* ci: update sjsonnet for benchmark

* doc: scratch benchmarks results

* feat: another Rust jsonnet implementation appears!

Ref: https://github.com/google/jsonnet/pull/1156

* chore: use jdk21-headless for sjsonnet

* chore: fix go-jsonnet benchmarking

* chore: increase default max-stack for rsjsonnet

* chore: update sjsonnet to v0.4.10

* chore: update skipped benchmarks list

* chore: rerun benchmarks

8 files changed

modifiedREADME.adocdiffbeforeafterboth
--- a/README.adoc
+++ b/README.adoc
@@ -69,6 +69,7 @@
 * https://github.com/google/jsonnet[C++];
 * https://github.com/google/go-jsonnet/[Go];
 * https://github.com/databricks/sjsonnet[Scala].
+* [NEW] https://github.com/eduardosm/rsjsonnet[Another Rust impl appears].
 
 This implementation shows performance better than all existing implementations.
 For more information see link:./docs/benchmarks.md[benchmarks]
modifieddocs/benchmarks.mddiffbeforeafterboth
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -9,10 +9,10 @@
 <details>
 <summary>Tested versions</summary>
 
-Go: Jsonnet commandline interpreter (Go implementation) v0.19.1
+Go: Jsonnet commandline interpreter (Go implementation) v0.20.0
 
 ```
-Jsonnet commandline interpreter (Go implementation) v0.19.1
+Jsonnet commandline interpreter (Go implementation) v0.20.0
 
 jsonnet {<option>} <filename>
 
@@ -68,10 +68,10 @@
   advised to use -- if the argument is unknown, e.g. jsonnet -- "$FILENAME".
 ```
 
-C++: Jsonnet commandline interpreter v0.19.1
+C++: Jsonnet commandline interpreter v0.20.0
 
 ```
-Jsonnet commandline interpreter v0.19.1
+Jsonnet commandline interpreter v0.20.0
 
 jsonnet {<option>} <filename>
 
@@ -121,45 +121,88 @@
 
 ```
 Missing argument: file <str>
-Expected Signature: Sjsonnet 0.4.3
+Expected Signature: Sjsonnet 0.4.10
 usage: sjsonnet [sjsonnet-options] script-file
-  -i --interactive         Run Mill in interactive mode, suitable for opening REPLs and taking user
-                           input
-  -J --jpath <str>         Specify an additional library search dir (right-most wins)
-  -o --output-file <str>   Write to the output file rather than stdout
-  -m --multi <str>         Write multiple files to the directory, list files on stdout
-  -c --create-output-dirs  Automatically creates all parent directories for files
-  -y --yaml-stream         Write output as a YAML stream of JSON documents
-  -S --string              Expect a string, manifest as plain text
-  -V --ext-str <str>       <var>[=<val>] Provide 'external' variable as string. 'If <val> is
-                           omitted, get from environment var <var>
-  --ext-str-file <str>     <var>=<file> Provide 'external' variable as string from the file
-  -V --ext-code <str>      <var>[=<code>] Provide 'external' variable as Jsonnet code. If <code> is
-                           omitted, get from environment var <var>
-  --ext-code-file <str>    <var>=<file> Provide 'external' variable as Jsonnet code from the file
-  -A --tla-str <str>       <var>[=<val>] Provide top-level arguments as string. 'If <val> is
-                           omitted, get from environment var <var>
-  --tla-str-file <str>     <var>=<file> Provide top-level arguments variable as string from the file
-  -V --tla-code <str>      <var>[=<val>] Provide top-level arguments as Jsonnet code. 'If <val> is
-                           omitted, get from environment var <var>
-  --tla-code-file <str>    <var>=<file> Provide top-level arguments variable as Jsonnet code from
-                           the file
-  -n --indent <int>        How much to indent your output JSON
-  -p --preserve-order      Preserves order of keys in the resulting JSON
-  --strict                 Enforce some additional syntax limitations
-  --yaml-out               Write output as a YAML document
-  file <str>               The jsonnet file you wish to evaluate
-  --yaml-debug             Generate source line comments in the output YAML doc to make it easier to
-                           figure out where values come from.
-  --no-static-errors       Turn static errors into warnings
-  --fatal-warnings         Fail if any warnings were emitted
+  -i --interactive                      Run Mill in interactive mode, suitable for opening REPLs and
+                                        taking user input
+  -J --jpath <str>                      Specify an additional library search dir (right-most wins)
+  -o --output-file <str>                Write to the output file rather than stdout
+  -m --multi <str>                      Write multiple files to the directory, list files on stdout
+  -c --create-output-dirs               Automatically creates all parent directories for files
+  -y --yaml-stream                      Write output as a YAML stream of JSON documents
+  -S --string                           Expect a string, manifest as plain text
+  -V --ext-str <str>                    <var>[=<val>] Provide 'external' variable as string. 'If
+                                        <val> is omitted, get from environment var <var>
+  --ext-str-file <str>                  <var>=<file> Provide 'external' variable as string from the
+                                        file
+  -V --ext-code <str>                   <var>[=<code>] Provide 'external' variable as Jsonnet code.
+                                        If <code> is omitted, get from environment var <var>
+  --ext-code-file <str>                 <var>=<file> Provide 'external' variable as Jsonnet code
+                                        from the file
+  -A --tla-str <str>                    <var>[=<val>] Provide top-level arguments as string. 'If
+                                        <val> is omitted, get from environment var <var>
+  --tla-str-file <str>                  <var>=<file> Provide top-level arguments variable as string
+                                        from the file
+  -V --tla-code <str>                   <var>[=<val>] Provide top-level arguments as Jsonnet code.
+                                        'If <val> is omitted, get from environment var <var>
+  --tla-code-file <str>                 <var>=<file> Provide top-level arguments variable as Jsonnet
+                                        code from the file
+  -n --indent <int>                     How much to indent your output JSON
+  -p --preserve-order                   Preserves order of keys in the resulting JSON
+  --strict                              Enforce some additional syntax limitations
+  --yaml-out                            Write output as a YAML document
+  file <str>                            The jsonnet file you wish to evaluate
+  --yaml-debug                          Generate source line comments in the output YAML doc to make
+                                        it easier to figure out where values come from.
+  --no-static-errors                    Turn static errors into warnings
+  --fatal-warnings                      Fail if any warnings were emitted
+  -e --exec                             Evaluate the given string as Jsonnet rather than treating it
+                                        as a file name
+  --no-duplicate-keys-in-comprehension  Raise an error if an object comprehension contains duplicate
+                                        keys
+  --strict-import-syntax                Raise an error if import expressions are used without proper
+                                        parentheses, e.g. import "foo".bar rather than (import
+                                        "foo").bar
+  --strict-inherited-assertions         Properly handle assertions defined in a Jsonnet dictionary
+                                        that is extended more than once
 
 
 ```
 
 </details>
 
+Rust (alternative):
 
+```
+Usage: rsjsonnet [OPTIONS] <filename>
+
+Arguments:
+  <filename>  
+
+Options:
+  -e, --exec                      Treat filename as code
+  -J, --jpath <dir>               Specify an additional library search dir (right-most wins)
+  -o, --output-file <file>        Write to the output file rather than stdout
+  -m, --multi <dir>               Write multiple files to the directory, list files on stdout
+  -y, --yaml-stream               Write output as a YAML stream of JSON documents
+  -S, --string                    Expect a string, manifest as plain text
+  -s, --max-stack <n>             Number of allowed stack frames
+  -t, --max-trace <n>             Max length of stack trace before cropping
+  -V, --ext-str <var=[val]>       
+      --ext-str-file <var=file>   
+      --ext-code <var[=code]>     
+      --ext-code-file <var=file>  
+  -A, --tla-str <var[=val]>       
+      --tla-str-file <var=file>   
+      --tla-code <var[=code]>     
+      --tla-code-file <var=file>  
+  -h, --help                      Print help
+
+```
+
+</details>
+
+
 ## Real world
 
 ### Graalvm CI
@@ -238,9 +281,10 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 122.0 ± 1.9 | 118.3 | 125.8 | 1.00 |
-| `Go` | 1402.0 ± 31.8 | 1359.7 | 1480.9 | 11.49 ± 0.32 |
-| `Scala` | 869.3 ± 9.6 | 852.0 | 889.4 | 7.12 ± 0.14 |
+| `Rust` | 93.6 ± 0.6 | 92.5 | 94.7 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 263.3 ± 1.8 | 259.0 | 268.4 | 2.81 ± 0.03 |
+| `Go` | 1204.6 ± 5.0 | 1196.0 | 1215.4 | 12.87 ± 0.10 |
+| `Scala` | 720.0 ± 2.9 | 713.4 | 725.3 | 7.70 ± 0.06 |
 
 ### Kube-prometheus manifests
 
@@ -291,10 +335,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 113.2 ± 2.9 | 109.7 | 120.1 | 1.00 |
-| `Go` | 2192.3 ± 25.1 | 2166.1 | 2262.2 | 19.36 ± 0.54 |
-| `Scala` | 1117.5 ± 21.9 | 1075.8 | 1152.1 | 9.87 ± 0.31 |
-| `C++` | 88573.2 ± 4833.2 | 84345.6 | 97409.2 | 782.15 ± 47.02 |
+| `Rust` | 129.4 ± 1.4 | 127.7 | 132.8 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 197.0 ± 1.9 | 193.6 | 200.7 | 1.52 ± 0.02 |
+| `Go` | 2127.7 ± 13.8 | 2105.4 | 2151.5 | 16.44 ± 0.21 |
+| `Scala` | 947.9 ± 11.1 | 926.1 | 967.9 | 7.33 ± 0.12 |
+| `C++` | 87633.8 ± 569.9 | 86464.2 | 89118.8 | 677.29 ± 8.72 |
 
 ## Benchmarks from C++ jsonnet (/perf_tests)
 
@@ -313,10 +358,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 7.4 ± 0.2 | 6.9 | 8.9 | 1.00 |
-| `Go` | 117.4 ± 5.1 | 112.1 | 139.2 | 15.87 ± 0.85 |
-| `Scala` | 373.1 ± 4.5 | 367.8 | 383.0 | 50.45 ± 1.70 |
-| `C++` | 85.9 ± 2.1 | 82.1 | 90.1 | 11.62 ± 0.46 |
+| `Rust` | 5.6 ± 0.1 | 5.4 | 6.2 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 45.3 ± 0.9 | 43.5 | 47.8 | 8.04 ± 0.25 |
+| `Go` | 98.2 ± 1.6 | 95.0 | 100.8 | 17.45 ± 0.50 |
+| `Scala` | 331.1 ± 5.8 | 322.3 | 343.7 | 58.79 ± 1.73 |
+| `C++` | 76.5 ± 0.7 | 75.4 | 78.0 | 13.59 ± 0.34 |
 
 ### Large string template
 
@@ -324,9 +370,10 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 8.2 ± 0.3 | 7.3 | 9.4 | 1.00 |
-| `Scala` | 423.8 ± 5.5 | 416.2 | 437.5 | 51.62 ± 2.00 |
-| `C++` | 15292.1 ± 204.0 | 15070.1 | 15840.7 | 1862.69 ± 72.39 |
+| `Rust` | 6.7 ± 0.1 | 6.4 | 7.1 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 7.1 ± 0.2 | 6.8 | 7.6 | 1.05 ± 0.03 |
+| `Scala` | 392.7 ± 2.7 | 388.2 | 399.8 | 58.29 ± 1.19 |
+| `C++` | 14376.2 ± 35.8 | 14317.5 | 14448.6 | 2133.86 ± 41.53 |
 
 ### Realistic 1
 
@@ -585,10 +632,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 14.6 ± 0.6 | 13.5 | 17.9 | 1.00 |
-| `Go` | 7663.9 ± 114.5 | 7518.9 | 7912.3 | 524.74 ± 24.02 |
-| `Scala` | 414.1 ± 5.4 | 406.8 | 429.4 | 28.35 ± 1.28 |
-| `C++` | 26376.7 ± 354.2 | 25755.3 | 26908.5 | 1806.00 ± 81.81 |
+| `Rust` | 12.6 ± 0.1 | 12.3 | 12.9 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 49.7 ± 1.0 | 47.3 | 53.0 | 3.96 ± 0.09 |
+| `Go` | 6858.1 ± 34.9 | 6815.9 | 6951.8 | 546.32 ± 6.87 |
+| `Scala` | 382.3 ± 3.9 | 371.7 | 387.1 | 30.46 ± 0.47 |
+| `C++` | 24472.7 ± 155.7 | 24230.3 | 24799.3 | 1949.51 ± 25.63 |
 
 ### Realistic 2
 
@@ -707,10 +755,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 295.6 ± 8.9 | 285.1 | 319.9 | 1.00 |
-| `Go` | 7540.7 ± 107.5 | 7357.3 | 7792.5 | 25.51 ± 0.85 |
-| `Scala` | 781.0 ± 27.4 | 745.3 | 854.7 | 2.64 ± 0.12 |
-| `C++` | 30144.1 ± 574.5 | 28895.0 | 30967.3 | 101.99 ± 3.62 |
+| `Rust` | 259.9 ± 4.4 | 250.9 | 266.7 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 550.2 ± 16.6 | 531.9 | 600.7 | 2.12 ± 0.07 |
+| `Go` | 6732.4 ± 51.4 | 6660.2 | 6858.6 | 25.90 ± 0.48 |
+| `Scala` | 675.4 ± 10.9 | 667.3 | 720.0 | 2.60 ± 0.06 |
+| `C++` | 26414.5 ± 259.9 | 25948.9 | 26934.4 | 101.63 ± 1.99 |
 
 ## Benchmarks from C++ jsonnet (/benchmarks)
 
@@ -748,10 +797,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 2.2 ± 0.1 | 1.9 | 3.6 | 1.00 |
-| `Go` | 6.7 ± 0.6 | 6.0 | 17.0 | 3.10 ± 0.32 |
-| `Scala` | 306.5 ± 2.1 | 302.8 | 311.8 | 141.78 ± 7.81 |
-| `C++` | 34.7 ± 2.2 | 32.6 | 52.1 | 16.04 ± 1.35 |
+| `Rust` | 1.8 ± 0.1 | 1.6 | 2.6 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 3.1 ± 0.1 | 2.8 | 3.4 | 1.69 ± 0.12 |
+| `Go` | 5.3 ± 0.2 | 4.9 | 5.9 | 2.90 ± 0.20 |
+| `Scala` | 287.4 ± 1.2 | 285.4 | 288.9 | 156.55 ± 9.92 |
+| `C++` | 32.1 ± 0.3 | 31.6 | 33.0 | 17.49 ± 1.12 |
 
 ### Inheritance recursion
 
@@ -788,10 +838,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 452.8 ± 20.3 | 425.2 | 494.6 | 1.00 |
-| `Go` | 1076.4 ± 16.2 | 1049.9 | 1111.1 | 2.38 ± 0.11 |
-| `Scala` | 475.5 ± 8.7 | 462.2 | 494.4 | 1.05 ± 0.05 |
-| `C++` | 3029.7 ± 121.1 | 2787.2 | 3214.5 | 6.69 ± 0.40 |
+| `Rust` | 376.1 ± 5.2 | 364.3 | 383.7 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 496.7 ± 13.0 | 470.6 | 506.8 | 1.32 ± 0.04 |
+| `Go` | 873.9 ± 7.1 | 859.3 | 888.0 | 2.32 ± 0.04 |
+| `Scala` | 420.8 ± 6.3 | 413.9 | 437.1 | 1.12 ± 0.02 |
+| `C++` | 2647.5 ± 14.2 | 2630.9 | 2675.6 | 7.04 ± 0.10 |
 
 ### Simple recursive call
 
@@ -828,10 +879,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 109.2 ± 2.4 | 104.8 | 114.3 | 1.00 |
-| `Go` | 429.0 ± 8.2 | 419.4 | 454.3 | 3.93 ± 0.12 |
-| `Scala` | 338.7 ± 3.2 | 334.6 | 345.6 | 3.10 ± 0.07 |
-| `C++` | 210.0 ± 8.0 | 195.5 | 227.5 | 1.92 ± 0.08 |
+| `Rust` | 81.3 ± 0.3 | 80.9 | 82.4 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 202.3 ± 12.0 | 183.1 | 237.6 | 2.49 ± 0.15 |
+| `Go` | 349.4 ± 1.7 | 347.2 | 354.2 | 4.30 ± 0.03 |
+| `Scala` | 322.5 ± 3.0 | 318.7 | 330.7 | 3.97 ± 0.04 |
+| `C++` | 193.5 ± 0.9 | 191.4 | 194.8 | 2.38 ± 0.01 |
 
 ### Foldl string concat
 
@@ -846,14 +898,15 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 12.5 ± 0.6 | 11.4 | 17.3 | 1.00 |
-| `Go` | 64.8 ± 3.1 | 60.8 | 78.8 | 5.16 ± 0.36 |
-| `Scala` | 630.6 ± 15.6 | 606.3 | 659.8 | 50.25 ± 2.83 |
-| `C++` | 1073.3 ± 18.2 | 1041.0 | 1107.6 | 85.53 ± 4.56 |
+| `Rust` | 8.9 ± 0.1 | 8.6 | 9.3 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 303.1 ± 3.8 | 297.6 | 308.1 | 34.06 ± 0.68 |
+| `Go` | 50.9 ± 0.8 | 48.3 | 52.2 | 5.72 ± 0.13 |
+| `Scala` | 550.6 ± 5.5 | 542.8 | 563.7 | 61.86 ± 1.13 |
+| `C++` | 902.6 ± 4.8 | 891.4 | 912.1 | 101.41 ± 1.65 |
 
 ### Array sorts
 
-> Note: No results for Scala, std.reverse is not implemented
+> Note: No results for Scala, sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204
 
 <details>
 <summary>Source</summary>
@@ -876,9 +929,10 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 4.2 ± 0.1 | 3.9 | 4.9 | 1.00 |
-| `Go` | 14.8 ± 0.5 | 13.7 | 17.8 | 3.51 ± 0.17 |
-| `C++` | 3964.0 ± 187.1 | 3700.2 | 4225.1 | 939.25 ± 55.45 |
+| `Rust` | 3.2 ± 0.2 | 2.9 | 4.1 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 14.4 ± 0.6 | 13.1 | 16.2 | 4.46 ± 0.28 |
+| `Go` | 12.1 ± 0.2 | 11.4 | 12.8 | 3.74 ± 0.20 |
+| `C++` | 3637.1 ± 24.9 | 3557.4 | 3664.7 | 1128.64 ± 56.81 |
 
 ### Lazy array
 
@@ -898,10 +952,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 58.8 ± 2.0 | 55.6 | 64.2 | 1.00 |
-| `Go` | 661.6 ± 26.6 | 629.6 | 730.1 | 11.25 ± 0.59 |
-| `Scala` | 348.8 ± 6.6 | 339.7 | 365.3 | 5.93 ± 0.23 |
-| `C++` | 206.4 ± 3.3 | 202.5 | 215.2 | 3.51 ± 0.13 |
+| `Rust` | 54.1 ± 1.1 | 52.3 | 56.3 | 1.30 ± 0.03 |
+| `Rust (alternative, rsjsonnet)` | 41.7 ± 0.7 | 39.6 | 43.8 | 1.00 |
+| `Go` | 596.5 ± 5.2 | 585.1 | 606.9 | 14.30 ± 0.29 |
+| `Scala` | 306.6 ± 1.6 | 303.5 | 310.4 | 7.35 ± 0.14 |
+| `C++` | 184.5 ± 1.8 | 182.0 | 187.9 | 4.42 ± 0.09 |
 
 ### Inheritance function recursion
 
@@ -926,10 +981,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 1.9 ± 0.1 | 1.6 | 3.3 | 1.00 |
-| `Go` | 3.0 ± 0.2 | 2.7 | 5.1 | 1.58 ± 0.15 |
-| `Scala` | 310.0 ± 9.5 | 299.9 | 341.1 | 161.51 ± 9.88 |
-| `C++` | 1505.6 ± 74.0 | 1363.5 | 1640.8 | 784.26 ± 56.71 |
+| `Rust` | 1.6 ± 0.1 | 1.5 | 2.5 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.7 | 3.7 | 1.78 ± 0.13 |
+| `Go` | 2.4 ± 0.1 | 2.1 | 2.9 | 1.48 ± 0.12 |
+| `Scala` | 293.2 ± 1.5 | 289.9 | 296.1 | 178.20 ± 12.35 |
+| `C++` | 999.1 ± 9.2 | 974.4 | 1010.9 | 607.23 ± 42.34 |
 
 ### String strips
 
@@ -951,10 +1007,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 9.4 ± 0.3 | 8.7 | 10.4 | 1.00 |
-| `Go` | 15.0 ± 0.5 | 13.9 | 17.6 | 1.60 ± 0.08 |
-| `Scala` | 340.6 ± 5.4 | 333.3 | 359.9 | 36.39 ± 1.36 |
-| `C++` | 37161.7 ± 525.9 | 36471.3 | 38275.4 | 3970.49 ± 145.97 |
+| `Rust` | 8.6 ± 0.2 | 8.4 | 9.5 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 18.4 ± 0.4 | 17.6 | 20.8 | 2.13 ± 0.06 |
+| `Go` | 11.2 ± 0.2 | 10.4 | 11.9 | 1.29 ± 0.04 |
+| `Scala` | 294.9 ± 2.0 | 292.2 | 301.9 | 34.10 ± 0.65 |
+| `C++` | 37345.3 ± 208.2 | 36567.2 | 37689.0 | 4318.10 ± 80.13 |
 
 ### Big object
 
@@ -987,10 +1044,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 2.5 ± 0.1 | 2.3 | 3.3 | 1.00 |
-| `Go` | 5.2 ± 0.3 | 4.6 | 7.0 | 2.07 ± 0.15 |
-| `Scala` | 302.6 ± 6.4 | 290.3 | 312.9 | 120.34 ± 6.71 |
-| `C++` | 29.4 ± 1.4 | 27.7 | 38.5 | 11.71 ± 0.82 |
+| `Rust` | 2.2 ± 0.1 | 1.9 | 3.0 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 3.2 ± 0.1 | 2.9 | 3.5 | 1.46 ± 0.10 |
+| `Go` | 4.1 ± 0.1 | 3.7 | 4.6 | 1.85 ± 0.13 |
+| `Scala` | 292.3 ± 2.9 | 289.3 | 304.0 | 132.11 ± 8.64 |
+| `C++` | 28.3 ± 0.3 | 27.7 | 29.1 | 12.80 ± 0.84 |
 
 ## Benchmarks from Go jsonnet (builtins)
 
@@ -1010,10 +1068,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 3.4 ± 0.1 | 3.1 | 4.9 | 1.00 |
-| `Go` | 21.6 ± 0.8 | 19.7 | 27.1 | 6.32 ± 0.34 |
-| `Scala` | 355.0 ± 6.0 | 340.7 | 367.4 | 103.77 ± 4.62 |
-| `C++` | 16359.4 ± 663.5 | 15526.0 | 17805.8 | 4782.25 ± 276.36 |
+| `Rust` | 2.7 ± 0.1 | 2.4 | 3.4 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 2195.9 ± 48.5 | 2098.1 | 2229.6 | 804.07 ± 39.58 |
+| `Go` | 17.0 ± 0.3 | 16.1 | 17.7 | 6.22 ± 0.29 |
+| `Scala` | 313.8 ± 1.2 | 311.3 | 316.1 | 114.91 ± 5.07 |
+| `C++` | 14621.6 ± 110.0 | 14473.9 | 14774.9 | 5354.05 ± 238.94 |
 
 ### std.base64Decode
 
@@ -1031,10 +1090,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 3.4 ± 0.2 | 3.1 | 5.3 | 1.00 |
-| `Go` | 21.0 ± 1.5 | 19.6 | 32.2 | 6.24 ± 0.52 |
-| `Scala` | 358.3 ± 7.1 | 349.0 | 379.9 | 106.23 ± 5.27 |
-| `C++` | 10921.3 ± 240.4 | 10653.9 | 11346.7 | 3237.97 ± 163.58 |
+| `Rust` | 2.6 ± 0.1 | 2.3 | 3.1 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 7889.2 ± 74.7 | 7711.6 | 8051.1 | 3045.22 ± 138.95 |
+| `Go` | 15.2 ± 0.3 | 14.6 | 16.5 | 5.85 ± 0.28 |
+| `Scala` | 313.0 ± 2.5 | 309.5 | 321.2 | 120.80 ± 5.47 |
+| `C++` | 9930.9 ± 26.5 | 9895.0 | 9997.9 | 3833.33 ± 171.41 |
 
 ### std.base64DecodeBytes
 
@@ -1052,10 +1112,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 45.7 ± 1.4 | 43.6 | 52.1 | 1.00 |
-| `Go` | 303.6 ± 8.4 | 292.1 | 324.7 | 6.64 ± 0.27 |
-| `Scala` | 406.3 ± 8.3 | 397.0 | 424.8 | 8.88 ± 0.32 |
-| `C++` | 10058.3 ± 337.9 | 9738.8 | 10641.5 | 219.86 ± 9.92 |
+| `Rust` | 34.8 ± 0.5 | 33.4 | 36.0 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 8116.2 ± 37.2 | 8049.1 | 8176.2 | 233.52 ± 3.24 |
+| `Go` | 271.1 ± 2.3 | 267.1 | 275.7 | 7.80 ± 0.12 |
+| `Scala` | 357.2 ± 2.3 | 353.5 | 363.4 | 10.28 ± 0.15 |
+| `C++` | 9653.7 ± 22.5 | 9621.4 | 9694.3 | 277.75 ± 3.69 |
 
 ### std.base64 (byte array)
 
@@ -1073,10 +1134,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 37.0 ± 1.8 | 33.7 | 42.5 | 1.00 |
-| `Go` | 195.8 ± 11.5 | 180.9 | 219.5 | 5.28 ± 0.41 |
-| `Scala` | 419.2 ± 13.0 | 396.5 | 454.6 | 11.32 ± 0.66 |
-| `C++` | 15605.3 ± 310.2 | 15173.1 | 16065.6 | 421.22 ± 22.47 |
+| `Rust` | 30.3 ± 0.4 | 29.1 | 31.6 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 1460.8 ± 47.6 | 1375.2 | 1568.3 | 48.26 ± 1.72 |
+| `Go` | 156.5 ± 1.9 | 152.5 | 159.4 | 5.17 ± 0.10 |
+| `Scala` | 355.9 ± 2.9 | 351.4 | 365.6 | 11.76 ± 0.20 |
+| `C++` | 13362.4 ± 88.3 | 13028.4 | 13473.8 | 441.41 ± 7.01 |
 
 ### std.foldl
 
@@ -1093,10 +1155,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 7.7 ± 0.7 | 6.8 | 18.5 | 1.00 |
-| `Go` | 32.6 ± 1.1 | 30.6 | 37.8 | 4.26 ± 0.41 |
-| `Scala` | 461.7 ± 8.5 | 445.0 | 484.5 | 60.35 ± 5.54 |
-| `C++` | 320.3 ± 6.0 | 308.6 | 335.0 | 41.87 ± 3.85 |
+| `Rust` | 5.3 ± 0.2 | 5.0 | 6.2 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 81.5 ± 1.6 | 78.9 | 83.8 | 15.38 ± 0.55 |
+| `Go` | 26.6 ± 0.5 | 25.0 | 27.5 | 5.02 ± 0.18 |
+| `Scala` | 405.4 ± 3.0 | 400.9 | 413.6 | 76.52 ± 2.34 |
+| `C++` | 265.3 ± 2.7 | 260.6 | 271.5 | 50.08 ± 1.57 |
 
 ### std.manifestJsonEx
 
@@ -1156,14 +1219,15 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 1.9 ± 0.1 | 1.7 | 3.0 | 1.00 |
-| `Go` | 4.2 ± 0.3 | 3.6 | 8.0 | 2.14 ± 0.18 |
-| `Scala` | 338.9 ± 3.4 | 334.9 | 348.8 | 174.49 ± 8.63 |
-| `C++` | 106.6 ± 3.2 | 102.8 | 113.2 | 54.90 ± 3.12 |
+| `Rust` | 1.7 ± 0.1 | 1.4 | 2.7 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.7 | 3.3 | 1.73 ± 0.13 |
+| `Go` | 3.2 ± 0.1 | 2.7 | 3.8 | 1.92 ± 0.15 |
+| `Scala` | 294.7 ± 1.7 | 291.6 | 299.0 | 175.84 ± 12.30 |
+| `C++` | 101.4 ± 0.8 | 99.3 | 102.9 | 60.48 ± 4.24 |
 
 ### std.manifestTomlEx
 
-> Note: No results for Scala, std.manifestTomlEx is not implemented
+> Note: No results for Scala, std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111
 
 <details>
 <summary>Source</summary>
@@ -1222,9 +1286,10 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 1.9 ± 0.1 | 1.8 | 2.7 | 1.00 |
-| `Go` | 4.2 ± 0.3 | 3.7 | 8.0 | 2.17 ± 0.19 |
-| `C++` | 1131.6 ± 28.1 | 1089.2 | 1198.7 | 584.01 ± 30.62 |
+| `Rust` | 1.7 ± 0.1 | 1.4 | 2.7 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 7.8 ± 0.2 | 7.3 | 8.5 | 4.65 ± 0.36 |
+| `Go` | 3.2 ± 0.1 | 2.9 | 3.7 | 1.90 ± 0.16 |
+| `C++` | 1046.1 ± 4.9 | 1039.7 | 1053.9 | 622.36 ± 44.25 |
 
 ### std.parseInt
 
@@ -1242,14 +1307,13 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 2.0 ± 0.1 | 1.7 | 3.2 | 1.00 |
-| `Go` | 3.7 ± 0.4 | 3.1 | 9.0 | 1.85 ± 0.21 |
-| `Scala` | 332.3 ± 7.2 | 326.1 | 359.7 | 168.32 ± 10.37 |
-| `C++` | 110.2 ± 1.9 | 106.9 | 115.1 | 55.82 ± 3.36 |
+| `Rust` | 1.7 ± 0.1 | 1.4 | 2.6 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.6 | 3.4 | 1.71 ± 0.16 |
+| `Go` | 2.8 ± 0.1 | 2.4 | 3.3 | 1.64 ± 0.16 |
+| `Scala` | 292.7 ± 1.7 | 289.2 | 295.8 | 170.68 ± 14.76 |
+| `C++` | 101.0 ± 0.7 | 99.0 | 102.0 | 58.88 ± 5.10 |
 
 ### std.reverse
-
-> Note: No results for Scala, std.reverse is not implemented
 
 <details>
 <summary>Source</summary>
@@ -1265,9 +1329,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 59.7 ± 1.9 | 56.3 | 64.7 | 1.00 |
-| `Go` | 363.6 ± 9.0 | 352.8 | 385.1 | 6.09 ± 0.25 |
-| `C++` | 865.9 ± 16.1 | 839.9 | 902.0 | 14.51 ± 0.54 |
+| `Rust` | 45.3 ± 0.5 | 44.0 | 46.5 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 784.1 ± 20.4 | 741.5 | 802.0 | 17.31 ± 0.49 |
+| `Go` | 327.4 ± 2.1 | 323.8 | 331.4 | 7.23 ± 0.10 |
+| `Scala` | 367.0 ± 2.4 | 361.1 | 370.2 | 8.10 ± 0.11 |
+| `C++` | 659.3 ± 5.4 | 644.6 | 666.0 | 14.55 ± 0.21 |
 
 ### std.substr
 
@@ -1285,10 +1351,11 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 2.6 ± 0.1 | 2.3 | 3.5 | 1.00 |
-| `Go` | 8.9 ± 0.6 | 8.0 | 15.3 | 3.46 ± 0.29 |
-| `Scala` | 346.6 ± 9.4 | 337.8 | 375.9 | 134.32 ± 7.51 |
-| `C++` | 31.4 ± 1.0 | 29.7 | 36.4 | 12.17 ± 0.72 |
+| `Rust` | 2.2 ± 0.2 | 1.9 | 3.2 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 3.2 ± 0.1 | 3.1 | 3.6 | 1.49 ± 0.11 |
+| `Go` | 6.8 ± 0.2 | 6.3 | 7.4 | 3.12 ± 0.24 |
+| `Scala` | 303.6 ± 3.6 | 300.5 | 318.2 | 139.43 ± 9.76 |
+| `C++` | 29.1 ± 0.4 | 28.3 | 30.2 | 13.36 ± 0.94 |
 
 ### Comparsion for array
 
@@ -1306,9 +1373,10 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 26.0 ± 0.6 | 25.4 | 30.2 | 1.00 |
-| `Go` | 121.5 ± 6.5 | 116.2 | 141.0 | 4.68 ± 0.28 |
-| `C++` | 147098.5 ± 3334.1 | 135178.4 | 150470.0 | 5663.91 ± 187.03 |
+| `Rust` | 25.7 ± 0.2 | 25.0 | 26.0 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 228.1 ± 7.9 | 214.0 | 245.1 | 8.89 ± 0.31 |
+| `Go` | 150.4 ± 10.4 | 135.9 | 169.4 | 5.86 ± 0.41 |
+| `C++` | 125755.8 ± 989.6 | 123095.8 | 127475.4 | 4901.85 ± 49.12 |
 
 ### Comparsion for primitives
 
@@ -1325,6 +1393,7 @@
 
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `Rust` | 284.7 ± 10.4 | 267.1 | 315.4 | 1.00 |
-| `Go` | 2009.5 ± 46.0 | 1931.3 | 2108.9 | 7.06 ± 0.31 |
-| `Scala` | 550.0 ± 14.8 | 526.3 | 575.4 | 1.93 ± 0.09 |
+| `Rust` | 385.9 ± 4.2 | 372.4 | 391.5 | 1.00 |
+| `Rust (alternative, rsjsonnet)` | 1287.4 ± 27.0 | 1244.2 | 1335.1 | 3.34 ± 0.08 |
+| `Go` | 1817.1 ± 18.6 | 1779.7 | 1842.9 | 4.71 ± 0.07 |
+| `Scala` | 453.1 ± 3.9 | 448.1 | 462.2 | 1.17 ± 0.02 |
modifiedflake.nixdiffbeforeafterboth
--- a/flake.nix
+++ b/flake.nix
@@ -39,6 +39,7 @@
           jsonnet = pkgs.callPackage ./nix/jsonnet.nix {};
           # I didn't managed to build it, and nixpkgs version is marked as broken
           # haskell-jsonnet = pkgs.callPackage ./nix/haskell-jsonnet.nix { };
+          rsjsonnet = pkgs.callPackage ./nix/rsjsonnet.nix {};
 
           jrsonnet = pkgs.callPackage ./nix/jrsonnet.nix {
             inherit craneLib;
@@ -60,55 +61,55 @@
           };
 
           benchmarks = pkgs.callPackage ./nix/benchmarks.nix {
-            inherit go-jsonnet sjsonnet jsonnet;
+            inherit go-jsonnet sjsonnet jsonnet rsjsonnet;
             jrsonnetVariants = [
               {
-                drv = jrsonnet.overrideAttrs {forBenchmarks = true;};
+                drv = jrsonnet.override {forBenchmarks = true;};
                 name = "";
               }
             ];
           };
           benchmarks-quick = pkgs.callPackage ./nix/benchmarks.nix {
-            inherit go-jsonnet sjsonnet jsonnet;
+            inherit go-jsonnet sjsonnet jsonnet rsjsonnet;
             quick = true;
             jrsonnetVariants = [
               {
-                drv = jrsonnet.overrideAttrs {forBenchmarks = true;};
+                drv = jrsonnet.override {forBenchmarks = true;};
                 name = "";
               }
             ];
           };
           benchmarks-against-release = pkgs.callPackage ./nix/benchmarks.nix {
-            inherit go-jsonnet sjsonnet jsonnet;
+            inherit go-jsonnet sjsonnet jsonnet rsjsonnet;
             jrsonnetVariants = [
               {
-                drv = jrsonnet.overrideAttrs {forBenchmarks = true;};
+                drv = jrsonnet.override {forBenchmarks = true;};
                 name = "current";
               }
               {
-                drv = jrsonnet-nightly.overrideAttrs {forBenchmarks = true;};
+                drv = jrsonnet-nightly.override {forBenchmarks = true;};
                 name = "current-nightly";
               }
               {
-                drv = jrsonnet-release.overrideAttrs {forBenchmarks = true;};
+                drv = jrsonnet-release.override {forBenchmarks = true;};
                 name = "release";
               }
             ];
           };
           benchmarks-quick-against-release = pkgs.callPackage ./nix/benchmarks.nix {
-            inherit go-jsonnet sjsonnet jsonnet;
+            inherit go-jsonnet sjsonnet jsonnet rsjsonnet;
             quick = true;
             jrsonnetVariants = [
               {
-                drv = jrsonnet;
+                drv = jrsonnet.override {forBenchmarks = true;};
                 name = "current";
               }
               {
-                drv = jrsonnet-nightly;
+                drv = jrsonnet-nightly.override {forBenchmarks = true;};
                 name = "current-nightly";
               }
               {
-                drv = jrsonnet-release;
+                drv = jrsonnet-release.override {forBenchmarks = true;};
                 name = "release";
               }
             ];
modifiednix/benchmarks.nixdiffbeforeafterboth
before · nix/benchmarks.nix
1{ lib2, runCommand3, jsonnet-bundler4, cacert5, stdenv6, fetchFromGitHub7, go-jsonnet8, sjsonnet9, jsonnet10, hyperfine11, quick ? false12, jrsonnetVariants13}:1415with lib;1617let18  jsonnetBench = fetchFromGitHub {19    rev = "v0.19.1";20    owner = "google";21    repo = "jsonnet";22    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";23  };24  goJsonnetBench = (fetchFromGitHub {25    owner = "google";26    repo = "go-jsonnet";27    rev = "v0.19.1";28    hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29  }) + "/builtin-benchmarks";30  graalvmBench = fetchFromGitHub {31    owner = "oracle";32    repo = "graal";33    rev = "bc305df3fe587960f7635f0185571500e5988475";34    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";35  };36  kubePrometheusBench =37    let38      src = fetchFromGitHub {39        owner = "prometheus-operator";40        repo = "kube-prometheus";41        rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42        hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43      };44    in45    runCommand "kube-prometheus-vendor"46      {47        outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48        outputHashMode = "recursive";49        buildInputs = [ cacert ];50      }51      ''52        mkdir -p $out53        cp -r ${src}/* $out/54        cd $out55        mkdir vendor56        ${jsonnet-bundler}/bin/jb install57      '';5859  # Removes outsiders from the output60  # Useful when comparing performance of different jrsonnet releases61  skipSlow = if quick then "slow benchmark, but only quick requested" else "";62in63stdenv.mkDerivation {64  name = "benchmarks";65  __impure = true;66  unpackPhase = "true";6768  buildInputs = [69    go-jsonnet70    sjsonnet71    jsonnet7273    hyperfine74  ];7576  installPhase =77    let78      mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: ''79        set -oux8081        echo >> $out82        echo "### ${name}" >> $out83        echo >> $out84        ${optionalString (skipGo != "") ''85          echo "> Note: No results for Go, ${skipGo}" >> $out86          echo >> $out87        ''}88        ${optionalString (skipScala != "") ''89          echo "> Note: No results for Scala, ${skipScala}" >> $out90          echo >> $out91        ''}92        ${optionalString (skipCpp != "") ''93          echo "> Note: No results for C++, ${skipCpp}" >> $out94          echo >> $out95        ''}96        ${optionalString (!quick && !omitSource) ''97          echo "<details>" >> $out98          echo "<summary>Source</summary>" >> $out99          echo >> $out100          echo "\`\`\`jsonnet" >> $out101          ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}102          cat ${path} >> $out103          echo >> $out104          echo "\`\`\`" >> $out105          echo "</details>" >> $out106          echo >> $out107        ''}108        path=${path}109        ${optionalString pathIsGenerator ''110          go-jsonnet $path > generated.jsonnet111          path=generated.jsonnet112        ''}113        hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \114          ${concatStringsSep " " (forEach jrsonnetVariants (variant:115            "\"${variant.drv}/bin/jrsonnet $path ${optionalString (vendor != "") "-J${vendor}"}\" -n \"Rust${if variant.name != "" then " (${variant.name})" else ""}\""116          ))} \117          ${optionalString (skipGo == "") "\"go-jsonnet $path ${optionalString (vendor != "") "-J ${vendor}"}\" -n \"Go\""} \118          ${optionalString (skipScala == "") "\"sjsonnet $path ${optionalString (vendor != "") "-J ${vendor}"}\" -n \"Scala\""} \119          ${optionalString (skipCpp == "") "\"jsonnet $path ${optionalString (vendor != "") "-J ${vendor}"}\" -n \"C++\""}120        cat result.md >> $out121      '';122    in123    ''124      touch $out125      ${optionalString (!quick) ''126        cat ${./benchmarks.md} >> $out127        echo >> $out128129        echo "<details>" >> $out130        echo "<summary>Tested versions</summary>" >> $out131        echo >> $out132        echo Go: $(go-jsonnet --version) >> $out133        echo >> $out134        echo "\`\`\`" >> $out135        go-jsonnet --help >> $out136        echo "\`\`\`" >> $out137        echo >> $out138        echo C++: $(jsonnet --version) >> $out139        echo >> $out140        echo "\`\`\`" >> $out141        jsonnet --help >> $out142        echo "\`\`\`" >> $out143        echo >> $out144        echo Scala: >> $out145        echo >> $out146        echo "\`\`\`" >> $out147        sjsonnet 2>> $out || true148        echo "\`\`\`" >> $out149        echo >> $out150        echo "</details>" >> $out151        echo >> $out152153        echo >> $out154      ''}155      echo "## Real world" >> $out156      ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour"; skipGo = skipSlow; skipScala = skipSlow;}}157      ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}158159      echo >> $out160      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out161      ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet"; skipScala = skipSlow;}}162      ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow; skipScala = skipSlow;}}163      ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}164      ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}}165166      echo >> $out167      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out168      ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet"; skipScala = skipSlow;}}169      ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow;}}170      ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet"; skipScala = skipSlow; skipGo = skipSlow;}}171      ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}172      ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}}173      ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet"; skipGo = skipSlow; skipScala = skipSlow;}}174      ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}175      ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}176      ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true; skipScala = skipSlow;}}177178      echo >> $out179      echo "## Benchmarks from Go jsonnet (builtins)" >> $out180      ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}181      ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}}182      ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}183      ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}}184      ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet"; skipScala = skipSlow;}}185      ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}186      ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented"; skipCpp=skipSlow;}}187      ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}}188      ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow; skipGo = skipSlow;}}189      ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet"; skipScala = skipSlow;}}190      ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}}191      ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM"; skipGo = skipSlow; skipScala = skipSlow;}}192    '';193}
after · nix/benchmarks.nix
1{2  lib,3  runCommand,4  jsonnet-bundler,5  cacert,6  stdenv,7  fetchFromGitHub,8  go-jsonnet,9  sjsonnet,10  jsonnet,11  rsjsonnet,12  hyperfine,13  quick ? false,14  jrsonnetVariants,15}:16with lib; let17  jsonnetBench = fetchFromGitHub {18    rev = "v0.19.1";19    owner = "google";20    repo = "jsonnet";21    hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU=";22  };23  goJsonnetBench =24    (fetchFromGitHub {25      owner = "google";26      repo = "go-jsonnet";27      rev = "v0.19.1";28      hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI=";29    })30    + "/builtin-benchmarks";31  graalvmBench = fetchFromGitHub {32    owner = "oracle";33    repo = "graal";34    rev = "bc305df3fe587960f7635f0185571500e5988475";35    hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc=";36  };37  kubePrometheusBench = let38    src = fetchFromGitHub {39      owner = "prometheus-operator";40      repo = "kube-prometheus";41      rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd";42      hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM=";43    };44  in45    runCommand "kube-prometheus-vendor"46    {47      outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw=";48      outputHashMode = "recursive";49      buildInputs = [cacert];50    }51    ''52      mkdir -p $out53      cp -r ${src}/* $out/54      cd $out55      mkdir vendor56      ${jsonnet-bundler}/bin/jb install57    '';5859  # Removes outsiders from the output60  # Useful when comparing performance of different jrsonnet releases61  skipSlow =62    if quick63    then "slow benchmark, but only quick requested"64    else "";65in66  stdenv.mkDerivation {67    name = "benchmarks";68    __impure = true;69    unpackPhase = "true";7071    buildInputs = [72      sjsonnet73      jsonnet74      rsjsonnet75      go-jsonnet7677      hyperfine78    ];7980    installPhase = let81      mkBench = {82        name,83        path,84        omitSource ? false,85        pathIsGenerator ? false,86        skipRustAlternative ? "",87        skipScala ? "",88        skipCpp ? "",89        skipGo ? "",90        vendor ? "",91      }: ''92        echo >> $out93        echo "### ${name}" >> $out94        echo >> $out95        ${optionalString (skipRustAlternative != "") ''96          echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out97          echo >> $out98        ''}99        ${optionalString (skipGo != "") ''100          echo "> Note: No results for Go, ${skipGo}" >> $out101          echo >> $out102        ''}103        ${optionalString (skipScala != "") ''104          echo "> Note: No results for Scala, ${skipScala}" >> $out105          echo >> $out106        ''}107        ${optionalString (skipCpp != "") ''108          echo "> Note: No results for C++, ${skipCpp}" >> $out109          echo >> $out110        ''}111        ${optionalString (!quick && !omitSource) ''112          echo "<details>" >> $out113          echo "<summary>Source</summary>" >> $out114          echo >> $out115          echo "\`\`\`jsonnet" >> $out116          ${optionalString pathIsGenerator "echo \"// Generator source\" >> $out"}117          cat ${path} >> $out118          echo >> $out119          echo "\`\`\`" >> $out120          echo "</details>" >> $out121          echo >> $out122        ''}123        path=${path}124        ${optionalString pathIsGenerator ''125          go-jsonnet $path > generated.jsonnet126          path=generated.jsonnet127        ''}128        hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \129          ${concatStringsSep " " (forEach jrsonnetVariants (130          variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${131            if variant.name != ""132            then " (${variant.name})"133            else ""134          }\""135        ))} \136          ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \137          ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \138          ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \139          ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""}140        cat result.md >> $out141      '';142    in ''143      set -oux144145      temp=$(mktemp -d)146      cd $temp147148      touch $out149      ${optionalString (!quick) ''150        cat ${./benchmarks.md} >> $out151        echo >> $out152153        echo "<details>" >> $out154        echo "<summary>Tested versions</summary>" >> $out155        echo >> $out156        echo Go: $(go-jsonnet --version) >> $out157        echo >> $out158        echo "\`\`\`" >> $out159        go-jsonnet --help >> $out160        echo "\`\`\`" >> $out161        echo >> $out162        echo C++: $(jsonnet --version) >> $out163        echo >> $out164        echo "\`\`\`" >> $out165        jsonnet --help >> $out166        echo "\`\`\`" >> $out167        echo >> $out168        echo Scala: >> $out169        echo >> $out170        echo "\`\`\`" >> $out171        sjsonnet 2>> $out || true172        echo "\`\`\`" >> $out173        echo >> $out174        echo "</details>" >> $out175        echo >> $out176        echo "Rust (alternative):" >> $out177        echo >> $out178        echo "\`\`\`" >> $out179        rsjsonnet --help 2>> $out || true180        echo "\`\`\`" >> $out181        echo >> $out182        echo "</details>" >> $out183        echo >> $out184185        echo >> $out186      ''}187      echo "## Real world" >> $out188      ${mkBench {189        name = "Graalvm CI";190        path = "${graalvmBench}/ci.jsonnet";191        skipCpp = "takes longer than a hour";192        skipGo = skipSlow;193        skipScala = skipSlow;194      }}195      ${mkBench {196        name = "Kube-prometheus manifests";197        vendor = "${kubePrometheusBench}/vendor";198        path = "${kubePrometheusBench}/example.jsonnet";199        skipCpp = skipSlow;200        skipGo = skipSlow;201        skipScala = skipSlow;202      }}203204      echo >> $out205      echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out206      ${mkBench {207        name = "Large string join";208        path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet";209        skipScala = skipSlow;210      }}211      ${mkBench {212        name = "Large string template";213        omitSource = true;214        path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet";215        skipGo = "fails with os stack size exhausion";216        skipCpp = skipSlow;217        skipScala = skipSlow;218      }}219      ${mkBench {220        name = "Realistic 1";221        path = "${jsonnetBench}/perf_tests/realistic1.jsonnet";222        skipGo = skipSlow;223        skipCpp = skipSlow;224        skipScala = skipSlow;225      }}226      ${mkBench {227        name = "Realistic 2";228        path = "${jsonnetBench}/perf_tests/realistic2.jsonnet";229        skipGo = skipSlow;230        skipCpp = skipSlow;231        skipScala = skipSlow;232      }}233234      echo >> $out235      echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out236      ${mkBench {237        name = "Tail call";238        path = "${jsonnetBench}/benchmarks/bench.01.jsonnet";239        skipScala = skipSlow;240      }}241      ${mkBench {242        name = "Inheritance recursion";243        path = "${jsonnetBench}/benchmarks/bench.02.jsonnet";244        skipCpp = skipSlow;245        skipGo = skipSlow;246      }}247      ${mkBench {248        name = "Simple recursive call";249        path = "${jsonnetBench}/benchmarks/bench.03.jsonnet";250        skipScala = skipSlow;251        skipGo = skipSlow;252      }}253      ${mkBench {254        name = "Foldl string concat";255        path = "${jsonnetBench}/benchmarks/bench.04.jsonnet";256        skipCpp = skipSlow;257        skipScala = skipSlow;258      }}259      ${mkBench {260        name = "Array sorts";261        path = "${jsonnetBench}/benchmarks/bench.06.jsonnet";262        skipCpp = skipSlow;263        # std.assertEqual(reverse(std.range(1, 1000)), sort(std.range(1, 1000), keyF=function(x) -x))264        skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204";265      }}266      ${mkBench {267        name = "Lazy array";268        path = "${jsonnetBench}/benchmarks/bench.07.jsonnet";269        skipGo = skipSlow;270        skipScala = skipSlow;271      }}272      ${mkBench {273        name = "Inheritance function recursion";274        path = "${jsonnetBench}/benchmarks/bench.08.jsonnet";275        skipCpp = skipSlow;276        skipScala = skipSlow;277      }}278      ${mkBench {279        name = "String strips";280        path = "${jsonnetBench}/benchmarks/bench.09.jsonnet";281        skipCpp = skipSlow;282        skipScala = skipSlow;283      }}284      ${mkBench {285        name = "Big object";286        path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet";287        pathIsGenerator = true;288        skipScala = skipSlow;289      }}290291      echo >> $out292      echo "## Benchmarks from Go jsonnet (builtins)" >> $out293      ${mkBench {294        name = "std.base64";295        path = "${goJsonnetBench}/base64.jsonnet";296        skipRustAlternative = skipSlow;297        skipCpp = skipSlow;298        skipScala = skipSlow;299      }}300      ${mkBench {301        name = "std.base64Decode";302        path = "${goJsonnetBench}/base64Decode.jsonnet";303        skipRustAlternative = skipSlow;304        skipCpp = skipSlow;305        skipScala = skipSlow;306      }}307      ${mkBench {308        name = "std.base64DecodeBytes";309        path = "${goJsonnetBench}/base64DecodeBytes.jsonnet";310        skipRustAlternative = skipSlow;311        skipCpp = skipSlow;312        skipGo = skipSlow;313        skipScala = skipSlow;314      }}315      ${mkBench {316        name = "std.base64 (byte array)";317        path = "${goJsonnetBench}/base64_byte_array.jsonnet";318        skipRustAlternative = skipSlow;319        skipCpp = skipSlow;320        skipGo = skipSlow;321        skipScala = skipSlow;322      }}323      ${mkBench {324        name = "std.foldl";325        path = "${goJsonnetBench}/foldl.jsonnet";326        skipScala = skipSlow;327      }}328      ${mkBench {329        name = "std.manifestJsonEx";330        path = "${goJsonnetBench}/manifestJsonEx.jsonnet";331        skipScala = skipSlow;332        skipCpp = skipSlow;333      }}334      ${mkBench {335        name = "std.manifestTomlEx";336        path = "${goJsonnetBench}/manifestTomlEx.jsonnet";337        skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111";338        skipCpp = skipSlow;339      }}340      ${mkBench {341        name = "std.parseInt";342        path = "${goJsonnetBench}/parseInt.jsonnet";343        skipScala = skipSlow;344        skipCpp = skipSlow;345      }}346      ${mkBench {347        name = "std.reverse";348        path = "${goJsonnetBench}/reverse.jsonnet";349        skipCpp = skipSlow;350        skipGo = skipSlow;351      }}352      ${mkBench {353        name = "std.substr";354        path = "${goJsonnetBench}/substr.jsonnet";355        skipScala = skipSlow;356      }}357      ${mkBench {358        name = "Comparsion for array";359        path = "${goJsonnetBench}/comparison.jsonnet";360        skipScala = "array comparsion is not implemented";361        skipCpp = skipSlow;362      }}363      ${mkBench {364        name = "Comparsion for primitives";365        path = "${goJsonnetBench}/comparison2.jsonnet";366        skipRustAlternative = skipSlow;367        skipCpp = "can't run: uses up to 192GB of RAM";368        skipGo = skipSlow;369        skipScala = skipSlow;370      }}371    '';372  }
modifiednix/go-jsonnet.nixdiffbeforeafterboth
--- a/nix/go-jsonnet.nix
+++ b/nix/go-jsonnet.nix
@@ -1,6 +1,9 @@
-{ lib, buildGo119Module, fetchFromGitHub, makeWrapper }:
-
-buildGo119Module rec {
+{
+  buildGoModule,
+  fetchFromGitHub,
+  makeWrapper,
+}:
+buildGoModule rec {
   pname = "go-jsonnet";
   version = "0.20.0";
 
@@ -12,7 +15,7 @@
   };
   vendorHash = "sha256-j1fTOUpLx34TgzW94A/BctLrg9XoTtb3cBizhVJoEEI=";
 
-  buildInputs = [ makeWrapper ];
+  buildInputs = [makeWrapper];
 
   postInstall = ''
     mv $out/bin/jsonnet $out/bin/go-jsonnet
@@ -21,5 +24,5 @@
 
   doCheck = false;
 
-  subPackages = [ "cmd/jsonnet" ];
+  subPackages = ["cmd/jsonnet"];
 }
modifiednix/jsonnet.nixdiffbeforeafterboth
--- a/nix/jsonnet.nix
+++ b/nix/jsonnet.nix
@@ -1,5 +1,8 @@
-{ stdenv, lib, jekyll, fetchFromGitHub, makeWrapper }:
-
+{
+  stdenv,
+  fetchFromGitHub,
+  makeWrapper,
+}:
 stdenv.mkDerivation rec {
   pname = "jsonnet";
   version = "0.20.0";
@@ -15,7 +18,7 @@
     "jsonnet"
   ];
 
-  nativeBuildInputs = [ makeWrapper ];
+  nativeBuildInputs = [makeWrapper];
 
   installPhase = ''
     mkdir -p $out/bin
addednix/rsjsonnet.nixdiffbeforeafterboth
--- /dev/null
+++ b/nix/rsjsonnet.nix
@@ -0,0 +1,24 @@
+{
+  fetchFromGitHub,
+  rustPlatform,
+  makeWrapper,
+}:
+rustPlatform.buildRustPackage rec {
+  pname = "rsjsonnet";
+  version = "0.1.1";
+
+  src = fetchFromGitHub {
+    owner = "eduardosm";
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-C6hZYGllKrKKMwMwss6PK2UD5Zb7bk2v8DrGpWnwP/A=";
+  };
+
+  cargoHash = "sha256-TsUN9oUu6S1l9oTaR6nET1ZdRvMrR29bkP3VEDre8aE=";
+
+  nativeBuildInputs = [makeWrapper];
+
+  postInstall = ''
+    wrapProgram $out/bin/rsjsonnet --add-flags "--max-stack=200000"
+  '';
+}
modifiednix/sjsonnet.nixdiffbeforeafterboth
--- a/nix/sjsonnet.nix
+++ b/nix/sjsonnet.nix
@@ -1,30 +1,30 @@
 # This derivation uses released sjsonnet binary, which most users will use
-
 # However, recommended way of using sjsonnet - is using a client-server model,
 # for which there is no released binaries: https://github.com/databricks/sjsonnet/issues/51
-
 # TODO: Somehow build client-server version of sjsonnet, and use it in benchmarks
-
-{ stdenv, lib, fetchurl, jdk20, makeWrapper }:
-
+{
+  stdenv,
+  fetchurl,
+  jdk21_headless,
+  makeWrapper,
+  java ? jdk21_headless,
+}:
 stdenv.mkDerivation rec {
   pname = "sjsonnet";
-  version = "0.4.6";
+  version = "0.4.10";
 
   src = fetchurl {
-    url =
-      "https://github.com/databricks/${pname}/releases/download/${version}/${pname}-${version}.jar";
-    hash = "sha256-ivVKMf7W9iLF/KSXYEl6iJeNGEP5jxw2uqPF7pz6rBU=";
+    url = "https://github.com/databricks/${pname}/releases/download/${version}/${pname}-${version}.jar";
+    hash = "sha256-zKlmh+jXm23x78CKwna8fGHNqziWuPipSbWTPUCADrY=";
   };
 
   unpackPhase = "true";
-  buildInputs =
-    [ jdk20 makeWrapper ];
+  buildInputs = [java makeWrapper];
 
   installPhase = ''
     mkdir -p $out/bin $out/lib
     cp $src $out/lib/sjsonnet.jar
-    makeWrapper ${jdk20}/bin/java $out/bin/sjsonnet --add-flags "-Xss100m -XX:+UseStringDeduplication -jar $out/lib/sjsonnet.jar"
+    makeWrapper ${java}/bin/java $out/bin/sjsonnet --add-flags "-Xss100m -XX:+UseStringDeduplication -jar $out/lib/sjsonnet.jar"
   '';
   separateDebugInfo = false;
 }