difftreelog
perf reimplement AST codegen
in: master
10 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -275,6 +275,7 @@
"jrsonnet-gcmodule",
"rustc-hash",
"serde",
+ "structdump",
]
[[package]]
@@ -292,10 +293,10 @@
dependencies = [
"jrsonnet-gcmodule",
"jrsonnet-interner",
- "jrsonnet-stdlib",
"peg",
"serde",
"static_assertions",
+ "structdump",
]
[[package]]
@@ -312,6 +313,7 @@
"serde",
"serde_json",
"serde_yaml_with_quirks",
+ "structdump",
]
[[package]]
@@ -576,6 +578,28 @@
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
+name = "structdump"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0570327507bf281d8a6e6b0d4c082b12cb6bcee27efce755aa5efacd44076c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "structdump-derive",
+]
+
+[[package]]
+name = "structdump-derive"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29cc0b59cfa11f1bceda09a9a7e37e6a6c3138575fd24ade8aa9af6d09aedf28"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
name = "syn"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,6 @@
opt-level = 3
lto = "fat"
codegen-units = 1
-debug = 0
+# debug = 0
panic = "abort"
-strip = true
+# strip = true
crates/jrsonnet-interner/Cargo.tomldiffbeforeafterboth--- a/crates/jrsonnet-interner/Cargo.toml
+++ b/crates/jrsonnet-interner/Cargo.toml
@@ -7,12 +7,19 @@
edition = "2021"
[features]
-default = ["serde"]
+default = []
+# Implement value serialization using structdump
+structdump = ["dep:structdump"]
+# Implement value serialization using serde
+#
+# Warning: serialized values won't be deduplicated
serde = ["dep:serde"]
[dependencies]
jrsonnet-gcmodule = { version = "0.3.4" }
serde = { version = "1.0", optional = true }
+structdump = { version = "0.2.0", optional = true }
+
rustc-hash = "1.1"
hashbrown = { version = "0.12.1", features = ["inline-more"] }
crates/jrsonnet-interner/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-interner/src/lib.rs
+++ b/crates/jrsonnet-interner/src/lib.rs
@@ -205,6 +205,7 @@
}
}
+#[cfg(feature = "serde")]
impl serde::Serialize for IStr {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
@@ -214,6 +215,7 @@
}
}
+#[cfg(feature = "serde")]
impl<'de> serde::Deserialize<'de> for IStr {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
@@ -224,6 +226,24 @@
}
}
+#[cfg(feature = "structdump")]
+impl structdump::Codegen for IStr {
+ fn gen_code(
+ &self,
+ res: &mut structdump::CodegenResult,
+ _unique: bool,
+ ) -> structdump::TokenStream {
+ let s: &str = self;
+ res.add_code(
+ structdump::quote! {
+ structdump_import::IStr::from(#s)
+ },
+ Some(structdump::quote![structdump_import::IStr]),
+ false,
+ )
+ }
+}
+
thread_local! {
static POOL: RefCell<HashMap<Inner, (), BuildHasherDefault<FxHasher>>> = RefCell::new(HashMap::with_capacity_and_hasher(200, BuildHasherDefault::default()));
}
crates/jrsonnet-parser/Cargo.tomldiffbeforeafterboth--- a/crates/jrsonnet-parser/Cargo.toml
+++ b/crates/jrsonnet-parser/Cargo.toml
@@ -7,7 +7,23 @@
edition = "2021"
[features]
+default = []
exp-destruct = []
+# Implement serialization of AST using structdump
+#
+# Structdump generates code, which exactly replicated passed AST
+# Contrary to serde, has no code bloat problem, and is recommended
+#
+# The only limitation is serialized form is only useable if built from build script
+structdump = ["dep:structdump", "jrsonnet-interner/structdump"]
+# Implement serialization of AST using serde
+#
+# Warning: as serde doesn't deduplicate strings, `Source` struct will bloat
+# output binary with repeating source code. To resolve this issue, you should either
+# override serialization of this struct using custom `Serializer`/`Deserializer`,
+# not rely on Source, and fill its `source_code` with empty value, or use `structdump`
+# instead
+serde = ["dep:serde"]
[dependencies]
jrsonnet-interner = { path = "../jrsonnet-interner", version = "0.4.2" }
@@ -18,6 +34,4 @@
peg = "0.8.0"
serde = { version = "1.0", features = ["derive", "rc"], optional = true }
-
-[dev-dependencies]
-jrsonnet-stdlib = { path = "../jrsonnet-stdlib", version = "0.4.2" }
+structdump = { version = "0.2.0", features = ["derive"], optional = true }
crates/jrsonnet-parser/src/expr.rsdiffbeforeafterboth8use jrsonnet_interner::IStr;8use jrsonnet_interner::IStr;9#[cfg(feature = "serde")]9#[cfg(feature = "serde")]10use serde::{Deserialize, Serialize};10use serde::{Deserialize, Serialize};11#[cfg(feature = "structdump")]12use structdump::Codegen;111312use crate::source::Source;14use crate::source::Source;131514#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]16#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]17#[cfg_attr(feature = "structdump", derive(Codegen))]15#[derive(Debug, PartialEq, Trace)]18#[derive(Debug, PartialEq, Trace)]16pub enum FieldName {19pub enum FieldName {17 /// {fixed: 2}20 /// {fixed: 2}20 Dyn(LocExpr),23 Dyn(LocExpr),21}24}222526#[cfg_attr(feature = "structdump", derive(Codegen))]23#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]27#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]24#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]28#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]25pub enum Visibility {29pub enum Visibility {37 }41 }38}42}394344#[cfg_attr(feature = "structdump", derive(Codegen))]40#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]45#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]41#[derive(Clone, Debug, PartialEq, Trace)]46#[derive(Clone, Debug, PartialEq, Trace)]42pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);47pub struct AssertStmt(pub LocExpr, pub Option<LocExpr>);434849#[cfg_attr(feature = "structdump", derive(Codegen))]44#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]50#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]45#[derive(Debug, PartialEq, Trace)]51#[derive(Debug, PartialEq, Trace)]46pub struct FieldMember {52pub struct FieldMember {51 pub value: LocExpr,57 pub value: LocExpr,52}58}535960#[cfg_attr(feature = "structdump", derive(Codegen))]54#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]61#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]55#[derive(Debug, PartialEq, Trace)]62#[derive(Debug, PartialEq, Trace)]56pub enum Member {63pub enum Member {59 AssertStmt(AssertStmt),66 AssertStmt(AssertStmt),60}67}616869#[cfg_attr(feature = "structdump", derive(Codegen))]62#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]70#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]63#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]71#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]64pub enum UnaryOpType {72pub enum UnaryOpType {84 }92 }85}93}869495#[cfg_attr(feature = "structdump", derive(Codegen))]87#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]96#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]88#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]97#[derive(Debug, Clone, Copy, PartialEq, Eq, Trace)]89pub enum BinaryOpType {98pub enum BinaryOpType {150}159}151160152/// name, default value161/// name, default value162#[cfg_attr(feature = "structdump", derive(Codegen))]153#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]163#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]154#[derive(Debug, PartialEq, Trace)]164#[derive(Debug, PartialEq, Trace)]155pub struct Param(pub Destruct, pub Option<LocExpr>);165pub struct Param(pub Destruct, pub Option<LocExpr>);156166157/// Defined function parameters167/// Defined function parameters168#[cfg_attr(feature = "structdump", derive(Codegen))]158#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]169#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]159#[derive(Debug, Clone, PartialEq, Trace)]170#[derive(Debug, Clone, PartialEq, Trace)]160pub struct ParamsDesc(pub Rc<Vec<Param>>);171pub struct ParamsDesc(pub Rc<Vec<Param>>);166 }177 }167}178}168179180#[cfg_attr(feature = "structdump", derive(Codegen))]169#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]181#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]170#[derive(Debug, PartialEq, Trace)]182#[derive(Debug, PartialEq, Trace)]171pub struct ArgsDesc {183pub struct ArgsDesc {187 Drop,199 Drop,188}200}189201202#[cfg_attr(feature = "structdump", derive(Codegen))]190#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]203#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]191#[derive(Debug, Clone, PartialEq, Trace)]204#[derive(Debug, Clone, PartialEq, Trace)]192pub enum Destruct {205pub enum Destruct {216 }229 }217}230}218231232#[cfg_attr(feature = "structdump", derive(Codegen))]219#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]233#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]220#[derive(Debug, Clone, PartialEq, Trace)]234#[derive(Debug, Clone, PartialEq, Trace)]221pub enum BindSpec {235pub enum BindSpec {230 },244 },231}245}232246247#[cfg_attr(feature = "structdump", derive(Codegen))]233#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]248#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]234#[derive(Debug, PartialEq, Trace)]249#[derive(Debug, PartialEq, Trace)]235pub struct IfSpecData(pub LocExpr);250pub struct IfSpecData(pub LocExpr);236251252#[cfg_attr(feature = "structdump", derive(Codegen))]237#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]253#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]238#[derive(Debug, PartialEq, Trace)]254#[derive(Debug, PartialEq, Trace)]239pub struct ForSpecData(pub IStr, pub LocExpr);255pub struct ForSpecData(pub IStr, pub LocExpr);240256257#[cfg_attr(feature = "structdump", derive(Codegen))]241#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]258#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]242#[derive(Debug, PartialEq, Trace)]259#[derive(Debug, PartialEq, Trace)]243pub enum CompSpec {260pub enum CompSpec {244 IfSpec(IfSpecData),261 IfSpec(IfSpecData),245 ForSpec(ForSpecData),262 ForSpec(ForSpecData),246}263}247264265#[cfg_attr(feature = "structdump", derive(Codegen))]248#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]266#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]249#[derive(Debug, PartialEq, Trace)]267#[derive(Debug, PartialEq, Trace)]250pub struct ObjComp {268pub struct ObjComp {256 pub compspecs: Vec<CompSpec>,274 pub compspecs: Vec<CompSpec>,257}275}258276277#[cfg_attr(feature = "structdump", derive(Codegen))]259#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]278#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]260#[derive(Debug, PartialEq, Trace)]279#[derive(Debug, PartialEq, Trace)]261pub enum ObjBody {280pub enum ObjBody {262 MemberList(Vec<Member>),281 MemberList(Vec<Member>),263 ObjComp(ObjComp),282 ObjComp(ObjComp),264}283}265284285#[cfg_attr(feature = "structdump", derive(Codegen))]266#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]286#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]267#[derive(Debug, PartialEq, Eq, Clone, Copy, Trace)]287#[derive(Debug, PartialEq, Eq, Clone, Copy, Trace)]268pub enum LiteralType {288pub enum LiteralType {274 False,294 False,275}295}276296297#[cfg_attr(feature = "structdump", derive(Codegen))]277#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]298#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]278#[derive(Debug, PartialEq, Trace)]299#[derive(Debug, PartialEq, Trace)]279pub struct SliceDesc {300pub struct SliceDesc {283}304}284305285/// Syntax base306/// Syntax base307#[cfg_attr(feature = "structdump", derive(Codegen))]286#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]308#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]287#[derive(Debug, PartialEq, Trace)]309#[derive(Debug, PartialEq, Trace)]288pub enum Expr {310pub enum Expr {351}373}352374353/// file, begin offset, end offset375/// file, begin offset, end offset376#[cfg_attr(feature = "structdump", derive(Codegen))]354#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]377#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]355#[derive(Clone, PartialEq, Eq, Trace)]378#[derive(Clone, PartialEq, Eq, Trace)]356#[trace(skip)]379#[trace(skip)]373396374/// Holds AST expression and its location in source file397/// Holds AST expression and its location in source file375#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]398#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]399#[cfg_attr(feature = "structdump", derive(Codegen))]376#[derive(Clone, PartialEq, Trace)]400#[derive(Clone, PartialEq, Trace)]377pub struct LocExpr(pub Rc<Expr>, pub ExprLocation);401pub struct LocExpr(pub Rc<Expr>, pub ExprLocation);378402crates/jrsonnet-parser/src/source.rsdiffbeforeafterboth--- a/crates/jrsonnet-parser/src/source.rs
+++ b/crates/jrsonnet-parser/src/source.rs
@@ -9,9 +9,12 @@
use jrsonnet_interner::IStr;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
+#[cfg(feature = "structdump")]
+use structdump::Codegen;
use crate::location::{location_to_offset, offset_to_location, CodeLocation};
+#[cfg_attr(feature = "structdump", derive(Codegen))]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(PartialEq, Eq, Debug, Hash, Clone)]
pub enum SourcePath {
@@ -39,9 +42,10 @@
/// Either real file, or virtual
/// Hash of FileName always have same value as raw Path, to make it possible to use with raw_entry_mut
+#[cfg_attr(feature = "structdump", derive(Codegen))]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct Source(Rc<(SourcePath, IStr)>);
+pub struct Source(pub Rc<(SourcePath, IStr)>);
static_assertions::assert_eq_size!(Source, *const ());
impl Trace for Source {
crates/jrsonnet-stdlib/Cargo.tomldiffbeforeafterboth--- a/crates/jrsonnet-stdlib/Cargo.toml
+++ b/crates/jrsonnet-stdlib/Cargo.toml
@@ -7,9 +7,10 @@
edition = "2021"
[features]
-default = []
-# Serializes standard library AST, and deserialize on start, instead of parsing it every run from text
-serialized-stdlib = ["bincode", "jrsonnet-parser/serde"]
+default = ["codegenerated-stdlib"]
+# Speed-up initialization by generating code for parsed stdlib, instead
+# of invoking parser for it
+codegenerated-stdlib = ["jrsonnet-parser/structdump"]
# Enables legacy `std.thisFile` support, at the cost of worse caching
legacy-this-file = []
# Add order preservation flag to some functions
@@ -45,8 +46,5 @@
serde_yaml_with_quirks = "0.8.24"
[build-dependencies]
-jrsonnet-parser = { path = "../jrsonnet-parser", version = "0.4.2", features = [
- "serde",
-] }
-serde = "1.0"
-bincode = "1.3"
+jrsonnet-parser = { path = "../jrsonnet-parser", version = "0.4.2" }
+structdump = { version = "0.2.0", features = ["derive"] }
crates/jrsonnet-stdlib/build.rsdiffbeforeafterboth--- a/crates/jrsonnet-stdlib/build.rs
+++ b/crates/jrsonnet-stdlib/build.rs
@@ -1,7 +1,7 @@
use std::{borrow::Cow, env, fs::File, io::Write, path::Path};
-use bincode::serialize;
use jrsonnet_parser::{parse, ParserSettings, Source};
+use structdump::CodegenResult;
fn main() {
let parsed = parse(
@@ -15,10 +15,15 @@
)
.expect("parse");
+ let mut out = CodegenResult::default();
+
+ let v = out.codegen(&parsed, true);
+
{
let out_dir = env::var("OUT_DIR").unwrap();
- let dest_path = Path::new(&out_dir).join("stdlib.bincode");
+ let dest_path = Path::new(&out_dir).join("stdlib.rs");
let mut f = File::create(&dest_path).unwrap();
- f.write_all(&serialize(&parsed).unwrap()).unwrap();
+ f.write_all(v.to_string().replace(';', ";\n").as_bytes())
+ .unwrap();
}
}
crates/jrsonnet-stdlib/src/expr.rsdiffbeforeafterboth--- a/crates/jrsonnet-stdlib/src/expr.rs
+++ b/crates/jrsonnet-stdlib/src/expr.rs
@@ -1,22 +1,103 @@
-use std::borrow::Cow;
+use jrsonnet_parser::LocExpr;
-use jrsonnet_parser::{LocExpr, ParserSettings, Source};
+mod structdump_import {
+ pub(super) use std::{borrow::Cow, rc::Rc};
-pub const STDLIB_STR: &str = include_str!("./std.jsonnet");
+ pub(super) use jrsonnet_parser::*;
+ pub(super) use vec;
+ pub(super) use Option;
+}
pub fn stdlib_expr() -> LocExpr {
#[cfg(feature = "serialized-stdlib")]
{
+ use bincode::{BincodeRead, DefaultOptions, Options};
+ use serde::{Deserialize, Deserializer};
+
+ struct LocDeserializer<R, O: Options> {
+ source: Source,
+ wrapped: bincode::Deserializer<R, O>,
+ }
+ macro_rules! delegate {
+ ($(fn $name:ident($($arg:ident: $ty:ty),*))+) => {$(
+ fn $name<V>(mut self $(, $arg: $ty)*, visitor: V) -> Result<V::Value, Self::Error>
+ where V: serde::de::Visitor<'de>,
+ {
+ self.wrapped.$name($($arg,)* visitor)
+ }
+ )+};
+ }
+ impl<'de, R, O> Deserializer<'de> for LocDeserializer<R, O>
+ where
+ R: BincodeRead<'de>,
+ O: Options,
+ {
+ type Error = <&'de mut bincode::Deserializer<R, O> as Deserializer<'de>>::Error;
+
+ delegate! {
+ fn deserialize_any()
+ fn deserialize_bool()
+ fn deserialize_u16()
+ fn deserialize_u32()
+ fn deserialize_u64()
+ fn deserialize_i16()
+ fn deserialize_i32()
+ fn deserialize_i64()
+ fn deserialize_f32()
+ fn deserialize_f64()
+ fn deserialize_u128()
+ fn deserialize_i128()
+ fn deserialize_u8()
+ fn deserialize_i8()
+ fn deserialize_unit()
+ fn deserialize_char()
+ fn deserialize_str()
+ fn deserialize_string()
+ fn deserialize_bytes()
+ fn deserialize_byte_buf()
+ fn deserialize_enum(name: &'static str, variants: &'static [&'static str])
+ fn deserialize_tuple(len: usize)
+ fn deserialize_option()
+ fn deserialize_seq()
+ fn deserialize_map()
+ fn deserialize_struct(name: &'static str, fields: &'static [&'static str])
+ fn deserialize_identifier()
+ fn deserialize_newtype_struct(name: &'static str)
+ fn deserialize_unit_struct(name: &'static str)
+ fn deserialize_tuple_struct(name: &'static str, len: usize)
+ fn deserialize_ignored_any()
+ }
+
+ fn is_human_readable(&self) -> bool {
+ false
+ }
+ }
+
+ // In build.rs, Source object is populated with empty values, deserializer wrapper loads correct values on deserialize
+ let mut deserializer = bincode::Deserializer::from_slice(
+ include_bytes!(concat!(env!("OUT_DIR"), "/stdlib.bincode")),
+ DefaultOptions::new()
+ .with_fixint_encoding()
+ .allow_trailing_bytes(),
+ );
+
// Should not panic, stdlib.bincode is generated in build.rs
- return bincode::deserialize(include_bytes!(concat!(env!("OUT_DIR"), "/stdlib.bincode")))
- .unwrap();
+ LocExpr::deserialize(&mut deserializer).unwrap()
}
- jrsonnet_parser::parse(
- STDLIB_STR,
- &ParserSettings {
- file_name: Source::new_virtual(Cow::Borrowed("<std>"), STDLIB_STR.into()),
- },
- )
- .unwrap()
+ #[cfg(feature = "codegenerated-stdlib")]
+ {
+ include!(concat!(env!("OUT_DIR"), "/stdlib.rs"))
+ }
+
+ #[cfg(not(feature = "codegenerated-stdlib"))]
+ {
+ jrsonnet_parser::parse(
+ STDLIB_STR,
+ &ParserSettings {
+ file_name: Source::new_virtual(Cow::Borrowed("<std>"), STDLIB_STR.into()),
+ },
+ )
+ .unwrap()
+ }
}