From 695a2f8d51fa9a76e951a16f1cfb4e5c612c7b76 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Wed, 06 Jan 2021 12:59:34 +0000 Subject: [PATCH] feat: string interning --- --- a/bindings/jsonnet/Cargo.toml +++ b/bindings/jsonnet/Cargo.toml @@ -5,6 +5,7 @@ edition = "2018" [dependencies] +jrsonnet-interner = { path = "../../crates/jrsonnet-interner", version = "0.3.3" } jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator", version = "0.3.3" } jrsonnet-parser = { path = "../../crates/jrsonnet-parser", version = "0.3.3" } --- a/bindings/jsonnet/src/import.rs +++ b/bindings/jsonnet/src/import.rs @@ -4,6 +4,7 @@ error::{Error::*, Result}, throw, EvaluationState, ImportResolver, }; +use jrsonnet_interner::IStr; use std::{ any::Any, cell::RefCell, @@ -30,7 +31,7 @@ cb: JsonnetImportCallback, ctx: *mut c_void, - out: RefCell>>, + out: RefCell>, } impl ImportResolver for CallbackImportResolver { fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { @@ -75,7 +76,7 @@ Ok(Rc::new(found_here_buf)) } - fn load_file_contents(&self, resolved: &PathBuf) -> Result> { + fn load_file_contents(&self, resolved: &PathBuf) -> Result { Ok(self.out.borrow().get(resolved).unwrap().clone()) } unsafe fn as_any(&self) -> &dyn Any { @@ -124,7 +125,7 @@ throw!(ImportFileNotFound(from.clone(), path.clone())) } } - fn load_file_contents(&self, id: &PathBuf) -> Result> { + fn load_file_contents(&self, id: &PathBuf) -> Result { let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.clone()))?; let mut out = String::new(); file.read_to_string(&mut out) --- a/bindings/jsonnet/src/lib.rs +++ b/bindings/jsonnet/src/lib.rs @@ -8,6 +8,7 @@ use import::NativeImportResolver; use jrsonnet_evaluator::{EvaluationState, ManifestFormat, Val}; +use jrsonnet_interner::IStr; use std::{ alloc::Layout, ffi::{CStr, CString}, @@ -161,7 +162,7 @@ }) } -fn multi_to_raw(multi: Vec<(Rc, Rc)>) -> *const c_char { +fn multi_to_raw(multi: Vec<(IStr, IStr)>) -> *const c_char { let mut out = Vec::new(); for (i, (k, v)) in multi.iter().enumerate() { if i != 0 { @@ -237,7 +238,7 @@ }) } -fn stream_to_raw(multi: Vec>) -> *const c_char { +fn stream_to_raw(multi: Vec) -> *const c_char { let mut out = Vec::new(); for (i, v) in multi.iter().enumerate() { if i != 0 { --- a/crates/jrsonnet-evaluator/Cargo.toml +++ b/crates/jrsonnet-evaluator/Cargo.toml @@ -12,8 +12,6 @@ serialized-stdlib = ["serde", "bincode", "jrsonnet-parser/deserialize"] # Allow to convert Val into serde_json::Value and backwards serde-json = ["serde", "serde_json"] -# Same as above, but with generated code instead of serde. Reduces memory usage, but increases binary size and compilation time -codegenerated-stdlib = [] # Replace some standard library functions with faster implementations (I.e manifestJsonEx) # Library works fine without this feature, but requires more memory and time for std function calls faster = [] @@ -24,6 +22,7 @@ unstable = [] [dependencies] +jrsonnet-interner = { path = "../jrsonnet-interner" } jrsonnet-parser = { path = "../jrsonnet-parser", version = "0.3.3" } jrsonnet-stdlib = { path = "../jrsonnet-stdlib", version = "0.3.3" } jrsonnet-types = { path = "../jrsonnet-types", version = "0.3.3" } @@ -58,8 +57,7 @@ optional = true [build-dependencies] -jrsonnet-parser = { path = "../jrsonnet-parser", features = ["dump", "serialize", "deserialize"], version = "0.3.3" } +jrsonnet-parser = { path = "../jrsonnet-parser", features = ["serialize", "deserialize"], version = "0.3.3" } jrsonnet-stdlib = { path = "../jrsonnet-stdlib", version = "0.3.3" } -structdump = "0.1.2" serde = "1.0" bincode = "1.3.1" --- a/crates/jrsonnet-evaluator/build.rs +++ b/crates/jrsonnet-evaluator/build.rs @@ -10,7 +10,6 @@ path::{Path, PathBuf}, rc::Rc, }; -use structdump::CodegenResult; fn main() { let parsed = parse( @@ -36,11 +35,11 @@ name: FieldName::Fixed(name), .. }) - if **name == *"join" || **name == *"manifestJsonEx" || - **name == *"escapeStringJson" || **name == *"equals" || - **name == *"base64" || **name == *"foldl" || **name == *"foldr" || - **name == *"sortImpl" || **name == *"format" || **name == *"range" || - **name == *"reverse" || **name == *"slice" || **name == *"mod" + if name == "join" || name == "manifestJsonEx" || + name == "escapeStringJson" || name == "equals" || + name == "base64" || name == "foldl" || name == "foldr" || + name == "sortImpl" || name == "format" || name == "range" || + name == "reverse" || name == "slice" || name == "mod" ) }) .collect(), @@ -52,15 +51,6 @@ } else { parsed }; - { - let mut codegen = CodegenResult::default(); - let code = codegen.codegen(&parsed); - - let out_dir = env::var("OUT_DIR").unwrap(); - let dest_path = Path::new(&out_dir).join("stdlib.rs"); - let mut f = File::create(&dest_path).unwrap(); - f.write_all(&code.as_bytes()).unwrap(); - } { let out_dir = env::var("OUT_DIR").unwrap(); let dest_path = Path::new(&out_dir).join("stdlib.bincode"); --- a/crates/jrsonnet-evaluator/src/builtin/format.rs +++ b/crates/jrsonnet-evaluator/src/builtin/format.rs @@ -2,6 +2,7 @@ #![allow(clippy::too_many_arguments)] use crate::{error::Error::*, throw, LocError, ObjValue, Result, Val}; +use jrsonnet_interner::IStr; use jrsonnet_types::ValType; use thiserror::Error; @@ -20,7 +21,7 @@ #[error("mapping keys required")] MappingKeysRequired, #[error("no such format field: {0}")] - NoSuchFormatField(Rc), + NoSuchFormatField(IStr), } impl From for LocError { @@ -29,7 +30,6 @@ } } -use std::rc::Rc; use FormatError::*; type ParseResult<'t, T> = std::result::Result<(T, &'t str), FormatError>; @@ -680,7 +680,7 @@ } Element::Code(c) => { // TODO: Operate on ref - let f: Rc = c.mkey.into(); + let f: IStr = c.mkey.into(); let width = match c.width { Width::Star => { throw!(CannotUseStarWidthWithObject); --- a/crates/jrsonnet-evaluator/src/builtin/mod.rs +++ b/crates/jrsonnet-evaluator/src/builtin/mod.rs @@ -6,6 +6,7 @@ with_state, ArrValue, Context, FuncVal, LazyVal, Val, }; use format::{format_arr, format_obj}; +use jrsonnet_interner::IStr; use jrsonnet_parser::{ArgsDesc, BinaryOpType, ExprLocation}; use jrsonnet_types::ty; use std::{collections::HashMap, path::PathBuf, rc::Rc}; @@ -19,7 +20,7 @@ pub mod manifest; pub mod sort; -fn std_format(str: Rc, vals: Val) -> Result { +fn std_format(str: IStr, vals: Val) -> Result { push( &Some(ExprLocation(Rc::from(PathBuf::from("std.jsonnet")), 0, 0)), || format!("std.format of {}", str), --- a/crates/jrsonnet-evaluator/src/ctx.rs +++ b/crates/jrsonnet-evaluator/src/ctx.rs @@ -2,6 +2,7 @@ error::Error::*, future_wrapper, map::LayeredHashMap, rc_fn_helper, resolved_lazy_val, LazyBinding, LazyVal, ObjValue, Result, Val, }; +use jrsonnet_interner::IStr; use rustc_hash::FxHashMap; use std::hash::BuildHasherDefault; use std::{cell::RefCell, collections::HashMap, fmt::Debug, rc::Rc}; @@ -18,7 +19,7 @@ dollar: Option, this: Option, super_obj: Option, - bindings: LayeredHashMap, LazyVal>, + bindings: LayeredHashMap, } impl Debug for ContextInternals { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -57,7 +58,7 @@ })) } - pub fn binding(&self, name: Rc) -> Result { + pub fn binding(&self, name: IStr) -> Result { Ok(self .0 .bindings @@ -72,7 +73,7 @@ ctx.unwrap() } - pub fn with_var(self, name: Rc, value: Val) -> Self { + pub fn with_var(self, name: IStr, value: Val) -> Self { let mut new_bindings = FxHashMap::with_capacity_and_hasher(1, BuildHasherDefault::default()); new_bindings.insert(name, resolved_lazy_val!(value)); @@ -81,7 +82,7 @@ pub fn extend( self, - new_bindings: FxHashMap, LazyVal>, + new_bindings: FxHashMap, new_dollar: Option, new_this: Option, new_super_obj: Option, @@ -123,7 +124,7 @@ } pub fn extend_unbound( self, - new_bindings: HashMap, LazyBinding>, + new_bindings: HashMap, new_dollar: Option, new_this: Option, new_super_obj: Option, --- a/crates/jrsonnet-evaluator/src/error.rs +++ b/crates/jrsonnet-evaluator/src/error.rs @@ -2,6 +2,7 @@ builtin::{format::FormatError, sort::SortError}, typed::TypeLocError, }; +use jrsonnet_interner::IStr; use jrsonnet_parser::{BinaryOpType, ExprLocation, UnaryOpType}; use jrsonnet_types::ValType; use std::{path::PathBuf, rc::Rc}; @@ -10,7 +11,7 @@ #[derive(Error, Debug, Clone)] pub enum Error { #[error("intrinsic not found: {0}")] - IntrinsicNotFound(Rc), + IntrinsicNotFound(IStr), #[error("argument reordering in intrisics not supported yet")] IntrinsicArgumentReorderingIsNotSupportedYet, @@ -33,36 +34,36 @@ ArrayBoundsError(usize, usize), #[error("assert failed: {0}")] - AssertionFailed(Rc), + AssertionFailed(IStr), #[error("variable is not defined: {0}")] - VariableIsNotDefined(Rc), + VariableIsNotDefined(IStr), #[error("type mismatch: expected {}, got {2} {0}", .1.iter().map(|e| format!("{}", e)).collect::>().join(", "))] TypeMismatch(&'static str, Vec, ValType), #[error("no such field: {0}")] - NoSuchField(Rc), + NoSuchField(IStr), #[error("only functions can be called, got {0}")] OnlyFunctionsCanBeCalledGot(ValType), #[error("parameter {0} is not defined")] UnknownFunctionParameter(String), #[error("argument {0} is already bound")] - BindingParameterASecondTime(Rc), + BindingParameterASecondTime(IStr), #[error("too many args, function has {0}")] TooManyArgsFunctionHas(usize), #[error("founction argument is not passed: {0}")] - FunctionParameterNotBoundInCall(Rc), + FunctionParameterNotBoundInCall(IStr), #[error("external variable is not defined: {0}")] - UndefinedExternalVariable(Rc), + UndefinedExternalVariable(IStr), #[error("native is not defined: {0}")] - UndefinedExternalFunction(Rc), + UndefinedExternalFunction(IStr), #[error("field name should be string, got {0}")] FieldMustBeStringGot(ValType), #[error("attempted to index array with string {0}")] - AttemptedIndexAnArrayWithString(Rc), + AttemptedIndexAnArrayWithString(IStr), #[error("{0} index type should be {1}, got {2}")] ValueIndexMustBeTypeGot(ValType, ValType, ValType), #[error("cant index into {0}")] @@ -86,12 +87,12 @@ )] ImportSyntaxError { path: Rc, - source_code: Rc, + source_code: IStr, error: Box, }, #[error("runtime error: {0}")] - RuntimeError(Rc), + RuntimeError(IStr), #[error("stack overflow, try to reduce recursion, or set --max-stack to bigger value")] StackOverflow, #[error("tried to index by fractional value")] --- a/crates/jrsonnet-evaluator/src/evaluate.rs +++ b/crates/jrsonnet-evaluator/src/evaluate.rs @@ -3,6 +3,7 @@ ContextCreator, FuncDesc, FuncVal, LazyBinding, LazyVal, ObjMember, ObjValue, Result, Val, }; use closure::closure; +use jrsonnet_interner::IStr; use jrsonnet_parser::{ ArgsDesc, AssertStmt, BinaryOpType, BindSpec, CompSpec, Expr, ExprLocation, FieldMember, ForSpecData, IfSpecData, LiteralType, LocExpr, Member, ObjBody, ParamsDesc, UnaryOpType, @@ -12,7 +13,7 @@ use rustc_hash::FxHashMap; use std::{collections::HashMap, rc::Rc}; -pub fn evaluate_binding(b: &BindSpec, context_creator: ContextCreator) -> (Rc, LazyBinding) { +pub fn evaluate_binding(b: &BindSpec, context_creator: ContextCreator) -> (IStr, LazyBinding) { let b = b.clone(); if let Some(params) = &b.params { let params = params.clone(); @@ -45,7 +46,7 @@ } } -pub fn evaluate_method(ctx: Context, name: Rc, params: ParamsDesc, body: LocExpr) -> Val { +pub fn evaluate_method(ctx: Context, name: IStr, params: ParamsDesc, body: LocExpr) -> Val { Val::Func(Rc::new(FuncVal::Normal(FuncDesc { name, ctx, @@ -57,7 +58,7 @@ pub fn evaluate_field_name( context: Context, field_name: &jrsonnet_parser::FieldName, -) -> Result>> { +) -> Result> { Ok(match field_name { jrsonnet_parser::FieldName::Fixed(n) => Some(n.clone()), jrsonnet_parser::FieldName::Dyn(expr) => { @@ -182,7 +183,7 @@ }) } -future_wrapper!(HashMap, LazyBinding>, FutureNewBindings); +future_wrapper!(HashMap, FutureNewBindings); future_wrapper!(ObjValue, FutureObjValue); pub fn evaluate_comp( @@ -230,7 +231,7 @@ }) ); { - let mut bindings: HashMap, LazyBinding> = HashMap::new(); + let mut bindings: HashMap = HashMap::new(); for (n, b) in members .iter() .filter_map(|m| match m { @@ -334,7 +335,7 @@ )?) }) ); - let mut bindings: HashMap, LazyBinding> = HashMap::new(); + let mut bindings: HashMap = HashMap::new(); for (n, b) in obj .pre_locals .iter() @@ -401,7 +402,7 @@ }) } -pub fn evaluate_named(context: Context, lexpr: &LocExpr, name: Rc) -> Result { +pub fn evaluate_named(context: Context, lexpr: &LocExpr, name: IStr) -> Result { use Expr::*; let LocExpr(expr, _loc) = lexpr; Ok(match &**expr { @@ -498,7 +499,7 @@ } } LocalExpr(bindings, returned) => { - let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); + let mut new_bindings: HashMap = HashMap::new(); let future_context = Context::new_future(); let context_creator = context_creator!( --- a/crates/jrsonnet-evaluator/src/function.rs +++ b/crates/jrsonnet-evaluator/src/function.rs @@ -1,8 +1,9 @@ use crate::{error::Error::*, evaluate, lazy_val, resolved_lazy_val, throw, Context, Result, Val}; use closure::closure; +use jrsonnet_interner::IStr; use jrsonnet_parser::{ArgsDesc, ParamsDesc}; use rustc_hash::FxHashMap; -use std::{collections::HashMap, hash::BuildHasherDefault, rc::Rc}; +use std::{collections::HashMap, hash::BuildHasherDefault}; const NO_DEFAULT_CONTEXT: &str = "no default context set for call with defined default parameter value"; @@ -66,7 +67,7 @@ ctx: Context, body_ctx: Option, params: &ParamsDesc, - args: &HashMap, Val>, + args: &HashMap, tailstrict: bool, ) -> Result { let mut out = FxHashMap::with_capacity_and_hasher(params.len(), BuildHasherDefault::default()); --- a/crates/jrsonnet-evaluator/src/import.rs +++ b/crates/jrsonnet-evaluator/src/import.rs @@ -3,6 +3,7 @@ throw, }; use fs::File; +use jrsonnet_interner::IStr; use std::fs; use std::io::Read; use std::{any::Any, cell::RefCell, collections::HashMap, path::PathBuf, rc::Rc}; @@ -15,7 +16,7 @@ fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result>; /// Reads file from filesystem, should be used only with path received from `resolve_file` - fn load_file_contents(&self, resolved: &PathBuf) -> Result>; + fn load_file_contents(&self, resolved: &PathBuf) -> Result; /// # Safety /// @@ -32,7 +33,7 @@ throw!(ImportNotSupported(from.clone(), path.clone())) } - fn load_file_contents(&self, _resolved: &PathBuf) -> Result> { + fn load_file_contents(&self, _resolved: &PathBuf) -> Result { // Can be only caused by library direct consumer, not by supplied jsonnet panic!("dummy resolver can't load any file") } @@ -72,7 +73,7 @@ throw!(ImportFileNotFound(from.clone(), path.clone())) } } - fn load_file_contents(&self, id: &PathBuf) -> Result> { + fn load_file_contents(&self, id: &PathBuf) -> Result { let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.clone()))?; let mut out = String::new(); file.read_to_string(&mut out) @@ -89,7 +90,7 @@ /// Caches results of the underlying resolver pub struct CachingImportResolver { resolution_cache: RefCell>>>, - loading_cache: RefCell>>>, + loading_cache: RefCell>>, inner: Box, } impl ImportResolver for CachingImportResolver { @@ -101,7 +102,7 @@ .clone() } - fn load_file_contents(&self, resolved: &PathBuf) -> Result> { + fn load_file_contents(&self, resolved: &PathBuf) -> Result { self.loading_cache .borrow_mut() .entry(resolved.clone()) --- a/crates/jrsonnet-evaluator/src/lib.rs +++ b/crates/jrsonnet-evaluator/src/lib.rs @@ -23,6 +23,7 @@ pub use evaluate::*; pub use function::parse_function_call; pub use import::*; +use jrsonnet_interner::IStr; use jrsonnet_parser::*; use native::NativeCallback; pub use obj::*; @@ -63,13 +64,13 @@ /// Limits amount of stack trace items preserved pub max_trace: usize, /// Used for s`td.extVar` - pub ext_vars: HashMap, Val>, + pub ext_vars: HashMap, /// Used for ext.native - pub ext_natives: HashMap, Rc>, + pub ext_natives: HashMap>, /// TLA vars - pub tla_vars: HashMap, Val>, + pub tla_vars: HashMap, /// Global variables are inserted in default context - pub globals: HashMap, Val>, + pub globals: HashMap, /// Used to resolve file locations/contents pub import_resolver: Box, /// Used in manifestification functions @@ -102,11 +103,11 @@ stack_depth: usize, /// Contains file source codes and evaluation results for imports and pretty-printed stacktraces files: HashMap, FileData>, - str_files: HashMap, Rc>, + str_files: HashMap, IStr>, } pub struct FileData { - source_code: Rc, + source_code: IStr, parsed: LocExpr, evaluated: Option, } @@ -144,7 +145,7 @@ impl EvaluationState { /// Parses and adds file as loaded - pub fn add_file(&self, path: Rc, source_code: Rc) -> Result<()> { + pub fn add_file(&self, path: Rc, source_code: IStr) -> Result<()> { self.add_parsed_file( path.clone(), source_code.clone(), @@ -169,7 +170,7 @@ pub fn add_parsed_file( &self, name: Rc, - source_code: Rc, + source_code: IStr, parsed: LocExpr, ) -> Result<()> { self.data_mut().files.insert( @@ -183,7 +184,7 @@ Ok(()) } - pub fn get_source(&self, name: &PathBuf) -> Option> { + pub fn get_source(&self, name: &PathBuf) -> Option { let ro_map = &self.data().files; ro_map.get(name).map(|value| value.source_code.clone()) } @@ -205,7 +206,7 @@ self.add_file(file_path.clone(), contents)?; self.evaluate_loaded_file_raw(&file_path) } - pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result> { + pub(crate) fn import_file_str(&self, from: &PathBuf, path: &PathBuf) -> Result { let path = self.resolve_file(from, path)?; if !self.data().str_files.contains_key(&path) { let file_str = self.load_file_contents(&path)?; @@ -257,7 +258,7 @@ /// Creates context with all passed global variables pub fn create_default_context(&self) -> Result { let globals = &self.settings().globals; - let mut new_bindings: HashMap, LazyBinding> = HashMap::new(); + let mut new_bindings: HashMap = HashMap::new(); for (name, value) in globals.iter() { new_bindings.insert( name.clone(), @@ -321,13 +322,13 @@ out } - pub fn manifest(&self, val: Val) -> Result> { + pub fn manifest(&self, val: Val) -> Result { self.run_in_state(|| val.manifest(&self.manifest_format())) } - pub fn manifest_multi(&self, val: Val) -> Result, Rc)>> { + pub fn manifest_multi(&self, val: Val) -> Result> { self.run_in_state(|| val.manifest_multi(&self.manifest_format())) } - pub fn manifest_stream(&self, val: Val) -> Result>> { + pub fn manifest_stream(&self, val: Val) -> Result> { self.run_in_state(|| val.manifest_stream(&self.manifest_format())) } @@ -371,7 +372,7 @@ self.run_in_state(|| self.import_file(&PathBuf::from("."), name)) } /// Parses and evaluates the given snippet - pub fn evaluate_snippet_raw(&self, source: Rc, code: Rc) -> Result { + pub fn evaluate_snippet_raw(&self, source: Rc, code: IStr) -> Result { let parsed = parse( &code, &ParserSettings { @@ -391,26 +392,26 @@ /// Settings utilities impl EvaluationState { - pub fn add_ext_var(&self, name: Rc, value: Val) { + pub fn add_ext_var(&self, name: IStr, value: Val) { self.settings_mut().ext_vars.insert(name, value); } - pub fn add_ext_str(&self, name: Rc, value: Rc) { + pub fn add_ext_str(&self, name: IStr, value: IStr) { self.add_ext_var(name, Val::Str(value)); } - pub fn add_ext_code(&self, name: Rc, code: Rc) -> Result<()> { + pub fn add_ext_code(&self, name: IStr, code: IStr) -> Result<()> { let value = self.evaluate_snippet_raw(Rc::new(PathBuf::from(format!("ext_code {}", name))), code)?; self.add_ext_var(name, value); Ok(()) } - pub fn add_tla(&self, name: Rc, value: Val) { + pub fn add_tla(&self, name: IStr, value: Val) { self.settings_mut().tla_vars.insert(name, value); } - pub fn add_tla_str(&self, name: Rc, value: Rc) { + pub fn add_tla_str(&self, name: IStr, value: IStr) { self.add_tla(name, Val::Str(value)); } - pub fn add_tla_code(&self, name: Rc, code: Rc) -> Result<()> { + pub fn add_tla_code(&self, name: IStr, code: IStr) -> Result<()> { let value = self.evaluate_snippet_raw(Rc::new(PathBuf::from(format!("tla_code {}", name))), code)?; self.add_tla(name, value); @@ -420,7 +421,7 @@ pub fn resolve_file(&self, from: &PathBuf, path: &PathBuf) -> Result> { Ok(self.settings().import_resolver.resolve_file(from, path)?) } - pub fn load_file_contents(&self, path: &PathBuf) -> Result> { + pub fn load_file_contents(&self, path: &PathBuf) -> Result { Ok(self.settings().import_resolver.load_file_contents(path)?) } @@ -431,7 +432,7 @@ self.settings_mut().import_resolver = resolver; } - pub fn add_native(&self, name: Rc, cb: Rc) { + pub fn add_native(&self, name: IStr, cb: Rc) { self.settings_mut().ext_natives.insert(name, cb); } @@ -468,6 +469,7 @@ pub mod tests { use super::Val; use crate::{error::Error::*, primitive_equals, EvaluationState}; + use jrsonnet_interner::IStr; use jrsonnet_parser::*; use std::{path::PathBuf, rc::Rc}; @@ -905,13 +907,13 @@ Ok(()) } - struct TestImportResolver(Rc); + struct TestImportResolver(IStr); impl crate::import::ImportResolver for TestImportResolver { fn resolve_file(&self, _: &PathBuf, _: &PathBuf) -> crate::error::Result> { Ok(Rc::new(PathBuf::from("/test"))) } - fn load_file_contents(&self, _: &PathBuf) -> crate::error::Result> { + fn load_file_contents(&self, _: &PathBuf) -> crate::error::Result { Ok(self.0.clone()) } --- a/crates/jrsonnet-evaluator/src/obj.rs +++ b/crates/jrsonnet-evaluator/src/obj.rs @@ -1,5 +1,6 @@ use crate::{evaluate_add_op, LazyBinding, Result, Val}; use indexmap::IndexMap; +use jrsonnet_interner::IStr; use jrsonnet_parser::{ExprLocation, Visibility}; use std::{cell::RefCell, collections::HashMap, fmt::Debug, rc::Rc}; @@ -12,11 +13,11 @@ } // Field => This -type CacheKey = (Rc, usize); +type CacheKey = (IStr, usize); #[derive(Debug)] pub struct ObjValueInternals { super_obj: Option, - this_entries: Rc, ObjMember>>, + this_entries: Rc>, value_cache: RefCell>>, } #[derive(Clone)] @@ -33,7 +34,7 @@ } let mut debug = f.debug_struct("ObjValue"); for (name, member) in self.0.this_entries.iter() { - debug.field(name, member); + debug.field(&name, member); } #[cfg(feature = "unstable")] { @@ -47,7 +48,7 @@ } impl ObjValue { - pub fn new(super_obj: Option, this_entries: Rc, ObjMember>>) -> Self { + pub fn new(super_obj: Option, this_entries: Rc>) -> Self { Self(Rc::new(ObjValueInternals { super_obj, this_entries, @@ -63,7 +64,7 @@ Some(v) => Self::new(Some(v.with_super(super_obj)), self.0.this_entries.clone()), } } - pub fn enum_fields(&self, handler: &impl Fn(&Rc, &Visibility)) { + pub fn enum_fields(&self, handler: &impl Fn(&IStr, &Visibility)) { if let Some(s) = &self.0.super_obj { s.enum_fields(handler); } @@ -71,7 +72,7 @@ handler(name, &member.visibility); } } - pub fn fields_visibility(&self) -> IndexMap, bool> { + pub fn fields_visibility(&self) -> IndexMap { let out = Rc::new(RefCell::new(IndexMap::new())); self.enum_fields(&|name, visibility| { let mut out = out.borrow_mut(); @@ -91,7 +92,7 @@ }); Rc::try_unwrap(out).unwrap().into_inner() } - pub fn visible_fields(&self) -> Vec> { + pub fn visible_fields(&self) -> Vec { let mut visible_fields: Vec<_> = self .fields_visibility() .into_iter() @@ -101,10 +102,10 @@ visible_fields.sort(); visible_fields } - pub fn get(&self, key: Rc) -> Result> { + pub fn get(&self, key: IStr) -> Result> { Ok(self.get_raw(key, None)?) } - pub(crate) fn get_raw(&self, key: Rc, real_this: Option<&Self>) -> Result> { + pub(crate) fn get_raw(&self, key: IStr, real_this: Option<&Self>) -> Result> { let real_this = real_this.unwrap_or(self); let cache_key = (key.clone(), Rc::as_ptr(&real_this.0) as usize); --- a/crates/jrsonnet-evaluator/src/val.rs +++ b/crates/jrsonnet-evaluator/src/val.rs @@ -9,6 +9,7 @@ native::NativeCallback, throw, with_state, Context, ObjValue, Result, }; +use jrsonnet_interner::IStr; use jrsonnet_parser::{el, Arg, ArgsDesc, Expr, ExprLocation, LiteralType, LocExpr, ParamsDesc}; use jrsonnet_types::ValType; use std::{cell::RefCell, collections::HashMap, fmt::Debug, rc::Rc}; @@ -61,7 +62,7 @@ #[derive(Debug, PartialEq)] pub struct FuncDesc { - pub name: Rc, + pub name: IStr, pub ctx: Context, pub params: ParamsDesc, pub body: LocExpr, @@ -72,9 +73,9 @@ /// Plain function implemented in jsonnet Normal(FuncDesc), /// Standard library function - Intrinsic(Rc), + Intrinsic(IStr), /// Library functions implemented in native - NativeExt(Rc, Rc), + NativeExt(IStr, Rc), } impl PartialEq for FuncVal { @@ -91,7 +92,7 @@ pub fn is_ident(&self) -> bool { matches!(&self, Self::Intrinsic(n) if n as &str == "id") } - pub fn name(&self) -> Rc { + pub fn name(&self) -> IStr { match self { Self::Normal(normal) => normal.name.clone(), Self::Intrinsic(name) => format!("std.{}", name).into(), @@ -131,7 +132,7 @@ pub fn evaluate_map( &self, call_ctx: Context, - args: &HashMap, Val>, + args: &HashMap, tailstrict: bool, ) -> Result { match self { @@ -270,7 +271,7 @@ pub enum Val { Bool(bool), Null, - Str(Rc), + Str(IStr), Num(f64), Arr(ArrValue), Obj(ObjValue), @@ -314,7 +315,7 @@ self.assert_type(context, ValType::Bool)?; Ok(matches_unwrap!(self, Self::Bool(v), v)) } - pub fn try_cast_str(self, context: &'static str) -> Result> { + pub fn try_cast_str(self, context: &'static str) -> Result { self.assert_type(context, ValType::Str)?; Ok(matches_unwrap!(self, Self::Str(v), v)) } @@ -334,7 +335,7 @@ } } - pub fn to_string(&self) -> Result> { + pub fn to_string(&self) -> Result { Ok(match self { Self::Bool(true) => "true".into(), Self::Bool(false) => "false".into(), @@ -352,7 +353,7 @@ } /// Expects value to be object, outputs (key, manifested value) pairs - pub fn manifest_multi(&self, ty: &ManifestFormat) -> Result, Rc)>> { + pub fn manifest_multi(&self, ty: &ManifestFormat) -> Result> { let obj = match self { Self::Obj(obj) => obj, _ => throw!(MultiManifestOutputIsNotAObject), @@ -370,7 +371,7 @@ } /// Expects value to be array, outputs manifested values - pub fn manifest_stream(&self, ty: &ManifestFormat) -> Result>> { + pub fn manifest_stream(&self, ty: &ManifestFormat) -> Result> { let arr = match self { Self::Arr(a) => a, _ => throw!(StreamManifestOutputIsNotAArray), @@ -382,7 +383,7 @@ Ok(out) } - pub fn manifest(&self, ty: &ManifestFormat) -> Result> { + pub fn manifest(&self, ty: &ManifestFormat) -> Result { Ok(match ty { ManifestFormat::YamlStream(format) => { let arr = match self { @@ -419,7 +420,7 @@ } /// For manifestification - pub fn to_json(&self, padding: usize) -> Result> { + pub fn to_json(&self, padding: usize) -> Result { manifest_json_ex( self, &ManifestJsonOptions { @@ -471,7 +472,7 @@ .try_cast_str("to json")?) }) } - pub fn to_yaml(&self, padding: usize) -> Result> { + pub fn to_yaml(&self, padding: usize) -> Result { with_state(|s| { let ctx = s .create_default_context()? --- /dev/null +++ b/crates/jrsonnet-interner/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock --- /dev/null +++ b/crates/jrsonnet-interner/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "jrsonnet-interner" +version = "0.3.3" +authors = ["Yaroslav Bolyukin "] +edition = "2018" + +[dependencies] +serde = { version = "1.0" } +rustc-hash = "1.1.0" --- /dev/null +++ b/crates/jrsonnet-interner/src/lib.rs @@ -0,0 +1,104 @@ +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use std::{ + cell::RefCell, + fmt::{self, Display}, + hash::{BuildHasherDefault, Hash, Hasher}, + ops::Deref, + rc::Rc, +}; + +#[derive(Clone, PartialOrd, Ord, Eq)] +pub struct IStr(Rc); + +impl Deref for IStr { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for IStr { + fn eq(&self, other: &Self) -> bool { + // It is ok, since all IStr should be inlined into same pool + Rc::ptr_eq(&self.0, &other.0) + } +} + +impl PartialEq for IStr { + fn eq(&self, other: &str) -> bool { + &self.0 as &str == other + } +} + +impl Hash for IStr { + fn hash(&self, state: &mut H) { + state.write_usize(Rc::as_ptr(&self.0) as *const () as usize) + } +} + +impl Drop for IStr { + fn drop(&mut self) { + // First reference - current object, second - POOL + if Rc::strong_count(&self.0) <= 2 { + STR_POOL.with(|pool| pool.borrow_mut().remove(&self.0)); + } + } +} + +impl fmt::Debug for IStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", &self.0) + } +} + +impl Display for IStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +thread_local! { + static STR_POOL: RefCell, ()>> = RefCell::new(FxHashMap::with_capacity_and_hasher(200, BuildHasherDefault::default())); +} + +impl From<&str> for IStr { + fn from(str: &str) -> Self { + IStr(STR_POOL.with(|pool| { + let mut pool = pool.borrow_mut(); + if let Some((k, _)) = pool.get_key_value(str) { + return k.clone(); + } else { + let rc: Rc = str.into(); + pool.insert(rc.clone(), ()); + rc + } + })) + } +} + +impl From for IStr { + fn from(str: String) -> Self { + (&str as &str).into() + } +} + +impl Serialize for IStr { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + (&self.0 as &str).serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for IStr { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = <&str>::deserialize(deserializer)?; + Ok(s.into()) + } +} --- a/crates/jrsonnet-parser/Cargo.toml +++ b/crates/jrsonnet-parser/Cargo.toml @@ -10,16 +10,14 @@ default = [] serialize = ["serde"] deserialize = ["serde"] -# Adds ability to dump AST as source code for easy embedding -dump = ["structdump", "structdump-derive"] [dependencies] +jrsonnet-interner = { path = "../jrsonnet-interner" } + peg = "0.6.3" unescape = "0.1.0" serde = { version = "1.0", features = ["derive", "rc"], optional = true } -structdump = { version = "0.1.2", optional = true } -structdump-derive = { version = "0.1.2", optional = true } [dev-dependencies] jrsonnet-stdlib = { path = "../jrsonnet-stdlib", version = "0.3.3" } --- a/crates/jrsonnet-parser/src/expr.rs +++ b/crates/jrsonnet-parser/src/expr.rs @@ -1,3 +1,4 @@ +use jrsonnet_interner::IStr; #[cfg(feature = "deserialize")] use serde::Deserialize; #[cfg(feature = "serialize")] @@ -8,21 +9,17 @@ path::PathBuf, rc::Rc, }; -#[cfg(feature = "dump")] -use structdump_derive::Codegen; -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] pub enum FieldName { /// {fixed: 2} - Fixed(Rc), + Fixed(IStr), /// {["dyn"+"amic"]: 3} Dyn(LocExpr), } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, Clone, Copy, PartialEq)] @@ -35,13 +32,11 @@ Unhide, } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] pub struct AssertStmt(pub LocExpr, pub Option); -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -53,7 +48,6 @@ pub value: LocExpr, } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -63,7 +57,6 @@ AssertStmt(AssertStmt), } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, Clone, Copy, PartialEq)] @@ -89,7 +82,6 @@ } } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, Clone, Copy, PartialEq)] @@ -147,14 +139,12 @@ } /// name, default value -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] -pub struct Param(pub Rc, pub Option); +pub struct Param(pub IStr, pub Option); /// Defined function parameters -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, Clone, PartialEq)] @@ -166,13 +156,11 @@ } } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] pub struct Arg(pub Option, pub LocExpr); -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -184,29 +172,25 @@ } } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, Clone, PartialEq)] pub struct BindSpec { - pub name: Rc, + pub name: IStr, pub params: Option, pub value: LocExpr, } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] pub struct IfSpecData(pub LocExpr); -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] -pub struct ForSpecData(pub Rc, pub LocExpr); +pub struct ForSpecData(pub IStr, pub LocExpr); -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -215,7 +199,6 @@ ForSpec(ForSpecData), } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -227,7 +210,6 @@ pub compspecs: Vec, } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -236,7 +218,6 @@ ObjComp(ObjComp), } -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq, Clone, Copy)] @@ -257,7 +238,6 @@ } /// Syntax base -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Debug, PartialEq)] @@ -265,11 +245,11 @@ Literal(LiteralType), /// String value: "hello" - Str(Rc), + Str(IStr), /// Number: 1, 2.0, 2e+20 Num(f64), /// Variable name: test - Var(Rc), + Var(IStr), /// Array of expressions: [1, 2, "Hello"] Arr(Vec), @@ -316,7 +296,7 @@ /// function(x) x Function(ParamsDesc, LocExpr), /// std.primitiveEquals - Intrinsic(Rc), + Intrinsic(IStr), /// if true == false then 1 else 2 IfElse { cond: IfSpecData, @@ -326,7 +306,6 @@ } /// file, begin offset, end offset -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Clone, PartialEq)] @@ -338,7 +317,6 @@ } /// Holds AST expression and its location in source file -#[cfg_attr(feature = "dump", derive(Codegen))] #[cfg_attr(feature = "serialize", derive(Serialize))] #[cfg_attr(feature = "deserialize", derive(Deserialize))] #[derive(Clone, PartialEq)] -- gitstuff