--- a/cmds/jrsonnet/Cargo.toml +++ b/cmds/jrsonnet/Cargo.toml @@ -11,6 +11,10 @@ workspace = true [features] +default = [ + "exp-regex", +] + experimental = [ "exp-preserve-order", "exp-destruct", @@ -18,7 +22,6 @@ "exp-object-iteration", "exp-bigint", "exp-apply", - "exp-regex", ] # Use mimalloc as allocator mimalloc = ["mimallocator"] --- a/crates/jrsonnet-evaluator/src/arr/mod.rs +++ b/crates/jrsonnet-evaluator/src/arr/mod.rs @@ -5,11 +5,11 @@ rc::Rc, }; -use jrsonnet_gcmodule::{cc_dyn, Cc}; +use jrsonnet_gcmodule::{cc_dyn, Cc, Trace}; use jrsonnet_interner::IBytes; use jrsonnet_parser::{Expr, Spanned}; -use crate::{function::NativeFn, Context, Result, Thunk, Val}; +use crate::{function::NativeFn, typed::Typed, Context, Result, Thunk, Val}; mod spec; pub use spec::{ArrayLike, *}; @@ -241,3 +241,4 @@ self.0.is_cheap() } } + --- a/crates/jrsonnet-evaluator/src/obj/mod.rs +++ b/crates/jrsonnet-evaluator/src/obj/mod.rs @@ -12,11 +12,12 @@ use educe::Educe; use jrsonnet_gcmodule::{cc_dyn, Acyclic, Cc, Trace, Weak}; use jrsonnet_interner::IStr; -use jrsonnet_parser::{Span, Visibility}; +use jrsonnet_parser::Span; use rustc_hash::{FxHashMap, FxHashSet}; mod oop; +pub use jrsonnet_parser::Visibility; pub use oop::ObjValueBuilder; use crate::{ @@ -30,7 +31,7 @@ }; #[cfg(not(feature = "exp-preserve-order"))] -mod ordering { +pub mod ordering { #![allow( // This module works as stub for preserve-order feature clippy::unused_self, @@ -41,6 +42,9 @@ #[derive(Clone, Copy, Default, Debug, Trace)] pub struct FieldIndex(()); impl FieldIndex { + pub fn absolute(_v: u32) -> Self { + Self(()) + } pub const fn next(self) -> Self { Self(()) } @@ -54,7 +58,7 @@ } #[cfg(feature = "exp-preserve-order")] -mod ordering { +pub mod ordering { use std::cmp::Reverse; use jrsonnet_gcmodule::Trace; @@ -62,6 +66,9 @@ #[derive(Clone, Copy, Default, Debug, Trace, PartialEq, Eq, PartialOrd, Ord)] pub struct FieldIndex(u32); impl FieldIndex { + pub fn absolute(v: u32) -> Self { + Self(v) + } pub fn next(self) -> Self { Self(self.0 + 1) } @@ -149,7 +156,7 @@ Pending, } -type EnumFieldsHandler<'a> = +pub type EnumFieldsHandler<'a> = dyn FnMut(SuperDepth, FieldIndex, IStr, EnumFields) -> ControlFlow<()> + 'a; pub enum EnumFields { --- a/crates/jrsonnet-interner/src/lib.rs +++ b/crates/jrsonnet-interner/src/lib.rs @@ -21,6 +21,8 @@ mod inner; use inner::Inner; +mod names; + /// Interned string /// /// Provides O(1) comparsions and hashing, cheap copy, and cheap conversion to [`IBytes`] --- a/crates/jrsonnet-macros/src/lib.rs +++ b/crates/jrsonnet-macros/src/lib.rs @@ -13,6 +13,11 @@ LitStr, Meta, Pat, Path, PathArguments, Result, ReturnType, Token, Type, }; +use self::typed::derive_typed_inner; + +mod typed; +mod names; + fn try_parse_attr_noargs(attrs: &[Attribute], ident: I) -> Result where Ident: PartialEq, @@ -435,278 +440,6 @@ } }; }) -} - -#[derive(Default)] -#[allow(clippy::struct_excessive_bools)] -struct TypedAttr { - rename: Option, - aliases: Vec, - flatten: bool, - /// flatten(ok) strategy for flattened optionals - /// field would be None in case of any parsing error (as in serde) - flatten_ok: bool, - // Should it be `field+:` instead of `field:` - add: bool, - // Should it be `field::` instead of `field:` - hide: bool, -} -impl Parse for TypedAttr { - fn parse(input: ParseStream) -> syn::Result { - let mut out = Self::default(); - loop { - let lookahead = input.lookahead1(); - if lookahead.peek(kw::rename) { - input.parse::()?; - input.parse::()?; - let name = input.parse::()?; - if out.rename.is_some() { - return Err(Error::new( - name.span(), - "rename attribute may only be specified once", - )); - } - out.rename = Some(name.value()); - } else if lookahead.peek(kw::alias) { - input.parse::()?; - input.parse::()?; - let alias = input.parse::()?; - out.aliases.push(alias.value()); - } else if lookahead.peek(kw::flatten) { - input.parse::()?; - out.flatten = true; - if input.peek(token::Paren) { - let content; - parenthesized!(content in input); - let lookahead = content.lookahead1(); - if lookahead.peek(kw::ok) { - content.parse::()?; - out.flatten_ok = true; - } else { - return Err(lookahead.error()); - } - } - } else if lookahead.peek(kw::add) { - input.parse::()?; - out.add = true; - } else if lookahead.peek(kw::hide) { - input.parse::()?; - out.hide = true; - } else if input.is_empty() { - break; - } else { - return Err(lookahead.error()); - } - if input.peek(Token![,]) { - input.parse::()?; - } else { - break; - } - } - Ok(out) - } -} - -struct TypedField { - attr: TypedAttr, - ident: Ident, - ty: Type, - is_option: bool, - is_lazy: bool, -} -impl TypedField { - fn parse(field: &syn::Field) -> Result { - let attr = parse_attr::(&field.attrs, "typed")?.unwrap_or_default(); - let Some(ident) = field.ident.clone() else { - return Err(Error::new( - field.span(), - "this field should appear in output object, but it has no visible name", - )); - }; - let (is_option, ty) = extract_type_from_option(&field.ty)? - .map_or_else(|| (false, field.ty.clone()), |ty| (true, ty.clone())); - if is_option && attr.flatten { - if !attr.flatten_ok { - return Err(Error::new( - field.span(), - "strategy should be set when flattening Option", - )); - } - } else if attr.flatten_ok { - return Err(Error::new( - field.span(), - "flatten(ok) is only useable on optional fields", - )); - } - - let is_lazy = type_is_path(&ty, "Thunk").is_some(); - - Ok(Self { - attr, - ident, - ty, - is_option, - is_lazy, - }) - } - /// None if this field is flattened in jsonnet output - fn name(&self) -> Option { - if self.attr.flatten { - return None; - } - Some( - self.attr - .rename - .clone() - .unwrap_or_else(|| self.ident.to_string()), - ) - } - - fn expand_field(&self) -> Option { - if self.is_option { - return None; - } - let name = self.name()?; - let ty = &self.ty; - Some(quote! { - (#name, <#ty as Typed>::TYPE) - }) - } - - fn expand_parse(&self) -> TokenStream { - if self.is_option { - self.expand_parse_optional() - } else { - self.expand_parse_mandatory() - } - } - - fn expand_parse_optional(&self) -> TokenStream { - let ident = &self.ident; - let ty = &self.ty; - - // optional flatten is handled in same way as serde - if self.attr.flatten { - return quote! { - #ident: <#ty as TypedObj>::parse(&obj).ok(), - }; - } - - let name = self.name().unwrap(); - let aliases = &self.attr.aliases; - - quote! { - #ident: { - let __value = if let Some(__v) = obj.get(#name.into())? { - Some(__v) - } #(else if let Some(__v) = obj.get(#aliases.into())? { - Some(__v) - })* else { - None - }; - - __value.map(<#ty as Typed>::from_untyped).transpose()? - }, - } - } - - fn expand_parse_mandatory(&self) -> TokenStream { - let ident = &self.ident; - let ty = &self.ty; - - // optional flatten is handled in same way as serde - if self.attr.flatten { - return quote! { - #ident: <#ty as TypedObj>::parse(&obj)?, - }; - } - - let name = self.name().unwrap(); - let aliases = &self.attr.aliases; - - let error_text = if aliases.is_empty() { - // clippy does not understand name variable usage in quote! macro - #[allow(clippy::redundant_clone)] - name.clone() - } else { - format!("{name} (alias {})", aliases.join(", ")) - }; - - quote! { - #ident: { - let __value = if let Some(__v) = obj.get(#name.into())? { - __v - } #(else if let Some(__v) = obj.get(#aliases.into())? { - __v - })* else { - return Err(ErrorKind::NoSuchField(#error_text.into(), vec![]).into()); - }; - - <#ty as Typed>::from_untyped(__value)? - }, - } - } - - fn expand_serialize(&self) -> TokenStream { - let ident = &self.ident; - let ty = &self.ty; - self.name().map_or_else( - || { - if self.is_option { - quote! { - if let Some(value) = self.#ident { - <#ty as TypedObj>::serialize(value, out)?; - } - } - } else { - quote! { - <#ty as TypedObj>::serialize(self.#ident, out)?; - } - } - }, - |name| { - let hide = if self.attr.hide { - quote! {.hide()} - } else { - quote! {} - }; - let add = if self.attr.add { - quote! {.add()} - } else { - quote! {} - }; - let value = if self.is_lazy { - quote! { - out.field(#name) - #hide - #add - .try_thunk(<#ty as Typed>::into_lazy_untyped(value))?; - } - } else { - quote! { - out.field(#name) - #hide - #add - .try_value(<#ty as Typed>::into_untyped(value)?)?; - } - }; - if self.is_option { - quote! { - if let Some(value) = self.#ident { - #value - } - } - } else { - quote! { - { - let value = self.#ident; - #value - } - } - } - }, - ) - } } #[proc_macro_derive(Typed, attributes(typed))] @@ -717,79 +450,6 @@ Ok(v) => v.into(), Err(e) => e.to_compile_error().into(), } -} - -fn derive_typed_inner(input: DeriveInput) -> Result { - let syn::Data::Struct(data) = &input.data else { - return Err(Error::new(input.span(), "only structs supported")); - }; - - let ident = &input.ident; - let fields = data - .fields - .iter() - .map(TypedField::parse) - .collect::>>()?; - - let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); - - let typed = { - let fields = fields - .iter() - .filter_map(TypedField::expand_field) - .collect::>(); - quote! { - impl #impl_generics Typed for #ident #ty_generics #where_clause { - const TYPE: &'static ComplexValType = &ComplexValType::ObjectRef(&[ - #(#fields,)* - ]); - - fn from_untyped(value: Val) -> JrResult { - let obj = value.as_obj().expect("shape is correct"); - Self::parse(&obj) - } - - fn into_untyped(value: Self) -> JrResult { - let mut out = ObjValueBuilder::new(); - value.serialize(&mut out)?; - Ok(Val::Obj(out.build())) - } - - } - } - }; - - let fields_parse = fields.iter().map(TypedField::expand_parse); - let fields_serialize = fields - .iter() - .map(TypedField::expand_serialize) - .collect::>(); - - Ok(quote! { - const _: () = { - use ::jrsonnet_evaluator::{ - typed::{ComplexValType, Typed, TypedObj, CheckType}, - Val, State, - error::{ErrorKind, Result as JrResult}, - ObjValueBuilder, ObjValue, - }; - - #typed - - impl #impl_generics TypedObj for #ident #ty_generics #where_clause { - fn serialize(self, out: &mut ObjValueBuilder) -> JrResult<()> { - #(#fields_serialize)* - - Ok(()) - } - fn parse(obj: &ObjValue) -> JrResult { - Ok(Self { - #(#fields_parse)* - }) - } - } - }; - }) } struct FormatInput { --- /dev/null +++ b/crates/jrsonnet-macros/src/names.rs @@ -0,0 +1,32 @@ +use proc_macro2::TokenStream; +use quote::quote; +use std::cell::RefCell; + +#[derive(Default)] +pub struct Names { + names: Vec, +} + +impl Names { + pub fn intern(&mut self, s: impl AsRef) -> usize { + let s = s.as_ref(); + if let Some(pos) = self.names.iter().position(|v| v == s) { + return pos; + } + let pos = self.names.len(); + self.names.push(s.to_owned()); + pos + } + + pub fn expand(&self) -> TokenStream { + let len = self.names.len(); + let name = self.names.iter(); + quote! { + thread_local! { + static NAMES: [::jrsonnet_evaluator::IStr; #len] = [ + #(::jrsonnet_evaluator::IStr::from(#name),)* + ]; + } + } + } +} --- /dev/null +++ b/crates/jrsonnet-macros/src/typed.rs @@ -0,0 +1,373 @@ +use crate::names::Names; +use crate::{extract_type_from_option, kw, parse_attr, type_is_path}; +use proc_macro2::TokenStream; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned as _; +use syn::{parenthesized, token, DeriveInput, Error, Ident, LitStr, Result, Token, Type}; + +#[derive(Default)] +#[allow(clippy::struct_excessive_bools)] +struct TypedAttr { + rename: Option, + aliases: Vec, + flatten: bool, + /// flatten(ok) strategy for flattened optionals + /// field would be None in case of any parsing error (as in serde) + flatten_ok: bool, + // Should it be `field+:` instead of `field:` + add: bool, + // Should it be `field::` instead of `field:` + hide: bool, +} +impl Parse for TypedAttr { + fn parse(input: ParseStream) -> syn::Result { + let mut out = Self::default(); + loop { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::rename) { + input.parse::()?; + input.parse::()?; + let name = input.parse::()?; + if out.rename.is_some() { + return Err(Error::new( + name.span(), + "rename attribute may only be specified once", + )); + } + out.rename = Some(name.value()); + } else if lookahead.peek(kw::alias) { + input.parse::()?; + input.parse::()?; + let alias = input.parse::()?; + out.aliases.push(alias.value()); + } else if lookahead.peek(kw::flatten) { + input.parse::()?; + out.flatten = true; + if input.peek(token::Paren) { + let content; + parenthesized!(content in input); + let lookahead = content.lookahead1(); + if lookahead.peek(kw::ok) { + content.parse::()?; + out.flatten_ok = true; + } else { + return Err(lookahead.error()); + } + } + } else if lookahead.peek(kw::add) { + input.parse::()?; + out.add = true; + } else if lookahead.peek(kw::hide) { + input.parse::()?; + out.hide = true; + } else if input.is_empty() { + break; + } else { + return Err(lookahead.error()); + } + if input.peek(Token![,]) { + input.parse::()?; + } else { + break; + } + } + Ok(out) + } +} +struct TypedField { + attr: TypedAttr, + ident: Ident, + ty: Type, + is_option: bool, + is_lazy: bool, +} +impl TypedField { + fn parse(field: &syn::Field) -> Result { + let attr = parse_attr::(&field.attrs, "typed")?.unwrap_or_default(); + let Some(ident) = field.ident.clone() else { + return Err(Error::new( + field.span(), + "this field should appear in output object, but it has no visible name", + )); + }; + let (is_option, ty) = extract_type_from_option(&field.ty)? + .map_or_else(|| (false, field.ty.clone()), |ty| (true, ty.clone())); + if is_option && attr.flatten { + if !attr.flatten_ok { + return Err(Error::new( + field.span(), + "strategy should be set when flattening Option", + )); + } + } else if attr.flatten_ok { + return Err(Error::new( + field.span(), + "flatten(ok) is only useable on optional fields", + )); + } + + let is_lazy = type_is_path(&ty, "Thunk").is_some(); + + Ok(Self { + attr, + ident, + ty, + is_option, + is_lazy, + }) + } + /// None if this field is flattened in jsonnet output + fn name(&self) -> Option { + if self.attr.flatten { + return None; + } + Some( + self.attr + .rename + .clone() + .unwrap_or_else(|| self.ident.to_string()), + ) + } + + fn expand_field(&self) -> Option { + if self.is_option { + return None; + } + let name = self.name()?; + let ty = &self.ty; + Some(quote! { + (#name, <#ty as Typed>::TYPE) + }) + } + + fn expand_parse(&self, names: &mut Names) -> TokenStream { + if self.is_option { + self.expand_parse_optional(names) + } else { + self.expand_parse_mandatory(names) + } + } + + fn expand_parse_optional(&self, names: &mut Names) -> TokenStream { + let ident = &self.ident; + let ty = &self.ty; + + // optional flatten is handled in same way as serde + if self.attr.flatten { + return quote! { + #ident: <#ty as TypedObj>::parse(&obj).ok(), + }; + } + + let name = names.intern(self.name().unwrap()); + let aliases = self + .attr + .aliases + .iter() + .map(|name| names.intern(name)) + .collect::>(); + + quote! { + #ident: { + let __value = if let Some(__v) = obj.get(__names[#name].clone())? { + Some(__v) + } #(else if let Some(__v) = obj.get(__names[#aliases].clone())? { + Some(__v) + })* else { + None + }; + + __value.map(<#ty as Typed>::from_untyped).transpose()? + }, + } + } + + fn expand_parse_mandatory(&self, names: &mut Names) -> TokenStream { + let ident = &self.ident; + let ty = &self.ty; + + // optional flatten is handled in same way as serde + if self.attr.flatten { + return quote! { + #ident: <#ty as TypedObj>::parse(&obj)?, + }; + } + + let name = self.name().unwrap(); + let aliases = &self.attr.aliases; + + let error_text = if aliases.is_empty() { + // clippy does not understand name variable usage in quote! macro + #[allow(clippy::redundant_clone)] + name.clone() + } else { + format!("{name} (alias {})", aliases.join(", ")) + }; + + let error_text = names.intern(error_text); + let name = names.intern(name); + let aliases = aliases.iter().map(|alias| names.intern(alias)); + + quote! { + #ident: { + let __value = if let Some(__v) = obj.get(__names[#name].clone())? { + __v + } #(else if let Some(__v) = obj.get(__names[#aliases].clone())? { + __v + })* else { + return Err(ErrorKind::NoSuchField(__names[#error_text].clone(), vec![]).into()); + }; + + <#ty as Typed>::from_untyped(__value)? + }, + } + } + + fn expand_serialize(&self, names: &mut Names) -> TokenStream { + let ident = &self.ident; + let ty = &self.ty; + self.name().map_or_else( + || { + if self.is_option { + quote! { + if let Some(value) = self.#ident { + <#ty as TypedObj>::serialize(value, out)?; + } + } + } else { + quote! { + <#ty as TypedObj>::serialize(self.#ident, out)?; + } + } + }, + |name| { + let name = names.intern(name); + let hide = if self.attr.hide { + quote! {.hide()} + } else { + quote! {} + }; + let add = if self.attr.add { + quote! {.add()} + } else { + quote! {} + }; + let value = if self.is_lazy { + quote! { + out.field(__names[#name].clone()) + #hide + #add + .try_thunk(<#ty as Typed>::into_lazy_untyped(value))?; + } + } else { + quote! { + out.field(__names[#name].clone()) + #hide + #add + .try_value(<#ty as Typed>::into_untyped(value)?)?; + } + }; + if self.is_option { + quote! { + if let Some(value) = self.#ident { + #value + } + } + } else { + quote! { + { + let value = self.#ident; + #value + } + } + } + }, + ) + } +} + +pub fn derive_typed_inner(input: DeriveInput) -> Result { + let syn::Data::Struct(data) = &input.data else { + return Err(Error::new(input.span(), "only structs supported")); + }; + + let ident = &input.ident; + let fields = data + .fields + .iter() + .map(TypedField::parse) + .collect::>>()?; + + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + + let capacity = fields.len(); + + let typed = { + let fields = fields + .iter() + .filter_map(TypedField::expand_field) + .collect::>(); + quote! { + impl #impl_generics Typed for #ident #ty_generics #where_clause { + const TYPE: &'static ComplexValType = &ComplexValType::ObjectRef(&[ + #(#fields,)* + ]); + + fn from_untyped(value: Val) -> JrResult { + let obj = value.as_obj().expect("shape is correct"); + Self::parse(&obj) + } + + fn into_untyped(value: Self) -> JrResult { + let mut out = ObjValueBuilder::with_capacity(#capacity); + value.serialize(&mut out)?; + Ok(Val::Obj(out.build())) + } + + } + } + }; + + let mut names = Names::default(); + + let fields_parse = fields + .iter() + .map(|f| f.expand_parse(&mut names)) + .collect::>(); + let fields_serialize = fields + .iter() + .map(|f| f.expand_serialize(&mut names)) + .collect::>(); + + let names_expanded = names.expand(); + Ok(quote! { + const _: () = { + use ::jrsonnet_evaluator::{ + typed::{ComplexValType, Typed, TypedObj, CheckType}, + Val, State, + error::{ErrorKind, Result as JrResult}, + ObjValueBuilder, ObjValue, IStr, + }; + + #typed + + #names_expanded + + impl #impl_generics TypedObj for #ident #ty_generics #where_clause { + fn serialize(self, out: &mut ObjValueBuilder) -> JrResult<()> { + NAMES.with(|__names| { + #(#fields_serialize)* + + Ok(()) + }) + } + fn parse(obj: &ObjValue) -> JrResult { + NAMES.with(|__names| Ok(Self { + #(#fields_parse)* + })) + } + } + }; + }) +} --- a/crates/jrsonnet-stdlib/src/regex.rs +++ b/crates/jrsonnet-stdlib/src/regex.rs @@ -4,8 +4,9 @@ use jrsonnet_evaluator::{ error::{ErrorKind::*, Result}, rustc_hash::FxBuildHasher, + typed::Typed, val::StrValue, - IStr, ObjValueBuilder, Val, + IStr, ObjValue, ObjValueBuilder, }; use jrsonnet_gcmodule::Acyclic; use jrsonnet_macros::builtin; @@ -20,7 +21,7 @@ Self { cache: RefCell::new(LruCache::with_hasher( NonZeroUsize::new(20).unwrap(), - FxBuildHasher::default(), + FxBuildHasher, )), } } @@ -40,21 +41,27 @@ } } -pub fn regex_match_inner(regex: &Regex, str: String) -> Result { - let mut out = ObjValueBuilder::with_capacity(3); +#[derive(Typed)] +pub struct RegexMatch { + string: IStr, + captures: Vec, + #[typed(rename = "namedCaptures")] + named_captures: ObjValue, +} +fn regex_match_inner(regex: &Regex, str: String) -> Result> { let mut captures = Vec::with_capacity(regex.captures_len()); let mut named_captures = ObjValueBuilder::with_capacity(regex.capture_names().len()); let Some(captured) = regex.captures(&str) else { - return Ok(Val::Null); + return Ok(None); }; for ele in captured.iter().skip(1) { if let Some(ele) = ele { - captures.push(Val::Str(StrValue::Flat(ele.as_str().into()))); + captures.push(ele.as_str().into()); } else { - captures.push(Val::Str(StrValue::Flat(IStr::empty()))); + captures.push(IStr::empty()); } } for (i, name) in regex @@ -67,13 +74,11 @@ named_captures.field(name).try_value(capture)?; } - out.field("string") - .value(Val::Str(captured.get(0).unwrap().as_str().into())); - out.field("captures").value(Val::Arr(captures.into())); - out.field("namedCaptures") - .value(Val::Obj(named_captures.build())); - - Ok(Val::Obj(out.build())) + Ok(Some(RegexMatch { + string: captured.get(0).expect("regex matched").as_str().into(), + named_captures: named_captures.build(), + captures, + })) } #[builtin(fields( @@ -83,7 +88,7 @@ this: &builtin_regex_partial_match, pattern: IStr, str: String, -) -> Result { +) -> Result> { let regex = this.cache.parse(pattern)?; regex_match_inner(®ex, str) } @@ -95,7 +100,7 @@ this: &builtin_regex_full_match, pattern: StrValue, str: String, -) -> Result { +) -> Result> { let pattern = format!("^{pattern}$").into(); let regex = this.cache.parse(pattern)?; regex_match_inner(®ex, str)