difftreelog
refactor cleanup
in: master
9 files changed
bindings/jsonnet/src/lib.rsdiffbeforeafterboth--- a/bindings/jsonnet/src/lib.rs
+++ b/bindings/jsonnet/src/lib.rs
@@ -45,8 +45,8 @@
/// If this does not match `LIB_JSONNET_VERSION`
/// then there is a mismatch between header and compiled library.
#[no_mangle]
-pub extern "C" fn jsonnet_version() -> &'static [u8; 8] {
- b"v0.20.0\0"
+pub extern "C" fn jsonnet_version() -> &'static [u8; 12] {
+ b"v0.22.0-rc1\0"
}
unsafe fn parse_path(input: &CStr) -> Cow<'_, Path> {
crates/jrsonnet-evaluator/src/async_import.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/async_import.rs
+++ b/crates/jrsonnet-evaluator/src/async_import.rs
@@ -3,11 +3,7 @@
use jrsonnet_gcmodule::Acyclic;
use jrsonnet_ir::visit::Visitor;
-use jrsonnet_ir::{
- ArgsDesc, AssertExpr, AssertStmt, BindSpec, CompSpec, Destruct, Expr, ExprParam, ExprParams,
- FieldMember, FieldName, ForSpecData, IStr, IfElse, IfSpecData, ImportKind, ObjBody, Slice,
- SliceDesc, Source, SourcePath, Spanned,
-};
+use jrsonnet_ir::{IStr, Source, SourcePath};
use rustc_hash::FxHashMap;
use crate::{AsPathLike, FileData, ImportResolver, ResolvePathOwned, State};
@@ -23,7 +19,7 @@
self.0.push(Import {
path: ResolvePathOwned::Str(value.to_string()),
expression,
- })
+ });
}
}
crates/jrsonnet-evaluator/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/lib.rs
+++ b/crates/jrsonnet-evaluator/src/lib.rs
@@ -5,7 +5,7 @@
extern crate self as jrsonnet_evaluator;
mod arr;
-// pub mod async_import;
+pub mod async_import;
mod ctx;
mod dynamic;
pub mod error;
crates/jrsonnet-evaluator/src/obj/mod.rsdiffbeforeafterboth--- a/crates/jrsonnet-evaluator/src/obj/mod.rs
+++ b/crates/jrsonnet-evaluator/src/obj/mod.rs
@@ -1,5 +1,13 @@
use std::{
- any::Any, cell::{Cell, RefCell}, clone::Clone, cmp::Reverse, collections::hash_map::Entry, fmt::{self, Debug}, hash::{Hash, Hasher}, num::Saturating, ops::ControlFlow
+ any::Any,
+ cell::{Cell, RefCell},
+ clone::Clone,
+ cmp::Reverse,
+ collections::hash_map::Entry,
+ fmt::{self, Debug},
+ hash::{Hash, Hasher},
+ num::Saturating,
+ ops::ControlFlow,
};
use educe::Educe;
crates/jrsonnet-ir-parser/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-ir-parser/src/lib.rs
+++ b/crates/jrsonnet-ir-parser/src/lib.rs
@@ -3,9 +3,9 @@
use jrsonnet_gcmodule::Acyclic;
use jrsonnet_ir::{
unescape, ArgsDesc, AssertExpr, AssertStmt, BinaryOp, BinaryOpType, BindSpec, CompSpec,
- Destruct, DestructRest, Expr, ExprParam, ExprParams, FieldMember, FieldName, ForSpecData, IStr,
- IfElse, IfSpecData, ImportKind, IndexPart, LiteralType, Member, ObjBody, ObjComp, ObjMembers,
- Slice, SliceDesc, Source, Span, Spanned, UnaryOpType, Visibility,
+ Destruct, Expr, ExprParam, ExprParams, FieldMember, FieldName, ForSpecData, IStr, IfElse,
+ IfSpecData, ImportKind, IndexPart, LiteralType, Member, ObjBody, ObjComp, ObjMembers, Slice,
+ SliceDesc, Source, Span, Spanned, UnaryOpType, Visibility,
};
use jrsonnet_lexer::{collect_lexed_str_block, Lexeme, Lexer, SyntaxKind, T};
@@ -30,7 +30,7 @@
}
}
-type R<T> = Result<T, ParseError>;
+type Result<T> = std::result::Result<T, ParseError>;
struct Parser<'a> {
lexemes: Vec<Lexeme<'a>>,
@@ -103,7 +103,7 @@
}
}
- fn eat(&mut self, t: SyntaxKind) -> R<()> {
+ fn eat(&mut self, t: SyntaxKind) -> Result<()> {
if !self.at(t) {
return Err(self.error(format!(
"expected {}, got {}",
@@ -138,15 +138,13 @@
}
}
- fn expect_ident(&mut self) -> R<IStr> {
+ fn expect_ident(&mut self) -> Result<IStr> {
if !self.at(SyntaxKind::IDENT) {
return Err(self.error(format!("expected identifier, got {}", self.current_desc())));
}
let text = self.text();
if is_reserved(text) {
- return Err(self.error(format!(
- "expected identifier, got reserved word '{text}'"
- )));
+ return Err(self.error(format!("expected identifier, got reserved word '{text}'")));
}
let s: IStr = text.into();
self.eat_any();
@@ -164,36 +162,38 @@
"assert"
| "else" | "error"
| "false" | "for"
- | "function" | "if"
- | "import" | "importstr"
- | "importbin" | "in"
- | "local" | "null"
- | "tailstrict" | "then"
- | "self" | "super"
- | "true"
+ | "function"
+ | "if" | "import"
+ | "importstr"
+ | "importbin"
+ | "in" | "local"
+ | "null" | "tailstrict"
+ | "then" | "self"
+ | "super" | "true"
)
}
-fn spanned<T: Acyclic>(p: &mut Parser<'_>, cb: impl FnOnce(&mut Parser<'_>) -> R<T>) -> R<Spanned<T>> {
+fn spanned<T: Acyclic>(
+ p: &mut Parser<'_>,
+ cb: impl FnOnce(&mut Parser<'_>) -> Result<T>,
+) -> Result<Spanned<T>> {
let start = p.span_start();
let v = cb(p)?;
let end = p.span_end();
Ok(Spanned::new(v, Span(p.source.clone(), start, end)))
}
-fn parse_string_content(p: &mut Parser<'_>) -> R<IStr> {
+fn parse_string_content(p: &mut Parser<'_>) -> Result<IStr> {
let kind = p.peek();
let text = p.text();
let s = match kind {
SyntaxKind::STRING_DOUBLE => {
let inner = &text[1..text.len() - 1];
- unescape::unescape(inner)
- .ok_or_else(|| p.error("invalid string escape".into()))?
+ unescape::unescape(inner).ok_or_else(|| p.error("invalid string escape".into()))?
}
SyntaxKind::STRING_SINGLE => {
let inner = &text[1..text.len() - 1];
- unescape::unescape(inner)
- .ok_or_else(|| p.error("invalid string escape".into()))?
+ unescape::unescape(inner).ok_or_else(|| p.error("invalid string escape".into()))?
}
SyntaxKind::STRING_DOUBLE_VERBATIM => {
let inner = &text[2..text.len() - 1];
@@ -236,7 +236,7 @@
)
}
-fn parse_number(p: &mut Parser<'_>) -> R<f64> {
+fn parse_number(p: &mut Parser<'_>) -> Result<f64> {
let text = p.text();
let n: f64 = text
.replace('_', "")
@@ -263,7 +263,7 @@
Some(t)
}
-fn assert_stmt(p: &mut Parser<'_>) -> R<AssertStmt> {
+fn assert_stmt(p: &mut Parser<'_>) -> Result<AssertStmt> {
p.eat(T![assert])?;
let cond = spanned(p, expr)?;
let msg = if p.try_eat(T![:]) {
@@ -274,13 +274,13 @@
Ok(AssertStmt(cond, msg))
}
-fn if_spec_data(p: &mut Parser<'_>) -> R<IfSpecData> {
+fn if_spec_data(p: &mut Parser<'_>) -> Result<IfSpecData> {
let v = spanned(p, |p| p.eat(T![if]))?;
let cond = expr(p)?;
Ok(IfSpecData { span: v.span, cond })
}
-fn if_else(p: &mut Parser<'_>) -> R<IfElse> {
+fn if_else(p: &mut Parser<'_>) -> Result<IfElse> {
let cond = if_spec_data(p)?;
p.eat(T![then])?;
let cond_then = expr(p)?;
@@ -296,7 +296,7 @@
})
}
-fn slice_desc(p: &mut Parser<'_>, start: Option<Spanned<Expr>>) -> R<SliceDesc> {
+fn slice_desc(p: &mut Parser<'_>, start: Option<Spanned<Expr>>) -> Result<SliceDesc> {
p.eat(T![:])?;
let end = if !p.at(T![:]) && !p.at(T![']']) {
Some(spanned(p, expr)?)
@@ -315,15 +315,12 @@
Ok(SliceDesc { start, end, step })
}
-fn destruct(p: &mut Parser<'_>) -> R<Destruct> {
+fn destruct(p: &mut Parser<'_>) -> Result<Destruct> {
if p.at_ident() {
return Ok(Destruct::Full(p.expect_ident()?));
}
#[cfg(not(feature = "exp-destruct"))]
- return Err(p.error(format!(
- "expected identifier, got {}",
- p.current_desc()
- )));
+ return Err(p.error(format!("expected identifier, got {}", p.current_desc())));
#[cfg(feature = "exp-destruct")]
{
if p.try_eat(T![?]) {
@@ -343,17 +340,17 @@
}
#[cfg(feature = "exp-destruct")]
-fn destruct_rest(p: &mut Parser<'_>) -> R<DestructRest> {
+fn destruct_rest(p: &mut Parser<'_>) -> Result<jrsonnet_ir::DestructRest> {
p.eat(T![...])?;
if p.at_ident() {
- Ok(DestructRest::Keep(p.expect_ident()?))
+ Ok(jrsonnet_ir::DestructRest::Keep(p.expect_ident()?))
} else {
- Ok(DestructRest::Drop)
+ Ok(jrsonnet_ir::DestructRest::Drop)
}
}
#[cfg(feature = "exp-destruct")]
-fn destruct_array(p: &mut Parser<'_>) -> R<Destruct> {
+fn destruct_array(p: &mut Parser<'_>) -> Result<Destruct> {
p.eat(T!['['])?;
let mut start = Vec::new();
let mut rest = None;
@@ -391,7 +388,7 @@
}
#[cfg(feature = "exp-destruct")]
-fn destruct_object(p: &mut Parser<'_>) -> R<Destruct> {
+fn destruct_object(p: &mut Parser<'_>) -> Result<Destruct> {
p.eat(T!['{'])?;
let mut fields = Vec::new();
let mut rest = None;
@@ -426,7 +423,7 @@
Ok(Destruct::Object { fields, rest })
}
-fn params(p: &mut Parser<'_>) -> R<ExprParams> {
+fn params(p: &mut Parser<'_>) -> Result<ExprParams> {
if p.at(T![')']) {
return Ok(ExprParams::new(Vec::new()));
}
@@ -452,7 +449,7 @@
Ok(ExprParams::new(result))
}
-fn args(p: &mut Parser<'_>) -> R<ArgsDesc> {
+fn args(p: &mut Parser<'_>) -> Result<ArgsDesc> {
if p.at(T![')']) {
return Ok(ArgsDesc::new(Vec::new(), Vec::new()));
}
@@ -489,7 +486,7 @@
Ok(ArgsDesc::new(unnamed, named))
}
-fn bind(p: &mut Parser<'_>) -> R<BindSpec> {
+fn bind(p: &mut Parser<'_>) -> Result<BindSpec> {
#[cfg(feature = "exp-destruct")]
{
if !p.at_ident() {
@@ -520,7 +517,7 @@
}
}
-fn visibility(p: &mut Parser<'_>) -> R<Visibility> {
+fn visibility(p: &mut Parser<'_>) -> Result<Visibility> {
p.eat(T![:])?;
if p.try_eat(T![:]) {
if p.try_eat(T![:]) {
@@ -533,7 +530,7 @@
}
}
-fn field_name(p: &mut Parser<'_>) -> R<FieldName> {
+fn field_name(p: &mut Parser<'_>) -> Result<FieldName> {
if p.at_ident() {
Ok(FieldName::Fixed(p.expect_ident()?))
} else if is_string_token(p.peek()) {
@@ -548,7 +545,7 @@
}
}
-fn field(p: &mut Parser<'_>) -> R<FieldMember> {
+fn field(p: &mut Parser<'_>) -> Result<FieldMember> {
let name = spanned(p, field_name)?;
if p.at(T!['(']) {
@@ -578,7 +575,7 @@
}
}
-fn member(p: &mut Parser<'_>) -> R<Member> {
+fn member(p: &mut Parser<'_>) -> Result<Member> {
if p.at(T![local]) {
p.eat(T![local])?;
Ok(Member::BindStmt(bind(p)?))
@@ -589,7 +586,7 @@
}
}
-fn for_spec(p: &mut Parser<'_>) -> R<ForSpecData> {
+fn for_spec(p: &mut Parser<'_>) -> Result<ForSpecData> {
p.eat(T![for])?;
let d = destruct(p)?;
p.eat(T![in])?;
@@ -597,7 +594,7 @@
Ok(ForSpecData { destruct: d, over })
}
-fn compspecs(p: &mut Parser<'_>) -> R<Vec<CompSpec>> {
+fn compspecs(p: &mut Parser<'_>) -> Result<Vec<CompSpec>> {
let mut specs = Vec::new();
specs.push(CompSpec::ForSpec(for_spec(p)?));
loop {
@@ -613,7 +610,7 @@
Ok(specs)
}
-fn objinside(p: &mut Parser<'_>) -> R<ObjBody> {
+fn objinside(p: &mut Parser<'_>) -> Result<ObjBody> {
if p.at(T!['}']) {
return Ok(ObjBody::MemberList(ObjMembers {
locals: Rc::new(Vec::new()),
@@ -641,26 +638,22 @@
match m {
Member::Field(f) => {
if field_member.is_some() {
- return Err(p.error(
- "object comprehension can only contain one field".into(),
- ));
+ return Err(
+ p.error("object comprehension can only contain one field".into())
+ );
}
field_member = Some(f);
}
Member::BindStmt(b) => locals.push(b),
Member::AssertStmt(_) => {
- return Err(p.error(
- "asserts are unsupported in object comprehension".into(),
- ));
+ return Err(p.error("asserts are unsupported in object comprehension".into()));
}
}
}
Ok(ObjBody::ObjComp(ObjComp {
locals: Rc::new(locals),
field: Rc::new(
- field_member.ok_or_else(|| {
- p.error("missing object comprehension field".into())
- })?,
+ field_member.ok_or_else(|| p.error("missing object comprehension field".into()))?,
),
compspecs: specs,
}))
@@ -683,7 +676,7 @@
}
}
-fn expr_basic(p: &mut Parser<'_>) -> R<Expr> {
+fn expr_basic(p: &mut Parser<'_>) -> Result<Expr> {
if let Some(lit) = literal(p) {
return Ok(Expr::Literal(lit));
}
@@ -825,7 +818,6 @@
}
}
-/// Flush accumulated index parts into an Expr::Index wrapping `e`.
fn flush_index_parts(e: &mut Expr, parts: &mut Vec<IndexPart>) {
if parts.is_empty() {
return;
@@ -837,7 +829,7 @@
};
}
-fn expr_suffix(p: &mut Parser<'_>) -> R<Expr> {
+fn expr_suffix(p: &mut Parser<'_>) -> Result<Expr> {
let mut e = expr_basic(p)?;
// Accumulate consecutive index parts (.field, [expr], ?.field, ?.[expr])
// into a single Expr::Index. This is critical for null-coalesce semantics:
@@ -1006,7 +998,7 @@
}
}
-fn expr_bp(p: &mut Parser<'_>, min_bp: u8) -> R<Expr> {
+fn expr_bp(p: &mut Parser<'_>, min_bp: u8) -> Result<Expr> {
let mut lhs = if let Some(op) = unary_op(p.peek()) {
p.eat_any();
let rbp = prefix_binding_power(op);
@@ -1038,11 +1030,11 @@
Ok(lhs)
}
-fn expr(p: &mut Parser<'_>) -> R<Expr> {
+fn expr(p: &mut Parser<'_>) -> Result<Expr> {
expr_bp(p, 0)
}
-pub fn parse(str: &str, settings: &ParserSettings) -> Result<Expr, ParseError> {
+pub fn parse(str: &str, settings: &ParserSettings) -> Result<Expr> {
let mut p = Parser::new(str, settings.source.clone());
for lexeme in &p.lexemes {
if let Some(desc) = lexeme.kind.error_description() {
@@ -1056,20 +1048,14 @@
}
let e = expr(&mut p)?;
if !p.at_eof() {
- return Err(p.error(format!(
- "expected end of file, got {}",
- p.current_desc(),
- )));
+ return Err(p.error(format!("expected end of file, got {}", p.current_desc(),)));
}
Ok(e)
}
pub fn string_to_expr(s: IStr, settings: &ParserSettings) -> Spanned<Expr> {
let len = s.len();
- Spanned::new(
- Expr::Str(s),
- Span(settings.source.clone(), 0, len as u32),
- )
+ Spanned::new(Expr::Str(s), Span(settings.source.clone(), 0, len as u32))
}
#[cfg(test)]
crates/jrsonnet-ir/src/visit.rsdiffbeforeafterboth--- a/crates/jrsonnet-ir/src/visit.rs
+++ b/crates/jrsonnet-ir/src/visit.rs
@@ -21,6 +21,7 @@
}
}
+#[allow(unused_variables, reason = "used with exp-destruct")]
pub fn visit_destruct<V: Visitor>(v: &mut V, destruct: &Destruct) {
match destruct {
Destruct::Full(_istr) => {}
crates/jrsonnet-rowan-parser/src/lex.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/lex.rs
+++ b/crates/jrsonnet-rowan-parser/src/lex.rs
@@ -11,9 +11,11 @@
}
pub fn lex(input: &str) -> Vec<Lexeme<'_>> {
- Lexer::new(input).map(|l| Lexeme {
- kind: SyntaxKind::from_raw(l.kind.into_raw()),
- text: l.text,
- range: TextRange::new(TextSize::from(l.range.0), TextSize::from(l.range.1)),
- }).collect()
+ Lexer::new(input)
+ .map(|l| Lexeme {
+ kind: SyntaxKind::from_raw(l.kind.into_raw()),
+ text: l.text,
+ range: TextRange::new(TextSize::from(l.range.0), TextSize::from(l.range.1)),
+ })
+ .collect()
}
crates/jrsonnet-stdlib/src/manifest/ini.rsdiffbeforeafterboth--- a/crates/jrsonnet-stdlib/src/manifest/ini.rs
+++ b/crates/jrsonnet-stdlib/src/manifest/ini.rs
@@ -3,8 +3,7 @@
use jrsonnet_evaluator::{
manifest::{ManifestFormat, ToStringFormat},
typed::{FromUntyped, Typed},
- ObjValue, Result, ResultExt, Val,
- IStr,
+ IStr, ObjValue, Result, ResultExt, Val,
};
pub struct IniFormat {
xtask/src/sourcegen/kinds.rsdiffbeforeafterboth1#[derive(Debug)]2pub struct KindsSrc {3 /// Key - how this token appears in ungrammar4 defined_tokens: IndexMap<String, TokenKind>,5 defined_node_names: HashSet<String>,6 pub nodes: Vec<String>,7}89#[derive(Debug, Clone)]10pub enum TokenKind {11 /// May exist in token tree, but never in source code12 Meta { grammar_name: String, name: String },13 /// Specific parsing/lexing errors may be emitted as this type of kind14 Error {15 grammar_name: String,16 name: String,17 #[allow(dead_code)]18 /// Is this error returned by lexer directly, or from lex.rs19 is_lexer_error: bool,20 regex: Option<String>,21 priority: Option<u32>,22 description: String,23 },24 /// Keyword - literal match of token25 Keyword {26 /// How this keyword appears in grammar/code, should be same as Kinds key27 code: String,28 name: String,29 },30 /// Literal - something defined by user, i.e strings, identifiers, smth31 Literal {32 /// How this keyword appears in grammar, should be same as Kinds key33 grammar_name: String,34 name: String,35 /// Regex for Logos lexer36 regex: String,37 /// Path to custom lexer38 lexer: Option<String>,39 },40}4142impl TokenKind {43 pub fn grammar_name(&self) -> &str {44 match self {45 Self::Keyword { code, .. } => code,46 Self::Literal { grammar_name, .. }47 | Self::Meta { grammar_name, .. }48 | Self::Error { grammar_name, .. } => grammar_name,49 }50 }51 /// How this keyword should appear in kinds enum, screaming snake cased52 pub fn name(&self) -> &str {53 match self {54 Self::Keyword { name, .. }55 | Self::Literal { name, .. }56 | Self::Meta { name, .. }57 | Self::Error { name, .. } => name,58 }59 }60 pub fn expand_kind(&self, lexer: bool) -> TokenStream {61 let name = format_ident!("{}", self.name());62 let attr = match self {63 Self::Keyword { code, .. } => quote! {#[token(#code)]},64 Self::Literal { regex, lexer, .. } => {65 let lexer = lexer66 .as_deref()67 .map(TokenStream::from_str)68 .map(|r| r.expect("path is correct"));69 quote! {#[regex(#regex, #lexer)]}70 }71 Self::Error {72 regex, priority, ..73 } if regex.is_some() => {74 let priority = priority.map(|p| quote! {, priority = #p});75 quote! {#[regex(#regex #priority)]}76 }77 _ => quote! {},78 };79 let attr = if lexer {80 attr81 } else {82 quote! {}83 };84 quote! {85 #attr86 #name87 }88 }89 pub fn expand_t_macros(&self) -> Option<TokenStream> {90 match self {91 Self::Keyword { code, name } => {92 let code = escape_token_macro(code);93 let name = format_ident!("{name}");94 Some(quote! {95 [#code] => {$crate::SyntaxKind::#name}96 })97 }98 // Meta items should not appear in T![_]99 _ => None,100 }101 }102103 /// How this token should be referenced in code104 /// Keywords are referenced with `T![_]` macro,105 /// and literals are referenced directly by name106 pub fn reference(&self) -> TokenStream {107 if let Self::Keyword { code, .. } = self {108 let code = escape_token_macro(code);109 quote! {T![#code]}110 } else {111 let name = self.name();112 let ident = format_ident!("{name}");113 quote! {#ident}114 }115 }116117 pub fn display_name(&self) -> String {118 match self {119 Self::Keyword { code, .. } => format!("'{code}'"),120 Self::Literal { name, .. } => match name.as_str() {121 "FLOAT" => "number".to_owned(),122 "IDENT" => "identifier".to_owned(),123 "STRING_DOUBLE" | "STRING_SINGLE" | "STRING_DOUBLE_VERBATIM"124 | "STRING_SINGLE_VERBATIM" | "STRING_BLOCK" => "string".to_owned(),125 "WHITESPACE" => "whitespace".to_owned(),126 "SINGLE_LINE_SLASH_COMMENT" | "SINGLE_LINE_HASH_COMMENT"127 | "MULTI_LINE_COMMENT" => "comment".to_owned(),128 _ => name.to_lowercase(),129 },130 Self::Meta { name, .. } => name.to_lowercase(),131 Self::Error { description, .. } => description.clone(),132 }133 }134135 pub fn method_name(&self) -> Ident {136 match self {137 Self::Keyword { name, .. } => {138 format_ident!("{}_token", name.to_lowercase())139 }140 Self::Literal { name, .. } => {141 format_ident!("{}_lit", name.to_lowercase())142 }143 Self::Meta { name, .. } => format_ident!("{}_meta", name.to_lowercase()),144 Self::Error { name, .. } => format_ident!("{}_error", name.to_lowercase()),145 }146 }147}148149#[macro_export]150macro_rules! define_kinds {151 ($into:ident = lit($name:literal) => $regex:literal $(, $lexer:literal)? $(; $($rest:tt)*)?) => {{152 $into.define_token(TokenKind::Literal {153 grammar_name: format!("LIT_{}!", $name),154 name: $name.to_owned(),155 regex: $regex.to_owned(),156 lexer: None $(.or_else(|| Some($lexer.to_string())))?,157 });158 $(define_kinds!($into = $($rest)*))?159 }};160 ($into:ident = error($name:literal, $desc:literal $(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{161 {162 let regex = None$(.or(Some($regex.to_owned())))?;163 let priority = None$(.or(Some($priority)))?;164 $into.define_token(TokenKind::Error {165 grammar_name: format!("ERROR_{}!", $name),166 name: format!("ERROR_{}", $name),167 is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),168 regex,169 priority,170 description: $desc.to_owned(),171 });172 }173 $(define_kinds!($into = $($rest)*))?174 }};175 ($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{176 $into.define_token(TokenKind::Keyword {177 code: format!("{}", $tok),178 name: $name.to_owned(),179 });180 $(define_kinds!($into = $($rest)*))?181 }};182 ($into:ident =) => {{}}183}184use std::{collections::HashSet, str::FromStr};185186use indexmap::IndexMap;187use proc_macro2::{Ident, TokenStream};188use quote::{format_ident, quote};189190use super::escape_token_macro;191192impl KindsSrc {193 pub fn new() -> Self {194 Self {195 defined_tokens: IndexMap::new(),196 defined_node_names: HashSet::new(),197 nodes: Vec::new(),198 }199 }200 pub fn define_token(&mut self, token: TokenKind) {201 assert!(202 self.defined_node_names.insert(token.name().to_owned()),203 "node name already defined: {}",204 token.name()205 );206 assert!(207 self.defined_tokens208 .insert(token.grammar_name().to_owned(), token.clone())209 .is_none(),210 "token already defined: {}",211 token.grammar_name()212 );213 }214 pub fn define_node(&mut self, node: &str) {215 assert!(216 self.defined_node_names.insert(node.to_owned()),217 "node name already defined: {node}"218 );219 self.nodes.push(node.to_string());220 }221 pub fn token(&self, tok: &str) -> Option<&TokenKind> {222 self.defined_tokens.get(tok)223 }224 pub fn is_token(&self, tok: &str) -> bool {225 self.defined_tokens.contains_key(tok)226 }227 pub fn tokens(&self) -> impl Iterator<Item = &TokenKind> {228 self.defined_tokens.iter().map(|(_, v)| v)229 }230}231232pub fn jsonnet_kinds() -> KindsSrc {233 let mut kinds = KindsSrc::new();234 define_kinds![kinds =235 "||" => "OR";236 "??" => "NULL_COAELSE";237 "&&" => "AND";238 "|" => "BIT_OR";239 "^" => "BIT_XOR";240 "&" => "BIT_AND";241 "==" => "EQ";242 "!=" => "NE";243 "<" => "LT";244 ">" => "GT";245 "<=" => "LE";246 ">=" => "GE";247 "<<" => "LHS";248 ">>" => "RHS";249 "+" => "PLUS";250 "-" => "MINUS";251 "*" => "MUL";252 "/" => "DIV";253 "%" => "MODULO";254 "!" => "NOT";255 "~" => "BIT_NOT";256 "[" => "L_BRACK";257 "]" => "R_BRACK";258 "(" => "L_PAREN";259 ")" => "R_PAREN";260 "{" => "L_BRACE";261 "}" => "R_BRACE";262 ":" => "COLON";263 ";" => "SEMI";264 "." => "DOT";265 "..." => "DOTDOTDOT";266 "," => "COMMA";267 "$" => "DOLLAR";268 "=" => "ASSIGN";269 "?" => "QUESTION_MARK";270 // Literals271 lit("FLOAT") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?(?:[eE][+-]?[0-9]+(?:_[0-9]+)*)?";272 error("FLOAT_JUNK_AFTER_POINT", "junk after decimal point in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)\.[^0-9]";273 error("FLOAT_JUNK_AFTER_EXPONENT", "junk after exponent in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][^+\-0-9]";274 error("FLOAT_JUNK_AFTER_EXPONENT_SIGN", "junk after exponent sign in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][+-][^0-9]";275 lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";276 error("STRING_DOUBLE_UNTERMINATED", "unterminated double-quoted string") => "\"(?s:[^\"\\\\]|\\\\.)*";277 lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";278 error("STRING_SINGLE_UNTERMINATED", "unterminated single-quoted string") => "'(?s:[^'\\\\]|\\\\.)*";279 lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";280 error("STRING_DOUBLE_VERBATIM_UNTERMINATED", "unterminated verbatim double-quoted string") => "@\"(?:[^\"]|\"\")*";281 lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";282 error("STRING_SINGLE_VERBATIM_UNTERMINATED", "unterminated verbatim single-quoted string") => "@'(?:[^']|'')*";283 error("STRING_VERBATIM_MISSING_QUOTES", "verbatim string missing opening quotes") => "@[^\"'\\s]\\S+";284 lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";285 error("STRING_BLOCK_UNEXPECTED_END", "unexpected end of text block", lexer = true);286 error("STRING_BLOCK_MISSING_NEW_LINE", "text block requires new line after |||", lexer = true);287 error("STRING_BLOCK_MISSING_TERMINATION", "unterminated text block", lexer = true);288 error("STRING_BLOCK_MISSING_INDENT", "text block first line must be indented", lexer = true);289 lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";290 lit("WHITESPACE") => r"[ \t\n\r]+";291 lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";292 lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";293 lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";294 error("COMMENT_TOO_SHORT", "comment too short") => r"/\*/";295 error("COMMENT_UNTERMINATED", "unterminated multi-line comment") => r"/\*([^*/]|\*[^/])+";296 error("NO_OPERATOR", "expected operator");297 error("MISSING_TOKEN", "missing token");298 error("UNEXPECTED_TOKEN", "unexpected token");299 error("CUSTOM", "error");300 ];301 kinds302}1#[derive(Debug)]2pub struct KindsSrc {3 /// Key - how this token appears in ungrammar4 defined_tokens: IndexMap<String, TokenKind>,5 defined_node_names: HashSet<String>,6 pub nodes: Vec<String>,7}89#[derive(Debug, Clone)]10pub enum TokenKind {11 /// May exist in token tree, but never in source code12 Meta { grammar_name: String, name: String },13 /// Specific parsing/lexing errors may be emitted as this type of kind14 Error {15 grammar_name: String,16 name: String,17 #[allow(dead_code)]18 /// Is this error returned by lexer directly, or from lex.rs19 is_lexer_error: bool,20 regex: Option<String>,21 priority: Option<u32>,22 description: String,23 },24 /// Keyword - literal match of token25 Keyword {26 /// How this keyword appears in grammar/code, should be same as Kinds key27 code: String,28 name: String,29 },30 /// Literal - something defined by user, i.e strings, identifiers, smth31 Literal {32 /// How this keyword appears in grammar, should be same as Kinds key33 grammar_name: String,34 name: String,35 /// Regex for Logos lexer36 regex: String,37 /// Path to custom lexer38 lexer: Option<String>,39 },40}4142impl TokenKind {43 pub fn grammar_name(&self) -> &str {44 match self {45 Self::Keyword { code, .. } => code,46 Self::Literal { grammar_name, .. }47 | Self::Meta { grammar_name, .. }48 | Self::Error { grammar_name, .. } => grammar_name,49 }50 }51 /// How this keyword should appear in kinds enum, screaming snake cased52 pub fn name(&self) -> &str {53 match self {54 Self::Keyword { name, .. }55 | Self::Literal { name, .. }56 | Self::Meta { name, .. }57 | Self::Error { name, .. } => name,58 }59 }60 pub fn expand_kind(&self, lexer: bool) -> TokenStream {61 let name = format_ident!("{}", self.name());62 let attr = match self {63 Self::Keyword { code, .. } => quote! {#[token(#code)]},64 Self::Literal { regex, lexer, .. } => {65 let lexer = lexer66 .as_deref()67 .map(TokenStream::from_str)68 .map(|r| r.expect("path is correct"));69 quote! {#[regex(#regex, #lexer)]}70 }71 Self::Error {72 regex, priority, ..73 } if regex.is_some() => {74 let priority = priority.map(|p| quote! {, priority = #p});75 quote! {#[regex(#regex #priority)]}76 }77 _ => quote! {},78 };79 let attr = if lexer {80 attr81 } else {82 quote! {}83 };84 quote! {85 #attr86 #name87 }88 }89 pub fn expand_t_macros(&self) -> Option<TokenStream> {90 match self {91 Self::Keyword { code, name } => {92 let code = escape_token_macro(code);93 let name = format_ident!("{name}");94 Some(quote! {95 [#code] => {$crate::SyntaxKind::#name}96 })97 }98 // Meta items should not appear in T![_]99 _ => None,100 }101 }102103 /// How this token should be referenced in code104 /// Keywords are referenced with `T![_]` macro,105 /// and literals are referenced directly by name106 pub fn reference(&self) -> TokenStream {107 if let Self::Keyword { code, .. } = self {108 let code = escape_token_macro(code);109 quote! {T![#code]}110 } else {111 let name = self.name();112 let ident = format_ident!("{name}");113 quote! {#ident}114 }115 }116117 pub fn display_name(&self) -> String {118 match self {119 Self::Keyword { code, .. } => format!("'{code}'"),120 Self::Literal { name, .. } => match name.as_str() {121 "FLOAT" => "number".to_owned(),122 "IDENT" => "identifier".to_owned(),123 "STRING_DOUBLE"124 | "STRING_SINGLE"125 | "STRING_DOUBLE_VERBATIM"126 | "STRING_SINGLE_VERBATIM"127 | "STRING_BLOCK" => "string".to_owned(),128 "WHITESPACE" => "whitespace".to_owned(),129 "SINGLE_LINE_SLASH_COMMENT" | "SINGLE_LINE_HASH_COMMENT" | "MULTI_LINE_COMMENT" => {130 "comment".to_owned()131 }132 _ => name.to_lowercase(),133 },134 Self::Meta { name, .. } => name.to_lowercase(),135 Self::Error { description, .. } => description.clone(),136 }137 }138139 pub fn method_name(&self) -> Ident {140 match self {141 Self::Keyword { name, .. } => {142 format_ident!("{}_token", name.to_lowercase())143 }144 Self::Literal { name, .. } => {145 format_ident!("{}_lit", name.to_lowercase())146 }147 Self::Meta { name, .. } => format_ident!("{}_meta", name.to_lowercase()),148 Self::Error { name, .. } => format_ident!("{}_error", name.to_lowercase()),149 }150 }151}152153#[macro_export]154macro_rules! define_kinds {155 ($into:ident = lit($name:literal) => $regex:literal $(, $lexer:literal)? $(; $($rest:tt)*)?) => {{156 $into.define_token(TokenKind::Literal {157 grammar_name: format!("LIT_{}!", $name),158 name: $name.to_owned(),159 regex: $regex.to_owned(),160 lexer: None $(.or_else(|| Some($lexer.to_string())))?,161 });162 $(define_kinds!($into = $($rest)*))?163 }};164 ($into:ident = error($name:literal, $desc:literal $(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{165 {166 let regex = None$(.or(Some($regex.to_owned())))?;167 let priority = None$(.or(Some($priority)))?;168 $into.define_token(TokenKind::Error {169 grammar_name: format!("ERROR_{}!", $name),170 name: format!("ERROR_{}", $name),171 is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),172 regex,173 priority,174 description: $desc.to_owned(),175 });176 }177 $(define_kinds!($into = $($rest)*))?178 }};179 ($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{180 $into.define_token(TokenKind::Keyword {181 code: format!("{}", $tok),182 name: $name.to_owned(),183 });184 $(define_kinds!($into = $($rest)*))?185 }};186 ($into:ident =) => {{}}187}188use std::{collections::HashSet, str::FromStr};189190use indexmap::IndexMap;191use proc_macro2::{Ident, TokenStream};192use quote::{format_ident, quote};193194use super::escape_token_macro;195196impl KindsSrc {197 pub fn new() -> Self {198 Self {199 defined_tokens: IndexMap::new(),200 defined_node_names: HashSet::new(),201 nodes: Vec::new(),202 }203 }204 pub fn define_token(&mut self, token: TokenKind) {205 assert!(206 self.defined_node_names.insert(token.name().to_owned()),207 "node name already defined: {}",208 token.name()209 );210 assert!(211 self.defined_tokens212 .insert(token.grammar_name().to_owned(), token.clone())213 .is_none(),214 "token already defined: {}",215 token.grammar_name()216 );217 }218 pub fn define_node(&mut self, node: &str) {219 assert!(220 self.defined_node_names.insert(node.to_owned()),221 "node name already defined: {node}"222 );223 self.nodes.push(node.to_string());224 }225 pub fn token(&self, tok: &str) -> Option<&TokenKind> {226 self.defined_tokens.get(tok)227 }228 pub fn is_token(&self, tok: &str) -> bool {229 self.defined_tokens.contains_key(tok)230 }231 pub fn tokens(&self) -> impl Iterator<Item = &TokenKind> {232 self.defined_tokens.iter().map(|(_, v)| v)233 }234}235236pub fn jsonnet_kinds() -> KindsSrc {237 let mut kinds = KindsSrc::new();238 define_kinds![kinds =239 "||" => "OR";240 "??" => "NULL_COAELSE";241 "&&" => "AND";242 "|" => "BIT_OR";243 "^" => "BIT_XOR";244 "&" => "BIT_AND";245 "==" => "EQ";246 "!=" => "NE";247 "<" => "LT";248 ">" => "GT";249 "<=" => "LE";250 ">=" => "GE";251 "<<" => "LHS";252 ">>" => "RHS";253 "+" => "PLUS";254 "-" => "MINUS";255 "*" => "MUL";256 "/" => "DIV";257 "%" => "MODULO";258 "!" => "NOT";259 "~" => "BIT_NOT";260 "[" => "L_BRACK";261 "]" => "R_BRACK";262 "(" => "L_PAREN";263 ")" => "R_PAREN";264 "{" => "L_BRACE";265 "}" => "R_BRACE";266 ":" => "COLON";267 ";" => "SEMI";268 "." => "DOT";269 "..." => "DOTDOTDOT";270 "," => "COMMA";271 "$" => "DOLLAR";272 "=" => "ASSIGN";273 "?" => "QUESTION_MARK";274 // Literals275 lit("FLOAT") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?(?:[eE][+-]?[0-9]+(?:_[0-9]+)*)?";276 error("FLOAT_JUNK_AFTER_POINT", "junk after decimal point in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)\.[^0-9]";277 error("FLOAT_JUNK_AFTER_EXPONENT", "junk after exponent in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][^+\-0-9]";278 error("FLOAT_JUNK_AFTER_EXPONENT_SIGN", "junk after exponent sign in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][+-][^0-9]";279 lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";280 error("STRING_DOUBLE_UNTERMINATED", "unterminated double-quoted string") => "\"(?s:[^\"\\\\]|\\\\.)*";281 lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";282 error("STRING_SINGLE_UNTERMINATED", "unterminated single-quoted string") => "'(?s:[^'\\\\]|\\\\.)*";283 lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";284 error("STRING_DOUBLE_VERBATIM_UNTERMINATED", "unterminated verbatim double-quoted string") => "@\"(?:[^\"]|\"\")*";285 lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";286 error("STRING_SINGLE_VERBATIM_UNTERMINATED", "unterminated verbatim single-quoted string") => "@'(?:[^']|'')*";287 error("STRING_VERBATIM_MISSING_QUOTES", "verbatim string missing opening quotes") => "@[^\"'\\s]\\S+";288 lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";289 error("STRING_BLOCK_UNEXPECTED_END", "unexpected end of text block", lexer = true);290 error("STRING_BLOCK_MISSING_NEW_LINE", "text block requires new line after |||", lexer = true);291 error("STRING_BLOCK_MISSING_TERMINATION", "unterminated text block", lexer = true);292 error("STRING_BLOCK_MISSING_INDENT", "text block first line must be indented", lexer = true);293 lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";294 lit("WHITESPACE") => r"[ \t\n\r]+";295 lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";296 lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";297 lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";298 error("COMMENT_TOO_SHORT", "comment too short") => r"/\*/";299 error("COMMENT_UNTERMINATED", "unterminated multi-line comment") => r"/\*([^*/]|\*[^/])+";300 error("NO_OPERATOR", "expected operator");301 error("MISSING_TOKEN", "missing token");302 error("UNEXPECTED_TOKEN", "unexpected token");303 error("CUSTOM", "error");304 ];305 kinds306}