difftreelog
feat(lexer) explicit token names
in: master
7 files changed
Cargo.tomldiffbeforeafterboth14jrsonnet-evaluator = { path = "./crates/jrsonnet-evaluator", version = "0.5.0-pre97" }14jrsonnet-evaluator = { path = "./crates/jrsonnet-evaluator", version = "0.5.0-pre97" }15jrsonnet-macros = { path = "./crates/jrsonnet-macros", version = "0.5.0-pre97" }15jrsonnet-macros = { path = "./crates/jrsonnet-macros", version = "0.5.0-pre97" }16jrsonnet-ir = { path = "./crates/jrsonnet-ir", version = "0.5.0-pre97" }16jrsonnet-ir = { path = "./crates/jrsonnet-ir", version = "0.5.0-pre97" }17jrsonnet-ir-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }17jrsonnet-ir-parser = { path = "./crates/jrsonnet-ir-parser", version = "0.5.0-pre97" }18jrsonnet-peg-parser = { path = "./crates/jrsonnet-peg-parser", version = "0.5.0-pre97" }18jrsonnet-peg-parser = { path = "./crates/jrsonnet-peg-parser", version = "0.5.0-pre97" }19jrsonnet-rowan-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }19jrsonnet-rowan-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }20jrsonnet-interner = { path = "./crates/jrsonnet-interner", version = "0.5.0-pre97" }20jrsonnet-interner = { path = "./crates/jrsonnet-interner", version = "0.5.0-pre97" }crates/jrsonnet-lexer/src/generated/syntax_kinds.rsdiffbeforeafterboth132 ERROR_COMMENT_TOO_SHORT,132 ERROR_COMMENT_TOO_SHORT,133 #[regex("/\\*([^*/]|\\*[^/])+")]133 #[regex("/\\*([^*/]|\\*[^/])+")]134 ERROR_COMMENT_UNTERMINATED,134 ERROR_COMMENT_UNTERMINATED,135 ERROR_NO_OPERATOR,136 ERROR_MISSING_TOKEN,137 ERROR_UNEXPECTED_TOKEN,138 ERROR_CUSTOM,135 #[token("tailstrict")]139 #[token("tailstrict")]136 TAILSTRICT_KW,140 TAILSTRICT_KW,137 #[token("local")]141 #[token("local")]155 #[token("in")]159 #[token("in")]156 IN_KW,160 IN_KW,157 META_OBJECT_APPLY,161 META_OBJECT_APPLY,158 ERROR_NO_OPERATOR,159 #[token("null")]162 #[token("null")]160 NULL_KW,163 NULL_KW,161 #[token("true")]164 #[token("true")]170 FOR_KW,173 FOR_KW,171 #[token("assert")]174 #[token("assert")]172 ASSERT_KW,175 ASSERT_KW,173 ERROR_MISSING_TOKEN,174 ERROR_UNEXPECTED_TOKEN,175 ERROR_CUSTOM,176 LEXING_ERROR,176 LEXING_ERROR,177 __LAST_TOKEN,177 __LAST_TOKEN,178 #[doc(hidden)]178 #[doc(hidden)]crates/jrsonnet-lexer/src/string_block.rsdiffbeforeafterboth211211212 // Process leading blank lines before calculating string block indent212 // Process leading blank lines before calculating string block indent213 while ctx.peek() == Some('\n') {213 while ctx.peek() == Some('\n') {214 lex.mark_line("");214 ctx.next();215 ctx.next();215 }216 }216217crates/jrsonnet-rowan-parser/jsonnet.ungramdiffbeforeafterboth209| FieldNameDynamic209| FieldNameDynamic210210211Visibility =211Visibility =212 ':' v1:':'? v2:':'?212 ':' ':'? ':'?213213214Literal =214Literal =215 'null'215 'null'crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rsdiffbeforeafterboth76 MULTI_LINE_COMMENT,76 MULTI_LINE_COMMENT,77 ERROR_COMMENT_TOO_SHORT,77 ERROR_COMMENT_TOO_SHORT,78 ERROR_COMMENT_UNTERMINATED,78 ERROR_COMMENT_UNTERMINATED,79 ERROR_NO_OPERATOR,80 ERROR_MISSING_TOKEN,81 ERROR_UNEXPECTED_TOKEN,82 ERROR_CUSTOM,79 TAILSTRICT_KW,83 TAILSTRICT_KW,80 LOCAL_KW,84 LOCAL_KW,81 IMPORTSTR_KW,85 IMPORTSTR_KW,88 ERROR_KW,92 ERROR_KW,89 IN_KW,93 IN_KW,90 META_OBJECT_APPLY,94 META_OBJECT_APPLY,91 ERROR_NO_OPERATOR,92 NULL_KW,95 NULL_KW,93 TRUE_KW,96 TRUE_KW,94 FALSE_KW,97 FALSE_KW,95 SELF_KW,98 SELF_KW,96 SUPER_KW,99 SUPER_KW,97 FOR_KW,100 FOR_KW,98 ASSERT_KW,101 ASSERT_KW,99 ERROR_MISSING_TOKEN,100 ERROR_UNEXPECTED_TOKEN,101 ERROR_CUSTOM,102 LEXING_ERROR,102 LEXING_ERROR,103 __LAST_TOKEN,103 __LAST_TOKEN,104 SOURCE_FILE,104 SOURCE_FILE,197 | FIELD_NAME | DESTRUCT | DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR197 | FIELD_NAME | DESTRUCT | DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR198 | LITERAL | TEXT | NUMBER | IMPORT_KIND | TRIVIA | CUSTOM_ERROR => true,198 | LITERAL | TEXT | NUMBER | IMPORT_KIND | TRIVIA | CUSTOM_ERROR => true,199 _ => false,199 _ => false,200 }201 }202 pub fn error_description(self) -> Option<&'static str> {203 match self {204 ERROR_FLOAT_JUNK_AFTER_POINT => {205 ::core::option::Option::Some("junk after decimal point in number literal")206 }207 ERROR_FLOAT_JUNK_AFTER_EXPONENT => {208 ::core::option::Option::Some("junk after exponent in number literal")209 }210 ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => {211 ::core::option::Option::Some("junk after exponent sign in number literal")212 }213 ERROR_STRING_DOUBLE_UNTERMINATED => {214 ::core::option::Option::Some("unterminated double-quoted string")215 }216 ERROR_STRING_SINGLE_UNTERMINATED => {217 ::core::option::Option::Some("unterminated single-quoted string")218 }219 ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => {220 ::core::option::Option::Some("unterminated verbatim double-quoted string")221 }222 ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => {223 ::core::option::Option::Some("unterminated verbatim single-quoted string")224 }225 ERROR_STRING_VERBATIM_MISSING_QUOTES => {226 ::core::option::Option::Some("verbatim string missing opening quotes")227 }228 ERROR_STRING_BLOCK_UNEXPECTED_END => {229 ::core::option::Option::Some("unexpected end of text block")230 }231 ERROR_STRING_BLOCK_MISSING_NEW_LINE => {232 ::core::option::Option::Some("text block requires new line after |||")233 }234 ERROR_STRING_BLOCK_MISSING_TERMINATION => {235 ::core::option::Option::Some("unterminated text block")236 }237 ERROR_STRING_BLOCK_MISSING_INDENT => {238 ::core::option::Option::Some("text block first line must be indented")239 }240 ERROR_COMMENT_TOO_SHORT => ::core::option::Option::Some("comment too short"),241 ERROR_COMMENT_UNTERMINATED => {242 ::core::option::Option::Some("unterminated multi-line comment")243 }244 ERROR_NO_OPERATOR => ::core::option::Option::Some("expected operator"),245 ERROR_MISSING_TOKEN => ::core::option::Option::Some("missing token"),246 ERROR_UNEXPECTED_TOKEN => ::core::option::Option::Some("unexpected token"),247 ERROR_CUSTOM => ::core::option::Option::Some("error"),248 LEXING_ERROR => ::core::option::Option::Some("unexpected character"),249 _ => None,250 }251 }252 pub fn display_name(self) -> &'static str {253 match self {254 OR => "'||'",255 NULL_COAELSE => "'??'",256 AND => "'&&'",257 BIT_OR => "'|'",258 BIT_XOR => "'^'",259 BIT_AND => "'&'",260 EQ => "'=='",261 NE => "'!='",262 LT => "'<'",263 GT => "'>'",264 LE => "'<='",265 GE => "'>='",266 LHS => "'<<'",267 RHS => "'>>'",268 PLUS => "'+'",269 MINUS => "'-'",270 MUL => "'*'",271 DIV => "'/'",272 MODULO => "'%'",273 NOT => "'!'",274 BIT_NOT => "'~'",275 L_BRACK => "'['",276 R_BRACK => "']'",277 L_PAREN => "'('",278 R_PAREN => "')'",279 L_BRACE => "'{'",280 R_BRACE => "'}'",281 COLON => "':'",282 SEMI => "';'",283 DOT => "'.'",284 DOTDOTDOT => "'...'",285 COMMA => "','",286 DOLLAR => "'$'",287 ASSIGN => "'='",288 QUESTION_MARK => "'?'",289 FLOAT => "number",290 ERROR_FLOAT_JUNK_AFTER_POINT => "junk after decimal point in number literal",291 ERROR_FLOAT_JUNK_AFTER_EXPONENT => "junk after exponent in number literal",292 ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => "junk after exponent sign in number literal",293 STRING_DOUBLE => "string",294 ERROR_STRING_DOUBLE_UNTERMINATED => "unterminated double-quoted string",295 STRING_SINGLE => "string",296 ERROR_STRING_SINGLE_UNTERMINATED => "unterminated single-quoted string",297 STRING_DOUBLE_VERBATIM => "string",298 ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => {299 "unterminated verbatim double-quoted string"300 }301 STRING_SINGLE_VERBATIM => "string",302 ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => {303 "unterminated verbatim single-quoted string"304 }305 ERROR_STRING_VERBATIM_MISSING_QUOTES => "verbatim string missing opening quotes",306 STRING_BLOCK => "string",307 ERROR_STRING_BLOCK_UNEXPECTED_END => "unexpected end of text block",308 ERROR_STRING_BLOCK_MISSING_NEW_LINE => "text block requires new line after |||",309 ERROR_STRING_BLOCK_MISSING_TERMINATION => "unterminated text block",310 ERROR_STRING_BLOCK_MISSING_INDENT => "text block first line must be indented",311 IDENT => "identifier",312 WHITESPACE => "whitespace",313 SINGLE_LINE_SLASH_COMMENT => "comment",314 SINGLE_LINE_HASH_COMMENT => "comment",315 MULTI_LINE_COMMENT => "comment",316 ERROR_COMMENT_TOO_SHORT => "comment too short",317 ERROR_COMMENT_UNTERMINATED => "unterminated multi-line comment",318 ERROR_NO_OPERATOR => "expected operator",319 ERROR_MISSING_TOKEN => "missing token",320 ERROR_UNEXPECTED_TOKEN => "unexpected token",321 ERROR_CUSTOM => "error",322 TAILSTRICT_KW => "'tailstrict'",323 LOCAL_KW => "'local'",324 IMPORTSTR_KW => "'importstr'",325 IMPORTBIN_KW => "'importbin'",326 IMPORT_KW => "'import'",327 IF_KW => "'if'",328 THEN_KW => "'then'",329 ELSE_KW => "'else'",330 FUNCTION_KW => "'function'",331 ERROR_KW => "'error'",332 IN_KW => "'in'",333 META_OBJECT_APPLY => "meta_object_apply",334 NULL_KW => "'null'",335 TRUE_KW => "'true'",336 FALSE_KW => "'false'",337 SELF_KW => "'self'",338 SUPER_KW => "'super'",339 FOR_KW => "'for'",340 ASSERT_KW => "'assert'",341 LEXING_ERROR => "unexpected character",342 _ => "unknown",200 }343 }201 }344 }202 pub fn from_raw(r: u16) -> Self {345 pub fn from_raw(r: u16) -> Self {xtask/src/sourcegen/kinds.rsdiffbeforeafterboth19 is_lexer_error: bool,19 is_lexer_error: bool,20 regex: Option<String>,20 regex: Option<String>,21 priority: Option<u32>,21 priority: Option<u32>,22 description: String,22 },23 },23 /// Keyword - literal match of token24 /// Keyword - literal match of token24 Keyword {25 Keyword {113 }114 }114 }115 }116117 pub fn display_name(&self) -> String {118 match self {119 Self::Keyword { code, .. } => format!("'{code}'"),120 Self::Literal { name, .. } => match name.as_str() {121 "FLOAT" => "number".to_owned(),122 "IDENT" => "identifier".to_owned(),123 "STRING_DOUBLE" | "STRING_SINGLE" | "STRING_DOUBLE_VERBATIM"124 | "STRING_SINGLE_VERBATIM" | "STRING_BLOCK" => "string".to_owned(),125 "WHITESPACE" => "whitespace".to_owned(),126 "SINGLE_LINE_SLASH_COMMENT" | "SINGLE_LINE_HASH_COMMENT"127 | "MULTI_LINE_COMMENT" => "comment".to_owned(),128 _ => name.to_lowercase(),129 },130 Self::Meta { name, .. } => name.to_lowercase(),131 Self::Error { description, .. } => description.clone(),132 }133 }115134116 pub fn method_name(&self) -> Ident {135 pub fn method_name(&self) -> Ident {117 match self {136 match self {138 });157 });139 $(define_kinds!($into = $($rest)*))?158 $(define_kinds!($into = $($rest)*))?140 }};159 }};141 ($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{160 ($into:ident = error($name:literal, $desc:literal $(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{142 {161 {143 let regex = None$(.or(Some($regex.to_owned())))?;162 let regex = None$(.or(Some($regex.to_owned())))?;144 let priority = None$(.or(Some($priority)))?;163 let priority = None$(.or(Some($priority)))?;148 is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),167 is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),149 regex,168 regex,150 priority,169 priority,170 description: $desc.to_owned(),151 });171 });152 }172 }153 $(define_kinds!($into = $($rest)*))?173 $(define_kinds!($into = $($rest)*))?248 "=" => "ASSIGN";268 "=" => "ASSIGN";249 "?" => "QUESTION_MARK";269 "?" => "QUESTION_MARK";250 // Literals270 // Literals251 lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?";271 lit("FLOAT") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?(?:[eE][+-]?[0-9]+(?:_[0-9]+)*)?";252 error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]";272 error("FLOAT_JUNK_AFTER_POINT", "junk after decimal point in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)\.[^0-9]";253 error("FLOAT_JUNK_AFTER_EXPONENT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]";273 error("FLOAT_JUNK_AFTER_EXPONENT", "junk after exponent in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][^+\-0-9]";254 error("FLOAT_JUNK_AFTER_EXPONENT_SIGN") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]";274 error("FLOAT_JUNK_AFTER_EXPONENT_SIGN", "junk after exponent sign in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][+-][^0-9]";255 lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";275 lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";256 error("STRING_DOUBLE_UNTERMINATED") => "\"(?s:[^\"\\\\]|\\\\.)*";276 error("STRING_DOUBLE_UNTERMINATED", "unterminated double-quoted string") => "\"(?s:[^\"\\\\]|\\\\.)*";257 lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";277 lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";258 error("STRING_SINGLE_UNTERMINATED") => "'(?s:[^'\\\\]|\\\\.)*";278 error("STRING_SINGLE_UNTERMINATED", "unterminated single-quoted string") => "'(?s:[^'\\\\]|\\\\.)*";259 lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";279 lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";260 error("STRING_DOUBLE_VERBATIM_UNTERMINATED") => "@\"(?:[^\"]|\"\")*";280 error("STRING_DOUBLE_VERBATIM_UNTERMINATED", "unterminated verbatim double-quoted string") => "@\"(?:[^\"]|\"\")*";261 lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";281 lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";262 error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";282 error("STRING_SINGLE_VERBATIM_UNTERMINATED", "unterminated verbatim single-quoted string") => "@'(?:[^']|'')*";263 error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";283 error("STRING_VERBATIM_MISSING_QUOTES", "verbatim string missing opening quotes") => "@[^\"'\\s]\\S+";264 lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";284 lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";265 error("STRING_BLOCK_UNEXPECTED_END", lexer = true);285 error("STRING_BLOCK_UNEXPECTED_END", "unexpected end of text block", lexer = true);266 error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);286 error("STRING_BLOCK_MISSING_NEW_LINE", "text block requires new line after |||", lexer = true);267 error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);287 error("STRING_BLOCK_MISSING_TERMINATION", "unterminated text block", lexer = true);268 error("STRING_BLOCK_MISSING_INDENT", lexer = true);288 error("STRING_BLOCK_MISSING_INDENT", "text block first line must be indented", lexer = true);269 lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";289 lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";270 lit("WHITESPACE") => r"[ \t\n\r]+";290 lit("WHITESPACE") => r"[ \t\n\r]+";271 lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";291 lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";272 lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";292 lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";273 lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";293 lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";274 error("COMMENT_TOO_SHORT") => r"/\*/";294 error("COMMENT_TOO_SHORT", "comment too short") => r"/\*/";275 error("COMMENT_UNTERMINATED") => r"/\*([^*/]|\*[^/])+";295 error("COMMENT_UNTERMINATED", "unterminated multi-line comment") => r"/\*([^*/]|\*[^/])+";296 error("NO_OPERATOR", "expected operator");297 error("MISSING_TOKEN", "missing token");298 error("UNEXPECTED_TOKEN", "unexpected token");299 error("CUSTOM", "error");276 ];300 ];277 kinds301 kinds278}302}xtask/src/sourcegen/mod.rsdiffbeforeafterboth56 });56 });57 }57 }58 SpecialName::Error => {58 SpecialName::Error => {59 eprintln!("implicit error: {name}");59 panic!("error token ERROR_{name} must be explicitly defined in jsonnet_kinds()");60 kinds.define_token(TokenKind::Error {61 grammar_name: token.to_owned(),62 name: format!("ERROR_{name}"),63 regex: None,64 priority: None,65 is_lexer_error: true,66 });67 }60 }68 }61 }69 continue;62 continue;170 quote! {}163 quote! {}171 };164 };165166 let error_desc_arms = kinds.tokens().filter_map(|t| {167 if let TokenKind::Error {168 name, description, ..169 } = t170 {171 let ident = format_ident!("{name}");172 Some(quote! { #ident => ::core::option::Option::Some(#description) })173 } else {174 None175 }176 });177178 let display_name_arms = kinds.tokens().map(|t| {179 let ident = format_ident!("{}", t.name());180 let display = t.display_name();181 quote! { #ident => #display }182 });172183173 let ast = quote! {184 let ast = quote! {174 #![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]185 #![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]200211201 #is_enum212 #is_enum213214 pub fn error_description(self) -> Option<&'static str> {215 match self {216 #(#error_desc_arms,)*217 LEXING_ERROR => ::core::option::Option::Some("unexpected character"),218 _ => None,219 }220 }221222 pub fn display_name(self) -> &'static str {223 match self {224 #(#display_name_arms,)*225 LEXING_ERROR => "unexpected character",226 _ => "unknown",227 }228 }202229203 pub fn from_raw(r: u16) -> Self {230 pub fn from_raw(r: u16) -> Self {204 assert!(r < Self::__LAST as u16);231 assert!(r < Self::__LAST as u16);