difftreelog
feat(lexer) explicit token names
in: master
7 files changed
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,7 +14,7 @@
jrsonnet-evaluator = { path = "./crates/jrsonnet-evaluator", version = "0.5.0-pre97" }
jrsonnet-macros = { path = "./crates/jrsonnet-macros", version = "0.5.0-pre97" }
jrsonnet-ir = { path = "./crates/jrsonnet-ir", version = "0.5.0-pre97" }
-jrsonnet-ir-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }
+jrsonnet-ir-parser = { path = "./crates/jrsonnet-ir-parser", version = "0.5.0-pre97" }
jrsonnet-peg-parser = { path = "./crates/jrsonnet-peg-parser", version = "0.5.0-pre97" }
jrsonnet-rowan-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }
jrsonnet-interner = { path = "./crates/jrsonnet-interner", version = "0.5.0-pre97" }
crates/jrsonnet-lexer/src/generated/syntax_kinds.rsdiffbeforeafterboth--- a/crates/jrsonnet-lexer/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-lexer/src/generated/syntax_kinds.rs
@@ -132,6 +132,10 @@
ERROR_COMMENT_TOO_SHORT,
#[regex("/\\*([^*/]|\\*[^/])+")]
ERROR_COMMENT_UNTERMINATED,
+ ERROR_NO_OPERATOR,
+ ERROR_MISSING_TOKEN,
+ ERROR_UNEXPECTED_TOKEN,
+ ERROR_CUSTOM,
#[token("tailstrict")]
TAILSTRICT_KW,
#[token("local")]
@@ -155,7 +159,6 @@
#[token("in")]
IN_KW,
META_OBJECT_APPLY,
- ERROR_NO_OPERATOR,
#[token("null")]
NULL_KW,
#[token("true")]
@@ -170,9 +173,6 @@
FOR_KW,
#[token("assert")]
ASSERT_KW,
- ERROR_MISSING_TOKEN,
- ERROR_UNEXPECTED_TOKEN,
- ERROR_CUSTOM,
LEXING_ERROR,
__LAST_TOKEN,
#[doc(hidden)]
crates/jrsonnet-lexer/src/string_block.rsdiffbeforeafterboth1#[derive(Clone, Copy, Debug, PartialEq, Eq)]2pub enum StringBlockError {3 UnexpectedEnd,4 MissingNewLine,5 MissingTermination,6 MissingIndent,7}89use logos::Lexer;10use StringBlockError::*;1112use crate::generated::syntax_kinds::SyntaxKind;1314pub(crate) fn lex_str_block_test(lex: &mut Lexer<'_, SyntaxKind>) {15 let _ = lex_str_block(lex);16}1718pub(crate) struct Context<'a> {19 source: &'a str,20 index: usize,21}2223impl<'a> Context<'a> {24 fn rest(&self) -> &'a str {25 &self.source[self.index..]26 }2728 fn next(&mut self) -> Option<char> {29 if self.index == self.source.len() {30 return None;31 }3233 match self.rest().chars().next() {34 None => None,35 Some(c) => {36 self.index += c.len_utf8();37 Some(c)38 }39 }40 }4142 fn peek(&self) -> Option<char> {43 if self.index == self.source.len() {44 return None;45 }4647 self.rest().chars().next()48 }4950 fn eat_if(&mut self, f: impl Fn(char) -> bool) -> usize {51 if self.peek().is_some_and(f) {52 self.index += 1;53 return 1;54 }55 056 }5758 fn eat_while(&mut self, f: impl Fn(char) -> bool) -> usize {59 if self.index == self.source.len() {60 return 0;61 }6263 let next_char = self.rest().char_indices().find(|(_, c)| !f(*c));6465 match next_char {66 None => {67 let diff = self.source.len() - self.index;68 self.index = self.source.len();69 diff70 }71 Some((idx, _)) => {72 self.index += idx;73 idx74 }75 }76 }7778 fn skip(&mut self, len: usize) {79 self.index = match self.index + len {80 n if n > self.source.len() => self.source.len(),81 n => n,82 };83 }84}8586// Check that b has at least the same whitespace prefix as a and returns the87// amount of this whitespace, otherwise returns 0. If a has no whitespace88// prefix than return 0.89fn check_whitespace(a: &str, b: &str) -> usize {90 let a = a.as_bytes();91 let b = b.as_bytes();9293 for i in 0..a.len() {94 if a[i] != b' ' && a[i] != b'\t' {95 // a has run out of whitespace and b matched up to this point. Return result.96 return i;97 }9899 if i >= b.len() {100 // We ran off the edge of b while a still has whitespace. Return 0 as failure.101 return 0;102 }103104 if a[i] != b[i] {105 // a has whitespace but b does not. Return 0 as failure.106 return 0;107 }108 }109110 // We ran off the end of a and b kept up111 a.len()112}113114pub(crate) trait StrBlockLexCtx<'d> {115 fn remainder(&self) -> &'d str;116 fn eat_error(&mut self, ctx: &Context<'d>);117 fn bump_pos(&mut self, s: usize);118 fn mark_truncating(&mut self);119 fn mark_line(&mut self, line: &'d str);120}121122impl<'d> StrBlockLexCtx<'d> for Lexer<'d, SyntaxKind> {123 fn remainder(&self) -> &'d str {124 self.remainder()125 }126 fn eat_error(&mut self, ctx: &Context<'d>) {127 let end_index = ctx128 .rest()129 .find("|||")130 .map_or_else(|| ctx.rest().len(), |v| v + 3);131 self.bump(ctx.index + end_index);132 }133 fn bump_pos(&mut self, s: usize) {134 self.bump(s);135 }136 fn mark_truncating(&mut self) {137 // Lexer test doesn't collect anything138 }139 fn mark_line(&mut self, _line: &'d str) {140 // Lexer test doesn't collect anything141 }142}143144pub fn collect_lexed_str_block(input: &str) -> Result<CollectStrBlock<'_>, StringBlockError> {145 let mut collect = CollectStrBlock {146 truncate: false,147 lines: vec![],148 input,149 offset: 0,150 };151 lex_str_block(&mut collect)?;152 Ok(collect)153}154155pub struct CollectStrBlock<'s> {156 pub truncate: bool,157 pub lines: Vec<&'s str>,158 input: &'s str,159 offset: usize,160}161162impl<'d> StrBlockLexCtx<'d> for CollectStrBlock<'d> {163 fn remainder(&self) -> &'d str {164 self.input165 }166167 fn eat_error(&mut self, _ctx: &Context<'d>) {168 // Error will be returned, no need to record it here169 }170171 fn bump_pos(&mut self, s: usize) {172 self.offset += s;173 }174175 fn mark_truncating(&mut self) {176 self.truncate = true;177 }178179 fn mark_line(&mut self, line: &'d str) {180 self.lines.push(line);181 }182}183184pub(crate) fn lex_str_block<'a>(lex: &mut impl StrBlockLexCtx<'a>) -> Result<(), StringBlockError> {185 // debug_assert_eq!(lex.slice(), "|||");186 let mut ctx = Context::<'a> {187 source: lex.remainder(),188 index: 0,189 };190191 if ctx.eat_if(|v| v == '-') != 0 {192 lex.mark_truncating();193 }194195 // Skip whitespaces196 ctx.eat_while(|r| r == ' ' || r == '\t' || r == '\r');197198 // Skip \n199 match ctx.next() {200 Some('\n') => (),201 None => {202 lex.eat_error(&ctx);203 return Err(UnexpectedEnd);204 }205 // Text block requires new line after |||.206 Some(_) => {207 lex.eat_error(&ctx);208 return Err(MissingNewLine);209 }210 }211212 // Process leading blank lines before calculating string block indent213 while ctx.peek() == Some('\n') {214 ctx.next();215 }216217 let mut num_whitespace = check_whitespace(ctx.rest(), ctx.rest());218 let str_block_indent = &ctx.rest()[..num_whitespace];219220 if num_whitespace == 0 {221 // Text block's first line must start with whitespace222 lex.eat_error(&ctx);223 return Err(MissingIndent);224 }225226 loop {227 debug_assert_ne!(num_whitespace, 0, "Unexpected value for num_whitespace");228 ctx.skip(num_whitespace);229230 let line_start = ctx.index;231 let mut line_size = 0;232 loop {233 match ctx.next() {234 None => {235 lex.eat_error(&ctx);236 return Err(UnexpectedEnd);237 }238 Some('\n') => {239 lex.mark_line(&ctx.source[line_start..line_start + line_size]);240 break;241 }242 Some(c) => {243 line_size += c.len_utf8();244 }245 }246 }247248 // Skip any blank lines249 while ctx.peek() == Some('\n') {250 lex.mark_line("");251 ctx.next();252 }253254 // Look at the next line255 num_whitespace = check_whitespace(str_block_indent, ctx.rest());256 if num_whitespace == 0 {257 // End of the text block258 // let mut term_indent = String::with_capacity(num_whitespace);259 while let Some(' ' | '\t') = ctx.peek() {260 // term_indent.push(261 ctx.next().unwrap();262 // );263 }264265 if !ctx.rest().starts_with("|||") {266 if ctx.rest().is_empty() {267 lex.bump_pos(ctx.index);268 return Err(UnexpectedEnd);269 }270 lex.eat_error(&ctx);271 return Err(MissingTermination);272 }273274 // Skip '|||'275 ctx.skip(3);276 break;277 }278 }279280 lex.bump_pos(ctx.index);281 Ok(())282}1#[derive(Clone, Copy, Debug, PartialEq, Eq)]2pub enum StringBlockError {3 UnexpectedEnd,4 MissingNewLine,5 MissingTermination,6 MissingIndent,7}89use logos::Lexer;10use StringBlockError::*;1112use crate::generated::syntax_kinds::SyntaxKind;1314pub(crate) fn lex_str_block_test(lex: &mut Lexer<'_, SyntaxKind>) {15 let _ = lex_str_block(lex);16}1718pub(crate) struct Context<'a> {19 source: &'a str,20 index: usize,21}2223impl<'a> Context<'a> {24 fn rest(&self) -> &'a str {25 &self.source[self.index..]26 }2728 fn next(&mut self) -> Option<char> {29 if self.index == self.source.len() {30 return None;31 }3233 match self.rest().chars().next() {34 None => None,35 Some(c) => {36 self.index += c.len_utf8();37 Some(c)38 }39 }40 }4142 fn peek(&self) -> Option<char> {43 if self.index == self.source.len() {44 return None;45 }4647 self.rest().chars().next()48 }4950 fn eat_if(&mut self, f: impl Fn(char) -> bool) -> usize {51 if self.peek().is_some_and(f) {52 self.index += 1;53 return 1;54 }55 056 }5758 fn eat_while(&mut self, f: impl Fn(char) -> bool) -> usize {59 if self.index == self.source.len() {60 return 0;61 }6263 let next_char = self.rest().char_indices().find(|(_, c)| !f(*c));6465 match next_char {66 None => {67 let diff = self.source.len() - self.index;68 self.index = self.source.len();69 diff70 }71 Some((idx, _)) => {72 self.index += idx;73 idx74 }75 }76 }7778 fn skip(&mut self, len: usize) {79 self.index = match self.index + len {80 n if n > self.source.len() => self.source.len(),81 n => n,82 };83 }84}8586// Check that b has at least the same whitespace prefix as a and returns the87// amount of this whitespace, otherwise returns 0. If a has no whitespace88// prefix than return 0.89fn check_whitespace(a: &str, b: &str) -> usize {90 let a = a.as_bytes();91 let b = b.as_bytes();9293 for i in 0..a.len() {94 if a[i] != b' ' && a[i] != b'\t' {95 // a has run out of whitespace and b matched up to this point. Return result.96 return i;97 }9899 if i >= b.len() {100 // We ran off the edge of b while a still has whitespace. Return 0 as failure.101 return 0;102 }103104 if a[i] != b[i] {105 // a has whitespace but b does not. Return 0 as failure.106 return 0;107 }108 }109110 // We ran off the end of a and b kept up111 a.len()112}113114pub(crate) trait StrBlockLexCtx<'d> {115 fn remainder(&self) -> &'d str;116 fn eat_error(&mut self, ctx: &Context<'d>);117 fn bump_pos(&mut self, s: usize);118 fn mark_truncating(&mut self);119 fn mark_line(&mut self, line: &'d str);120}121122impl<'d> StrBlockLexCtx<'d> for Lexer<'d, SyntaxKind> {123 fn remainder(&self) -> &'d str {124 self.remainder()125 }126 fn eat_error(&mut self, ctx: &Context<'d>) {127 let end_index = ctx128 .rest()129 .find("|||")130 .map_or_else(|| ctx.rest().len(), |v| v + 3);131 self.bump(ctx.index + end_index);132 }133 fn bump_pos(&mut self, s: usize) {134 self.bump(s);135 }136 fn mark_truncating(&mut self) {137 // Lexer test doesn't collect anything138 }139 fn mark_line(&mut self, _line: &'d str) {140 // Lexer test doesn't collect anything141 }142}143144pub fn collect_lexed_str_block(input: &str) -> Result<CollectStrBlock<'_>, StringBlockError> {145 let mut collect = CollectStrBlock {146 truncate: false,147 lines: vec![],148 input,149 offset: 0,150 };151 lex_str_block(&mut collect)?;152 Ok(collect)153}154155pub struct CollectStrBlock<'s> {156 pub truncate: bool,157 pub lines: Vec<&'s str>,158 input: &'s str,159 offset: usize,160}161162impl<'d> StrBlockLexCtx<'d> for CollectStrBlock<'d> {163 fn remainder(&self) -> &'d str {164 self.input165 }166167 fn eat_error(&mut self, _ctx: &Context<'d>) {168 // Error will be returned, no need to record it here169 }170171 fn bump_pos(&mut self, s: usize) {172 self.offset += s;173 }174175 fn mark_truncating(&mut self) {176 self.truncate = true;177 }178179 fn mark_line(&mut self, line: &'d str) {180 self.lines.push(line);181 }182}183184pub(crate) fn lex_str_block<'a>(lex: &mut impl StrBlockLexCtx<'a>) -> Result<(), StringBlockError> {185 // debug_assert_eq!(lex.slice(), "|||");186 let mut ctx = Context::<'a> {187 source: lex.remainder(),188 index: 0,189 };190191 if ctx.eat_if(|v| v == '-') != 0 {192 lex.mark_truncating();193 }194195 // Skip whitespaces196 ctx.eat_while(|r| r == ' ' || r == '\t' || r == '\r');197198 // Skip \n199 match ctx.next() {200 Some('\n') => (),201 None => {202 lex.eat_error(&ctx);203 return Err(UnexpectedEnd);204 }205 // Text block requires new line after |||.206 Some(_) => {207 lex.eat_error(&ctx);208 return Err(MissingNewLine);209 }210 }211212 // Process leading blank lines before calculating string block indent213 while ctx.peek() == Some('\n') {214 lex.mark_line("");215 ctx.next();216 }217218 let mut num_whitespace = check_whitespace(ctx.rest(), ctx.rest());219 let str_block_indent = &ctx.rest()[..num_whitespace];220221 if num_whitespace == 0 {222 // Text block's first line must start with whitespace223 lex.eat_error(&ctx);224 return Err(MissingIndent);225 }226227 loop {228 debug_assert_ne!(num_whitespace, 0, "Unexpected value for num_whitespace");229 ctx.skip(num_whitespace);230231 let line_start = ctx.index;232 let mut line_size = 0;233 loop {234 match ctx.next() {235 None => {236 lex.eat_error(&ctx);237 return Err(UnexpectedEnd);238 }239 Some('\n') => {240 lex.mark_line(&ctx.source[line_start..line_start + line_size]);241 break;242 }243 Some(c) => {244 line_size += c.len_utf8();245 }246 }247 }248249 // Skip any blank lines250 while ctx.peek() == Some('\n') {251 lex.mark_line("");252 ctx.next();253 }254255 // Look at the next line256 num_whitespace = check_whitespace(str_block_indent, ctx.rest());257 if num_whitespace == 0 {258 // End of the text block259 // let mut term_indent = String::with_capacity(num_whitespace);260 while let Some(' ' | '\t') = ctx.peek() {261 // term_indent.push(262 ctx.next().unwrap();263 // );264 }265266 if !ctx.rest().starts_with("|||") {267 if ctx.rest().is_empty() {268 lex.bump_pos(ctx.index);269 return Err(UnexpectedEnd);270 }271 lex.eat_error(&ctx);272 return Err(MissingTermination);273 }274275 // Skip '|||'276 ctx.skip(3);277 break;278 }279 }280281 lex.bump_pos(ctx.index);282 Ok(())283}crates/jrsonnet-rowan-parser/jsonnet.ungramdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/jsonnet.ungram
+++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram
@@ -209,7 +209,7 @@
| FieldNameDynamic
Visibility =
- ':' v1:':'? v2:':'?
+ ':' ':'? ':'?
Literal =
'null'
crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
@@ -76,6 +76,10 @@
MULTI_LINE_COMMENT,
ERROR_COMMENT_TOO_SHORT,
ERROR_COMMENT_UNTERMINATED,
+ ERROR_NO_OPERATOR,
+ ERROR_MISSING_TOKEN,
+ ERROR_UNEXPECTED_TOKEN,
+ ERROR_CUSTOM,
TAILSTRICT_KW,
LOCAL_KW,
IMPORTSTR_KW,
@@ -88,7 +92,6 @@
ERROR_KW,
IN_KW,
META_OBJECT_APPLY,
- ERROR_NO_OPERATOR,
NULL_KW,
TRUE_KW,
FALSE_KW,
@@ -96,9 +99,6 @@
SUPER_KW,
FOR_KW,
ASSERT_KW,
- ERROR_MISSING_TOKEN,
- ERROR_UNEXPECTED_TOKEN,
- ERROR_CUSTOM,
LEXING_ERROR,
__LAST_TOKEN,
SOURCE_FILE,
@@ -199,6 +199,149 @@
_ => false,
}
}
+ pub fn error_description(self) -> Option<&'static str> {
+ match self {
+ ERROR_FLOAT_JUNK_AFTER_POINT => {
+ ::core::option::Option::Some("junk after decimal point in number literal")
+ }
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT => {
+ ::core::option::Option::Some("junk after exponent in number literal")
+ }
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => {
+ ::core::option::Option::Some("junk after exponent sign in number literal")
+ }
+ ERROR_STRING_DOUBLE_UNTERMINATED => {
+ ::core::option::Option::Some("unterminated double-quoted string")
+ }
+ ERROR_STRING_SINGLE_UNTERMINATED => {
+ ::core::option::Option::Some("unterminated single-quoted string")
+ }
+ ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => {
+ ::core::option::Option::Some("unterminated verbatim double-quoted string")
+ }
+ ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => {
+ ::core::option::Option::Some("unterminated verbatim single-quoted string")
+ }
+ ERROR_STRING_VERBATIM_MISSING_QUOTES => {
+ ::core::option::Option::Some("verbatim string missing opening quotes")
+ }
+ ERROR_STRING_BLOCK_UNEXPECTED_END => {
+ ::core::option::Option::Some("unexpected end of text block")
+ }
+ ERROR_STRING_BLOCK_MISSING_NEW_LINE => {
+ ::core::option::Option::Some("text block requires new line after |||")
+ }
+ ERROR_STRING_BLOCK_MISSING_TERMINATION => {
+ ::core::option::Option::Some("unterminated text block")
+ }
+ ERROR_STRING_BLOCK_MISSING_INDENT => {
+ ::core::option::Option::Some("text block first line must be indented")
+ }
+ ERROR_COMMENT_TOO_SHORT => ::core::option::Option::Some("comment too short"),
+ ERROR_COMMENT_UNTERMINATED => {
+ ::core::option::Option::Some("unterminated multi-line comment")
+ }
+ ERROR_NO_OPERATOR => ::core::option::Option::Some("expected operator"),
+ ERROR_MISSING_TOKEN => ::core::option::Option::Some("missing token"),
+ ERROR_UNEXPECTED_TOKEN => ::core::option::Option::Some("unexpected token"),
+ ERROR_CUSTOM => ::core::option::Option::Some("error"),
+ LEXING_ERROR => ::core::option::Option::Some("unexpected character"),
+ _ => None,
+ }
+ }
+ pub fn display_name(self) -> &'static str {
+ match self {
+ OR => "'||'",
+ NULL_COAELSE => "'??'",
+ AND => "'&&'",
+ BIT_OR => "'|'",
+ BIT_XOR => "'^'",
+ BIT_AND => "'&'",
+ EQ => "'=='",
+ NE => "'!='",
+ LT => "'<'",
+ GT => "'>'",
+ LE => "'<='",
+ GE => "'>='",
+ LHS => "'<<'",
+ RHS => "'>>'",
+ PLUS => "'+'",
+ MINUS => "'-'",
+ MUL => "'*'",
+ DIV => "'/'",
+ MODULO => "'%'",
+ NOT => "'!'",
+ BIT_NOT => "'~'",
+ L_BRACK => "'['",
+ R_BRACK => "']'",
+ L_PAREN => "'('",
+ R_PAREN => "')'",
+ L_BRACE => "'{'",
+ R_BRACE => "'}'",
+ COLON => "':'",
+ SEMI => "';'",
+ DOT => "'.'",
+ DOTDOTDOT => "'...'",
+ COMMA => "','",
+ DOLLAR => "'$'",
+ ASSIGN => "'='",
+ QUESTION_MARK => "'?'",
+ FLOAT => "number",
+ ERROR_FLOAT_JUNK_AFTER_POINT => "junk after decimal point in number literal",
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT => "junk after exponent in number literal",
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => "junk after exponent sign in number literal",
+ STRING_DOUBLE => "string",
+ ERROR_STRING_DOUBLE_UNTERMINATED => "unterminated double-quoted string",
+ STRING_SINGLE => "string",
+ ERROR_STRING_SINGLE_UNTERMINATED => "unterminated single-quoted string",
+ STRING_DOUBLE_VERBATIM => "string",
+ ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => {
+ "unterminated verbatim double-quoted string"
+ }
+ STRING_SINGLE_VERBATIM => "string",
+ ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => {
+ "unterminated verbatim single-quoted string"
+ }
+ ERROR_STRING_VERBATIM_MISSING_QUOTES => "verbatim string missing opening quotes",
+ STRING_BLOCK => "string",
+ ERROR_STRING_BLOCK_UNEXPECTED_END => "unexpected end of text block",
+ ERROR_STRING_BLOCK_MISSING_NEW_LINE => "text block requires new line after |||",
+ ERROR_STRING_BLOCK_MISSING_TERMINATION => "unterminated text block",
+ ERROR_STRING_BLOCK_MISSING_INDENT => "text block first line must be indented",
+ IDENT => "identifier",
+ WHITESPACE => "whitespace",
+ SINGLE_LINE_SLASH_COMMENT => "comment",
+ SINGLE_LINE_HASH_COMMENT => "comment",
+ MULTI_LINE_COMMENT => "comment",
+ ERROR_COMMENT_TOO_SHORT => "comment too short",
+ ERROR_COMMENT_UNTERMINATED => "unterminated multi-line comment",
+ ERROR_NO_OPERATOR => "expected operator",
+ ERROR_MISSING_TOKEN => "missing token",
+ ERROR_UNEXPECTED_TOKEN => "unexpected token",
+ ERROR_CUSTOM => "error",
+ TAILSTRICT_KW => "'tailstrict'",
+ LOCAL_KW => "'local'",
+ IMPORTSTR_KW => "'importstr'",
+ IMPORTBIN_KW => "'importbin'",
+ IMPORT_KW => "'import'",
+ IF_KW => "'if'",
+ THEN_KW => "'then'",
+ ELSE_KW => "'else'",
+ FUNCTION_KW => "'function'",
+ ERROR_KW => "'error'",
+ IN_KW => "'in'",
+ META_OBJECT_APPLY => "meta_object_apply",
+ NULL_KW => "'null'",
+ TRUE_KW => "'true'",
+ FALSE_KW => "'false'",
+ SELF_KW => "'self'",
+ SUPER_KW => "'super'",
+ FOR_KW => "'for'",
+ ASSERT_KW => "'assert'",
+ LEXING_ERROR => "unexpected character",
+ _ => "unknown",
+ }
+ }
pub fn from_raw(r: u16) -> Self {
assert!(r < Self::__LAST as u16);
unsafe { std::mem::transmute(r) }
xtask/src/sourcegen/kinds.rsdiffbeforeafterboth--- a/xtask/src/sourcegen/kinds.rs
+++ b/xtask/src/sourcegen/kinds.rs
@@ -19,6 +19,7 @@
is_lexer_error: bool,
regex: Option<String>,
priority: Option<u32>,
+ description: String,
},
/// Keyword - literal match of token
Keyword {
@@ -113,6 +114,24 @@
}
}
+ pub fn display_name(&self) -> String {
+ match self {
+ Self::Keyword { code, .. } => format!("'{code}'"),
+ Self::Literal { name, .. } => match name.as_str() {
+ "FLOAT" => "number".to_owned(),
+ "IDENT" => "identifier".to_owned(),
+ "STRING_DOUBLE" | "STRING_SINGLE" | "STRING_DOUBLE_VERBATIM"
+ | "STRING_SINGLE_VERBATIM" | "STRING_BLOCK" => "string".to_owned(),
+ "WHITESPACE" => "whitespace".to_owned(),
+ "SINGLE_LINE_SLASH_COMMENT" | "SINGLE_LINE_HASH_COMMENT"
+ | "MULTI_LINE_COMMENT" => "comment".to_owned(),
+ _ => name.to_lowercase(),
+ },
+ Self::Meta { name, .. } => name.to_lowercase(),
+ Self::Error { description, .. } => description.clone(),
+ }
+ }
+
pub fn method_name(&self) -> Ident {
match self {
Self::Keyword { name, .. } => {
@@ -138,7 +157,7 @@
});
$(define_kinds!($into = $($rest)*))?
}};
- ($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
+ ($into:ident = error($name:literal, $desc:literal $(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
{
let regex = None$(.or(Some($regex.to_owned())))?;
let priority = None$(.or(Some($priority)))?;
@@ -148,6 +167,7 @@
is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),
regex,
priority,
+ description: $desc.to_owned(),
});
}
$(define_kinds!($into = $($rest)*))?
@@ -248,31 +268,35 @@
"=" => "ASSIGN";
"?" => "QUESTION_MARK";
// Literals
- lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?";
- error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]";
- error("FLOAT_JUNK_AFTER_EXPONENT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]";
- error("FLOAT_JUNK_AFTER_EXPONENT_SIGN") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]";
+ lit("FLOAT") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?(?:[eE][+-]?[0-9]+(?:_[0-9]+)*)?";
+ error("FLOAT_JUNK_AFTER_POINT", "junk after decimal point in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)\.[^0-9]";
+ error("FLOAT_JUNK_AFTER_EXPONENT", "junk after exponent in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][^+\-0-9]";
+ error("FLOAT_JUNK_AFTER_EXPONENT_SIGN", "junk after exponent sign in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][+-][^0-9]";
lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";
- error("STRING_DOUBLE_UNTERMINATED") => "\"(?s:[^\"\\\\]|\\\\.)*";
+ error("STRING_DOUBLE_UNTERMINATED", "unterminated double-quoted string") => "\"(?s:[^\"\\\\]|\\\\.)*";
lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";
- error("STRING_SINGLE_UNTERMINATED") => "'(?s:[^'\\\\]|\\\\.)*";
+ error("STRING_SINGLE_UNTERMINATED", "unterminated single-quoted string") => "'(?s:[^'\\\\]|\\\\.)*";
lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";
- error("STRING_DOUBLE_VERBATIM_UNTERMINATED") => "@\"(?:[^\"]|\"\")*";
+ error("STRING_DOUBLE_VERBATIM_UNTERMINATED", "unterminated verbatim double-quoted string") => "@\"(?:[^\"]|\"\")*";
lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";
- error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";
- error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";
+ error("STRING_SINGLE_VERBATIM_UNTERMINATED", "unterminated verbatim single-quoted string") => "@'(?:[^']|'')*";
+ error("STRING_VERBATIM_MISSING_QUOTES", "verbatim string missing opening quotes") => "@[^\"'\\s]\\S+";
lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";
- error("STRING_BLOCK_UNEXPECTED_END", lexer = true);
- error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);
- error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);
- error("STRING_BLOCK_MISSING_INDENT", lexer = true);
+ error("STRING_BLOCK_UNEXPECTED_END", "unexpected end of text block", lexer = true);
+ error("STRING_BLOCK_MISSING_NEW_LINE", "text block requires new line after |||", lexer = true);
+ error("STRING_BLOCK_MISSING_TERMINATION", "unterminated text block", lexer = true);
+ error("STRING_BLOCK_MISSING_INDENT", "text block first line must be indented", lexer = true);
lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";
lit("WHITESPACE") => r"[ \t\n\r]+";
lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";
lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";
lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";
- error("COMMENT_TOO_SHORT") => r"/\*/";
- error("COMMENT_UNTERMINATED") => r"/\*([^*/]|\*[^/])+";
+ error("COMMENT_TOO_SHORT", "comment too short") => r"/\*/";
+ error("COMMENT_UNTERMINATED", "unterminated multi-line comment") => r"/\*([^*/]|\*[^/])+";
+ error("NO_OPERATOR", "expected operator");
+ error("MISSING_TOKEN", "missing token");
+ error("UNEXPECTED_TOKEN", "unexpected token");
+ error("CUSTOM", "error");
];
kinds
}
xtask/src/sourcegen/mod.rsdiffbeforeafterboth--- a/xtask/src/sourcegen/mod.rs
+++ b/xtask/src/sourcegen/mod.rs
@@ -56,14 +56,7 @@
});
}
SpecialName::Error => {
- eprintln!("implicit error: {name}");
- kinds.define_token(TokenKind::Error {
- grammar_name: token.to_owned(),
- name: format!("ERROR_{name}"),
- regex: None,
- priority: None,
- is_lexer_error: true,
- });
+ panic!("error token ERROR_{name} must be explicitly defined in jsonnet_kinds()");
}
}
continue;
@@ -170,6 +163,24 @@
quote! {}
};
+ let error_desc_arms = kinds.tokens().filter_map(|t| {
+ if let TokenKind::Error {
+ name, description, ..
+ } = t
+ {
+ let ident = format_ident!("{name}");
+ Some(quote! { #ident => ::core::option::Option::Some(#description) })
+ } else {
+ None
+ }
+ });
+
+ let display_name_arms = kinds.tokens().map(|t| {
+ let ident = format_ident!("{}", t.name());
+ let display = t.display_name();
+ quote! { #ident => #display }
+ });
+
let ast = quote! {
#![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]
@@ -200,6 +211,22 @@
#is_enum
+ pub fn error_description(self) -> Option<&'static str> {
+ match self {
+ #(#error_desc_arms,)*
+ LEXING_ERROR => ::core::option::Option::Some("unexpected character"),
+ _ => None,
+ }
+ }
+
+ pub fn display_name(self) -> &'static str {
+ match self {
+ #(#display_name_arms,)*
+ LEXING_ERROR => "unexpected character",
+ _ => "unknown",
+ }
+ }
+
pub fn from_raw(r: u16) -> Self {
assert!(r < Self::__LAST as u16);
unsafe { std::mem::transmute(r) }