git.delta.rocks / jrsonnet / refs/commits / 7c03fc40023d

--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,7 +14,7 @@
 jrsonnet-evaluator = { path = "./crates/jrsonnet-evaluator", version = "0.5.0-pre97" }
 jrsonnet-macros = { path = "./crates/jrsonnet-macros", version = "0.5.0-pre97" }
 jrsonnet-ir = { path = "./crates/jrsonnet-ir", version = "0.5.0-pre97" }
-jrsonnet-ir-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }
+jrsonnet-ir-parser = { path = "./crates/jrsonnet-ir-parser", version = "0.5.0-pre97" }
 jrsonnet-peg-parser = { path = "./crates/jrsonnet-peg-parser", version = "0.5.0-pre97" }
 jrsonnet-rowan-parser = { path = "./crates/jrsonnet-rowan-parser", version = "0.5.0-pre97" }
 jrsonnet-interner = { path = "./crates/jrsonnet-interner", version = "0.5.0-pre97" }

--- a/crates/jrsonnet-lexer/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-lexer/src/generated/syntax_kinds.rs
@@ -132,6 +132,10 @@
 	ERROR_COMMENT_TOO_SHORT,
 	#[regex("/\\*([^*/]|\\*[^/])+")]
 	ERROR_COMMENT_UNTERMINATED,
+	ERROR_NO_OPERATOR,
+	ERROR_MISSING_TOKEN,
+	ERROR_UNEXPECTED_TOKEN,
+	ERROR_CUSTOM,
 	#[token("tailstrict")]
 	TAILSTRICT_KW,
 	#[token("local")]
@@ -155,7 +159,6 @@
 	#[token("in")]
 	IN_KW,
 	META_OBJECT_APPLY,
-	ERROR_NO_OPERATOR,
 	#[token("null")]
 	NULL_KW,
 	#[token("true")]
@@ -170,9 +173,6 @@
 	FOR_KW,
 	#[token("assert")]
 	ASSERT_KW,
-	ERROR_MISSING_TOKEN,
-	ERROR_UNEXPECTED_TOKEN,
-	ERROR_CUSTOM,
 	LEXING_ERROR,
 	__LAST_TOKEN,
 	#[doc(hidden)]

--- a/crates/jrsonnet-lexer/src/string_block.rs
+++ b/crates/jrsonnet-lexer/src/string_block.rs
@@ -211,6 +211,7 @@
 
 	// Process leading blank lines before calculating string block indent
 	while ctx.peek() == Some('\n') {
+		lex.mark_line("");
 		ctx.next();
 	}
 

--- a/crates/jrsonnet-rowan-parser/jsonnet.ungram
+++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram
@@ -209,7 +209,7 @@
 |   FieldNameDynamic
 
 Visibility =
-    ':' v1:':'? v2:':'?
+    ':' ':'? ':'?
 
 Literal =
     'null'

--- a/xtask/src/sourcegen/kinds.rs
+++ b/xtask/src/sourcegen/kinds.rs
@@ -19,6 +19,7 @@
 		is_lexer_error: bool,
 		regex: Option<String>,
 		priority: Option<u32>,
+		description: String,
 	},
 	/// Keyword - literal match of token
 	Keyword {
@@ -113,6 +114,24 @@
 		}
 	}
 
+	pub fn display_name(&self) -> String {
+		match self {
+			Self::Keyword { code, .. } => format!("'{code}'"),
+			Self::Literal { name, .. } => match name.as_str() {
+				"FLOAT" => "number".to_owned(),
+				"IDENT" => "identifier".to_owned(),
+				"STRING_DOUBLE" | "STRING_SINGLE" | "STRING_DOUBLE_VERBATIM"
+				| "STRING_SINGLE_VERBATIM" | "STRING_BLOCK" => "string".to_owned(),
+				"WHITESPACE" => "whitespace".to_owned(),
+				"SINGLE_LINE_SLASH_COMMENT" | "SINGLE_LINE_HASH_COMMENT"
+				| "MULTI_LINE_COMMENT" => "comment".to_owned(),
+				_ => name.to_lowercase(),
+			},
+			Self::Meta { name, .. } => name.to_lowercase(),
+			Self::Error { description, .. } => description.clone(),
+		}
+	}
+
 	pub fn method_name(&self) -> Ident {
 		match self {
 			Self::Keyword { name, .. } => {
@@ -138,7 +157,7 @@
 		});
 		$(define_kinds!($into = $($rest)*))?
 	}};
-	($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
+	($into:ident = error($name:literal, $desc:literal $(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
 		{
 			let regex = None$(.or(Some($regex.to_owned())))?;
 			let priority = None$(.or(Some($priority)))?;
@@ -148,6 +167,7 @@
 				is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),
 				regex,
 				priority,
+				description: $desc.to_owned(),
 			});
 		}
 		$(define_kinds!($into = $($rest)*))?
@@ -248,31 +268,35 @@
 		"=" => "ASSIGN";
 		"?" => "QUESTION_MARK";
 		// Literals
-		lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?";
-		error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]";
-		error("FLOAT_JUNK_AFTER_EXPONENT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]";
-		error("FLOAT_JUNK_AFTER_EXPONENT_SIGN") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]";
+		lit("FLOAT") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?(?:[eE][+-]?[0-9]+(?:_[0-9]+)*)?";
+		error("FLOAT_JUNK_AFTER_POINT", "junk after decimal point in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)\.[^0-9]";
+		error("FLOAT_JUNK_AFTER_EXPONENT", "junk after exponent in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][^+\-0-9]";
+		error("FLOAT_JUNK_AFTER_EXPONENT_SIGN", "junk after exponent sign in number literal") => r"(?:0|[1-9][0-9]*(?:_[0-9]+)*)(?:\.[0-9]+(?:_[0-9]+)*)?[eE][+-][^0-9]";
 		lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";
-		error("STRING_DOUBLE_UNTERMINATED") => "\"(?s:[^\"\\\\]|\\\\.)*";
+		error("STRING_DOUBLE_UNTERMINATED", "unterminated double-quoted string") => "\"(?s:[^\"\\\\]|\\\\.)*";
 		lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";
-		error("STRING_SINGLE_UNTERMINATED") => "'(?s:[^'\\\\]|\\\\.)*";
+		error("STRING_SINGLE_UNTERMINATED", "unterminated single-quoted string") => "'(?s:[^'\\\\]|\\\\.)*";
 		lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";
-		error("STRING_DOUBLE_VERBATIM_UNTERMINATED") => "@\"(?:[^\"]|\"\")*";
+		error("STRING_DOUBLE_VERBATIM_UNTERMINATED", "unterminated verbatim double-quoted string") => "@\"(?:[^\"]|\"\")*";
 		lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";
-		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";
-		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";
+		error("STRING_SINGLE_VERBATIM_UNTERMINATED", "unterminated verbatim single-quoted string") => "@'(?:[^']|'')*";
+		error("STRING_VERBATIM_MISSING_QUOTES", "verbatim string missing opening quotes") => "@[^\"'\\s]\\S+";
 		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";
-		error("STRING_BLOCK_UNEXPECTED_END", lexer = true);
-		error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);
-		error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);
-		error("STRING_BLOCK_MISSING_INDENT", lexer = true);
+		error("STRING_BLOCK_UNEXPECTED_END", "unexpected end of text block", lexer = true);
+		error("STRING_BLOCK_MISSING_NEW_LINE", "text block requires new line after |||", lexer = true);
+		error("STRING_BLOCK_MISSING_TERMINATION", "unterminated text block", lexer = true);
+		error("STRING_BLOCK_MISSING_INDENT", "text block first line must be indented", lexer = true);
 		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";
 		lit("WHITESPACE") => r"[ \t\n\r]+";
 		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*?(\r\n|\n)?";
 		lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*?(\r\n|\n)?";
 		lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";
-		error("COMMENT_TOO_SHORT") => r"/\*/";
-		error("COMMENT_UNTERMINATED") =>  r"/\*([^*/]|\*[^/])+";
+		error("COMMENT_TOO_SHORT", "comment too short") => r"/\*/";
+		error("COMMENT_UNTERMINATED", "unterminated multi-line comment") =>  r"/\*([^*/]|\*[^/])+";
+		error("NO_OPERATOR", "expected operator");
+		error("MISSING_TOKEN", "missing token");
+		error("UNEXPECTED_TOKEN", "unexpected token");
+		error("CUSTOM", "error");
 	];
 	kinds
 }

--- a/xtask/src/sourcegen/mod.rs
+++ b/xtask/src/sourcegen/mod.rs
@@ -56,14 +56,7 @@
 						});
 					}
 					SpecialName::Error => {
-						eprintln!("implicit error: {name}");
-						kinds.define_token(TokenKind::Error {
-							grammar_name: token.to_owned(),
-							name: format!("ERROR_{name}"),
-							regex: None,
-							priority: None,
-							is_lexer_error: true,
-						});
+						panic!("error token ERROR_{name} must be explicitly defined in jsonnet_kinds()");
 					}
 				}
 				continue;
@@ -170,6 +163,24 @@
 		quote! {}
 	};
 
+	let error_desc_arms = kinds.tokens().filter_map(|t| {
+		if let TokenKind::Error {
+			name, description, ..
+		} = t
+		{
+			let ident = format_ident!("{name}");
+			Some(quote! { #ident => ::core::option::Option::Some(#description) })
+		} else {
+			None
+		}
+	});
+
+	let display_name_arms = kinds.tokens().map(|t| {
+		let ident = format_ident!("{}", t.name());
+		let display = t.display_name();
+		quote! { #ident => #display }
+	});
+
 	let ast = quote! {
 		#![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]
 
@@ -200,6 +211,22 @@
 
 			#is_enum
 
+			pub fn error_description(self) -> Option<&'static str> {
+				match self {
+					#(#error_desc_arms,)*
+					LEXING_ERROR => ::core::option::Option::Some("unexpected character"),
+					_ => None,
+				}
+			}
+
+			pub fn display_name(self) -> &'static str {
+				match self {
+					#(#display_name_arms,)*
+					LEXING_ERROR => "unexpected character",
+					_ => "unknown",
+				}
+			}
+
 			pub fn from_raw(r: u16) -> Self {
 				assert!(r < Self::__LAST as u16);
 				unsafe { std::mem::transmute(r) }