git.delta.rocks / jrsonnet / refs/commits / 5ad3c0601af8

--- a/cmds/jrsonnet-fmt/src/main.rs
+++ b/cmds/jrsonnet-fmt/src/main.rs
@@ -5,7 +5,7 @@
 	nodes::{
 		ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart,
 		DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal,
-		Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, String,
+		Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text,
 		UnaryOperator,
 	},
 	AstToken, SyntaxToken,
@@ -91,7 +91,7 @@
 	}
 }
 
-impl Printable for String {
+impl Printable for Text {
 	fn print(&self) -> PrintItems {
 		p!(new: str(&format!("{}", self)))
 	}
@@ -168,7 +168,7 @@
 			FieldName::FieldNameFixed(f) => {
 				if let Some(id) = f.id() {
 					p!(new: {id})
-				} else if let Some(str) = f.string() {
+				} else if let Some(str) = f.text() {
 					p!(new: {str})
 				} else {
 					p!(new: str("/*missing FieldName*/"))
@@ -371,7 +371,7 @@
 			Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")),
 			Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")),
 			Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")),
-			Expr::ExprString(s) => p!(new: {s.string()}),
+			Expr::ExprString(s) => p!(new: {s.text()}),
 			Expr::ExprNumber(n) => p!(new: {n.number()}),
 			Expr::ExprArray(a) => {
 				let mut pi = p!(new: str("[") >i nl);
@@ -393,7 +393,7 @@
 				pi
 			}
 			Expr::ExprImport(v) => {
-				p!(new: {v.import_kind()} str(" ") {v.string()})
+				p!(new: {v.import_kind()} str(" ") {v.text()})
 			}
 			Expr::ExprVar(n) => p!(new: {n.name()}),
 			Expr::ExprLocal(l) => {

--- a/crates/jrsonnet-rowan-parser/jsonnet.ungram
+++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram
@@ -48,7 +48,7 @@
     name:Name
     ')'
 ExprString =
-    String
+    Text
 ExprNumber =
     Number
 ExprArray =
@@ -67,7 +67,7 @@
     ']'
 
 ExprImport =
-    ImportKind String
+    ImportKind Text
 
 ImportKind =
     'importstr'
@@ -217,7 +217,7 @@
 
 FieldNameFixed =
     id:Name
-|   String
+|   Text
 FieldNameDynamic =
     '['
     Expr
@@ -239,16 +239,27 @@
 |   '$'
 |   'super'
 
-String =
+Text =
     'LIT_STRING_DOUBLE!'
+|   'ERROR_STRING_DOUBLE_UNTERMINATED!'
 |   'LIT_STRING_SINGLE!'
+|   'ERROR_STRING_SINGLE_UNTERMINATED!'
 |   'LIT_STRING_DOUBLE_VERBATIM!'
+|   'ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED!'
 |   'LIT_STRING_SINGLE_VERBATIM!'
+|   'ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED!'
+|   'ERROR_STRING_VERBATIM_MISSING_QUOTES!'
 |   'LIT_STRING_BLOCK!'
+|   'ERROR_STRING_BLOCK_UNEXPECTED_END!'
+|   'ERROR_STRING_BLOCK_MISSING_NEW_LINE!'
+|   'ERROR_STRING_BLOCK_MISSING_TERMINATION!'
+|   'ERROR_STRING_BLOCK_MISSING_INDENT!'
 
 Number =
     'LIT_FLOAT!'
-|   'META_FORCE_ENUM!'
+|   'ERROR_FLOAT_JUNK_AFTER_POINT!'
+|   'ERROR_FLOAT_JUNK_AFTER_EXPONENT!'
+|   'ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN!'
 
 ForSpec =
     'for'
@@ -347,3 +358,12 @@
 TrueExpr=Expr
 FalseExpr=Expr
 LhsExpr=Expr
+
+// Trivia - tokens which will be implicitly skipped for parser
+Trivia =
+    'LIT_WHITESPACE!'
+|   'LIT_MULTI_LINE_COMMENT!'
+|   'ERROR_COMMENT_TOO_SHORT!'
+|   'ERROR_COMMENT_UNTERMINATED!'
+|   'LIT_SINGLE_LINE_HASH_COMMENT!'
+|   'LIT_SINGLE_LINE_SLASH_COMMENT!'

--- a/crates/jrsonnet-rowan-parser/src/event.rs
+++ b/crates/jrsonnet-rowan-parser/src/event.rs
@@ -4,8 +4,9 @@
 
 use crate::{
 	lex::Lexeme,
+	nodes::Trivia,
 	parser::{Parse, SyntaxError},
-	JsonnetLanguage, SyntaxKind,
+	AstToken, JsonnetLanguage, SyntaxKind,
 };
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -144,7 +145,7 @@
 	}
 	fn skip_whitespace(&mut self) {
 		while let Some(lexeme) = self.lexemes.get(self.offset) {
-			if !lexeme.kind.is_trivia() {
+			if !Trivia::can_cast(lexeme.kind) {
 				break;
 			}
 

255	pub(crate) syntax: SyntaxNode,255	pub(crate) syntax: SyntaxNode,
256}256}
257impl ExprString {257impl ExprString {
258	pub fn string(&self) -> Option<String> {258	pub fn text(&self) -> Option<Text> {
259		support::token_child(&self.syntax)259		support::token_child(&self.syntax)
260	}260	}
261}261}
332	pub fn import_kind(&self) -> Option<ImportKind> {332	pub fn import_kind(&self) -> Option<ImportKind> {
333		support::token_child(&self.syntax)333		support::token_child(&self.syntax)
334	}334	}
335	pub fn string(&self) -> Option<String> {335	pub fn text(&self) -> Option<Text> {
336		support::token_child(&self.syntax)336		support::token_child(&self.syntax)
337	}337	}
338}338}
692	pub fn id(&self) -> Option<Name> {692	pub fn id(&self) -> Option<Name> {
693		support::child(&self.syntax)693		support::child(&self.syntax)
694	}694	}
695	pub fn string(&self) -> Option<String> {695	pub fn text(&self) -> Option<Text> {
696		support::token_child(&self.syntax)696		support::token_child(&self.syntax)
697	}697	}
698}698}
1038}1038}
10391039
1040#[derive(Debug, Clone, PartialEq, Eq, Hash)]1040#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1041pub struct String {1041pub struct Text {
1042	syntax: SyntaxToken,1042	syntax: SyntaxToken,
1043	kind: StringKind,1043	kind: TextKind,
1044}1044}
10451045
1046#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]1046#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1047pub enum StringKind {1047pub enum TextKind {
1048	StringDouble,1048	StringDouble,
1049	ErrorStringDoubleUnterminated,
1049	StringSingle,1050	StringSingle,
1051	ErrorStringSingleUnterminated,
1050	StringDoubleVerbatim,1052	StringDoubleVerbatim,
1053	ErrorStringDoubleVerbatimUnterminated,
1051	StringSingleVerbatim,1054	StringSingleVerbatim,
1055	ErrorStringSingleVerbatimUnterminated,
1056	ErrorStringVerbatimMissingQuotes,
1052	StringBlock,1057	StringBlock,
1058	ErrorStringBlockUnexpectedEnd,
1059	ErrorStringBlockMissingNewLine,
1060	ErrorStringBlockMissingTermination,
1061	ErrorStringBlockMissingIndent,
1053}1062}
10541063
1055#[derive(Debug, Clone, PartialEq, Eq, Hash)]1064#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1061#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]1070#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1062pub enum NumberKind {1071pub enum NumberKind {
1063	Float,1072	Float,
1064	MetaForceEnum,1073	ErrorFloatJunkAfterPoint,
1074	ErrorFloatJunkAfterExponent,
1075	ErrorFloatJunkAfterExponentSign,
1065}1076}
10661077
1067#[derive(Debug, Clone, PartialEq, Eq, Hash)]1078#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1090	Colon,1101	Colon,
1091}1102}
1103
1104#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1105pub struct Trivia {
1106	syntax: SyntaxToken,
1107	kind: TriviaKind,
1108}
1109
1110#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1111pub enum TriviaKind {
1112	Whitespace,
1113	MultiLineComment,
1114	ErrorCommentTooShort,
1115	ErrorCommentUnterminated,
1116	SingleLineHashComment,
1117	SingleLineSlashComment,
1118}
1092impl AstNode for SourceFile {1119impl AstNode for SourceFile {
1093	fn can_cast(kind: SyntaxKind) -> bool {1120	fn can_cast(kind: SyntaxKind) -> bool {
1094		kind == SOURCE_FILE1121		kind == SOURCE_FILE
2677		std::fmt::Display::fmt(self.syntax(), f)2704		std::fmt::Display::fmt(self.syntax(), f)
2678	}2705	}
2679}2706}
2680impl AstToken for String {2707impl AstToken for Text {
2681	fn can_cast(kind: SyntaxKind) -> bool {2708	fn can_cast(kind: SyntaxKind) -> bool {
2682		match kind {2709		match kind {
2683			STRING_DOUBLE2710			STRING_DOUBLE
2711			| ERROR_STRING_DOUBLE_UNTERMINATED
2684			| STRING_SINGLE2712			| STRING_SINGLE
2713			| ERROR_STRING_SINGLE_UNTERMINATED
2714			| STRING_DOUBLE_VERBATIM
2715			| ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED
2716			| STRING_SINGLE_VERBATIM
2717			| ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED
2718			| ERROR_STRING_VERBATIM_MISSING_QUOTES
2719			| STRING_BLOCK
2720			| ERROR_STRING_BLOCK_UNEXPECTED_END
2685			| STRING_DOUBLE_VERBATIM2721			| ERROR_STRING_BLOCK_MISSING_NEW_LINE
2686			| STRING_SINGLE_VERBATIM2722			| ERROR_STRING_BLOCK_MISSING_TERMINATION
2687			| STRING_BLOCK => true,2723			| ERROR_STRING_BLOCK_MISSING_INDENT => true,
2688			_ => false,2724			_ => false,
2689		}2725		}
2690	}2726	}
2691	fn cast(syntax: SyntaxToken) -> Option<Self> {2727	fn cast(syntax: SyntaxToken) -> Option<Self> {
2692		let res = match syntax.kind() {2728		let res = match syntax.kind() {
2693			STRING_DOUBLE => String {2729			STRING_DOUBLE => Text {
2694				syntax,2730				syntax,
2695				kind: StringKind::StringDouble,2731				kind: TextKind::StringDouble,
2696			},2732			},
2733			ERROR_STRING_DOUBLE_UNTERMINATED => Text {
2734				syntax,
2735				kind: TextKind::ErrorStringDoubleUnterminated,
2736			},
2697			STRING_SINGLE => String {2737			STRING_SINGLE => Text {
2698				syntax,2738				syntax,
2699				kind: StringKind::StringSingle,2739				kind: TextKind::StringSingle,
2700			},2740			},
2741			ERROR_STRING_SINGLE_UNTERMINATED => Text {
2742				syntax,
2743				kind: TextKind::ErrorStringSingleUnterminated,
2744			},
2701			STRING_DOUBLE_VERBATIM => String {2745			STRING_DOUBLE_VERBATIM => Text {
2702				syntax,2746				syntax,
2703				kind: StringKind::StringDoubleVerbatim,2747				kind: TextKind::StringDoubleVerbatim,
2704			},2748			},
2749			ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => Text {
2750				syntax,
2751				kind: TextKind::ErrorStringDoubleVerbatimUnterminated,
2752			},
2705			STRING_SINGLE_VERBATIM => String {2753			STRING_SINGLE_VERBATIM => Text {
2706				syntax,2754				syntax,
2707				kind: StringKind::StringSingleVerbatim,2755				kind: TextKind::StringSingleVerbatim,
2708			},2756			},
2757			ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => Text {
2758				syntax,
2759				kind: TextKind::ErrorStringSingleVerbatimUnterminated,
2760			},
2761			ERROR_STRING_VERBATIM_MISSING_QUOTES => Text {
2762				syntax,
2763				kind: TextKind::ErrorStringVerbatimMissingQuotes,
2764			},
2709			STRING_BLOCK => String {2765			STRING_BLOCK => Text {
2710				syntax,2766				syntax,
2711				kind: StringKind::StringBlock,2767				kind: TextKind::StringBlock,
2712			},2768			},
2769			ERROR_STRING_BLOCK_UNEXPECTED_END => Text {
2770				syntax,
2771				kind: TextKind::ErrorStringBlockUnexpectedEnd,
2772			},
2773			ERROR_STRING_BLOCK_MISSING_NEW_LINE => Text {
2774				syntax,
2775				kind: TextKind::ErrorStringBlockMissingNewLine,
2776			},
2777			ERROR_STRING_BLOCK_MISSING_TERMINATION => Text {
2778				syntax,
2779				kind: TextKind::ErrorStringBlockMissingTermination,
2780			},
2781			ERROR_STRING_BLOCK_MISSING_INDENT => Text {
2782				syntax,
2783				kind: TextKind::ErrorStringBlockMissingIndent,
2784			},
2713			_ => return None,2785			_ => return None,
2714		};2786		};
2715		Some(res)2787		Some(res)
2718		&self.syntax2790		&self.syntax
2719	}2791	}
2720}2792}
2721impl String {2793impl Text {
2722	pub fn kind(&self) -> StringKind {2794	pub fn kind(&self) -> TextKind {
2723		self.kind2795		self.kind
2724	}2796	}
2725}2797}
2726impl std::fmt::Display for String {2798impl std::fmt::Display for Text {
2727	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {2799	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2728		std::fmt::Display::fmt(self.syntax(), f)2800		std::fmt::Display::fmt(self.syntax(), f)
2729	}2801	}
2731impl AstToken for Number {2803impl AstToken for Number {
2732	fn can_cast(kind: SyntaxKind) -> bool {2804	fn can_cast(kind: SyntaxKind) -> bool {
2733		match kind {2805		match kind {
2734			FLOAT | META_FORCE_ENUM => true,2806			FLOAT
2807			| ERROR_FLOAT_JUNK_AFTER_POINT
2808			| ERROR_FLOAT_JUNK_AFTER_EXPONENT
2809			| ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => true,
2735			_ => false,2810			_ => false,
2736		}2811		}
2737	}2812	}
2741				syntax,2816				syntax,
2742				kind: NumberKind::Float,2817				kind: NumberKind::Float,
2743			},2818			},
2744			META_FORCE_ENUM => Number {2819			ERROR_FLOAT_JUNK_AFTER_POINT => Number {
2745				syntax,2820				syntax,
2746				kind: NumberKind::MetaForceEnum,2821				kind: NumberKind::ErrorFloatJunkAfterPoint,
2747			},2822			},
2823			ERROR_FLOAT_JUNK_AFTER_EXPONENT => Number {
2824				syntax,
2825				kind: NumberKind::ErrorFloatJunkAfterExponent,
2826			},
2827			ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => Number {
2828				syntax,
2829				kind: NumberKind::ErrorFloatJunkAfterExponentSign,
2830			},
2748			_ => return None,2831			_ => return None,
2749		};2832		};
2750		Some(res)2833		Some(res)
2841		std::fmt::Display::fmt(self.syntax(), f)2924		std::fmt::Display::fmt(self.syntax(), f)
2842	}2925	}
2843}2926}
2927impl AstToken for Trivia {
2928	fn can_cast(kind: SyntaxKind) -> bool {
2929		match kind {
2930			WHITESPACE
2931			| MULTI_LINE_COMMENT
2932			| ERROR_COMMENT_TOO_SHORT
2933			| ERROR_COMMENT_UNTERMINATED
2934			| SINGLE_LINE_HASH_COMMENT
2935			| SINGLE_LINE_SLASH_COMMENT => true,
2936			_ => false,
2937		}
2938	}
2939	fn cast(syntax: SyntaxToken) -> Option<Self> {
2940		let res = match syntax.kind() {
2941			WHITESPACE => Trivia {
2942				syntax,
2943				kind: TriviaKind::Whitespace,
2944			},
2945			MULTI_LINE_COMMENT => Trivia {
2946				syntax,
2947				kind: TriviaKind::MultiLineComment,
2948			},
2949			ERROR_COMMENT_TOO_SHORT => Trivia {
2950				syntax,
2951				kind: TriviaKind::ErrorCommentTooShort,
2952			},
2953			ERROR_COMMENT_UNTERMINATED => Trivia {
2954				syntax,
2955				kind: TriviaKind::ErrorCommentUnterminated,
2956			},
2957			SINGLE_LINE_HASH_COMMENT => Trivia {
2958				syntax,
2959				kind: TriviaKind::SingleLineHashComment,
2960			},
2961			SINGLE_LINE_SLASH_COMMENT => Trivia {
2962				syntax,
2963				kind: TriviaKind::SingleLineSlashComment,
2964			},
2965			_ => return None,
2966		};
2967		Some(res)
2968	}
2969	fn syntax(&self) -> &SyntaxToken {
2970		&self.syntax
2971	}
2972}
2973impl Trivia {
2974	pub fn kind(&self) -> TriviaKind {
2975		self.kind
2976	}
2977}
2978impl std::fmt::Display for Trivia {
2979	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2980		std::fmt::Display::fmt(self.syntax(), f)
2981	}
2982}
2844impl std::fmt::Display for Expr {2983impl std::fmt::Display for Expr {
2845	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {2984	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2846		std::fmt::Display::fmt(self.syntax(), f)2985		std::fmt::Display::fmt(self.syntax(), f)

--- a/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
@@ -174,7 +174,6 @@
 	SELF_KW,
 	#[token("super")]
 	SUPER_KW,
-	META_FORCE_ENUM,
 	#[token("for")]
 	FOR_KW,
 	#[token("assert")]
@@ -253,10 +252,11 @@
 	BINARY_OPERATOR,
 	UNARY_OPERATOR,
 	LITERAL,
-	STRING,
+	TEXT,
 	NUMBER,
 	IMPORT_KIND,
 	VISIBILITY,
+	TRIVIA,
 	#[doc(hidden)]
 	__LAST,
 }
@@ -277,8 +277,8 @@
 	pub fn is_enum(self) -> bool {
 		match self {
 			EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER | FIELD | FIELD_NAME | DESTRUCT
-			| DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | STRING
-			| NUMBER | IMPORT_KIND | VISIBILITY => true,
+			| DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | TEXT | NUMBER
+			| IMPORT_KIND | VISIBILITY | TRIVIA => true,
 			_ => false,
 		}
 	}

--- a/crates/jrsonnet-rowan-parser/src/lex.rs
+++ b/crates/jrsonnet-rowan-parser/src/lex.rs
@@ -4,7 +4,10 @@
 use logos::Logos;
 use rowan::{TextRange, TextSize};
 
-use crate::SyntaxKind;
+use crate::{
+	string_block::{lex_str_block, StringBlockError},
+	SyntaxKind,
+};
 
 pub struct Lexer<'a> {
 	inner: logos::Lexer<'a, SyntaxKind>,
@@ -22,9 +25,34 @@
 	type Item = Lexeme<'a>;
 
 	fn next(&mut self) -> Option<Self::Item> {
-		let kind = self.inner.next()?;
+		use SyntaxKind::*;
+
+		let mut kind = self.inner.next()?;
 		let text = self.inner.slice();
 
+		if kind == STRING_BLOCK {
+			// We use custom lexer, which skips enough bytes, but not returns error
+			// Instead we should call lexer again to verify if there is something wrong with string block
+			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);
+			// In kinds, string blocks is parsed at least as `|||`
+			lexer.bump(3);
+			let res = lex_str_block(&mut lexer);
+			debug_assert!(lexer.next().is_none(), "str_block is lexed");
+			match res {
+				Ok(_) => {}
+				Err(e) => {
+					kind = match e {
+						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
+						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
+						StringBlockError::MissingTermination => {
+							ERROR_STRING_BLOCK_MISSING_TERMINATION
+						}
+						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
+					}
+				}
+			}
+		}
+
 		Some(Self::Item {
 			kind,
 			text,

--- a/crates/jrsonnet-rowan-parser/src/lib.rs
+++ b/crates/jrsonnet-rowan-parser/src/lib.rs
@@ -2,7 +2,6 @@
 
 mod ast;
 mod binary;
-mod classify;
 mod event;
 mod generated;
 mod language;

--- a/crates/jrsonnet-rowan-parser/src/marker.rs
+++ b/crates/jrsonnet-rowan-parser/src/marker.rs
@@ -44,10 +44,10 @@
 			!kind.is_enum(),
 			"{kind:?} is a enum kind, you should use variant kinds instead"
 		);
-		// TODO: is_parser should return true if enum variant has #[regex]/#[token] over it
+		// TODO: is_lexer should return true if enum variant has #[regex]/#[token] over it, or it is defined as lexer error explicitly
 		// debug_assert!(
-		// 	!kind.is_parser(),
-		// 	"{kind:?} should be only emitted by parser, not used directly"
+		// 	!kind.is_lexer(),
+		// 	"{kind:?} should be only emitted by lexer, not used directly"
 		// );
 		let event_at_pos = &mut p.events[self.start_event_idx];
 		assert_eq!(*event_at_pos, Event::Pending);

--- a/crates/jrsonnet-rowan-parser/src/parser.rs
+++ b/crates/jrsonnet-rowan-parser/src/parser.rs
@@ -8,10 +8,10 @@
 	event::Event,
 	lex::Lexeme,
 	marker::{AsRange, CompletedMarker, Marker, Ranger},
-	string_block::{lex_str_block, StringBlockError},
+	nodes::{Literal, Number, Text, Trivia},
 	token_set::SyntaxKindSet,
 	unary::UnaryOperator,
-	SyntaxKind,
+	AstToken, SyntaxKind,
 	SyntaxKind::*,
 	SyntaxNode, T, TS,
 };
@@ -36,6 +36,7 @@
 }
 
 pub struct Parser<'i> {
+	// TODO: remove all trivia before feeding to parser?
 	lexemes: &'i [Lexeme<'i>],
 	pub offset: usize,
 	pub events: Vec<Event>,
@@ -191,7 +192,7 @@
 		while self
 			.lexemes
 			.get(previous_token_idx)
-			.map_or(false, |l| l.kind.is_trivia())
+			.map_or(false, |l| Trivia::can_cast(l.kind))
 			&& previous_token_idx != 0
 		{
 			previous_token_idx -= 1;
@@ -200,13 +201,13 @@
 		Some(self.lexemes[previous_token_idx])
 	}
 	pub fn start_of_token(&self, mut idx: usize) -> TextSize {
-		while self.lexemes[idx].kind.is_trivia() {
+		while Trivia::can_cast(self.lexemes[idx].kind) {
 			idx += 1;
 		}
 		self.lexemes[idx].range.start()
 	}
 	pub fn end_of_token(&self, mut idx: usize) -> TextSize {
-		while self.lexemes[idx].kind.is_trivia() {
+		while Trivia::can_cast(self.lexemes[idx].kind) {
 			idx -= 1;
 		}
 		self.lexemes[idx].range.end()
@@ -267,7 +268,11 @@
 		self.bump();
 		Some(m.complete(self, SyntaxKind::ERROR))
 	}
-
+	fn bump_assert(&mut self, kind: SyntaxKind) {
+		self.skip_trivia();
+		assert!(self.at(kind), "expected {:?}", kind);
+		self.bump_remap(self.current());
+	}
 	fn bump(&mut self) {
 		self.skip_trivia();
 		self.bump_remap(self.current());
@@ -314,7 +319,7 @@
 			while self
 				.lexemes
 				.get(offset)
-				.map(|l| l.kind.is_trivia())
+				.map(|l| Trivia::can_cast(l.kind))
 				.unwrap_or(false)
 			{
 				offset += 1;
@@ -324,7 +329,7 @@
 		while self
 			.lexemes
 			.get(offset)
-			.map(|l| l.kind.is_trivia())
+			.map(|l| Trivia::can_cast(l.kind))
 			.unwrap_or(false)
 		{
 			offset += 1;
@@ -335,14 +340,10 @@
 		self.nth(0)
 	}
 	fn skip_trivia(&mut self) {
-		while self.peek_raw().is_trivia() {
+		while Trivia::can_cast(self.peek_raw()) {
 			self.offset += 1;
 		}
 	}
-	fn current_lexeme(&mut self) -> Option<&Lexeme> {
-		self.skip_trivia();
-		self.lexemes.get(self.offset)
-	}
 	fn peek_raw(&mut self) -> SyntaxKind {
 		self.lexemes
 			.get(self.offset)
@@ -516,8 +517,8 @@
 	} else if p.at(IDENT) {
 		name(p);
 		m.complete(p, FIELD_NAME_FIXED);
-	} else if p.current().is_string() {
-		string(p);
+	} else if Text::can_cast(p.current()) {
+		text(p);
 		m.complete(p, FIELD_NAME_FIXED);
 	} else {
 		p.error_with_recovery_set(TS![;]);
@@ -564,9 +565,8 @@
 	};
 }
 fn assertion(p: &mut Parser) {
-	assert!(p.at(T![assert]));
 	let m = p.start();
-	p.bump();
+	p.bump_assert(T![assert]);
 	expr(p).map(|c| c.wrap(p, LHS_EXPR));
 	if p.at(T![:]) {
 		p.bump();
@@ -575,10 +575,9 @@
 	m.complete(p, ASSERTION);
 }
 fn object(p: &mut Parser) -> CompletedMarker {
-	assert!(p.at(T!['{']));
 	let m_t = p.start();
 	let m = p.start();
-	p.bump();
+	p.bump_assert(T!['{']);
 
 	loop {
 		if p.at(T!['}']) {
@@ -619,9 +618,8 @@
 	m.complete(p, PARAM);
 }
 fn params_desc(p: &mut Parser) -> CompletedMarker {
-	assert!(p.at(T!['(']));
 	let m = p.start();
-	p.bump();
+	p.bump_assert(T!['(']);
 
 	loop {
 		if p.at(T![')']) {
@@ -640,8 +638,7 @@
 }
 fn args_desc(p: &mut Parser) {
 	let m = p.start();
-	assert!(p.at(T!['(']));
-	p.bump();
+	p.bump_assert(T!['(']);
 
 	let started_named = Cell::new(false);
 
@@ -674,10 +671,9 @@
 }
 
 fn array(p: &mut Parser) -> CompletedMarker {
-	assert!(p.at(T!['[']));
 	// Start the list node
 	let m = p.start();
-	p.bump(); // '['
+	p.bump_assert(T!['[']);
 
 	// This vec will have at most one element in case of correct input
 	let mut compspecs = Vec::with_capacity(1);
@@ -795,9 +791,8 @@
 	m.complete(p, NAME);
 }
 fn destruct_rest(p: &mut Parser) {
-	assert!(p.at(T![...]));
-	p.bump();
 	let m = p.start();
+	p.bump_assert(T![...]);
 	if p.at(IDENT) {
 		p.bump()
 	}
@@ -817,9 +812,8 @@
 	m.complete(p, DESTRUCT_OBJECT_FIELD);
 }
 fn obj_local(p: &mut Parser) {
-	assert!(p.at(T![local]));
 	let m = p.start();
-	p.bump();
+	p.bump_assert(T![local]);
 	bind(p);
 	m.complete(p, OBJ_LOCAL);
 }
@@ -903,52 +897,29 @@
 		m.complete(p, BIND_DESTRUCT)
 	};
 }
-fn string(p: &mut Parser) {
-	assert!(p.current().is_string());
-	if p.at(STRING_BLOCK) {
-		// We use custom lexer, which skips enough bytes, but not returns error
-		// Instead we should call lexer again to verify if there is something wrong with string block
-		let mut lexer = logos::Lexer::<SyntaxKind>::new(dbg!(
-			&p.current_lexeme().expect("parser is at string block").text
-		));
-		// In kinds, string blocks is parsed at least as `|||`
-		lexer.bump(3);
-		let res = lex_str_block(&mut lexer);
-		debug_assert!(lexer.next().is_none(), "str_block is lexed");
-		match res {
-			Ok(_) => {
-				p.bump();
-			}
-			Err(e) => p.bump_remap(match e {
-				StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
-				StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
-				StringBlockError::MissingTermination => ERROR_STRING_BLOCK_MISSING_TERMINATION,
-				StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
-			}),
-		}
-	} else {
-		p.bump();
-	}
+fn text(p: &mut Parser) {
+	assert!(Text::can_cast(p.current()));
+	p.bump();
 }
 fn number(p: &mut Parser) {
-	assert!(p.current().is_number());
+	assert!(Number::can_cast(p.current()));
 	p.bump();
 }
 fn literal(p: &mut Parser) {
-	assert!(p.current().is_literal());
+	assert!(Literal::can_cast(p.current()));
 	p.bump();
 }
 fn lhs_basic(p: &mut Parser) -> Option<CompletedMarker> {
 	let _e = p.expected_syntax_name("value");
-	Some(if p.current().is_literal() {
+	Some(if Literal::can_cast(p.current()) {
 		let m = p.start();
 		literal(p);
 		m.complete(p, EXPR_LITERAL)
-	} else if p.current().is_string() {
+	} else if Text::can_cast(p.current()) {
 		let m = p.start();
-		string(p);
+		text(p);
 		m.complete(p, EXPR_STRING)
-	} else if p.current().is_number() {
+	} else if Number::can_cast(p.current()) {
 		let m = p.start();
 		number(p);
 		m.complete(p, EXPR_NUMBER)
@@ -1025,7 +996,7 @@
 	} else if p.at(T![import]) || p.at(T![importstr]) || p.at(T![importbin]) {
 		let m = p.start();
 		p.bump();
-		string(p);
+		text(p);
 		m.complete(p, EXPR_IMPORT)
 	} else if p.at(T![-]) || p.at(T![!]) || p.at(T![~]) {
 		let op = match p.current() {
@@ -1044,8 +1015,7 @@
 		let m = p.start();
 		p.bump();
 		expr(p);
-		assert!(p.at(T![')']));
-		p.bump();
+		p.expect(T![')']);
 		m.complete(p, EXPR_PARENED)
 	} else {
 		p.error_with_recovery_set(TS![]);

--- a/xtask/src/sourcegen/kinds.rs
+++ b/xtask/src/sourcegen/kinds.rs
@@ -10,10 +10,12 @@
 pub enum TokenKind {
 	/// May exist in token tree, but never in source code
 	Meta { grammar_name: String, name: String },
-	/// Specific parsing errors may be emitted as this type of kind
+	/// Specific parsing/lexing errors may be emitted as this type of kind
 	Error {
 		grammar_name: String,
 		name: String,
+		/// Is this error returned by lexer directly, or from lex.rs
+		is_lexer_error: bool,
 		regex: Option<String>,
 		priority: Option<u32>,
 	},
@@ -133,13 +135,18 @@
 		});
 		$(define_kinds!($into = $($rest)*))?
 	}};
-	($into:ident = error($name:literal$(, priority = $priority:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
-		$into.define_token(TokenKind::Error {
-			grammar_name: format!("ERROR_{}!", $name),
-			name: format!("ERROR_{}", $name),
-			regex: None$(.or(Some($regex.to_owned())))?,
-			priority: None$(.or(Some($priority)))?,
-		});
+	($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
+		{
+			let regex = None$(.or(Some($regex.to_owned())))?;
+			let priority = None$(.or(Some($priority)))?;
+			$into.define_token(TokenKind::Error {
+				grammar_name: format!("ERROR_{}!", $name),
+				name: format!("ERROR_{}", $name),
+				is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),
+				regex,
+				priority,
+			});
+		}
 		$(define_kinds!($into = $($rest)*))?
 	}};
 	($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{
@@ -258,10 +265,10 @@
 		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";
 		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";
 		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";
-		error("STRING_BLOCK_UNEXPECTED_END");
-		error("STRING_BLOCK_MISSING_NEW_LINE");
-		error("STRING_BLOCK_MISSING_TERMINATION");
-		error("STRING_BLOCK_MISSING_INDENT");
+		error("STRING_BLOCK_UNEXPECTED_END", lexer = true);
+		error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);
+		error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);
+		error("STRING_BLOCK_MISSING_INDENT", lexer = true);
 		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";
 		lit("WHITESPACE") => r"[ \t\n\r]+";
 		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";

--- a/xtask/src/sourcegen/mod.rs
+++ b/xtask/src/sourcegen/mod.rs
@@ -48,20 +48,28 @@
 			if let Some((special, name)) = classify_special(token) {
 				match special {
 					SpecialName::Literal => panic!("literal is not defined: {name}"),
-					SpecialName::Meta => kinds.define_token(TokenKind::Meta {
-						grammar_name: token.to_owned(),
-						name: format!("META_{}", name),
-					}),
-					SpecialName::Error => kinds.define_token(TokenKind::Error {
-						grammar_name: token.to_owned(),
-						name: format!("ERROR_{}", name),
-						regex: None,
-						priority: None,
-					}),
+					SpecialName::Meta => {
+						eprintln!("implicit meta: {}", name);
+						kinds.define_token(TokenKind::Meta {
+							grammar_name: token.to_owned(),
+							name: format!("META_{}", name),
+						})
+					}
+					SpecialName::Error => {
+						eprintln!("implicit error: {}", name);
+						kinds.define_token(TokenKind::Error {
+							grammar_name: token.to_owned(),
+							name: format!("ERROR_{}", name),
+							regex: None,
+							priority: None,
+							is_lexer_error: true,
+						})
+					}
 				};
 				continue;
 			};
 			let name = to_upper_snake_case(token);
+			eprintln!("implicit kw: {}", token);
 			kinds.define_token(TokenKind::Keyword {
 				code: token.to_owned(),
 				name: format!("{name}_KW"),

--- a/xtask/src/sourcegen/util.rs
+++ b/xtask/src/sourcegen/util.rs
@@ -13,10 +13,7 @@
 		}
 	}
 
-	eprintln!(" {} was not up-to-date, updating\n", file.display());
-	if std::env::var("CI").is_ok() {
-		eprintln!("NOTE: run `cargo xtask` locally and commit the updated files\n");
-	}
+	eprintln!("{} was not up-to-date, updating", file.display());
 	if let Some(parent) = file.parent() {
 		let _ = fs::create_dir_all(parent);
 	}