git.delta.rocks / jrsonnet / refs/commits / 5ad3c0601af8

5	nodes::{5	nodes::{
6		ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart,6		ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart,
7		DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal,7		DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal,
8		Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, String,8		Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text,
9		UnaryOperator,9		UnaryOperator,
10	},10	},
11	AstToken, SyntaxToken,11	AstToken, SyntaxToken,
91	}91	}
92}92}
9393
94impl Printable for String {94impl Printable for Text {
95	fn print(&self) -> PrintItems {95	fn print(&self) -> PrintItems {
96		p!(new: str(&format!("{}", self)))96		p!(new: str(&format!("{}", self)))
97	}97	}
168			FieldName::FieldNameFixed(f) => {168			FieldName::FieldNameFixed(f) => {
169				if let Some(id) = f.id() {169				if let Some(id) = f.id() {
170					p!(new: {id})170					p!(new: {id})
171				} else if let Some(str) = f.string() {171				} else if let Some(str) = f.text() {
172					p!(new: {str})172					p!(new: {str})
173				} else {173				} else {
174					p!(new: str("/*missing FieldName*/"))174					p!(new: str("/*missing FieldName*/"))
371			Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")),371			Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")),
372			Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")),372			Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")),
373			Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")),373			Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")),
374			Expr::ExprString(s) => p!(new: {s.string()}),374			Expr::ExprString(s) => p!(new: {s.text()}),
375			Expr::ExprNumber(n) => p!(new: {n.number()}),375			Expr::ExprNumber(n) => p!(new: {n.number()}),
376			Expr::ExprArray(a) => {376			Expr::ExprArray(a) => {
377				let mut pi = p!(new: str("[") >i nl);377				let mut pi = p!(new: str("[") >i nl);
393				pi393				pi
394			}394			}
395			Expr::ExprImport(v) => {395			Expr::ExprImport(v) => {
396				p!(new: {v.import_kind()} str(" ") {v.string()})396				p!(new: {v.import_kind()} str(" ") {v.text()})
397			}397			}
398			Expr::ExprVar(n) => p!(new: {n.name()}),398			Expr::ExprVar(n) => p!(new: {n.name()}),
399			Expr::ExprLocal(l) => {399			Expr::ExprLocal(l) => {

48    name:Name48    name:Name
49    ')'49    ')'
50ExprString =50ExprString =
51    String51    Text
52ExprNumber =52ExprNumber =
53    Number53    Number
54ExprArray =54ExprArray =
67    ']'67    ']'
6868
69ExprImport =69ExprImport =
70    ImportKind String70    ImportKind Text
7171
72ImportKind =72ImportKind =
73    'importstr'73    'importstr'
217217
218FieldNameFixed =218FieldNameFixed =
219    id:Name219    id:Name
220|   String220|   Text
221FieldNameDynamic =221FieldNameDynamic =
222    '['222    '['
223    Expr223    Expr
239|   '$'239|   '$'
240|   'super'240|   'super'
241241
242String =242Text =
243    'LIT_STRING_DOUBLE!'243    'LIT_STRING_DOUBLE!'
244|   'ERROR_STRING_DOUBLE_UNTERMINATED!'
244|   'LIT_STRING_SINGLE!'245|   'LIT_STRING_SINGLE!'
246|   'ERROR_STRING_SINGLE_UNTERMINATED!'
245|   'LIT_STRING_DOUBLE_VERBATIM!'247|   'LIT_STRING_DOUBLE_VERBATIM!'
248|   'ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED!'
246|   'LIT_STRING_SINGLE_VERBATIM!'249|   'LIT_STRING_SINGLE_VERBATIM!'
250|   'ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED!'
251|   'ERROR_STRING_VERBATIM_MISSING_QUOTES!'
247|   'LIT_STRING_BLOCK!'252|   'LIT_STRING_BLOCK!'
253|   'ERROR_STRING_BLOCK_UNEXPECTED_END!'
254|   'ERROR_STRING_BLOCK_MISSING_NEW_LINE!'
255|   'ERROR_STRING_BLOCK_MISSING_TERMINATION!'
256|   'ERROR_STRING_BLOCK_MISSING_INDENT!'
248257
249Number =258Number =
250    'LIT_FLOAT!'259    'LIT_FLOAT!'
251|   'META_FORCE_ENUM!'260|   'ERROR_FLOAT_JUNK_AFTER_POINT!'
261|   'ERROR_FLOAT_JUNK_AFTER_EXPONENT!'
262|   'ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN!'
252263
253ForSpec =264ForSpec =
254    'for'265    'for'
347TrueExpr=Expr358TrueExpr=Expr
348FalseExpr=Expr359FalseExpr=Expr
349LhsExpr=Expr360LhsExpr=Expr
361
362// Trivia - tokens which will be implicitly skipped for parser
363Trivia =
364    'LIT_WHITESPACE!'
365|   'LIT_MULTI_LINE_COMMENT!'
366|   'ERROR_COMMENT_TOO_SHORT!'
367|   'ERROR_COMMENT_UNTERMINATED!'
368|   'LIT_SINGLE_LINE_HASH_COMMENT!'
369|   'LIT_SINGLE_LINE_SLASH_COMMENT!'
350370

44
5use crate::{5use crate::{
6	lex::Lexeme,6	lex::Lexeme,
7	nodes::Trivia,
7	parser::{Parse, SyntaxError},8	parser::{Parse, SyntaxError},
8	JsonnetLanguage, SyntaxKind,9	AstToken, JsonnetLanguage, SyntaxKind,
9};10};
1011
11#[derive(Clone, Debug, PartialEq, Eq)]12#[derive(Clone, Debug, PartialEq, Eq)]
144	}145	}
145	fn skip_whitespace(&mut self) {146	fn skip_whitespace(&mut self) {
146		while let Some(lexeme) = self.lexemes.get(self.offset) {147		while let Some(lexeme) = self.lexemes.get(self.offset) {
147			if !lexeme.kind.is_trivia() {148			if !Trivia::can_cast(lexeme.kind) {
148				break;149				break;
149			}150			}
150151

255	pub(crate) syntax: SyntaxNode,255	pub(crate) syntax: SyntaxNode,
256}256}
257impl ExprString {257impl ExprString {
258	pub fn string(&self) -> Option<String> {258	pub fn text(&self) -> Option<Text> {
259		support::token_child(&self.syntax)259		support::token_child(&self.syntax)
260	}260	}
261}261}
332	pub fn import_kind(&self) -> Option<ImportKind> {332	pub fn import_kind(&self) -> Option<ImportKind> {
333		support::token_child(&self.syntax)333		support::token_child(&self.syntax)
334	}334	}
335	pub fn string(&self) -> Option<String> {335	pub fn text(&self) -> Option<Text> {
336		support::token_child(&self.syntax)336		support::token_child(&self.syntax)
337	}337	}
338}338}
692	pub fn id(&self) -> Option<Name> {692	pub fn id(&self) -> Option<Name> {
693		support::child(&self.syntax)693		support::child(&self.syntax)
694	}694	}
695	pub fn string(&self) -> Option<String> {695	pub fn text(&self) -> Option<Text> {
696		support::token_child(&self.syntax)696		support::token_child(&self.syntax)
697	}697	}
698}698}
1038}1038}
10391039
1040#[derive(Debug, Clone, PartialEq, Eq, Hash)]1040#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1041pub struct String {1041pub struct Text {
1042	syntax: SyntaxToken,1042	syntax: SyntaxToken,
1043	kind: StringKind,1043	kind: TextKind,
1044}1044}
10451045
1046#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]1046#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1047pub enum StringKind {1047pub enum TextKind {
1048	StringDouble,1048	StringDouble,
1049	ErrorStringDoubleUnterminated,
1049	StringSingle,1050	StringSingle,
1051	ErrorStringSingleUnterminated,
1050	StringDoubleVerbatim,1052	StringDoubleVerbatim,
1053	ErrorStringDoubleVerbatimUnterminated,
1051	StringSingleVerbatim,1054	StringSingleVerbatim,
1055	ErrorStringSingleVerbatimUnterminated,
1056	ErrorStringVerbatimMissingQuotes,
1052	StringBlock,1057	StringBlock,
1058	ErrorStringBlockUnexpectedEnd,
1059	ErrorStringBlockMissingNewLine,
1060	ErrorStringBlockMissingTermination,
1061	ErrorStringBlockMissingIndent,
1053}1062}
10541063
1055#[derive(Debug, Clone, PartialEq, Eq, Hash)]1064#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1061#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]1070#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1062pub enum NumberKind {1071pub enum NumberKind {
1063	Float,1072	Float,
1064	MetaForceEnum,1073	ErrorFloatJunkAfterPoint,
1074	ErrorFloatJunkAfterExponent,
1075	ErrorFloatJunkAfterExponentSign,
1065}1076}
10661077
1067#[derive(Debug, Clone, PartialEq, Eq, Hash)]1078#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1090	Colon,1101	Colon,
1091}1102}
1103
1104#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1105pub struct Trivia {
1106	syntax: SyntaxToken,
1107	kind: TriviaKind,
1108}
1109
1110#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1111pub enum TriviaKind {
1112	Whitespace,
1113	MultiLineComment,
1114	ErrorCommentTooShort,
1115	ErrorCommentUnterminated,
1116	SingleLineHashComment,
1117	SingleLineSlashComment,
1118}
1092impl AstNode for SourceFile {1119impl AstNode for SourceFile {
1093	fn can_cast(kind: SyntaxKind) -> bool {1120	fn can_cast(kind: SyntaxKind) -> bool {
1094		kind == SOURCE_FILE1121		kind == SOURCE_FILE
2677		std::fmt::Display::fmt(self.syntax(), f)2704		std::fmt::Display::fmt(self.syntax(), f)
2678	}2705	}
2679}2706}
2680impl AstToken for String {2707impl AstToken for Text {
2681	fn can_cast(kind: SyntaxKind) -> bool {2708	fn can_cast(kind: SyntaxKind) -> bool {
2682		match kind {2709		match kind {
2683			STRING_DOUBLE2710			STRING_DOUBLE
2711			| ERROR_STRING_DOUBLE_UNTERMINATED
2684			| STRING_SINGLE2712			| STRING_SINGLE
2713			| ERROR_STRING_SINGLE_UNTERMINATED
2714			| STRING_DOUBLE_VERBATIM
2715			| ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED
2716			| STRING_SINGLE_VERBATIM
2717			| ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED
2718			| ERROR_STRING_VERBATIM_MISSING_QUOTES
2719			| STRING_BLOCK
2720			| ERROR_STRING_BLOCK_UNEXPECTED_END
2685			| STRING_DOUBLE_VERBATIM2721			| ERROR_STRING_BLOCK_MISSING_NEW_LINE
2686			| STRING_SINGLE_VERBATIM2722			| ERROR_STRING_BLOCK_MISSING_TERMINATION
2687			| STRING_BLOCK => true,2723			| ERROR_STRING_BLOCK_MISSING_INDENT => true,
2688			_ => false,2724			_ => false,
2689		}2725		}
2690	}2726	}
2691	fn cast(syntax: SyntaxToken) -> Option<Self> {2727	fn cast(syntax: SyntaxToken) -> Option<Self> {
2692		let res = match syntax.kind() {2728		let res = match syntax.kind() {
2693			STRING_DOUBLE => String {2729			STRING_DOUBLE => Text {
2694				syntax,2730				syntax,
2695				kind: StringKind::StringDouble,2731				kind: TextKind::StringDouble,
2696			},2732			},
2733			ERROR_STRING_DOUBLE_UNTERMINATED => Text {
2734				syntax,
2735				kind: TextKind::ErrorStringDoubleUnterminated,
2736			},
2697			STRING_SINGLE => String {2737			STRING_SINGLE => Text {
2698				syntax,2738				syntax,
2699				kind: StringKind::StringSingle,2739				kind: TextKind::StringSingle,
2700			},2740			},
2741			ERROR_STRING_SINGLE_UNTERMINATED => Text {
2742				syntax,
2743				kind: TextKind::ErrorStringSingleUnterminated,
2744			},
2701			STRING_DOUBLE_VERBATIM => String {2745			STRING_DOUBLE_VERBATIM => Text {
2702				syntax,2746				syntax,
2703				kind: StringKind::StringDoubleVerbatim,2747				kind: TextKind::StringDoubleVerbatim,
2704			},2748			},
2749			ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => Text {
2750				syntax,
2751				kind: TextKind::ErrorStringDoubleVerbatimUnterminated,
2752			},
2705			STRING_SINGLE_VERBATIM => String {2753			STRING_SINGLE_VERBATIM => Text {
2706				syntax,2754				syntax,
2707				kind: StringKind::StringSingleVerbatim,2755				kind: TextKind::StringSingleVerbatim,
2708			},2756			},
2757			ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => Text {
2758				syntax,
2759				kind: TextKind::ErrorStringSingleVerbatimUnterminated,
2760			},
2761			ERROR_STRING_VERBATIM_MISSING_QUOTES => Text {
2762				syntax,
2763				kind: TextKind::ErrorStringVerbatimMissingQuotes,
2764			},
2709			STRING_BLOCK => String {2765			STRING_BLOCK => Text {
2710				syntax,2766				syntax,
2711				kind: StringKind::StringBlock,2767				kind: TextKind::StringBlock,
2712			},2768			},
2769			ERROR_STRING_BLOCK_UNEXPECTED_END => Text {
2770				syntax,
2771				kind: TextKind::ErrorStringBlockUnexpectedEnd,
2772			},
2773			ERROR_STRING_BLOCK_MISSING_NEW_LINE => Text {
2774				syntax,
2775				kind: TextKind::ErrorStringBlockMissingNewLine,
2776			},
2777			ERROR_STRING_BLOCK_MISSING_TERMINATION => Text {
2778				syntax,
2779				kind: TextKind::ErrorStringBlockMissingTermination,
2780			},
2781			ERROR_STRING_BLOCK_MISSING_INDENT => Text {
2782				syntax,
2783				kind: TextKind::ErrorStringBlockMissingIndent,
2784			},
2713			_ => return None,2785			_ => return None,
2714		};2786		};
2715		Some(res)2787		Some(res)
2718		&self.syntax2790		&self.syntax
2719	}2791	}
2720}2792}
2721impl String {2793impl Text {
2722	pub fn kind(&self) -> StringKind {2794	pub fn kind(&self) -> TextKind {
2723		self.kind2795		self.kind
2724	}2796	}
2725}2797}
2726impl std::fmt::Display for String {2798impl std::fmt::Display for Text {
2727	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {2799	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2728		std::fmt::Display::fmt(self.syntax(), f)2800		std::fmt::Display::fmt(self.syntax(), f)
2729	}2801	}
2731impl AstToken for Number {2803impl AstToken for Number {
2732	fn can_cast(kind: SyntaxKind) -> bool {2804	fn can_cast(kind: SyntaxKind) -> bool {
2733		match kind {2805		match kind {
2734			FLOAT | META_FORCE_ENUM => true,2806			FLOAT
2807			| ERROR_FLOAT_JUNK_AFTER_POINT
2808			| ERROR_FLOAT_JUNK_AFTER_EXPONENT
2809			| ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => true,
2735			_ => false,2810			_ => false,
2736		}2811		}
2737	}2812	}
2741				syntax,2816				syntax,
2742				kind: NumberKind::Float,2817				kind: NumberKind::Float,
2743			},2818			},
2744			META_FORCE_ENUM => Number {2819			ERROR_FLOAT_JUNK_AFTER_POINT => Number {
2745				syntax,2820				syntax,
2746				kind: NumberKind::MetaForceEnum,2821				kind: NumberKind::ErrorFloatJunkAfterPoint,
2747			},2822			},
2823			ERROR_FLOAT_JUNK_AFTER_EXPONENT => Number {
2824				syntax,
2825				kind: NumberKind::ErrorFloatJunkAfterExponent,
2826			},
2827			ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => Number {
2828				syntax,
2829				kind: NumberKind::ErrorFloatJunkAfterExponentSign,
2830			},
2748			_ => return None,2831			_ => return None,
2749		};2832		};
2750		Some(res)2833		Some(res)
2841		std::fmt::Display::fmt(self.syntax(), f)2924		std::fmt::Display::fmt(self.syntax(), f)
2842	}2925	}
2843}2926}
2927impl AstToken for Trivia {
2928	fn can_cast(kind: SyntaxKind) -> bool {
2929		match kind {
2930			WHITESPACE
2931			| MULTI_LINE_COMMENT
2932			| ERROR_COMMENT_TOO_SHORT
2933			| ERROR_COMMENT_UNTERMINATED
2934			| SINGLE_LINE_HASH_COMMENT
2935			| SINGLE_LINE_SLASH_COMMENT => true,
2936			_ => false,
2937		}
2938	}
2939	fn cast(syntax: SyntaxToken) -> Option<Self> {
2940		let res = match syntax.kind() {
2941			WHITESPACE => Trivia {
2942				syntax,
2943				kind: TriviaKind::Whitespace,
2944			},
2945			MULTI_LINE_COMMENT => Trivia {
2946				syntax,
2947				kind: TriviaKind::MultiLineComment,
2948			},
2949			ERROR_COMMENT_TOO_SHORT => Trivia {
2950				syntax,
2951				kind: TriviaKind::ErrorCommentTooShort,
2952			},
2953			ERROR_COMMENT_UNTERMINATED => Trivia {
2954				syntax,
2955				kind: TriviaKind::ErrorCommentUnterminated,
2956			},
2957			SINGLE_LINE_HASH_COMMENT => Trivia {
2958				syntax,
2959				kind: TriviaKind::SingleLineHashComment,
2960			},
2961			SINGLE_LINE_SLASH_COMMENT => Trivia {
2962				syntax,
2963				kind: TriviaKind::SingleLineSlashComment,
2964			},
2965			_ => return None,
2966		};
2967		Some(res)
2968	}
2969	fn syntax(&self) -> &SyntaxToken {
2970		&self.syntax
2971	}
2972}
2973impl Trivia {
2974	pub fn kind(&self) -> TriviaKind {
2975		self.kind
2976	}
2977}
2978impl std::fmt::Display for Trivia {
2979	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2980		std::fmt::Display::fmt(self.syntax(), f)
2981	}
2982}
2844impl std::fmt::Display for Expr {2983impl std::fmt::Display for Expr {
2845	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {2984	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2846		std::fmt::Display::fmt(self.syntax(), f)2985		std::fmt::Display::fmt(self.syntax(), f)

174	SELF_KW,174	SELF_KW,
175	#[token("super")]175	#[token("super")]
176	SUPER_KW,176	SUPER_KW,
177	META_FORCE_ENUM,
178	#[token("for")]177	#[token("for")]
179	FOR_KW,178	FOR_KW,
180	#[token("assert")]179	#[token("assert")]
253	BINARY_OPERATOR,252	BINARY_OPERATOR,
254	UNARY_OPERATOR,253	UNARY_OPERATOR,
255	LITERAL,254	LITERAL,
256	STRING,255	TEXT,
257	NUMBER,256	NUMBER,
258	IMPORT_KIND,257	IMPORT_KIND,
259	VISIBILITY,258	VISIBILITY,
259	TRIVIA,
260	#[doc(hidden)]260	#[doc(hidden)]
261	__LAST,261	__LAST,
262}262}
277	pub fn is_enum(self) -> bool {277	pub fn is_enum(self) -> bool {
278		match self {278		match self {
279			EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER | FIELD | FIELD_NAME | DESTRUCT279			EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER | FIELD | FIELD_NAME | DESTRUCT
280			| DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | STRING280			| DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | TEXT | NUMBER
281			| NUMBER | IMPORT_KIND | VISIBILITY => true,281			| IMPORT_KIND | VISIBILITY | TRIVIA => true,
282			_ => false,282			_ => false,
283		}283		}
284	}284	}

4use logos::Logos;4use logos::Logos;
5use rowan::{TextRange, TextSize};5use rowan::{TextRange, TextSize};
66
7use crate::SyntaxKind;7use crate::{
8	string_block::{lex_str_block, StringBlockError},
9	SyntaxKind,
10};
811
9pub struct Lexer<'a> {12pub struct Lexer<'a> {
10	inner: logos::Lexer<'a, SyntaxKind>,13	inner: logos::Lexer<'a, SyntaxKind>,
22	type Item = Lexeme<'a>;25	type Item = Lexeme<'a>;
2326
24	fn next(&mut self) -> Option<Self::Item> {27	fn next(&mut self) -> Option<Self::Item> {
28		use SyntaxKind::*;
29
25		let kind = self.inner.next()?;30		let mut kind = self.inner.next()?;
26		let text = self.inner.slice();31		let text = self.inner.slice();
32
33		if kind == STRING_BLOCK {
34			// We use custom lexer, which skips enough bytes, but not returns error
35			// Instead we should call lexer again to verify if there is something wrong with string block
36			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);
37			// In kinds, string blocks is parsed at least as `|||`
38			lexer.bump(3);
39			let res = lex_str_block(&mut lexer);
40			debug_assert!(lexer.next().is_none(), "str_block is lexed");
41			match res {
42				Ok(_) => {}
43				Err(e) => {
44					kind = match e {
45						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
46						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
47						StringBlockError::MissingTermination => {
48							ERROR_STRING_BLOCK_MISSING_TERMINATION
49						}
50						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
51					}
52				}
53			}
54		}
2755
28		Some(Self::Item {56		Some(Self::Item {
29			kind,57			kind,

22
3mod ast;3mod ast;
4mod binary;4mod binary;
5mod classify;
6mod event;5mod event;
7mod generated;6mod generated;
8mod language;7mod language;

44			!kind.is_enum(),44			!kind.is_enum(),
45			"{kind:?} is a enum kind, you should use variant kinds instead"45			"{kind:?} is a enum kind, you should use variant kinds instead"
46		);46		);
47		// TODO: is_parser should return true if enum variant has #[regex]/#[token] over it47		// TODO: is_lexer should return true if enum variant has #[regex]/#[token] over it, or it is defined as lexer error explicitly
48		// debug_assert!(48		// debug_assert!(
49		// 	!kind.is_parser(),49		// 	!kind.is_lexer(),
50		// 	"{kind:?} should be only emitted by parser, not used directly"50		// 	"{kind:?} should be only emitted by lexer, not used directly"
51		// );51		// );
52		let event_at_pos = &mut p.events[self.start_event_idx];52		let event_at_pos = &mut p.events[self.start_event_idx];
53		assert_eq!(*event_at_pos, Event::Pending);53		assert_eq!(*event_at_pos, Event::Pending);

8	event::Event,8	event::Event,
9	lex::Lexeme,9	lex::Lexeme,
10	marker::{AsRange, CompletedMarker, Marker, Ranger},10	marker::{AsRange, CompletedMarker, Marker, Ranger},
11	string_block::{lex_str_block, StringBlockError},11	nodes::{Literal, Number, Text, Trivia},
12	token_set::SyntaxKindSet,12	token_set::SyntaxKindSet,
13	unary::UnaryOperator,13	unary::UnaryOperator,
14	SyntaxKind,14	AstToken, SyntaxKind,
15	SyntaxKind::*,15	SyntaxKind::*,
16	SyntaxNode, T, TS,16	SyntaxNode, T, TS,
17};17};
36}36}
3737
38pub struct Parser<'i> {38pub struct Parser<'i> {
39	// TODO: remove all trivia before feeding to parser?
39	lexemes: &'i [Lexeme<'i>],40	lexemes: &'i [Lexeme<'i>],
40	pub offset: usize,41	pub offset: usize,
41	pub events: Vec<Event>,42	pub events: Vec<Event>,
191		while self192		while self
192			.lexemes193			.lexemes
193			.get(previous_token_idx)194			.get(previous_token_idx)
194			.map_or(false, |l| l.kind.is_trivia())195			.map_or(false, |l| Trivia::can_cast(l.kind))
195			&& previous_token_idx != 0196			&& previous_token_idx != 0
196		{197		{
197			previous_token_idx -= 1;198			previous_token_idx -= 1;
200		Some(self.lexemes[previous_token_idx])201		Some(self.lexemes[previous_token_idx])
201	}202	}
202	pub fn start_of_token(&self, mut idx: usize) -> TextSize {203	pub fn start_of_token(&self, mut idx: usize) -> TextSize {
203		while self.lexemes[idx].kind.is_trivia() {204		while Trivia::can_cast(self.lexemes[idx].kind) {
204			idx += 1;205			idx += 1;
205		}206		}
206		self.lexemes[idx].range.start()207		self.lexemes[idx].range.start()
207	}208	}
208	pub fn end_of_token(&self, mut idx: usize) -> TextSize {209	pub fn end_of_token(&self, mut idx: usize) -> TextSize {
209		while self.lexemes[idx].kind.is_trivia() {210		while Trivia::can_cast(self.lexemes[idx].kind) {
210			idx -= 1;211			idx -= 1;
211		}212		}
212		self.lexemes[idx].range.end()213		self.lexemes[idx].range.end()
267		self.bump();268		self.bump();
268		Some(m.complete(self, SyntaxKind::ERROR))269		Some(m.complete(self, SyntaxKind::ERROR))
269	}270	}
270271	fn bump_assert(&mut self, kind: SyntaxKind) {
272		self.skip_trivia();
273		assert!(self.at(kind), "expected {:?}", kind);
274		self.bump_remap(self.current());
275	}
271	fn bump(&mut self) {276	fn bump(&mut self) {
272		self.skip_trivia();277		self.skip_trivia();
273		self.bump_remap(self.current());278		self.bump_remap(self.current());
314			while self319			while self
315				.lexemes320				.lexemes
316				.get(offset)321				.get(offset)
317				.map(|l| l.kind.is_trivia())322				.map(|l| Trivia::can_cast(l.kind))
318				.unwrap_or(false)323				.unwrap_or(false)
319			{324			{
320				offset += 1;325				offset += 1;
324		while self329		while self
325			.lexemes330			.lexemes
326			.get(offset)331			.get(offset)
327			.map(|l| l.kind.is_trivia())332			.map(|l| Trivia::can_cast(l.kind))
328			.unwrap_or(false)333			.unwrap_or(false)
329		{334		{
330			offset += 1;335			offset += 1;
335		self.nth(0)340		self.nth(0)
336	}341	}
337	fn skip_trivia(&mut self) {342	fn skip_trivia(&mut self) {
338		while self.peek_raw().is_trivia() {343		while Trivia::can_cast(self.peek_raw()) {
339			self.offset += 1;344			self.offset += 1;
340		}345		}
341	}346	}
342	fn current_lexeme(&mut self) -> Option<&Lexeme> {
343		self.skip_trivia();
344		self.lexemes.get(self.offset)
345	}
346	fn peek_raw(&mut self) -> SyntaxKind {347	fn peek_raw(&mut self) -> SyntaxKind {
347		self.lexemes348		self.lexemes
348			.get(self.offset)349			.get(self.offset)
516	} else if p.at(IDENT) {517	} else if p.at(IDENT) {
517		name(p);518		name(p);
518		m.complete(p, FIELD_NAME_FIXED);519		m.complete(p, FIELD_NAME_FIXED);
519	} else if p.current().is_string() {520	} else if Text::can_cast(p.current()) {
520		string(p);521		text(p);
521		m.complete(p, FIELD_NAME_FIXED);522		m.complete(p, FIELD_NAME_FIXED);
522	} else {523	} else {
523		p.error_with_recovery_set(TS![;]);524		p.error_with_recovery_set(TS![;]);
564	};565	};
565}566}
566fn assertion(p: &mut Parser) {567fn assertion(p: &mut Parser) {
567	assert!(p.at(T![assert]));
568	let m = p.start();568	let m = p.start();
569	p.bump();569	p.bump_assert(T![assert]);
570	expr(p).map(|c| c.wrap(p, LHS_EXPR));570	expr(p).map(|c| c.wrap(p, LHS_EXPR));
571	if p.at(T![:]) {571	if p.at(T![:]) {
572		p.bump();572		p.bump();
575	m.complete(p, ASSERTION);575	m.complete(p, ASSERTION);
576}576}
577fn object(p: &mut Parser) -> CompletedMarker {577fn object(p: &mut Parser) -> CompletedMarker {
578	assert!(p.at(T!['{']));
579	let m_t = p.start();578	let m_t = p.start();
580	let m = p.start();579	let m = p.start();
581	p.bump();580	p.bump_assert(T!['{']);
582581
583	loop {582	loop {
584		if p.at(T!['}']) {583		if p.at(T!['}']) {
619	m.complete(p, PARAM);618	m.complete(p, PARAM);
620}619}
621fn params_desc(p: &mut Parser) -> CompletedMarker {620fn params_desc(p: &mut Parser) -> CompletedMarker {
622	assert!(p.at(T!['(']));
623	let m = p.start();621	let m = p.start();
624	p.bump();622	p.bump_assert(T!['(']);
625623
626	loop {624	loop {
627		if p.at(T![')']) {625		if p.at(T![')']) {
640}638}
641fn args_desc(p: &mut Parser) {639fn args_desc(p: &mut Parser) {
642	let m = p.start();640	let m = p.start();
643	assert!(p.at(T!['(']));641	p.bump_assert(T!['(']);
644	p.bump();
645642
646	let started_named = Cell::new(false);643	let started_named = Cell::new(false);
647644
674}671}
675672
676fn array(p: &mut Parser) -> CompletedMarker {673fn array(p: &mut Parser) -> CompletedMarker {
677	assert!(p.at(T!['[']));
678	// Start the list node674	// Start the list node
679	let m = p.start();675	let m = p.start();
680	p.bump(); // '['676	p.bump_assert(T!['[']);
681677
682	// This vec will have at most one element in case of correct input678	// This vec will have at most one element in case of correct input
683	let mut compspecs = Vec::with_capacity(1);679	let mut compspecs = Vec::with_capacity(1);
795	m.complete(p, NAME);791	m.complete(p, NAME);
796}792}
797fn destruct_rest(p: &mut Parser) {793fn destruct_rest(p: &mut Parser) {
798	assert!(p.at(T![...]));
799	p.bump();
800	let m = p.start();794	let m = p.start();
795	p.bump_assert(T![...]);
801	if p.at(IDENT) {796	if p.at(IDENT) {
802		p.bump()797		p.bump()
803	}798	}
817	m.complete(p, DESTRUCT_OBJECT_FIELD);812	m.complete(p, DESTRUCT_OBJECT_FIELD);
818}813}
819fn obj_local(p: &mut Parser) {814fn obj_local(p: &mut Parser) {
820	assert!(p.at(T![local]));
821	let m = p.start();815	let m = p.start();
822	p.bump();816	p.bump_assert(T![local]);
823	bind(p);817	bind(p);
824	m.complete(p, OBJ_LOCAL);818	m.complete(p, OBJ_LOCAL);
825}819}
903		m.complete(p, BIND_DESTRUCT)897		m.complete(p, BIND_DESTRUCT)
904	};898	};
905}899}
906fn string(p: &mut Parser) {900fn text(p: &mut Parser) {
907	assert!(p.current().is_string());901	assert!(Text::can_cast(p.current()));
908	if p.at(STRING_BLOCK) {
909		// We use custom lexer, which skips enough bytes, but not returns error
910		// Instead we should call lexer again to verify if there is something wrong with string block
911		let mut lexer = logos::Lexer::<SyntaxKind>::new(dbg!(
912			&p.current_lexeme().expect("parser is at string block").text
913		));
914		// In kinds, string blocks is parsed at least as `|||`902	p.bump();
915		lexer.bump(3);
916		let res = lex_str_block(&mut lexer);
917		debug_assert!(lexer.next().is_none(), "str_block is lexed");
918		match res {
919			Ok(_) => {
920				p.bump();
921			}
922			Err(e) => p.bump_remap(match e {
923				StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
924				StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
925				StringBlockError::MissingTermination => ERROR_STRING_BLOCK_MISSING_TERMINATION,
926				StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
927			}),
928		}
929	} else {
930		p.bump();
931	}
932}903}
933fn number(p: &mut Parser) {904fn number(p: &mut Parser) {
934	assert!(p.current().is_number());905	assert!(Number::can_cast(p.current()));
935	p.bump();906	p.bump();
936}907}
937fn literal(p: &mut Parser) {908fn literal(p: &mut Parser) {
938	assert!(p.current().is_literal());909	assert!(Literal::can_cast(p.current()));
939	p.bump();910	p.bump();
940}911}
941fn lhs_basic(p: &mut Parser) -> Option<CompletedMarker> {912fn lhs_basic(p: &mut Parser) -> Option<CompletedMarker> {
942	let _e = p.expected_syntax_name("value");913	let _e = p.expected_syntax_name("value");
943	Some(if p.current().is_literal() {914	Some(if Literal::can_cast(p.current()) {
944		let m = p.start();915		let m = p.start();
945		literal(p);916		literal(p);
946		m.complete(p, EXPR_LITERAL)917		m.complete(p, EXPR_LITERAL)
947	} else if p.current().is_string() {918	} else if Text::can_cast(p.current()) {
948		let m = p.start();919		let m = p.start();
949		string(p);920		text(p);
950		m.complete(p, EXPR_STRING)921		m.complete(p, EXPR_STRING)
951	} else if p.current().is_number() {922	} else if Number::can_cast(p.current()) {
952		let m = p.start();923		let m = p.start();
953		number(p);924		number(p);
954		m.complete(p, EXPR_NUMBER)925		m.complete(p, EXPR_NUMBER)
1025	} else if p.at(T![import]) || p.at(T![importstr]) || p.at(T![importbin]) {996	} else if p.at(T![import]) || p.at(T![importstr]) || p.at(T![importbin]) {
1026		let m = p.start();997		let m = p.start();
1027		p.bump();998		p.bump();
1028		string(p);999		text(p);
1029		m.complete(p, EXPR_IMPORT)1000		m.complete(p, EXPR_IMPORT)
1030	} else if p.at(T![-]) || p.at(T![!]) || p.at(T![~]) {1001	} else if p.at(T![-]) || p.at(T![!]) || p.at(T![~]) {
1031		let op = match p.current() {1002		let op = match p.current() {
1044		let m = p.start();1015		let m = p.start();
1045		p.bump();1016		p.bump();
1046		expr(p);1017		expr(p);
1047		assert!(p.at(T![')']));1018		p.expect(T![')']);
1048		p.bump();
1049		m.complete(p, EXPR_PARENED)1019		m.complete(p, EXPR_PARENED)
1050	} else {1020	} else {
1051		p.error_with_recovery_set(TS![]);1021		p.error_with_recovery_set(TS![]);

10pub enum TokenKind {10pub enum TokenKind {
11	/// May exist in token tree, but never in source code11	/// May exist in token tree, but never in source code
12	Meta { grammar_name: String, name: String },12	Meta { grammar_name: String, name: String },
13	/// Specific parsing errors may be emitted as this type of kind13	/// Specific parsing/lexing errors may be emitted as this type of kind
14	Error {14	Error {
15		grammar_name: String,15		grammar_name: String,
16		name: String,16		name: String,
17		/// Is this error returned by lexer directly, or from lex.rs
18		is_lexer_error: bool,
17		regex: Option<String>,19		regex: Option<String>,
18		priority: Option<u32>,20		priority: Option<u32>,
19	},21	},
133		});135		});
134		$(define_kinds!($into = $($rest)*))?136		$(define_kinds!($into = $($rest)*))?
135	}};137	}};
136	($into:ident = error($name:literal$(, priority = $priority:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{138	($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
139		{
140			let regex = None$(.or(Some($regex.to_owned())))?;
141			let priority = None$(.or(Some($priority)))?;
137		$into.define_token(TokenKind::Error {142			$into.define_token(TokenKind::Error {
138			grammar_name: format!("ERROR_{}!", $name),143				grammar_name: format!("ERROR_{}!", $name),
139			name: format!("ERROR_{}", $name),144				name: format!("ERROR_{}", $name),
140			regex: None$(.or(Some($regex.to_owned())))?,145				is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),
141			priority: None$(.or(Some($priority)))?,146				regex,
147				priority,
142		});148			});
149		}
143		$(define_kinds!($into = $($rest)*))?150		$(define_kinds!($into = $($rest)*))?
144	}};151	}};
145	($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{152	($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{
258		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";265		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";
259		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";266		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";
260		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";267		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";
261		error("STRING_BLOCK_UNEXPECTED_END");268		error("STRING_BLOCK_UNEXPECTED_END", lexer = true);
262		error("STRING_BLOCK_MISSING_NEW_LINE");269		error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);
263		error("STRING_BLOCK_MISSING_TERMINATION");270		error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);
264		error("STRING_BLOCK_MISSING_INDENT");271		error("STRING_BLOCK_MISSING_INDENT", lexer = true);
265		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";272		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";
266		lit("WHITESPACE") => r"[ \t\n\r]+";273		lit("WHITESPACE") => r"[ \t\n\r]+";
267		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";274		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";