git.delta.rocks / jrsonnet / refs/commits / 93f08ca017e4

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -716,6 +716,13 @@
 ]
 
 [[package]]
+name = "jrsonnet-lexer"
+version = "0.5.0-pre97"
+dependencies = [
+ "logos",
+]
+
+[[package]]
 name = "jrsonnet-macros"
 version = "0.5.0-pre97"
 dependencies = [
@@ -744,7 +751,7 @@
  "hi-doc",
  "indoc",
  "insta",
- "logos",
+ "jrsonnet-lexer",
  "rowan",
  "strip-ansi-escapes",
  "thiserror",

--- a/Cargo.toml
+++ b/Cargo.toml
@@ -81,9 +81,6 @@
 itertools = "0.14.0"
 xshell = "0.2.7"
 
-lsp-server = "0.7.9"
-lsp-types = "0.97.0"
-
 regex = "1.12"
 lru = "0.16.3"
 

--- a/crates/jrsonnet-rowan-parser/Cargo.toml
+++ b/crates/jrsonnet-rowan-parser/Cargo.toml
@@ -14,7 +14,7 @@
 drop_bomb.workspace = true
 hi-doc.workspace = true
 indoc.workspace = true
-logos.workspace = true
+jrsonnet-lexer = { version = "0.5.0-pre97", path = "../jrsonnet-lexer" }
 rowan.workspace = true
 thiserror.workspace = true
 

--- a/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
@@ -8,172 +8,95 @@
 	clippy::manual_non_exhaustive,
 	clippy::match_like_matches_macro
 )]
-use logos::Logos;
 #[doc = r" The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`."]
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Logos)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 #[repr(u16)]
 pub enum SyntaxKind {
 	#[doc(hidden)]
 	TOMBSTONE,
 	#[doc(hidden)]
 	EOF,
-	#[token("||")]
 	OR,
-	#[token("??")]
 	NULL_COAELSE,
-	#[token("&&")]
 	AND,
-	#[token("|")]
 	BIT_OR,
-	#[token("^")]
 	BIT_XOR,
-	#[token("&")]
 	BIT_AND,
-	#[token("==")]
 	EQ,
-	#[token("!=")]
 	NE,
-	#[token("<")]
 	LT,
-	#[token(">")]
 	GT,
-	#[token("<=")]
 	LE,
-	#[token(">=")]
 	GE,
-	#[token("<<")]
 	LHS,
-	#[token(">>")]
 	RHS,
-	#[token("+")]
 	PLUS,
-	#[token("-")]
 	MINUS,
-	#[token("*")]
 	MUL,
-	#[token("/")]
 	DIV,
-	#[token("%")]
 	MODULO,
-	#[token("!")]
 	NOT,
-	#[token("~")]
 	BIT_NOT,
-	#[token("[")]
 	L_BRACK,
-	#[token("]")]
 	R_BRACK,
-	#[token("(")]
 	L_PAREN,
-	#[token(")")]
 	R_PAREN,
-	#[token("{")]
 	L_BRACE,
-	#[token("}")]
 	R_BRACE,
-	#[token(":")]
 	COLON,
-	#[token("::")]
 	COLONCOLON,
-	#[token(":::")]
 	COLONCOLONCOLON,
-	#[token(";")]
 	SEMI,
-	#[token(".")]
 	DOT,
-	#[token("...")]
 	DOTDOTDOT,
-	#[token(",")]
 	COMMA,
-	#[token("$")]
 	DOLLAR,
-	#[token("=")]
 	ASSIGN,
-	#[token("?")]
 	QUESTION_MARK,
-	#[regex("(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")]
 	FLOAT,
-	#[regex("(?:0|[1-9][0-9]*)\\.[^0-9]")]
 	ERROR_FLOAT_JUNK_AFTER_POINT,
-	#[regex("(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?[eE][^+\\-0-9]")]
 	ERROR_FLOAT_JUNK_AFTER_EXPONENT,
-	#[regex("(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?[eE][+-][^0-9]")]
 	ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN,
-	#[regex("\"(?s:[^\"\\\\]|\\\\.)*\"")]
 	STRING_DOUBLE,
-	#[regex("\"(?s:[^\"\\\\]|\\\\.)*")]
 	ERROR_STRING_DOUBLE_UNTERMINATED,
-	#[regex("'(?s:[^'\\\\]|\\\\.)*'")]
 	STRING_SINGLE,
-	#[regex("'(?s:[^'\\\\]|\\\\.)*")]
 	ERROR_STRING_SINGLE_UNTERMINATED,
-	#[regex("@\"(?:[^\"]|\"\")*\"")]
 	STRING_DOUBLE_VERBATIM,
-	#[regex("@\"(?:[^\"]|\"\")*")]
 	ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED,
-	#[regex("@'(?:[^']|'')*'")]
 	STRING_SINGLE_VERBATIM,
-	#[regex("@'(?:[^']|'')*")]
 	ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED,
-	#[regex("@[^\"'\\s]\\S+")]
 	ERROR_STRING_VERBATIM_MISSING_QUOTES,
-	#[regex("\\|\\|\\|", crate::string_block::lex_str_block_test)]
 	STRING_BLOCK,
 	ERROR_STRING_BLOCK_UNEXPECTED_END,
 	ERROR_STRING_BLOCK_MISSING_NEW_LINE,
 	ERROR_STRING_BLOCK_MISSING_TERMINATION,
 	ERROR_STRING_BLOCK_MISSING_INDENT,
-	#[regex("[_a-zA-Z][_a-zA-Z0-9]*")]
 	IDENT,
-	#[regex("[ \\t\\n\\r]+")]
 	WHITESPACE,
-	#[regex("//[^\\r\\n]*?(\\r\\n|\\n)?")]
 	SINGLE_LINE_SLASH_COMMENT,
-	#[regex("#[^\\r\\n]*?(\\r\\n|\\n)?")]
 	SINGLE_LINE_HASH_COMMENT,
-	#[regex("/\\*([^*]|\\*[^/])*\\*/")]
 	MULTI_LINE_COMMENT,
-	#[regex("/\\*/")]
 	ERROR_COMMENT_TOO_SHORT,
-	#[regex("/\\*([^*/]|\\*[^/])+")]
 	ERROR_COMMENT_UNTERMINATED,
-	#[token("tailstrict")]
 	TAILSTRICT_KW,
-	#[token("local")]
 	LOCAL_KW,
-	#[token("importstr")]
 	IMPORTSTR_KW,
-	#[token("importbin")]
 	IMPORTBIN_KW,
-	#[token("import")]
 	IMPORT_KW,
-	#[token("if")]
 	IF_KW,
-	#[token("then")]
 	THEN_KW,
-	#[token("else")]
 	ELSE_KW,
-	#[token("function")]
 	FUNCTION_KW,
-	#[token("error")]
 	ERROR_KW,
-	#[token("in")]
 	IN_KW,
 	META_OBJECT_APPLY,
 	ERROR_NO_OPERATOR,
-	#[token("null")]
 	NULL_KW,
-	#[token("true")]
 	TRUE_KW,
-	#[token("false")]
 	FALSE_KW,
-	#[token("self")]
 	SELF_KW,
-	#[token("super")]
 	SUPER_KW,
-	#[token("for")]
 	FOR_KW,
-	#[token("assert")]
 	ASSERT_KW,
 	ERROR_MISSING_TOKEN,
 	ERROR_UNEXPECTED_TOKEN,

--- a/crates/jrsonnet-rowan-parser/src/lex.rs
+++ b/crates/jrsonnet-rowan-parser/src/lex.rs
@@ -1,81 +1,19 @@
-use core::ops::Range;
-use std::convert::TryFrom;
-
-use logos::Logos;
+use jrsonnet_lexer::Lexer;
 use rowan::{TextRange, TextSize};
 
-use crate::{
-	string_block::{lex_str_block, StringBlockError},
-	SyntaxKind,
-};
-
-pub struct Lexer<'a> {
-	inner: logos::Lexer<'a, SyntaxKind>,
-}
-
-impl<'a> Lexer<'a> {
-	pub fn new(input: &'a str) -> Self {
-		Self {
-			inner: SyntaxKind::lexer(input),
-		}
-	}
-}
-
-impl<'a> Iterator for Lexer<'a> {
-	type Item = Lexeme<'a>;
-
-	fn next(&mut self) -> Option<Self::Item> {
-		use SyntaxKind::*;
-
-		let mut kind = self.inner.next()?;
-		let text = self.inner.slice();
-
-		if kind == Ok(STRING_BLOCK) {
-			// We use custom lexer, which skips enough bytes, but not returns error
-			// Instead we should call lexer again to verify if there is something wrong with string block
-			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);
-			// In kinds, string blocks is parsed at least as `|||`
-			lexer.bump(3);
-			let res = lex_str_block(&mut lexer);
-			let next = lexer.next();
-			assert!(next.is_none(), "str_block is lexed");
-			match res {
-				Ok(()) => {}
-				Err(e) => {
-					kind = Ok(match e {
-						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
-						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
-						StringBlockError::MissingTermination => {
-							ERROR_STRING_BLOCK_MISSING_TERMINATION
-						}
-						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
-					});
-				}
-			}
-		}
-
-		Some(Self::Item {
-			kind: kind.unwrap_or(SyntaxKind::LEXING_ERROR),
-			text,
-			range: {
-				let Range { start, end } = self.inner.span();
-
-				TextRange::new(
-					TextSize::try_from(start).unwrap(),
-					TextSize::try_from(end).unwrap(),
-				)
-			},
-		})
-	}
-}
+use crate::SyntaxKind;
 
 #[derive(Clone, Copy, Debug)]
-pub struct Lexeme<'i> {
+pub struct Lexeme<'s> {
 	pub kind: SyntaxKind,
-	pub text: &'i str,
+	pub text: &'s str,
 	pub range: TextRange,
 }
 
 pub fn lex(input: &str) -> Vec<Lexeme<'_>> {
-	Lexer::new(input).collect()
+	Lexer::new(input).map(|l| Lexeme {
+		kind: SyntaxKind::from_raw(l.kind.into_raw()),
+		text: l.text,
+		range: TextRange::new(TextSize::from(l.range.0), TextSize::from(l.range.1)),
+	}).collect()
 }

--- a/crates/jrsonnet-rowan-parser/src/lib.rs
+++ b/crates/jrsonnet-rowan-parser/src/lib.rs
@@ -2,7 +2,6 @@
 
 use event::Sink;
 use generated::nodes::{SourceFile, Trivia};
-use lex::lex;
 use parser::{LocatedSyntaxError, Parser};
 pub use rowan;
 
@@ -14,14 +13,12 @@
 mod marker;
 mod parser;
 mod precedence;
-mod string_block;
 mod tests;
 mod token_set;
 
 pub use ast::{AstChildren, AstNode, AstToken};
 pub use generated::{nodes, syntax_kinds::SyntaxKind};
 pub use language::*;
-pub use string_block::{collect_lexed_str_block, CollectStrBlock};
 pub use token_set::SyntaxKindSet;
 
 use self::{
@@ -30,7 +27,7 @@
 };
 
 pub fn parse(input: &str) -> (SourceFile, Vec<LocatedSyntaxError>) {
-	let lexemes = lex(input);
+	let lexemes = lex::lex(input);
 	let kinds = lexemes
 		.iter()
 		.map(|l| l.kind)

--- a/xtask/src/sourcegen/kinds.rs
+++ b/xtask/src/sourcegen/kinds.rs
@@ -56,7 +56,7 @@
 			| Self::Error { name, .. } => name,
 		}
 	}
-	pub fn expand_kind(&self) -> TokenStream {
+	pub fn expand_kind(&self, lexer: bool) -> TokenStream {
 		let name = format_ident!("{}", self.name());
 		let attr = match self {
 			Self::Keyword { code, .. } => quote! {#[token(#code)]},
@@ -75,6 +75,11 @@
 			}
 			_ => quote! {},
 		};
+		let attr = if lexer {
+			attr
+		} else {
+			quote! {}
+		};
 		quote! {
 			#attr
 			#name

--- a/xtask/src/sourcegen/mod.rs
+++ b/xtask/src/sourcegen/mod.rs
@@ -89,7 +89,7 @@
 		kinds.define_node(&name);
 	}
 
-	let syntax_kinds = generate_syntax_kinds(&kinds, &ast)?;
+	let syntax_kinds = generate_syntax_kinds(&kinds, &ast, false)?;
 
 	let nodes = generate_nodes(&kinds, &ast)?;
 	ensure_file_contents(
@@ -106,12 +106,21 @@
 		)),
 		&nodes,
 	);
+
+	let lexer_syntax_kinds = generate_syntax_kinds(&kinds, &ast, true)?;
+	ensure_file_contents(
+		&PathBuf::from(concat!(
+			env!("CARGO_MANIFEST_DIR"),
+			"/../crates/jrsonnet-lexer/src/generated/syntax_kinds.rs",
+		)),
+		&lexer_syntax_kinds,
+	);
 	Ok(())
 }
 
-fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {
+fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc, lexer: bool) -> Result<String> {
 	let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);
-	let token_kinds = kinds.tokens().map(TokenKind::expand_kind);
+	let token_kinds = kinds.tokens().map(|t| t.expand_kind(lexer));
 
 	let keywords = kinds
 		.tokens()
@@ -119,12 +128,16 @@
 		.map(TokenKind::name)
 		.map(|n| format_ident!("{n}"));
 
-	let nodes = kinds
+	let mut nodes = kinds
 		.nodes
 		.iter()
 		.map(|name| format_ident!("{}", name))
 		.collect::<Vec<_>>();
 
+	if lexer {
+		nodes.clear();
+	}
+
 	let enums = grammar
 		.enums
 		.iter()
@@ -134,14 +147,34 @@
 				.token_enums
 				.iter()
 				.map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),
-		);
+		)
+		.collect::<Vec<_>>();
+	let is_enum = if lexer {
+		quote! {}
+	} else {
+		quote! {
+			pub fn is_enum(self) -> bool {
+				match self {
+					#(#enums)|* => true,
+					_ => false,
+				}
+			}
+		}
+	};
 
+	let derive_logos = if lexer {
+		quote! {
+			, logos::Logos
+		}
+	} else {
+		quote! {}
+	};
+
 	let ast = quote! {
 		#![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]
-		use logos::Logos;
 
 		/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`.
-		#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Logos)]
+		#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug #derive_logos)]
 		#[repr(u16)]
 		pub enum SyntaxKind {
 			#[doc(hidden)]
@@ -164,13 +197,9 @@
 					_ => false,
 				}
 			}
-			pub fn is_enum(self) -> bool {
-				match self {
-					#(#enums)|* => true,
-					_ => false,
-				}
-			}
 
+			#is_enum
+
 			pub fn from_raw(r: u16) -> Self {
 				assert!(r < Self::__LAST as u16);
 				unsafe { std::mem::transmute(r) }