`cc55b2dd2d98`

source

xtask/src/sourcegen/kinds.rs8.0 KiBsource history
1#[derive(Debug)]2pub struct KindsSrc {3	/// Key - how this token appears in ungrammar4	defined_tokens: IndexMap<String, TokenKind>,5	defined_node_names: HashSet<String>,6	pub nodes: Vec<String>,7}89#[derive(Debug, Clone)]10pub enum TokenKind {11	/// May exist in token tree, but never in source code12	Meta { grammar_name: String, name: String },13	/// Specific parsing/lexing errors may be emitted as this type of kind14	Error {15		grammar_name: String,16		name: String,17		#[allow(dead_code)]18		/// Is this error returned by lexer directly, or from lex.rs19		is_lexer_error: bool,20		regex: Option<String>,21		priority: Option<u32>,22	},23	/// Keyword - literal match of token24	Keyword {25		/// How this keyword appears in grammar/code, should be same as Kinds key26		code: String,27		name: String,28	},29	/// Literal - something defined by user, i.e strings, identifiers, smth30	Literal {31		/// How this keyword appears in grammar, should be same as Kinds key32		grammar_name: String,33		name: String,34		/// Regex for Logos lexer35		regex: String,36		/// Path to custom lexer37		lexer: Option<String>,38	},39}4041impl TokenKind {42	pub fn grammar_name(&self) -> &str {43		match self {44			TokenKind::Keyword { code, .. } => code,45			TokenKind::Literal { grammar_name, .. } => grammar_name,46			TokenKind::Meta { grammar_name, .. } => grammar_name,47			TokenKind::Error { grammar_name, .. } => grammar_name,48		}49	}50	/// How this keyword should appear in kinds enum, screaming snake cased51	pub fn name(&self) -> &str {52		match self {53			TokenKind::Keyword { name, .. } => name,54			TokenKind::Literal { name, .. } => name,55			TokenKind::Meta { name, .. } => name,56			TokenKind::Error { name, .. } => name,57		}58	}59	pub fn expand_kind(&self) -> TokenStream {60		let name = format_ident!("{}", self.name());61		let attr = match self {62			TokenKind::Keyword { code, .. } => quote! {#[token(#code)]},63			TokenKind::Literal { regex, lexer, .. } => {64				let lexer = lexer65					.as_deref()66					.map(TokenStream::from_str)67					.map(|r| r.expect("path is correct"));68				quote! {#[regex(#regex, #lexer)]}69			}70			TokenKind::Error {71				regex, priority, ..72			} if regex.is_some() => {73				let priority = priority.map(|p| quote! {, priority = #p});74				quote! {#[regex(#regex #priority)]}75			}76			_ => quote! {},77		};78		quote! {79			#attr80			#name81		}82	}83	pub fn expand_t_macros(&self) -> Option<TokenStream> {84		match self {85			TokenKind::Keyword { code, name } => {86				let code = escape_token_macro(code);87				let name = format_ident!("{name}");88				Some(quote! {89					[#code] => {$crate::SyntaxKind::#name}90				})91			}92			// Meta items should not appear in T![_]93			_ => None,94		}95	}9697	/// How this token should be referenced in code98	/// Keywords are referenced with `T![_]` macro,99	/// and literals are referenced directly by name100	pub fn reference(&self) -> TokenStream {101		match self {102			TokenKind::Keyword { code, .. } => {103				let code = escape_token_macro(code);104				quote! {T![#code]}105			}106			_ => {107				let name = self.name();108				let ident = format_ident!("{name}");109				quote! {#ident}110			}111		}112	}113114	pub fn method_name(&self) -> Ident {115		match self {116			TokenKind::Keyword { name, .. } => {117				format_ident!("{}_token", name.to_lowercase())118			}119			TokenKind::Literal { name, .. } => {120				format_ident!("{}_lit", name.to_lowercase())121			}122			TokenKind::Meta { name, .. } => format_ident!("{}_meta", name.to_lowercase()),123			TokenKind::Error { name, .. } => format_ident!("{}_error", name.to_lowercase()),124		}125	}126}127128#[macro_export]129macro_rules! define_kinds {130	($into:ident = lit($name:literal) => $regex:literal $(, $lexer:literal)? $(; $($rest:tt)*)?) => {{131		$into.define_token(TokenKind::Literal {132			grammar_name: format!("LIT_{}!", $name),133			name: $name.to_owned(),134			regex: $regex.to_owned(),135			lexer: None $(.or_else(|| Some($lexer.to_string())))?,136		});137		$(define_kinds!($into = $($rest)*))?138	}};139	($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{140		{141			let regex = None$(.or(Some($regex.to_owned())))?;142			let priority = None$(.or(Some($priority)))?;143			$into.define_token(TokenKind::Error {144				grammar_name: format!("ERROR_{}!", $name),145				name: format!("ERROR_{}", $name),146				is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),147				regex,148				priority,149			});150		}151		$(define_kinds!($into = $($rest)*))?152	}};153	($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{154		$into.define_token(TokenKind::Keyword {155			code: format!("{}", $tok),156			name: $name.to_owned(),157		});158		$(define_kinds!($into = $($rest)*))?159	}};160	($into:ident =) => {{}}161}162use std::{collections::HashSet, str::FromStr};163164pub use define_kinds;165use indexmap::IndexMap;166use proc_macro2::{Ident, TokenStream};167use quote::{format_ident, quote};168169use super::escape_token_macro;170171impl KindsSrc {172	pub fn new() -> Self {173		Self {174			defined_tokens: IndexMap::new(),175			defined_node_names: HashSet::new(),176			nodes: Vec::new(),177		}178	}179	pub fn define_token(&mut self, token: TokenKind) {180		assert!(181			self.defined_node_names.insert(token.name().to_owned()),182			"node name already defined: {}",183			token.name()184		);185		assert!(186			self.defined_tokens187				.insert(token.grammar_name().to_owned(), token.clone())188				.is_none(),189			"token already defined: {}",190			token.grammar_name()191		)192	}193	pub fn define_node(&mut self, node: &str) {194		assert!(195			self.defined_node_names.insert(node.to_owned()),196			"node name already defined: {}",197			node198		);199		self.nodes.push(node.to_string())200	}201	pub fn token(&self, tok: &str) -> Option<&TokenKind> {202		self.defined_tokens.get(tok)203	}204	pub fn is_token(&self, tok: &str) -> bool {205		self.defined_tokens.contains_key(tok)206	}207	pub fn tokens(&self) -> impl Iterator<Item = &TokenKind> {208		self.defined_tokens.iter().map(|(_, v)| v)209	}210}211212pub fn jsonnet_kinds() -> KindsSrc {213	let mut kinds = KindsSrc::new();214	define_kinds![kinds =215		"||" => "OR";216		"??" => "NULL_COAELSE";217		"&&" => "AND";218		"|" => "BIT_OR";219		"^" => "BIT_XOR";220		"&" => "BIT_AND";221		"==" => "EQ";222		"!=" => "NE";223		"<" => "LT";224		">" => "GT";225		"<=" => "LE";226		">=" => "GE";227		"<<" => "LHS";228		">>" => "RHS";229		"+" => "PLUS";230		"-" => "MINUS";231		"*" => "MUL";232		"/" => "DIV";233		"%" => "MODULO";234		"!" => "NOT";235		"~" => "BIT_NOT";236		"[" => "L_BRACK";237		"]" => "R_BRACK";238		"(" => "L_PAREN";239		")" => "R_PAREN";240		"{" => "L_BRACE";241		"}" => "R_BRACE";242		":" => "COLON";243		"::" => "COLONCOLON";244		":::" => "COLONCOLONCOLON";245		";" => "SEMI";246		"." => "DOT";247		"..." => "DOTDOTDOT";248		"," => "COMMA";249		"$" => "DOLLAR";250		"=" => "ASSIGN";251		"?" => "QUESTION_MARK";252		// Literals253		lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?";254		error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]";255		error("FLOAT_JUNK_AFTER_EXPONENT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]";256		error("FLOAT_JUNK_AFTER_EXPONENT_SIGN") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]";257		lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";258		error("STRING_DOUBLE_UNTERMINATED") => "\"(?s:[^\"\\\\]|\\\\.)*";259		lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";260		error("STRING_SINGLE_UNTERMINATED") => "'(?s:[^'\\\\]|\\\\.)*";261		lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";262		error("STRING_DOUBLE_VERBATIM_UNTERMINATED") => "@\"(?:[^\"]|\"\")*";263		lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";264		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";265		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";266		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";267		error("STRING_BLOCK_UNEXPECTED_END", lexer = true);268		error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);269		error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);270		error("STRING_BLOCK_MISSING_INDENT", lexer = true);271		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";272		lit("WHITESPACE") => r"[ \t\n\r]+";273		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";274		lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*(\r\n|\n)?";275		lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";276		error("COMMENT_TOO_SHORT") => r"/\*/";277		error("COMMENT_UNTERMINATED") =>  r"/\*([^*/]|\*[^/])+";278	];279	kinds280}