`dfc47a63aff8`

source

xtask/src/sourcegen/kinds.rs7.8 KiBsource history
1#[derive(Debug)]2pub struct KindsSrc {3	/// Key - how this token appears in ungrammar4	defined_tokens: IndexMap<String, TokenKind>,5	defined_node_names: HashSet<String>,6	pub nodes: Vec<String>,7}89#[derive(Debug, Clone)]10pub enum TokenKind {11	/// May exist in token tree, but never in source code12	Meta { grammar_name: String, name: String },13	/// Specific parsing errors may be emitted as this type of kind14	Error {15		grammar_name: String,16		name: String,17		regex: Option<String>,18		priority: Option<u32>,19	},20	/// Keyword - literal match of token21	Keyword {22		/// How this keyword appears in grammar/code, should be same as Kinds key23		code: String,24		name: String,25	},26	/// Literal - something defined by user, i.e strings, identifiers, smth27	Literal {28		/// How this keyword appears in grammar, should be same as Kinds key29		grammar_name: String,30		name: String,31		/// Regex for Logos lexer32		regex: String,33		/// Path to custom lexer34		lexer: Option<String>,35	},36}3738impl TokenKind {39	pub fn grammar_name(&self) -> &str {40		match self {41			TokenKind::Keyword { code, .. } => code,42			TokenKind::Literal { grammar_name, .. } => grammar_name,43			TokenKind::Meta { grammar_name, .. } => grammar_name,44			TokenKind::Error { grammar_name, .. } => grammar_name,45		}46	}47	/// How this keyword should appear in kinds enum, screaming snake cased48	pub fn name(&self) -> &str {49		match self {50			TokenKind::Keyword { name, .. } => name,51			TokenKind::Literal { name, .. } => name,52			TokenKind::Meta { name, .. } => name,53			TokenKind::Error { name, .. } => name,54		}55	}56	pub fn expand_kind(&self) -> TokenStream {57		let name = format_ident!("{}", self.name());58		let attr = match self {59			TokenKind::Keyword { code, .. } => quote! {#[token(#code)]},60			TokenKind::Literal { regex, lexer, .. } => {61				let lexer = lexer62					.as_deref()63					.map(TokenStream::from_str)64					.map(|r| r.expect("path is correct"));65				quote! {#[regex(#regex, #lexer)]}66			}67			TokenKind::Error {68				regex, priority, ..69			} if regex.is_some() => {70				let priority = priority.map(|p| quote! {, priority = #p});71				quote! {#[regex(#regex #priority)]}72			}73			_ => quote! {},74		};75		quote! {76			#attr77			#name78		}79	}80	pub fn expand_t_macros(&self) -> Option<TokenStream> {81		match self {82			TokenKind::Keyword { code, name } => {83				let code = escape_token_macro(code);84				let name = format_ident!("{name}");85				Some(quote! {86					[#code] => {$crate::SyntaxKind::#name}87				})88			}89			// Meta items should not appear in T![_]90			_ => None,91		}92	}9394	/// How this token should be referenced in code95	/// Keywords are referenced with `T![_]` macro,96	/// and literals are referenced directly by name97	pub fn reference(&self) -> TokenStream {98		match self {99			TokenKind::Keyword { code, .. } => {100				let code = escape_token_macro(code);101				quote! {T![#code]}102			}103			_ => {104				let name = self.name();105				let ident = format_ident!("{name}");106				quote! {#ident}107			}108		}109	}110111	pub fn method_name(&self) -> Ident {112		match self {113			TokenKind::Keyword { name, .. } => {114				format_ident!("{}_token", name.to_lowercase())115			}116			TokenKind::Literal { name, .. } => {117				format_ident!("{}_lit", name.to_lowercase())118			}119			TokenKind::Meta { name, .. } => format_ident!("{}_meta", name.to_lowercase()),120			TokenKind::Error { name, .. } => format_ident!("{}_error", name.to_lowercase()),121		}122	}123}124125#[macro_export]126macro_rules! define_kinds {127	($into:ident = lit($name:literal) => $regex:literal $(, $lexer:literal)? $(; $($rest:tt)*)?) => {{128		$into.define_token(TokenKind::Literal {129			grammar_name: format!("LIT_{}!", $name),130			name: $name.to_owned(),131			regex: $regex.to_owned(),132			lexer: None $(.or_else(|| Some($lexer.to_string())))?,133		});134		$(define_kinds!($into = $($rest)*))?135	}};136	($into:ident = error($name:literal$(, priority = $priority:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{137		$into.define_token(TokenKind::Error {138			grammar_name: format!("ERROR_{}!", $name),139			name: format!("ERROR_{}", $name),140			regex: None$(.or(Some($regex.to_owned())))?,141			priority: None$(.or(Some($priority)))?,142		});143		$(define_kinds!($into = $($rest)*))?144	}};145	($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{146		$into.define_token(TokenKind::Keyword {147			code: format!("{}", $tok),148			name: $name.to_owned(),149		});150		$(define_kinds!($into = $($rest)*))?151	}};152	($into:ident =) => {{}}153}154use std::{collections::HashSet, str::FromStr};155156pub use define_kinds;157use indexmap::IndexMap;158use proc_macro2::{Ident, TokenStream};159use quote::{format_ident, quote};160161use super::escape_token_macro;162163impl KindsSrc {164	pub fn new() -> Self {165		Self {166			defined_tokens: IndexMap::new(),167			defined_node_names: HashSet::new(),168			nodes: Vec::new(),169		}170	}171	pub fn define_token(&mut self, token: TokenKind) {172		assert!(173			self.defined_node_names.insert(token.name().to_owned()),174			"node name already defined: {}",175			token.name()176		);177		assert!(178			self.defined_tokens179				.insert(token.grammar_name().to_owned(), token.clone())180				.is_none(),181			"token already defined: {}",182			token.grammar_name()183		)184	}185	pub fn define_node(&mut self, node: &str) {186		assert!(187			self.defined_node_names.insert(node.to_owned()),188			"node name already defined: {}",189			node190		);191		self.nodes.push(node.to_string())192	}193	pub fn token(&self, tok: &str) -> Option<&TokenKind> {194		self.defined_tokens.get(tok)195	}196	pub fn is_token(&self, tok: &str) -> bool {197		self.defined_tokens.contains_key(tok)198	}199	pub fn tokens(&self) -> impl Iterator<Item = &TokenKind> {200		self.defined_tokens.iter().map(|(_, v)| v)201	}202}203204pub fn jsonnet_kinds() -> KindsSrc {205	let mut kinds = KindsSrc::new();206	define_kinds![kinds =207		"||" => "OR";208		"&&" => "AND";209		"|" => "BIT_OR";210		"^" => "BIT_XOR";211		"&" => "BIT_AND";212		"==" => "EQ";213		"!=" => "NE";214		"<" => "LT";215		">" => "GT";216		"<=" => "LE";217		">=" => "GE";218		"<<" => "LHS";219		">>" => "RHS";220		"+" => "PLUS";221		"-" => "MINUS";222		"*" => "MUL";223		"/" => "DIV";224		"%" => "MODULO";225		"!" => "NOT";226		"~" => "BIT_NOT";227		"[" => "L_BRACK";228		"]" => "R_BRACK";229		"(" => "L_PAREN";230		")" => "R_PAREN";231		"{" => "L_BRACE";232		"}" => "R_BRACE";233		":" => "COLON";234		"::" => "COLONCOLON";235		":::" => "COLONCOLONCOLON";236		";" => "SEMI";237		"." => "DOT";238		"..." => "DOTDOTDOT";239		"," => "COMMA";240		"$" => "DOLLAR";241		"=" => "ASSIGN";242		"?" => "QUESTION_MARK";243		"$intrinsicThisFile" => "INTRINSIC_THIS_FILE";244		"$intrinsicId" => "INTRINSIC_ID";245		"$intrinsic" => "INTRINSIC";246		// Literals247		lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?";248		error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]";249		error("FLOAT_JUNK_AFTER_EXPONENT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]";250		error("FLOAT_JUNK_AFTER_EXPONENT_SIGN") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]";251		lit("STRING_DOUBLE") => "\"(?s:[^\"\\\\]|\\\\.)*\"";252		error("STRING_DOUBLE_UNTERMINATED") => "\"(?s:[^\"\\\\]|\\\\.)*";253		lit("STRING_SINGLE") => "'(?s:[^'\\\\]|\\\\.)*'";254		error("STRING_SINGLE_UNTERMINATED") => "'(?s:[^'\\\\]|\\\\.)*";255		lit("STRING_DOUBLE_VERBATIM") => "@\"(?:[^\"]|\"\")*\"";256		error("STRING_DOUBLE_VERBATIM_UNTERMINATED") => "@\"(?:[^\"]|\"\")*";257		lit("STRING_SINGLE_VERBATIM") => "@'(?:[^']|'')*'";258		error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";259		error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";260		lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";261		error("STRING_BLOCK_UNEXPECTED_END");262		error("STRING_BLOCK_MISSING_NEW_LINE");263		error("STRING_BLOCK_MISSING_TERMINATION");264		error("STRING_BLOCK_MISSING_INDENT");265		lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";266		lit("WHITESPACE") => r"[ \t\n\r]+";267		lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";268		lit("SINGLE_LINE_HASH_COMMENT") => r"#[^\r\n]*(\r\n|\n)?";269		lit("MULTI_LINE_COMMENT") => r"/\*([^*]|\*[^/])*\*/";270		error("COMMENT_TOO_SHORT") => r"/\*/";271		error("COMMENT_UNTERMINATED") =>  r"/\*([^*]|\*[^/])+";272	];273	kinds274}