git.delta.rocks / jrsonnet / refs/commits / 7c03fc40023d

1use std::{collections::HashMap, path::PathBuf};23use anyhow::Result;4use ast::{lower, AstSrc};5use itertools::Itertools;6use kinds::{KindsSrc, TokenKind};7use proc_macro2::{Ident, Punct, Spacing, Span, TokenStream};8use quote::{format_ident, quote};9use ungrammar::Grammar;10use util::{ensure_file_contents, reformat, to_pascal_case, to_upper_snake_case};1112mod ast;13mod kinds;14mod util;1516enum SpecialName {17	Literal,18	Meta,19	Error,20}21fn classify_special(name: &str) -> Option<(SpecialName, &str)> {22	let name = name.strip_suffix('!')?;23	Some(if let Some(name) = name.strip_prefix("LIT_") {24		(SpecialName::Literal, name)25	} else if let Some(name) = name.strip_prefix("META_") {26		(SpecialName::Meta, name)27	} else if let Some(name) = name.strip_prefix("ERROR_") {28		(SpecialName::Error, name)29	} else {30		return None;31	})32}3334pub fn generate_ungrammar() -> Result<()> {35	let grammar: Grammar = include_str!(concat!(36		env!("CARGO_MANIFEST_DIR"),37		"/../crates/jrsonnet-rowan-parser/jsonnet.ungram"38	))39	.parse()?;4041	let mut kinds = kinds::jsonnet_kinds();42	let ast = lower(&kinds, &grammar);4344	for token in grammar.tokens() {45		let token = &grammar[token];46		let token = &token.name.clone();47		if !kinds.is_token(token) {48			if let Some((special, name)) = classify_special(token) {49				match special {50					SpecialName::Literal => panic!("literal is not defined: {name}"),51					SpecialName::Meta => {52						eprintln!("implicit meta: {name}");53						kinds.define_token(TokenKind::Meta {54							grammar_name: token.to_owned(),55							name: format!("META_{name}"),56						});57					}58					SpecialName::Error => {59						eprintln!("implicit error: {name}");60						kinds.define_token(TokenKind::Error {61							grammar_name: token.to_owned(),62							name: format!("ERROR_{name}"),63							regex: None,64							priority: None,65							is_lexer_error: true,66						});67					}68				}69				continue;70			}71			let name = to_upper_snake_case(token);72			eprintln!("implicit kw: {token}");73			kinds.define_token(TokenKind::Keyword {74				code: token.to_owned(),75				name: format!("{name}_KW"),76			});77		}78	}79	for node in &ast.nodes {80		let name = to_upper_snake_case(&node.name);81		kinds.define_node(&name);82	}83	for enum_ in &ast.enums {84		let name = to_upper_snake_case(&enum_.name);85		kinds.define_node(&name);86	}87	for token_enum in &ast.token_enums {88		let name = to_upper_snake_case(&token_enum.name);89		kinds.define_node(&name);90	}9192	let syntax_kinds = generate_syntax_kinds(&kinds, &ast, false)?;9394	let nodes = generate_nodes(&kinds, &ast)?;95	ensure_file_contents(96		&PathBuf::from(concat!(97			env!("CARGO_MANIFEST_DIR"),98			"/../crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs",99		)),100		&syntax_kinds,101	);102	ensure_file_contents(103		&PathBuf::from(concat!(104			env!("CARGO_MANIFEST_DIR"),105			"/../crates/jrsonnet-rowan-parser/src/generated/nodes.rs",106		)),107		&nodes,108	);109110	let lexer_syntax_kinds = generate_syntax_kinds(&kinds, &ast, true)?;111	ensure_file_contents(112		&PathBuf::from(concat!(113			env!("CARGO_MANIFEST_DIR"),114			"/../crates/jrsonnet-lexer/src/generated/syntax_kinds.rs",115		)),116		&lexer_syntax_kinds,117	);118	Ok(())119}120121fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc, lexer: bool) -> Result<String> {122	let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);123	let token_kinds = kinds.tokens().map(|t| t.expand_kind(lexer));124125	let keywords = kinds126		.tokens()127		.filter(|k| matches!(k, TokenKind::Keyword { .. }))128		.map(TokenKind::name)129		.map(|n| format_ident!("{n}"));130131	let mut nodes = kinds132		.nodes133		.iter()134		.map(|name| format_ident!("{}", name))135		.collect::<Vec<_>>();136137	if lexer {138		nodes.clear();139	}140141	let enums = grammar142		.enums143		.iter()144		.map(|e| format_ident!("{}", to_upper_snake_case(&e.name)))145		.chain(146			grammar147				.token_enums148				.iter()149				.map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),150		)151		.collect::<Vec<_>>();152	let is_enum = if lexer {153		quote! {}154	} else {155		quote! {156			pub fn is_enum(self) -> bool {157				match self {158					#(#enums)|* => true,159					_ => false,160				}161			}162		}163	};164165	let derive_logos = if lexer {166		quote! {167			, logos::Logos168		}169	} else {170		quote! {}171	};172173	let ast = quote! {174		#![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]175176		/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`.177		#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug #derive_logos)]178		#[repr(u16)]179		pub enum SyntaxKind {180			#[doc(hidden)]181			TOMBSTONE,182			#[doc(hidden)]183			EOF,184			#(#token_kinds,)*185			LEXING_ERROR,186			__LAST_TOKEN,187			#(#nodes,)*188			#[doc(hidden)]189			__LAST,190		}191		use self::SyntaxKind::*;192193		impl SyntaxKind {194			pub fn is_keyword(self) -> bool {195				match self {196					#(#keywords)|* => true,197					_ => false,198				}199			}200201			#is_enum202203			pub fn from_raw(r: u16) -> Self {204				assert!(r < Self::__LAST as u16);205				unsafe { std::mem::transmute(r) }206			}207			pub fn into_raw(self) -> u16 {208				self as u16209			}210		}211212		#[macro_export]213		macro_rules! T {#(#t_macros);*}214		#[allow(unused_imports)]215		pub use T;216	};217218	reformat(&ast.to_string())219}220221#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]222fn generate_nodes(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {223	let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar224		.nodes225		.iter()226		.map(|node| {227			let name = format_ident!("{}", node.name);228			let kind = format_ident!("{}", to_upper_snake_case(&node.name));229			let traits = node.traits.iter().map(|trait_name| {230				let trait_name = format_ident!("{}", trait_name);231				quote!(impl ast::#trait_name for #name {})232			});233234			let mut type_positions: HashMap<String, usize> = HashMap::new();235			let field_positions: Vec<_> = node236				.fields237				.iter()238				.map(|field| {239					let ty_str = field.ty().to_string();240					let pos = *type_positions.get(&ty_str).unwrap_or(&0);241					type_positions.insert(ty_str, pos + 1);242					pos243				})244				.collect();245246			let methods = node247				.fields248				.iter()249				.zip(field_positions.iter())250				.map(|(field, &pos)| {251					let method_name = field.method_name(kinds);252					let ty = field.ty();253254					if field.is_many() {255						quote! {256							pub fn #method_name(&self) -> AstChildren<#ty> {257								support::children(&self.syntax)258							}259						}260					} else if let Some(token_kind) = field.token_kind(kinds) {261						quote! {262							pub fn #method_name(&self) -> Option<#ty> {263								support::token(&self.syntax, #token_kind)264							}265						}266					} else if field.is_token_enum(grammar) {267						quote! {268							pub fn #method_name(&self) -> Option<#ty> {269								support::token_child(&self.syntax)270							}271						}272					} else if pos == 0 {273						quote! {274							pub fn #method_name(&self) -> Option<#ty> {275								support::children(&self.syntax).next()276							}277						}278					} else {279						quote! {280							pub fn #method_name(&self) -> Option<#ty> {281								support::children(&self.syntax).nth(#pos)282							}283						}284					}285				});286			(287				quote! {288					#[pretty_doc_comment_placeholder_workaround]289					#[derive(Debug, Clone, PartialEq, Eq, Hash)]290					pub struct #name {291						pub(crate) syntax: SyntaxNode,292					}293294					#(#traits)*295296					impl #name {297						#(#methods)*298					}299				},300				quote! {301					impl AstNode for #name {302						fn can_cast(kind: SyntaxKind) -> bool {303							kind == #kind304						}305						fn cast(syntax: SyntaxNode) -> Option<Self> {306							if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }307						}308						fn syntax(&self) -> &SyntaxNode { &self.syntax }309					}310				},311			)312		})313		.unzip();314315	let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar316		.enums317		.iter()318		.map(|en| {319			let variants: Vec<_> = en320				.variants321				.iter()322				.map(|var| format_ident!("{}", var))323				.collect();324			let name = format_ident!("{}", en.name);325			let kinds: Vec<_> = variants326				.iter()327				.map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))328				.collect();329			let traits = en.traits.iter().map(|trait_name| {330				let trait_name = format_ident!("{}", trait_name);331				quote!(impl ast::#trait_name for #name {})332			});333334			let ast_node = quote! {335				impl AstNode for #name {336					fn can_cast(kind: SyntaxKind) -> bool {337						match kind {338							#(#kinds)|* => true,339							_ => false,340						}341					}342					fn cast(syntax: SyntaxNode) -> Option<Self> {343						let res = match syntax.kind() {344							#(345							#kinds => #name::#variants(#variants { syntax }),346							)*347							_ => return None,348						};349						Some(res)350					}351					fn syntax(&self) -> &SyntaxNode {352						match self {353							#(354							#name::#variants(it) => &it.syntax,355							)*356						}357					}358				}359			};360361			(362				quote! {363					#[pretty_doc_comment_placeholder_workaround]364					#[derive(Debug, Clone, PartialEq, Eq, Hash)]365					pub enum #name {366						#(#variants(#variants),)*367					}368369					#(#traits)*370				},371				quote! {372					#(373						impl From<#variants> for #name {374							fn from(node: #variants) -> #name {375								#name::#variants(node)376							}377						}378					)*379					#ast_node380				},381			)382		})383		.unzip();384385	let (token_enum_defs, token_enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar386		.token_enums387		.iter()388		.map(|en| {389			let variants: Vec<_> = en390				.variants391				.iter()392				.map(|token| {393					format_ident!(394						"{}",395						to_pascal_case(kinds.token(token).expect("token exists").name())396					)397				})398				.collect();399			let name = format_ident!("{}", en.name);400			let kind_name = format_ident!("{}Kind", en.name);401			let kinds: Vec<_> = variants402				.iter()403				.map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))404				.collect();405406			let ast_node = quote! {407				impl AstToken for #name {408					fn can_cast(kind: SyntaxKind) -> bool {409						#kind_name::can_cast(kind)410					}411					fn cast(syntax: SyntaxToken) -> Option<Self> {412						let kind = #kind_name::cast(syntax.kind())?;413						Some(#name { syntax, kind })414					}415					fn syntax(&self) -> &SyntaxToken {416						&self.syntax417					}418				}419420				impl #kind_name {421					fn can_cast(kind: SyntaxKind) -> bool {422						match kind {423							#(#kinds)|* => true,424							_ => false,425						}426					}427					pub fn cast(kind: SyntaxKind) -> Option<Self> {428						let res = match kind {429							#(#kinds => Self::#variants,)*430							_ => return None,431						};432						Some(res)433					}434				}435			};436437			(438				quote! {439					#[pretty_doc_comment_placeholder_workaround]440					#[derive(Debug, Clone, PartialEq, Eq, Hash)]441					pub struct #name { syntax: SyntaxToken, kind: #kind_name }442443					#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]444					pub enum #kind_name {445						#(#variants,)*446					}447				},448				quote! {449					#ast_node450451					impl #name {452						pub fn kind(&self) -> #kind_name {453							self.kind454						}455					}456457					impl std::fmt::Display for #name {458						fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {459							std::fmt::Display::fmt(self.syntax(), f)460						}461					}462				},463			)464		})465		.unzip();466467	let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar468		.nodes469		.iter()470		.flat_map(|node| node.traits.iter().map(move |t| (t, node)))471		.into_group_map()472		.into_iter()473		.sorted_by_key(|(k, _)| *k)474		.map(|(trait_name, nodes)| {475			let name = format_ident!("Any{}", trait_name);476			let trait_name = format_ident!("{}", trait_name);477			let kinds: Vec<_> = nodes478				.iter()479				.map(|name| format_ident!("{}", to_upper_snake_case(&name.name)))480				.collect();481482			(483				quote! {484					#[pretty_doc_comment_placeholder_workaround]485					#[derive(Debug, Clone, PartialEq, Eq, Hash)]486					pub struct #name {487						pub(crate) syntax: SyntaxNode,488					}489					impl ast::#trait_name for #name {}490				},491				quote! {492					impl #name {493						#[inline]494						pub fn new<T: ast::#trait_name>(node: T) -> #name {495							#name {496								syntax: node.syntax().clone()497							}498						}499					}500					impl AstNode for #name {501						fn can_cast(kind: SyntaxKind) -> bool {502							match kind {503								#(#kinds)|* => true,504								_ => false,505							}506						}507						fn cast(syntax: SyntaxNode) -> Option<Self> {508							Self::can_cast(syntax.kind()).then(|| #name { syntax })509						}510						fn syntax(&self) -> &SyntaxNode {511							&self.syntax512						}513					}514				},515			)516		})517		.unzip();518519	let enum_names = grammar.enums.iter().map(|it| &it.name);520	let node_names = grammar.nodes.iter().map(|it| &it.name);521522	let display_impls = enum_names523		.chain(node_names.clone())524		.map(|it| format_ident!("{}", it))525		.map(|name| {526			quote! {527				impl std::fmt::Display for #name {528					fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {529						std::fmt::Display::fmt(self.syntax(), f)530					}531				}532			}533		});534535	let ast = quote! {536		#![allow(non_snake_case, clippy::match_like_matches_macro)]537538		use crate::{539			SyntaxNode, SyntaxToken, SyntaxKind::{self, *},540			ast::{AstNode, AstToken, AstChildren, support},541			T,542		};543544		#(#node_defs)*545		#(#enum_defs)*546		#(#token_enum_defs)*547		#(#any_node_defs)*548		#(#node_boilerplate_impls)*549		#(#enum_boilerplate_impls)*550		#(#token_enum_boilerplate_impls)*551		#(#any_node_boilerplate_impls)*552		#(#display_impls)*553	};554555	let ast = ast.to_string().replace("T ! [", "T![");556557	let mut res = String::with_capacity(ast.len() * 2);558559	let mut docs = grammar560		.nodes561		.iter()562		.map(|it| &it.doc)563		.chain(grammar.enums.iter().map(|it| &it.doc));564565	for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {566		res.push_str(chunk);567		if let Some(doc) = docs.next() {568			write_doc_comment(doc, &mut res);569		}570	}571572	let res = reformat(&res)?;573	Ok(res.replace("#[derive", "\n#[derive"))574}575576fn write_doc_comment(contents: &[String], dest: &mut String) {577	use std::fmt::Write;578	for line in contents {579		writeln!(dest, "///{line}").unwrap();580	}581}582583pub fn escape_token_macro(token: &str) -> TokenStream {584	if "{}[]()$".contains(token) {585		let c = token.chars().next().unwrap();586		quote! { #c }587	} else if token.contains('$') {588		quote! { #token }589	} else if token.chars().all(|v: char| v.is_ascii_lowercase()) {590		let i = Ident::new(token, Span::call_site());591		quote! { #i }592	} else {593		let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));594		quote! { #(#cs)* }595	}596}