git.delta.rocks / jrsonnet / refs/commits / da64a7455472

1use std::path::PathBuf;23use anyhow::Result;4use ast::{lower, AstSrc};5use itertools::Itertools;6use kinds::{KindsSrc, TokenKind};7use proc_macro2::{Punct, Spacing, TokenStream};8use quote::{format_ident, quote};9use ungrammar::Grammar;10use util::{ensure_file_contents, reformat, to_pascal_case, to_upper_snake_case};1112mod ast;13mod kinds;14mod util;1516enum SpecialName {17	Literal,18	Meta,19	Error,20}21fn classify_special(name: &str) -> Option<(SpecialName, &str)> {22	let name = name.strip_suffix('!')?;23	Some(if let Some(name) = name.strip_prefix("LIT_") {24		(SpecialName::Literal, name)25	} else if let Some(name) = name.strip_prefix("META_") {26		(SpecialName::Meta, name)27	} else if let Some(name) = name.strip_prefix("ERROR_") {28		(SpecialName::Error, name)29	} else {30		return None;31	})32}3334pub fn generate_ungrammar() -> Result<()> {35	let grammar: Grammar = include_str!(concat!(36		env!("CARGO_MANIFEST_DIR"),37		"/../crates/jrsonnet-rowan-parser/jsonnet.ungram"38	))39	.parse()?;4041	let mut kinds = kinds::jsonnet_kinds();42	let ast = lower(&kinds, &grammar);4344	for token in grammar.tokens() {45		let token = &grammar[token];46		let token = &token.name.clone();47		if !kinds.is_token(token) {48			if let Some((special, name)) = classify_special(token) {49				match special {50					SpecialName::Literal => panic!("literal is not defined: {name}"),51					SpecialName::Meta => {52						eprintln!("implicit meta: {}", name);53						kinds.define_token(TokenKind::Meta {54							grammar_name: token.to_owned(),55							name: format!("META_{}", name),56						})57					}58					SpecialName::Error => {59						eprintln!("implicit error: {}", name);60						kinds.define_token(TokenKind::Error {61							grammar_name: token.to_owned(),62							name: format!("ERROR_{}", name),63							regex: None,64							priority: None,65							is_lexer_error: true,66						})67					}68				};69				continue;70			};71			let name = to_upper_snake_case(token);72			eprintln!("implicit kw: {}", token);73			kinds.define_token(TokenKind::Keyword {74				code: token.to_owned(),75				name: format!("{name}_KW"),76			});77		}78	}79	for node in &ast.nodes {80		let name = to_upper_snake_case(&node.name);81		kinds.define_node(&name);82	}83	for enum_ in &ast.enums {84		let name = to_upper_snake_case(&enum_.name);85		kinds.define_node(&name);86	}87	for token_enum in &ast.token_enums {88		let name = to_upper_snake_case(&token_enum.name);89		kinds.define_node(&name);90	}9192	let syntax_kinds = generate_syntax_kinds(&kinds, &ast)?;9394	let nodes = generate_nodes(&kinds, &ast)?;95	ensure_file_contents(96		&PathBuf::from(concat!(97			env!("CARGO_MANIFEST_DIR"),98			"/../crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs",99		)),100		&syntax_kinds,101	)?;102	ensure_file_contents(103		&PathBuf::from(concat!(104			env!("CARGO_MANIFEST_DIR"),105			"/../crates/jrsonnet-rowan-parser/src/generated/nodes.rs",106		)),107		&nodes,108	)?;109	Ok(())110}111112fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {113	let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);114	let token_kinds = kinds.tokens().map(TokenKind::expand_kind);115116	let keywords = kinds117		.tokens()118		.filter(|k| matches!(k, TokenKind::Keyword { .. }))119		.map(TokenKind::name)120		.map(|n| format_ident!("{n}"));121122	let nodes = kinds123		.nodes124		.iter()125		.map(|name| format_ident!("{}", name))126		.collect::<Vec<_>>();127128	let enums = grammar129		.enums130		.iter()131		.map(|e| format_ident!("{}", to_upper_snake_case(&e.name)))132		.chain(133			grammar134				.token_enums135				.iter()136				.map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),137		);138139	let ast = quote! {140		#![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]141		use logos::Logos;142143		/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`.144		#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Logos)]145		#[repr(u16)]146		pub enum SyntaxKind {147			#[doc(hidden)]148			TOMBSTONE,149			#[doc(hidden)]150			EOF,151			#(#token_kinds,)*152			/// Also acts as __LAST_TOKEN153			#[error]154			LEXING_ERROR,155			#(#nodes,)*156			#[doc(hidden)]157			__LAST,158		}159		use self::SyntaxKind::*;160161		impl SyntaxKind {162			pub fn is_keyword(self) -> bool {163				match self {164					#(#keywords)|* => true,165					_ => false,166				}167			}168			pub fn is_enum(self) -> bool {169				match self {170					#(#enums)|* => true,171					_ => false,172				}173			}174175			pub fn from_raw(r: u16) -> Self {176				assert!(r < Self::__LAST as u16);177				unsafe { std::mem::transmute(r) }178			}179			pub fn into_raw(self) -> u16 {180				self as u16181			}182		}183184		#[macro_export]185		macro_rules! T {#(#t_macros);*}186		#[allow(unused_imports)]187		pub use T;188	};189190	reformat(&ast.to_string())191}192193fn generate_nodes(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {194	let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar195		.nodes196		.iter()197		.map(|node| {198			let name = format_ident!("{}", node.name);199			let kind = format_ident!("{}", to_upper_snake_case(&node.name));200			let traits = node.traits.iter().map(|trait_name| {201				let trait_name = format_ident!("{}", trait_name);202				quote!(impl ast::#trait_name for #name {})203			});204205			let methods = node.fields.iter().map(|field| {206				let method_name = field.method_name(kinds);207				let ty = field.ty();208209				if field.is_many() {210					quote! {211						pub fn #method_name(&self) -> AstChildren<#ty> {212							support::children(&self.syntax)213						}214					}215				} else if let Some(token_kind) = field.token_kind(kinds) {216					quote! {217						pub fn #method_name(&self) -> Option<#ty> {218							support::token(&self.syntax, #token_kind)219						}220					}221				} else if field.is_token_enum(grammar) {222					quote! {223						pub fn #method_name(&self) -> Option<#ty> {224							support::token_child(&self.syntax)225						}226					}227				} else {228					quote! {229						pub fn #method_name(&self) -> Option<#ty> {230							support::child(&self.syntax)231						}232					}233				}234			});235			(236				quote! {237					#[pretty_doc_comment_placeholder_workaround]238					#[derive(Debug, Clone, PartialEq, Eq, Hash)]239					pub struct #name {240						pub(crate) syntax: SyntaxNode,241					}242243					#(#traits)*244245					impl #name {246						#(#methods)*247					}248				},249				quote! {250					impl AstNode for #name {251						fn can_cast(kind: SyntaxKind) -> bool {252							kind == #kind253						}254						fn cast(syntax: SyntaxNode) -> Option<Self> {255							if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }256						}257						fn syntax(&self) -> &SyntaxNode { &self.syntax }258					}259				},260			)261		})262		.unzip();263264	let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar265		.enums266		.iter()267		.map(|en| {268			let variants: Vec<_> = en269				.variants270				.iter()271				.map(|var| format_ident!("{}", var))272				.collect();273			let name = format_ident!("{}", en.name);274			let kinds: Vec<_> = variants275				.iter()276				.map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))277				.collect();278			let traits = en.traits.iter().map(|trait_name| {279				let trait_name = format_ident!("{}", trait_name);280				quote!(impl ast::#trait_name for #name {})281			});282283			let ast_node = quote! {284				impl AstNode for #name {285					fn can_cast(kind: SyntaxKind) -> bool {286						match kind {287							#(#kinds)|* => true,288							_ => false,289						}290					}291					fn cast(syntax: SyntaxNode) -> Option<Self> {292						let res = match syntax.kind() {293							#(294							#kinds => #name::#variants(#variants { syntax }),295							)*296							_ => return None,297						};298						Some(res)299					}300					fn syntax(&self) -> &SyntaxNode {301						match self {302							#(303							#name::#variants(it) => &it.syntax,304							)*305						}306					}307				}308			};309310			(311				quote! {312					#[pretty_doc_comment_placeholder_workaround]313					#[derive(Debug, Clone, PartialEq, Eq, Hash)]314					pub enum #name {315						#(#variants(#variants),)*316					}317318					#(#traits)*319				},320				quote! {321					#(322						impl From<#variants> for #name {323							fn from(node: #variants) -> #name {324								#name::#variants(node)325							}326						}327					)*328					#ast_node329				},330			)331		})332		.unzip();333334	let (token_enum_defs, token_enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar335		.token_enums336		.iter()337		.map(|en| {338			let variants: Vec<_> = en339				.variants340				.iter()341				.map(|token| {342					format_ident!(343						"{}",344						to_pascal_case(kinds.token(token).expect("token exists").name())345					)346				})347				.collect();348			let name = format_ident!("{}", en.name);349			let kind_name = format_ident!("{}Kind", en.name);350			let kinds: Vec<_> = variants351				.iter()352				.map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))353				.collect();354355			let ast_node = quote! {356				impl AstToken for #name {357					fn can_cast(kind: SyntaxKind) -> bool {358						#kind_name::can_cast(kind)359					}360					fn cast(syntax: SyntaxToken) -> Option<Self> {361						let kind = #kind_name::cast(syntax.kind())?;362						Some(#name { syntax, kind })363					}364					fn syntax(&self) -> &SyntaxToken {365						&self.syntax366					}367				}368369				impl #kind_name {370					fn can_cast(kind: SyntaxKind) -> bool {371						match kind {372							#(#kinds)|* => true,373							_ => false,374						}375					}376					pub fn cast(kind: SyntaxKind) -> Option<Self> {377						let res = match kind {378							#(#kinds => Self::#variants,)*379							_ => return None,380						};381						Some(res)382					}383				}384			};385386			(387				quote! {388					#[pretty_doc_comment_placeholder_workaround]389					#[derive(Debug, Clone, PartialEq, Eq, Hash)]390					pub struct #name { syntax: SyntaxToken, kind: #kind_name }391392					#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]393					pub enum #kind_name {394						#(#variants,)*395					}396				},397				quote! {398					#ast_node399400					impl #name {401						pub fn kind(&self) -> #kind_name {402							self.kind403						}404					}405406					impl std::fmt::Display for #name {407						fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {408							std::fmt::Display::fmt(self.syntax(), f)409						}410					}411				},412			)413		})414		.unzip();415416	let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar417		.nodes418		.iter()419		.flat_map(|node| node.traits.iter().map(move |t| (t, node)))420		.into_group_map()421		.into_iter()422		.sorted_by_key(|(k, _)| *k)423		.map(|(trait_name, nodes)| {424			let name = format_ident!("Any{}", trait_name);425			let trait_name = format_ident!("{}", trait_name);426			let kinds: Vec<_> = nodes427				.iter()428				.map(|name| format_ident!("{}", to_upper_snake_case(&name.name.to_string())))429				.collect();430431			(432				quote! {433					#[pretty_doc_comment_placeholder_workaround]434					#[derive(Debug, Clone, PartialEq, Eq, Hash)]435					pub struct #name {436						pub(crate) syntax: SyntaxNode,437					}438					impl ast::#trait_name for #name {}439				},440				quote! {441					impl #name {442						#[inline]443						pub fn new<T: ast::#trait_name>(node: T) -> #name {444							#name {445								syntax: node.syntax().clone()446							}447						}448					}449					impl AstNode for #name {450						fn can_cast(kind: SyntaxKind) -> bool {451							match kind {452								#(#kinds)|* => true,453								_ => false,454							}455						}456						fn cast(syntax: SyntaxNode) -> Option<Self> {457							Self::can_cast(syntax.kind()).then(|| #name { syntax })458						}459						fn syntax(&self) -> &SyntaxNode {460							&self.syntax461						}462					}463				},464			)465		})466		.unzip();467468	let enum_names = grammar.enums.iter().map(|it| &it.name);469	let node_names = grammar.nodes.iter().map(|it| &it.name);470471	let display_impls = enum_names472		.chain(node_names.clone())473		.map(|it| format_ident!("{}", it))474		.map(|name| {475			quote! {476				impl std::fmt::Display for #name {477					fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {478						std::fmt::Display::fmt(self.syntax(), f)479					}480				}481			}482		});483484	let ast = quote! {485		#![allow(non_snake_case, clippy::match_like_matches_macro)]486487		use crate::{488			SyntaxNode, SyntaxToken, SyntaxKind::{self, *},489			ast::{AstNode, AstToken, AstChildren, support},490			T,491		};492493		#(#node_defs)*494		#(#enum_defs)*495		#(#token_enum_defs)*496		#(#any_node_defs)*497		#(#node_boilerplate_impls)*498		#(#enum_boilerplate_impls)*499		#(#token_enum_boilerplate_impls)*500		#(#any_node_boilerplate_impls)*501		#(#display_impls)*502	};503504	let ast = ast.to_string().replace("T ! [", "T![");505506	let mut res = String::with_capacity(ast.len() * 2);507508	let mut docs = grammar509		.nodes510		.iter()511		.map(|it| &it.doc)512		.chain(grammar.enums.iter().map(|it| &it.doc));513514	for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {515		res.push_str(chunk);516		if let Some(doc) = docs.next() {517			write_doc_comment(doc, &mut res);518		}519	}520521	let res = reformat(&res)?;522	Ok(res.replace("#[derive", "\n#[derive"))523}524525fn write_doc_comment(contents: &[String], dest: &mut String) {526	use std::fmt::Write;527	for line in contents {528		writeln!(dest, "///{}", line).unwrap();529	}530}531532pub fn escape_token_macro(token: &str) -> TokenStream {533	if "{}[]()$".contains(token) {534		let c = token.chars().next().unwrap();535		quote! { #c }536	} else if token.contains('$') {537		quote! { #token }538	} else {539		let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));540		quote! { #(#cs)* }541	}542}