1use std::path::PathBuf;23use anyhow::Result;4use ast::{lower, AstSrc};5use itertools::Itertools;6use kinds::{KindsSrc, TokenKind};7use proc_macro2::{Ident, Punct, Spacing, Span, TokenStream};8use quote::{format_ident, quote};9use ungrammar::Grammar;10use util::{ensure_file_contents, reformat, to_pascal_case, to_upper_snake_case};1112mod ast;13mod kinds;14mod util;1516enum SpecialName {17 Literal,18 Meta,19 Error,20}21fn classify_special(name: &str) -> Option<(SpecialName, &str)> {22 let name = name.strip_suffix('!')?;23 Some(if let Some(name) = name.strip_prefix("LIT_") {24 (SpecialName::Literal, name)25 } else if let Some(name) = name.strip_prefix("META_") {26 (SpecialName::Meta, name)27 } else if let Some(name) = name.strip_prefix("ERROR_") {28 (SpecialName::Error, name)29 } else {30 return None;31 })32}3334pub fn generate_ungrammar() -> Result<()> {35 let grammar: Grammar = include_str!(concat!(36 env!("CARGO_MANIFEST_DIR"),37 "/../crates/jrsonnet-rowan-parser/jsonnet.ungram"38 ))39 .parse()?;4041 let mut kinds = kinds::jsonnet_kinds();42 let ast = lower(&kinds, &grammar);4344 for token in grammar.tokens() {45 let token = &grammar[token];46 let token = &token.name.clone();47 if !kinds.is_token(token) {48 if let Some((special, name)) = classify_special(token) {49 match special {50 SpecialName::Literal => panic!("literal is not defined: {name}"),51 SpecialName::Meta => {52 eprintln!("implicit meta: {name}");53 kinds.define_token(TokenKind::Meta {54 grammar_name: token.to_owned(),55 name: format!("META_{name}"),56 });57 }58 SpecialName::Error => {59 eprintln!("implicit error: {name}");60 kinds.define_token(TokenKind::Error {61 grammar_name: token.to_owned(),62 name: format!("ERROR_{name}"),63 regex: None,64 priority: None,65 is_lexer_error: true,66 });67 }68 };69 continue;70 };71 let name = to_upper_snake_case(token);72 eprintln!("implicit kw: {token}");73 kinds.define_token(TokenKind::Keyword {74 code: token.to_owned(),75 name: format!("{name}_KW"),76 });77 }78 }79 for node in &ast.nodes {80 let name = to_upper_snake_case(&node.name);81 kinds.define_node(&name);82 }83 for enum_ in &ast.enums {84 let name = to_upper_snake_case(&enum_.name);85 kinds.define_node(&name);86 }87 for token_enum in &ast.token_enums {88 let name = to_upper_snake_case(&token_enum.name);89 kinds.define_node(&name);90 }9192 let syntax_kinds = generate_syntax_kinds(&kinds, &ast)?;9394 let nodes = generate_nodes(&kinds, &ast)?;95 ensure_file_contents(96 &PathBuf::from(concat!(97 env!("CARGO_MANIFEST_DIR"),98 "/../crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs",99 )),100 &syntax_kinds,101 );102 ensure_file_contents(103 &PathBuf::from(concat!(104 env!("CARGO_MANIFEST_DIR"),105 "/../crates/jrsonnet-rowan-parser/src/generated/nodes.rs",106 )),107 &nodes,108 );109 Ok(())110}111112fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {113 let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);114 let token_kinds = kinds.tokens().map(TokenKind::expand_kind);115116 let keywords = kinds117 .tokens()118 .filter(|k| matches!(k, TokenKind::Keyword { .. }))119 .map(TokenKind::name)120 .map(|n| format_ident!("{n}"));121122 let nodes = kinds123 .nodes124 .iter()125 .map(|name| format_ident!("{}", name))126 .collect::<Vec<_>>();127128 let enums = grammar129 .enums130 .iter()131 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name)))132 .chain(133 grammar134 .token_enums135 .iter()136 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),137 );138139 let ast = quote! {140 #![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]141 use logos::Logos;142143 144 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Logos)]145 #[repr(u16)]146 pub enum SyntaxKind {147 #[doc(hidden)]148 TOMBSTONE,149 #[doc(hidden)]150 EOF,151 #(#token_kinds,)*152 LEXING_ERROR,153 __LAST_TOKEN,154 #(#nodes,)*155 #[doc(hidden)]156 __LAST,157 }158 use self::SyntaxKind::*;159160 impl SyntaxKind {161 pub fn is_keyword(self) -> bool {162 match self {163 #(#keywords)|* => true,164 _ => false,165 }166 }167 pub fn is_enum(self) -> bool {168 match self {169 #(#enums)|* => true,170 _ => false,171 }172 }173174 pub fn from_raw(r: u16) -> Self {175 assert!(r < Self::__LAST as u16);176 unsafe { std::mem::transmute(r) }177 }178 pub fn into_raw(self) -> u16 {179 self as u16180 }181 }182183 #[macro_export]184 macro_rules! T {#(#t_macros);*}185 #[allow(unused_imports)]186 pub use T;187 };188189 reformat(&ast.to_string())190}191192#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]193fn generate_nodes(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {194 let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar195 .nodes196 .iter()197 .map(|node| {198 let name = format_ident!("{}", node.name);199 let kind = format_ident!("{}", to_upper_snake_case(&node.name));200 let traits = node.traits.iter().map(|trait_name| {201 let trait_name = format_ident!("{}", trait_name);202 quote!(impl ast::#trait_name for #name {})203 });204205 let methods = node.fields.iter().map(|field| {206 let method_name = field.method_name(kinds);207 let ty = field.ty();208209 if field.is_many() {210 quote! {211 pub fn #method_name(&self) -> AstChildren<#ty> {212 support::children(&self.syntax)213 }214 }215 } else if let Some(token_kind) = field.token_kind(kinds) {216 quote! {217 pub fn #method_name(&self) -> Option<#ty> {218 support::token(&self.syntax, #token_kind)219 }220 }221 } else if field.is_token_enum(grammar) {222 quote! {223 pub fn #method_name(&self) -> Option<#ty> {224 support::token_child(&self.syntax)225 }226 }227 } else {228 quote! {229 pub fn #method_name(&self) -> Option<#ty> {230 support::child(&self.syntax)231 }232 }233 }234 });235 (236 quote! {237 #[pretty_doc_comment_placeholder_workaround]238 #[derive(Debug, Clone, PartialEq, Eq, Hash)]239 pub struct #name {240 pub(crate) syntax: SyntaxNode,241 }242243 #(#traits)*244245 impl #name {246 #(#methods)*247 }248 },249 quote! {250 impl AstNode for #name {251 fn can_cast(kind: SyntaxKind) -> bool {252 kind == #kind253 }254 fn cast(syntax: SyntaxNode) -> Option<Self> {255 if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }256 }257 fn syntax(&self) -> &SyntaxNode { &self.syntax }258 }259 },260 )261 })262 .unzip();263264 let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar265 .enums266 .iter()267 .map(|en| {268 let variants: Vec<_> = en269 .variants270 .iter()271 .map(|var| format_ident!("{}", var))272 .collect();273 let name = format_ident!("{}", en.name);274 let kinds: Vec<_> = variants275 .iter()276 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))277 .collect();278 let traits = en.traits.iter().map(|trait_name| {279 let trait_name = format_ident!("{}", trait_name);280 quote!(impl ast::#trait_name for #name {})281 });282283 let ast_node = quote! {284 impl AstNode for #name {285 fn can_cast(kind: SyntaxKind) -> bool {286 match kind {287 #(#kinds)|* => true,288 _ => false,289 }290 }291 fn cast(syntax: SyntaxNode) -> Option<Self> {292 let res = match syntax.kind() {293 #(294 #kinds => #name::#variants(#variants { syntax }),295 )*296 _ => return None,297 };298 Some(res)299 }300 fn syntax(&self) -> &SyntaxNode {301 match self {302 #(303 #name::#variants(it) => &it.syntax,304 )*305 }306 }307 }308 };309310 (311 quote! {312 #[pretty_doc_comment_placeholder_workaround]313 #[derive(Debug, Clone, PartialEq, Eq, Hash)]314 pub enum #name {315 #(#variants(#variants),)*316 }317318 #(#traits)*319 },320 quote! {321 #(322 impl From<#variants> for #name {323 fn from(node: #variants) -> #name {324 #name::#variants(node)325 }326 }327 )*328 #ast_node329 },330 )331 })332 .unzip();333334 let (token_enum_defs, token_enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar335 .token_enums336 .iter()337 .map(|en| {338 let variants: Vec<_> = en339 .variants340 .iter()341 .map(|token| {342 format_ident!(343 "{}",344 to_pascal_case(kinds.token(token).expect("token exists").name())345 )346 })347 .collect();348 let name = format_ident!("{}", en.name);349 let kind_name = format_ident!("{}Kind", en.name);350 let kinds: Vec<_> = variants351 .iter()352 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))353 .collect();354355 let ast_node = quote! {356 impl AstToken for #name {357 fn can_cast(kind: SyntaxKind) -> bool {358 #kind_name::can_cast(kind)359 }360 fn cast(syntax: SyntaxToken) -> Option<Self> {361 let kind = #kind_name::cast(syntax.kind())?;362 Some(#name { syntax, kind })363 }364 fn syntax(&self) -> &SyntaxToken {365 &self.syntax366 }367 }368369 impl #kind_name {370 fn can_cast(kind: SyntaxKind) -> bool {371 match kind {372 #(#kinds)|* => true,373 _ => false,374 }375 }376 pub fn cast(kind: SyntaxKind) -> Option<Self> {377 let res = match kind {378 #(#kinds => Self::#variants,)*379 _ => return None,380 };381 Some(res)382 }383 }384 };385386 (387 quote! {388 #[pretty_doc_comment_placeholder_workaround]389 #[derive(Debug, Clone, PartialEq, Eq, Hash)]390 pub struct #name { syntax: SyntaxToken, kind: #kind_name }391392 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]393 pub enum #kind_name {394 #(#variants,)*395 }396 },397 quote! {398 #ast_node399400 impl #name {401 pub fn kind(&self) -> #kind_name {402 self.kind403 }404 }405406 impl std::fmt::Display for #name {407 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {408 std::fmt::Display::fmt(self.syntax(), f)409 }410 }411 },412 )413 })414 .unzip();415416 let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar417 .nodes418 .iter()419 .flat_map(|node| node.traits.iter().map(move |t| (t, node)))420 .into_group_map()421 .into_iter()422 .sorted_by_key(|(k, _)| *k)423 .map(|(trait_name, nodes)| {424 let name = format_ident!("Any{}", trait_name);425 let trait_name = format_ident!("{}", trait_name);426 let kinds: Vec<_> = nodes427 .iter()428 .map(|name| format_ident!("{}", to_upper_snake_case(&name.name.to_string())))429 .collect();430431 (432 quote! {433 #[pretty_doc_comment_placeholder_workaround]434 #[derive(Debug, Clone, PartialEq, Eq, Hash)]435 pub struct #name {436 pub(crate) syntax: SyntaxNode,437 }438 impl ast::#trait_name for #name {}439 },440 quote! {441 impl #name {442 #[inline]443 pub fn new<T: ast::#trait_name>(node: T) -> #name {444 #name {445 syntax: node.syntax().clone()446 }447 }448 }449 impl AstNode for #name {450 fn can_cast(kind: SyntaxKind) -> bool {451 match kind {452 #(#kinds)|* => true,453 _ => false,454 }455 }456 fn cast(syntax: SyntaxNode) -> Option<Self> {457 Self::can_cast(syntax.kind()).then(|| #name { syntax })458 }459 fn syntax(&self) -> &SyntaxNode {460 &self.syntax461 }462 }463 },464 )465 })466 .unzip();467468 let enum_names = grammar.enums.iter().map(|it| &it.name);469 let node_names = grammar.nodes.iter().map(|it| &it.name);470471 let display_impls = enum_names472 .chain(node_names.clone())473 .map(|it| format_ident!("{}", it))474 .map(|name| {475 quote! {476 impl std::fmt::Display for #name {477 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {478 std::fmt::Display::fmt(self.syntax(), f)479 }480 }481 }482 });483484 let ast = quote! {485 #![allow(non_snake_case, clippy::match_like_matches_macro)]486487 use crate::{488 SyntaxNode, SyntaxToken, SyntaxKind::{self, *},489 ast::{AstNode, AstToken, AstChildren, support},490 T,491 };492493 #(#node_defs)*494 #(#enum_defs)*495 #(#token_enum_defs)*496 #(#any_node_defs)*497 #(#node_boilerplate_impls)*498 #(#enum_boilerplate_impls)*499 #(#token_enum_boilerplate_impls)*500 #(#any_node_boilerplate_impls)*501 #(#display_impls)*502 };503504 let ast = ast.to_string().replace("T ! [", "T![");505506 let mut res = String::with_capacity(ast.len() * 2);507508 let mut docs = grammar509 .nodes510 .iter()511 .map(|it| &it.doc)512 .chain(grammar.enums.iter().map(|it| &it.doc));513514 for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {515 res.push_str(chunk);516 if let Some(doc) = docs.next() {517 write_doc_comment(doc, &mut res);518 }519 }520521 let res = reformat(&res)?;522 Ok(res.replace("#[derive", "\n#[derive"))523}524525fn write_doc_comment(contents: &[String], dest: &mut String) {526 use std::fmt::Write;527 for line in contents {528 writeln!(dest, "///{line}").unwrap();529 }530}531532pub fn escape_token_macro(token: &str) -> TokenStream {533 if "{}[]()$".contains(token) {534 let c = token.chars().next().unwrap();535 quote! { #c }536 } else if token.contains(|v| v == '$') {537 quote! { #token }538 } else if token.chars().all(|v| ('a'..='z').contains(&v)) {539 let i = Ident::new(&token, Span::call_site());540 quote! { #i }541 } else {542 let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));543 quote! { #(#cs)* }544 }545}