1use std::{collections::HashMap, path::PathBuf};23use anyhow::Result;4use ast::{lower, AstSrc};5use itertools::Itertools;6use kinds::{KindsSrc, TokenKind};7use proc_macro2::{Ident, Punct, Spacing, Span, TokenStream};8use quote::{format_ident, quote};9use ungrammar::Grammar;10use util::{ensure_file_contents, reformat, to_pascal_case, to_upper_snake_case};1112mod ast;13mod kinds;14mod util;1516enum SpecialName {17 Literal,18 Meta,19 Error,20}21fn classify_special(name: &str) -> Option<(SpecialName, &str)> {22 let name = name.strip_suffix('!')?;23 Some(if let Some(name) = name.strip_prefix("LIT_") {24 (SpecialName::Literal, name)25 } else if let Some(name) = name.strip_prefix("META_") {26 (SpecialName::Meta, name)27 } else if let Some(name) = name.strip_prefix("ERROR_") {28 (SpecialName::Error, name)29 } else {30 return None;31 })32}3334pub fn generate_ungrammar() -> Result<()> {35 let grammar: Grammar = include_str!(concat!(36 env!("CARGO_MANIFEST_DIR"),37 "/../crates/jrsonnet-rowan-parser/jsonnet.ungram"38 ))39 .parse()?;4041 let mut kinds = kinds::jsonnet_kinds();42 let ast = lower(&kinds, &grammar);4344 for token in grammar.tokens() {45 let token = &grammar[token];46 let token = &token.name.clone();47 if !kinds.is_token(token) {48 if let Some((special, name)) = classify_special(token) {49 match special {50 SpecialName::Literal => panic!("literal is not defined: {name}"),51 SpecialName::Meta => {52 eprintln!("implicit meta: {name}");53 kinds.define_token(TokenKind::Meta {54 grammar_name: token.to_owned(),55 name: format!("META_{name}"),56 });57 }58 SpecialName::Error => {59 eprintln!("implicit error: {name}");60 kinds.define_token(TokenKind::Error {61 grammar_name: token.to_owned(),62 name: format!("ERROR_{name}"),63 regex: None,64 priority: None,65 is_lexer_error: true,66 });67 }68 }69 continue;70 }71 let name = to_upper_snake_case(token);72 eprintln!("implicit kw: {token}");73 kinds.define_token(TokenKind::Keyword {74 code: token.to_owned(),75 name: format!("{name}_KW"),76 });77 }78 }79 for node in &ast.nodes {80 let name = to_upper_snake_case(&node.name);81 kinds.define_node(&name);82 }83 for enum_ in &ast.enums {84 let name = to_upper_snake_case(&enum_.name);85 kinds.define_node(&name);86 }87 for token_enum in &ast.token_enums {88 let name = to_upper_snake_case(&token_enum.name);89 kinds.define_node(&name);90 }9192 let syntax_kinds = generate_syntax_kinds(&kinds, &ast, false)?;9394 let nodes = generate_nodes(&kinds, &ast)?;95 ensure_file_contents(96 &PathBuf::from(concat!(97 env!("CARGO_MANIFEST_DIR"),98 "/../crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs",99 )),100 &syntax_kinds,101 );102 ensure_file_contents(103 &PathBuf::from(concat!(104 env!("CARGO_MANIFEST_DIR"),105 "/../crates/jrsonnet-rowan-parser/src/generated/nodes.rs",106 )),107 &nodes,108 );109110 let lexer_syntax_kinds = generate_syntax_kinds(&kinds, &ast, true)?;111 ensure_file_contents(112 &PathBuf::from(concat!(113 env!("CARGO_MANIFEST_DIR"),114 "/../crates/jrsonnet-lexer/src/generated/syntax_kinds.rs",115 )),116 &lexer_syntax_kinds,117 );118 Ok(())119}120121fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc, lexer: bool) -> Result<String> {122 let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);123 let token_kinds = kinds.tokens().map(|t| t.expand_kind(lexer));124125 let keywords = kinds126 .tokens()127 .filter(|k| matches!(k, TokenKind::Keyword { .. }))128 .map(TokenKind::name)129 .map(|n| format_ident!("{n}"));130131 let mut nodes = kinds132 .nodes133 .iter()134 .map(|name| format_ident!("{}", name))135 .collect::<Vec<_>>();136137 if lexer {138 nodes.clear();139 }140141 let enums = grammar142 .enums143 .iter()144 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name)))145 .chain(146 grammar147 .token_enums148 .iter()149 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),150 )151 .collect::<Vec<_>>();152 let is_enum = if lexer {153 quote! {}154 } else {155 quote! {156 pub fn is_enum(self) -> bool {157 match self {158 #(#enums)|* => true,159 _ => false,160 }161 }162 }163 };164165 let derive_logos = if lexer {166 quote! {167 , logos::Logos168 }169 } else {170 quote! {}171 };172173 let ast = quote! {174 #![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]175176 177 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug #derive_logos)]178 #[repr(u16)]179 pub enum SyntaxKind {180 #[doc(hidden)]181 TOMBSTONE,182 #[doc(hidden)]183 EOF,184 #(#token_kinds,)*185 LEXING_ERROR,186 __LAST_TOKEN,187 #(#nodes,)*188 #[doc(hidden)]189 __LAST,190 }191 use self::SyntaxKind::*;192193 impl SyntaxKind {194 pub fn is_keyword(self) -> bool {195 match self {196 #(#keywords)|* => true,197 _ => false,198 }199 }200201 #is_enum202203 pub fn from_raw(r: u16) -> Self {204 assert!(r < Self::__LAST as u16);205 unsafe { std::mem::transmute(r) }206 }207 pub fn into_raw(self) -> u16 {208 self as u16209 }210 }211212 #[macro_export]213 macro_rules! T {#(#t_macros);*}214 #[allow(unused_imports)]215 pub use T;216 };217218 reformat(&ast.to_string())219}220221#[allow(clippy::too_many_lines, clippy::cognitive_complexity)]222fn generate_nodes(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {223 let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar224 .nodes225 .iter()226 .map(|node| {227 let name = format_ident!("{}", node.name);228 let kind = format_ident!("{}", to_upper_snake_case(&node.name));229 let traits = node.traits.iter().map(|trait_name| {230 let trait_name = format_ident!("{}", trait_name);231 quote!(impl ast::#trait_name for #name {})232 });233234 let mut type_positions: HashMap<String, usize> = HashMap::new();235 let field_positions: Vec<_> = node236 .fields237 .iter()238 .map(|field| {239 let ty_str = field.ty().to_string();240 let pos = *type_positions.get(&ty_str).unwrap_or(&0);241 type_positions.insert(ty_str, pos + 1);242 pos243 })244 .collect();245246 let methods = node247 .fields248 .iter()249 .zip(field_positions.iter())250 .map(|(field, &pos)| {251 let method_name = field.method_name(kinds);252 let ty = field.ty();253254 if field.is_many() {255 quote! {256 pub fn #method_name(&self) -> AstChildren<#ty> {257 support::children(&self.syntax)258 }259 }260 } else if let Some(token_kind) = field.token_kind(kinds) {261 quote! {262 pub fn #method_name(&self) -> Option<#ty> {263 support::token(&self.syntax, #token_kind)264 }265 }266 } else if field.is_token_enum(grammar) {267 quote! {268 pub fn #method_name(&self) -> Option<#ty> {269 support::token_child(&self.syntax)270 }271 }272 } else if pos == 0 {273 quote! {274 pub fn #method_name(&self) -> Option<#ty> {275 support::children(&self.syntax).next()276 }277 }278 } else {279 quote! {280 pub fn #method_name(&self) -> Option<#ty> {281 support::children(&self.syntax).nth(#pos)282 }283 }284 }285 });286 (287 quote! {288 #[pretty_doc_comment_placeholder_workaround]289 #[derive(Debug, Clone, PartialEq, Eq, Hash)]290 pub struct #name {291 pub(crate) syntax: SyntaxNode,292 }293294 #(#traits)*295296 impl #name {297 #(#methods)*298 }299 },300 quote! {301 impl AstNode for #name {302 fn can_cast(kind: SyntaxKind) -> bool {303 kind == #kind304 }305 fn cast(syntax: SyntaxNode) -> Option<Self> {306 if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }307 }308 fn syntax(&self) -> &SyntaxNode { &self.syntax }309 }310 },311 )312 })313 .unzip();314315 let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar316 .enums317 .iter()318 .map(|en| {319 let variants: Vec<_> = en320 .variants321 .iter()322 .map(|var| format_ident!("{}", var))323 .collect();324 let name = format_ident!("{}", en.name);325 let kinds: Vec<_> = variants326 .iter()327 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))328 .collect();329 let traits = en.traits.iter().map(|trait_name| {330 let trait_name = format_ident!("{}", trait_name);331 quote!(impl ast::#trait_name for #name {})332 });333334 let ast_node = quote! {335 impl AstNode for #name {336 fn can_cast(kind: SyntaxKind) -> bool {337 match kind {338 #(#kinds)|* => true,339 _ => false,340 }341 }342 fn cast(syntax: SyntaxNode) -> Option<Self> {343 let res = match syntax.kind() {344 #(345 #kinds => #name::#variants(#variants { syntax }),346 )*347 _ => return None,348 };349 Some(res)350 }351 fn syntax(&self) -> &SyntaxNode {352 match self {353 #(354 #name::#variants(it) => &it.syntax,355 )*356 }357 }358 }359 };360361 (362 quote! {363 #[pretty_doc_comment_placeholder_workaround]364 #[derive(Debug, Clone, PartialEq, Eq, Hash)]365 pub enum #name {366 #(#variants(#variants),)*367 }368369 #(#traits)*370 },371 quote! {372 #(373 impl From<#variants> for #name {374 fn from(node: #variants) -> #name {375 #name::#variants(node)376 }377 }378 )*379 #ast_node380 },381 )382 })383 .unzip();384385 let (token_enum_defs, token_enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar386 .token_enums387 .iter()388 .map(|en| {389 let variants: Vec<_> = en390 .variants391 .iter()392 .map(|token| {393 format_ident!(394 "{}",395 to_pascal_case(kinds.token(token).expect("token exists").name())396 )397 })398 .collect();399 let name = format_ident!("{}", en.name);400 let kind_name = format_ident!("{}Kind", en.name);401 let kinds: Vec<_> = variants402 .iter()403 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))404 .collect();405406 let ast_node = quote! {407 impl AstToken for #name {408 fn can_cast(kind: SyntaxKind) -> bool {409 #kind_name::can_cast(kind)410 }411 fn cast(syntax: SyntaxToken) -> Option<Self> {412 let kind = #kind_name::cast(syntax.kind())?;413 Some(#name { syntax, kind })414 }415 fn syntax(&self) -> &SyntaxToken {416 &self.syntax417 }418 }419420 impl #kind_name {421 fn can_cast(kind: SyntaxKind) -> bool {422 match kind {423 #(#kinds)|* => true,424 _ => false,425 }426 }427 pub fn cast(kind: SyntaxKind) -> Option<Self> {428 let res = match kind {429 #(#kinds => Self::#variants,)*430 _ => return None,431 };432 Some(res)433 }434 }435 };436437 (438 quote! {439 #[pretty_doc_comment_placeholder_workaround]440 #[derive(Debug, Clone, PartialEq, Eq, Hash)]441 pub struct #name { syntax: SyntaxToken, kind: #kind_name }442443 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]444 pub enum #kind_name {445 #(#variants,)*446 }447 },448 quote! {449 #ast_node450451 impl #name {452 pub fn kind(&self) -> #kind_name {453 self.kind454 }455 }456457 impl std::fmt::Display for #name {458 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {459 std::fmt::Display::fmt(self.syntax(), f)460 }461 }462 },463 )464 })465 .unzip();466467 let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar468 .nodes469 .iter()470 .flat_map(|node| node.traits.iter().map(move |t| (t, node)))471 .into_group_map()472 .into_iter()473 .sorted_by_key(|(k, _)| *k)474 .map(|(trait_name, nodes)| {475 let name = format_ident!("Any{}", trait_name);476 let trait_name = format_ident!("{}", trait_name);477 let kinds: Vec<_> = nodes478 .iter()479 .map(|name| format_ident!("{}", to_upper_snake_case(&name.name)))480 .collect();481482 (483 quote! {484 #[pretty_doc_comment_placeholder_workaround]485 #[derive(Debug, Clone, PartialEq, Eq, Hash)]486 pub struct #name {487 pub(crate) syntax: SyntaxNode,488 }489 impl ast::#trait_name for #name {}490 },491 quote! {492 impl #name {493 #[inline]494 pub fn new<T: ast::#trait_name>(node: T) -> #name {495 #name {496 syntax: node.syntax().clone()497 }498 }499 }500 impl AstNode for #name {501 fn can_cast(kind: SyntaxKind) -> bool {502 match kind {503 #(#kinds)|* => true,504 _ => false,505 }506 }507 fn cast(syntax: SyntaxNode) -> Option<Self> {508 Self::can_cast(syntax.kind()).then(|| #name { syntax })509 }510 fn syntax(&self) -> &SyntaxNode {511 &self.syntax512 }513 }514 },515 )516 })517 .unzip();518519 let enum_names = grammar.enums.iter().map(|it| &it.name);520 let node_names = grammar.nodes.iter().map(|it| &it.name);521522 let display_impls = enum_names523 .chain(node_names.clone())524 .map(|it| format_ident!("{}", it))525 .map(|name| {526 quote! {527 impl std::fmt::Display for #name {528 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {529 std::fmt::Display::fmt(self.syntax(), f)530 }531 }532 }533 });534535 let ast = quote! {536 #![allow(non_snake_case, clippy::match_like_matches_macro)]537538 use crate::{539 SyntaxNode, SyntaxToken, SyntaxKind::{self, *},540 ast::{AstNode, AstToken, AstChildren, support},541 T,542 };543544 #(#node_defs)*545 #(#enum_defs)*546 #(#token_enum_defs)*547 #(#any_node_defs)*548 #(#node_boilerplate_impls)*549 #(#enum_boilerplate_impls)*550 #(#token_enum_boilerplate_impls)*551 #(#any_node_boilerplate_impls)*552 #(#display_impls)*553 };554555 let ast = ast.to_string().replace("T ! [", "T![");556557 let mut res = String::with_capacity(ast.len() * 2);558559 let mut docs = grammar560 .nodes561 .iter()562 .map(|it| &it.doc)563 .chain(grammar.enums.iter().map(|it| &it.doc));564565 for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {566 res.push_str(chunk);567 if let Some(doc) = docs.next() {568 write_doc_comment(doc, &mut res);569 }570 }571572 let res = reformat(&res)?;573 Ok(res.replace("#[derive", "\n#[derive"))574}575576fn write_doc_comment(contents: &[String], dest: &mut String) {577 use std::fmt::Write;578 for line in contents {579 writeln!(dest, "///{line}").unwrap();580 }581}582583pub fn escape_token_macro(token: &str) -> TokenStream {584 if "{}[]()$".contains(token) {585 let c = token.chars().next().unwrap();586 quote! { #c }587 } else if token.contains('$') {588 quote! { #token }589 } else if token.chars().all(|v: char| v.is_ascii_lowercase()) {590 let i = Ident::new(token, Span::call_site());591 quote! { #i }592 } else {593 let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));594 quote! { #(#cs)* }595 }596}