1use std::path::PathBuf;23use anyhow::Result;4use ast::{lower, AstSrc};5use itertools::Itertools;6use kinds::{KindsSrc, TokenKind};7use proc_macro2::{Punct, Spacing, TokenStream};8use quote::{format_ident, quote};9use ungrammar::Grammar;10use util::{ensure_file_contents, reformat, to_pascal_case, to_upper_snake_case};1112mod ast;13mod kinds;14mod util;1516enum SpecialName {17 Literal,18 Meta,19 Error,20}21fn classify_special(name: &str) -> Option<(SpecialName, &str)> {22 let name = name.strip_suffix('!')?;23 Some(if let Some(name) = name.strip_prefix("LIT_") {24 (SpecialName::Literal, name)25 } else if let Some(name) = name.strip_prefix("META_") {26 (SpecialName::Meta, name)27 } else if let Some(name) = name.strip_prefix("ERROR_") {28 (SpecialName::Error, name)29 } else {30 return None;31 })32}3334pub fn generate_ungrammar() -> Result<()> {35 let grammar: Grammar = include_str!(concat!(36 env!("CARGO_MANIFEST_DIR"),37 "/../crates/jrsonnet-rowan-parser/jsonnet.ungram"38 ))39 .parse()?;4041 let mut kinds = kinds::jsonnet_kinds();42 let ast = lower(&kinds, &grammar);4344 for token in grammar.tokens() {45 let token = &grammar[token];46 let token = &token.name.clone();47 if !kinds.is_token(token) {48 if let Some((special, name)) = classify_special(token) {49 match special {50 SpecialName::Literal => panic!("literal is not defined: {name}"),51 SpecialName::Meta => kinds.define_token(TokenKind::Meta {52 grammar_name: token.to_owned(),53 name: format!("META_{}", name),54 }),55 SpecialName::Error => kinds.define_token(TokenKind::Error {56 grammar_name: token.to_owned(),57 name: format!("ERROR_{}", name),58 regex: None,59 priority: None,60 }),61 };62 continue;63 };64 let name = to_upper_snake_case(token);65 kinds.define_token(TokenKind::Keyword {66 code: token.to_owned(),67 name: format!("{name}_KW"),68 });69 }70 }71 for node in &ast.nodes {72 let name = to_upper_snake_case(&node.name);73 kinds.define_node(&name);74 }75 for enum_ in &ast.enums {76 let name = to_upper_snake_case(&enum_.name);77 kinds.define_node(&name);78 }79 for token_enum in &ast.token_enums {80 let name = to_upper_snake_case(&token_enum.name);81 kinds.define_node(&name);82 }8384 let syntax_kinds = generate_syntax_kinds(&kinds, &ast)?;8586 let nodes = generate_nodes(&kinds, &ast)?;87 ensure_file_contents(88 &PathBuf::from(concat!(89 env!("CARGO_MANIFEST_DIR"),90 "/../crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs",91 )),92 &syntax_kinds,93 )?;94 ensure_file_contents(95 &PathBuf::from(concat!(96 env!("CARGO_MANIFEST_DIR"),97 "/../crates/jrsonnet-rowan-parser/src/generated/nodes.rs",98 )),99 &nodes,100 )?;101 Ok(())102}103104fn generate_syntax_kinds(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {105 let t_macros = kinds.tokens().filter_map(TokenKind::expand_t_macros);106 let token_kinds = kinds.tokens().map(TokenKind::expand_kind);107108 let keywords = kinds109 .tokens()110 .filter(|k| matches!(k, TokenKind::Keyword { .. }))111 .map(TokenKind::name)112 .map(|n| format_ident!("{n}"));113114 let nodes = kinds115 .nodes116 .iter()117 .map(|name| format_ident!("{}", name))118 .collect::<Vec<_>>();119120 let enums = grammar121 .enums122 .iter()123 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name)))124 .chain(125 grammar126 .token_enums127 .iter()128 .map(|e| format_ident!("{}", to_upper_snake_case(&e.name))),129 );130131 let ast = quote! {132 #![allow(bad_style, missing_docs, unreachable_pub, clippy::manual_non_exhaustive, clippy::match_like_matches_macro)]133 use logos::Logos;134135 136 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Logos)]137 #[repr(u16)]138 pub enum SyntaxKind {139 #[doc(hidden)]140 TOMBSTONE,141 #[doc(hidden)]142 EOF,143 #(#token_kinds,)*144 #[error]145 ERROR,146 #(#nodes,)*147 #[doc(hidden)]148 __LAST,149 }150 use self::SyntaxKind::*;151152 impl SyntaxKind {153 pub fn is_keyword(self) -> bool {154 match self {155 #(#keywords)|* => true,156 _ => false,157 }158 }159 pub fn is_enum(self) -> bool {160 match self {161 #(#enums)|* => true,162 _ => false,163 }164 }165166 pub fn from_raw(r: u16) -> Self {167 assert!(r < Self::__LAST as u16);168 unsafe { std::mem::transmute(r) }169 }170 pub fn into_raw(self) -> u16 {171 self as u16172 }173 }174175 #[macro_export]176 macro_rules! T {#(#t_macros);*}177 pub use T;178 };179180 reformat(&ast.to_string())181}182183fn generate_nodes(kinds: &KindsSrc, grammar: &AstSrc) -> Result<String> {184 let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar185 .nodes186 .iter()187 .map(|node| {188 let name = format_ident!("{}", node.name);189 let kind = format_ident!("{}", to_upper_snake_case(&node.name));190 let traits = node.traits.iter().map(|trait_name| {191 let trait_name = format_ident!("{}", trait_name);192 quote!(impl ast::#trait_name for #name {})193 });194195 let methods = node.fields.iter().map(|field| {196 let method_name = field.method_name(kinds);197 let ty = field.ty();198199 if field.is_many() {200 quote! {201 pub fn #method_name(&self) -> AstChildren<#ty> {202 support::children(&self.syntax)203 }204 }205 } else if let Some(token_kind) = field.token_kind(kinds) {206 quote! {207 pub fn #method_name(&self) -> Option<#ty> {208 support::token(&self.syntax, #token_kind)209 }210 }211 } else if field.is_token_enum(grammar) {212 quote! {213 pub fn #method_name(&self) -> Option<#ty> {214 support::token_child(&self.syntax)215 }216 }217 } else {218 quote! {219 pub fn #method_name(&self) -> Option<#ty> {220 support::child(&self.syntax)221 }222 }223 }224 });225 (226 quote! {227 #[pretty_doc_comment_placeholder_workaround]228 #[derive(Debug, Clone, PartialEq, Eq, Hash)]229 pub struct #name {230 pub(crate) syntax: SyntaxNode,231 }232233 #(#traits)*234235 impl #name {236 #(#methods)*237 }238 },239 quote! {240 impl AstNode for #name {241 fn can_cast(kind: SyntaxKind) -> bool {242 kind == #kind243 }244 fn cast(syntax: SyntaxNode) -> Option<Self> {245 if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }246 }247 fn syntax(&self) -> &SyntaxNode { &self.syntax }248 }249 },250 )251 })252 .unzip();253254 let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar255 .enums256 .iter()257 .map(|en| {258 let variants: Vec<_> = en259 .variants260 .iter()261 .map(|var| format_ident!("{}", var))262 .collect();263 let name = format_ident!("{}", en.name);264 let kinds: Vec<_> = variants265 .iter()266 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))267 .collect();268 let traits = en.traits.iter().map(|trait_name| {269 let trait_name = format_ident!("{}", trait_name);270 quote!(impl ast::#trait_name for #name {})271 });272273 let ast_node = quote! {274 impl AstNode for #name {275 fn can_cast(kind: SyntaxKind) -> bool {276 match kind {277 #(#kinds)|* => true,278 _ => false,279 }280 }281 fn cast(syntax: SyntaxNode) -> Option<Self> {282 let res = match syntax.kind() {283 #(284 #kinds => #name::#variants(#variants { syntax }),285 )*286 _ => return None,287 };288 Some(res)289 }290 fn syntax(&self) -> &SyntaxNode {291 match self {292 #(293 #name::#variants(it) => &it.syntax,294 )*295 }296 }297 }298 };299300 (301 quote! {302 #[pretty_doc_comment_placeholder_workaround]303 #[derive(Debug, Clone, PartialEq, Eq, Hash)]304 pub enum #name {305 #(#variants(#variants),)*306 }307308 #(#traits)*309 },310 quote! {311 #(312 impl From<#variants> for #name {313 fn from(node: #variants) -> #name {314 #name::#variants(node)315 }316 }317 )*318 #ast_node319 },320 )321 })322 .unzip();323324 let (token_enum_defs, token_enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar325 .token_enums326 .iter()327 .map(|en| {328 let variants: Vec<_> = en329 .variants330 .iter()331 .map(|token| {332 format_ident!(333 "{}",334 to_pascal_case(kinds.token(token).expect("token exists").name())335 )336 })337 .collect();338 let name = format_ident!("{}", en.name);339 let kind_name = format_ident!("{}Kind", en.name);340 let kinds: Vec<_> = variants341 .iter()342 .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))343 .collect();344345 let ast_node = quote! {346 impl AstToken for #name {347 fn can_cast(kind: SyntaxKind) -> bool {348 match kind {349 #(#kinds)|* => true,350 _ => false,351 }352 }353 fn cast(syntax: SyntaxToken) -> Option<Self> {354 let res = match syntax.kind() {355 #(356 #kinds => #name { syntax, kind: #kind_name::#variants },357 )*358 _ => return None,359 };360 Some(res)361 }362 fn syntax(&self) -> &SyntaxToken {363 &self.syntax364 }365 }366 };367368 (369 quote! {370 #[pretty_doc_comment_placeholder_workaround]371 #[derive(Debug, Clone, PartialEq, Eq, Hash)]372 pub struct #name { syntax: SyntaxToken, kind: #kind_name }373374 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]375 pub enum #kind_name {376 #(#variants,)*377 }378 },379 quote! {380 #ast_node381382 impl #name {383 pub fn kind(&self) -> #kind_name {384 self.kind385 }386 }387388 impl std::fmt::Display for #name {389 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {390 std::fmt::Display::fmt(self.syntax(), f)391 }392 }393 },394 )395 })396 .unzip();397398 let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar399 .nodes400 .iter()401 .flat_map(|node| node.traits.iter().map(move |t| (t, node)))402 .into_group_map()403 .into_iter()404 .sorted_by_key(|(k, _)| *k)405 .map(|(trait_name, nodes)| {406 let name = format_ident!("Any{}", trait_name);407 let trait_name = format_ident!("{}", trait_name);408 let kinds: Vec<_> = nodes409 .iter()410 .map(|name| format_ident!("{}", to_upper_snake_case(&name.name.to_string())))411 .collect();412413 (414 quote! {415 #[pretty_doc_comment_placeholder_workaround]416 #[derive(Debug, Clone, PartialEq, Eq, Hash)]417 pub struct #name {418 pub(crate) syntax: SyntaxNode,419 }420 impl ast::#trait_name for #name {}421 },422 quote! {423 impl #name {424 #[inline]425 pub fn new<T: ast::#trait_name>(node: T) -> #name {426 #name {427 syntax: node.syntax().clone()428 }429 }430 }431 impl AstNode for #name {432 fn can_cast(kind: SyntaxKind) -> bool {433 match kind {434 #(#kinds)|* => true,435 _ => false,436 }437 }438 fn cast(syntax: SyntaxNode) -> Option<Self> {439 Self::can_cast(syntax.kind()).then(|| #name { syntax })440 }441 fn syntax(&self) -> &SyntaxNode {442 &self.syntax443 }444 }445 },446 )447 })448 .unzip();449450 let enum_names = grammar.enums.iter().map(|it| &it.name);451 let node_names = grammar.nodes.iter().map(|it| &it.name);452453 let display_impls = enum_names454 .chain(node_names.clone())455 .map(|it| format_ident!("{}", it))456 .map(|name| {457 quote! {458 impl std::fmt::Display for #name {459 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {460 std::fmt::Display::fmt(self.syntax(), f)461 }462 }463 }464 });465466 let ast = quote! {467 #![allow(non_snake_case, clippy::match_like_matches_macro)]468469 use crate::{470 SyntaxNode, SyntaxToken, SyntaxKind::{self, *},471 ast::{AstNode, AstToken, AstChildren, support},472 T,473 };474475 #(#node_defs)*476 #(#enum_defs)*477 #(#token_enum_defs)*478 #(#any_node_defs)*479 #(#node_boilerplate_impls)*480 #(#enum_boilerplate_impls)*481 #(#token_enum_boilerplate_impls)*482 #(#any_node_boilerplate_impls)*483 #(#display_impls)*484 };485486 let ast = ast.to_string().replace("T ! [", "T![");487488 let mut res = String::with_capacity(ast.len() * 2);489490 let mut docs = grammar491 .nodes492 .iter()493 .map(|it| &it.doc)494 .chain(grammar.enums.iter().map(|it| &it.doc));495496 for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {497 res.push_str(chunk);498 if let Some(doc) = docs.next() {499 write_doc_comment(doc, &mut res);500 }501 }502503 let res = reformat(&res)?;504 Ok(res.replace("#[derive", "\n#[derive"))505}506507fn write_doc_comment(contents: &[String], dest: &mut String) {508 use std::fmt::Write;509 for line in contents {510 writeln!(dest, "///{}", line).unwrap();511 }512}513514pub fn escape_token_macro(token: &str) -> TokenStream {515 if "{}[]()$".contains(token) {516 let c = token.chars().next().unwrap();517 quote! { #c }518 } else if token.contains('$') {519 quote! { #token }520 } else {521 let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));522 quote! { #(#cs)* }523 }524}