From 26a3b24cc9cfd491d6e220e696c0a93b04d169e5 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Wed, 14 Dec 2022 01:05:48 +0000 Subject: [PATCH] refactor(rowan-parser): remove intrinsic syntax --- --- a/cmds/jrsonnet-fmt/src/children.rs +++ b/cmds/jrsonnet-fmt/src/children.rs @@ -3,13 +3,11 @@ use std::{fmt::Debug, mem}; use jrsonnet_rowan_parser::{ - nodes::{Trivia, TriviaKind}, - AstNode, AstToken, SyntaxElement, - SyntaxKind::*, - SyntaxNode, TS, + nodes::{CustomError, Trivia, TriviaKind}, + AstNode, AstToken, SyntaxElement, SyntaxNode, TS, }; -pub type ChildTrivia = Vec; +pub type ChildTrivia = Vec>; /// Node should have no non-trivia tokens before element pub fn trivia_before(node: SyntaxNode, end: Option<&SyntaxElement>) -> ChildTrivia { @@ -20,12 +18,14 @@ } if let Some(trivia) = item.as_token().cloned().and_then(Trivia::cast) { - out.push(trivia); + out.push(Ok(trivia)); + } else if CustomError::can_cast(item.kind()) { + out.push(Err(item.to_string())); } else if end.is_none() { break; } else { assert!( - TS![, ;].contains(item.kind()) || item.kind() == ERROR, + TS![, ;].contains(item.kind()), "silently eaten token: {:?}", item.kind() ) @@ -46,10 +46,12 @@ let mut out = Vec::new(); for item in iter { if let Some(trivia) = item.as_token().cloned().and_then(Trivia::cast) { - out.push(trivia); + out.push(Ok(trivia)); + } else if CustomError::can_cast(item.kind()) { + out.push(Err(item.to_string())) } else { assert!( - TS![, ;].contains(item.kind()) || item.kind() == ERROR, + TS![, ;].contains(item.kind()), "silently eaten token: {:?}", item.kind() ) @@ -74,12 +76,14 @@ let mut out = Vec::new(); for item in iter.take_while(|i| Some(i) != end) { if let Some(trivia) = item.as_token().cloned().and_then(Trivia::cast) { - out.push(trivia); + out.push(Ok(trivia)); + } else if CustomError::can_cast(item.kind()) { + out.push(Err(item.to_string())) } else if loose { break; } else { assert!( - TS![, ;].contains(item.kind()) || item.kind() == ERROR, + TS![, ;].contains(item.kind()), "silently eaten token: {:?}", item.kind() ) @@ -120,11 +124,16 @@ fn count_newlines_before(tt: &ChildTrivia) -> usize { let mut nl_count = 0; for t in tt { - match t.kind() { - TriviaKind::Whitespace => { - nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); + match t { + Ok(t) => match t.kind() { + TriviaKind::Whitespace => { + nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); + } + _ => break, + }, + Err(_) => { + nl_count += 1; } - _ => break, } } nl_count @@ -132,19 +141,22 @@ fn count_newlines_after(tt: &ChildTrivia) -> usize { let mut nl_count = 0; for t in tt.iter().rev() { - match t.kind() { - TriviaKind::Whitespace => { - nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); - } - TriviaKind::SingleLineHashComment => { - nl_count += 1; - break; - } - TriviaKind::SingleLineSlashComment => { - nl_count += 1; - break; - } - _ => {} + match t { + Ok(t) => match t.kind() { + TriviaKind::Whitespace => { + nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); + } + TriviaKind::SingleLineHashComment => { + nl_count += 1; + break; + } + TriviaKind::SingleLineSlashComment => { + nl_count += 1; + break; + } + _ => {} + }, + Err(_) => nl_count += 1, } } nl_count @@ -187,16 +199,18 @@ || current_child.is_none() || trivia.text().contains('\n') && !is_single_line_comment { - next.push(trivia.clone()); + next.push(Ok(trivia.clone())); started_next = true; } else { let cur = current_child.as_mut().expect("checked not none"); - cur.inline_trivia.push(trivia); + cur.inline_trivia.push(Ok(trivia)); if is_single_line_comment { started_next = true; } } had_some = true; + } else if CustomError::can_cast(item.kind()) { + next.push(Err(item.to_string())) } else if loose { if had_some { break; @@ -204,7 +218,7 @@ started_next = true; } else { assert!( - TS![, ;].contains(item.kind()) || item.kind() == ERROR, + TS![, ;].contains(item.kind()), "silently eaten token: {:?}", item.kind() ) --- a/cmds/jrsonnet-fmt/src/comments.rs +++ b/cmds/jrsonnet-fmt/src/comments.rs @@ -17,6 +17,24 @@ let mut pi = p!(new:); for c in comments { + let Ok(c) = c else { + let mut text = c.as_ref().unwrap_err() as &str; + while !text.is_empty() { + let pos = text.find(|c| c == '\n' || c == '\t').unwrap_or(text.len()); + let sliced = &text[..pos]; + p!(pi: string(sliced.to_string())); + text = &text[pos..]; + if! text.is_empty(){ + match text.as_bytes()[0] { + b'\n' => p!(pi: nl), + b'\t' => p!(pi: tab), + _ => unreachable!() + } + text = &text[1..]; + } + } + continue; + }; match c.kind() { TriviaKind::Whitespace => {} TriviaKind::MultiLineComment => { @@ -37,7 +55,7 @@ let mut immediate_start = true; let mut lines = text .split('\n') - .map(|l| l.trim_end()) + .map(|l| l.trim_end().to_string()) .skip_while(|l| { if l.is_empty() { immediate_start = false; @@ -51,7 +69,7 @@ lines.pop(); } if lines.len() == 1 && !doc { - p!(pi: str("/* ") str(lines[0].trim()) str(" */") nl) + p!(pi: str("/* ") string(lines[0].trim().to_string()) str(" */") nl) } else if !lines.is_empty() { fn common_ws_prefix<'a>(a: &'a str, b: &str) -> &'a str { let offset = a @@ -62,17 +80,18 @@ &a[..offset] } // First line is not empty, extract ws prefix of it - let mut common_ws_padding = if immediate_start && lines.len() > 1 { - common_ws_prefix(lines[1], lines[1]) + let mut common_ws_padding = (if immediate_start && lines.len() > 1 { + common_ws_prefix(&lines[1], &lines[1]) } else { - common_ws_prefix(lines[0], lines[0]) - }; + common_ws_prefix(&lines[0], &lines[0]) + }) + .to_string(); for line in lines .iter() .skip(if immediate_start { 2 } else { 1 }) .filter(|l| !l.is_empty()) { - common_ws_padding = common_ws_prefix(common_ws_padding, line); + common_ws_padding = common_ws_prefix(&common_ws_padding, line).to_string(); } for line in lines .iter_mut() @@ -80,8 +99,9 @@ .filter(|l| !l.is_empty()) { *line = line - .strip_prefix(common_ws_padding) - .expect("all non-empty lines start with this padding"); + .strip_prefix(&common_ws_padding) + .expect("all non-empty lines start with this padding") + .to_string(); } p!(pi: str("/*")); @@ -105,9 +125,9 @@ } else { p!(pi: tab); } - line = new_line; + line = new_line.to_string(); } - p!(pi: str(line) nl) + p!(pi: string(line.to_string()) nl) } } if doc { @@ -136,7 +156,7 @@ if matches!(loc, CommentLocation::ItemInline) { p!(pi: str(" ")) } - p!(pi: str("# ") str(c.text().strip_prefix('#').expect("hash comment starts with #").trim())); + p!(pi: str("# ") string(c.text().strip_prefix('#').expect("hash comment starts with #").trim().to_string())); if !matches!(loc, CommentLocation::ItemInline) { p!(pi: nl) } @@ -145,14 +165,14 @@ if matches!(loc, CommentLocation::ItemInline) { p!(pi: str(" ")) } - p!(pi: str("// ") str(c.text().strip_prefix("//").expect("comment starts with //").trim())); + p!(pi: str("// ") string(c.text().strip_prefix("//").expect("comment starts with //").trim().to_string())); if !matches!(loc, CommentLocation::ItemInline) { p!(pi: nl) } } // Garbage in - garbage out TriviaKind::ErrorCommentTooShort => p!(pi: str("/*/")), - TriviaKind::ErrorCommentUnterminated => p!(pi: str(c.text())), + TriviaKind::ErrorCommentUnterminated => p!(pi: string(c.text().to_string())), } } --- a/cmds/jrsonnet-fmt/src/main.rs +++ b/cmds/jrsonnet-fmt/src/main.rs @@ -5,10 +5,11 @@ use jrsonnet_rowan_parser::{ nodes::{ ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart, - DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal, - Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text, - UnaryOperator, + DestructRest, Expr, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal, Member, Name, + Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text, UnaryOperator, + Visibility, VisibilityKind, }, + rowan::NodeOrToken, AstNode, AstToken, SyntaxToken, }; @@ -37,6 +38,10 @@ $o.push_str($e); pi!(@s; $o: $($t)*); }}; + (@s; $o:ident: string($e:expr $(,)?) $($t:tt)*) => {{ + $o.push_string($e); + pi!(@s; $o: $($t)*); + }}; (@s; $o:ident: nl $($t:tt)*) => {{ $o.push_signal(dprint_core::formatting::Signal::NewLine); pi!(@s; $o: $($t)*); @@ -96,8 +101,8 @@ if let Some(v) = self { v.print() } else { - p!(new: str( - &format!( + p!(new: string( + format!( "/*missing {}*/", type_name::

().replace("jrsonnet_rowan_parser::generated::nodes::", "") ), @@ -108,18 +113,18 @@ impl Printable for SyntaxToken { fn print(&self) -> PrintItems { - p!(new: str(&self.to_string())) + p!(new: string(self.to_string())) } } impl Printable for Text { fn print(&self) -> PrintItems { - p!(new: str(&format!("{}", self))) + p!(new: string(format!("{}", self))) } } impl Printable for Number { fn print(&self) -> PrintItems { - p!(new: str(&format!("{}", self))) + p!(new: string(format!("{}", self))) } } @@ -201,22 +206,10 @@ } } } -impl Printable for Field { + +impl Printable for Visibility { fn print(&self) -> PrintItems { - let mut pi = p!(new:); - match self { - Field::FieldNormal(n) => { - p!(pi: {n.field_name()}); - if n.plus_token().is_some() { - p!(pi: str("+")); - } - p!(pi: str(": ") {n.expr()}); - } - Field::FieldMethod(m) => { - p!(pi: {m.field_name()} {m.params_desc()} str(": ") {m.expr()}); - } - } - pi + p!(new: string(self.to_string())) } } @@ -282,10 +275,82 @@ } } +impl Printable for Member { + fn print(&self) -> PrintItems { + match self { + Member::MemberBindStmt(b) => { + p!(new: {b.obj_local()}) + } + Member::MemberAssertStmt(ass) => { + p!(new: {ass.assertion()}) + } + Member::MemberFieldNormal(n) => { + p!(new: {n.field_name()} if(n.plus_token().is_some())({n.plus_token()}) {n.visibility()} str(" ") {n.expr()}) + } + Member::MemberFieldMethod(_) => todo!(), + } + } +} + impl Printable for ObjBody { fn print(&self) -> PrintItems { match self { - ObjBody::ObjBodyComp(_) => todo!(), + ObjBody::ObjBodyComp(l) => { + let mut pi = p!(new: str("{") >i nl); + let (children, end_comments) = children_between::( + l.syntax().clone(), + l.l_brace_token().map(Into::into).as_ref(), + Some( + &(l.comp_specs() + .next() + .expect("at least one spec is defined") + .syntax() + .clone()) + .into(), + ), + ); + for mem in children.into_iter() { + if mem.should_start_with_newline { + p!(pi: nl); + } + p!(pi: items(format_comments(&mem.before_trivia, CommentLocation::AboveItem))); + p!(pi: {mem.value} str(",")); + p!(pi: items(format_comments(&mem.inline_trivia, CommentLocation::ItemInline))); + p!(pi: nl) + } + + if end_comments.should_start_with_newline { + p!(pi: nl); + } + p!(pi: items(format_comments(&end_comments.trivia, CommentLocation::EndOfItems))); + + let (compspecs, end_comments) = children_between::( + l.syntax().clone(), + l.member_comps() + .last() + .map(|m| m.syntax().clone()) + .map(Into::into) + .or_else(|| l.l_brace_token().map(Into::into)) + .as_ref(), + l.r_brace_token().map(Into::into).as_ref(), + ); + for mem in compspecs.into_iter() { + if mem.should_start_with_newline { + p!(pi: nl); + } + p!(pi: items(format_comments(&mem.before_trivia, CommentLocation::AboveItem))); + p!(pi: {mem.value}); + p!(pi: items(format_comments(&mem.inline_trivia, CommentLocation::ItemInline))); + p!(pi: nl) + } + if end_comments.should_start_with_newline { + p!(pi: nl); + } + p!(pi: items(format_comments(&end_comments.trivia, CommentLocation::EndOfItems))); + + p!(pi: { let mut pi = p!(new: str("{") >i nl); let (children, end_comments) = children_between::( @@ -298,18 +363,7 @@ p!(pi: nl); } p!(pi: items(format_comments(&mem.before_trivia, CommentLocation::AboveItem))); - match mem.value { - Member::MemberBindStmt(b) => { - p!(pi: {b.obj_local()}) - } - Member::MemberAssertStmt(ass) => { - p!(pi: {ass.assertion()}) - } - Member::MemberField(f) => { - p!(pi: {f.field()}) - } - } - p!(pi: str(",")); + p!(pi: {mem.value} str(",")); p!(pi: items(format_comments(&mem.inline_trivia, CommentLocation::ItemInline))); p!(pi: nl) } @@ -326,12 +380,12 @@ } impl Printable for UnaryOperator { fn print(&self) -> PrintItems { - p!(new: str(self.text())) + p!(new: string(self.text().to_string())) } } impl Printable for BinaryOperator { fn print(&self) -> PrintItems { - p!(new: str(self.text())) + p!(new: string(self.text().to_string())) } } impl Printable for Bind { @@ -348,12 +402,12 @@ } impl Printable for Literal { fn print(&self) -> PrintItems { - p!(new: str(&self.syntax().to_string())) + p!(new: string(self.syntax().to_string())) } } impl Printable for ImportKind { fn print(&self) -> PrintItems { - p!(new: str(&self.syntax().to_string())) + p!(new: string(self.syntax().to_string())) } } impl Printable for LhsExpr { @@ -406,9 +460,6 @@ Expr::ExprParened(p) => { p!(new: str("(") {p.expr()} str(")")) } - Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")), - Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")), - Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")), Expr::ExprString(s) => p!(new: {s.text()}), Expr::ExprNumber(n) => p!(new: {n.number()}), Expr::ExprArray(a) => { @@ -543,10 +594,6 @@ local ocomp = {[k]: 1 for k in v}; local ? = skip; - - local intr = $intrinsic(test); - local intrId = $intrinsicId; - local intrThisFile = $intrinsicThisFile; local ie = a[expr]; @@ -643,7 +690,13 @@ 2 - else Template {} + else Template {}, + + compspecs: { + obj_with_no_item: {for i in [1, 2, 3]}, + obj_with_2_items: {a:1, b:2, for i in [1,2,3]}, + } + } + Template --- a/crates/jrsonnet-rowan-parser/jsonnet.ungram +++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram @@ -38,15 +38,6 @@ ExprLiteral = Literal -ExprIntrinsicThisFile = - '$intrinsicThisFile' -ExprIntrinsicId = - '$intrinsicId' -ExprIntrinsic = - '$intrinsic' - '(' - name:Name - ')' ExprString = Text ExprNumber = @@ -110,9 +101,6 @@ | ExprApply | ExprObjExtend | ExprParened -| ExprIntrinsicThisFile -| ExprIntrinsicId -| ExprIntrinsic | ExprString | ExprNumber | ExprLiteral @@ -167,14 +155,7 @@ ObjBodyComp = '{' - pre:ObjLocalPostComma* - '[' - key:LhsExpr - ']' - '+'? - ':' - value:Expr - post:ObjLocalPreComma* + (MemberComp (',' MemberComp)* ','?)? CompSpec* '}' ObjBodyMemberList = @@ -185,13 +166,6 @@ ObjBodyComp | ObjBodyMemberList -ObjLocalPostComma = - ObjLocal - ',' -ObjLocalPreComma = - ',' - ObjLocal - MemberBindStmt = ObjLocal MemberAssertStmt = Assertion MemberFieldNormal = @@ -204,6 +178,10 @@ ParamsDesc Visibility Expr +MemberComp = + MemberBindStmt +| MemberFieldNormal +| MemberFieldMethod Member = MemberBindStmt | MemberAssertStmt @@ -367,7 +345,7 @@ | 'LIT_SINGLE_LINE_HASH_COMMENT!' | 'LIT_SINGLE_LINE_SLASH_COMMENT!' -ParsingError = +CustomError = 'ERROR_MISSING_TOKEN!' | 'ERROR_UNEXPECTED_TOKEN!' | 'ERROR_CUSTOM!' --- a/crates/jrsonnet-rowan-parser/src/generated/nodes.rs +++ b/crates/jrsonnet-rowan-parser/src/generated/nodes.rs @@ -212,45 +212,6 @@ } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ExprIntrinsicThisFile { - pub(crate) syntax: SyntaxNode, -} -impl ExprIntrinsicThisFile { - pub fn intrinsic_this_file_token(&self) -> Option { - support::token(&self.syntax, T!["$intrinsicThisFile"]) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ExprIntrinsicId { - pub(crate) syntax: SyntaxNode, -} -impl ExprIntrinsicId { - pub fn intrinsic_id_token(&self) -> Option { - support::token(&self.syntax, T!["$intrinsicId"]) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ExprIntrinsic { - pub(crate) syntax: SyntaxNode, -} -impl ExprIntrinsic { - pub fn intrinsic_token(&self) -> Option { - support::token(&self.syntax, T!["$intrinsic"]) - } - pub fn l_paren_token(&self) -> Option { - support::token(&self.syntax, T!['(']) - } - pub fn name(&self) -> Option { - support::child(&self.syntax) - } - pub fn r_paren_token(&self) -> Option { - support::token(&self.syntax, T![')']) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ExprString { pub(crate) syntax: SyntaxNode, } @@ -535,28 +496,7 @@ pub fn l_brace_token(&self) -> Option { support::token(&self.syntax, T!['{']) } - pub fn pre(&self) -> AstChildren { - support::children(&self.syntax) - } - pub fn l_brack_token(&self) -> Option { - support::token(&self.syntax, T!['[']) - } - pub fn key(&self) -> Option { - support::child(&self.syntax) - } - pub fn r_brack_token(&self) -> Option { - support::token(&self.syntax, T![']']) - } - pub fn plus_token(&self) -> Option { - support::token(&self.syntax, T![+]) - } - pub fn colon_token(&self) -> Option { - support::token(&self.syntax, T![:]) - } - pub fn value(&self) -> Option { - support::child(&self.syntax) - } - pub fn post(&self) -> AstChildren { + pub fn member_comps(&self) -> AstChildren { support::children(&self.syntax) } pub fn comp_specs(&self) -> AstChildren { @@ -568,32 +508,6 @@ } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ObjLocalPostComma { - pub(crate) syntax: SyntaxNode, -} -impl ObjLocalPostComma { - pub fn obj_local(&self) -> Option { - support::child(&self.syntax) - } - pub fn comma_token(&self) -> Option { - support::token(&self.syntax, T![,]) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ObjLocalPreComma { - pub(crate) syntax: SyntaxNode, -} -impl ObjLocalPreComma { - pub fn comma_token(&self) -> Option { - support::token(&self.syntax, T![,]) - } - pub fn obj_local(&self) -> Option { - support::child(&self.syntax) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ObjBodyMemberList { pub(crate) syntax: SyntaxNode, } @@ -610,24 +524,24 @@ } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ObjLocal { +pub struct MemberBindStmt { pub(crate) syntax: SyntaxNode, } -impl ObjLocal { - pub fn local_kw_token(&self) -> Option { - support::token(&self.syntax, T![local]) - } - pub fn bind(&self) -> Option { +impl MemberBindStmt { + pub fn obj_local(&self) -> Option { support::child(&self.syntax) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct MemberBindStmt { +pub struct ObjLocal { pub(crate) syntax: SyntaxNode, } -impl MemberBindStmt { - pub fn obj_local(&self) -> Option { +impl ObjLocal { + pub fn local_kw_token(&self) -> Option { + support::token(&self.syntax, T![local]) + } + pub fn bind(&self) -> Option { support::child(&self.syntax) } } @@ -905,9 +819,6 @@ ExprApply(ExprApply), ExprObjExtend(ExprObjExtend), ExprParened(ExprParened), - ExprIntrinsicThisFile(ExprIntrinsicThisFile), - ExprIntrinsicId(ExprIntrinsicId), - ExprIntrinsic(ExprIntrinsic), ExprString(ExprString), ExprNumber(ExprNumber), ExprLiteral(ExprLiteral), @@ -942,6 +853,13 @@ } #[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum MemberComp { + MemberBindStmt(MemberBindStmt), + MemberFieldNormal(MemberFieldNormal), + MemberFieldMethod(MemberFieldMethod), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Member { MemberBindStmt(MemberBindStmt), MemberAssertStmt(MemberAssertStmt), @@ -1110,13 +1028,13 @@ } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ParsingError { +pub struct CustomError { syntax: SyntaxToken, - kind: ParsingErrorKind, + kind: CustomErrorKind, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ParsingErrorKind { +pub enum CustomErrorKind { ErrorMissingToken, ErrorUnexpectedToken, ErrorCustom, @@ -1319,36 +1237,6 @@ impl AstNode for ExprLiteral { fn can_cast(kind: SyntaxKind) -> bool { kind == EXPR_LITERAL - } - fn cast(syntax: SyntaxNode) -> Option { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { - &self.syntax - } -} -impl AstNode for ExprIntrinsicThisFile { - fn can_cast(kind: SyntaxKind) -> bool { - kind == EXPR_INTRINSIC_THIS_FILE - } - fn cast(syntax: SyntaxNode) -> Option { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { - &self.syntax - } -} -impl AstNode for ExprIntrinsicId { - fn can_cast(kind: SyntaxKind) -> bool { - kind == EXPR_INTRINSIC_ID } fn cast(syntax: SyntaxNode) -> Option { if Self::can_cast(syntax.kind()) { @@ -1361,21 +1249,6 @@ &self.syntax } } -impl AstNode for ExprIntrinsic { - fn can_cast(kind: SyntaxKind) -> bool { - kind == EXPR_INTRINSIC - } - fn cast(syntax: SyntaxNode) -> Option { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { - &self.syntax - } -} impl AstNode for ExprString { fn can_cast(kind: SyntaxKind) -> bool { kind == EXPR_STRING @@ -1676,36 +1549,6 @@ &self.syntax } } -impl AstNode for ObjLocalPostComma { - fn can_cast(kind: SyntaxKind) -> bool { - kind == OBJ_LOCAL_POST_COMMA - } - fn cast(syntax: SyntaxNode) -> Option { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { - &self.syntax - } -} -impl AstNode for ObjLocalPreComma { - fn can_cast(kind: SyntaxKind) -> bool { - kind == OBJ_LOCAL_PRE_COMMA - } - fn cast(syntax: SyntaxNode) -> Option { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { - &self.syntax - } -} impl AstNode for ObjBodyMemberList { fn can_cast(kind: SyntaxKind) -> bool { kind == OBJ_BODY_MEMBER_LIST @@ -1721,9 +1564,9 @@ &self.syntax } } -impl AstNode for ObjLocal { +impl AstNode for MemberBindStmt { fn can_cast(kind: SyntaxKind) -> bool { - kind == OBJ_LOCAL + kind == MEMBER_BIND_STMT } fn cast(syntax: SyntaxNode) -> Option { if Self::can_cast(syntax.kind()) { @@ -1736,9 +1579,9 @@ &self.syntax } } -impl AstNode for MemberBindStmt { +impl AstNode for ObjLocal { fn can_cast(kind: SyntaxKind) -> bool { - kind == MEMBER_BIND_STMT + kind == OBJ_LOCAL } fn cast(syntax: SyntaxNode) -> Option { if Self::can_cast(syntax.kind()) { @@ -2044,21 +1887,6 @@ impl From for Expr { fn from(node: ExprParened) -> Expr { Expr::ExprParened(node) - } -} -impl From for Expr { - fn from(node: ExprIntrinsicThisFile) -> Expr { - Expr::ExprIntrinsicThisFile(node) - } -} -impl From for Expr { - fn from(node: ExprIntrinsicId) -> Expr { - Expr::ExprIntrinsicId(node) - } -} -impl From for Expr { - fn from(node: ExprIntrinsic) -> Expr { - Expr::ExprIntrinsic(node) } } impl From for Expr { @@ -2129,30 +1957,10 @@ impl AstNode for Expr { fn can_cast(kind: SyntaxKind) -> bool { match kind { - EXPR_BINARY - | EXPR_UNARY - | EXPR_SLICE - | EXPR_INDEX - | EXPR_INDEX_EXPR - | EXPR_APPLY - | EXPR_OBJ_EXTEND - | EXPR_PARENED - | EXPR_INTRINSIC_THIS_FILE - | EXPR_INTRINSIC_ID - | EXPR_INTRINSIC - | EXPR_STRING - | EXPR_NUMBER - | EXPR_LITERAL - | EXPR_ARRAY - | EXPR_OBJECT - | EXPR_ARRAY_COMP - | EXPR_IMPORT - | EXPR_VAR - | EXPR_LOCAL - | EXPR_IF_THEN_ELSE - | EXPR_FUNCTION - | EXPR_ASSERT - | EXPR_ERROR => true, + EXPR_BINARY | EXPR_UNARY | EXPR_SLICE | EXPR_INDEX | EXPR_INDEX_EXPR | EXPR_APPLY + | EXPR_OBJ_EXTEND | EXPR_PARENED | EXPR_STRING | EXPR_NUMBER | EXPR_LITERAL + | EXPR_ARRAY | EXPR_OBJECT | EXPR_ARRAY_COMP | EXPR_IMPORT | EXPR_VAR | EXPR_LOCAL + | EXPR_IF_THEN_ELSE | EXPR_FUNCTION | EXPR_ASSERT | EXPR_ERROR => true, _ => false, } } @@ -2166,11 +1974,6 @@ EXPR_APPLY => Expr::ExprApply(ExprApply { syntax }), EXPR_OBJ_EXTEND => Expr::ExprObjExtend(ExprObjExtend { syntax }), EXPR_PARENED => Expr::ExprParened(ExprParened { syntax }), - EXPR_INTRINSIC_THIS_FILE => { - Expr::ExprIntrinsicThisFile(ExprIntrinsicThisFile { syntax }) - } - EXPR_INTRINSIC_ID => Expr::ExprIntrinsicId(ExprIntrinsicId { syntax }), - EXPR_INTRINSIC => Expr::ExprIntrinsic(ExprIntrinsic { syntax }), EXPR_STRING => Expr::ExprString(ExprString { syntax }), EXPR_NUMBER => Expr::ExprNumber(ExprNumber { syntax }), EXPR_LITERAL => Expr::ExprLiteral(ExprLiteral { syntax }), @@ -2198,9 +2001,6 @@ Expr::ExprApply(it) => &it.syntax, Expr::ExprObjExtend(it) => &it.syntax, Expr::ExprParened(it) => &it.syntax, - Expr::ExprIntrinsicThisFile(it) => &it.syntax, - Expr::ExprIntrinsicId(it) => &it.syntax, - Expr::ExprIntrinsic(it) => &it.syntax, Expr::ExprString(it) => &it.syntax, Expr::ExprNumber(it) => &it.syntax, Expr::ExprLiteral(it) => &it.syntax, @@ -2313,6 +2113,45 @@ } } } +impl From for MemberComp { + fn from(node: MemberBindStmt) -> MemberComp { + MemberComp::MemberBindStmt(node) + } +} +impl From for MemberComp { + fn from(node: MemberFieldNormal) -> MemberComp { + MemberComp::MemberFieldNormal(node) + } +} +impl From for MemberComp { + fn from(node: MemberFieldMethod) -> MemberComp { + MemberComp::MemberFieldMethod(node) + } +} +impl AstNode for MemberComp { + fn can_cast(kind: SyntaxKind) -> bool { + match kind { + MEMBER_BIND_STMT | MEMBER_FIELD_NORMAL | MEMBER_FIELD_METHOD => true, + _ => false, + } + } + fn cast(syntax: SyntaxNode) -> Option { + let res = match syntax.kind() { + MEMBER_BIND_STMT => MemberComp::MemberBindStmt(MemberBindStmt { syntax }), + MEMBER_FIELD_NORMAL => MemberComp::MemberFieldNormal(MemberFieldNormal { syntax }), + MEMBER_FIELD_METHOD => MemberComp::MemberFieldMethod(MemberFieldMethod { syntax }), + _ => return None, + }; + Some(res) + } + fn syntax(&self) -> &SyntaxNode { + match self { + MemberComp::MemberBindStmt(it) => &it.syntax, + MemberComp::MemberFieldNormal(it) => &it.syntax, + MemberComp::MemberFieldMethod(it) => &it.syntax, + } + } +} impl From for Member { fn from(node: MemberBindStmt) -> Member { Member::MemberBindStmt(node) @@ -2847,19 +2686,19 @@ std::fmt::Display::fmt(self.syntax(), f) } } -impl AstToken for ParsingError { +impl AstToken for CustomError { fn can_cast(kind: SyntaxKind) -> bool { - ParsingErrorKind::can_cast(kind) + CustomErrorKind::can_cast(kind) } fn cast(syntax: SyntaxToken) -> Option { - let kind = ParsingErrorKind::cast(syntax.kind())?; - Some(ParsingError { syntax, kind }) + let kind = CustomErrorKind::cast(syntax.kind())?; + Some(CustomError { syntax, kind }) } fn syntax(&self) -> &SyntaxToken { &self.syntax } } -impl ParsingErrorKind { +impl CustomErrorKind { fn can_cast(kind: SyntaxKind) -> bool { match kind { ERROR_MISSING_TOKEN | ERROR_UNEXPECTED_TOKEN | ERROR_CUSTOM => true, @@ -2876,12 +2715,12 @@ Some(res) } } -impl ParsingError { - pub fn kind(&self) -> ParsingErrorKind { +impl CustomError { + pub fn kind(&self) -> CustomErrorKind { self.kind } } -impl std::fmt::Display for ParsingError { +impl std::fmt::Display for CustomError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) } @@ -2906,6 +2745,11 @@ std::fmt::Display::fmt(self.syntax(), f) } } +impl std::fmt::Display for MemberComp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } +} impl std::fmt::Display for Member { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) @@ -2996,21 +2840,6 @@ std::fmt::Display::fmt(self.syntax(), f) } } -impl std::fmt::Display for ExprIntrinsicThisFile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} -impl std::fmt::Display for ExprIntrinsicId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} -impl std::fmt::Display for ExprIntrinsic { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} impl std::fmt::Display for ExprString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) @@ -3111,27 +2940,17 @@ std::fmt::Display::fmt(self.syntax(), f) } } -impl std::fmt::Display for ObjLocalPostComma { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} -impl std::fmt::Display for ObjLocalPreComma { +impl std::fmt::Display for ObjBodyMemberList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) } } -impl std::fmt::Display for ObjBodyMemberList { +impl std::fmt::Display for MemberBindStmt { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) } } impl std::fmt::Display for ObjLocal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} -impl std::fmt::Display for MemberBindStmt { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) } --- a/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs +++ b/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs @@ -89,12 +89,6 @@ ASSIGN, #[token("?")] QUESTION_MARK, - #[token("$intrinsicThisFile")] - INTRINSIC_THIS_FILE, - #[token("$intrinsicId")] - INTRINSIC_ID, - #[token("$intrinsic")] - INTRINSIC, #[regex("(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")] FLOAT, #[regex("(?:0|[1-9][0-9]*)\\.[^0-9]")] @@ -199,9 +193,6 @@ EXPR_OBJ_EXTEND, EXPR_PARENED, EXPR_LITERAL, - EXPR_INTRINSIC_THIS_FILE, - EXPR_INTRINSIC_ID, - EXPR_INTRINSIC, EXPR_STRING, EXPR_NUMBER, EXPR_ARRAY, @@ -222,11 +213,9 @@ SLICE_DESC_STEP, ARG, OBJ_BODY_COMP, - OBJ_LOCAL_POST_COMMA, - OBJ_LOCAL_PRE_COMMA, OBJ_BODY_MEMBER_LIST, - OBJ_LOCAL, MEMBER_BIND_STMT, + OBJ_LOCAL, MEMBER_ASSERT_STMT, MEMBER_FIELD_NORMAL, MEMBER_FIELD_METHOD, @@ -248,6 +237,7 @@ OBJ_BODY, COMP_SPEC, BIND, + MEMBER_COMP, MEMBER, FIELD_NAME, DESTRUCT, @@ -260,7 +250,7 @@ IMPORT_KIND, VISIBILITY, TRIVIA, - PARSING_ERROR, + CUSTOM_ERROR, #[doc(hidden)] __LAST, } @@ -271,18 +261,18 @@ OR | AND | BIT_OR | BIT_XOR | BIT_AND | EQ | NE | LT | GT | LE | GE | LHS | RHS | PLUS | MINUS | MUL | DIV | MODULO | NOT | BIT_NOT | L_BRACK | R_BRACK | L_PAREN | R_PAREN | L_BRACE | R_BRACE | COLON | COLONCOLON | COLONCOLONCOLON | SEMI | DOT - | DOTDOTDOT | COMMA | DOLLAR | ASSIGN | QUESTION_MARK | INTRINSIC_THIS_FILE - | INTRINSIC_ID | INTRINSIC | TAILSTRICT_KW | IMPORTSTR_KW | IMPORTBIN_KW - | IMPORT_KW | LOCAL_KW | IF_KW | THEN_KW | ELSE_KW | FUNCTION_KW | ERROR_KW | IN_KW - | NULL_KW | TRUE_KW | FALSE_KW | SELF_KW | SUPER_KW | FOR_KW | ASSERT_KW => true, + | DOTDOTDOT | COMMA | DOLLAR | ASSIGN | QUESTION_MARK | TAILSTRICT_KW + | IMPORTSTR_KW | IMPORTBIN_KW | IMPORT_KW | LOCAL_KW | IF_KW | THEN_KW | ELSE_KW + | FUNCTION_KW | ERROR_KW | IN_KW | NULL_KW | TRUE_KW | FALSE_KW | SELF_KW + | SUPER_KW | FOR_KW | ASSERT_KW => true, _ => false, } } pub fn is_enum(self) -> bool { match self { - EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER | FIELD_NAME | DESTRUCT + EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER_COMP | MEMBER | FIELD_NAME | DESTRUCT | DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | TEXT | NUMBER - | IMPORT_KIND | VISIBILITY | TRIVIA | PARSING_ERROR => true, + | IMPORT_KIND | VISIBILITY | TRIVIA | CUSTOM_ERROR => true, _ => false, } } @@ -295,5 +285,5 @@ } } #[macro_export] -macro_rules ! T { [||] => { $ crate :: SyntaxKind :: OR } ; [&&] => { $ crate :: SyntaxKind :: AND } ; [|] => { $ crate :: SyntaxKind :: BIT_OR } ; [^] => { $ crate :: SyntaxKind :: BIT_XOR } ; [&] => { $ crate :: SyntaxKind :: BIT_AND } ; [==] => { $ crate :: SyntaxKind :: EQ } ; [!=] => { $ crate :: SyntaxKind :: NE } ; [<] => { $ crate :: SyntaxKind :: LT } ; [>] => { $ crate :: SyntaxKind :: GT } ; [<=] => { $ crate :: SyntaxKind :: LE } ; [>=] => { $ crate :: SyntaxKind :: GE } ; [<<] => { $ crate :: SyntaxKind :: LHS } ; [>>] => { $ crate :: SyntaxKind :: RHS } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [*] => { $ crate :: SyntaxKind :: MUL } ; [/] => { $ crate :: SyntaxKind :: DIV } ; [%] => { $ crate :: SyntaxKind :: MODULO } ; [!] => { $ crate :: SyntaxKind :: NOT } ; [~] => { $ crate :: SyntaxKind :: BIT_NOT } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_BRACE } ; ['}'] => { $ crate :: SyntaxKind :: R_BRACE } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLONCOLON } ; [:::] => { $ crate :: SyntaxKind :: COLONCOLONCOLON } ; [;] => { $ crate :: SyntaxKind :: SEMI } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [...] => { $ crate :: SyntaxKind :: DOTDOTDOT } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['$'] => { $ crate :: SyntaxKind :: DOLLAR } ; [=] => { $ crate :: SyntaxKind :: ASSIGN } ; [?] => { $ crate :: SyntaxKind :: QUESTION_MARK } ; ["$intrinsicThisFile"] => { $ crate :: SyntaxKind :: INTRINSIC_THIS_FILE } ; ["$intrinsicId"] => { $ crate :: SyntaxKind :: INTRINSIC_ID } ; ["$intrinsic"] => { $ crate :: SyntaxKind :: INTRINSIC } ; [tailstrict] => { $ crate :: SyntaxKind :: TAILSTRICT_KW } ; [importstr] => { $ crate :: SyntaxKind :: IMPORTSTR_KW } ; [importbin] => { $ crate :: SyntaxKind :: IMPORTBIN_KW } ; [import] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [local] => { $ crate :: SyntaxKind :: LOCAL_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [then] => { $ crate :: SyntaxKind :: THEN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [function] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [error] => { $ crate :: SyntaxKind :: ERROR_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [null] => { $ crate :: SyntaxKind :: NULL_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [assert] => { $ crate :: SyntaxKind :: ASSERT_KW } } +macro_rules ! T { [||] => { $ crate :: SyntaxKind :: OR } ; [&&] => { $ crate :: SyntaxKind :: AND } ; [|] => { $ crate :: SyntaxKind :: BIT_OR } ; [^] => { $ crate :: SyntaxKind :: BIT_XOR } ; [&] => { $ crate :: SyntaxKind :: BIT_AND } ; [==] => { $ crate :: SyntaxKind :: EQ } ; [!=] => { $ crate :: SyntaxKind :: NE } ; [<] => { $ crate :: SyntaxKind :: LT } ; [>] => { $ crate :: SyntaxKind :: GT } ; [<=] => { $ crate :: SyntaxKind :: LE } ; [>=] => { $ crate :: SyntaxKind :: GE } ; [<<] => { $ crate :: SyntaxKind :: LHS } ; [>>] => { $ crate :: SyntaxKind :: RHS } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [*] => { $ crate :: SyntaxKind :: MUL } ; [/] => { $ crate :: SyntaxKind :: DIV } ; [%] => { $ crate :: SyntaxKind :: MODULO } ; [!] => { $ crate :: SyntaxKind :: NOT } ; [~] => { $ crate :: SyntaxKind :: BIT_NOT } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_BRACE } ; ['}'] => { $ crate :: SyntaxKind :: R_BRACE } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLONCOLON } ; [:::] => { $ crate :: SyntaxKind :: COLONCOLONCOLON } ; [;] => { $ crate :: SyntaxKind :: SEMI } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [...] => { $ crate :: SyntaxKind :: DOTDOTDOT } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['$'] => { $ crate :: SyntaxKind :: DOLLAR } ; [=] => { $ crate :: SyntaxKind :: ASSIGN } ; [?] => { $ crate :: SyntaxKind :: QUESTION_MARK } ; [tailstrict] => { $ crate :: SyntaxKind :: TAILSTRICT_KW } ; [importstr] => { $ crate :: SyntaxKind :: IMPORTSTR_KW } ; [importbin] => { $ crate :: SyntaxKind :: IMPORTBIN_KW } ; [import] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [local] => { $ crate :: SyntaxKind :: LOCAL_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [then] => { $ crate :: SyntaxKind :: THEN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [function] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [error] => { $ crate :: SyntaxKind :: ERROR_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [null] => { $ crate :: SyntaxKind :: NULL_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [assert] => { $ crate :: SyntaxKind :: ASSERT_KW } } pub use T; --- a/crates/jrsonnet-rowan-parser/src/parser.rs +++ b/crates/jrsonnet-rowan-parser/src/parser.rs @@ -114,7 +114,13 @@ pub fn parse(mut self) -> Vec { let m = self.start(); expr(&mut self); - self.expect(EOF); + if !self.at(EOF) { + let m = self.start(); + while !self.at(EOF) { + self.bump(); + } + m.complete_error(&mut self, "unexpected tokens after end"); + } m.complete(&mut self, SOURCE_FILE); self.events @@ -832,21 +838,6 @@ let m = p.start(); name(p); m.complete(p, EXPR_VAR) - } else if p.at(INTRINSIC_THIS_FILE) { - let m = p.start(); - p.bump(); - m.complete(p, EXPR_INTRINSIC_THIS_FILE) - } else if p.at(INTRINSIC_ID) { - let m = p.start(); - p.bump(); - m.complete(p, EXPR_INTRINSIC_ID) - } else if p.at(INTRINSIC) { - let m = p.start(); - p.bump(); - p.expect(T!['(']); - name(p); - p.expect(T![')']); - m.complete(p, EXPR_INTRINSIC) } else if p.at(T![if]) { let m = p.start(); p.bump(); --- /dev/null +++ b/crates/jrsonnet-rowan-parser/src/snapshots/jrsonnet_rowan_parser__tests__continue_after_total_failure.snap @@ -0,0 +1,74 @@ +--- +source: crates/jrsonnet-rowan-parser/src/tests.rs +expression: "local intr = $intrinsic(test);\n\nlocal a = 1, b = 2, c = a + b;\n\n[c]\n" +--- +SOURCE_FILE@0..68 + EXPR_LOCAL@0..29 + LOCAL_KW@0..5 "local" + WHITESPACE@5..6 " " + BIND_DESTRUCT@6..14 + DESTRUCT_FULL@6..10 + NAME@6..10 + IDENT@6..10 "intr" + WHITESPACE@10..11 " " + ASSIGN@11..12 "=" + WHITESPACE@12..13 " " + EXPR_LITERAL@13..14 + DOLLAR@13..14 "$" + ERROR_UNEXPECTED_TOKEN@14..23 + IDENT@14..23 "intrinsic" + EXPR_PARENED@23..29 + L_PAREN@23..24 "(" + EXPR_VAR@24..28 + NAME@24..28 + IDENT@24..28 "test" + R_PAREN@28..29 ")" + ERROR_CUSTOM@29..67 + SEMI@29..30 ";" + WHITESPACE@30..32 "\n\n" + LOCAL_KW@32..37 "local" + WHITESPACE@37..38 " " + IDENT@38..39 "a" + WHITESPACE@39..40 " " + ASSIGN@40..41 "=" + WHITESPACE@41..42 " " + FLOAT@42..43 "1" + COMMA@43..44 "," + WHITESPACE@44..45 " " + IDENT@45..46 "b" + WHITESPACE@46..47 " " + ASSIGN@47..48 "=" + WHITESPACE@48..49 " " + FLOAT@49..50 "2" + COMMA@50..51 "," + WHITESPACE@51..52 " " + IDENT@52..53 "c" + WHITESPACE@53..54 " " + ASSIGN@54..55 "=" + WHITESPACE@55..56 " " + IDENT@56..57 "a" + WHITESPACE@57..58 " " + PLUS@58..59 "+" + WHITESPACE@59..60 " " + IDENT@60..61 "b" + SEMI@61..62 ";" + WHITESPACE@62..64 "\n\n" + L_BRACK@64..65 "[" + IDENT@65..66 "c" + R_BRACK@66..67 "]" + WHITESPACE@67..68 "\n" +=== +LocatedSyntaxError { error: Unexpected { expected: Unnamed(SyntaxKindSet([L_BRACK, L_PAREN, L_BRACE, SEMI, DOT, COMMA])), found: IDENT }, range: 14..23 } +LocatedSyntaxError { error: Custom { error: "unexpected tokens after end" }, range: 29..67 } +=== + x syntax error + ,-[1:1] + 1 | ,-> local intr = $intrinsic(test); + : || ^^^^|^^^^ + : || `-- expected L_BRACK, L_PAREN, L_BRACE, SEMI, DOT or COMMA, found IDENT + 2 | | + 3 | | local a = 1, b = 2, c = a + b; + 4 | | + 5 | |-> [c] + : `---- unexpected tokens after end + `---- --- a/crates/jrsonnet-rowan-parser/src/snapshots/jrsonnet_rowan_parser__tests__no_lhs.snap +++ b/crates/jrsonnet-rowan-parser/src/snapshots/jrsonnet_rowan_parser__tests__no_lhs.snap @@ -2,19 +2,21 @@ source: crates/jrsonnet-rowan-parser/src/tests.rs expression: "+ 2\n" --- -SOURCE_FILE@0..2 +SOURCE_FILE@0..4 ERROR_MISSING_TOKEN@0..0 - ERROR_UNEXPECTED_TOKEN@0..1 + ERROR_CUSTOM@0..3 PLUS@0..1 "+" - WHITESPACE@1..2 " " + WHITESPACE@1..2 " " + FLOAT@2..3 "2" + WHITESPACE@3..4 "\n" === LocatedSyntaxError { error: Missing { expected: Named("expression") }, range: 0..0 } -LocatedSyntaxError { error: Unexpected { expected: Unnamed(SyntaxKindSet([EOF])), found: PLUS }, range: 0..1 } +LocatedSyntaxError { error: Custom { error: "unexpected tokens after end" }, range: 0..3 } === x syntax error ,---- 1 | + 2 - : ^| - : |`-- expected EOF, found PLUS + : ^^| + : | `-- unexpected tokens after end : `-- missing expression `---- --- a/crates/jrsonnet-rowan-parser/src/snapshots/jrsonnet_rowan_parser__tests__no_operator.snap +++ b/crates/jrsonnet-rowan-parser/src/snapshots/jrsonnet_rowan_parser__tests__no_operator.snap @@ -6,15 +6,15 @@ EXPR_NUMBER@0..1 FLOAT@0..1 "2" WHITESPACE@1..2 " " - ERROR_UNEXPECTED_TOKEN@2..3 + ERROR_CUSTOM@2..3 FLOAT@2..3 "2" WHITESPACE@3..4 "\n" === -LocatedSyntaxError { error: Unexpected { expected: Unnamed(SyntaxKindSet([EOF, L_BRACK, L_PAREN, L_BRACE, DOT])), found: FLOAT }, range: 2..3 } +LocatedSyntaxError { error: Custom { error: "unexpected tokens after end" }, range: 2..3 } === x syntax error ,---- 1 | 2 2 : | - : `-- expected EOF, L_BRACK, L_PAREN, L_BRACE or DOT, found FLOAT + : `-- unexpected tokens after end `---- --- a/crates/jrsonnet-rowan-parser/src/tests.rs +++ b/crates/jrsonnet-rowan-parser/src/tests.rs @@ -228,6 +228,14 @@ a: function(x) x, } "# + + continue_after_total_failure => r#" + local intr = $intrinsic(test); + + local a = 1, b = 2, c = a + b; + + [c] + "# ); #[test] --- a/crates/jrsonnet-rowan-parser/src/token_set.rs +++ b/crates/jrsonnet-rowan-parser/src/token_set.rs @@ -27,7 +27,10 @@ SyntaxKindSet(self.0 | mask(kind)) } - pub const fn contains(&self, kind: SyntaxKind) -> bool { + pub fn contains(&self, kind: SyntaxKind) -> bool { + if !is_token(kind) { + return false; + } self.0 & mask(kind) != 0 } } @@ -74,6 +77,9 @@ } const fn mask(kind: SyntaxKind) -> u128 { + if kind as u32 > 128 { + panic!("mask for not a token kind") + } 1u128 << (kind as u128) } @@ -95,3 +101,6 @@ "can't keep KindSet as bitset" ); } +fn is_token(kind: SyntaxKind) -> bool { + (kind as u32) < 127 +} --- a/xtask/src/sourcegen/kinds.rs +++ b/xtask/src/sourcegen/kinds.rs @@ -247,9 +247,6 @@ "$" => "DOLLAR"; "=" => "ASSIGN"; "?" => "QUESTION_MARK"; - "$intrinsicThisFile" => "INTRINSIC_THIS_FILE"; - "$intrinsicId" => "INTRINSIC_ID"; - "$intrinsic" => "INTRINSIC"; // Literals lit("FLOAT") => r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?"; error("FLOAT_JUNK_AFTER_POINT") => r"(?:0|[1-9][0-9]*)\.[^0-9]"; -- gitstuff