difftreelog
refactor use grammar to classify tokens
in: master
13 files changed
cmds/jrsonnet-fmt/src/main.rsdiffbeforeafterboth1use std::any::type_name;23use dprint_core::formatting::{PrintItems, PrintOptions, Signal};4use jrsonnet_rowan_parser::{5 nodes::{6 ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart,7 DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal,8 Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, String,9 UnaryOperator,10 },11 AstToken, SyntaxToken,12};1314pub trait Printable {15 fn print(&self) -> PrintItems;16}1718macro_rules! pi {19 (@i; $($t:tt)*) => {{20 #[allow(unused_mut)]21 let mut o = PrintItems::new();22 pi!(@s; o: $($t)*);23 o24 }};25 (@s; $o:ident: str($e:expr $(,)?) $($t:tt)*) => {{26 $o.push_str($e);27 pi!(@s; $o: $($t)*);28 }};29 (@s; $o:ident: nl $($t:tt)*) => {{30 $o.push_signal(Signal::NewLine);31 pi!(@s; $o: $($t)*);32 }};33 (@s; $o:ident: >i $($t:tt)*) => {{34 $o.push_signal(Signal::StartIndent);35 pi!(@s; $o: $($t)*);36 }};37 (@s; $o:ident: <i $($t:tt)*) => {{38 $o.push_signal(Signal::FinishIndent);39 pi!(@s; $o: $($t)*);40 }};41 (@s; $o:ident: {$expr:expr} $($t:tt)*) => {{42 $o.extend($expr.print());43 pi!(@s; $o: $($t)*);44 }};45 (@s; $o:ident: if ($e:expr)($($then:tt)*) $($t:tt)*) => {{46 if $e {47 pi!(@s; $o: $($then)*);48 }49 pi!(@s; $o: $($t)*);50 }};51 (@s; $o:ident: ifelse ($e:expr)($($then:tt)*)($($else:tt)*) $($t:tt)*) => {{52 if $e {53 pi!(@s; $o: $($then)*);54 } else {55 pi!(@s; $o: $($else)*);56 }57 pi!(@s; $o: $($t)*);58 }};59 (@s; $i:ident:) => {}60}61macro_rules! p {62 (new: $($t:tt)*) => {63 pi!(@i; $($t)*)64 };65 ($o:ident: $($t:tt)*) => {66 pi!(@s; $o: $($t)*)67 };68}6970impl<P> Printable for Option<P>71where72 P: Printable,73{74 fn print(&self) -> PrintItems {75 if let Some(v) = self {76 v.print()77 } else {78 p!(new: str(79 &format!(80 "/*missing {}*/",81 type_name::<P>().replace("jrsonnet_rowan_parser::generated::nodes::", "")82 ),83 ))84 }85 }86}8788impl Printable for SyntaxToken {89 fn print(&self) -> PrintItems {90 p!(new: str(&self.to_string()))91 }92}9394impl Printable for String {95 fn print(&self) -> PrintItems {96 p!(new: str(&format!("{}", self)))97 }98}99impl Printable for Number {100 fn print(&self) -> PrintItems {101 p!(new: str(&format!("{}", self)))102 }103}104105impl Printable for Name {106 fn print(&self) -> PrintItems {107 p!(new: {self.ident_lit()})108 }109}110111impl Printable for DestructRest {112 fn print(&self) -> PrintItems {113 let mut pi = p!(new: str("..."));114 if let Some(name) = self.into() {115 p!(pi: {name});116 }117 pi118 }119}120121impl Printable for Destruct {122 fn print(&self) -> PrintItems {123 let mut pi = p!(new:);124 match self {125 Destruct::DestructFull(f) => {126 p!(pi: {f.name()})127 }128 Destruct::DestructSkip(_) => p!(pi: str("?")),129 Destruct::DestructArray(a) => {130 p!(pi: str("[") >i nl);131 for el in a.destruct_array_parts() {132 match el {133 DestructArrayPart::DestructArrayElement(e) => {134 p!(pi: {e.destruct()} str(",") nl)135 }136 DestructArrayPart::DestructRest(d) => {137 p!(pi: {d} str(",") nl)138 }139 }140 }141 p!(pi: <i str("]"));142 }143 Destruct::DestructObject(o) => {144 p!(pi: str("{") >i nl);145 for item in o.destruct_object_fields() {146 p!(pi: {item.field()});147 if let Some(des) = item.destruct() {148 p!(pi: str(": ") {des})149 }150 if let Some(def) = item.expr() {151 p!(pi: str(" = ") {def});152 }153 p!(pi: str(",") nl);154 }155 if let Some(rest) = o.destruct_rest() {156 p!(pi: {rest} nl)157 }158 p!(pi: <i str("}"));159 }160 }161 pi162 }163}164165impl Printable for FieldName {166 fn print(&self) -> PrintItems {167 match self {168 FieldName::FieldNameFixed(f) => {169 if let Some(id) = f.id() {170 p!(new: {id})171 } else if let Some(str) = f.string() {172 p!(new: {str})173 } else {174 p!(new: str("/*missing FieldName*/"))175 }176 }177 FieldName::FieldNameDynamic(d) => {178 p!(new: str("[") {d.expr()} str("]"))179 }180 }181 }182}183impl Printable for Field {184 fn print(&self) -> PrintItems {185 let mut pi = p!(new:);186 match self {187 Field::FieldNormal(n) => {188 p!(pi: {n.field_name()});189 if n.plus_token().is_some() {190 p!(pi: str("+"));191 }192 p!(pi: str(": ") {n.expr()});193 }194 Field::FieldMethod(m) => {195 p!(pi: {m.field_name()} {m.params_desc()} str(": ") {m.expr()});196 }197 }198 pi199 }200}201202impl Printable for ObjLocal {203 fn print(&self) -> PrintItems {204 p!(new: str("local ") {self.bind()})205 }206}207208impl Printable for Assertion {209 fn print(&self) -> PrintItems {210 let mut pi = p!(new: str("assert ") {self.condition()});211 if self.colon_token().is_some() || self.message().is_some() {212 p!(pi: str(": ") {self.message()})213 }214 pi215 }216}217218impl Printable for ParamsDesc {219 fn print(&self) -> PrintItems {220 let mut pi = p!(new: str("(") >i nl);221 for param in self.params() {222 p!(pi: {param.destruct()});223 if param.assign_token().is_some() || param.expr().is_some() {224 p!(pi: str(" = ") {param.expr()})225 }226 p!(pi: str(",") nl)227 }228 p!(pi: <i str(")"));229 pi230 }231}232impl Printable for ArgsDesc {233 fn print(&self) -> PrintItems {234 let mut pi = p!(new: str("(") >i nl);235 for arg in self.args() {236 if arg.name().is_some() || arg.assign_token().is_some() {237 p!(pi: {arg.name()} str(" = "));238 }239 p!(pi: {arg.expr()} str(",") nl)240 }241 p!(pi: <i str(")"));242 pi243 }244}245impl Printable for SliceDesc {246 fn print(&self) -> PrintItems {247 let mut pi = p!(new: str("["));248 if self.from().is_some() {249 p!(pi: {self.from()});250 }251 p!(pi: str(":"));252 if self.end().is_some() {253 p!(pi: {self.end().map(|e|e.expr())})254 }255 // Keep only one : in case if we don't need step256 if self.step().is_some() {257 p!(pi: str(":") {self.step().map(|e|e.expr())});258 }259 p!(pi: str("]"));260 pi261 }262}263264impl Printable for ObjBody {265 fn print(&self) -> PrintItems {266 match self {267 ObjBody::ObjBodyComp(_) => todo!(),268 ObjBody::ObjBodyMemberList(l) => {269 let mut pi = p!(new:);270 for mem in l.members() {271 match mem {272 Member::MemberBindStmt(b) => {273 p!(pi: {b.obj_local()})274 }275 Member::MemberAssertStmt(ass) => {276 p!(pi: {ass.assertion()})277 }278 Member::MemberField(f) => {279 p!(pi: {f.field()})280 }281 }282 p!(pi: str(",") nl)283 }284 pi285 }286 }287 }288}289impl Printable for UnaryOperator {290 fn print(&self) -> PrintItems {291 p!(new: str(self.text()))292 }293}294impl Printable for BinaryOperator {295 fn print(&self) -> PrintItems {296 p!(new: str(self.text()))297 }298}299impl Printable for Bind {300 fn print(&self) -> PrintItems {301 match self {302 Bind::BindDestruct(d) => {303 p!(new: {d.into()} str(" = ") {d.value()})304 }305 Bind::BindFunction(f) => {306 p!(new: str("function") {f.params()} str(" = ") {f.value()})307 }308 }309 }310}311impl Printable for Literal {312 fn print(&self) -> PrintItems {313 p!(new: str(&self.syntax().to_string()))314 }315}316impl Printable for ImportKind {317 fn print(&self) -> PrintItems {318 p!(new: str(&self.syntax().to_string()))319 }320}321impl Printable for LhsExpr {322 fn print(&self) -> PrintItems {323 p!(new: {self.expr()})324 }325}326impl Printable for ForSpec {327 fn print(&self) -> PrintItems {328 p!(new: str("for ") {self.bind()} str(" in ") {self.expr()})329 }330}331impl Printable for IfSpec {332 fn print(&self) -> PrintItems {333 p!(new: str("if ") {self.expr()})334 }335}336impl Printable for CompSpec {337 fn print(&self) -> PrintItems {338 match self {339 CompSpec::ForSpec(f) => f.print(),340 CompSpec::IfSpec(i) => i.print(),341 }342 }343}344impl Printable for Expr {345 fn print(&self) -> PrintItems {346 match self {347 Expr::ExprBinary(b) => {348 p!(new: {b.lhs()} str(" ") {b.binary_operator()} str(" ") {b.rhs()})349 }350 Expr::ExprUnary(u) => p!(new: {u.unary_operator()} {u.rhs()}),351 Expr::ExprSlice(s) => {352 p!(new: {s.expr()} {s.slice_desc()})353 }354 Expr::ExprIndex(i) => {355 p!(new: {i.expr()} str(".") {i.index()})356 }357 Expr::ExprIndexExpr(i) => p!(new: {i.base()} str("[") {i.index()} str("]")),358 Expr::ExprApply(a) => {359 let mut pi = p!(new: {a.expr()} {a.args_desc()});360 if a.tailstrict_kw_token().is_some() {361 p!(pi: str(" tailstrict"));362 }363 pi364 }365 Expr::ExprObjExtend(ex) => {366 p!(new: {ex.lhs_expr()} str(" ") {ex.expr()})367 }368 Expr::ExprParened(p) => {369 p!(new: str("(") {p.expr()} str(")"))370 }371 Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")),372 Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")),373 Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")),374 Expr::ExprString(s) => p!(new: {s.string()}),375 Expr::ExprNumber(n) => p!(new: {n.number()}),376 Expr::ExprArray(a) => {377 let mut pi = p!(new: str("[") >i nl);378 for el in a.exprs() {379 p!(pi: {el} str(",") nl);380 }381 p!(pi: <i str("]"));382 pi383 }384 Expr::ExprObject(o) => {385 p!(new: str("{") >i nl {o.obj_body()} <i str("}"))386 }387 Expr::ExprArrayComp(arr) => {388 let mut pi = p!(new: str("[") {arr.expr()});389 for spec in arr.comp_specs() {390 p!(pi: str(" ") {spec});391 }392 p!(pi: str("]"));393 pi394 }395 Expr::ExprImport(v) => {396 p!(new: {v.import_kind()} str(" ") {v.string()})397 }398 Expr::ExprVar(n) => p!(new: {n.name()}),399 Expr::ExprLocal(l) => {400 let mut pi = p!(new: str("local") >i nl);401 for bind in l.binds() {402 p!(pi: {bind} str(",") nl);403 }404 p!(pi: <i str(";") nl {l.expr()});405 pi406 }407 Expr::ExprIfThenElse(ite) => {408 let mut pi =409 p!(new: str("if ") {ite.cond()} str(" then ") {ite.then().map(|t| t.expr())});410 if ite.else_kw_token().is_some() || ite.else_().is_some() {411 p!(pi: str(" else ") {ite.else_().map(|t| t.expr())})412 }413 pi414 }415 Expr::ExprFunction(f) => p!(new: str("function") {f.params_desc()} str(" ") {f.expr()}),416 Expr::ExprAssert(a) => p!(new: {a.assertion()} str("; ") {a.expr()}),417 Expr::ExprError(e) => p!(new: str("error ") {e.expr()}),418 Expr::ExprLiteral(l) => {419 p!(new: {l.literal()})420 }421 }422 }423}424425impl Printable for SourceFile {426 fn print(&self) -> PrintItems {427 assert!(self.expr().is_some());428 self.expr().print()429 }430}431432fn main() {433 let (parsed, _errors) = jrsonnet_rowan_parser::parse(434 r#"435436437 # Edit me!438 local b = import "b.libsonnet"; # comment439 local a = import "a.libsonnet";440441 local f(x,y)=x+y;442443 local {a: [b, ..., c], d, ...e} = null;444445 local ass = assert false : false; false;446447 local fn = function(a, b, c = 3) 4;448449 local comp = [a for b in c if d == e];450 local ocomp = {[k]: 1 for k in v};451452 local ? = skip;453454 local intr = $intrinsic(test);455 local intrId = $intrinsicId;456 local intrThisFile = $intrinsicThisFile;457458 local ie = a[expr];459460 local unary = !a;461462 local Template = {z: "foo"};463464 {465 local466467 h = 3,468 assert self.a == 1469470 : "error",471 "f": ((((((3)))))) ,472 "g g":473 f(4,2),474 arr: [[475 1, 2,476 ],477 3,478 {479 b: {480 c: {481 k: [16]482 }483 }484 }485 ],486 m: a[1::],487 m: b[::],488 k: if a == b then489490491 2492493 else Template {}494 } + Template495496497"#,498 );499500 // dbg!(errors);501 dbg!(&parsed);502503 let o = dprint_core::formatting::format(504 || parsed.print(),505 PrintOptions {506 indent_width: 2,507 max_width: 100,508 use_tabs: false,509 new_line_text: "\n",510 },511 );512 println!("{}", o);513}1use std::any::type_name;23use dprint_core::formatting::{PrintItems, PrintOptions, Signal};4use jrsonnet_rowan_parser::{5 nodes::{6 ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart,7 DestructRest, Expr, Field, FieldName, ForSpec, IfSpec, ImportKind, LhsExpr, Literal,8 Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text,9 UnaryOperator,10 },11 AstToken, SyntaxToken,12};1314pub trait Printable {15 fn print(&self) -> PrintItems;16}1718macro_rules! pi {19 (@i; $($t:tt)*) => {{20 #[allow(unused_mut)]21 let mut o = PrintItems::new();22 pi!(@s; o: $($t)*);23 o24 }};25 (@s; $o:ident: str($e:expr $(,)?) $($t:tt)*) => {{26 $o.push_str($e);27 pi!(@s; $o: $($t)*);28 }};29 (@s; $o:ident: nl $($t:tt)*) => {{30 $o.push_signal(Signal::NewLine);31 pi!(@s; $o: $($t)*);32 }};33 (@s; $o:ident: >i $($t:tt)*) => {{34 $o.push_signal(Signal::StartIndent);35 pi!(@s; $o: $($t)*);36 }};37 (@s; $o:ident: <i $($t:tt)*) => {{38 $o.push_signal(Signal::FinishIndent);39 pi!(@s; $o: $($t)*);40 }};41 (@s; $o:ident: {$expr:expr} $($t:tt)*) => {{42 $o.extend($expr.print());43 pi!(@s; $o: $($t)*);44 }};45 (@s; $o:ident: if ($e:expr)($($then:tt)*) $($t:tt)*) => {{46 if $e {47 pi!(@s; $o: $($then)*);48 }49 pi!(@s; $o: $($t)*);50 }};51 (@s; $o:ident: ifelse ($e:expr)($($then:tt)*)($($else:tt)*) $($t:tt)*) => {{52 if $e {53 pi!(@s; $o: $($then)*);54 } else {55 pi!(@s; $o: $($else)*);56 }57 pi!(@s; $o: $($t)*);58 }};59 (@s; $i:ident:) => {}60}61macro_rules! p {62 (new: $($t:tt)*) => {63 pi!(@i; $($t)*)64 };65 ($o:ident: $($t:tt)*) => {66 pi!(@s; $o: $($t)*)67 };68}6970impl<P> Printable for Option<P>71where72 P: Printable,73{74 fn print(&self) -> PrintItems {75 if let Some(v) = self {76 v.print()77 } else {78 p!(new: str(79 &format!(80 "/*missing {}*/",81 type_name::<P>().replace("jrsonnet_rowan_parser::generated::nodes::", "")82 ),83 ))84 }85 }86}8788impl Printable for SyntaxToken {89 fn print(&self) -> PrintItems {90 p!(new: str(&self.to_string()))91 }92}9394impl Printable for Text {95 fn print(&self) -> PrintItems {96 p!(new: str(&format!("{}", self)))97 }98}99impl Printable for Number {100 fn print(&self) -> PrintItems {101 p!(new: str(&format!("{}", self)))102 }103}104105impl Printable for Name {106 fn print(&self) -> PrintItems {107 p!(new: {self.ident_lit()})108 }109}110111impl Printable for DestructRest {112 fn print(&self) -> PrintItems {113 let mut pi = p!(new: str("..."));114 if let Some(name) = self.into() {115 p!(pi: {name});116 }117 pi118 }119}120121impl Printable for Destruct {122 fn print(&self) -> PrintItems {123 let mut pi = p!(new:);124 match self {125 Destruct::DestructFull(f) => {126 p!(pi: {f.name()})127 }128 Destruct::DestructSkip(_) => p!(pi: str("?")),129 Destruct::DestructArray(a) => {130 p!(pi: str("[") >i nl);131 for el in a.destruct_array_parts() {132 match el {133 DestructArrayPart::DestructArrayElement(e) => {134 p!(pi: {e.destruct()} str(",") nl)135 }136 DestructArrayPart::DestructRest(d) => {137 p!(pi: {d} str(",") nl)138 }139 }140 }141 p!(pi: <i str("]"));142 }143 Destruct::DestructObject(o) => {144 p!(pi: str("{") >i nl);145 for item in o.destruct_object_fields() {146 p!(pi: {item.field()});147 if let Some(des) = item.destruct() {148 p!(pi: str(": ") {des})149 }150 if let Some(def) = item.expr() {151 p!(pi: str(" = ") {def});152 }153 p!(pi: str(",") nl);154 }155 if let Some(rest) = o.destruct_rest() {156 p!(pi: {rest} nl)157 }158 p!(pi: <i str("}"));159 }160 }161 pi162 }163}164165impl Printable for FieldName {166 fn print(&self) -> PrintItems {167 match self {168 FieldName::FieldNameFixed(f) => {169 if let Some(id) = f.id() {170 p!(new: {id})171 } else if let Some(str) = f.text() {172 p!(new: {str})173 } else {174 p!(new: str("/*missing FieldName*/"))175 }176 }177 FieldName::FieldNameDynamic(d) => {178 p!(new: str("[") {d.expr()} str("]"))179 }180 }181 }182}183impl Printable for Field {184 fn print(&self) -> PrintItems {185 let mut pi = p!(new:);186 match self {187 Field::FieldNormal(n) => {188 p!(pi: {n.field_name()});189 if n.plus_token().is_some() {190 p!(pi: str("+"));191 }192 p!(pi: str(": ") {n.expr()});193 }194 Field::FieldMethod(m) => {195 p!(pi: {m.field_name()} {m.params_desc()} str(": ") {m.expr()});196 }197 }198 pi199 }200}201202impl Printable for ObjLocal {203 fn print(&self) -> PrintItems {204 p!(new: str("local ") {self.bind()})205 }206}207208impl Printable for Assertion {209 fn print(&self) -> PrintItems {210 let mut pi = p!(new: str("assert ") {self.condition()});211 if self.colon_token().is_some() || self.message().is_some() {212 p!(pi: str(": ") {self.message()})213 }214 pi215 }216}217218impl Printable for ParamsDesc {219 fn print(&self) -> PrintItems {220 let mut pi = p!(new: str("(") >i nl);221 for param in self.params() {222 p!(pi: {param.destruct()});223 if param.assign_token().is_some() || param.expr().is_some() {224 p!(pi: str(" = ") {param.expr()})225 }226 p!(pi: str(",") nl)227 }228 p!(pi: <i str(")"));229 pi230 }231}232impl Printable for ArgsDesc {233 fn print(&self) -> PrintItems {234 let mut pi = p!(new: str("(") >i nl);235 for arg in self.args() {236 if arg.name().is_some() || arg.assign_token().is_some() {237 p!(pi: {arg.name()} str(" = "));238 }239 p!(pi: {arg.expr()} str(",") nl)240 }241 p!(pi: <i str(")"));242 pi243 }244}245impl Printable for SliceDesc {246 fn print(&self) -> PrintItems {247 let mut pi = p!(new: str("["));248 if self.from().is_some() {249 p!(pi: {self.from()});250 }251 p!(pi: str(":"));252 if self.end().is_some() {253 p!(pi: {self.end().map(|e|e.expr())})254 }255 // Keep only one : in case if we don't need step256 if self.step().is_some() {257 p!(pi: str(":") {self.step().map(|e|e.expr())});258 }259 p!(pi: str("]"));260 pi261 }262}263264impl Printable for ObjBody {265 fn print(&self) -> PrintItems {266 match self {267 ObjBody::ObjBodyComp(_) => todo!(),268 ObjBody::ObjBodyMemberList(l) => {269 let mut pi = p!(new:);270 for mem in l.members() {271 match mem {272 Member::MemberBindStmt(b) => {273 p!(pi: {b.obj_local()})274 }275 Member::MemberAssertStmt(ass) => {276 p!(pi: {ass.assertion()})277 }278 Member::MemberField(f) => {279 p!(pi: {f.field()})280 }281 }282 p!(pi: str(",") nl)283 }284 pi285 }286 }287 }288}289impl Printable for UnaryOperator {290 fn print(&self) -> PrintItems {291 p!(new: str(self.text()))292 }293}294impl Printable for BinaryOperator {295 fn print(&self) -> PrintItems {296 p!(new: str(self.text()))297 }298}299impl Printable for Bind {300 fn print(&self) -> PrintItems {301 match self {302 Bind::BindDestruct(d) => {303 p!(new: {d.into()} str(" = ") {d.value()})304 }305 Bind::BindFunction(f) => {306 p!(new: str("function") {f.params()} str(" = ") {f.value()})307 }308 }309 }310}311impl Printable for Literal {312 fn print(&self) -> PrintItems {313 p!(new: str(&self.syntax().to_string()))314 }315}316impl Printable for ImportKind {317 fn print(&self) -> PrintItems {318 p!(new: str(&self.syntax().to_string()))319 }320}321impl Printable for LhsExpr {322 fn print(&self) -> PrintItems {323 p!(new: {self.expr()})324 }325}326impl Printable for ForSpec {327 fn print(&self) -> PrintItems {328 p!(new: str("for ") {self.bind()} str(" in ") {self.expr()})329 }330}331impl Printable for IfSpec {332 fn print(&self) -> PrintItems {333 p!(new: str("if ") {self.expr()})334 }335}336impl Printable for CompSpec {337 fn print(&self) -> PrintItems {338 match self {339 CompSpec::ForSpec(f) => f.print(),340 CompSpec::IfSpec(i) => i.print(),341 }342 }343}344impl Printable for Expr {345 fn print(&self) -> PrintItems {346 match self {347 Expr::ExprBinary(b) => {348 p!(new: {b.lhs()} str(" ") {b.binary_operator()} str(" ") {b.rhs()})349 }350 Expr::ExprUnary(u) => p!(new: {u.unary_operator()} {u.rhs()}),351 Expr::ExprSlice(s) => {352 p!(new: {s.expr()} {s.slice_desc()})353 }354 Expr::ExprIndex(i) => {355 p!(new: {i.expr()} str(".") {i.index()})356 }357 Expr::ExprIndexExpr(i) => p!(new: {i.base()} str("[") {i.index()} str("]")),358 Expr::ExprApply(a) => {359 let mut pi = p!(new: {a.expr()} {a.args_desc()});360 if a.tailstrict_kw_token().is_some() {361 p!(pi: str(" tailstrict"));362 }363 pi364 }365 Expr::ExprObjExtend(ex) => {366 p!(new: {ex.lhs_expr()} str(" ") {ex.expr()})367 }368 Expr::ExprParened(p) => {369 p!(new: str("(") {p.expr()} str(")"))370 }371 Expr::ExprIntrinsicThisFile(_) => p!(new: str("$intrinsicThisFile")),372 Expr::ExprIntrinsicId(_) => p!(new: str("$intrinsicId")),373 Expr::ExprIntrinsic(i) => p!(new: str("$intrinsic(") {i.name()} str(")")),374 Expr::ExprString(s) => p!(new: {s.text()}),375 Expr::ExprNumber(n) => p!(new: {n.number()}),376 Expr::ExprArray(a) => {377 let mut pi = p!(new: str("[") >i nl);378 for el in a.exprs() {379 p!(pi: {el} str(",") nl);380 }381 p!(pi: <i str("]"));382 pi383 }384 Expr::ExprObject(o) => {385 p!(new: str("{") >i nl {o.obj_body()} <i str("}"))386 }387 Expr::ExprArrayComp(arr) => {388 let mut pi = p!(new: str("[") {arr.expr()});389 for spec in arr.comp_specs() {390 p!(pi: str(" ") {spec});391 }392 p!(pi: str("]"));393 pi394 }395 Expr::ExprImport(v) => {396 p!(new: {v.import_kind()} str(" ") {v.text()})397 }398 Expr::ExprVar(n) => p!(new: {n.name()}),399 Expr::ExprLocal(l) => {400 let mut pi = p!(new: str("local") >i nl);401 for bind in l.binds() {402 p!(pi: {bind} str(",") nl);403 }404 p!(pi: <i str(";") nl {l.expr()});405 pi406 }407 Expr::ExprIfThenElse(ite) => {408 let mut pi =409 p!(new: str("if ") {ite.cond()} str(" then ") {ite.then().map(|t| t.expr())});410 if ite.else_kw_token().is_some() || ite.else_().is_some() {411 p!(pi: str(" else ") {ite.else_().map(|t| t.expr())})412 }413 pi414 }415 Expr::ExprFunction(f) => p!(new: str("function") {f.params_desc()} str(" ") {f.expr()}),416 Expr::ExprAssert(a) => p!(new: {a.assertion()} str("; ") {a.expr()}),417 Expr::ExprError(e) => p!(new: str("error ") {e.expr()}),418 Expr::ExprLiteral(l) => {419 p!(new: {l.literal()})420 }421 }422 }423}424425impl Printable for SourceFile {426 fn print(&self) -> PrintItems {427 assert!(self.expr().is_some());428 self.expr().print()429 }430}431432fn main() {433 let (parsed, _errors) = jrsonnet_rowan_parser::parse(434 r#"435436437 # Edit me!438 local b = import "b.libsonnet"; # comment439 local a = import "a.libsonnet";440441 local f(x,y)=x+y;442443 local {a: [b, ..., c], d, ...e} = null;444445 local ass = assert false : false; false;446447 local fn = function(a, b, c = 3) 4;448449 local comp = [a for b in c if d == e];450 local ocomp = {[k]: 1 for k in v};451452 local ? = skip;453454 local intr = $intrinsic(test);455 local intrId = $intrinsicId;456 local intrThisFile = $intrinsicThisFile;457458 local ie = a[expr];459460 local unary = !a;461462 local Template = {z: "foo"};463464 {465 local466467 h = 3,468 assert self.a == 1469470 : "error",471 "f": ((((((3)))))) ,472 "g g":473 f(4,2),474 arr: [[475 1, 2,476 ],477 3,478 {479 b: {480 c: {481 k: [16]482 }483 }484 }485 ],486 m: a[1::],487 m: b[::],488 k: if a == b then489490491 2492493 else Template {}494 } + Template495496497"#,498 );499500 // dbg!(errors);501 dbg!(&parsed);502503 let o = dprint_core::formatting::format(504 || parsed.print(),505 PrintOptions {506 indent_width: 2,507 max_width: 100,508 use_tabs: false,509 new_line_text: "\n",510 },511 );512 println!("{}", o);513}crates/jrsonnet-rowan-parser/jsonnet.ungramdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/jsonnet.ungram
+++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram
@@ -48,7 +48,7 @@
name:Name
')'
ExprString =
- String
+ Text
ExprNumber =
Number
ExprArray =
@@ -67,7 +67,7 @@
']'
ExprImport =
- ImportKind String
+ ImportKind Text
ImportKind =
'importstr'
@@ -217,7 +217,7 @@
FieldNameFixed =
id:Name
-| String
+| Text
FieldNameDynamic =
'['
Expr
@@ -239,16 +239,27 @@
| '$'
| 'super'
-String =
+Text =
'LIT_STRING_DOUBLE!'
+| 'ERROR_STRING_DOUBLE_UNTERMINATED!'
| 'LIT_STRING_SINGLE!'
+| 'ERROR_STRING_SINGLE_UNTERMINATED!'
| 'LIT_STRING_DOUBLE_VERBATIM!'
+| 'ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED!'
| 'LIT_STRING_SINGLE_VERBATIM!'
+| 'ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED!'
+| 'ERROR_STRING_VERBATIM_MISSING_QUOTES!'
| 'LIT_STRING_BLOCK!'
+| 'ERROR_STRING_BLOCK_UNEXPECTED_END!'
+| 'ERROR_STRING_BLOCK_MISSING_NEW_LINE!'
+| 'ERROR_STRING_BLOCK_MISSING_TERMINATION!'
+| 'ERROR_STRING_BLOCK_MISSING_INDENT!'
Number =
'LIT_FLOAT!'
-| 'META_FORCE_ENUM!'
+| 'ERROR_FLOAT_JUNK_AFTER_POINT!'
+| 'ERROR_FLOAT_JUNK_AFTER_EXPONENT!'
+| 'ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN!'
ForSpec =
'for'
@@ -347,3 +358,12 @@
TrueExpr=Expr
FalseExpr=Expr
LhsExpr=Expr
+
+// Trivia - tokens which will be implicitly skipped for parser
+Trivia =
+ 'LIT_WHITESPACE!'
+| 'LIT_MULTI_LINE_COMMENT!'
+| 'ERROR_COMMENT_TOO_SHORT!'
+| 'ERROR_COMMENT_UNTERMINATED!'
+| 'LIT_SINGLE_LINE_HASH_COMMENT!'
+| 'LIT_SINGLE_LINE_SLASH_COMMENT!'
crates/jrsonnet-rowan-parser/src/classify.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/classify.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-use crate::SyntaxKind;
-
-impl SyntaxKind {
- pub fn is_trivia(self) -> bool {
- matches!(
- self,
- Self::WHITESPACE
- | Self::MULTI_LINE_COMMENT
- | Self::ERROR_COMMENT_TOO_SHORT
- | Self::ERROR_COMMENT_UNTERMINATED
- | Self::SINGLE_LINE_HASH_COMMENT
- | Self::SINGLE_LINE_SLASH_COMMENT
- )
- }
- pub fn is_string(self) -> bool {
- matches!(
- self,
- Self::STRING_SINGLE
- | Self::ERROR_STRING_SINGLE_UNTERMINATED
- | Self::STRING_DOUBLE
- | Self::ERROR_STRING_DOUBLE_UNTERMINATED
- | Self::STRING_SINGLE_VERBATIM
- | Self::ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED
- | Self::STRING_DOUBLE_VERBATIM
- | Self::ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED
- | Self::STRING_BLOCK
- | Self::ERROR_STRING_BLOCK_UNEXPECTED_END
- | Self::ERROR_STRING_BLOCK_MISSING_NEW_LINE
- | Self::ERROR_STRING_BLOCK_MISSING_TERMINATION
- | Self::ERROR_STRING_BLOCK_MISSING_INDENT
- )
- }
- pub fn is_number(self) -> bool {
- matches!(
- self,
- Self::FLOAT
- | Self::ERROR_FLOAT_JUNK_AFTER_POINT
- | Self::ERROR_FLOAT_JUNK_AFTER_EXPONENT
- | Self::ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN
- )
- }
- pub fn is_literal(self) -> bool {
- matches!(
- self,
- Self::NULL_KW
- | Self::TRUE_KW | Self::FALSE_KW
- | Self::SELF_KW | Self::DOLLAR
- | Self::SUPER_KW
- )
- }
-}
crates/jrsonnet-rowan-parser/src/event.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/event.rs
+++ b/crates/jrsonnet-rowan-parser/src/event.rs
@@ -4,8 +4,9 @@
use crate::{
lex::Lexeme,
+ nodes::Trivia,
parser::{Parse, SyntaxError},
- JsonnetLanguage, SyntaxKind,
+ AstToken, JsonnetLanguage, SyntaxKind,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -144,7 +145,7 @@
}
fn skip_whitespace(&mut self) {
while let Some(lexeme) = self.lexemes.get(self.offset) {
- if !lexeme.kind.is_trivia() {
+ if !Trivia::can_cast(lexeme.kind) {
break;
}
crates/jrsonnet-rowan-parser/src/generated/nodes.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/generated/nodes.rs
+++ b/crates/jrsonnet-rowan-parser/src/generated/nodes.rs
@@ -255,7 +255,7 @@
pub(crate) syntax: SyntaxNode,
}
impl ExprString {
- pub fn string(&self) -> Option<String> {
+ pub fn text(&self) -> Option<Text> {
support::token_child(&self.syntax)
}
}
@@ -332,7 +332,7 @@
pub fn import_kind(&self) -> Option<ImportKind> {
support::token_child(&self.syntax)
}
- pub fn string(&self) -> Option<String> {
+ pub fn text(&self) -> Option<Text> {
support::token_child(&self.syntax)
}
}
@@ -692,7 +692,7 @@
pub fn id(&self) -> Option<Name> {
support::child(&self.syntax)
}
- pub fn string(&self) -> Option<String> {
+ pub fn text(&self) -> Option<Text> {
support::token_child(&self.syntax)
}
}
@@ -1038,18 +1038,27 @@
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct String {
+pub struct Text {
syntax: SyntaxToken,
- kind: StringKind,
+ kind: TextKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum StringKind {
+pub enum TextKind {
StringDouble,
+ ErrorStringDoubleUnterminated,
StringSingle,
+ ErrorStringSingleUnterminated,
StringDoubleVerbatim,
+ ErrorStringDoubleVerbatimUnterminated,
StringSingleVerbatim,
+ ErrorStringSingleVerbatimUnterminated,
+ ErrorStringVerbatimMissingQuotes,
StringBlock,
+ ErrorStringBlockUnexpectedEnd,
+ ErrorStringBlockMissingNewLine,
+ ErrorStringBlockMissingTermination,
+ ErrorStringBlockMissingIndent,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -1061,7 +1070,9 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum NumberKind {
Float,
- MetaForceEnum,
+ ErrorFloatJunkAfterPoint,
+ ErrorFloatJunkAfterExponent,
+ ErrorFloatJunkAfterExponentSign,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -1089,6 +1100,22 @@
Coloncolon,
Colon,
}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Trivia {
+ syntax: SyntaxToken,
+ kind: TriviaKind,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum TriviaKind {
+ Whitespace,
+ MultiLineComment,
+ ErrorCommentTooShort,
+ ErrorCommentUnterminated,
+ SingleLineHashComment,
+ SingleLineSlashComment,
+}
impl AstNode for SourceFile {
fn can_cast(kind: SyntaxKind) -> bool {
kind == SOURCE_FILE
@@ -2677,39 +2704,84 @@
std::fmt::Display::fmt(self.syntax(), f)
}
}
-impl AstToken for String {
+impl AstToken for Text {
fn can_cast(kind: SyntaxKind) -> bool {
match kind {
STRING_DOUBLE
+ | ERROR_STRING_DOUBLE_UNTERMINATED
| STRING_SINGLE
+ | ERROR_STRING_SINGLE_UNTERMINATED
| STRING_DOUBLE_VERBATIM
+ | ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED
| STRING_SINGLE_VERBATIM
- | STRING_BLOCK => true,
+ | ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED
+ | ERROR_STRING_VERBATIM_MISSING_QUOTES
+ | STRING_BLOCK
+ | ERROR_STRING_BLOCK_UNEXPECTED_END
+ | ERROR_STRING_BLOCK_MISSING_NEW_LINE
+ | ERROR_STRING_BLOCK_MISSING_TERMINATION
+ | ERROR_STRING_BLOCK_MISSING_INDENT => true,
_ => false,
}
}
fn cast(syntax: SyntaxToken) -> Option<Self> {
let res = match syntax.kind() {
- STRING_DOUBLE => String {
+ STRING_DOUBLE => Text {
syntax,
- kind: StringKind::StringDouble,
+ kind: TextKind::StringDouble,
},
- STRING_SINGLE => String {
+ ERROR_STRING_DOUBLE_UNTERMINATED => Text {
syntax,
- kind: StringKind::StringSingle,
+ kind: TextKind::ErrorStringDoubleUnterminated,
},
- STRING_DOUBLE_VERBATIM => String {
+ STRING_SINGLE => Text {
syntax,
- kind: StringKind::StringDoubleVerbatim,
+ kind: TextKind::StringSingle,
},
- STRING_SINGLE_VERBATIM => String {
+ ERROR_STRING_SINGLE_UNTERMINATED => Text {
syntax,
- kind: StringKind::StringSingleVerbatim,
+ kind: TextKind::ErrorStringSingleUnterminated,
},
- STRING_BLOCK => String {
+ STRING_DOUBLE_VERBATIM => Text {
syntax,
- kind: StringKind::StringBlock,
+ kind: TextKind::StringDoubleVerbatim,
+ },
+ ERROR_STRING_DOUBLE_VERBATIM_UNTERMINATED => Text {
+ syntax,
+ kind: TextKind::ErrorStringDoubleVerbatimUnterminated,
+ },
+ STRING_SINGLE_VERBATIM => Text {
+ syntax,
+ kind: TextKind::StringSingleVerbatim,
},
+ ERROR_STRING_SINGLE_VERBATIM_UNTERMINATED => Text {
+ syntax,
+ kind: TextKind::ErrorStringSingleVerbatimUnterminated,
+ },
+ ERROR_STRING_VERBATIM_MISSING_QUOTES => Text {
+ syntax,
+ kind: TextKind::ErrorStringVerbatimMissingQuotes,
+ },
+ STRING_BLOCK => Text {
+ syntax,
+ kind: TextKind::StringBlock,
+ },
+ ERROR_STRING_BLOCK_UNEXPECTED_END => Text {
+ syntax,
+ kind: TextKind::ErrorStringBlockUnexpectedEnd,
+ },
+ ERROR_STRING_BLOCK_MISSING_NEW_LINE => Text {
+ syntax,
+ kind: TextKind::ErrorStringBlockMissingNewLine,
+ },
+ ERROR_STRING_BLOCK_MISSING_TERMINATION => Text {
+ syntax,
+ kind: TextKind::ErrorStringBlockMissingTermination,
+ },
+ ERROR_STRING_BLOCK_MISSING_INDENT => Text {
+ syntax,
+ kind: TextKind::ErrorStringBlockMissingIndent,
+ },
_ => return None,
};
Some(res)
@@ -2718,12 +2790,12 @@
&self.syntax
}
}
-impl String {
- pub fn kind(&self) -> StringKind {
+impl Text {
+ pub fn kind(&self) -> TextKind {
self.kind
}
}
-impl std::fmt::Display for String {
+impl std::fmt::Display for Text {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
@@ -2731,7 +2803,10 @@
impl AstToken for Number {
fn can_cast(kind: SyntaxKind) -> bool {
match kind {
- FLOAT | META_FORCE_ENUM => true,
+ FLOAT
+ | ERROR_FLOAT_JUNK_AFTER_POINT
+ | ERROR_FLOAT_JUNK_AFTER_EXPONENT
+ | ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => true,
_ => false,
}
}
@@ -2741,10 +2816,18 @@
syntax,
kind: NumberKind::Float,
},
- META_FORCE_ENUM => Number {
+ ERROR_FLOAT_JUNK_AFTER_POINT => Number {
+ syntax,
+ kind: NumberKind::ErrorFloatJunkAfterPoint,
+ },
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT => Number {
syntax,
- kind: NumberKind::MetaForceEnum,
+ kind: NumberKind::ErrorFloatJunkAfterExponent,
},
+ ERROR_FLOAT_JUNK_AFTER_EXPONENT_SIGN => Number {
+ syntax,
+ kind: NumberKind::ErrorFloatJunkAfterExponentSign,
+ },
_ => return None,
};
Some(res)
@@ -2841,6 +2924,62 @@
std::fmt::Display::fmt(self.syntax(), f)
}
}
+impl AstToken for Trivia {
+ fn can_cast(kind: SyntaxKind) -> bool {
+ match kind {
+ WHITESPACE
+ | MULTI_LINE_COMMENT
+ | ERROR_COMMENT_TOO_SHORT
+ | ERROR_COMMENT_UNTERMINATED
+ | SINGLE_LINE_HASH_COMMENT
+ | SINGLE_LINE_SLASH_COMMENT => true,
+ _ => false,
+ }
+ }
+ fn cast(syntax: SyntaxToken) -> Option<Self> {
+ let res = match syntax.kind() {
+ WHITESPACE => Trivia {
+ syntax,
+ kind: TriviaKind::Whitespace,
+ },
+ MULTI_LINE_COMMENT => Trivia {
+ syntax,
+ kind: TriviaKind::MultiLineComment,
+ },
+ ERROR_COMMENT_TOO_SHORT => Trivia {
+ syntax,
+ kind: TriviaKind::ErrorCommentTooShort,
+ },
+ ERROR_COMMENT_UNTERMINATED => Trivia {
+ syntax,
+ kind: TriviaKind::ErrorCommentUnterminated,
+ },
+ SINGLE_LINE_HASH_COMMENT => Trivia {
+ syntax,
+ kind: TriviaKind::SingleLineHashComment,
+ },
+ SINGLE_LINE_SLASH_COMMENT => Trivia {
+ syntax,
+ kind: TriviaKind::SingleLineSlashComment,
+ },
+ _ => return None,
+ };
+ Some(res)
+ }
+ fn syntax(&self) -> &SyntaxToken {
+ &self.syntax
+ }
+}
+impl Trivia {
+ pub fn kind(&self) -> TriviaKind {
+ self.kind
+ }
+}
+impl std::fmt::Display for Trivia {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ std::fmt::Display::fmt(self.syntax(), f)
+ }
+}
impl std::fmt::Display for Expr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
+++ b/crates/jrsonnet-rowan-parser/src/generated/syntax_kinds.rs
@@ -174,7 +174,6 @@
SELF_KW,
#[token("super")]
SUPER_KW,
- META_FORCE_ENUM,
#[token("for")]
FOR_KW,
#[token("assert")]
@@ -253,10 +252,11 @@
BINARY_OPERATOR,
UNARY_OPERATOR,
LITERAL,
- STRING,
+ TEXT,
NUMBER,
IMPORT_KIND,
VISIBILITY,
+ TRIVIA,
#[doc(hidden)]
__LAST,
}
@@ -277,8 +277,8 @@
pub fn is_enum(self) -> bool {
match self {
EXPR | OBJ_BODY | COMP_SPEC | BIND | MEMBER | FIELD | FIELD_NAME | DESTRUCT
- | DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | STRING
- | NUMBER | IMPORT_KIND | VISIBILITY => true,
+ | DESTRUCT_ARRAY_PART | BINARY_OPERATOR | UNARY_OPERATOR | LITERAL | TEXT | NUMBER
+ | IMPORT_KIND | VISIBILITY | TRIVIA => true,
_ => false,
}
}
crates/jrsonnet-rowan-parser/src/lex.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/lex.rs
+++ b/crates/jrsonnet-rowan-parser/src/lex.rs
@@ -4,7 +4,10 @@
use logos::Logos;
use rowan::{TextRange, TextSize};
-use crate::SyntaxKind;
+use crate::{
+ string_block::{lex_str_block, StringBlockError},
+ SyntaxKind,
+};
pub struct Lexer<'a> {
inner: logos::Lexer<'a, SyntaxKind>,
@@ -22,9 +25,34 @@
type Item = Lexeme<'a>;
fn next(&mut self) -> Option<Self::Item> {
- let kind = self.inner.next()?;
+ use SyntaxKind::*;
+
+ let mut kind = self.inner.next()?;
let text = self.inner.slice();
+ if kind == STRING_BLOCK {
+ // We use custom lexer, which skips enough bytes, but not returns error
+ // Instead we should call lexer again to verify if there is something wrong with string block
+ let mut lexer = logos::Lexer::<SyntaxKind>::new(text);
+ // In kinds, string blocks is parsed at least as `|||`
+ lexer.bump(3);
+ let res = lex_str_block(&mut lexer);
+ debug_assert!(lexer.next().is_none(), "str_block is lexed");
+ match res {
+ Ok(_) => {}
+ Err(e) => {
+ kind = match e {
+ StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
+ StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
+ StringBlockError::MissingTermination => {
+ ERROR_STRING_BLOCK_MISSING_TERMINATION
+ }
+ StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
+ }
+ }
+ }
+ }
+
Some(Self::Item {
kind,
text,
crates/jrsonnet-rowan-parser/src/lib.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/lib.rs
+++ b/crates/jrsonnet-rowan-parser/src/lib.rs
@@ -2,7 +2,6 @@
mod ast;
mod binary;
-mod classify;
mod event;
mod generated;
mod language;
crates/jrsonnet-rowan-parser/src/marker.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/marker.rs
+++ b/crates/jrsonnet-rowan-parser/src/marker.rs
@@ -44,10 +44,10 @@
!kind.is_enum(),
"{kind:?} is a enum kind, you should use variant kinds instead"
);
- // TODO: is_parser should return true if enum variant has #[regex]/#[token] over it
+ // TODO: is_lexer should return true if enum variant has #[regex]/#[token] over it, or it is defined as lexer error explicitly
// debug_assert!(
- // !kind.is_parser(),
- // "{kind:?} should be only emitted by parser, not used directly"
+ // !kind.is_lexer(),
+ // "{kind:?} should be only emitted by lexer, not used directly"
// );
let event_at_pos = &mut p.events[self.start_event_idx];
assert_eq!(*event_at_pos, Event::Pending);
crates/jrsonnet-rowan-parser/src/parser.rsdiffbeforeafterboth--- a/crates/jrsonnet-rowan-parser/src/parser.rs
+++ b/crates/jrsonnet-rowan-parser/src/parser.rs
@@ -8,10 +8,10 @@
event::Event,
lex::Lexeme,
marker::{AsRange, CompletedMarker, Marker, Ranger},
- string_block::{lex_str_block, StringBlockError},
+ nodes::{Literal, Number, Text, Trivia},
token_set::SyntaxKindSet,
unary::UnaryOperator,
- SyntaxKind,
+ AstToken, SyntaxKind,
SyntaxKind::*,
SyntaxNode, T, TS,
};
@@ -36,6 +36,7 @@
}
pub struct Parser<'i> {
+ // TODO: remove all trivia before feeding to parser?
lexemes: &'i [Lexeme<'i>],
pub offset: usize,
pub events: Vec<Event>,
@@ -191,7 +192,7 @@
while self
.lexemes
.get(previous_token_idx)
- .map_or(false, |l| l.kind.is_trivia())
+ .map_or(false, |l| Trivia::can_cast(l.kind))
&& previous_token_idx != 0
{
previous_token_idx -= 1;
@@ -200,13 +201,13 @@
Some(self.lexemes[previous_token_idx])
}
pub fn start_of_token(&self, mut idx: usize) -> TextSize {
- while self.lexemes[idx].kind.is_trivia() {
+ while Trivia::can_cast(self.lexemes[idx].kind) {
idx += 1;
}
self.lexemes[idx].range.start()
}
pub fn end_of_token(&self, mut idx: usize) -> TextSize {
- while self.lexemes[idx].kind.is_trivia() {
+ while Trivia::can_cast(self.lexemes[idx].kind) {
idx -= 1;
}
self.lexemes[idx].range.end()
@@ -267,7 +268,11 @@
self.bump();
Some(m.complete(self, SyntaxKind::ERROR))
}
-
+ fn bump_assert(&mut self, kind: SyntaxKind) {
+ self.skip_trivia();
+ assert!(self.at(kind), "expected {:?}", kind);
+ self.bump_remap(self.current());
+ }
fn bump(&mut self) {
self.skip_trivia();
self.bump_remap(self.current());
@@ -314,7 +319,7 @@
while self
.lexemes
.get(offset)
- .map(|l| l.kind.is_trivia())
+ .map(|l| Trivia::can_cast(l.kind))
.unwrap_or(false)
{
offset += 1;
@@ -324,7 +329,7 @@
while self
.lexemes
.get(offset)
- .map(|l| l.kind.is_trivia())
+ .map(|l| Trivia::can_cast(l.kind))
.unwrap_or(false)
{
offset += 1;
@@ -335,14 +340,10 @@
self.nth(0)
}
fn skip_trivia(&mut self) {
- while self.peek_raw().is_trivia() {
+ while Trivia::can_cast(self.peek_raw()) {
self.offset += 1;
}
}
- fn current_lexeme(&mut self) -> Option<&Lexeme> {
- self.skip_trivia();
- self.lexemes.get(self.offset)
- }
fn peek_raw(&mut self) -> SyntaxKind {
self.lexemes
.get(self.offset)
@@ -516,8 +517,8 @@
} else if p.at(IDENT) {
name(p);
m.complete(p, FIELD_NAME_FIXED);
- } else if p.current().is_string() {
- string(p);
+ } else if Text::can_cast(p.current()) {
+ text(p);
m.complete(p, FIELD_NAME_FIXED);
} else {
p.error_with_recovery_set(TS![;]);
@@ -564,9 +565,8 @@
};
}
fn assertion(p: &mut Parser) {
- assert!(p.at(T![assert]));
let m = p.start();
- p.bump();
+ p.bump_assert(T![assert]);
expr(p).map(|c| c.wrap(p, LHS_EXPR));
if p.at(T![:]) {
p.bump();
@@ -575,10 +575,9 @@
m.complete(p, ASSERTION);
}
fn object(p: &mut Parser) -> CompletedMarker {
- assert!(p.at(T!['{']));
let m_t = p.start();
let m = p.start();
- p.bump();
+ p.bump_assert(T!['{']);
loop {
if p.at(T!['}']) {
@@ -619,9 +618,8 @@
m.complete(p, PARAM);
}
fn params_desc(p: &mut Parser) -> CompletedMarker {
- assert!(p.at(T!['(']));
let m = p.start();
- p.bump();
+ p.bump_assert(T!['(']);
loop {
if p.at(T![')']) {
@@ -640,8 +638,7 @@
}
fn args_desc(p: &mut Parser) {
let m = p.start();
- assert!(p.at(T!['(']));
- p.bump();
+ p.bump_assert(T!['(']);
let started_named = Cell::new(false);
@@ -674,10 +671,9 @@
}
fn array(p: &mut Parser) -> CompletedMarker {
- assert!(p.at(T!['[']));
// Start the list node
let m = p.start();
- p.bump(); // '['
+ p.bump_assert(T!['[']);
// This vec will have at most one element in case of correct input
let mut compspecs = Vec::with_capacity(1);
@@ -795,9 +791,8 @@
m.complete(p, NAME);
}
fn destruct_rest(p: &mut Parser) {
- assert!(p.at(T![...]));
- p.bump();
let m = p.start();
+ p.bump_assert(T![...]);
if p.at(IDENT) {
p.bump()
}
@@ -817,9 +812,8 @@
m.complete(p, DESTRUCT_OBJECT_FIELD);
}
fn obj_local(p: &mut Parser) {
- assert!(p.at(T![local]));
let m = p.start();
- p.bump();
+ p.bump_assert(T![local]);
bind(p);
m.complete(p, OBJ_LOCAL);
}
@@ -903,52 +897,29 @@
m.complete(p, BIND_DESTRUCT)
};
}
-fn string(p: &mut Parser) {
- assert!(p.current().is_string());
- if p.at(STRING_BLOCK) {
- // We use custom lexer, which skips enough bytes, but not returns error
- // Instead we should call lexer again to verify if there is something wrong with string block
- let mut lexer = logos::Lexer::<SyntaxKind>::new(dbg!(
- &p.current_lexeme().expect("parser is at string block").text
- ));
- // In kinds, string blocks is parsed at least as `|||`
- lexer.bump(3);
- let res = lex_str_block(&mut lexer);
- debug_assert!(lexer.next().is_none(), "str_block is lexed");
- match res {
- Ok(_) => {
- p.bump();
- }
- Err(e) => p.bump_remap(match e {
- StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,
- StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,
- StringBlockError::MissingTermination => ERROR_STRING_BLOCK_MISSING_TERMINATION,
- StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,
- }),
- }
- } else {
- p.bump();
- }
+fn text(p: &mut Parser) {
+ assert!(Text::can_cast(p.current()));
+ p.bump();
}
fn number(p: &mut Parser) {
- assert!(p.current().is_number());
+ assert!(Number::can_cast(p.current()));
p.bump();
}
fn literal(p: &mut Parser) {
- assert!(p.current().is_literal());
+ assert!(Literal::can_cast(p.current()));
p.bump();
}
fn lhs_basic(p: &mut Parser) -> Option<CompletedMarker> {
let _e = p.expected_syntax_name("value");
- Some(if p.current().is_literal() {
+ Some(if Literal::can_cast(p.current()) {
let m = p.start();
literal(p);
m.complete(p, EXPR_LITERAL)
- } else if p.current().is_string() {
+ } else if Text::can_cast(p.current()) {
let m = p.start();
- string(p);
+ text(p);
m.complete(p, EXPR_STRING)
- } else if p.current().is_number() {
+ } else if Number::can_cast(p.current()) {
let m = p.start();
number(p);
m.complete(p, EXPR_NUMBER)
@@ -1025,7 +996,7 @@
} else if p.at(T![import]) || p.at(T![importstr]) || p.at(T![importbin]) {
let m = p.start();
p.bump();
- string(p);
+ text(p);
m.complete(p, EXPR_IMPORT)
} else if p.at(T![-]) || p.at(T![!]) || p.at(T![~]) {
let op = match p.current() {
@@ -1044,8 +1015,7 @@
let m = p.start();
p.bump();
expr(p);
- assert!(p.at(T![')']));
- p.bump();
+ p.expect(T![')']);
m.complete(p, EXPR_PARENED)
} else {
p.error_with_recovery_set(TS![]);
xtask/src/sourcegen/kinds.rsdiffbeforeafterboth--- a/xtask/src/sourcegen/kinds.rs
+++ b/xtask/src/sourcegen/kinds.rs
@@ -10,10 +10,12 @@
pub enum TokenKind {
/// May exist in token tree, but never in source code
Meta { grammar_name: String, name: String },
- /// Specific parsing errors may be emitted as this type of kind
+ /// Specific parsing/lexing errors may be emitted as this type of kind
Error {
grammar_name: String,
name: String,
+ /// Is this error returned by lexer directly, or from lex.rs
+ is_lexer_error: bool,
regex: Option<String>,
priority: Option<u32>,
},
@@ -133,13 +135,18 @@
});
$(define_kinds!($into = $($rest)*))?
}};
- ($into:ident = error($name:literal$(, priority = $priority:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
- $into.define_token(TokenKind::Error {
- grammar_name: format!("ERROR_{}!", $name),
- name: format!("ERROR_{}", $name),
- regex: None$(.or(Some($regex.to_owned())))?,
- priority: None$(.or(Some($priority)))?,
- });
+ ($into:ident = error($name:literal$(, priority = $priority:literal)? $(, lexer = $lexer:literal)?) $(=> $regex:literal)? $(; $($rest:tt)*)?) => {{
+ {
+ let regex = None$(.or(Some($regex.to_owned())))?;
+ let priority = None$(.or(Some($priority)))?;
+ $into.define_token(TokenKind::Error {
+ grammar_name: format!("ERROR_{}!", $name),
+ name: format!("ERROR_{}", $name),
+ is_lexer_error: false $(|| $lexer)? || regex.is_some() || priority.is_some(),
+ regex,
+ priority,
+ });
+ }
$(define_kinds!($into = $($rest)*))?
}};
($into:ident = $tok:literal => $name:literal $(; $($rest:tt)*)?) => {{
@@ -258,10 +265,10 @@
error("STRING_SINGLE_VERBATIM_UNTERMINATED") => "@'(?:[^']|'')*";
error("STRING_VERBATIM_MISSING_QUOTES") => "@[^\"'\\s]\\S+";
lit("STRING_BLOCK") => r"\|\|\|", "crate::string_block::lex_str_block_test";
- error("STRING_BLOCK_UNEXPECTED_END");
- error("STRING_BLOCK_MISSING_NEW_LINE");
- error("STRING_BLOCK_MISSING_TERMINATION");
- error("STRING_BLOCK_MISSING_INDENT");
+ error("STRING_BLOCK_UNEXPECTED_END", lexer = true);
+ error("STRING_BLOCK_MISSING_NEW_LINE", lexer = true);
+ error("STRING_BLOCK_MISSING_TERMINATION", lexer = true);
+ error("STRING_BLOCK_MISSING_INDENT", lexer = true);
lit("IDENT") => r"[_a-zA-Z][_a-zA-Z0-9]*";
lit("WHITESPACE") => r"[ \t\n\r]+";
lit("SINGLE_LINE_SLASH_COMMENT") => r"//[^\r\n]*(\r\n|\n)?";
xtask/src/sourcegen/mod.rsdiffbeforeafterboth--- a/xtask/src/sourcegen/mod.rs
+++ b/xtask/src/sourcegen/mod.rs
@@ -48,20 +48,28 @@
if let Some((special, name)) = classify_special(token) {
match special {
SpecialName::Literal => panic!("literal is not defined: {name}"),
- SpecialName::Meta => kinds.define_token(TokenKind::Meta {
- grammar_name: token.to_owned(),
- name: format!("META_{}", name),
- }),
- SpecialName::Error => kinds.define_token(TokenKind::Error {
- grammar_name: token.to_owned(),
- name: format!("ERROR_{}", name),
- regex: None,
- priority: None,
- }),
+ SpecialName::Meta => {
+ eprintln!("implicit meta: {}", name);
+ kinds.define_token(TokenKind::Meta {
+ grammar_name: token.to_owned(),
+ name: format!("META_{}", name),
+ })
+ }
+ SpecialName::Error => {
+ eprintln!("implicit error: {}", name);
+ kinds.define_token(TokenKind::Error {
+ grammar_name: token.to_owned(),
+ name: format!("ERROR_{}", name),
+ regex: None,
+ priority: None,
+ is_lexer_error: true,
+ })
+ }
};
continue;
};
let name = to_upper_snake_case(token);
+ eprintln!("implicit kw: {}", token);
kinds.define_token(TokenKind::Keyword {
code: token.to_owned(),
name: format!("{name}_KW"),
xtask/src/sourcegen/util.rsdiffbeforeafterboth--- a/xtask/src/sourcegen/util.rs
+++ b/xtask/src/sourcegen/util.rs
@@ -13,10 +13,7 @@
}
}
- eprintln!(" {} was not up-to-date, updating\n", file.display());
- if std::env::var("CI").is_ok() {
- eprintln!("NOTE: run `cargo xtask` locally and commit the updated files\n");
- }
+ eprintln!("{} was not up-to-date, updating", file.display());
if let Some(parent) = file.parent() {
let _ = fs::create_dir_all(parent);
}