--- a/.cargo/config +++ /dev/null @@ -1,2 +0,0 @@ -[alias] -xtask = "run --manifest-path ./xtask/Cargo.toml --" --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --bin xtask --" --- a/cmds/jrsonnet-fmt/Cargo.toml +++ b/cmds/jrsonnet-fmt/Cargo.toml @@ -6,3 +6,5 @@ [dependencies] dprint-core = "0.58.2" jrsonnet-rowan-parser = { path = "../../crates/jrsonnet-rowan-parser" } +insta = "1.15" +indoc = "1.0" --- /dev/null +++ b/cmds/jrsonnet-fmt/src/children.rs @@ -0,0 +1,168 @@ +use std::{fmt::Debug, marker::PhantomData, mem}; + +use jrsonnet_rowan_parser::{ + nodes::{Trivia, TriviaKind}, + AstNode, AstToken, SyntaxElement, + SyntaxKind::*, + SyntaxNode, TS, +}; + +pub type ChildTrivia = Vec; + +pub struct ChildIterator { + inner: I, + _marker: PhantomData, +} + +pub fn children_between( + node: SyntaxNode, + start: Option<&SyntaxElement>, + end: Option<&SyntaxElement>, +) -> (Vec>, ChildTrivia) { + let mut iter = node.children_with_tokens().peekable(); + while iter.peek() == start { + iter.next(); + } + children( + iter.take_while(|i| Some(i) != end), + start.is_none() || end.is_none(), + ) +} + +pub fn should_start_with_newline(tt: &ChildTrivia) -> bool { + // First for previous item end + count_newlines_before(&tt) >= 2 +} + +fn count_newlines_before(tt: &ChildTrivia) -> usize { + let mut nl_count = 0; + for t in tt { + match t.kind() { + TriviaKind::Whitespace => { + nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); + } + _ => break, + } + } + nl_count +} +fn count_newlines_after(tt: &ChildTrivia) -> usize { + let mut nl_count = 0; + for t in tt.iter().rev() { + match t.kind() { + TriviaKind::Whitespace => { + nl_count += t.text().bytes().filter(|b| *b == b'\n').count(); + } + TriviaKind::SingleLineHashComment => { + nl_count += 1; + break; + } + TriviaKind::SingleLineSlashComment => { + nl_count += 1; + break; + } + _ => {} + } + } + nl_count +} + +pub fn children<'a, T: AstNode + Debug>( + items: impl Iterator, + loose: bool, +) -> (Vec>, ChildTrivia) { + let mut out = Vec::new(); + let mut current_child = None::>; + let mut next = ChildTrivia::new(); + // Previous element ended, do not add more inline comments + let mut started_next = false; + let mut had_some = false; + + for item in items { + if let Some(value) = item.as_node().cloned().and_then(T::cast) { + let before_trivia = mem::take(&mut next); + let last_child = current_child.replace(Child { + newlines_above: if had_some { + count_newlines_before(&before_trivia) + + current_child + .as_ref() + .map(|c| count_newlines_after(&c.inline_trivia)) + .unwrap_or_default() + } else { + 0 + }, + before_trivia, + value, + inline_trivia: Vec::new(), + }); + if let Some(last_child) = last_child { + out.push(last_child) + } + had_some = true; + started_next = false; + } else if let Some(trivia) = item.as_token().cloned().and_then(Trivia::cast) { + let is_single_line_comment = trivia.kind() == TriviaKind::SingleLineHashComment + || trivia.kind() == TriviaKind::SingleLineSlashComment; + if started_next + || current_child.is_none() + || trivia.text().contains('\n') && !is_single_line_comment + { + next.push(trivia.clone()); + started_next = true; + } else { + let cur = current_child.as_mut().expect("checked not none"); + cur.inline_trivia.push(trivia); + if is_single_line_comment { + started_next = true; + } + } + had_some = true; + } else if loose { + if had_some { + break; + } + started_next = true; + } else { + assert!( + TS![, ;].contains(item.kind()) || item.kind() == ERROR, + "silently eaten token: {:?}", + item.kind() + ) + } + } + + if let Some(current_child) = current_child { + out.push(current_child); + } + + (out, next) +} + +#[derive(Debug)] +pub struct Child { + newlines_above: usize, + /// Comment before item, i.e + /// + /// ```ignore + /// // Comment + /// item + /// ``` + pub before_trivia: ChildTrivia, + pub value: T, + /// Comment after line, but located at same line + /// + /// ```ignore + /// item1, // Inline comment + /// // Not inline comment + /// item2, + /// ``` + pub inline_trivia: ChildTrivia, +} + +impl Child { + /// If this child has two newlines above in source code, so it needs to have it in output + pub fn needs_newline_above(&self) -> bool { + // First line for end of previous item + self.newlines_above >= 2 + } +} --- /dev/null +++ b/cmds/jrsonnet-fmt/src/comments.rs @@ -0,0 +1,159 @@ +use dprint_core::formatting::PrintItems; +use jrsonnet_rowan_parser::{nodes::TriviaKind, AstToken}; + +use crate::{children::ChildTrivia, p, pi}; + +pub enum CommentLocation { + /// Above local, field, other things + AboveItem, + /// After item + ItemInline, + /// After all items in object + EndOfItems, +} + +pub fn format_comments(comments: &ChildTrivia, loc: CommentLocation) -> PrintItems { + let mut pi = p!(new:); + + for c in comments { + match c.kind() { + TriviaKind::Whitespace => {} + TriviaKind::MultiLineComment => { + let mut text = c + .text() + .strip_prefix("/*") + .expect("ml comment starts with /*") + .strip_suffix("*/") + .expect("ml comment ends with */"); + // doc-style comment, /** + let doc = if text.starts_with('*') { + text = &text[1..]; + true + } else { + false + }; + // Is comment starts with text immediatly, i.e /*text + let mut immediate_start = true; + let mut lines = text + .split('\n') + .map(|l| l.trim_end()) + .skip_while(|l| { + if l.is_empty() { + immediate_start = false; + true + } else { + false + } + }) + .collect::>(); + while lines.last().map(|l| l.is_empty()).unwrap_or(false) { + lines.pop(); + } + if lines.len() == 1 && !doc { + p!(pi: str("/* ") str(lines[0].trim()) str(" */") nl) + } else if !lines.is_empty() { + fn common_ws_prefix<'a>(a: &'a str, b: &str) -> &'a str { + let offset = a + .bytes() + .zip(b.bytes()) + .take_while(|(a, b)| a == b && (a.is_ascii_whitespace() || *a == b'*')) + .count(); + &a[..offset] + } + // First line is not empty, extract ws prefix of it + let mut common_ws_padding = if immediate_start && lines.len() > 1 { + common_ws_prefix(lines[1], lines[1]) + } else { + common_ws_prefix(lines[0], lines[0]) + }; + for line in lines + .iter() + .skip(if immediate_start { 2 } else { 1 }) + .filter(|l| !l.is_empty()) + { + common_ws_padding = common_ws_prefix(common_ws_padding, line); + } + for line in lines + .iter_mut() + .skip(if immediate_start { 1 } else { 0 }) + .filter(|l| !l.is_empty()) + { + *line = line + .strip_prefix(common_ws_padding) + .expect("all non-empty lines start with this padding"); + } + + p!(pi: str("/*")); + if doc { + p!(pi: str("*")); + } + p!(pi: nl); + for mut line in lines { + if doc { + p!(pi: str(" *")); + } + if line.is_empty() { + p!(pi: nl); + } else { + if doc { + p!(pi: str(" ")); + } + while let Some(new_line) = line.strip_prefix('\t') { + if doc { + p!(pi: str(" ")); + } else { + p!(pi: tab); + } + line = new_line; + } + p!(pi: str(line) nl) + } + } + if doc { + p!(pi: str(" ")); + } + p!(pi: str("*/") nl) + } + } + // TODO: Keep common padding for multiple continous lines of single-line comments + // I.e + // ``` + // # Line1 + // # Line2 + // ``` + // Should be reformatted as + // ``` + // # Line1 + // # Line2 + // ``` + // But currently comment formatter is not aware of continous comment lines, and reformats it as + // ``` + // # Line1 + // # Line2 + // ``` + TriviaKind::SingleLineHashComment => { + if matches!(loc, CommentLocation::ItemInline) { + p!(pi: str(" ")) + } + p!(pi: str("# ") str(c.text().strip_prefix('#').expect("hash comment starts with #").trim())); + if !matches!(loc, CommentLocation::ItemInline) { + p!(pi: nl) + } + } + TriviaKind::SingleLineSlashComment => { + if matches!(loc, CommentLocation::ItemInline) { + p!(pi: str(" ")) + } + p!(pi: str("// ") str(c.text().strip_prefix("//").expect("comment starts with //").trim())); + if !matches!(loc, CommentLocation::ItemInline) { + p!(pi: nl) + } + } + // Garbage in - garbage out + TriviaKind::ErrorCommentTooShort => p!(pi: str("/*/")), + TriviaKind::ErrorCommentUnterminated => p!(pi: str(c.text())), + } + } + + pi +} --- a/cmds/jrsonnet-fmt/src/main.rs +++ b/cmds/jrsonnet-fmt/src/main.rs @@ -1,6 +1,7 @@ use std::any::type_name; -use dprint_core::formatting::{PrintItems, PrintOptions, Signal}; +use children::children_between; +use dprint_core::formatting::{PrintItems, PrintOptions}; use jrsonnet_rowan_parser::{ nodes::{ ArgsDesc, Assertion, BinaryOperator, Bind, CompSpec, Destruct, DestructArrayPart, @@ -8,9 +9,19 @@ Member, Name, Number, ObjBody, ObjLocal, ParamsDesc, SliceDesc, SourceFile, Text, UnaryOperator, }, - AstToken, SyntaxToken, + AstNode, AstToken, SyntaxToken, }; +use crate::{ + children::should_start_with_newline, + comments::{format_comments, CommentLocation}, +}; + +mod children; +mod comments; +#[cfg(test)] +mod tests; + pub trait Printable { fn print(&self) -> PrintItems; } @@ -18,7 +29,7 @@ macro_rules! pi { (@i; $($t:tt)*) => {{ #[allow(unused_mut)] - let mut o = PrintItems::new(); + let mut o = dprint_core::formatting::PrintItems::new(); pi!(@s; o: $($t)*); o }}; @@ -27,21 +38,29 @@ pi!(@s; $o: $($t)*); }}; (@s; $o:ident: nl $($t:tt)*) => {{ - $o.push_signal(Signal::NewLine); + $o.push_signal(dprint_core::formatting::Signal::NewLine); pi!(@s; $o: $($t)*); }}; + (@s; $o:ident: tab $($t:tt)*) => {{ + $o.push_signal(dprint_core::formatting::Signal::Tab); + pi!(@s; $o: $($t)*); + }}; (@s; $o:ident: >i $($t:tt)*) => {{ - $o.push_signal(Signal::StartIndent); + $o.push_signal(dprint_core::formatting::Signal::StartIndent); pi!(@s; $o: $($t)*); }}; (@s; $o:ident: {{ - $o.push_signal(Signal::FinishIndent); + $o.push_signal(dprint_core::formatting::Signal::FinishIndent); pi!(@s; $o: $($t)*); }}; (@s; $o:ident: {$expr:expr} $($t:tt)*) => {{ $o.extend($expr.print()); pi!(@s; $o: $($t)*); }}; + (@s; $o:ident: items($expr:expr) $($t:tt)*) => {{ + $o.extend($expr); + pi!(@s; $o: $($t)*); + }}; (@s; $o:ident: if ($e:expr)($($then:tt)*) $($t:tt)*) => {{ if $e { pi!(@s; $o: $($then)*); @@ -66,6 +85,8 @@ pi!(@s; $o: $($t)*) }; } +pub(crate) use p; +pub(crate) use pi; impl

Printable for Option

where @@ -266,9 +287,18 @@ match self { ObjBody::ObjBodyComp(_) => todo!(), ObjBody::ObjBodyMemberList(l) => { - let mut pi = p!(new:); - for mem in l.members() { - match mem { + let mut pi = p!(new: str("{") >i nl); + let (children, end_comments) = children_between::( + l.syntax().clone(), + l.l_brace_token().map(Into::into).as_ref(), + l.r_brace_token().map(Into::into).as_ref(), + ); + for mem in children.into_iter() { + if mem.needs_newline_above() { + p!(pi: nl); + } + p!(pi: items(format_comments(&mem.before_trivia, CommentLocation::AboveItem))); + match mem.value { Member::MemberBindStmt(b) => { p!(pi: {b.obj_local()}) } @@ -279,8 +309,17 @@ p!(pi: {f.field()}) } } - p!(pi: str(",") nl) + p!(pi: str(",")); + p!(pi: items(format_comments(&mem.inline_trivia, CommentLocation::ItemInline))); + p!(pi: nl) } + + // TODO: implement same thing as needs_newline_above, but for end comments + if should_start_with_newline(&end_comments) { + p!(pi: nl); + } + p!(pi: items(format_comments(&end_comments, CommentLocation::EndOfItems))); + p!(pi: { - p!(new: str("{") >i nl {o.obj_body()} { let mut pi = p!(new: str("[") {arr.expr()}); @@ -485,6 +524,47 @@ ], m: a[1::], m: b[::], + + comments: { + _: '', + // Plain comment + a: '', + + # Plain comment with empty line before + b: '', + /*Single-line multiline comment + + */ + c: '', + + /**Single-line multiline doc comment + + */ + c: '', + + /**multiline doc comment + s + */ + c: '', + + /* + + Multi-line + + comment + */ + d: '', + + e: '', // Inline comment + + k: '', + + // Text after everything + }, + comments2: { + k: '', + // Text after everything, but no newline above + }, k: if a == b then --- /dev/null +++ b/cmds/jrsonnet-fmt/src/snapshots/jrsonnet_fmt__tests__complex_comments_snapshot.snap @@ -0,0 +1,53 @@ +--- +source: cmds/jrsonnet-fmt/src/tests.rs +expression: "reformat(indoc!(\"{\n\t\t comments: {\n\t\t\t_: '',\n\t\t\t// Plain comment\n\t\t\ta: '',\n\n\t\t\t# Plain comment with empty line before\n\t\t\tb: '',\n\t\t\t/*Single-line multiline comment\n\n\t\t\t*/\n\t\t\tc: '',\n\n\t\t\t/**Single-line multiline doc comment\n\n\t\t\t*/\n\t\t\tc: '',\n\n\t\t\t/**Multiline doc\n\t\t\tComment\n\t\t\t*/\n\t\t\tc: '',\n\n\t\t\t/*\n\n\tMulti-line\n\n\tcomment\n\t\t\t*/\n\t\t\td: '',\n\n\t\t\te: '', // Inline comment\n\n\t\t\tk: '',\n\n\t\t\t// Text after everything\n\t\t },\n\t\t comments2: {\n\t\t\tk: '',\n\t\t\t// Text after everything, but no newline above\n\t\t },\n spacing: {\n a: '',\n\n b: '',\n },\n noSpacing: {\n a: '',\n b: '',\n },\n }\"))" +--- +{ + comments: { + _: '', + // Plain comment + a: '', + + # Plain comment with empty line before + b: '', + /* Single-line multiline comment */ + c: '', + + /** + * Single-line multiline doc comment + */ + c: '', + + /** + * Multiline doc + * Comment + */ + c: '', + + /* + Multi-line + + comment + */ + d: '', + + e: '', // Inline comment + + k: '', + + // Text after everything + }, + comments2: { + k: '', + // Text after everything, but no newline above + }, + spacing: { + a: '', + + b: '', + }, + noSpacing: { + a: '', + b: '', + }, +} --- /dev/null +++ b/cmds/jrsonnet-fmt/src/tests.rs @@ -0,0 +1,124 @@ +use dprint_core::formatting::PrintOptions; +use indoc::indoc; + +use crate::Printable; + +fn reformat(input: &str) -> String { + let (source, _) = jrsonnet_rowan_parser::parse(input); + + dprint_core::formatting::format( + || source.print(), + PrintOptions { + indent_width: 2, + max_width: 100, + use_tabs: false, + new_line_text: "\n", + }, + ) +} + +macro_rules! assert_formatted { + ($input:literal, $output:literal) => { + let formatted = reformat(indoc!($input)); + let expected = indoc!($output); + if formatted != expected { + panic!( + "bad formatting, expected\n```\n{formatted}\n```\nto be equal to\n```\n{expected}\n```", + ) + } + }; +} + +#[test] +fn padding_stripped_for_multiline_comment() { + assert_formatted!( + "{ + /* + Hello + World + */ + _: null, + }", + "{ + /* + Hello + World + */ + _: null, + }" + ); +} + +// Fails +#[test] +fn last_comment_respects_spacing_with_inline_comment_above() { + assert_formatted!( + "{ + a: '', // Inline + + // Comment + }", + "{ + a: '', // Inline + + // Comment + }" + ); +} + +#[test] +fn complex_comments_snapshot() { + insta::assert_display_snapshot!(reformat(indoc!( + "{ + comments: { + _: '', + // Plain comment + a: '', + + # Plain comment with empty line before + b: '', + /*Single-line multiline comment + + */ + c: '', + + /**Single-line multiline doc comment + + */ + c: '', + + /**Multiline doc + Comment + */ + c: '', + + /* + + Multi-line + + comment + */ + d: '', + + e: '', // Inline comment + + k: '', + + // Text after everything + }, + comments2: { + k: '', + // Text after everything, but no newline above + }, + spacing: { + a: '', + + b: '', + }, + noSpacing: { + a: '', + b: '', + }, + }" + ))) +} --- /dev/null +++ b/cmds/jrsonnet-lsp/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "jrsonnet-lsp" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.48" +jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator" } +jrsonnet-rowan-parser = { path = "../../crates/jrsonnet-rowan-parser" } +lsp-server = "0.6.0" +lsp-types = "0.93.0" +serde = "1.0.130" +serde_json = "1.0.71" --- /dev/null +++ b/cmds/jrsonnet-lsp/src/main.rs @@ -0,0 +1,188 @@ +use std::{fs::File, io::Write, path::PathBuf, str::FromStr}; + +use lsp_server::{Connection, ErrorCode, Message, Request, RequestId, Response}; +use lsp_types::{ + notification::{DidChangeTextDocument, DidOpenTextDocument, Notification}, + request::{DocumentLinkRequest, HoverRequest}, + CompletionOptions, DidChangeTextDocumentParams, DidOpenTextDocumentParams, DocumentLink, + DocumentLinkOptions, ServerCapabilities, TextDocumentSyncCapability, TextDocumentSyncKind, + TextDocumentSyncOptions, Url, WorkDoneProgressOptions, +}; + +fn main() { + let mut log = File::create("test").unwrap(); + writeln!(log, "start").unwrap(); + let (connection, io_threads) = Connection::stdio(); + let capabilities = serde_json::to_value(&ServerCapabilities { + completion_provider: Some(CompletionOptions::default()), + definition_provider: Some(lsp_types::OneOf::Left(true)), + document_link_provider: Some(DocumentLinkOptions { + resolve_provider: Some(false), + work_done_progress_options: WorkDoneProgressOptions::default(), + }), + hover_provider: Some(lsp_types::HoverProviderCapability::Simple(true)), + text_document_sync: Some(TextDocumentSyncCapability::Options( + TextDocumentSyncOptions { + change: Some(TextDocumentSyncKind::FULL), + open_close: Some(true), + ..TextDocumentSyncOptions::default() + }, + )), + ..ServerCapabilities::default() + }) + .expect("failed to convert capabilities to json"); + + connection + .initialize(capabilities) + .expect("failed to initialize connection"); + + writeln!(log, "initialized").unwrap(); + + main_loop(&mut log, &connection).expect("main loop failed"); + + io_threads.join().expect("failed to join io_threads"); +} +fn main_loop(log: &mut File, connection: &Connection) -> anyhow::Result<()> { + // let mut es = EvaluationState::default(); + // es.set_import_resolver(Box::new(FileImportResolver::default())); + + let reply = |response: Response| { + connection + .sender + .send(Message::Response(response)) + .expect("failed to respond"); + }; + + for msg in &connection.receiver { + match msg { + Message::Response(_) => (), + Message::Request(req) => { + if connection.handle_shutdown(&req)? { + return Ok(()); + } + if let Some((id, params)) = cast::(&req) { + reply(Response::new_ok(id, >::new())); + } else if let Some((id, params)) = cast::(&req) { + let pos = params + .text_document_position_params + .text_document + .uri + .path(); + let buf = PathBuf::from_str(pos).unwrap(); + // let pos = es + // .map_from_source_location( + // &buf, + // params.text_document_position_params.position.line as usize + 1, + // params.text_document_position_params.position.character as usize + 1, + // ) + // .unwrap(); + // let el = ExprLocation(buf.clone().into(), pos as usize, pos as usize); + // let es2 = es.clone(); + // reply(Response::new_ok( + // id, + // Some(Hover { + // range: None, + // contents: HoverContents::Markup(MarkupContent { + // kind: MarkupKind::Markdown, + // value: es + // .run_in_state_with_breakpoint(el, move || { + // es2.reset_evaluation_state(&buf); + // es2.import_file(&PathBuf::new(), &buf)? + // .to_string() + // .map(|_| ()) + // }) + // .unwrap() + // .unwrap_or_else(|| Val::Null) + // .value_type() + // .to_string(), + // }), + // }), + // )); + } else { + reply(Response::new_err( + req.id, + ErrorCode::MethodNotFound as i32, + format!("unrecognized request {}", req.method), + )) + } + /* + if let Some((id, params)) = cast::(&req) { + let links = handle_links(&files, params).unwrap_or_default(); + reply(Response::new_ok(id, links)); + } else if let Some((id, params)) = cast::(&req) { + if let Some(loc) = handle_goto(&files, params) { + reply(Response::new_ok(id, loc)) + } else { + reply(Response::new_ok(id, ())) + } + } else if let Some((id, params)) = cast::(&req) { + match handle_hover(&files, params) { + Some((range, markdown)) => { + reply(Response::new_ok( + id, + Hover { + contents: HoverContents::Markup(MarkupContent { + kind: MarkupKind::Markdown, + value: markdown, + }), + range, + }, + )); + } + None => { + reply(Response::new_ok(id, ())); + } + } + } else if let Some((id, params)) = cast::(&req) { + let completions = handle_completion(&files, params.text_document_position) + .unwrap_or_default(); + reply(Response::new_ok(id, completions)); + } else + */ + } + Message::Notification(req) => { + let mut handle = |text: String, uri: Url| { + writeln!(log, "updated file: {:?}", uri).unwrap(); + let path = match PathBuf::from_str(uri.path()) { + Ok(x) => x, + Err(_) => return, + }; + let (ast, errors) = jrsonnet_rowan_parser::parse(&text); + // es.add_parsed_file(path.into(), text.into(), parsed) + // .unwrap(); + writeln!(log, "parsed: {:?}", uri).unwrap(); + }; + + match &*req.method { + DidOpenTextDocument::METHOD => { + let params: DidOpenTextDocumentParams = + match serde_json::from_value(req.params) { + Ok(x) => x, + Err(_) => continue, + }; + handle(params.text_document.text, params.text_document.uri); + } + DidChangeTextDocument::METHOD => { + let params: DidChangeTextDocumentParams = + match serde_json::from_value(req.params) { + Ok(x) => x, + Err(_) => continue, + }; + for change in params.content_changes.into_iter() { + handle(change.text, params.text_document.uri.clone()); + } + } + _ => continue, + } + } + } + } + Ok(()) +} +fn cast(req: &Request) -> Option<(RequestId, R::Params)> +where + R: lsp_types::request::Request, + R::Params: serde::de::DeserializeOwned, +{ + req.clone().extract(R::METHOD).ok() +} --- a/crates/jrsonnet-rowan-parser/Cargo.toml +++ b/crates/jrsonnet-rowan-parser/Cargo.toml @@ -4,19 +4,19 @@ edition = "2021" [dependencies] -anyhow = "1.0.51" +anyhow = "1.0" backtrace = "0.3.63" drop_bomb = "0.1.5" -indoc = "1.0.3" -logos = "0.12.0" -miette = { version = "4.2.1", features = ["fancy"] } -rowan = "0.15.0" -text-size = "1.1.0" -thiserror = "1.0.30" +indoc = "1.0" +logos = "0.12" +miette = { version = "4.2", features = ["fancy"] } +rowan = "0.15" +text-size = "1.1" +thiserror = "1.0" [dev-dependencies] backtrace = "0.3.63" -indoc = "1.0.3" -insta = "1.10.0" -anyhow = "1.0.57" +indoc = "1.0" +insta = "1.15" +anyhow = "1.0" jrsonnet-stdlib = { path = "../jrsonnet-stdlib" } --- a/crates/jrsonnet-rowan-parser/jsonnet.ungram +++ b/crates/jrsonnet-rowan-parser/jsonnet.ungram @@ -56,9 +56,7 @@ (Expr (',' Expr)* ','?)? ']' ExprObject = - '{' ObjBody - '}' ExprArrayComp = '[' Expr @@ -168,6 +166,7 @@ (name:Name '=')? Expr ObjBodyComp = + '{' pre:ObjLocalPostComma* '[' key:LhsExpr @@ -177,8 +176,11 @@ value:Expr post:ObjLocalPreComma* CompSpec* + '}' ObjBodyMemberList = + '{' (Member (',' Member)* ','?)? + '}' ObjBody = ObjBodyComp | ObjBodyMemberList --- a/crates/jrsonnet-rowan-parser/src/event.rs +++ b/crates/jrsonnet-rowan-parser/src/event.rs @@ -24,6 +24,10 @@ Token { kind: SyntaxKind, }, + /// Push token, but do not eat anything, + VirtualToken { + kind: SyntaxKind, + }, /// Position of finished node Finish { /// Same as forward_parent of Start, but for wrapping @@ -105,6 +109,13 @@ self.token(kind); eat_start_whitespace = true; } + Event::VirtualToken { kind } => { + if eat_start_whitespace { + self.skip_whitespace(); + } + self.virtual_token(kind); + eat_start_whitespace = false; + } Event::Finish { wrapper } => { self.builder.finish_node(); depth -= 1; @@ -124,7 +135,7 @@ } eat_start_whitespace = true; } - Event::Pending => panic!("placeholder should not end in events"), + Event::Pending => panic!("pending event should not appear in finished events"), Event::Noop => {} Event::Error(e) => { self.errors.push(e); @@ -137,6 +148,9 @@ errors: self.errors, } } + fn virtual_token(&mut self, kind: SyntaxKind) { + self.builder.token(JsonnetLanguage::kind_to_raw(kind), "") + } fn token(&mut self, kind: SyntaxKind) { let lexeme = self.lexemes[self.offset]; self.builder --- a/crates/jrsonnet-rowan-parser/src/generated/nodes.rs +++ b/crates/jrsonnet-rowan-parser/src/generated/nodes.rs @@ -291,15 +291,9 @@ pub(crate) syntax: SyntaxNode, } impl ExprObject { - pub fn l_brace_token(&self) -> Option { - support::token(&self.syntax, T!['{']) - } pub fn obj_body(&self) -> Option { support::child(&self.syntax) } - pub fn r_brace_token(&self) -> Option { - support::token(&self.syntax, T!['}']) - } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -538,6 +532,9 @@ pub(crate) syntax: SyntaxNode, } impl ObjBodyComp { + pub fn l_brace_token(&self) -> Option { + support::token(&self.syntax, T!['{']) + } pub fn pre(&self) -> AstChildren { support::children(&self.syntax) } @@ -565,6 +562,9 @@ pub fn comp_specs(&self) -> AstChildren { support::children(&self.syntax) } + pub fn r_brace_token(&self) -> Option { + support::token(&self.syntax, T!['}']) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -598,9 +598,15 @@ pub(crate) syntax: SyntaxNode, } impl ObjBodyMemberList { + pub fn l_brace_token(&self) -> Option { + support::token(&self.syntax, T!['{']) + } pub fn members(&self) -> AstChildren { support::children(&self.syntax) } + pub fn r_brace_token(&self) -> Option { + support::token(&self.syntax, T!['}']) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] --- a/crates/jrsonnet-rowan-parser/src/lib.rs +++ b/crates/jrsonnet-rowan-parser/src/lib.rs @@ -1,5 +1,11 @@ #![deny(unused_must_use)] +use event::Sink; +use generated::nodes::{SourceFile, Trivia}; +use lex::lex; +use parser::{Parser, SyntaxError}; +pub use rowan; + mod ast; mod event; mod generated; @@ -13,18 +19,18 @@ mod token_set; pub use ast::{AstChildren, AstNode, AstToken}; -use event::Sink; -use generated::nodes::SourceFile; pub use generated::{nodes, syntax_kinds::SyntaxKind}; -pub use language::{ - JsonnetLanguage, PreorderWithTokens, SyntaxElement, SyntaxElementChildren, SyntaxNode, - SyntaxNodeChildren, SyntaxToken, -}; -use lex::lex; -use parser::{Parser, SyntaxError}; +pub use language::*; +pub use token_set::SyntaxKindSet; + pub fn parse(input: &str) -> (SourceFile, Vec) { let lexemes = lex(input); - let parser = Parser::new(&lexemes); + let kinds = lexemes + .iter() + .map(|l| l.kind) + .filter(|k| !Trivia::can_cast(*k)) + .collect(); + let parser = Parser::new(kinds); let events = parser.parse(); let sink = Sink::new(events, &lexemes); --- a/crates/jrsonnet-rowan-parser/src/marker.rs +++ b/crates/jrsonnet-rowan-parser/src/marker.rs @@ -113,21 +113,3 @@ completed } } - -pub trait AsRange { - fn as_range(&self, p: &Parser) -> TextRange; - fn end_token(&self) -> usize; -} - -impl AsRange for FinishedRanger { - fn as_range(&self, p: &Parser) -> TextRange { - TextRange::new( - p.start_of_token(self.start_token), - p.end_of_token(self.end_token), - ) - } - - fn end_token(&self) -> usize { - self.end_token - } -} --- a/crates/jrsonnet-rowan-parser/src/parser.rs +++ b/crates/jrsonnet-rowan-parser/src/parser.rs @@ -6,7 +6,7 @@ use crate::{ event::Event, lex::Lexeme, - marker::{AsRange, CompletedMarker, Marker, Ranger}, + marker::{CompletedMarker, Marker, Ranger}, nodes::{BinaryOperatorKind, Literal, Number, Text, Trivia, UnaryOperatorKind}, token_set::SyntaxKindSet, AstToken, SyntaxKind, @@ -33,9 +33,9 @@ } } -pub struct Parser<'i> { +pub struct Parser { // TODO: remove all trivia before feeding to parser? - lexemes: &'i [Lexeme<'i>], + kinds: Vec, pub offset: usize, pub events: Vec, pub entered: u32, @@ -103,10 +103,10 @@ } } -impl<'i> Parser<'i> { - pub fn new(lexemes: &'i [Lexeme<'i>]) -> Self { +impl Parser { + pub fn new(kinds: Vec) -> Self { Self { - lexemes, + kinds, offset: 0, events: vec![], entered: 0, @@ -134,14 +134,12 @@ .set(ExpectedSyntaxTrackingState::Unnamed); } pub fn start(&mut self) -> Marker { - self.skip_trivia(); let start_event_idx = self.events.len(); self.events.push(Event::Pending); self.entered += 1; Marker::new(start_event_idx) } pub fn start_ranger(&mut self) -> Ranger { - self.skip_trivia(); let pos = self.offset; Ranger { pos } } @@ -178,45 +176,7 @@ } else { self.error_with_no_skip(); } - } - fn current_token(&self) -> Lexeme<'i> { - self.lexemes[self.offset] - } - fn previous_token(&mut self) -> Option> { - if self.offset == 0 { - return None; - } - let mut previous_token_idx = self.offset - 1; - while self - .lexemes - .get(previous_token_idx) - .map_or(false, |l| Trivia::can_cast(l.kind)) - && previous_token_idx != 0 - { - previous_token_idx -= 1; - } - - Some(self.lexemes[previous_token_idx]) - } - pub fn start_of_token(&self, mut idx: usize) -> TextSize { - while Trivia::can_cast(self.lexemes[idx].kind) { - idx += 1; - } - self.lexemes[idx].range.start() } - pub fn end_of_token(&self, mut idx: usize) -> TextSize { - while Trivia::can_cast(self.lexemes[idx].kind) { - idx -= 1; - } - self.lexemes[idx].range.end() - } - pub(crate) fn custom_error(&mut self, marker: impl AsRange, error: impl AsRef) { - self.last_error_token = marker.end_token(); - self.events.push(Event::Error(SyntaxError::Custom { - error: error.as_ref().to_string(), - range: marker.as_range(self), - })); - } pub(crate) fn error_with_recovery_set( &mut self, recovery_set: SyntaxKindSet, @@ -238,27 +198,26 @@ self.expected_syntax_tracking_state .set(ExpectedSyntaxTrackingState::Unnamed); - self.skip_trivia(); if self.at_end() || self.at_ts(recovery_set) { - let range = self - .previous_token() - .map(|t| t.range) - .unwrap_or_else(|| TextRange::at(TextSize::from(0), TextSize::from(0))); + // let range = self + // .previous_token() + // .map(|t| t.range) + // .unwrap_or_else(|| TextRange::at(TextSize::from(0), TextSize::from(0))); - self.events.push(Event::Error(SyntaxError::Missing { - expected: expected_syntax, - offset: range.end(), - })); + // self.events.push(Event::Error(SyntaxError::Missing { + // expected: expected_syntax, + // offset: range.end(), + // })); return None; } - let current_token = self.current_token(); + let current_token = self.current(); - self.events.push(Event::Error(SyntaxError::Unexpected { - expected: expected_syntax, - found: current_token.kind, - range: current_token.range, - })); + // self.events.push(Event::Error(SyntaxError::Unexpected { + // expected: expected_syntax, + // found: current_token.kind, + // range: current_token.range, + // })); self.clear_expected_syntaxes(); self.last_error_token = self.offset; @@ -267,17 +226,14 @@ Some(m.complete(self, SyntaxKind::ERROR)) } fn bump_assert(&mut self, kind: SyntaxKind) { - self.skip_trivia(); assert!(self.at(kind), "expected {:?}", kind); self.bump_remap(self.current()); } fn bump(&mut self) { - self.skip_trivia(); self.bump_remap(self.current()); } fn bump_remap(&mut self, kind: SyntaxKind) { - self.skip_trivia(); - assert_ne!(self.offset, self.lexemes.len(), "already at end"); + assert_ne!(self.offset, self.kinds.len(), "already at end"); self.events.push(Event::Token { kind }); self.offset += 1; self.clear_expected_syntaxes(); @@ -302,7 +258,7 @@ { let next = 20; write!(out, "\n\nNext {next} tokens:").unwrap(); - for (i, tok) in self.lexemes.iter().skip(self.offset).take(next).enumerate() { + for (i, tok) in self.kinds.iter().skip(self.offset).take(next).enumerate() { write!(out, "\n{i}. {tok:?}").unwrap(); } } @@ -314,39 +270,12 @@ self.step(); let mut offset = self.offset; for _ in 0..i { - while self - .lexemes - .get(offset) - .map(|l| Trivia::can_cast(l.kind)) - .unwrap_or(false) - { - offset += 1; - } offset += 1; } - while self - .lexemes - .get(offset) - .map(|l| Trivia::can_cast(l.kind)) - .unwrap_or(false) - { - offset += 1; - } - self.lexemes.get(offset).map(|l| l.kind).unwrap_or(EOF) + self.kinds.get(offset).copied().unwrap_or(EOF) } fn current(&self) -> SyntaxKind { self.nth(0) - } - fn skip_trivia(&mut self) { - while Trivia::can_cast(self.peek_raw()) { - self.offset += 1; - } - } - fn peek_raw(&mut self) -> SyntaxKind { - self.lexemes - .get(self.offset) - .map(|l| l.kind) - .unwrap_or(SyntaxKind::EOF) } #[must_use] pub(crate) fn expected_syntax_name(&mut self, name: &'static str) -> ExpectedSyntaxGuard { @@ -507,15 +436,15 @@ None }; let params = if p.at(T!['(']) { - if let Some(plus) = plus { - p.custom_error(plus, "can't extend with method"); - } + // if let Some(plus) = plus { + // p.custom_error(plus, "can't extend with method"); + // } params_desc(p); - if p.at(T![+]) { - let r = p.start_ranger(); - p.bump(); - p.custom_error(r.finish(p), "can't extend with method"); - } + // if p.at(T![+]) { + // let r = p.start_ranger(); + // p.bump(); + // p.custom_error(r.finish(p), "can't extend with method"); + // } true } else { false @@ -669,10 +598,10 @@ if elems > 1 && !compspecs.is_empty() { for spec in compspecs { - p.custom_error( - spec, - "compspec may only be used if there is only one array element", - ) + // p.custom_error( + // spec, + // "compspec may only be used if there is only one array element", + // ) } m.complete(p, EXPR_ARRAY) @@ -797,9 +726,9 @@ } else if p.at(T![...]) { let m_err = p.start_ranger(); destruct_rest(p); - if had_rest { - p.custom_error(m_err.finish(p), "only one rest can be present in array"); - } + // if had_rest { + // p.custom_error(m_err.finish(p), "only one rest can be present in array"); + // } had_rest = true; } else { destruct(p); @@ -822,9 +751,9 @@ } else if p.at(T![...]) { let m_err = p.start_ranger(); destruct_rest(p); - if had_rest { - p.custom_error(m_err.finish(p), "only one rest can be present in object"); - } + // if had_rest { + // p.custom_error(m_err.finish(p), "only one rest can be present in object"); + // } had_rest = true; } else { if had_rest { --- a/crates/jrsonnet-rowan-parser/src/token_set.rs +++ b/crates/jrsonnet-rowan-parser/src/token_set.rs @@ -34,9 +34,9 @@ #[macro_export] macro_rules! TS { ($($tt:tt)*) => { - SyntaxKindSet::new(&[ + $crate::SyntaxKindSet::new(&[ $( - T![$tt] + $crate::T![$tt] ),* ]) }; --- a/jrsonnet-lsp/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "jrsonnet-lsp" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = "1.0.48" -jrsonnet-evaluator = { path = "../jrsonnet-evaluator" } -jrsonnet-parser = { path = "../jrsonnet-parser" } -lsp-server = "0.5.2" -lsp-types = "0.92.0" -serde = "1.0.130" -serde_json = "1.0.71" --- a/jrsonnet-lsp/src/main.rs +++ /dev/null @@ -1,211 +0,0 @@ -use std::{ - collections::HashMap, - fs::File, - path::{Path, PathBuf}, - str::FromStr, -}; - -use jrsonnet_evaluator::{EvaluationState, FileImportResolver, Val}; -use jrsonnet_parser::{ExprLocation, ParserSettings}; -use lsp_server::{Connection, ErrorCode, Message, Request, RequestId, Response}; -use lsp_types::{ - notification::{DidChangeTextDocument, DidOpenTextDocument, Notification}, - request::{DocumentLinkRequest, HoverRequest}, - CompletionOptions, DidChangeTextDocumentParams, DidOpenTextDocumentParams, DocumentLink, - DocumentLinkOptions, Hover, HoverContents, MarkupContent, MarkupKind, ServerCapabilities, - TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions, Url, - WorkDoneProgressOptions, -}; - -use std::io::Write; - -fn main() { - let mut log = File::create("test").unwrap(); - writeln!(log, "start").unwrap(); - let (connection, io_threads) = Connection::stdio(); - let capabilities = serde_json::to_value(&ServerCapabilities { - completion_provider: Some(CompletionOptions::default()), - definition_provider: Some(lsp_types::OneOf::Left(true)), - document_link_provider: Some(DocumentLinkOptions { - resolve_provider: Some(false), - work_done_progress_options: WorkDoneProgressOptions::default(), - }), - hover_provider: Some(lsp_types::HoverProviderCapability::Simple(true)), - text_document_sync: Some(TextDocumentSyncCapability::Options( - TextDocumentSyncOptions { - change: Some(TextDocumentSyncKind::FULL), - open_close: Some(true), - ..TextDocumentSyncOptions::default() - }, - )), - ..ServerCapabilities::default() - }) - .expect("failed to convert capabilities to json"); - - connection - .initialize(capabilities) - .expect("failed to initialize connection"); - - writeln!(log, "initialized").unwrap(); - - main_loop(&mut log, &connection).expect("main loop failed"); - - io_threads.join().expect("failed to join io_threads"); -} -fn main_loop(log: &mut File, connection: &Connection) -> anyhow::Result<()> { - let mut es = EvaluationState::default(); - es.set_import_resolver(Box::new(FileImportResolver::default())); - - let reply = |response: Response| { - connection - .sender - .send(Message::Response(response)) - .expect("failed to respond"); - }; - - for msg in &connection.receiver { - match msg { - Message::Response(_) => (), - Message::Request(req) => { - if connection.handle_shutdown(&req)? { - return Ok(()); - } - if let Some((id, params)) = cast::(&req) { - reply(Response::new_ok(id, >::new())); - } else if let Some((id, params)) = cast::(&req) { - let pos = params - .text_document_position_params - .text_document - .uri - .path(); - let buf = PathBuf::from_str(pos).unwrap(); - let pos = es - .map_from_source_location( - &buf, - params.text_document_position_params.position.line as usize + 1, - params.text_document_position_params.position.character as usize + 1, - ) - .unwrap(); - let el = ExprLocation(buf.clone().into(), pos as usize, pos as usize); - let es2 = es.clone(); - // reply(Response::new_ok( - // id, - // Some(Hover { - // range: None, - // contents: HoverContents::Markup(MarkupContent { - // kind: MarkupKind::Markdown, - // value: es - // .run_in_state_with_breakpoint(el, move || { - // es2.reset_evaluation_state(&buf); - // es2.import_file(&PathBuf::new(), &buf)? - // .to_string() - // .map(|_| ()) - // }) - // .unwrap() - // .unwrap_or_else(|| Val::Null) - // .value_type() - // .to_string(), - // }), - // }), - // )); - } else - /* - if let Some((id, params)) = cast::(&req) { - let links = handle_links(&files, params).unwrap_or_default(); - reply(Response::new_ok(id, links)); - } else if let Some((id, params)) = cast::(&req) { - if let Some(loc) = handle_goto(&files, params) { - reply(Response::new_ok(id, loc)) - } else { - reply(Response::new_ok(id, ())) - } - } else if let Some((id, params)) = cast::(&req) { - match handle_hover(&files, params) { - Some((range, markdown)) => { - reply(Response::new_ok( - id, - Hover { - contents: HoverContents::Markup(MarkupContent { - kind: MarkupKind::Markdown, - value: markdown, - }), - range, - }, - )); - } - None => { - reply(Response::new_ok(id, ())); - } - } - } else if let Some((id, params)) = cast::(&req) { - let completions = handle_completion(&files, params.text_document_position) - .unwrap_or_default(); - reply(Response::new_ok(id, completions)); - } else - */ - { - reply(Response::new_err( - req.id, - ErrorCode::MethodNotFound as i32, - format!("unrecognized request {}", req.method), - )) - } - } - Message::Notification(req) => { - let mut handle = |text: String, uri: Url| { - writeln!(log, "updated file: {:?}", uri).unwrap(); - let path = match PathBuf::from_str(uri.path()) { - Ok(x) => x, - Err(_) => return, - }; - let parsed = match jrsonnet_parser::parse( - &text, - &ParserSettings { - file_name: path.clone().into(), - }, - ) { - Ok(v) => v, - Err(e) => { - writeln!(log, "fuck D: {:?}", e).unwrap(); - return; - // connection.sender.send(Message::Notification(Notification::new_err(req.id, ErrorCode::ParseError as i32, format!("Fuck D: {:?}", e)))) - } - }; - es.add_parsed_file(path.into(), text.into(), parsed) - .unwrap(); - writeln!(log, "parsed: {:?}", uri).unwrap(); - }; - - match &*req.method { - DidOpenTextDocument::METHOD => { - let params: DidOpenTextDocumentParams = - match serde_json::from_value(req.params) { - Ok(x) => x, - Err(_) => continue, - }; - handle(params.text_document.text, params.text_document.uri); - } - DidChangeTextDocument::METHOD => { - let params: DidChangeTextDocumentParams = - match serde_json::from_value(req.params) { - Ok(x) => x, - Err(_) => continue, - }; - for change in params.content_changes.into_iter() { - handle(change.text, params.text_document.uri.clone()); - } - } - _ => continue, - } - } - } - } - Ok(()) -} -fn cast(req: &Request) -> Option<(RequestId, R::Params)> -where - R: lsp_types::request::Request, - R::Params: serde::de::DeserializeOwned, -{ - req.clone().extract(R::METHOD).ok() -} --- a/xtask/src/sourcegen/ast.rs +++ b/xtask/src/sourcegen/ast.rs @@ -151,6 +151,19 @@ if let Some(old) = types.insert(field.ty(), field.method_name(kinds)) { panic!("{name}.{} has same type as {name}.{}, resolve conflict by wrapping one field: {}", old, field.method_name(kinds), field.ty()); } + // TODO: check for assignable field types, i.e you can have + // ``` + // SomeEnum = + // SomeItem + // | SomeOtherItem + // ``` + // And check above will fail to detect conflict in + // ``` + // SomeStruct = + // SomeEnum + // SomeItem + // ``` + // Despite generating getters, which will both return SomeEnum } res.nodes.push(AstNodeSrc { doc: Vec::new(),