From 4ad9956e5f9bbcf7ddb04d82957e33caa72b0cc3 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Fri, 05 Aug 2022 18:53:31 +0000 Subject: [PATCH] refactor: keep source code alongside source path --- --- a/bindings/jsonnet/src/import.rs +++ b/bindings/jsonnet/src/import.rs @@ -16,6 +16,7 @@ error::{Error::*, Result}, throw, ImportResolver, State, }; +use jrsonnet_parser::SourcePath; pub type JsonnetImportCallback = unsafe extern "C" fn( ctx: *mut c_void, @@ -29,10 +30,10 @@ pub struct CallbackImportResolver { cb: JsonnetImportCallback, ctx: *mut c_void, - out: RefCell>>, + out: RefCell>>, } impl ImportResolver for CallbackImportResolver { - fn resolve_file(&self, from: &Path, path: &str) -> Result { + fn resolve_file_relative(&self, from: &Path, path: &str) -> Result { let base = CString::new(from.to_str().unwrap()).unwrap().into_raw(); let rel = CString::new(path).unwrap().into_raw(); let found_here: *mut c_char = null_mut(); @@ -61,7 +62,7 @@ } let found_here_raw = unsafe { CStr::from_ptr(found_here) }; - let found_here_buf = PathBuf::from(found_here_raw.to_str().unwrap()); + let found_here_buf = SourcePath::Path(PathBuf::from(found_here_raw.to_str().unwrap())); unsafe { let _ = CString::from_raw(found_here); } @@ -74,7 +75,7 @@ Ok(found_here_buf) } - fn load_file_contents(&self, resolved: &Path) -> Result> { + fn load_file_contents(&self, resolved: &SourcePath) -> Result> { Ok(self.out.borrow().get(resolved).unwrap().clone()) } @@ -108,24 +109,28 @@ } } impl ImportResolver for NativeImportResolver { - fn resolve_file(&self, from: &Path, path: &str) -> Result { + fn resolve_file_relative(&self, from: &Path, path: &str) -> Result { let mut new_path = from.to_owned(); new_path.push(path); if new_path.exists() { - Ok(new_path) + Ok(SourcePath::Path(new_path)) } else { for library_path in self.library_paths.borrow().iter() { let mut cloned = library_path.clone(); cloned.push(path); if cloned.exists() { - return Ok(cloned); + return Ok(SourcePath::Path(cloned)); } } throw!(ImportFileNotFound(from.to_owned(), path.to_owned())) } } - fn load_file_contents(&self, id: &Path) -> Result> { - let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?; + fn load_file_contents(&self, id: &SourcePath) -> Result> { + let path = match id { + SourcePath::Path(path) => path, + _ => unreachable!("NativeImportResolver::resolve_file may only return plain paths"), + }; + let mut file = File::open(path).map_err(|_e| ResolvedFileNotFound(id.clone()))?; let mut out = Vec::new(); file.read_to_end(&mut out) .map_err(|e| ImportIo(e.to_string()))?; --- a/bindings/jsonnet/src/lib.rs +++ b/bindings/jsonnet/src/lib.rs @@ -10,9 +10,9 @@ use std::{ alloc::Layout, + env, ffi::{CStr, CString}, os::raw::{c_char, c_double, c_int, c_uint}, - path::PathBuf, }; use import::NativeImportResolver; @@ -112,7 +112,10 @@ ) -> *const c_char { let filename = CStr::from_ptr(filename); match vm - .import(PathBuf::from(filename.to_str().unwrap())) + .import( + &env::current_dir().expect("cwd"), + filename.to_str().unwrap(), + ) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest(v)) { @@ -141,10 +144,7 @@ let filename = CStr::from_ptr(filename); let snippet = CStr::from_ptr(snippet); match vm - .evaluate_snippet( - filename.to_str().unwrap().into(), - snippet.to_str().unwrap().into(), - ) + .evaluate_snippet(filename.to_str().unwrap().into(), snippet.to_str().unwrap()) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest(v)) { @@ -186,7 +186,10 @@ ) -> *const c_char { let filename = CStr::from_ptr(filename); match vm - .import(PathBuf::from(filename.to_str().unwrap())) + .import( + &env::current_dir().expect("cwd"), + filename.to_str().unwrap(), + ) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest_multi(v)) { @@ -213,10 +216,7 @@ let filename = CStr::from_ptr(filename); let snippet = CStr::from_ptr(snippet); match vm - .evaluate_snippet( - filename.to_str().unwrap().into(), - snippet.to_str().unwrap().into(), - ) + .evaluate_snippet(filename.to_str().unwrap().into(), snippet.to_str().unwrap()) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest_multi(v)) { @@ -256,7 +256,10 @@ ) -> *const c_char { let filename = CStr::from_ptr(filename); match vm - .import(PathBuf::from(filename.to_str().unwrap())) + .import( + &env::current_dir().expect("cwd"), + filename.to_str().unwrap(), + ) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest_stream(v)) { @@ -283,10 +286,7 @@ let filename = CStr::from_ptr(filename); let snippet = CStr::from_ptr(snippet); match vm - .evaluate_snippet( - filename.to_str().unwrap().into(), - snippet.to_str().unwrap().into(), - ) + .evaluate_snippet(filename.to_str().unwrap().into(), snippet.to_str().unwrap()) .and_then(|v| vm.with_tla(v)) .and_then(|v| vm.manifest_stream(v)) { --- a/bindings/jsonnet/src/vars_tlas.rs +++ b/bindings/jsonnet/src/vars_tlas.rs @@ -32,7 +32,7 @@ .as_any() .downcast_ref::() .expect("only stdlib context initializer supported") - .add_ext_code(name.to_str().unwrap(), value.to_str().unwrap().into()) + .add_ext_code(name.to_str().unwrap(), value.to_str().unwrap()) .unwrap() } /// # Safety --- a/cmds/jrsonnet/src/main.rs +++ b/cmds/jrsonnet/src/main.rs @@ -133,14 +133,14 @@ let input = opts.input.input.ok_or(Error::MissingInputArgument)?; let val = if opts.input.exec { - s.evaluate_snippet("".to_owned(), (&input as &str).into())? + s.evaluate_snippet("".to_owned(), &input as &str)? } else if input == "-" { let mut input = Vec::new(); std::io::stdin().read_to_end(&mut input)?; - let input_str = std::str::from_utf8(&input)?.into(); + let input_str = std::str::from_utf8(&input)?; s.evaluate_snippet("".to_owned(), input_str)? } else { - s.import(s.resolve_file(¤t_dir().expect("cwd"), &input)?)? + s.import(¤t_dir().expect("cwd"), &input)? }; let val = s.with_tla(val)?; --- a/crates/jrsonnet-cli/src/stdlib.rs +++ b/crates/jrsonnet-cli/src/stdlib.rs @@ -118,10 +118,10 @@ ctx.add_ext_str((&ext.name as &str).into(), (&ext.value as &str).into()); } for ext in self.ext_code.iter() { - ctx.add_ext_code(&ext.name as &str, (&ext.value as &str).into())?; + ctx.add_ext_code(&ext.name as &str, &ext.value as &str)?; } for ext in self.ext_code_file.iter() { - ctx.add_ext_code(&ext.name as &str, (&ext.value as &str).into())?; + ctx.add_ext_code(&ext.name as &str, &ext.value as &str)?; } s.settings_mut().context_initializer = Box::new(ctx); Ok(()) --- a/crates/jrsonnet-evaluator/src/error.rs +++ b/crates/jrsonnet-evaluator/src/error.rs @@ -2,7 +2,7 @@ use jrsonnet_gcmodule::Trace; use jrsonnet_interner::IStr; -use jrsonnet_parser::{BinaryOpType, ExprLocation, Source, UnaryOpType}; +use jrsonnet_parser::{BinaryOpType, ExprLocation, Source, SourcePath, UnaryOpType}; use jrsonnet_types::ValType; use thiserror::Error; @@ -138,10 +138,10 @@ #[error("can't resolve {1} from {0}")] ImportFileNotFound(PathBuf, String), - #[error("resolved file not found: {0}")] - ResolvedFileNotFound(PathBuf), + #[error("resolved file not found: {:?}", .0)] + ResolvedFileNotFound(SourcePath), #[error("imported file is not valid utf-8: {0:?}")] - ImportBadFileUtf8(PathBuf), + ImportBadFileUtf8(SourcePath), #[error("import io error: {0}")] ImportIo(String), #[error("tried to import {1} from {0}, but imports is not supported")] @@ -151,12 +151,11 @@ #[error( "syntax error: expected {}, got {:?}", .error.expected, - .source_code.chars().nth(error.location.offset) + .path.code().chars().nth(error.location.offset) .map_or_else(|| "EOF".into(), |c| c.to_string()) )] ImportSyntaxError { path: Source, - source_code: IStr, #[trace(skip)] error: Box, }, --- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs +++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs @@ -416,13 +416,13 @@ Literal(LiteralType::This) => { Val::Obj(ctx.this().clone().ok_or(CantUseSelfOutsideOfObject)?) } - Literal(LiteralType::Super) => { - Val::Obj(ctx.super_obj().clone().ok_or(NoSuperFound)?.with_this( + Literal(LiteralType::Super) => Val::Obj( + ctx.super_obj().clone().ok_or(NoSuperFound)?.with_this( ctx.this() .clone() .expect("if super exists - then this should to"), - )) - } + ), + ), Literal(LiteralType::Dollar) => { Val::Obj(ctx.dollar().clone().ok_or(NoTopLevelObjectFound)?) } @@ -643,10 +643,10 @@ Import(_) => s.push( CallLocation::new(loc), || format!("import {:?}", path.clone()), - || s.import(resolved_path.clone()), + || s.import_resolved(resolved_path), )?, - ImportStr(_) => Val::Str(s.import_str(resolved_path)?), - ImportBin(_) => Val::Arr(ArrValue::Bytes(s.import_bin(resolved_path)?)), + ImportStr(_) => Val::Str(s.import_resolved_str(resolved_path)?), + ImportBin(_) => Val::Arr(ArrValue::Bytes(s.import_resolved_bin(resolved_path)?)), _ => unreachable!(), } } --- a/crates/jrsonnet-evaluator/src/import.rs +++ b/crates/jrsonnet-evaluator/src/import.rs @@ -6,6 +6,7 @@ }; use fs::File; +use jrsonnet_parser::SourcePath; use crate::{ error::{Error::*, Result}, @@ -17,9 +18,12 @@ /// Resolves real file path, e.g. `(/home/user/manifests, b.libjsonnet)` can correspond /// both to `/home/user/manifests/b.libjsonnet` and to `/home/user/${vendor}/b.libjsonnet` /// where `${vendor}` is a library path. - fn resolve_file(&self, from: &Path, path: &str) -> Result; + fn resolve_file_relative(&self, from: &Path, path: &str) -> Result; - fn load_file_contents(&self, resolved: &Path) -> Result>; + /// Load resolved file + /// This should only be called with value returned from `resolve_file`, this cannot be resolved using associated type, + /// as evaluator uses object instead of generic for [`ImportResolver`] + fn load_file_contents(&self, resolved: &SourcePath) -> Result>; /// # Safety /// @@ -32,11 +36,11 @@ /// Dummy resolver, can't resolve/load any file pub struct DummyImportResolver; impl ImportResolver for DummyImportResolver { - fn resolve_file(&self, from: &Path, path: &str) -> Result { + fn resolve_file_relative(&self, from: &Path, path: &str) -> Result { throw!(ImportNotSupported(from.into(), path.into())) } - fn load_file_contents(&self, _resolved: &Path) -> Result> { + fn load_file_contents(&self, _resolved: &SourcePath) -> Result> { panic!("dummy resolver can't load any file") } @@ -59,25 +63,35 @@ pub library_paths: Vec, } impl ImportResolver for FileImportResolver { - fn resolve_file(&self, from: &Path, path: &str) -> Result { + fn resolve_file_relative(&self, from: &Path, path: &str) -> Result { let mut direct = from.to_path_buf(); direct.push(path); if direct.exists() { - Ok(direct.canonicalize().map_err(|e| ImportIo(e.to_string()))?) + Ok(SourcePath::Path( + direct.canonicalize().map_err(|e| ImportIo(e.to_string()))?, + )) } else { for library_path in &self.library_paths { let mut cloned = library_path.clone(); cloned.push(path); if cloned.exists() { - return Ok(cloned.canonicalize().map_err(|e| ImportIo(e.to_string()))?); + return Ok(SourcePath::Path( + cloned.canonicalize().map_err(|e| ImportIo(e.to_string()))?, + )); } } throw!(ImportFileNotFound(from.to_owned(), path.to_owned())) } } - fn load_file_contents(&self, id: &Path) -> Result> { - let mut file = File::open(id).map_err(|_e| ResolvedFileNotFound(id.to_owned()))?; + fn load_file_contents(&self, id: &SourcePath) -> Result> { + let path = match id { + SourcePath::Path(path) => path, + _ => { + panic!("this resolver can only resolve to path") + } + }; + let mut file = File::open(path).map_err(|_e| ResolvedFileNotFound(id.clone()))?; let mut out = Vec::new(); file.read_to_end(&mut out) .map_err(|e| ImportIo(e.to_string()))?; --- a/crates/jrsonnet-evaluator/src/lib.rs +++ b/crates/jrsonnet-evaluator/src/lib.rs @@ -48,7 +48,7 @@ cell::{Ref, RefCell, RefMut}, collections::HashMap, fmt::{self, Debug}, - path::{Path, PathBuf}, + path::Path, rc::Rc, }; @@ -65,7 +65,7 @@ pub use jrsonnet_parser as parser; use jrsonnet_parser::*; pub use obj::*; -use trace::{location_to_offset, offset_to_location, CodeLocation, CompactFormat, TraceFormat}; +use trace::{CompactFormat, TraceFormat}; pub use val::{ManifestFormat, Thunk, Val}; pub trait Unbound: Trace { @@ -170,10 +170,7 @@ breakpoints: Breakpoints, /// Contains file source codes and evaluation results for imports and pretty-printed stacktraces - files: GcHashMap, - /// Contains tla arguments and others, which aren't needed to be obtained by name, however may be used for receiving source - /// TODO: look into nix approach, storing source code in `Source` object - volatile_files: GcHashMap, + files: GcHashMap, } struct FileData { string: Option, @@ -249,7 +246,8 @@ pub struct State(Rc); impl State { - pub fn import_str(&self, path: PathBuf) -> Result { + /// Should only be called with path retrieved from [`resolve_path`], may panic otherwise + pub fn import_resolved_str(&self, path: SourcePath) -> Result { let mut data = self.data_mut(); let mut file = data.files.raw_entry_mut().from_key(&path); @@ -283,7 +281,8 @@ } Ok(file.string.as_ref().expect("just set").clone()) } - pub fn import_bin(&self, path: PathBuf) -> Result { + /// Should only be called with path retrieved from [`resolve_path`], may panic otherwise + pub fn import_resolved_bin(&self, path: SourcePath) -> Result { let mut data = self.data_mut(); let mut file = data.files.raw_entry_mut().from_key(&path); @@ -309,7 +308,8 @@ } Ok(file.bytes.as_ref().expect("just set").clone()) } - pub fn import(&self, path: PathBuf) -> Result { + /// Should only be called with path retrieved from [`resolve_path`], may panic otherwise + pub fn import_resolved(&self, path: SourcePath) -> Result { let mut data = self.data_mut(); let mut file = data.files.raw_entry_mut().from_key(&path); @@ -343,7 +343,8 @@ ); } let code = file.string.as_ref().expect("just set"); - let file_name = Source::new(path.clone()).expect("resolver should return correct name"); + let file_name = + Source::new(path.clone(), code.clone()).expect("resolver should return correct name"); if file.parsed.is_none() { file.parsed = Some( jrsonnet_parser::parse( @@ -354,7 +355,6 @@ ) .map_err(|e| ImportSyntaxError { path: file_name.clone(), - source_code: code.clone(), error: Box::new(e), })?, ); @@ -386,34 +386,11 @@ Ok(v) } Err(e) => Err(e), - } - } - - pub fn get_source(&self, name: Source) -> Option { - let data = self.data(); - match name.repr() { - Ok(real) => data - .files - .get(real) - .and_then(|f| f.string.as_ref()) - .map(ToString::to_string), - Err(e) => data.volatile_files.get(e).map(ToOwned::to_owned), } - } - pub fn map_source_locations(&self, file: Source, locs: &[u32]) -> Vec { - offset_to_location(&self.get_source(file).unwrap_or_else(|| "".into()), locs) } - pub fn map_from_source_location( - &self, - file: Source, - line: usize, - column: usize, - ) -> Option { - location_to_offset( - &self.get_source(file).expect("file not found"), - line, - column, - ) + pub fn import(&self, from: &Path, path: &str) -> Result { + let resolved = self.resolve_file(from, path)?; + self.import_resolved(resolved) } /// Creates context with all passed global variables @@ -554,7 +531,10 @@ || { func.evaluate( self.clone(), - self.create_default_context(Source::new_virtual(Cow::Borrowed(""))), + self.create_default_context(Source::new_virtual( + Cow::Borrowed(""), + IStr::empty(), + )), CallLocation::native(), &self.settings().tla_vars, true, @@ -568,9 +548,9 @@ /// Internals impl State { - fn data(&self) -> Ref { - self.0.data.borrow() - } + // fn data(&self) -> Ref { + // self.0.data.borrow() + // } fn data_mut(&self) -> RefMut { self.0.data.borrow_mut() } @@ -585,8 +565,9 @@ /// Raw methods evaluate passed values but don't perform TLA execution impl State { /// Parses and evaluates the given snippet - pub fn evaluate_snippet(&self, name: String, code: String) -> Result { - let source = Source::new_virtual(Cow::Owned(name.clone())); + pub fn evaluate_snippet(&self, name: String, code: impl Into) -> Result { + let code = code.into(); + let source = Source::new_virtual(Cow::Owned(name), code.clone()); let parsed = jrsonnet_parser::parse( &code, &ParserSettings { @@ -595,10 +576,8 @@ ) .map_err(|e| ImportSyntaxError { path: source.clone(), - source_code: code.clone().into(), error: Box::new(e), })?; - self.data_mut().volatile_files.insert(name, code); evaluate(self.clone(), self.create_default_context(source), &parsed) } } @@ -617,7 +596,7 @@ } pub fn add_tla_code(&self, name: IStr, code: &str) -> Result<()> { let source_name = format!("", name); - let source = Source::new_virtual(Cow::Owned(source_name.clone())); + let source = Source::new_virtual(Cow::Owned(source_name.clone()), code.into()); let parsed = jrsonnet_parser::parse( code, &ParserSettings { @@ -626,22 +605,18 @@ ) .map_err(|e| ImportSyntaxError { path: source, - source_code: code.into(), error: Box::new(e), })?; - self.data_mut() - .volatile_files - .insert(source_name, code.to_owned()); self.settings_mut() .tla_vars .insert(name, TlaArg::Code(parsed)); Ok(()) } - pub fn resolve_file(&self, from: &Path, path: &str) -> Result { + pub fn resolve_file(&self, from: &Path, path: &str) -> Result { self.settings() .import_resolver - .resolve_file(from, path.as_ref()) + .resolve_file_relative(from, path.as_ref()) } pub fn import_resolver(&self) -> Ref { --- a/crates/jrsonnet-evaluator/src/trace/location.rs +++ /dev/null @@ -1,124 +0,0 @@ -#[allow(clippy::module_name_repetitions)] -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct CodeLocation { - pub offset: usize, - - pub line: usize, - pub column: usize, - - pub line_start_offset: usize, - pub line_end_offset: usize, -} - -#[allow(clippy::module_name_repetitions)] -pub fn location_to_offset(mut file: &str, mut line: usize, column: usize) -> Option { - let mut offset = 0; - while line > 1 { - let pos = file.find('\n')?; - offset += pos + 1; - file = &file[pos + 1..]; - line -= 1; - } - offset += column - 1; - Some(offset) -} - -#[allow(clippy::module_name_repetitions)] -pub fn offset_to_location(file: &str, offsets: &[u32]) -> Vec { - if offsets.is_empty() { - return vec![]; - } - let mut line = 1; - let mut column = 1; - let max_offset = *offsets.iter().max().expect("offsets is not empty"); - - let mut offset_map = offsets - .iter() - .enumerate() - .map(|(pos, offset)| (*offset, pos)) - .collect::>(); - offset_map.sort_by_key(|v| v.0); - offset_map.reverse(); - - let mut out = vec![ - CodeLocation { - offset: 0, - column: 0, - line: 0, - line_start_offset: 0, - line_end_offset: 0 - }; - offsets.len() - ]; - let mut with_no_known_line_ending = vec![]; - let mut this_line_offset = 0; - for (pos, ch) in file - .chars() - .enumerate() - .chain(std::iter::once((file.len(), ' '))) - { - column += 1; - match offset_map.last() { - Some(x) if x.0 == pos as u32 => { - let out_idx = x.1; - with_no_known_line_ending.push(out_idx); - out[out_idx].offset = pos; - out[out_idx].line = line; - out[out_idx].column = column; - out[out_idx].line_start_offset = this_line_offset; - offset_map.pop(); - } - _ => {} - } - if ch == '\n' { - line += 1; - column = 1; - - for idx in with_no_known_line_ending.drain(..) { - out[idx].line_end_offset = pos; - } - this_line_offset = pos + 1; - - if pos == max_offset as usize + 1 { - break; - } - } - } - let file_end = file.chars().count(); - for idx in with_no_known_line_ending { - out[idx].line_end_offset = file_end; - } - - out -} - -#[cfg(test)] -pub mod tests { - use super::{offset_to_location, CodeLocation}; - - #[test] - fn test() { - assert_eq!( - offset_to_location( - "hello world\n_______________________________________________________", - &[0, 14] - ), - vec![ - CodeLocation { - offset: 0, - line: 1, - column: 2, - line_start_offset: 0, - line_end_offset: 11, - }, - CodeLocation { - offset: 14, - line: 2, - column: 4, - line_start_offset: 12, - line_end_offset: 67 - } - ] - ) - } -} --- a/crates/jrsonnet-evaluator/src/trace/mod.rs +++ b/crates/jrsonnet-evaluator/src/trace/mod.rs @@ -1,9 +1,6 @@ -mod location; - use std::path::{Path, PathBuf}; -use jrsonnet_parser::Source; -pub use location::*; +use jrsonnet_parser::{CodeLocation, Source}; use crate::{error::Error, LocError, State}; @@ -84,31 +81,27 @@ fn write_trace( &self, out: &mut dyn std::fmt::Write, - s: &State, + _s: &State, error: &LocError, ) -> Result<(), std::fmt::Error> { write!(out, "{}", error.error())?; - if let Error::ImportSyntaxError { - path, - source_code, - error, - } = error.error() - { + if let Error::ImportSyntaxError { path, error } = error.error() { use std::fmt::Write; writeln!(out)?; - let mut n = match path.repr() { - Ok(r) => self.resolver.resolve(r), - Err(v) => v.to_string(), + let mut n = match path.path() { + Some(r) => self.resolver.resolve(r), + None => path.short_display().to_string(), }; let mut offset = error.location.offset; - let is_eof = if offset >= source_code.len() { - offset = source_code.len().saturating_sub(1); + let is_eof = if offset >= path.code().len() { + offset = path.code().len().saturating_sub(1); true } else { false }; - let mut location = offset_to_location(source_code, &[offset as u32]) + let mut location = path + .map_source_locations(&[offset as u32]) .into_iter() .next() .unwrap(); @@ -129,13 +122,12 @@ use std::fmt::Write; #[allow(clippy::option_if_let_else)] if let Some(location) = location { - let mut resolved_path = match location.0.repr() { - Ok(r) => self.resolver.resolve(r), - Err(v) => v.to_string(), + let mut resolved_path = match location.0.path() { + Some(r) => self.resolver.resolve(r), + None => location.0.short_display().to_string(), }; // TODO: Process all trace elements first - let location = - s.map_source_locations(location.0.clone(), &[location.1, location.2]); + let location = location.0.map_source_locations(&[location.1, location.2]); write!(resolved_path, ":").unwrap(); print_code_location(&mut resolved_path, &location[0], &location[1]).unwrap(); write!(resolved_path, ":").unwrap(); @@ -176,7 +168,7 @@ fn write_trace( &self, out: &mut dyn std::fmt::Write, - s: &State, + _s: &State, error: &LocError, ) -> Result<(), std::fmt::Error> { write!(out, "{}", error.error())?; @@ -184,10 +176,10 @@ writeln!(out)?; let desc = &item.desc; if let Some(source) = &item.location { - let start_end = s.map_source_locations(source.0.clone(), &[source.1, source.2]); - let resolved_path = match source.0.repr() { - Ok(r) => r.display().to_string(), - Err(v) => v.to_string(), + let start_end = source.0.map_source_locations(&[source.1, source.2]); + let resolved_path = match source.0.path() { + Some(r) => r.display().to_string(), + None => source.0.short_display().to_string(), }; write!( @@ -213,19 +205,15 @@ fn write_trace( &self, out: &mut dyn std::fmt::Write, - s: &State, + _s: &State, error: &LocError, ) -> Result<(), std::fmt::Error> { write!(out, "{}", error.error())?; - if let Error::ImportSyntaxError { - path, - source_code, - error, - } = error.error() - { + if let Error::ImportSyntaxError { path, error } = error.error() { writeln!(out)?; let offset = error.location.offset; - let location = offset_to_location(source_code, &[offset as u32]) + let location = path + .map_source_locations(&[offset as u32]) .into_iter() .next() .unwrap(); @@ -234,7 +222,7 @@ self.print_snippet( out, - source_code, + path.code(), path, &location, &end_location, @@ -246,10 +234,10 @@ writeln!(out)?; let desc = &item.desc; if let Some(source) = &item.location { - let start_end = s.map_source_locations(source.0.clone(), &[source.1, source.2]); + let start_end = source.0.map_source_locations(&[source.1, source.2]); self.print_snippet( out, - &s.get_source(source.0.clone()).unwrap(), + &source.0.code(), &source.0, &start_end[0], &start_end[1], @@ -284,9 +272,9 @@ .take(end.line_end_offset - end.line_start_offset) .collect(); - let origin = match origin.repr() { - Ok(r) => self.resolver.resolve(r), - Err(v) => v.to_string(), + let origin = match origin.path() { + Some(r) => self.resolver.resolve(r), + None => origin.short_display().to_string(), }; let snippet = Snippet { opt: FormatOptions { --- a/crates/jrsonnet-interner/src/lib.rs +++ b/crates/jrsonnet-interner/src/lib.rs @@ -33,6 +33,10 @@ impl IStr { #[must_use] + pub fn empty() -> Self { + "".into() + } + #[must_use] pub fn as_str(&self) -> &str { self as &str } --- a/crates/jrsonnet-parser/src/lib.rs +++ b/crates/jrsonnet-parser/src/lib.rs @@ -7,9 +7,11 @@ pub use expr::*; pub use jrsonnet_interner::IStr; pub use peg; +mod location; mod source; mod unescape; -pub use source::Source; +pub use location::CodeLocation; +pub use source::{Source, SourcePath}; pub struct ParserSettings { pub file_name: Source, --- /dev/null +++ b/crates/jrsonnet-parser/src/location.rs @@ -0,0 +1,124 @@ +#[allow(clippy::module_name_repetitions)] +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct CodeLocation { + pub offset: usize, + + pub line: usize, + pub column: usize, + + pub line_start_offset: usize, + pub line_end_offset: usize, +} + +#[allow(clippy::module_name_repetitions)] +pub fn location_to_offset(mut file: &str, mut line: usize, column: usize) -> Option { + let mut offset = 0; + while line > 1 { + let pos = file.find('\n')?; + offset += pos + 1; + file = &file[pos + 1..]; + line -= 1; + } + offset += column - 1; + Some(offset) +} + +#[allow(clippy::module_name_repetitions)] +pub fn offset_to_location(file: &str, offsets: &[u32]) -> Vec { + if offsets.is_empty() { + return vec![]; + } + let mut line = 1; + let mut column = 1; + let max_offset = *offsets.iter().max().expect("offsets is not empty"); + + let mut offset_map = offsets + .iter() + .enumerate() + .map(|(pos, offset)| (*offset, pos)) + .collect::>(); + offset_map.sort_by_key(|v| v.0); + offset_map.reverse(); + + let mut out = vec![ + CodeLocation { + offset: 0, + column: 0, + line: 0, + line_start_offset: 0, + line_end_offset: 0 + }; + offsets.len() + ]; + let mut with_no_known_line_ending = vec![]; + let mut this_line_offset = 0; + for (pos, ch) in file + .chars() + .enumerate() + .chain(std::iter::once((file.len(), ' '))) + { + column += 1; + match offset_map.last() { + Some(x) if x.0 == pos as u32 => { + let out_idx = x.1; + with_no_known_line_ending.push(out_idx); + out[out_idx].offset = pos; + out[out_idx].line = line; + out[out_idx].column = column; + out[out_idx].line_start_offset = this_line_offset; + offset_map.pop(); + } + _ => {} + } + if ch == '\n' { + line += 1; + column = 1; + + for idx in with_no_known_line_ending.drain(..) { + out[idx].line_end_offset = pos; + } + this_line_offset = pos + 1; + + if pos == max_offset as usize + 1 { + break; + } + } + } + let file_end = file.chars().count(); + for idx in with_no_known_line_ending { + out[idx].line_end_offset = file_end; + } + + out +} + +#[cfg(test)] +pub mod tests { + use super::{offset_to_location, CodeLocation}; + + #[test] + fn test() { + assert_eq!( + offset_to_location( + "hello world\n_______________________________________________________", + &[0, 14] + ), + vec![ + CodeLocation { + offset: 0, + line: 1, + column: 2, + line_start_offset: 0, + line_end_offset: 11, + }, + CodeLocation { + offset: 14, + line: 2, + column: 4, + line_start_offset: 12, + line_end_offset: 67 + } + ] + ) + } +} --- a/crates/jrsonnet-parser/src/source.rs +++ b/crates/jrsonnet-parser/src/source.rs @@ -6,21 +6,42 @@ }; use jrsonnet_gcmodule::{Trace, Tracer}; +use jrsonnet_interner::IStr; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use crate::location::{location_to_offset, offset_to_location, CodeLocation}; + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(PartialEq, Eq, Debug, Hash)] -enum Inner { - Real(PathBuf), +#[derive(PartialEq, Eq, Debug, Hash, Clone)] +pub enum SourcePath { + /// This file is located on disk + Path(PathBuf), + /// This file is located somewhere else (I.e http), but it can refer to relative paths, and is egilible for caching + Custom(String), + /// This file is only located in memory, and can't be cached Virtual(Cow<'static, str>), } +impl Trace for SourcePath { + fn trace(&self, _tracer: &mut Tracer) {} + fn is_type_tracked() -> bool { + false + } +} + +impl SourcePath { + /// Should import resolver be able to read file by this path? + pub fn can_load(&self) -> bool { + matches!(self, Self::Path(_) | Self::Custom(_)) + } +} + /// Either real file, or virtual /// Hash of FileName always have same value as raw Path, to make it possible to use with raw_entry_mut #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, PartialEq, Eq, Debug)] -pub struct Source(Rc); +pub struct Source(Rc<(SourcePath, IStr)>); static_assertions::assert_eq_size!(Source, *const ()); impl Trace for Source { @@ -33,55 +54,63 @@ impl Source { /// Fails when path contains inner /../ or /./ references, or not absolute - pub fn new(path: PathBuf) -> Option { - if !path.is_absolute() - || path - .components() - .any(|c| matches!(c, Component::CurDir | Component::ParentDir)) - { - return None; + pub fn new(path: SourcePath, code: IStr) -> Option { + if let SourcePath::Path(path) = &path { + if !path.is_absolute() + || path + .components() + .any(|c| matches!(c, Component::CurDir | Component::ParentDir)) + { + return None; + } } - Some(Self(Rc::new(Inner::Real(path)))) + Some(Self(Rc::new((path, code)))) } - pub fn new_virtual(n: Cow<'static, str>) -> Self { - Self(Rc::new(Inner::Virtual(n))) + pub fn new_virtual(n: Cow<'static, str>, code: IStr) -> Self { + Self(Rc::new((SourcePath::Virtual(n), code))) } pub fn short_display(&self) -> ShortDisplay { ShortDisplay(self.clone()) } - /// Returns None if file is virtual + /// Returns Some if this file is loaded from FS pub fn path(&self) -> Option<&Path> { - match self.inner() { - Inner::Real(r) => Some(r), - Inner::Virtual(_) => None, + match self.source_path() { + SourcePath::Path(r) => Some(r), + SourcePath::Custom(_) => None, + SourcePath::Virtual(_) => None, } } - pub fn repr(&self) -> Result<&Path, &str> { - match self.inner() { - Inner::Real(r) => Ok(r), - Inner::Virtual(v) => Err(v.as_ref()), - } + pub fn code(&self) -> &str { + &self.0 .1 + } + + pub fn source_path(&self) -> &SourcePath { + &self.0 .0 as &SourcePath } - fn inner(&self) -> &Inner { - &self.0 as &Inner + pub fn map_source_locations(&self, locs: &[u32]) -> Vec { + offset_to_location(&self.0 .1, locs) + } + pub fn map_from_source_location(&self, line: usize, column: usize) -> Option { + location_to_offset(&self.0 .1, line, column) } } pub struct ShortDisplay(Source); impl fmt::Display for ShortDisplay { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.0 .0 as &Inner { - Inner::Real(r) => { + match &self.0 .0 .0 as &SourcePath { + SourcePath::Path(r) => { write!( f, "{}", r.file_name().expect("path is valid").to_string_lossy() ) } - Inner::Virtual(n) => write!(f, "{}", n), + SourcePath::Custom(r) => write!(f, "{}", r), + SourcePath::Virtual(n) => write!(f, "{}", n), } } } --- a/crates/jrsonnet-stdlib/build.rs +++ b/crates/jrsonnet-stdlib/build.rs @@ -7,7 +7,10 @@ let parsed = parse( include_str!("./src/std.jsonnet"), &ParserSettings { - file_name: Source::new_virtual(Cow::Borrowed("")), + file_name: Source::new_virtual( + Cow::Borrowed(""), + include_str!("./src/std.jsonnet").into(), + ), }, ) .expect("parse"); --- a/crates/jrsonnet-stdlib/src/expr.rs +++ b/crates/jrsonnet-stdlib/src/expr.rs @@ -15,7 +15,7 @@ jrsonnet_parser::parse( STDLIB_STR, &ParserSettings { - file_name: Source::new_virtual(Cow::Borrowed("")), + file_name: Source::new_virtual(Cow::Borrowed(""), STDLIB_STR.into()), }, ) .unwrap() --- a/crates/jrsonnet-stdlib/src/lib.rs +++ b/crates/jrsonnet-stdlib/src/lib.rs @@ -183,10 +183,10 @@ pub struct StdTracePrinter; impl TracePrinter for StdTracePrinter { - fn print_trace(&self, s: State, loc: CallLocation, value: IStr) { + fn print_trace(&self, _s: State, loc: CallLocation, value: IStr) { eprint!("TRACE:"); if let Some(loc) = loc.0 { - let locs = s.map_source_locations(loc.0.clone(), &[loc.1]); + let locs = loc.0.map_source_locations(&[loc.1]); eprint!(" {}:{}", loc.0.short_display(), locs[0].line); } eprintln!(" {}", value); @@ -212,9 +212,9 @@ } } -pub fn extvar_source(name: &str) -> Source { +pub fn extvar_source(name: &str, code: impl Into) -> Source { let source_name = format!("", name); - Source::new_virtual(Cow::Owned(source_name)) + Source::new_virtual(Cow::Owned(source_name), code.into()) } pub struct ContextInitializer { @@ -260,8 +260,9 @@ .ext_vars .insert(name, TlaArg::String(value)); } - pub fn add_ext_code(&self, name: &str, code: String) -> Result<()> { - let source = extvar_source(name); + pub fn add_ext_code(&self, name: &str, code: impl Into) -> Result<()> { + let code = code.into(); + let source = extvar_source(name, code.clone()); let parsed = jrsonnet_parser::parse( &code, &jrsonnet_parser::ParserSettings { @@ -270,7 +271,6 @@ ) .map_err(|e| ImportSyntaxError { path: source, - source_code: code.clone().into(), error: Box::new(e), })?; // self.data_mut().volatile_files.insert(source_name, code); @@ -297,11 +297,13 @@ .hide() .value( s, - Val::Str(match source.repr() { - Ok(p) => p.display().to_string().into(), - // Virtual files end up as empty strings in std.thisFile - Err(_e) => "".into(), - }), + Val::Str( + source + .path() + .map(|p| p.display().to_string()) + .unwrap_or_else(String::new) + .into(), + ), ) .expect("this object builder is empty"); let stdlib_with_this_file = builder.build(); @@ -343,7 +345,7 @@ settings: Rc>, ))] fn builtin_ext_var(this: &builtin_ext_var, s: State, x: IStr) -> Result { - let ctx = s.create_default_context(extvar_source(&x)); + let ctx = s.create_default_context(extvar_source(&x, "")); Ok(Any(this .settings .borrow() -- gitstuff