git.delta.rocks / jrsonnet / refs/commits / bfcd43bbf560

difftreelog

feat support imports from fifo on unix

Yaroslav Bolyukin2024-03-17parent: #d78cbb7.patch.diff
in: master

3 files changed

modifiedcrates/jrsonnet-evaluator/src/import.rsdiffbeforeafterboth
before · crates/jrsonnet-evaluator/src/import.rs
1use std::{2	any::Any,3	cell::RefCell,4	env::current_dir,5	fs,6	io::{ErrorKind, Read},7	path::{Path, PathBuf},8};910use fs::File;11use jrsonnet_gcmodule::Trace;12use jrsonnet_parser::{SourceDirectory, SourceFile, SourcePath};1314use crate::{15	bail,16	error::{ErrorKind::*, Result},17};1819/// Implements file resolution logic for `import` and `importStr`20pub trait ImportResolver: Trace {21	/// Resolves file path, e.g. `(/home/user/manifests, b.libjsonnet)` can correspond22	/// both to `/home/user/manifests/b.libjsonnet` and to `/home/user/${vendor}/b.libjsonnet`23	/// where `${vendor}` is a library path.24	///25	/// `from` should only be returned from [`ImportResolver::resolve`], or from other defined file, any other value26	/// may result in panic27	fn resolve_from(&self, from: &SourcePath, path: &str) -> Result<SourcePath> {28		bail!(ImportNotSupported(from.clone(), path.into()))29	}30	fn resolve_from_default(&self, path: &str) -> Result<SourcePath> {31		self.resolve_from(&SourcePath::default(), path)32	}33	/// Resolves absolute path, doesn't supports jpath and other fancy things34	fn resolve(&self, path: &Path) -> Result<SourcePath> {35		bail!(AbsoluteImportNotSupported(path.to_owned()))36	}3738	/// Load resolved file39	/// This should only be called with value returned from [`ImportResolver::resolve_file`]/[`ImportResolver::resolve`],40	/// this cannot be resolved using associated type, as evaluator uses object instead of generic for [`ImportResolver`]41	fn load_file_contents(&self, resolved: &SourcePath) -> Result<Vec<u8>>;4243	/// For downcasts44	fn as_any(&self) -> &dyn Any;45}4647/// Dummy resolver, can't resolve/load any file48#[derive(Trace)]49pub struct DummyImportResolver;50impl ImportResolver for DummyImportResolver {51	fn load_file_contents(&self, _resolved: &SourcePath) -> Result<Vec<u8>> {52		panic!("dummy resolver can't load any file")53	}5455	fn as_any(&self) -> &dyn Any {56		self57	}58}59#[allow(clippy::use_self)]60impl Default for Box<dyn ImportResolver> {61	fn default() -> Self {62		Box::new(DummyImportResolver)63	}64}6566/// File resolver, can load file from both FS and library paths67#[derive(Default, Trace)]68pub struct FileImportResolver {69	/// Library directories to search for file.70	/// Referred to as `jpath` in original jsonnet implementation.71	library_paths: RefCell<Vec<PathBuf>>,72}73impl FileImportResolver {74	pub fn new(jpath: Vec<PathBuf>) -> Self {75		Self {76			library_paths: RefCell::new(jpath),77		}78	}79	/// Dynamically add new jpath, used by bindings80	pub fn add_jpath(&self, path: PathBuf) {81		self.library_paths.borrow_mut().push(path);82	}83}8485impl ImportResolver for FileImportResolver {86	fn resolve_from(&self, from: &SourcePath, path: &str) -> Result<SourcePath> {87		let mut direct = if let Some(f) = from.downcast_ref::<SourceFile>() {88			let mut o = f.path().to_owned();89			o.pop();90			o91		} else if let Some(d) = from.downcast_ref::<SourceDirectory>() {92			d.path().to_owned()93		} else if from.is_default() {94			current_dir().map_err(|e| ImportIo(e.to_string()))?95		} else {96			unreachable!("resolver can't return this path")97		};98		direct.push(path);99		if direct.is_file() {100			Ok(SourcePath::new(SourceFile::new(101				direct.canonicalize().map_err(|e| ImportIo(e.to_string()))?,102			)))103		} else {104			for library_path in self.library_paths.borrow().iter() {105				let mut cloned = library_path.clone();106				cloned.push(path);107				if cloned.exists() {108					return Ok(SourcePath::new(SourceFile::new(109						cloned.canonicalize().map_err(|e| ImportIo(e.to_string()))?,110					)));111				}112			}113			bail!(ImportFileNotFound(from.clone(), path.to_owned()))114		}115	}116	fn resolve(&self, path: &Path) -> Result<SourcePath> {117		let meta = match fs::metadata(path) {118			Ok(v) => v,119			Err(e) if e.kind() == ErrorKind::NotFound => {120				bail!(AbsoluteImportFileNotFound(path.to_owned()))121			}122			Err(e) => bail!(ImportIo(e.to_string())),123		};124		if meta.is_file() {125			Ok(SourcePath::new(SourceFile::new(126				path.canonicalize().map_err(|e| ImportIo(e.to_string()))?,127			)))128		} else if meta.is_dir() {129			Ok(SourcePath::new(SourceDirectory::new(130				path.canonicalize().map_err(|e| ImportIo(e.to_string()))?,131			)))132		} else {133			unreachable!("this can't be a symlink")134		}135	}136137	fn load_file_contents(&self, id: &SourcePath) -> Result<Vec<u8>> {138		let path = if let Some(f) = id.downcast_ref::<SourceFile>() {139			f.path()140		} else if id.downcast_ref::<SourceDirectory>().is_some() || id.is_default() {141			bail!(ImportIsADirectory(id.clone()))142		} else {143			unreachable!("other types are not supported in resolve");144		};145		let mut file = File::open(path).map_err(|_e| ResolvedFileNotFound(id.clone()))?;146		let mut out = Vec::new();147		file.read_to_end(&mut out)148			.map_err(|e| ImportIo(e.to_string()))?;149		Ok(out)150	}151152	fn as_any(&self) -> &dyn Any {153		self154	}155156	fn resolve_from_default(&self, path: &str) -> Result<SourcePath> {157		self.resolve_from(&SourcePath::default(), path)158	}159}
after · crates/jrsonnet-evaluator/src/import.rs
1use std::{2	any::Any,3	cell::RefCell,4	env::current_dir,5	fs,6	io::{ErrorKind, Read},7	path::{Path, PathBuf},8};910use fs::File;11use jrsonnet_gcmodule::Trace;12use jrsonnet_interner::IBytes;13use jrsonnet_parser::{SourceDirectory, SourceFile, SourcePath, SourceFifo};1415use crate::{16	bail,17	error::{ErrorKind::*, Result},18};1920/// Implements file resolution logic for `import` and `importStr`21pub trait ImportResolver: Trace {22	/// Resolves file path, e.g. `(/home/user/manifests, b.libjsonnet)` can correspond23	/// both to `/home/user/manifests/b.libjsonnet` and to `/home/user/${vendor}/b.libjsonnet`24	/// where `${vendor}` is a library path.25	///26	/// `from` should only be returned from [`ImportResolver::resolve`], or from other defined file, any other value27	/// may result in panic28	fn resolve_from(&self, from: &SourcePath, path: &str) -> Result<SourcePath> {29		bail!(ImportNotSupported(from.clone(), path.into()))30	}31	fn resolve_from_default(&self, path: &str) -> Result<SourcePath> {32		self.resolve_from(&SourcePath::default(), path)33	}34	/// Resolves absolute path, doesn't supports jpath and other fancy things35	fn resolve(&self, path: &Path) -> Result<SourcePath> {36		bail!(AbsoluteImportNotSupported(path.to_owned()))37	}3839	/// Load resolved file40	/// This should only be called with value returned from [`ImportResolver::resolve_file`]/[`ImportResolver::resolve`],41	/// this cannot be resolved using associated type, as evaluator uses object instead of generic for [`ImportResolver`]42	fn load_file_contents(&self, resolved: &SourcePath) -> Result<Vec<u8>>;4344	/// For downcasts45	fn as_any(&self) -> &dyn Any;46}4748/// Dummy resolver, can't resolve/load any file49#[derive(Trace)]50pub struct DummyImportResolver;51impl ImportResolver for DummyImportResolver {52	fn load_file_contents(&self, _resolved: &SourcePath) -> Result<Vec<u8>> {53		panic!("dummy resolver can't load any file")54	}5556	fn as_any(&self) -> &dyn Any {57		self58	}59}60#[allow(clippy::use_self)]61impl Default for Box<dyn ImportResolver> {62	fn default() -> Self {63		Box::new(DummyImportResolver)64	}65}6667/// File resolver, can load file from both FS and library paths68#[derive(Default, Trace)]69pub struct FileImportResolver {70	/// Library directories to search for file.71	/// Referred to as `jpath` in original jsonnet implementation.72	library_paths: RefCell<Vec<PathBuf>>,73}74impl FileImportResolver {75	pub fn new(jpath: Vec<PathBuf>) -> Self {76		Self {77			library_paths: RefCell::new(jpath),78		}79	}80	/// Dynamically add new jpath, used by bindings81	pub fn add_jpath(&self, path: PathBuf) {82		self.library_paths.borrow_mut().push(path);83	}84}8586/// Create `SourcePath` from path, handling directories/Fifo files (on unix)/etc87fn check_path(path: &Path) -> Result<Option<SourcePath>> {88	let meta = match fs::metadata(path) {89		Ok(v) => v,90		Err(e) if e.kind() == ErrorKind::NotFound => {91			return Ok(None);92		}93		Err(e) => bail!(ImportIo(e.to_string())),94	};95	let ty = meta.file_type();96	if ty.is_file() {97		return Ok(Some(SourcePath::new(SourceFile::new(98			path.canonicalize().map_err(|e| ImportIo(e.to_string()))?,99		))));100	}101	let ty = meta.file_type();102	#[cfg(unix)]103	{104		use std::os::unix::fs::FileTypeExt;105		if ty.is_fifo() {106			let file = fs::read(path).map_err(|e| ImportIo(format!("FIFO read failed: {e}")))?;107			return Ok(Some(SourcePath::new(SourceFifo(108				format!("{}", path.display()),109				IBytes::from(file.as_slice()),110			))));111		}112	}113	// Block device/some other magic thing.114	Err(RuntimeError("special file can't be imported".into()).into())115}116117impl ImportResolver for FileImportResolver {118	fn resolve_from(&self, from: &SourcePath, path: &str) -> Result<SourcePath> {119		let mut direct = if let Some(f) = from.downcast_ref::<SourceFile>() {120			let mut o = f.path().to_owned();121			o.pop();122			o123		} else if let Some(d) = from.downcast_ref::<SourceDirectory>() {124			d.path().to_owned()125		} else if from.is_default() {126			current_dir().map_err(|e| ImportIo(e.to_string()))?127		} else {128			unreachable!("resolver can't return this path")129		};130131		direct.push(path);132		if let Some(direct) = check_path(&direct)? {133			return Ok(direct);134		}135		for library_path in self.library_paths.borrow().iter() {136			let mut cloned = library_path.clone();137			cloned.push(path);138			if let Some(cloned) = check_path(&cloned)? {139				return Ok(cloned);140			}141		}142		bail!(ImportFileNotFound(from.clone(), path.to_owned()))143	}144	fn resolve(&self, path: &Path) -> Result<SourcePath> {145		let Some(source) = check_path(path)? else {146			bail!(AbsoluteImportFileNotFound(path.to_owned()))147		};148		Ok(source)149	}150151	fn load_file_contents(&self, id: &SourcePath) -> Result<Vec<u8>> {152		let path = if let Some(f) = id.downcast_ref::<SourceFile>() {153			f.path()154		} else if id.downcast_ref::<SourceDirectory>().is_some() {155			bail!(ImportIsADirectory(id.clone()))156		} else if let Some(f) = id.downcast_ref::<SourceFifo>() {157			return Ok(f.1.to_vec());158		} else {159			unreachable!("other types are not supported in resolve");160		};161		let mut file = File::open(path).map_err(|_e| ResolvedFileNotFound(id.clone()))?;162		let mut out = Vec::new();163		file.read_to_end(&mut out)164			.map_err(|e| ImportIo(e.to_string()))?;165		Ok(out)166	}167168	fn as_any(&self) -> &dyn Any {169		self170	}171172	fn resolve_from_default(&self, path: &str) -> Result<SourcePath> {173		self.resolve_from(&SourcePath::default(), path)174	}175}
modifiedcrates/jrsonnet-parser/src/lib.rsdiffbeforeafterboth
--- a/crates/jrsonnet-parser/src/lib.rs
+++ b/crates/jrsonnet-parser/src/lib.rs
@@ -11,7 +11,9 @@
 mod source;
 mod unescape;
 pub use location::CodeLocation;
-pub use source::{Source, SourceDirectory, SourceFile, SourcePath, SourcePathT, SourceVirtual};
+pub use source::{
+	Source, SourceDirectory, SourceFifo, SourceFile, SourcePath, SourcePathT, SourceVirtual,
+};
 
 pub struct ParserSettings {
 	pub source: Source,
modifiedcrates/jrsonnet-parser/src/source.rsdiffbeforeafterboth
--- a/crates/jrsonnet-parser/src/source.rs
+++ b/crates/jrsonnet-parser/src/source.rs
@@ -7,7 +7,7 @@
 };
 
 use jrsonnet_gcmodule::{Trace, Tracer};
-use jrsonnet_interner::IStr;
+use jrsonnet_interner::{IBytes, IStr};
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "structdump")]
@@ -75,6 +75,7 @@
 /// - [`SourceFile`] - for any file
 /// - [`SourceDirectory`] - for resolution from CWD
 /// - [`SourceVirtual`] - for stdlib/ext-str
+/// - [`SourceFifo`] - for /dev/fd/X (This path may appear with `jrsonnet <(command_that_produces_jsonnet)`)
 ///
 /// From all of those, only [`SourceVirtual`] may be constructed manually, any other path kind should be only obtained
 /// from assigned `ImportResolver`
@@ -254,6 +255,37 @@
 	any_ext_impl!(SourcePathT);
 }
 
+/// Represents resolved FIFO file, those files may only be read once, and this type is only used for
+/// unix, where user might want to do `jrsonnet <(command_that_produces_jsonnet_source)`
+/// In most cases, user most probably want to use `jrsonnet -` instead of `jrsonnet /dev/stdin`
+/// for better cross-platform support.
+// PartialEq is limited to ptr equality
+#[allow(clippy::derived_hash_with_manual_eq)]
+#[derive(Trace, Debug, Hash)]
+pub struct SourceFifo(pub String, pub IBytes);
+impl PartialEq for SourceFifo {
+	fn eq(&self, other: &Self) -> bool {
+		std::ptr::eq(self, other)
+	}
+}
+impl fmt::Display for SourceFifo {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "fifo({:?})", self.0)
+	}
+}
+impl SourcePathT for SourceFifo {
+	fn is_default(&self) -> bool {
+		// In case of FD input, user won't expect relative paths to be resolved from /dev/fd/
+		true
+	}
+
+	fn path(&self) -> Option<&Path> {
+		None
+	}
+
+	any_ext_impl!(SourcePathT);
+}
+
 /// Either real file, or virtual
 /// Hash of FileName always have same value as raw Path, to make it possible to use with raw_entry_mut
 #[cfg_attr(feature = "structdump", derive(Codegen))]