difftreelog
feat async import ergonomics
in: master
1 file changed
crates/jrsonnet-evaluator/src/async_import.rsdiffbeforeafterboth1use std::{any::Any, cell::RefCell, future::Future, rc::Rc};23use jrsonnet_gcmodule::Acyclic;4use jrsonnet_ir::{IStr, Source, SourcePath, visit::Visitor};5use rustc_hash::FxHashMap;67use crate::{AsPathLike, FileData, ImportResolver, ResolvePathOwned, Result, State};89pub struct Import {10 path: ResolvePathOwned,11 expression: bool,12}1314pub struct FoundImports(Vec<Import>);15impl Visitor for FoundImports {16 fn visit_import(&mut self, expression: bool, value: IStr) {17 self.0.push(Import {18 path: ResolvePathOwned::Str(value.to_string()),19 expression,20 });21 }22}2324pub trait AsyncImportResolver {25 type Error;26 /// Resolves file path, e.g. `(/home/user/manifests, b.libjsonnet)` can correspond27 /// both to `/home/user/manifests/b.libjsonnet` and to `/home/user/${vendor}/b.libjsonnet`28 /// where `${vendor}` is a library path.29 ///30 /// `from` should only be returned from [`ImportResolver::resolve`],31 /// or from other defined file, any other value may result in panic32 fn resolve_from(33 &self,34 from: &SourcePath,35 path: &dyn AsPathLike,36 ) -> impl Future<Output = Result<SourcePath, Self::Error>>;37 fn resolve_from_default(38 &self,39 path: &dyn AsPathLike,40 ) -> impl Future<Output = Result<SourcePath, Self::Error>> {41 async { self.resolve_from(&SourcePath::default(), path).await }42 }4344 /// Load resolved file45 /// This should only be called with value returned46 /// from [`ImportResolver::resolve_file`]/[`ImportResolver::resolve`],47 /// this cannot be resolved using associated type,48 /// as the evaluator uses object instead of generic for [`ImportResolver`]49 fn load_file_contents(50 &self,51 resolved: &SourcePath,52 ) -> impl Future<Output = Result<Vec<u8>, Self::Error>>;53}5455#[derive(Acyclic)]56struct ResolvedImportResolver {57 resolved: RefCell<FxHashMap<(SourcePath, ResolvePathOwned), (SourcePath, bool)>>,58}59impl ImportResolver for ResolvedImportResolver {60 fn load_file_contents(&self, _resolved: &SourcePath) -> crate::Result<Vec<u8>> {61 unreachable!("all files should be loaded at this point");62 }6364 fn resolve_from(&self, from: &SourcePath, path: &dyn AsPathLike) -> crate::Result<SourcePath> {65 Ok(self66 .resolved67 .borrow()68 .get(&(from.clone(), path.as_path().to_owned()))69 .expect("all imports should be resolved at this point")70 .071 .clone())72 }7374 fn resolve_from_default(&self, path: &dyn AsPathLike) -> crate::Result<SourcePath> {75 self.resolve_from(&SourcePath::default(), path)76 }77}7879enum Job {80 LoadFile { path: SourcePath, parse: bool },81 ParseFile(SourcePath),82 ResolveImport { from: SourcePath, import: Import },83}8485#[allow(clippy::future_not_send)]86pub async fn async_import<H>(s: State, handler: H, path: &dyn AsPathLike) -> Result<(), H::Error>87where88 H: AsyncImportResolver,89{90 let resolved = (s.import_resolver() as &dyn Any)91 .downcast_ref::<ResolvedImportResolver>()92 .expect("for async imports, import_resolver should be set to ResolvedImportResolver");9394 let mut queue = vec![Job::LoadFile {95 path: handler.resolve_from_default(path).await?,96 parse: true,97 }];98 while let Some(job) = queue.pop() {99 match job {100 Job::LoadFile { path, parse } => {101 if !s.0.file_cache.borrow().contains_key(&path) {102 let data = handler.load_file_contents(&path).await?;103 s.0.file_cache104 .borrow_mut()105 .insert(path.clone(), FileData::new_bytes(data.as_slice().into()));106 }107 if parse {108 queue.push(Job::ParseFile(path));109 }110 }111 Job::ParseFile(path) => {112 if let Some(file) = s.0.file_cache.borrow_mut().get_mut(&path)113 && file.parsed.is_none()114 {115 let Some(code) = file.get_string() else {116 continue;117 };118 let source = Source::new(path.clone(), code.clone());119 // If failed - then skip import120 file.parsed = crate::parse_jsonnet(&code, source).map(Rc::new).ok();121 if let Some(parsed) = &file.parsed {122 let mut imports = FoundImports(vec![]);123 imports.visit_expr(parsed);124 for import in imports.0 {125 queue.push(Job::ResolveImport {126 from: path.clone(),127 import,128 });129 }130 }131 }132 }133 Job::ResolveImport { from, import } => {134 {135 let mut resolved_map = resolved.resolved.borrow_mut();136 if let Some((resolved, expression)) =137 resolved_map.get_mut(&(from.clone(), import.path.clone()))138 {139 if import.expression && !*expression {140 *expression = true;141 queue.push(Job::ParseFile(resolved.clone()));142 }143 continue;144 }145 }146 let resolved = handler.resolve_from(&from, &import.path).await?;147 queue.push(Job::LoadFile {148 path: resolved,149 parse: import.expression,150 });151 }152 }153 }154 Ok(())155}1use std::{any::Any, cell::RefCell, future::Future, rc::Rc};23use jrsonnet_gcmodule::Acyclic;4use jrsonnet_ir::{IStr, Source, SourcePath, visit::Visitor};5use rustc_hash::FxHashMap;67use crate::{AsPathLike, FileData, ImportResolver, ResolvePathOwned, Result, State};89pub struct Import {10 path: ResolvePathOwned,11 expression: bool,12}1314pub struct FoundImports(Vec<Import>);15impl Visitor for FoundImports {16 fn visit_import(&mut self, expression: bool, value: IStr) {17 self.0.push(Import {18 path: ResolvePathOwned::Str(value.to_string()),19 expression,20 });21 }22}2324pub trait AsyncImportResolver {25 type Error;26 /// Resolves file path, e.g. `(/home/user/manifests, b.libjsonnet)` can correspond27 /// both to `/home/user/manifests/b.libjsonnet` and to `/home/user/${vendor}/b.libjsonnet`28 /// where `${vendor}` is a library path.29 ///30 /// `from` should only be returned from [`ImportResolver::resolve`],31 /// or from other defined file, any other value may result in panic32 fn resolve_from(33 &self,34 from: &SourcePath,35 path: &dyn AsPathLike,36 ) -> impl Future<Output = Result<SourcePath, Self::Error>>;37 fn resolve_from_default(38 &self,39 path: &dyn AsPathLike,40 ) -> impl Future<Output = Result<SourcePath, Self::Error>> {41 async { self.resolve_from(&SourcePath::default(), path).await }42 }4344 /// Load resolved file45 /// This should only be called with value returned46 /// from [`ImportResolver::resolve_file`]/[`ImportResolver::resolve`],47 /// this cannot be resolved using associated type,48 /// as the evaluator uses object instead of generic for [`ImportResolver`]49 fn load_file_contents(50 &self,51 resolved: &SourcePath,52 ) -> impl Future<Output = Result<Vec<u8>, Self::Error>>;53}5455#[derive(Acyclic, Default)]56pub struct ResolvedImportResolver {57 resolved: RefCell<FxHashMap<(SourcePath, ResolvePathOwned), (SourcePath, bool)>>,58}59impl ResolvedImportResolver {60 pub fn new() -> Self {61 Self::default()62 }63}64impl ImportResolver for ResolvedImportResolver {65 fn load_file_contents(&self, _resolved: &SourcePath) -> crate::Result<Vec<u8>> {66 unreachable!("all files should be loaded at this point");67 }6869 fn resolve_from(&self, from: &SourcePath, path: &dyn AsPathLike) -> crate::Result<SourcePath> {70 Ok(self71 .resolved72 .borrow()73 .get(&(from.clone(), path.as_path().to_owned()))74 .expect("all imports should be resolved at this point")75 .076 .clone())77 }7879 fn resolve_from_default(&self, path: &dyn AsPathLike) -> crate::Result<SourcePath> {80 self.resolve_from(&SourcePath::default(), path)81 }82}8384enum Job {85 LoadFile { path: SourcePath, parse: bool },86 ParseFile(SourcePath),87 ResolveImport { from: SourcePath, import: Import },88}8990#[allow(clippy::future_not_send)]91pub async fn async_import<H>(92 s: State,93 handler: H,94 path: &dyn AsPathLike,95) -> Result<SourcePath, H::Error>96where97 H: AsyncImportResolver,98{99 let resolved = (s.import_resolver() as &dyn Any)100 .downcast_ref::<ResolvedImportResolver>()101 .expect("for async imports, import_resolver should be set to ResolvedImportResolver");102103 let entry = handler.resolve_from_default(path).await?;104 let mut queue = vec![Job::LoadFile {105 path: entry.clone(),106 parse: true,107 }];108 while let Some(job) = queue.pop() {109 match job {110 Job::LoadFile { path, parse } => {111 if !s.0.file_cache.borrow().contains_key(&path) {112 let data = handler.load_file_contents(&path).await?;113 s.0.file_cache114 .borrow_mut()115 .insert(path.clone(), FileData::new_bytes(data.as_slice().into()));116 }117 if parse {118 queue.push(Job::ParseFile(path));119 }120 }121 Job::ParseFile(path) => {122 if let Some(file) = s.0.file_cache.borrow_mut().get_mut(&path)123 && file.parsed.is_none()124 {125 let Some(code) = file.get_string() else {126 continue;127 };128 let source = Source::new(path.clone(), code.clone());129 // If failed - then skip import130 file.parsed = crate::parse_jsonnet(&code, source).map(Rc::new).ok();131 if let Some(parsed) = &file.parsed {132 let mut imports = FoundImports(vec![]);133 imports.visit_expr(parsed);134 for import in imports.0 {135 queue.push(Job::ResolveImport {136 from: path.clone(),137 import,138 });139 }140 }141 }142 }143 Job::ResolveImport { from, import } => {144 {145 let mut resolved_map = resolved.resolved.borrow_mut();146 if let Some((resolved, expression)) =147 resolved_map.get_mut(&(from.clone(), import.path.clone()))148 {149 if import.expression && !*expression {150 *expression = true;151 queue.push(Job::ParseFile(resolved.clone()));152 }153 continue;154 }155 }156 let resolved_path = handler.resolve_from(&from, &import.path).await?;157 resolved.resolved.borrow_mut().insert(158 (from.clone(), import.path.clone()),159 (resolved_path.clone(), import.expression),160 );161 queue.push(Job::LoadFile {162 path: resolved_path,163 parse: import.expression,164 });165 }166 }167 }168 Ok(entry)169}