use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::Path; use std::rc::Rc; use avril_core::{Error, ModuleFormat, ModuleLoader, ModuleSource}; use futures::future::LocalBoxFuture; use oxc::allocator::Allocator; use oxc::codegen::{Codegen, CodegenOptions}; use oxc::diagnostics::{Diagnostics, NamedSource}; use oxc::parser::Parser; use oxc::semantic::SemanticBuilder; use oxc::span::SourceType; use oxc::transformer::{EnvOptions, Module, TransformOptions, Transformer}; use oxc_sourcemap::Token; use crate::data_modules::DataModule; use crate::local_packages::ResolveMode; use crate::modules::{ModuleGraph, ResolutionMode, ResolutionTarget, data_url, http_url}; use crate::registry::npm::package::NpmModuleKind; #[derive(Default)] pub(crate) struct LoaderState { pub(crate) entry: Option, pub(crate) records: HashMap, } #[derive(Clone, Copy, PartialEq, Eq)] enum ModuleState { Ready, Complete, } pub(crate) struct ModuleRecord { state: ModuleState, source: Option, specifiers: Option>, imports: HashMap, source_map: Option>, } /// Warm-only notice for a skipped edge: the unsupported specifier plus the /// importing module. Strict walks never invoke it. pub type BuiltinWarning = Rc; impl ModuleGraph { pub fn set_entry(&self, name: &str) { self.loader.borrow_mut().entry = Some(name.to_owned()); } pub fn entry_url(&self) -> String { self.loader .borrow() .entry .as_deref() .and_then(|name| self.module_url(name)) .unwrap_or_default() } pub fn info(&self, root: &str) -> Result { let complete: Vec<(String, HashMap)> = { let loader = self.loader.borrow(); let records = &loader.records; records .iter() .filter(|(_, record)| record.state == ModuleState::Complete) .map(|(name, record)| (name.clone(), record.imports.clone())) .collect() }; let modules: BTreeMap<_, _> = complete .into_iter() .map(|(name, imports)| { self.check_access(&name)?; let imports: BTreeMap<_, _> = imports.into_iter().collect(); let metadata = self.package_metadata(&name)?; Ok((name.clone(), serde_json::json!({"url": self.module_url(&name), "imports": imports, "package": metadata}))) }) .collect::>()?; Ok(serde_json::json!({"root": root, "modules": modules})) } /// Local execution format by extension. `.js` stays ESM per avril /// policy (no `package.json` `type` behavior); explicit `.cjs`/`.cts` /// are CommonJS and `.json` is JSON for `require`. fn local_format(name: &str) -> ModuleFormat { let lower = name.to_ascii_lowercase(); if lower.ends_with(".cjs") || lower.ends_with(".cts") { ModuleFormat::CommonJs } else if lower.ends_with(".json") { ModuleFormat::Json } else { ModuleFormat::Esm } } fn module_format_sync(&self, name: &str) -> Result { if name.starts_with("avril:") || http_url(name).is_some() || data_url(name).is_some() { // Remote CJS stays explicitly unsupported; remote files execute // as ESM so `require` of them fails with the ESM message. return Ok(ModuleFormat::Esm); } if self.npm.is_reserved_npm_path(Path::new(name)) { return match self.npm.check_access(Path::new(name)) { Ok(NpmModuleKind::Esm) | Ok(NpmModuleKind::TypeScript) => Ok(ModuleFormat::Esm), Ok(NpmModuleKind::CommonJs) => Ok(ModuleFormat::CommonJs), Ok(NpmModuleKind::Json) => Ok(ModuleFormat::Json), Ok(NpmModuleKind::Declaration) => Err(Error::Module(format!( "{name}: npm declaration files have no runtime code" ))), Err(error) => Err(Error::Module(error.to_string())), }; } Ok(Self::local_format(name)) } pub fn prepare_for_add( &self, specifier: String, referrer: String, warn_unsupported: BuiltinWarning, ) -> LocalBoxFuture<'static, Result> { self.prepare_with(specifier, referrer, Some(warn_unsupported)) } fn prepare_with( &self, specifier: String, referrer: String, warn_unsupported: Option, ) -> LocalBoxFuture<'static, Result> { let graph = self.clone(); Box::pin(async move { let (root, request) = if referrer.is_empty() { (data_url(&specifier).map_or(specifier, String::from), None) } else { graph.resolve_tracked(&specifier, &referrer).await? }; let mut requests: BTreeSet<_> = request.into_iter().collect(); let mut pending = vec![root.clone()]; let mut seen = HashSet::new(); let mut links = HashMap::new(); while let Some(name) = pending.pop() { if name.starts_with("avril:") || !seen.insert(name.clone()) { continue; } graph.check_access(&name)?; graph.initialize_package(&name).await?; let is_complete = { graph .loader .borrow() .records .get(&name) .is_some_and(|record| record.state == ModuleState::Complete) }; if is_complete { continue; } let cached = { graph.loader.borrow().records.get(&name).and_then(|record| { Some((record.source.clone()?, record.specifiers.clone()?)) }) }; let format = graph.module_format_sync(&name)?; if format != ModuleFormat::Esm { if cached.is_none() { let raw = graph.original_source(&name).await?; let (code, maps) = if format == ModuleFormat::Json { (raw, None) } else if name.to_ascii_lowercase().ends_with(".cts") { let source_type = graph.source_type(&name)?; let (code, maps) = transpile(&name, &raw, source_type, TransformOptions::default())?; (code, if maps.is_empty() { None } else { Some(maps) }) } else { (raw, None) }; let source: ModuleSource = code.into(); { let mut loader = graph.loader.borrow_mut(); let records = &mut loader.records; match records.get_mut(&name) { Some(record) => { record.source = Some(source); record.specifiers = Some(Vec::new()); record.source_map = maps; } None => { records.insert( name.clone(), ModuleRecord { state: ModuleState::Ready, source: Some(source), specifiers: Some(Vec::new()), imports: HashMap::new(), source_map: maps, }, ); } } } } requests.extend(graph.prepare_npm_roots(&name, format, &[]).await?); links.insert(name, HashMap::new()); continue; } let specifiers = match cached { Some((_, specifiers)) => specifiers, None => { let raw = if name.starts_with(graph.store.options().jsr_registry.as_str()) { let hints = graph.store.hints(&name).await.unwrap_or_default(); let prefetch = futures::future::join_all( hints .iter() .filter(|hint| graph.check_access(hint).is_ok()) .map(|hint| graph.store.source(hint)), ); let (source, _) = futures::future::join(graph.original_source(&name), prefetch).await; source? } else { graph.original_source(&name).await? }; let (code, mappings) = match graph.source_type(&name)? { source_type if source_type.is_typescript() || source_type.is_jsx() => { let options = if source_type.is_jsx() { graph.jsx_config(&name)?.transform_options().map_err( |error| Error::Module(format!("{name}: {error}")), )? } else { TransformOptions::default() }; let (code, mappings) = transpile(&name, &raw, source_type, options)?; (code, Some(mappings)) } _ => (raw, None), }; let specifiers = match import_specifiers(&name, &code) { Ok(specifiers) => specifiers, Err(error) => { // Another prepare may have stored or completed the record // while this one awaited; never clear its source, state, // or imports, only fill a missing source map. if let Some(mappings) = mappings { let mut loader = graph.loader.borrow_mut(); let records = &mut loader.records; match records.get_mut(&name) { Some(record) if record.state != ModuleState::Complete && record.source.is_none() => { record.source_map = Some(mappings); } None => { records.insert( name.clone(), ModuleRecord { state: ModuleState::Ready, source: None, specifiers: None, imports: HashMap::new(), source_map: Some(mappings), }, ); } _ => {} } } return Err(error); } }; let source: ModuleSource = code.into(); { let mut loader = graph.loader.borrow_mut(); let records = &mut loader.records; match records.get_mut(&name) { Some(record) => { // Refresh only the fetch artifacts; a concurrent // prepare's state and imports must survive. record.source = Some(source); record.specifiers = Some(specifiers.clone()); record.source_map = mappings; } None => { records.insert( name.clone(), ModuleRecord { state: ModuleState::Ready, source: Some(source), specifiers: Some(specifiers.clone()), imports: HashMap::new(), source_map: mappings, }, ); } } } specifiers } }; graph.prepare_npm_roots(&name, format, &specifiers).await?; let mut dependencies = Vec::new(); let mut imports = HashMap::new(); for specifier in &specifiers { if let Some(warn) = &warn_unsupported && specifier.starts_with("node:") { warn(specifier, &name); continue; } let (dependency, request) = graph.resolve_tracked(specifier, &name).await?; requests.extend(request); imports.insert(specifier.clone(), dependency.clone()); dependencies.push(dependency); } pending.extend(dependencies.into_iter().rev()); links.insert(name, imports); } graph .store .save_graph(&requests, &seen) .map_err(|error| Error::Module(error.to_string()))?; { let mut loader = graph.loader.borrow_mut(); let records = &mut loader.records; // Every name needing completion is in links; names in seen but not // in links were already Complete and skipped above. for (name, imports) in links { if let Some(record) = records.get_mut(&name) && record.state != ModuleState::Complete { record.imports = imports; // Warm-only walks cache sources without completing // records, so a later strict walk still validates // every edge including skipped builtins. if warn_unsupported.is_none() { record.state = ModuleState::Complete; } } } } Ok(root) }) } } impl ModuleLoader for ModuleGraph { fn resolve(&self, specifier: &str, referrer: &str) -> Result { if specifier.starts_with("avril:") { return Ok(specifier.to_owned()); } self.loader .borrow() .records .get(referrer) .and_then(|record| record.imports.get(specifier)) .cloned() .ok_or_else(|| { Error::Module(format!( "Import was not prepared: {specifier} from {referrer}" )) }) } fn check_access(&self, name: &str) -> Result<(), Error> { if data_url(name).is_some() { DataModule::parse(name)?; return Ok(()); } if name.starts_with("avril:") { return Err(Error::Module(format!( "Unsupported module specifier: {name} (unknown builtin)" ))); } if let Some(url) = http_url(name) { if !url.username().is_empty() || url.password().is_some() { return Err(Error::Module(format!( "HTTP module credentials are not supported: {name}" ))); } if name.starts_with(self.store.options().jsr_registry.as_str()) { self.store .module_base(name) .map_err(|error| Error::Module(error.to_string()))?; } return self .store .check_acquired_remote(name, &self.authority) .map_err(|error| Error::Module(error.to_string())); } // Reserved npm paths always authorize through virtual-URL // authority: project filesystem grants never cover them, and // unowned cache paths fail instead of loading as local files. // Declarations have no executable code even when readable. if self.npm.is_reserved_npm_path(Path::new(name)) { return match self.npm.check_access(Path::new(name)) { Ok(NpmModuleKind::Declaration) => Err(Error::Module(format!( "{name}: npm declaration files have no runtime code" ))), Ok(_) => Ok(()), Err(error) => Err(Error::Module(error.to_string())), }; } self.target(name).map(|_| ()) } fn load(&self, name: &str) -> Result { self.check_access(name)?; if let Some(source) = self .loader .borrow() .records .get(name) .and_then(|record| record.source.clone()) { return Ok(source); } let format = self.module_format_sync(name)?; match format { ModuleFormat::Esm => Err(Error::Module(format!("Module was not prepared: {name}"))), ModuleFormat::CommonJs | ModuleFormat::Json => { let (code, maps) = if matches!(format, ModuleFormat::CommonJs) && name.to_ascii_lowercase().ends_with(".cts") { let raw = self.sync_source(name)?; let source_type = self.source_type(name)?; let (code, maps) = transpile(name, &raw, source_type, TransformOptions::default())?; (code, if maps.is_empty() { None } else { Some(maps) }) } else { (self.sync_source(name)?, None) }; let source: ModuleSource = code.into(); let mut loader = self.loader.borrow_mut(); let records = &mut loader.records; match records.get_mut(name) { Some(record) => { record.source = Some(source.clone()); record.specifiers = Some(Vec::new()); record.source_map = maps; } None => { records.insert( name.to_owned(), ModuleRecord { state: ModuleState::Complete, source: Some(source.clone()), specifiers: Some(Vec::new()), imports: HashMap::new(), source_map: maps, }, ); } } Ok(source) } } } fn module_format(&self, name: &str) -> Result { self.module_format_sync(name) } fn require_resolve(&self, specifier: &str, referrer: &str) -> Result { self.require_resolve_sync(specifier, referrer) } fn cjs_named_exports(&self, name: &str) -> Result, Error> { self.check_access(name)?; let source = self.load(name)?; self.check_access(name)?; Ok(crate::commonjs::named_exports(&source)) } fn prepare( &self, specifier: String, referrer: String, ) -> LocalBoxFuture<'static, Result> { self.prepare_with(specifier, referrer, None) } /// Mirrors `resolve_tracked`'s decision order, but never acquires: unprepared /// jsr/npm packages error instead of fetching, missing targets are allowed, /// local names present as `file:` URLs, and no lock key is produced. fn resolve_import_meta(&self, specifier: &str, referrer: &str) -> Result { let referrer = self.classify_referrer(referrer)?; if let Some(name) = Self::early_scheme(specifier)? { return Ok(name); } if let Some(path) = referrer.path.as_deref() && referrer.npm { if let Some(url) = data_url(specifier) { return Ok(url.to_string()); } return self .npm .resolve_from_file(specifier, path, ResolutionMode::Import) .map_err(|error| error.to_string()) .and_then(|resolved| { let path = resolved .path .to_str() .ok_or_else(|| "Invalid module path".to_owned())?; url::Url::from_file_path(path) .map(String::from) .map_err(|_| "Invalid module path".to_owned()) }) .map_err(Error::Module); } let mapped = if referrer.inline { None } else if referrer.remote { self.remote.resolve_import_meta(specifier, &referrer.name)? } else { self.packages .resolve_import_meta(specifier, &referrer.name, &self.remote)? }; let effective = match &mapped { Some(crate::local_packages::Resolution::Specifier(target)) => target.as_str(), _ => specifier, }; if effective.starts_with("npm:") { return self.resolve_import_meta_npm(effective, &referrer.name, referrer.remote); } match self.resolve_target(specifier, &referrer, mapped, ResolveMode::PreparedOnly)? { ResolutionTarget::Jsr(specifier) => crate::registry::parse_jsr_request(&specifier) .and_then(|request| self.store.resolve_import_meta_request(&request)) .map_err(|error| Error::Module(error.to_string())), ResolutionTarget::Npm(specifier) => { self.resolve_import_meta_npm(&specifier, &referrer.name, referrer.remote) } ResolutionTarget::Remote(name) => Ok(name), ResolutionTarget::Module(name) if name.starts_with("avril:") || http_url(&name).is_some() || data_url(&name).is_some() => { Ok(name) } ResolutionTarget::Module(name) => self .module_url(&name) .ok_or_else(|| Error::Module("Invalid module path".into())), } } fn module_url(&self, name: &str) -> Option { if http_url(name).is_some() || data_url(name).is_some() { return Some(name.to_owned()); } url::Url::from_file_path(name).ok().map(String::from) } fn map_source_location( &self, name: &str, line: u32, column: u32, ) -> Option<(String, u32, u32)> { let loader = self.loader.borrow(); let records = &loader.records; let mappings = records.get(name)?.source_map.as_ref()?; let position = (line.checked_sub(1)?, column.checked_sub(1)?); let index = mappings .partition_point(|token| (token.get_dst_line(), token.get_dst_col()) <= position) .checked_sub(1)?; let token = mappings[index]; token.get_source_id()?; (token.get_dst_line() == position.0).then(|| { ( name.to_owned(), token.get_src_line() + 1, token.get_src_col() + 1, ) }) } } fn import_specifiers(name: &str, source: &str) -> Result, Error> { let allocator = Allocator::default(); let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse(); if !parsed.diagnostics.is_empty() { let diagnostics: Vec<_> = parsed .diagnostics .into_iter() .map(|diagnostic| diagnostic.render_with_source_code(NamedSource::new(name, source))) .collect(); return Err(Error::Syntax(format!("{name}\n{}", diagnostics.join("")))); } let mut imports: Vec<_> = parsed.module_record.requested_modules.iter().collect(); imports.sort_by_key(|(_, requests)| requests.iter().map(|request| request.span.start).min()); Ok(imports .into_iter() .map(|(specifier, _)| specifier.to_string()) .collect()) } pub(crate) fn transpile( name: &str, source: &str, source_type: SourceType, options: TransformOptions, ) -> Result<(String, Vec), Error> { let (code, map) = compile(name, Path::new(name), source, source_type, options)?; Ok((code, map.get_tokens().collect())) } /// Strips types and compiles JSX, keeping CommonJS source types' module /// syntax. `path` is the file name the transformer sees, as in JSX /// development metadata; maps always name `name`. pub(crate) fn compile( name: &str, path: &Path, source: &str, source_type: SourceType, mut options: TransformOptions, ) -> Result<(String, oxc_sourcemap::SourceMap<'static>), Error> { if source_type.is_typescript_definition() { return Err(Error::Module(format!( "{name}: TypeScript declaration files have no runtime code" ))); } let commonjs = source_type.is_commonjs(); let report = |diagnostics: Diagnostics| { let rendered: Vec<_> = diagnostics .into_iter() .map(|diagnostic| diagnostic.render_with_source_code(NamedSource::new(name, source))) .collect(); format!( "Failed to transpile {name}\n{}", rendered.join("").trim_end() ) }; let allocator = Allocator::default(); let parsed = Parser::new( &allocator, source, if commonjs { source_type } else { source_type.with_module(true) }, ) .parse(); if !parsed.diagnostics.is_empty() { return Err(Error::Syntax(report(parsed.diagnostics))); } let mut program = parsed.program; let semantic = SemanticBuilder::new_compiler() .with_enum_eval(true) .build(&program); if !semantic.diagnostics.is_empty() { return Err(Error::Syntax(report(semantic.diagnostics))); } options.env = EnvOptions { module: if commonjs { Module::Preserve } else { Module::Esm }, ..EnvOptions::default() }; let transformed = Transformer::new(&allocator, path, &options) .build_with_scoping(semantic.semantic.into_scoping(), &mut program); // Oxc warns where its output would not match TypeScript, such as for CommonJS-only syntax. if !transformed.diagnostics.is_empty() { return Err(Error::Module(report(transformed.diagnostics))); } let generated = Codegen::new() .with_options(CodegenOptions { source_map_path: Some(name.into()), ..CodegenOptions::default() }) .build(&program); Ok(( generated.code, generated.map.expect("maps are requested").into_owned(), )) } #[cfg(test)] mod warm_builtin_tests { use avril_core::ModuleLoader; use std::cell::RefCell; use std::rc::Rc; #[test] fn strict_prepare_still_rejects_builtin_after_warm() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("avril.conf"), "").unwrap(); std::fs::write( dir.path().join("parent.mjs"), "import './child.mjs';\nexport default 1;\n", ) .unwrap(); std::fs::write( dir.path().join("child.mjs"), "import 'node:fs';\nimport './sibling.mjs';\nexport default 2;\n", ) .unwrap(); std::fs::write(dir.path().join("sibling.mjs"), "export default 3;\n").unwrap(); let (_runtime, graph) = crate::modules::offline_graph(dir.path()); let warned = Rc::new(RefCell::new(Vec::new())); let warn = { let warned = warned.clone(); Rc::new(move |specifier: &str, referrer: &str| { warned .borrow_mut() .push((specifier.to_owned(), referrer.to_owned())); }) }; let parent = dir.path().join("parent.mjs").to_string_lossy().into_owned(); futures::executor::block_on(graph.prepare_for_add(parent.clone(), String::new(), warn)) .unwrap(); assert!( warned.borrow().iter().any(|(specifier, referrer)| { specifier == "node:fs" && referrer.ends_with("child.mjs") }), "{warned:?}", ); let error = futures::executor::block_on(graph.prepare(parent, String::new())).unwrap_err(); assert!( error .to_string() .contains("builtin modules are not supported"), "{error:?}" ); } }