char/avril

git clone https://git.t4t.associates/char/avril

Charlotte Somdeslop: factor module prep/transpilatione/etc into loader.rsab5df082026-10-27

use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::path::Path;
use std::rc::Rc;

use avril_core::{Error, ModuleFormat, ModuleLoader, ModuleSource};
use futures::future::LocalBoxFuture;
use oxc::allocator::Allocator;
use oxc::codegen::{Codegen, CodegenOptions};
use oxc::diagnostics::{Diagnostics, NamedSource};
use oxc::parser::Parser;
use oxc::semantic::SemanticBuilder;
use oxc::span::SourceType;
use oxc::transformer::{EnvOptions, Module, TransformOptions, Transformer};
use oxc_sourcemap::Token;

use crate::data_modules::DataModule;
use crate::local_packages::ResolveMode;
use crate::modules::{ModuleGraph, ResolutionMode, ResolutionTarget, data_url, http_url};
use crate::registry::npm::package::NpmModuleKind;

#[derive(Default)]
pub(crate) struct LoaderState {
    pub(crate) entry: Option<String>,
    pub(crate) records: HashMap<String, ModuleRecord>,
}

#[derive(Clone, Copy, PartialEq, Eq)]
enum ModuleState {
    Ready,
    Complete,
}

pub(crate) struct ModuleRecord {
    state: ModuleState,
    source: Option<ModuleSource>,
    specifiers: Option<Vec<String>>,
    imports: HashMap<String, String>,
    source_map: Option<Vec<Token>>,
}

/// Warm-only notice for a skipped edge: the unsupported specifier plus the
/// importing module. Strict walks never invoke it.
pub type BuiltinWarning = Rc<dyn Fn(&str, &str)>;

impl ModuleGraph {
    pub fn set_entry(&self, name: &str) {
        self.loader.borrow_mut().entry = Some(name.to_owned());
    }

    pub fn entry_url(&self) -> String {
        self.loader
            .borrow()
            .entry
            .as_deref()
            .and_then(|name| self.module_url(name))
            .unwrap_or_default()
    }

    pub fn info(&self, root: &str) -> Result<serde_json::Value, Error> {
        let complete: Vec<(String, HashMap<String, String>)> = {
            let loader = self.loader.borrow();
            let records = &loader.records;
            records
                .iter()
                .filter(|(_, record)| record.state == ModuleState::Complete)
                .map(|(name, record)| (name.clone(), record.imports.clone()))
                .collect()
        };
        let modules: BTreeMap<_, _> = complete
            .into_iter()
            .map(|(name, imports)| {
                self.check_access(&name)?;
                let imports: BTreeMap<_, _> = imports.into_iter().collect();
                let metadata = self.package_metadata(&name)?;
                Ok((name.clone(), serde_json::json!({"url": self.module_url(&name), "imports": imports, "package": metadata})))
            })
            .collect::<Result<_, Error>>()?;
        Ok(serde_json::json!({"root": root, "modules": modules}))
    }

    /// Local execution format by extension. `.js` stays ESM per avril
    /// policy (no `package.json` `type` behavior); explicit `.cjs`/`.cts`
    /// are CommonJS and `.json` is JSON for `require`.
    fn local_format(name: &str) -> ModuleFormat {
        let lower = name.to_ascii_lowercase();
        if lower.ends_with(".cjs") || lower.ends_with(".cts") {
            ModuleFormat::CommonJs
        } else if lower.ends_with(".json") {
            ModuleFormat::Json
        } else {
            ModuleFormat::Esm
        }
    }

    fn module_format_sync(&self, name: &str) -> Result<ModuleFormat, Error> {
        if name.starts_with("avril:") || http_url(name).is_some() || data_url(name).is_some() {
            // Remote CJS stays explicitly unsupported; remote files execute
            // as ESM so `require` of them fails with the ESM message.
            return Ok(ModuleFormat::Esm);
        }
        if self.npm.is_reserved_npm_path(Path::new(name)) {
            return match self.npm.check_access(Path::new(name)) {
                Ok(NpmModuleKind::Esm) | Ok(NpmModuleKind::TypeScript) => Ok(ModuleFormat::Esm),
                Ok(NpmModuleKind::CommonJs) => Ok(ModuleFormat::CommonJs),
                Ok(NpmModuleKind::Json) => Ok(ModuleFormat::Json),
                Ok(NpmModuleKind::Declaration) => Err(Error::Module(format!(
                    "{name}: npm declaration files have no runtime code"
                ))),
                Err(error) => Err(Error::Module(error.to_string())),
            };
        }
        Ok(Self::local_format(name))
    }

    pub fn prepare_for_add(
        &self,
        specifier: String,
        referrer: String,
        warn_unsupported: BuiltinWarning,
    ) -> LocalBoxFuture<'static, Result<String, Error>> {
        self.prepare_with(specifier, referrer, Some(warn_unsupported))
    }

    fn prepare_with(
        &self,
        specifier: String,
        referrer: String,
        warn_unsupported: Option<BuiltinWarning>,
    ) -> LocalBoxFuture<'static, Result<String, Error>> {
        let graph = self.clone();
        Box::pin(async move {
            let (root, request) = if referrer.is_empty() {
                (data_url(&specifier).map_or(specifier, String::from), None)
            } else {
                graph.resolve_tracked(&specifier, &referrer).await?
            };
            let mut requests: BTreeSet<_> = request.into_iter().collect();
            let mut pending = vec![root.clone()];
            let mut seen = HashSet::new();
            let mut links = HashMap::new();
            while let Some(name) = pending.pop() {
                if name.starts_with("avril:") || !seen.insert(name.clone()) {
                    continue;
                }
                graph.check_access(&name)?;
                graph.initialize_package(&name).await?;
                let is_complete = {
                    graph
                        .loader
                        .borrow()
                        .records
                        .get(&name)
                        .is_some_and(|record| record.state == ModuleState::Complete)
                };
                if is_complete {
                    continue;
                }
                let cached = {
                    graph.loader.borrow().records.get(&name).and_then(|record| {
                        Some((record.source.clone()?, record.specifiers.clone()?))
                    })
                };
                let format = graph.module_format_sync(&name)?;
                if format != ModuleFormat::Esm {
                    if cached.is_none() {
                        let raw = graph.original_source(&name).await?;
                        let (code, maps) = if format == ModuleFormat::Json {
                            (raw, None)
                        } else if name.to_ascii_lowercase().ends_with(".cts") {
                            let source_type = graph.source_type(&name)?;
                            let (code, maps) =
                                transpile(&name, &raw, source_type, TransformOptions::default())?;
                            (code, if maps.is_empty() { None } else { Some(maps) })
                        } else {
                            (raw, None)
                        };
                        let source: ModuleSource = code.into();
                        {
                            let mut loader = graph.loader.borrow_mut();
                            let records = &mut loader.records;
                            match records.get_mut(&name) {
                                Some(record) => {
                                    record.source = Some(source);
                                    record.specifiers = Some(Vec::new());
                                    record.source_map = maps;
                                }
                                None => {
                                    records.insert(
                                        name.clone(),
                                        ModuleRecord {
                                            state: ModuleState::Ready,
                                            source: Some(source),
                                            specifiers: Some(Vec::new()),
                                            imports: HashMap::new(),
                                            source_map: maps,
                                        },
                                    );
                                }
                            }
                        }
                    }
                    requests.extend(graph.prepare_npm_roots(&name, format, &[]).await?);
                    links.insert(name, HashMap::new());
                    continue;
                }
                let specifiers = match cached {
                    Some((_, specifiers)) => specifiers,
                    None => {
                        let raw = if name.starts_with(graph.store.options().jsr_registry.as_str()) {
                            let hints = graph.store.hints(&name).await.unwrap_or_default();
                            let prefetch = futures::future::join_all(
                                hints
                                    .iter()
                                    .filter(|hint| graph.check_access(hint).is_ok())
                                    .map(|hint| graph.store.source(hint)),
                            );
                            let (source, _) =
                                futures::future::join(graph.original_source(&name), prefetch).await;
                            source?
                        } else {
                            graph.original_source(&name).await?
                        };
                        let (code, mappings) = match graph.source_type(&name)? {
                            source_type if source_type.is_typescript() || source_type.is_jsx() => {
                                let options = if source_type.is_jsx() {
                                    graph.jsx_config(&name)?.transform_options().map_err(
                                        |error| Error::Module(format!("{name}: {error}")),
                                    )?
                                } else {
                                    TransformOptions::default()
                                };
                                let (code, mappings) =
                                    transpile(&name, &raw, source_type, options)?;
                                (code, Some(mappings))
                            }
                            _ => (raw, None),
                        };
                        let specifiers = match import_specifiers(&name, &code) {
                            Ok(specifiers) => specifiers,
                            Err(error) => {
                                // Another prepare may have stored or completed the record
                                // while this one awaited; never clear its source, state,
                                // or imports, only fill a missing source map.
                                if let Some(mappings) = mappings {
                                    let mut loader = graph.loader.borrow_mut();
                                    let records = &mut loader.records;
                                    match records.get_mut(&name) {
                                        Some(record)
                                            if record.state != ModuleState::Complete
                                                && record.source.is_none() =>
                                        {
                                            record.source_map = Some(mappings);
                                        }
                                        None => {
                                            records.insert(
                                                name.clone(),
                                                ModuleRecord {
                                                    state: ModuleState::Ready,
                                                    source: None,
                                                    specifiers: None,
                                                    imports: HashMap::new(),
                                                    source_map: Some(mappings),
                                                },
                                            );
                                        }
                                        _ => {}
                                    }
                                }
                                return Err(error);
                            }
                        };
                        let source: ModuleSource = code.into();
                        {
                            let mut loader = graph.loader.borrow_mut();
                            let records = &mut loader.records;
                            match records.get_mut(&name) {
                                Some(record) => {
                                    // Refresh only the fetch artifacts; a concurrent
                                    // prepare's state and imports must survive.
                                    record.source = Some(source);
                                    record.specifiers = Some(specifiers.clone());
                                    record.source_map = mappings;
                                }
                                None => {
                                    records.insert(
                                        name.clone(),
                                        ModuleRecord {
                                            state: ModuleState::Ready,
                                            source: Some(source),
                                            specifiers: Some(specifiers.clone()),
                                            imports: HashMap::new(),
                                            source_map: mappings,
                                        },
                                    );
                                }
                            }
                        }
                        specifiers
                    }
                };
                graph.prepare_npm_roots(&name, format, &specifiers).await?;
                let mut dependencies = Vec::new();
                let mut imports = HashMap::new();
                for specifier in &specifiers {
                    if let Some(warn) = &warn_unsupported
                        && specifier.starts_with("node:")
                    {
                        warn(specifier, &name);
                        continue;
                    }
                    let (dependency, request) = graph.resolve_tracked(specifier, &name).await?;
                    requests.extend(request);
                    imports.insert(specifier.clone(), dependency.clone());
                    dependencies.push(dependency);
                }
                pending.extend(dependencies.into_iter().rev());
                links.insert(name, imports);
            }
            graph
                .store
                .save_graph(&requests, &seen)
                .map_err(|error| Error::Module(error.to_string()))?;
            {
                let mut loader = graph.loader.borrow_mut();
                let records = &mut loader.records;
                // Every name needing completion is in links; names in seen but not
                // in links were already Complete and skipped above.
                for (name, imports) in links {
                    if let Some(record) = records.get_mut(&name)
                        && record.state != ModuleState::Complete
                    {
                        record.imports = imports;
                        // Warm-only walks cache sources without completing
                        // records, so a later strict walk still validates
                        // every edge including skipped builtins.
                        if warn_unsupported.is_none() {
                            record.state = ModuleState::Complete;
                        }
                    }
                }
            }
            Ok(root)
        })
    }
}

impl ModuleLoader for ModuleGraph {
    fn resolve(&self, specifier: &str, referrer: &str) -> Result<String, Error> {
        if specifier.starts_with("avril:") {
            return Ok(specifier.to_owned());
        }
        self.loader
            .borrow()
            .records
            .get(referrer)
            .and_then(|record| record.imports.get(specifier))
            .cloned()
            .ok_or_else(|| {
                Error::Module(format!(
                    "Import was not prepared: {specifier} from {referrer}"
                ))
            })
    }

    fn check_access(&self, name: &str) -> Result<(), Error> {
        if data_url(name).is_some() {
            DataModule::parse(name)?;
            return Ok(());
        }
        if name.starts_with("avril:") {
            return Err(Error::Module(format!(
                "Unsupported module specifier: {name} (unknown builtin)"
            )));
        }
        if let Some(url) = http_url(name) {
            if !url.username().is_empty() || url.password().is_some() {
                return Err(Error::Module(format!(
                    "HTTP module credentials are not supported: {name}"
                )));
            }
            if name.starts_with(self.store.options().jsr_registry.as_str()) {
                self.store
                    .module_base(name)
                    .map_err(|error| Error::Module(error.to_string()))?;
            }
            return self
                .store
                .check_acquired_remote(name, &self.authority)
                .map_err(|error| Error::Module(error.to_string()));
        }
        // Reserved npm paths always authorize through virtual-URL
        // authority: project filesystem grants never cover them, and
        // unowned cache paths fail instead of loading as local files.
        // Declarations have no executable code even when readable.
        if self.npm.is_reserved_npm_path(Path::new(name)) {
            return match self.npm.check_access(Path::new(name)) {
                Ok(NpmModuleKind::Declaration) => Err(Error::Module(format!(
                    "{name}: npm declaration files have no runtime code"
                ))),
                Ok(_) => Ok(()),
                Err(error) => Err(Error::Module(error.to_string())),
            };
        }
        self.target(name).map(|_| ())
    }

    fn load(&self, name: &str) -> Result<ModuleSource, Error> {
        self.check_access(name)?;
        if let Some(source) = self
            .loader
            .borrow()
            .records
            .get(name)
            .and_then(|record| record.source.clone())
        {
            return Ok(source);
        }
        let format = self.module_format_sync(name)?;
        match format {
            ModuleFormat::Esm => Err(Error::Module(format!("Module was not prepared: {name}"))),
            ModuleFormat::CommonJs | ModuleFormat::Json => {
                let (code, maps) = if matches!(format, ModuleFormat::CommonJs)
                    && name.to_ascii_lowercase().ends_with(".cts")
                {
                    let raw = self.sync_source(name)?;
                    let source_type = self.source_type(name)?;
                    let (code, maps) =
                        transpile(name, &raw, source_type, TransformOptions::default())?;
                    (code, if maps.is_empty() { None } else { Some(maps) })
                } else {
                    (self.sync_source(name)?, None)
                };
                let source: ModuleSource = code.into();
                let mut loader = self.loader.borrow_mut();
                let records = &mut loader.records;
                match records.get_mut(name) {
                    Some(record) => {
                        record.source = Some(source.clone());
                        record.specifiers = Some(Vec::new());
                        record.source_map = maps;
                    }
                    None => {
                        records.insert(
                            name.to_owned(),
                            ModuleRecord {
                                state: ModuleState::Complete,
                                source: Some(source.clone()),
                                specifiers: Some(Vec::new()),
                                imports: HashMap::new(),
                                source_map: maps,
                            },
                        );
                    }
                }
                Ok(source)
            }
        }
    }

    fn module_format(&self, name: &str) -> Result<ModuleFormat, Error> {
        self.module_format_sync(name)
    }

    fn require_resolve(&self, specifier: &str, referrer: &str) -> Result<String, Error> {
        self.require_resolve_sync(specifier, referrer)
    }

    fn cjs_named_exports(&self, name: &str) -> Result<Vec<String>, Error> {
        self.check_access(name)?;
        let source = self.load(name)?;
        self.check_access(name)?;
        Ok(crate::commonjs::named_exports(&source))
    }

    fn prepare(
        &self,
        specifier: String,
        referrer: String,
    ) -> LocalBoxFuture<'static, Result<String, Error>> {
        self.prepare_with(specifier, referrer, None)
    }

    /// Mirrors `resolve_tracked`'s decision order, but never acquires: unprepared
    /// jsr/npm packages error instead of fetching, missing targets are allowed,
    /// local names present as `file:` URLs, and no lock key is produced.
    fn resolve_import_meta(&self, specifier: &str, referrer: &str) -> Result<String, Error> {
        let referrer = self.classify_referrer(referrer)?;
        if let Some(name) = Self::early_scheme(specifier)? {
            return Ok(name);
        }
        if let Some(path) = referrer.path.as_deref()
            && referrer.npm
        {
            if let Some(url) = data_url(specifier) {
                return Ok(url.to_string());
            }
            return self
                .npm
                .resolve_from_file(specifier, path, ResolutionMode::Import)
                .map_err(|error| error.to_string())
                .and_then(|resolved| {
                    let path = resolved
                        .path
                        .to_str()
                        .ok_or_else(|| "Invalid module path".to_owned())?;
                    url::Url::from_file_path(path)
                        .map(String::from)
                        .map_err(|_| "Invalid module path".to_owned())
                })
                .map_err(Error::Module);
        }
        let mapped = if referrer.inline {
            None
        } else if referrer.remote {
            self.remote.resolve_import_meta(specifier, &referrer.name)?
        } else {
            self.packages
                .resolve_import_meta(specifier, &referrer.name, &self.remote)?
        };
        let effective = match &mapped {
            Some(crate::local_packages::Resolution::Specifier(target)) => target.as_str(),
            _ => specifier,
        };
        if effective.starts_with("npm:") {
            return self.resolve_import_meta_npm(effective, &referrer.name, referrer.remote);
        }
        match self.resolve_target(specifier, &referrer, mapped, ResolveMode::PreparedOnly)? {
            ResolutionTarget::Jsr(specifier) => crate::registry::parse_jsr_request(&specifier)
                .and_then(|request| self.store.resolve_import_meta_request(&request))
                .map_err(|error| Error::Module(error.to_string())),
            ResolutionTarget::Npm(specifier) => {
                self.resolve_import_meta_npm(&specifier, &referrer.name, referrer.remote)
            }
            ResolutionTarget::Remote(name) => Ok(name),
            ResolutionTarget::Module(name)
                if name.starts_with("avril:")
                    || http_url(&name).is_some()
                    || data_url(&name).is_some() =>
            {
                Ok(name)
            }
            ResolutionTarget::Module(name) => self
                .module_url(&name)
                .ok_or_else(|| Error::Module("Invalid module path".into())),
        }
    }

    fn module_url(&self, name: &str) -> Option<String> {
        if http_url(name).is_some() || data_url(name).is_some() {
            return Some(name.to_owned());
        }
        url::Url::from_file_path(name).ok().map(String::from)
    }

    fn map_source_location(
        &self,
        name: &str,
        line: u32,
        column: u32,
    ) -> Option<(String, u32, u32)> {
        let loader = self.loader.borrow();
        let records = &loader.records;
        let mappings = records.get(name)?.source_map.as_ref()?;
        let position = (line.checked_sub(1)?, column.checked_sub(1)?);
        let index = mappings
            .partition_point(|token| (token.get_dst_line(), token.get_dst_col()) <= position)
            .checked_sub(1)?;
        let token = mappings[index];
        token.get_source_id()?;
        (token.get_dst_line() == position.0).then(|| {
            (
                name.to_owned(),
                token.get_src_line() + 1,
                token.get_src_col() + 1,
            )
        })
    }
}

fn import_specifiers(name: &str, source: &str) -> Result<Vec<String>, Error> {
    let allocator = Allocator::default();
    let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse();
    if !parsed.diagnostics.is_empty() {
        let diagnostics: Vec<_> = parsed
            .diagnostics
            .into_iter()
            .map(|diagnostic| diagnostic.render_with_source_code(NamedSource::new(name, source)))
            .collect();
        return Err(Error::Syntax(format!("{name}\n{}", diagnostics.join(""))));
    }
    let mut imports: Vec<_> = parsed.module_record.requested_modules.iter().collect();
    imports.sort_by_key(|(_, requests)| requests.iter().map(|request| request.span.start).min());
    Ok(imports
        .into_iter()
        .map(|(specifier, _)| specifier.to_string())
        .collect())
}

pub(crate) fn transpile(
    name: &str,
    source: &str,
    source_type: SourceType,
    options: TransformOptions,
) -> Result<(String, Vec<Token>), Error> {
    let (code, map) = compile(name, Path::new(name), source, source_type, options)?;
    Ok((code, map.get_tokens().collect()))
}

/// Strips types and compiles JSX, keeping CommonJS source types' module
/// syntax. `path` is the file name the transformer sees, as in JSX
/// development metadata; maps always name `name`.
pub(crate) fn compile(
    name: &str,
    path: &Path,
    source: &str,
    source_type: SourceType,
    mut options: TransformOptions,
) -> Result<(String, oxc_sourcemap::SourceMap<'static>), Error> {
    if source_type.is_typescript_definition() {
        return Err(Error::Module(format!(
            "{name}: TypeScript declaration files have no runtime code"
        )));
    }
    let commonjs = source_type.is_commonjs();
    let report = |diagnostics: Diagnostics| {
        let rendered: Vec<_> = diagnostics
            .into_iter()
            .map(|diagnostic| diagnostic.render_with_source_code(NamedSource::new(name, source)))
            .collect();
        format!(
            "Failed to transpile {name}\n{}",
            rendered.join("").trim_end()
        )
    };

    let allocator = Allocator::default();
    let parsed = Parser::new(
        &allocator,
        source,
        if commonjs {
            source_type
        } else {
            source_type.with_module(true)
        },
    )
    .parse();
    if !parsed.diagnostics.is_empty() {
        return Err(Error::Syntax(report(parsed.diagnostics)));
    }
    let mut program = parsed.program;
    let semantic = SemanticBuilder::new_compiler()
        .with_enum_eval(true)
        .build(&program);
    if !semantic.diagnostics.is_empty() {
        return Err(Error::Syntax(report(semantic.diagnostics)));
    }
    options.env = EnvOptions {
        module: if commonjs {
            Module::Preserve
        } else {
            Module::Esm
        },
        ..EnvOptions::default()
    };
    let transformed = Transformer::new(&allocator, path, &options)
        .build_with_scoping(semantic.semantic.into_scoping(), &mut program);
    // Oxc warns where its output would not match TypeScript, such as for CommonJS-only syntax.
    if !transformed.diagnostics.is_empty() {
        return Err(Error::Module(report(transformed.diagnostics)));
    }

    let generated = Codegen::new()
        .with_options(CodegenOptions {
            source_map_path: Some(name.into()),
            ..CodegenOptions::default()
        })
        .build(&program);
    Ok((
        generated.code,
        generated.map.expect("maps are requested").into_owned(),
    ))
}

#[cfg(test)]
mod warm_builtin_tests {
    use avril_core::ModuleLoader;
    use std::cell::RefCell;
    use std::rc::Rc;

    #[test]
    fn strict_prepare_still_rejects_builtin_after_warm() {
        let dir = tempfile::tempdir().unwrap();
        std::fs::write(dir.path().join("avril.conf"), "").unwrap();
        std::fs::write(
            dir.path().join("parent.mjs"),
            "import './child.mjs';\nexport default 1;\n",
        )
        .unwrap();
        std::fs::write(
            dir.path().join("child.mjs"),
            "import 'node:fs';\nimport './sibling.mjs';\nexport default 2;\n",
        )
        .unwrap();
        std::fs::write(dir.path().join("sibling.mjs"), "export default 3;\n").unwrap();
        let (_runtime, graph) = crate::modules::offline_graph(dir.path());
        let warned = Rc::new(RefCell::new(Vec::new()));
        let warn = {
            let warned = warned.clone();
            Rc::new(move |specifier: &str, referrer: &str| {
                warned
                    .borrow_mut()
                    .push((specifier.to_owned(), referrer.to_owned()));
            })
        };
        let parent = dir.path().join("parent.mjs").to_string_lossy().into_owned();
        futures::executor::block_on(graph.prepare_for_add(parent.clone(), String::new(), warn))
            .unwrap();
        assert!(
            warned.borrow().iter().any(|(specifier, referrer)| {
                specifier == "node:fs" && referrer.ends_with("child.mjs")
            }),
            "{warned:?}",
        );
        let error = futures::executor::block_on(graph.prepare(parent, String::new())).unwrap_err();
        assert!(
            error
                .to_string()
                .contains("builtin modules are not supported"),
            "{error:?}"
        );
    }
}