use std::collections::BTreeMap; use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; use bytes::Bytes; use deno_semver::npm::NpmPackageReqReference; use http_body_util::{BodyExt, Empty}; use hyper::Request; use hyper_rustls::HttpsConnector; use hyper_util::client::legacy::{Client, connect::HttpConnector}; use serde::Deserialize; use url::Url; use super::{ Access, NpmAcquisition, NpmLock, NpmPin, Options, PackageStore, artifact_key, checksum, }; use storage::NpmArchive; /// Failures resolving, downloading, and verifying npm packages. #[derive(Clone, Debug, thiserror::Error)] pub enum NpmError { #[error("Locked npm package changed for {key}")] LockedNpmPackageChanged { key: String }, #[error("npm file integrity mismatch: {rel}")] NpmFileIntegrityMismatch { rel: String }, #[error("Frozen lockfile is missing {name}")] FrozenLockfileMissing { name: String }, #[error("Invalid npm package.json: {source}")] InvalidNpmPackageJson { source: Arc }, #[error("npm manifest does not match its locked files")] NpmManifestFilesMismatch, #[error("npm archive changed for {name}@{version}")] NpmArchiveChangedFor { name: String, version: String }, #[error("npm pin is missing its manifest")] NpmPinMissingManifest, #[error("npm package.json is missing its name")] NpmPackageJsonMissingName, #[error("npm package.json is missing its version")] NpmPackageJsonMissingVersion, #[error("npm package.json does not match its archive identity")] NpmPackageJsonIdentityMismatch, #[error("npm file is not in the package manifest: {path}")] NpmFileNotInManifest { path: String }, #[error("Expected an npm: request, not {specifier:?}")] ExpectedNpmRequest { specifier: String }, #[error("Invalid npm request: {specifier:?}")] InvalidNpmRequest { specifier: String }, #[error("{detail}")] NpmVersionReq { detail: String }, #[error("{detail}")] NpmPackageReqReference { detail: String }, #[error("npm subpaths resolve later; acquire the package first")] NpmSubpathNeedsAcquire, #[error("Invalid npm package name: {name}")] InvalidNpmPackageName { name: String }, #[error("Invalid npm version: {version}")] InvalidNpmVersion { version: String }, #[error("Non-canonical npm version: {version}")] NonCanonicalNpmVersion { version: String }, #[error("Invalid npm registry URL")] InvalidNpmRegistryUrl, #[error("Invalid locked npm request: {specifier}")] InvalidLockedNpmRequest { specifier: String }, #[error("Invalid locked npm root: {key}")] InvalidLockedNpmRoot { key: String }, #[error("Lockfile is missing npm instance {instance}")] LockfileMissingNpmInstance { instance: String }, #[error("Lockfile is missing npm package {}", package)] LockfileMissingNpmPackage { package: String }, #[error("Lockfile is missing npm instance {edge}")] LockfileMissingNpmEdge { edge: String }, #[error("Invalid npm instance: {instance}")] InvalidNpmInstance { instance: String }, #[error("Invalid npm lock identity")] InvalidNpmLockIdentity, #[error("Invalid automatic type association for {identity}")] InvalidNpmTypesAssociation { identity: String }, #[error("Invalid npm manifest digest")] InvalidNpmManifestDigest, #[error("Invalid npm manifest")] InvalidNpmManifest, #[error("Invalid npm file checksum: {path}")] InvalidNpmFileChecksum { path: String }, #[error("npm pin is missing package.json")] NpmPinMissingPackageJson, #[error("Invalid npm file path: {path}")] InvalidNpmFilePath { path: String }, #[error("npm tarball path must be UTF-8")] NpmTarballPathNotUtf8, #[error("npm tarball mixes archive roots: {raw}")] NpmTarballMixedRoots { raw: String }, #[error("Invalid npm tarball URL: {raw}")] InvalidNpmTarballUrl { raw: String }, #[error("npm tarball credentials are not supported: {raw}")] NpmTarballCredentials { raw: String }, #[error("npm URLs require HTTPS: {url}")] NpmUrlRequiresHttps { url: String }, #[error("Unsupported npm integrity algorithm: {algorithm}")] UnsupportedNpmIntegrityAlgorithm { algorithm: String }, #[error("Invalid npm integrity encoding")] InvalidNpmIntegrityEncoding, #[error("Invalid npm integrity length")] InvalidNpmIntegrityLength, #[error("Invalid npm integrity")] InvalidNpmIntegrity, #[error("Invalid npm shasum")] InvalidNpmShasum, #[error("npm tarball integrity mismatch")] NpmTarballIntegrityMismatch, #[error("npm tag {tag:?} does not exist for {}", name)] NpmTagMissing { tag: String, name: String }, #[error("npm tag {tag:?} points at invalid version {version}")] NpmTagInvalidVersion { tag: String, version: String }, #[error("npm tag {tag:?} points at missing version {version}")] NpmTagMissingVersion { tag: String, version: String }, #[error("npm package {name} has no latest tag")] NpmPackageNoLatestTag { name: String }, #[error("npm latest tag points at invalid version {version}")] NpmLatestTagInvalidVersion { version: String }, #[error("npm latest tag points at missing version {version}")] NpmLatestTagMissingVersion { version: String }, #[error("No npm version satisfies {}", version_text)] NoNpmVersionSatisfies { version_text: String }, #[error("npm origin is outside the registry: {url}")] NpmOriginOutsideRegistry { url: String }, #[error("{detail}")] NpmHttp { detail: String }, #[error("npm redirect has no valid Location: {url}")] NpmRedirectNoLocation { url: String }, #[error("npm redirect escapes the registry: {next}")] NpmRedirectEscapesRegistry { next: String }, #[error("npm server returned HTML instead of {what}: {url}")] NpmServerHtml { what: String, url: String }, #[error("npm {what} is too large: {url}")] NpmResponseTooLarge { what: String, url: String }, #[error("Invalid npm tarball")] InvalidNpmTarball, #[error("Invalid npm tarball path")] InvalidNpmTarballPath, #[error("Invalid npm tarball entry")] InvalidNpmTarballEntry, #[error("npm tarball has too many entries")] NpmTarballTooManyEntries, #[error("npm file is too large: {path}")] NpmFileTooLarge { path: String }, #[error("npm tarball expands to too many bytes")] NpmTarballExpandsTooMuch, #[error("npm tarball has a duplicate file: {path}")] NpmTarballDuplicate { path: String }, #[error("npm tarball has too many files")] NpmTarballTooManyFiles, #[error("npm tarball is empty")] NpmTarballEmpty, #[error("npm download worker stopped")] NpmDownloadWorkerStopped, #[error("npm metadata worker stopped")] NpmMetadataWorkerStopped, #[error("npm acquisition worker stopped")] NpmAcquisitionWorkerStopped, #[error("Locked version {version} does not satisfy {specifier}")] NpmLockedVersionUnsatisfied { version: String, specifier: String }, #[error("Too many npm redirects: {url}")] NpmTooManyRedirects { url: String }, #[error("npm redirect credentials are not supported: {next}")] NpmRedirectCredentials { next: String }, #[error("npm server returned {status} for {url}")] NpmServerStatus { status: u16, url: String }, #[error( "Type packages are not prepared for {identity}; run `avril cache` or configure `types.imports`" )] NpmTypesNotPrepared { identity: String }, #[error("npm request timed out: {url}")] NpmRequestTimeout { url: String }, #[error("npm tarball is missing package.json")] NpmTarballMissingPackageJson, #[error("npm tarball is too large")] NpmTarballTooLargeBare, #[error("npm tarball escapes the registry: {tarball}")] NpmTarballEscapesRegistry { tarball: String }, #[error("Offline npm cache miss: {name}")] NpmOfflineCacheMiss { name: String }, #[error("Offline npm metadata miss: {name}")] NpmOfflineMetadataMiss { name: String }, #[error("Invalid npm packument for {name}: {source}")] InvalidNpmPackument { name: String, source: Arc, }, #[error("npm packument name mismatch for {name}")] NpmPackumentNameMismatch { name: String }, #[error("Conflicting npm version selections for {request_key}")] NpmConflictingVersionSelections { request_key: String }, #[error("npm version is missing tarball integrity")] NpmVersionMissingIntegrity, #[error("npm tarball must contain only regular files and directories")] NpmTarballMustContainRegularFiles, #[error(transparent)] Storage(#[from] storage::StorageError), #[error(transparent)] Package(#[from] package::PackageError), #[error(transparent)] Url(#[from] url::ParseError), #[error(transparent)] Caps(#[from] crate::caps::CapabilitiesError), } mod fs; pub mod package; pub mod resolution; pub(crate) mod storage; pub(super) type SharedPackument = futures::future::Shared< futures::future::BoxFuture<'static, Result, NpmError>>, >; pub const NPM_DEFAULT_INSTANCE: &str = "default"; pub(super) const MAX_NPM_CONCURRENT_DOWNLOADS: usize = 8; const MAX_NPM_PACKUMENT_BYTES: usize = 8 * 1024 * 1024; const MAX_NPM_TARBALL_COMPRESSED: usize = 32 * 1024 * 1024; const MAX_NPM_TARBALL_EXPANDED: usize = 128 * 1024 * 1024; const MAX_NPM_FILES: usize = 65_536; const MAX_NPM_FILE_BYTES: usize = 32 * 1024 * 1024; const MAX_NPM_TAR_STREAM: usize = MAX_NPM_TARBALL_EXPANDED + MAX_NPM_FILES * 512 + 8192; const NPM_FETCH_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum NpmVerification { Full, OnUse, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct NpmIdentity { pub registry: Url, pub name: String, pub version: String, } impl NpmIdentity { pub fn key(&self) -> String { format!("{}|{}|{}", self.registry.as_str(), self.name, self.version) } } #[derive(Clone, Debug)] pub struct NpmArtifact { pub identity: NpmIdentity, pub tarball: String, pub integrity: String, pub dir: PathBuf, pub files: BTreeMap, pub package: serde_json::Value, } impl NpmArtifact { /// Read one verified package file. This is the loader-facing primitive: /// callers must use these bytes rather than reading the tree directly, so /// verification cannot be bypassed between check and use. pub fn read_file(&self, project_root: &Path, path: &str) -> Result, NpmError> { let expected = self .files .get(path) .ok_or_else(|| NpmError::NpmFileNotInManifest { path: path.to_owned() })?; let bytes = storage::read_project_file(project_root, &self.dir, path, MAX_NPM_FILE_BYTES)?; if checksum(&bytes) != *expected { return Err(NpmError::NpmFileIntegrityMismatch { rel: path.to_owned() }); } Ok(bytes) } } #[derive(Deserialize, Clone)] pub(super) struct NpmPackument { name: String, #[serde(default, rename = "dist-tags")] dist_tags: BTreeMap, versions: BTreeMap, } #[derive(Deserialize, Default, Clone)] struct NpmVersionEntry { #[serde(default)] dist: NpmDist, } #[derive(Deserialize, Default, Clone)] struct NpmDist { #[serde(default)] tarball: String, #[serde(default)] integrity: Option, #[serde(default)] shasum: Option, } /// A parsed `npm:` request using manifest range grammar (unions, spaces, /// comparators), which URL-grammar parsers reject. `request_key` preserves /// the declared range text so lock entries stay stable. #[derive(Clone, Debug, PartialEq, Eq)] pub struct NpmRequest { pub name: String, pub range: Option, pub version_req: deno_semver::VersionReq, pub subpath: Option, } impl NpmRequest { pub fn request_key(&self) -> String { match &self.range { None => format!("npm:{}", self.name), Some(range) => format!("npm:{}@{range}", self.name), } } } pub fn parse_npm_request(specifier: &str) -> Result { let text = specifier .strip_prefix("npm:") .ok_or_else(|| NpmError::ExpectedNpmRequest { specifier: specifier.to_owned() })?; if text.is_empty() { return Err(NpmError::ExpectedNpmRequest { specifier: specifier.to_owned() }); } let (head, subpath): (String, Option) = if let Some(rest) = text.strip_prefix('@') { match rest.split_once('/') { None => (text.to_owned(), None), Some((_, tail)) => match tail.split_once('/') { None => (text.to_owned(), None), Some((_, sub)) => ( text[..text.len() - sub.len() - 1].to_owned(), Some(sub.to_owned()), ), }, } } else { match text.split_once('/') { None => (text.to_owned(), None), Some((name, sub)) => (name.to_owned(), Some(sub.to_owned())), } }; let (name, range): (String, Option) = if let Some(rest) = head.strip_prefix('@') { match rest.rsplit_once('@') { None => (head.clone(), None), Some((scope_name, range)) => { if range.is_empty() { return Err(NpmError::InvalidNpmRequest { specifier: specifier.to_owned() }); } (format!("@{scope_name}"), Some(range.to_owned())) } } } else { match head.rsplit_once('@') { None => (head.clone(), None), Some((name, range)) => { if range.is_empty() { return Err(NpmError::InvalidNpmRequest { specifier: specifier.to_owned() }); } (name.to_owned(), Some(range.to_owned())) } } }; validate_npm_name(&name)?; let version_req = match &range { None => deno_semver::VersionReq::parse_from_npm("*") .map_err(|detail| NpmError::NpmVersionReq { detail: detail.to_string() })?, Some(range) => deno_semver::VersionReq::parse_from_npm(range) .map_err(|detail| NpmError::NpmVersionReq { detail: detail.to_string() })?, }; Ok(NpmRequest { name, range, version_req, subpath, }) } pub fn parse_npm_specifier(specifier: &str) -> Result { let request = parse_npm_request(specifier)?; if request.subpath.is_some() { return Err(NpmError::NpmSubpathNeedsAcquire); } NpmPackageReqReference::from_str(&request.request_key()) .map_err(|detail| NpmError::NpmPackageReqReference { detail: detail.to_string() }) } pub fn validate_npm_name(name: &str) -> Result<(), NpmError> { if name.is_empty() || name.len() > 214 { return Err(NpmError::InvalidNpmPackageName { name: name.to_owned() }); } if let Some(rest) = name.strip_prefix('@') { let Some((scope, leaf)) = rest.split_once('/') else { return Err(NpmError::InvalidNpmPackageName { name: name.to_owned() }); }; validate_npm_part(scope) .map_err(|_| NpmError::InvalidNpmPackageName { name: name.to_owned() })?; validate_npm_part(leaf) .map_err(|_| NpmError::InvalidNpmPackageName { name: name.to_owned() })?; if name.split('/').count() != 2 { return Err(NpmError::InvalidNpmPackageName { name: name.to_owned() }); } } else { if name.contains('/') { return Err(NpmError::InvalidNpmPackageName { name: name.to_owned() }); } validate_npm_part(name) .map_err(|_| NpmError::InvalidNpmPackageName { name: name.to_owned() })?; } Ok(()) } fn validate_npm_part(part: &str) -> Result<(), ()> { if part.is_empty() || part.len() > 214 || part.starts_with('.') || part.starts_with('_') || part .bytes() .any(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, b'-' | b'_' | b'.' | b'~'))) { return Err(()); } Ok(()) } fn validate_npm_version(version: &str) -> Result { let parsed = deno_semver::Version::parse_from_npm(version) .map_err(|_| NpmError::InvalidNpmVersion { version: version.to_owned() })?; if parsed.to_string() != version { return Err(NpmError::NonCanonicalNpmVersion { version: version.to_owned() }); } Ok(parsed) } /// Provenance base for acquisition checks: always slash-terminated so sibling /// names cannot collide through string-prefix grants. pub fn npm_package_base(registry: &Url, name: &str) -> Result { validate_npm_name(name)?; let mut base = registry.join(&format!("{name}/"))?; base.set_query(None); base.set_fragment(None); Ok(base) } /// Version base for execution checks and virtual file URLs. pub fn npm_version_base(registry: &Url, name: &str, version: &str) -> Result { validate_npm_version(version)?; let package = npm_package_base(registry, name)?; Ok(package.join(&format!("{version}/"))?) } /// Canonical virtual URL of one package file, for capability grants and the /// next loader chunk. Transport never fetches these; they name authority. pub fn npm_virtual_module( registry: &Url, name: &str, version: &str, path: &str, ) -> Result { validate_npm_relpath(path)?; let mut url = npm_version_base(registry, name, version)?; url.path_segments_mut() .map_err(|_| NpmError::InvalidNpmRegistryUrl)? .pop_if_empty() .extend(path.split('/')); Ok(url) } pub(super) fn validate_lock_npm(npm: &NpmLock, expected_registry: &Url) -> Result<(), NpmError> { for (identity, request) in &npm.types { let (registry, name, version) = parse_npm_key(identity) .map_err(|_| NpmError::InvalidNpmTypesAssociation { identity: identity.clone() })?; let _ = npm_package_base(®istry, &name)?; if ®istry != expected_registry || !npm.packages.contains_key(identity) { return Err(NpmError::InvalidNpmTypesAssociation { identity: identity.clone() }); } let source_version = validate_npm_version(&version)?; if let Some(request) = request { let parsed = parse_npm_request(request) .map_err(|_| NpmError::InvalidNpmTypesAssociation { identity: identity.clone() })?; let expected = package::types_package_name(&name); let next_major = source_version.major.checked_add(1).ok_or_else(|| { NpmError::InvalidNpmTypesAssociation { identity: identity.clone() } })?; let expected_range = format!(">={}.0.0 <{next_major}.0.0", source_version.major); let type_version = npm.specifiers.get(request).ok_or_else(|| { NpmError::InvalidNpmTypesAssociation { identity: identity.clone() } })?; let type_key = format!("{}|{}|{type_version}", expected_registry, parsed.name); if name.starts_with("@types/") || parsed.name != expected || parsed.subpath.is_some() || parsed.range.as_deref() != Some(expected_range.as_str()) || parsed.request_key() != *request || !npm.packages.contains_key(&type_key) { return Err(NpmError::InvalidNpmTypesAssociation { identity: identity.clone() }); } } } for (key, pin) in &npm.packages { let (registry, name, version) = parse_npm_key(key)?; let _ = npm_package_base(®istry, &name)?; validate_npm_version(&version)?; validate_npm_pin(pin)?; let tarball = npm_tarball_url(&pin.tarball)?; if !same_origin(®istry, &tarball) { return Err(NpmError::NpmTarballEscapesRegistry { tarball: pin.tarball.clone() }); } check_https(&tarball)?; } for (specifier, version) in &npm.specifiers { let request = parse_npm_request(specifier)?; if request.subpath.is_some() { return Err(NpmError::InvalidLockedNpmRequest { specifier: specifier.to_owned() }); } let locked = validate_npm_version(version)?; if !locked_satisfies(request.version_req.clone(), &locked) { return Err(NpmError::NpmLockedVersionUnsatisfied { version: version.clone(), specifier: specifier.to_owned(), }); } } for (key, instance) in &npm.roots { let (request, context) = match key.split_once('\0') { None => (key.as_str(), None), Some((request, suffix)) => { let context = suffix .strip_prefix("ctx:") .ok_or_else(|| NpmError::InvalidLockedNpmRoot { key: key.to_owned() })?; (request, Some(context)) } }; let parsed = parse_npm_request(request)?; if parsed.subpath.is_some() { return Err(NpmError::InvalidLockedNpmRoot { key: key.to_owned() }); } if let Some(context) = context { if context.is_empty() { return Err(NpmError::InvalidLockedNpmRoot { key: key.to_owned() }); } for entry in context.split(',') { let (peer, provider) = entry .split_once('=') .ok_or_else(|| NpmError::InvalidLockedNpmRoot { key: key.to_owned() })?; validate_npm_name(peer)?; parse_npm_key(provider)?; } } validate_npm_instance_id(instance)?; if !npm.instances.contains_key(instance) { return Err(NpmError::LockfileMissingNpmInstance { instance: instance.to_owned() }); } } for (instance, pin) in &npm.instances { validate_npm_instance_id(instance)?; let (registry, name, version) = parse_npm_key(&pin.package)?; let _ = npm_package_base(®istry, &name)?; validate_npm_version(&version)?; if !npm.packages.contains_key(&pin.package) { return Err(NpmError::LockfileMissingNpmPackage { package: pin.package.clone() }); } if instance != &pin.package && !instance.starts_with("p-") { return Err(NpmError::InvalidNpmInstance { instance: instance.to_owned() }); } for (alias, edge) in pin.deps.iter().chain(pin.peers.iter()) { validate_npm_name(alias)?; validate_npm_instance_id(edge)?; if !npm.instances.contains_key(edge) { return Err(NpmError::LockfileMissingNpmEdge { edge: edge.to_owned() }); } } } Ok(()) } fn validate_npm_instance_id(instance: &str) -> Result<(), NpmError> { if instance == NPM_DEFAULT_INSTANCE { return Ok(()); } if let Some(rest) = instance.strip_prefix("p-") { if (16..=64).contains(&rest.len()) && rest.bytes().all(|ch| ch.is_ascii_hexdigit()) { return Ok(()); } return Err(NpmError::InvalidNpmInstance { instance: instance.to_owned() }); } parse_npm_key(instance)?; Ok(()) } pub(super) fn parse_npm_key(key: &str) -> Result<(Url, String, String), NpmError> { let mut parts = key.splitn(3, '|'); let (Some(registry), Some(name), Some(version)) = (parts.next(), parts.next(), parts.next()) else { return Err(NpmError::InvalidNpmLockIdentity); }; if parts.next().is_some() { return Err(NpmError::InvalidNpmLockIdentity); } let url = Url::parse(registry).map_err(|_| NpmError::InvalidNpmLockIdentity)?; if url.as_str() != registry { return Err(NpmError::InvalidNpmLockIdentity); } validate_npm_name(name)?; validate_npm_version(version)?; Ok((url, name.to_owned(), version.to_owned())) } #[cfg(test)] mod types_lock_tests { use super::{NpmLock, NpmPin, parse_npm_request, validate_lock_npm}; use std::collections::BTreeMap; #[test] fn validates_automatic_type_associations() { let source = "https://registry.npmjs.org/|@scope/library|0.7.4"; let request = "npm:@types/scope__library@>=0.0.0 <1.0.0"; let lock = NpmLock { specifiers: BTreeMap::from([(request.to_owned(), "0.9.0".to_owned())]), packages: BTreeMap::from([ (source.to_owned(), test_pin()), ( "https://registry.npmjs.org/|@types/scope__library|0.9.0".to_owned(), test_pin(), ), ]), types: BTreeMap::from([(source.to_owned(), Some(request.to_owned()))]), ..NpmLock::default() }; validate_lock_npm( &lock, &url::Url::parse("https://registry.npmjs.org/").unwrap(), ) .unwrap(); assert_eq!( parse_npm_request(request).unwrap().name, "@types/scope__library" ); let registry = url::Url::parse("https://registry.npmjs.org/").unwrap(); let mut invalid = lock.clone(); invalid .types .insert(source.to_owned(), Some("npm:@types/other@*".to_owned())); assert!(validate_lock_npm(&invalid, ®istry).is_err()); let mut dangling = lock.clone(); dangling .packages .remove("https://registry.npmjs.org/|@types/scope__library|0.9.0"); assert!(validate_lock_npm(&dangling, ®istry).is_err()); let mut missing_source = lock.clone(); missing_source.packages.remove(source); assert!(validate_lock_npm(&missing_source, ®istry).is_err()); assert!( validate_lock_npm(&lock, &url::Url::parse("https://elsewhere.test/").unwrap()).is_err() ); } #[test] fn type_packages_can_have_negative_discovery_records() { let identity = "https://registry.npmjs.org/|@types/node|22.0.0"; let lock = NpmLock { packages: BTreeMap::from([(identity.to_owned(), test_pin())]), types: BTreeMap::from([(identity.to_owned(), None)]), ..NpmLock::default() }; validate_lock_npm( &lock, &url::Url::parse("https://registry.npmjs.org/").unwrap(), ) .unwrap(); } fn test_pin() -> NpmPin { NpmPin { tarball: "https://registry.npmjs.org/package.tgz".to_owned(), integrity: "sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==".to_owned(), manifest: Some(format!("sha256-{}", "0".repeat(64))), files: None, } } } fn locked_satisfies(version_req: deno_semver::VersionReq, version: &deno_semver::Version) -> bool { if version_req.tag().is_some() || version_req.version_text() == "*" { return true; } version_req.matches(version) } fn validate_npm_pin(pin: &NpmPin) -> Result<(), NpmError> { npm_tarball_url(&pin.tarball)?; select_sri(&pin.integrity)?; match (&pin.manifest, &pin.files) { (Some(manifest), Some(files)) => { manifest_file_name(manifest)?; validate_files_map(files)?; if storage::manifest_digest(files)? != *manifest { return Err(NpmError::NpmManifestFilesMismatch); } } (Some(manifest), None) => { manifest_file_name(manifest)?; } (None, Some(files)) => validate_files_map(files)?, (None, None) => return Err(NpmError::NpmPinMissingManifest), } Ok(()) } pub(crate) fn manifest_file_name(digest: &str) -> Result { let hex = digest .strip_prefix("sha256-") .ok_or(NpmError::InvalidNpmManifestDigest)?; if hex.len() != 64 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(NpmError::InvalidNpmManifestDigest); } Ok(format!("{hex}.json")) } pub(crate) fn validate_files_map(files: &BTreeMap) -> Result<(), NpmError> { if files.is_empty() || files.len() > MAX_NPM_FILES { return Err(NpmError::InvalidNpmManifest); } for (path, hash) in files { validate_npm_relpath(path)?; if !valid_npm_checksum(hash) { return Err(NpmError::InvalidNpmFileChecksum { path: path.to_owned() }); } } if !files.contains_key("package.json") { return Err(NpmError::NpmPinMissingPackageJson); } Ok(()) } fn valid_npm_checksum(hash: &str) -> bool { hash.strip_prefix("sha256-") .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|ch| ch.is_ascii_hexdigit())) } fn validate_npm_relpath(path: &str) -> Result<(), NpmError> { if path.is_empty() || path.len() > 1024 { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } if path.starts_with('/') || path.contains('\\') || path.contains('\0') { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } if path.bytes().any(|byte| byte.is_ascii_control()) { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } let lower = path.to_ascii_lowercase(); if lower.contains("%2f") || lower.contains("%5c") { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } for part in path.split('/') { if part.is_empty() || part == "." || part == ".." { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } if part.len() > 255 || part.contains(':') { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } if part.ends_with('.') || part.ends_with(' ') { return Err(NpmError::InvalidNpmFilePath { path: path.to_owned() }); } } Ok(()) } /// Tar entry names arrive as platform paths: one consistent safe top-level /// directory is accepted for the whole archive and stripped, harmless `.` /// segments beneath it normalize away, while anything else must already be /// a clean relative posix path. Returns empty only for ignored root directories. fn npm_relpath_from_tar( path: &std::path::Path, is_dir: bool, root: &mut Option, ) -> Result { let bytes = path.as_os_str().as_encoded_bytes(); let raw = std::str::from_utf8(bytes).map_err(|_| NpmError::NpmTarballPathNotUtf8)?; if raw.contains('\\') { return Err(NpmError::InvalidNpmFilePath { path: raw.to_owned() }); } let (candidate, stripped) = match raw.find('/') { Some(slash) => (&raw[..slash], &raw[slash + 1..]), None => (raw, ""), }; validate_npm_relpath(candidate) .map_err(|_| NpmError::InvalidNpmFilePath { path: raw.to_owned() })?; match root { Some(bound) if *bound != candidate => { return Err(NpmError::NpmTarballMixedRoots { raw: raw.to_owned() }); } Some(_) => {} None => *root = Some(candidate.to_owned()), } if stripped.len() > 1024 { return Err(NpmError::InvalidNpmFilePath { path: stripped.to_owned() }); } if is_dir { if stripped.is_empty() { return Ok(String::new()); } let inner = stripped.strip_suffix('/').unwrap_or(stripped); if inner.is_empty() || inner.split('/').any(|part| part.is_empty()) { return Err(NpmError::InvalidNpmFilePath { path: stripped.to_owned() }); } let canonical: Vec<&str> = inner.split('/').filter(|part| *part != ".").collect(); let canonical = canonical.join("/"); if canonical.is_empty() { return Ok(String::new()); } validate_npm_relpath(&canonical)?; return Ok(canonical); } let canonical: Vec<&str> = stripped.split('/').filter(|part| *part != ".").collect(); let canonical = canonical.join("/"); validate_npm_relpath(&canonical)?; Ok(canonical) } fn npm_tarball_url(raw: &str) -> Result { let url = Url::parse(raw).map_err(|_| NpmError::InvalidNpmTarballUrl { raw: raw.to_owned() })?; if !matches!(url.scheme(), "http" | "https") || url.host().is_none() { return Err(NpmError::InvalidNpmTarballUrl { raw: raw.to_owned() }); } if !url.username().is_empty() || url.password().is_some() { return Err(NpmError::NpmTarballCredentials { raw: raw.to_owned() }); } if url.query().is_some() || url.fragment().is_some() { return Err(NpmError::InvalidNpmTarballUrl { raw: raw.to_owned() }); } Ok(url) } fn same_origin(left: &Url, right: &Url) -> bool { left.scheme() == right.scheme() && left.host_str() == right.host_str() && left.port_or_known_default() == right.port_or_known_default() } fn is_loopback(url: &Url) -> bool { matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "[::1]")) } fn check_https(url: &Url) -> Result<(), NpmError> { if url.scheme() != "https" && !(url.scheme() == "http" && is_loopback(url)) { return Err(NpmError::NpmUrlRequiresHttps { url: url.to_string() }); } Ok(()) } fn packument_url(registry: &Url, name: &str) -> Result { validate_npm_name(name)?; let mut url = registry.join(name)?; url.set_query(None); url.set_fragment(None); Ok(url) } fn validate_instance(instance: &str) -> Result<(), NpmError> { if instance.is_empty() || instance.len() > 64 || instance.starts_with('.') || instance.bytes().any(|byte| { byte == b'/' || byte == b'\\' || byte == b':' || byte == b'\0' || byte.is_ascii_control() }) { return Err(NpmError::InvalidNpmInstance { instance: instance.to_owned() }); } Ok(()) } fn select_sri(sri: &str) -> Result<(String, Vec), NpmError> { let mut best: Option<(u8, String, Vec)> = None; for token in sri.split_whitespace() { let (algorithm, encoded) = token.split_once('-').ok_or(NpmError::InvalidNpmIntegrity)?; let rank: u8 = match algorithm { "sha512" => 3, "sha256" => 2, "sha1" => 1, _ => { return Err(NpmError::UnsupportedNpmIntegrityAlgorithm { algorithm: algorithm.to_owned(), }); } }; use base64::Engine; let digest = base64::engine::general_purpose::STANDARD .decode(encoded) .map_err(|_| NpmError::InvalidNpmIntegrityEncoding)?; let expected = match rank { 3 => 64, 2 => 32, _ => 20, }; if digest.len() != expected { return Err(NpmError::InvalidNpmIntegrityLength); } if best .as_ref() .is_some_and(|(current, _, _)| *current >= rank) { continue; } let normalized = format!( "{algorithm}-{}", base64::engine::general_purpose::STANDARD.encode(&digest) ); best = Some((rank, normalized, digest)); } best.map(|(_, normalized, digest)| (normalized, digest)) .ok_or(NpmError::InvalidNpmIntegrity) } fn verify_sri(bytes: &[u8], sri: &str) -> Result<(), NpmError> { let (normalized, expected) = select_sri(sri)?; let algorithm = normalized.split_once('-').unwrap().0; let actual: Vec = match algorithm { "sha512" => { use sha2::Digest; sha2::Sha512::digest(bytes).to_vec() } "sha256" => { use sha2::Digest; sha2::Sha256::digest(bytes).to_vec() } _ => { use sha1::Digest as _; sha1::Sha1::digest(bytes).to_vec() } }; if actual != expected { return Err(NpmError::NpmTarballIntegrityMismatch); } Ok(()) } fn normalized_sri(sri: &str) -> Result { let (normalized, _) = select_sri(sri)?; Ok(normalized) } fn sri_from_shasum(shasum: &str) -> Result { if shasum.len() != 40 || !shasum.bytes().all(|ch| ch.is_ascii_hexdigit()) { return Err(NpmError::InvalidNpmShasum); } let mut raw = Vec::with_capacity(20); for chunk in 0..20 { let byte = u8::from_str_radix(&shasum[chunk * 2..chunk * 2 + 2], 16) .map_err(|_| NpmError::InvalidNpmShasum)?; raw.push(byte); } use base64::Engine; Ok(format!( "sha1-{}", base64::engine::general_purpose::STANDARD.encode(raw) )) } fn pin_integrity(dist: &NpmDist) -> Result { if let Some(integrity) = &dist.integrity { normalized_sri(integrity) } else if let Some(shasum) = &dist.shasum { sri_from_shasum(shasum) } else { Err(NpmError::NpmVersionMissingIntegrity) } } fn select_npm_version( packument: &NpmPackument, version_req: &deno_semver::VersionReq, ) -> Result<(String, NpmDist), NpmError> { if let Some(tag) = version_req.tag() { let version = packument .dist_tags .get(tag) .ok_or_else(|| NpmError::NpmTagMissing { tag: tag.to_owned(), name: packument.name.clone(), })?; validate_npm_version(version).map_err(|_| NpmError::NpmTagInvalidVersion { tag: tag.to_owned(), version: version.to_owned(), })?; let entry = packument .versions .get(version) .ok_or_else(|| NpmError::NpmTagMissingVersion { tag: tag.to_owned(), version: version.to_owned(), })?; return Ok((version.clone(), entry.dist.clone())); } if version_req.version_text() == "*" { let version = packument .dist_tags .get("latest") .ok_or_else(|| NpmError::NpmPackageNoLatestTag { name: packument.name.clone() })?; validate_npm_version(version) .map_err(|_| NpmError::NpmLatestTagInvalidVersion { version: version.to_owned() })?; let entry = packument .versions .get(version) .ok_or_else(|| NpmError::NpmLatestTagMissingVersion { version: version.to_owned() })?; return Ok((version.clone(), entry.dist.clone())); } let mut best: Option<(deno_semver::Version, String)> = None; for version_text in packument.versions.keys() { let Ok(version) = deno_semver::Version::parse_from_npm(version_text) else { continue; }; if !version_req.matches(&version) { continue; } if best .as_ref() .is_some_and(|(current, _)| current >= &version) { continue; } best = Some((version, version_text.clone())); } let (_, version_text) = best.ok_or_else(|| NpmError::NoNpmVersionSatisfies { version_text: version_req.version_text().to_owned(), })?; let entry = &packument.versions[&version_text]; Ok((version_text, entry.dist.clone())) } /// A pinned package already satisfying a new range is reused without /// consulting mutable packument metadata. fn find_pinned_satisfying( lock: &super::Lockfile, registry: &Url, name: &str, version_req: &deno_semver::VersionReq, ) -> Option<(String, NpmPin)> { if version_req.tag().is_some() || version_req.version_text() == "*" { return None; } let mut best: Option<(deno_semver::Version, String)> = None; for key in lock.npm.packages.keys() { let Ok((key_registry, key_name, key_version)) = parse_npm_key(key) else { continue; }; if key_registry != *registry || key_name != name { continue; } let Ok(version) = deno_semver::Version::parse_from_npm(&key_version) else { continue; }; if !version_req.matches(&version) { continue; } if best .as_ref() .is_some_and(|(current, _)| current >= &version) { continue; } best = Some((version, key_version)); } best.and_then(|(_, version)| { let key = format!("{}|{name}|{version}", registry.as_str()); lock.npm .packages .get(&key) .map(|pin| (version, pin.clone())) }) } struct BoundedTarReader { inner: R, remaining: usize, } impl std::io::Read for BoundedTarReader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { if self.remaining == 0 { return Err(std::io::Error::new( std::io::ErrorKind::FileTooLarge, "npm tarball expands to too many bytes", )); } let capped = buf.len().min(self.remaining); let read = self.inner.read(&mut buf[..capped])?; self.remaining -= read; Ok(read) } } fn collect_npm_files(tarball: &[u8]) -> Result>, NpmError> { use flate2::read::GzDecoder; let decoder = GzDecoder::new(tarball); let bounded = BoundedTarReader { inner: decoder, remaining: MAX_NPM_TAR_STREAM, }; let mut archive = tar::Archive::new(bounded); let mut files: BTreeMap> = BTreeMap::new(); let mut root: Option = None; let mut entries = 0usize; let mut expanded: usize = 0; let entries_iter = archive.entries().map_err(|_| NpmError::InvalidNpmTarball)?; for entry in entries_iter { let entry = entry.map_err(|_| NpmError::InvalidNpmTarball)?; entries += 1; if entries > MAX_NPM_FILES { return Err(NpmError::NpmTarballTooManyEntries); } let kind = entry.header().entry_type(); if kind.is_dir() { let path = entry.path().map_err(|_| NpmError::InvalidNpmTarballPath)?; npm_relpath_from_tar(&path, true, &mut root)?; continue; } if !kind.is_file() { return Err(NpmError::NpmTarballMustContainRegularFiles); } let path = entry.path().map_err(|_| NpmError::InvalidNpmTarballPath)?; let stripped = npm_relpath_from_tar(&path, false, &mut root)?; let mut bytes = Vec::new(); entry .take(MAX_NPM_FILE_BYTES as u64 + 1) .read_to_end(&mut bytes) .map_err(|_| NpmError::InvalidNpmTarballEntry)?; if bytes.len() > MAX_NPM_FILE_BYTES { return Err(NpmError::NpmFileTooLarge { path: stripped.to_owned() }); } expanded = expanded .checked_add(bytes.len()) .ok_or(NpmError::NpmTarballTooLargeBare)?; if expanded > MAX_NPM_TARBALL_EXPANDED { return Err(NpmError::NpmTarballExpandsTooMuch); } // Some published archives repeat the same file under a `./` alias. if let Some(previous) = files.get(&stripped) { if *previous != bytes { return Err(NpmError::NpmTarballDuplicate { path: stripped.to_owned() }); } continue; } if files.len() >= MAX_NPM_FILES { return Err(NpmError::NpmTarballTooManyFiles); } files.insert(stripped, bytes); } if files.is_empty() { return Err(NpmError::NpmTarballEmpty); } Ok(files) } fn bind_npm_files( data: &BTreeMap>, name: &str, version: &str, ) -> Result<(BTreeMap, serde_json::Value), NpmError> { let package_bytes = data .get("package.json") .ok_or(NpmError::NpmTarballMissingPackageJson)?; let package: serde_json::Value = serde_json::from_slice(package_bytes) .map_err(|source| NpmError::InvalidNpmPackageJson { source: Arc::new(source) })?; let published_name = package .get("name") .and_then(|value| value.as_str()) .ok_or(NpmError::NpmPackageJsonMissingName)?; let published_version = package .get("version") .and_then(|value| value.as_str()) .ok_or(NpmError::NpmPackageJsonMissingVersion)?; if published_name != name || published_version != version { return Err(NpmError::NpmPackageJsonIdentityMismatch); } let mut hashes = BTreeMap::new(); for (path, bytes) in data { hashes.insert(path.clone(), checksum(bytes)); } Ok((hashes, package)) } pub(super) fn check_npm_url( authority: Option<&crate::authority::SourceAuthority>, url: &Url, ) -> Result<(), NpmError> { Ok(Access::Module(url.clone()).check(authority)?) } fn check_npm_exec( authority: Option<&crate::authority::SourceAuthority>, registry: &Url, name: &str, version: &str, files: &BTreeMap, ) -> Result<(), NpmError> { let version_base = npm_version_base(registry, name, version)?; if Access::Module(version_base).check(authority).is_ok() { return Ok(()); } for path in files.keys() { let url = npm_virtual_module(registry, name, version, path)?; if Access::Module(url).check(authority).is_ok() { return Ok(()); } } let version_base = npm_version_base(registry, name, version)?; Ok(Access::Module(version_base).check(authority)?) } /// Network and policy context for one registry request. Owned, because /// spawned workers carry their own copy rather than borrowing the store. #[derive(Clone)] struct Fetcher { options: Options, client: Client, Empty>, authority: Option, progress: Option, registry: Url, } /// Packument dedupe and download admission shared by every acquisition. #[derive(Clone)] struct Workers { packuments: Arc>>, downloads: Arc, runtime: tokio::runtime::Handle, } impl Fetcher { fn archive<'a>( &'a self, name: &'a str, version: &'a str, tarball: &'a str, integrity: &'a str, ) -> NpmArchive<'a> { NpmArchive { registry: &self.registry, name, version, tarball, integrity, } } async fn fetch_bytes( &self, mut url: Url, accept: &'static str, max_bytes: usize, what: &str, access: Access, ) -> Result, NpmError> { let authority = &self.authority; let registry = &self.registry; let client = &self.client; let download = async { access.check(authority.as_ref())?; check_https(&url)?; if !same_origin(&url, registry) { return Err(NpmError::NpmOriginOutsideRegistry { url: url.to_string() }); } for _ in 0..=10 { access.check(authority.as_ref())?; let mut request_url = url.clone(); request_url.set_fragment(None); let request = Request::get(request_url.as_str()) .header("Accept", accept) .header("User-Agent", concat!("avril/", env!("CARGO_PKG_VERSION"))) .body(Empty::new()) .map_err(|detail| NpmError::NpmHttp { detail: detail.to_string() })?; let response = client .request(request) .await .map_err(|detail| NpmError::NpmHttp { detail: detail.to_string() })?; access.check(authority.as_ref())?; if matches!(response.status().as_u16(), 301 | 302 | 303 | 307 | 308) { let location = response .headers() .get("location") .and_then(|value| value.to_str().ok()) .ok_or_else(|| NpmError::NpmRedirectNoLocation { url: url.to_string() })?; let next = url.join(location)?; if !next.username().is_empty() || next.password().is_some() { return Err(NpmError::NpmRedirectCredentials { next: next.to_string() }); } check_https(&next)?; if !same_origin(&next, registry) { return Err(NpmError::NpmRedirectEscapesRegistry { next: next.to_string(), }); } url = next; continue; } if !response.status().is_success() { return Err(NpmError::NpmServerStatus { status: response.status().as_u16(), url: url.to_string(), }); } if response .headers() .get("content-type") .and_then(|value| value.to_str().ok()) .is_some_and(|value| value.starts_with("text/html")) { return Err(NpmError::NpmServerHtml { what: what.to_owned(), url: url.to_string(), }); } let mut body = response.into_body(); let mut bytes = Vec::new(); while let Some(frame) = body.frame().await { if let Ok(data) = frame .map_err(|detail| NpmError::NpmHttp { detail: detail.to_string() })? .into_data() { if bytes.len() + data.len() > max_bytes { return Err(NpmError::NpmResponseTooLarge { what: what.to_owned(), url: url.to_string(), }); } bytes.extend_from_slice(&data); } } return Ok(bytes); } Err(NpmError::NpmTooManyRedirects { url: url.to_string() }) }; tokio::time::timeout(NPM_FETCH_TIMEOUT, download) .await .map_err(|_| NpmError::NpmRequestTimeout { url: url.to_string() })? } async fn npm_acquire_task( &self, workers: &Workers, lock_snapshot: super::Lockfile, instance: String, request: NpmRequest, verification: NpmVerification, ) -> Result { let options = &self.options; let authority = &self.authority; let npm_registry = &self.registry; let request_key = request.request_key(); let name = request.name; let version_req = request.version_req; let package_base = npm_package_base(npm_registry, &name)?; Access::Package(package_base.clone()).check(authority.as_ref())?; // One permit per executing worker; waiter clones only await the shared // result. Metadata fetching takes no permit, so holding one across the // shared packument wait cannot deadlock. Recheck caps after admission // for revocation while queued; later validation rechecks versions. let _download_permit = workers .downloads .clone() .acquire_owned() .await .map_err(|_| NpmError::NpmDownloadWorkerStopped)?; Access::Package(package_base.clone()).check(authority.as_ref())?; if let Some(version_text) = lock_snapshot.npm.specifiers.get(&request_key) { let version = validate_npm_version(version_text)?; if !locked_satisfies(version_req.clone(), &version) { return Err(NpmError::NpmLockedVersionUnsatisfied { version: version_text.clone(), specifier: request_key.clone(), }); } let identity_key = format!("{}|{name}|{version_text}", npm_registry.as_str()); if let Some(pin) = lock_snapshot.npm.packages.get(&identity_key) { return self .pinned_npm(&instance, &name, version_text, pin, verification) .await; } } else if !options.update && !options.frozen && let Some((version_text, pin)) = find_pinned_satisfying(&lock_snapshot, npm_registry, &name, &version_req) { return self .pinned_npm(&instance, &name, &version_text, &pin, verification) .await; } if options.frozen { return Err(NpmError::FrozenLockfileMissing { name: request_key.clone() }); } let packument = self.shared_packument(&name, workers).await?; let (version_text, dist) = select_npm_version(&packument, &version_req)?; validate_npm_version(&version_text)?; Access::Package(npm_version_base(npm_registry, &name, &version_text)?) .check(authority.as_ref())?; let tarball_url = npm_tarball_url(&dist.tarball)?; if !same_origin(&tarball_url, npm_registry) { return Err(NpmError::NpmTarballEscapesRegistry { tarball: dist.tarball.clone() }); } check_https(&tarball_url)?; Access::Package(package_base.clone()).check(authority.as_ref())?; let integrity = pin_integrity(&dist)?; if options.offline { let cached = storage::read_tarball_cache(options, &dist.tarball, MAX_NPM_TARBALL_COMPRESSED)? .ok_or_else(|| NpmError::NpmOfflineCacheMiss { name: request_key.clone() })?; verify_sri(&cached, &integrity)?; return self .spawn_materialize( &instance, self.archive(&name, &version_text, &dist.tarball, &integrity), cached, None, ) .await; } let bytes = self .fetch_npm_tarball(&name, &version_text, &dist.tarball, &integrity) .await?; self.spawn_materialize( &instance, self.archive(&name, &version_text, &dist.tarball, &integrity), bytes, None, ) .await } /// Run the blocking stage-publish-verify sequence off the async workers. fn spawn_materialize( &self, instance: &str, archive: NpmArchive<'_>, bytes: Vec, expected: Option, ) -> impl std::future::Future> { let options = self.options.clone(); let authority = self.authority.clone(); let progress = self.progress.clone(); let instance = instance.to_owned(); let registry = archive.registry.clone(); let name = archive.name.to_owned(); let version = archive.version.to_owned(); let tarball = archive.tarball.to_owned(); let integrity = archive.integrity.to_owned(); async move { let access = Access::Package(npm_package_base(®istry, &name)?); access.check(authority.as_ref())?; if let Some(sink) = &progress { sink(&super::ProgressEvent::NpmMaterializing { name: name.clone(), version: version.clone(), }); access.check(authority.as_ref())?; } let artifact = tokio::task::spawn_blocking(move || { storage::materialize_npm( &options, &authority, NpmArchive { registry: ®istry, name: &name, version: &version, tarball: &tarball, integrity: &integrity, }, &instance, bytes, expected.as_ref(), ) }) .await .map_err(|detail| NpmError::NpmHttp { detail: detail.to_string() })??; if let Some(sink) = &progress { sink(&super::ProgressEvent::NpmMaterialized { name: artifact.identity.name.clone(), version: artifact.identity.version.clone(), }); } Ok(artifact) } } /// Authenticated file map behind a pin: legacy inline maps directly, /// compact pins through the project sidecar, the vendor mirror sidecar, /// or an SRI-verified tarball. Recovery republishes the sidecar; a warm /// tree alone never authenticates. async fn resolve_pin_files( &self, name: &str, version: &str, pin: &NpmPin, ) -> Result, NpmError> { let options = &self.options; if let Some(files) = &pin.files { return Ok(files.clone()); } let manifest = pin .manifest .as_ref() .ok_or(NpmError::NpmPinMissingManifest)?; if let Some(map) = storage::load_manifest_map(options, manifest)? { return Ok(map); } if let Some(map) = storage::load_vendor_map(options, manifest)? { storage::ensure_manifest_sidecar(options, &map)?; return Ok(map); } let bytes = match storage::read_tarball_cache(options, &pin.tarball, MAX_NPM_TARBALL_COMPRESSED)? { Some(bytes) => { verify_sri(&bytes, &pin.integrity)?; bytes } None if options.offline => { return Err(NpmError::NpmOfflineCacheMiss { name: format!("{name}@{version}"), }); } None => { self.fetch_npm_tarball(name, version, &pin.tarball, &pin.integrity) .await? } }; let data = collect_npm_files(&bytes)?; let (derived, _) = bind_npm_files(&data, name, version)?; if storage::manifest_digest(&derived)? != *manifest { return Err(NpmError::NpmArchiveChangedFor { name: name.to_owned(), version: version.to_owned(), }); } storage::ensure_manifest_sidecar(options, &derived)?; Ok(derived) } async fn pinned_npm( &self, instance: &str, name: &str, version: &str, pin: &NpmPin, verification: NpmVerification, ) -> Result { let options = &self.options; let authority = &self.authority; let npm_registry = &self.registry; validate_npm_version(version)?; validate_npm_pin(pin)?; let package_base = npm_package_base(npm_registry, name)?; let tarball_url = npm_tarball_url(&pin.tarball)?; if !same_origin(&tarball_url, npm_registry) { return Err(NpmError::NpmTarballEscapesRegistry { tarball: pin.tarball.clone() }); } check_https(&tarball_url)?; Access::Package(package_base).check(authority.as_ref())?; let files = self.resolve_pin_files(name, version, pin).await?; check_npm_exec(authority.as_ref(), npm_registry, name, version, &files)?; let dir = storage::npm_package_dir(options, instance, name, version); let package = match verification { NpmVerification::Full => { match storage::npm_dir_state(&options.project_root, &dir, &files)? { storage::NpmDirState::Verified(package) => Some(package), storage::NpmDirState::Absent => None, } } NpmVerification::OnUse => { storage::verify_npm_metadata(&options.project_root, &dir, &files)? } }; if let Some(package) = package { if pin.is_legacy() && !options.frozen { storage::ensure_manifest_sidecar(options, &files)?; } return Ok(NpmArtifact { identity: NpmIdentity { registry: npm_registry.clone(), name: name.to_owned(), version: version.to_owned(), }, tarball: pin.tarball.clone(), integrity: pin.integrity.clone(), dir, files, package, }); } if let Some(seeded) = storage::seed_npm_from_vendor( options, npm_registry, instance, name, version, pin, &files, )? { return Ok(seeded); } let cached = storage::read_tarball_cache(options, &pin.tarball, MAX_NPM_TARBALL_COMPRESSED)?; if let Some(bytes) = cached { verify_sri(&bytes, &pin.integrity)?; return self .spawn_materialize( instance, self.archive(name, version, &pin.tarball, &pin.integrity), bytes, Some(pin.clone()), ) .await; } if options.offline { return Err(NpmError::NpmOfflineCacheMiss { name: format!("{name}@{version}"), }); } let bytes = self .fetch_npm_tarball(name, version, &pin.tarball, &pin.integrity) .await?; self.spawn_materialize( instance, self.archive(name, version, &pin.tarball, &pin.integrity), bytes, Some(pin.clone()), ) .await } /// One packument per package shared by selection and acquisition. Completed /// successes stay for this store; malformed fetches evict so retries refetch. async fn shared_packument( &self, name: &str, workers: &Workers, ) -> Result, NpmError> { use futures::FutureExt; let name = name.to_owned(); let future = { let mut guard = workers.packuments.lock().unwrap(); if let Some(current) = guard.get(&name) { current.clone() } else { let fetcher = self.clone(); let task_name = name.clone(); let (send, receive) = futures::channel::oneshot::channel(); workers.runtime.spawn(async move { let result = async { let bytes = fetcher.fetch_npm_packument(&task_name).await?; let packument: NpmPackument = serde_json::from_slice(&bytes).map_err(|error| { NpmError::InvalidNpmPackument { name: task_name.clone(), source: Arc::new(error), } })?; if packument.name != task_name { return Err(NpmError::NpmPackumentNameMismatch { name: task_name.clone(), }); } Ok(Arc::new(packument)) } .await; let _ = send.send(result); }); let shared: SharedPackument = async move { receive .await .map_err(|_| NpmError::NpmMetadataWorkerStopped)? } .boxed() .shared(); guard.insert(name.clone(), shared.clone()); shared } }; let result = future.clone().await; if result.is_err() { let mut guard = workers.packuments.lock().unwrap(); if guard .get(&name) .is_some_and(|current| current.ptr_eq(&future)) { guard.remove(&name); } } result } async fn fetch_npm_packument(&self, name: &str) -> Result, NpmError> { let options = &self.options; let npm_registry = &self.registry; let base = npm_package_base(npm_registry, name)?; Access::Package(base.clone()).check(self.authority.as_ref())?; if options.offline { return storage::read_packument_cache(options, name, MAX_NPM_PACKUMENT_BYTES)? .ok_or(NpmError::NpmOfflineMetadataMiss { name: name.to_owned() }); } if let Some(sink) = &self.progress { sink(&super::ProgressEvent::NpmMetadata { name: name.to_owned() }); } let url = packument_url(npm_registry, name)?; let bytes = self .fetch_bytes( url, "application/vnd.npm.install-v1+json", MAX_NPM_PACKUMENT_BYTES, "metadata", Access::Package(base), ) .await?; serde_json::from_slice::(&bytes).map_err(|source| { NpmError::InvalidNpmPackument { name: name.to_owned(), source: Arc::new(source), } })?; storage::write_packument_cache(options, name, &bytes)?; Ok(bytes) } async fn fetch_npm_tarball( &self, name: &str, version: &str, tarball: &str, expected: &str, ) -> Result, NpmError> { let options = &self.options; let npm_registry = &self.registry; let url = npm_tarball_url(tarball)?; if !same_origin(&url, npm_registry) { return Err(NpmError::NpmTarballEscapesRegistry { tarball: tarball.to_owned() }); } check_https(&url)?; let access = Access::Package(npm_package_base(npm_registry, name)?); access.check(self.authority.as_ref())?; if let Some(bytes) = storage::read_tarball_cache(options, tarball, MAX_NPM_TARBALL_COMPRESSED)? { verify_sri(&bytes, expected)?; return Ok(bytes); } if let Some(sink) = &self.progress { sink(&super::ProgressEvent::NpmArchive { name: name.to_owned(), version: version.to_owned(), }); } let bytes = self .fetch_bytes( url, "application/octet-stream", MAX_NPM_TARBALL_COMPRESSED, "tarball", access, ) .await?; verify_sri(&bytes, expected)?; storage::write_tarball_cache(options, tarball, &bytes)?; Ok(bytes) } } impl PackageStore { fn fetcher(&self) -> Fetcher { Fetcher { options: self.options.clone(), client: self.client.clone(), authority: self.authority.clone(), progress: self.progress.clone(), registry: self.options.npm_registry.clone(), } } fn workers(&self) -> Workers { Workers { packuments: self.npm_packuments.clone(), downloads: self.npm_downloads.clone(), runtime: self.runtime.clone(), } } /// Select and lock automatic DefinitelyTyped discovery for one resolved package. pub async fn npm_types_request( &self, identity: &NpmIdentity, discover: bool, ) -> Result, NpmError> { let key = identity.key(); if identity.registry != self.options.npm_registry { return Err(NpmError::InvalidNpmTypesAssociation { identity: key }); } if identity.name.starts_with("@types/") { return Ok(None); } let replay = || { let lock = self.lock.lock().unwrap(); let mut used = self.used.lock().unwrap(); if (!self.options.update || used.npm_types_discovered.contains(&key)) && let Some(record) = lock.npm.types.get(&key) { used.npm_types.insert(key.clone()); Some(record.clone()) } else { None } }; if let Some(record) = replay() { return Ok(record); } if !discover || self.options.frozen { return Err(NpmError::NpmTypesNotPrepared { identity: key }); } let _gate = self.npm_types_gate.lock().await; if let Some(record) = replay() { return Ok(record); } let request = package::types_package_name(&identity.name); let library_version = validate_npm_version(&identity.version)?; let major = library_version.major; let next_major = major .checked_add(1) .ok_or_else(|| NpmError::InvalidNpmVersion { version: identity.version.clone(), })?; let range = format!(">={major}.0.0 <{next_major}.0.0"); let type_request = format!("npm:{request}@{range}"); let result = match self.npm_select_version(&type_request).await { Ok(_) => Some(parse_npm_request(&type_request)?.request_key()), Err(NpmError::NpmServerStatus { status: 404, .. }) | Err(NpmError::NoNpmVersionSatisfies { .. }) => None, Err(error) => return Err(error), }; let mut lock = self.lock.lock().unwrap(); let mut used = self.used.lock().unwrap(); used.npm_types.insert(key.clone()); used.npm_types_discovered.insert(key.clone()); if lock.npm.types.get(&key) != Some(&result) || self.options.update { used.pending_npm_types.insert(key.clone()); } lock.npm.types.insert(key, result.clone()); Ok(result) } /// Metadata-only version selection: lock reuse or one shared packument /// fetch, never archives, graphs, pins, or usage marks. pub async fn npm_select_version(&self, specifier: &str) -> Result { let request = parse_npm_request(specifier)?; let package_base = npm_package_base(&self.options.npm_registry, &request.name)?; Access::Package(package_base.clone()).check(self.authority.as_ref())?; let lock_snapshot = self.lock.lock().unwrap().clone(); let version_text = if let Some(locked) = lock_snapshot.npm.specifiers.get(&request.request_key()) { let version = validate_npm_version(locked)?; if !locked_satisfies(request.version_req.clone(), &version) { return Err(NpmError::NpmLockedVersionUnsatisfied { version: locked.clone(), specifier: request.request_key(), }); } locked.clone() } else if !self.options.update && !self.options.frozen && let Some((pinned, _)) = find_pinned_satisfying( &lock_snapshot, &self.options.npm_registry, &request.name, &request.version_req, ) { pinned } else { if self.options.frozen { return Err(NpmError::FrozenLockfileMissing { name: request.request_key() }); } let packument = self .fetcher() .shared_packument(&request.name, &self.workers()) .await?; select_npm_version(&packument, &request.version_req)?.0 }; validate_npm_version(&version_text)?; Access::Package(npm_version_base( &self.options.npm_registry, &request.name, &version_text, )?) .check(self.authority.as_ref())?; Access::Package(package_base).check(self.authority.as_ref())?; Ok(version_text) } pub async fn npm_acquire(&self, specifier: &str) -> Result { self.npm_acquire_in(NPM_DEFAULT_INSTANCE, specifier).await } pub async fn npm_acquire_in( &self, instance: &str, specifier: &str, ) -> Result { self.npm_acquire_mode(instance, specifier, NpmVerification::Full) .await } pub(crate) async fn npm_acquire_for_resolution( &self, specifier: &str, ) -> Result { self.npm_acquire_mode(NPM_DEFAULT_INSTANCE, specifier, NpmVerification::OnUse) .await } /// Materialize an already selected package without introducing a new request /// selection. In particular, frozen replay need not pin an exact-version alias. pub(crate) async fn npm_acquire_locked( &self, identity: &NpmIdentity, pin: &NpmPin, ) -> Result { if identity.registry != self.options.npm_registry { return Err(NpmError::NpmOriginOutsideRegistry { url: identity.registry.to_string(), }); } let fetcher = self.fetcher(); let downloads = self.npm_downloads.clone(); let identity = identity.clone(); let pin = pin.clone(); let artifact = self .runtime .spawn(async move { let _permit = downloads .acquire_owned() .await .map_err(|_| NpmError::NpmDownloadWorkerStopped)?; fetcher .pinned_npm( NPM_DEFAULT_INSTANCE, &identity.name, &identity.version, &pin, NpmVerification::OnUse, ) .await }) .await .map_err(|_| NpmError::NpmAcquisitionWorkerStopped)??; check_npm_exec( self.authority.as_ref(), &artifact.identity.registry, &artifact.identity.name, &artifact.identity.version, &artifact.files, )?; self.finish_npm_acquire(None, &artifact, NpmVerification::OnUse) } async fn npm_acquire_mode( &self, instance: &str, specifier: &str, verification: NpmVerification, ) -> Result { use futures::FutureExt; validate_instance(instance)?; let request = parse_npm_request(specifier)?; if request.subpath.is_some() { return Err(NpmError::NpmSubpathNeedsAcquire); } let request_key = request.request_key(); let package_base = npm_package_base(&self.options.npm_registry, &request.name)?; Access::Package(package_base.clone()).check(self.authority.as_ref())?; let key = format!("{instance}\0{request_key}\0{verification:?}"); let future: NpmAcquisition = { let mut inflight = self.npm_inflight.lock().unwrap(); inflight .entry(key.clone()) .or_insert_with(|| { let fetcher = self.fetcher(); let workers = self.workers(); let lock_snapshot = self.lock.lock().unwrap().clone(); let instance = instance.to_owned(); let request = request.clone(); let (send, receive) = futures::channel::oneshot::channel(); self.runtime.spawn(async move { let result = fetcher .npm_acquire_task( &workers, lock_snapshot, instance, request, verification, ) .await .map(Arc::new); let _ = send.send(result); }); async move { receive .await .map_err(|_| NpmError::NpmAcquisitionWorkerStopped)? } .boxed() .shared() }) .clone() }; let result = future.clone().await; { let mut inflight = self.npm_inflight.lock().unwrap(); if inflight .get(&key) .is_some_and(|current| current.ptr_eq(&future)) { inflight.remove(&key); } } let artifact = result?; Access::Package(package_base).check(self.authority.as_ref())?; check_npm_exec( self.authority.as_ref(), &artifact.identity.registry, &artifact.identity.name, &artifact.identity.version, &artifact.files, )?; self.finish_npm_acquire(Some(&request_key), &artifact, verification) } fn finish_npm_acquire( &self, request_key: Option<&str>, artifact: &NpmArtifact, verification: NpmVerification, ) -> Result { if verification == NpmVerification::Full { storage::verify_and_load_npm_dir( &self.options.project_root, &artifact.dir, &artifact.files, )?; } else { storage::verify_npm_metadata( &self.options.project_root, &artifact.dir, &artifact.files, )? .ok_or(storage::StorageError::NpmManifestMissing)?; } let mut lock = self.lock.lock().unwrap(); let mut used = self.used.lock().unwrap(); let identity_key = artifact.identity.key(); if let Some(request_key) = request_key && let Some(existing) = lock.npm.specifiers.get(request_key) && existing != &artifact.identity.version { return Err(NpmError::NpmConflictingVersionSelections { request_key: request_key.to_owned(), }); } let compact = NpmPin { tarball: artifact.tarball.clone(), integrity: artifact.integrity.clone(), manifest: Some(storage::manifest_digest(&artifact.files)?), files: None, }; if let Some(existing) = lock.npm.packages.get(&identity_key) && !existing.same_content(&compact) { return Err(NpmError::LockedNpmPackageChanged { key: identity_key.clone() }); } // A warm frozen legacy hit needs no sidecar; keep its inline map. let pin = if self.options.frozen { lock.npm .packages .get(&identity_key) .cloned() .unwrap_or(compact) } else { compact }; used.npm_packages.insert(identity_key.clone()); if let Some(request_key) = request_key { used.npm_specifiers.insert(request_key.to_owned()); if lock.npm.specifiers.get(request_key) != Some(&artifact.identity.version) || self.options.update { used.pending_npm_specifiers.insert(request_key.to_owned()); } lock.npm .specifiers .insert(request_key.to_owned(), artifact.identity.version.clone()); } if lock.npm.packages.get(&identity_key) != Some(&pin) { used.pending_npm_packages.insert(identity_key.clone()); } lock.npm.packages.insert(identity_key, pin); Ok((*artifact).clone()) } }