char/avril

(wip) why'd you have to go and make things so complicaaaaated?

git clone https://git.t4t.associates/char/avril

Charlotte Somadd progress to module imports from registries7b01146

main
branchesmainall refs →
12.1 KiB321 linesraw
1use std::path::{Path, PathBuf};
2use std::time::Duration;
3
4use std::sync::Arc;
5
6use http_body_util::{BodyExt, Empty};
7use hyper::Request;
8use url::Url;
9
10use super::{
11    Access, AcquisitionContext, MAX_ARTIFACT_SIZE, RemoteArtifact, RemoteEntry, artifact_key,
12    atomic_write, checksum, read_artifact, valid_checksum, verify,
13};
14
15/// Failures fetching remote modules over HTTP(S).
16#[derive(Clone, Debug, thiserror::Error)]
17pub enum HttpError {
18    #[error("Invalid HTTP module URL {name}: {source}")]
19    InvalidHttpModuleUrl {
20        name: String,
21        source: url::ParseError,
22    },
23    #[error("Expected an HTTP(S) module URL: {name}")]
24    ExpectedHttpModuleUrl { name: String },
25    #[error("HTTP module credentials are not supported: {name}")]
26    HttpModuleCredentials { name: String },
27    #[error("Invalid remote source checksum")]
28    InvalidRemoteChecksum,
29    #[error("Invalid HTTP cache metadata for {name}: {source}")]
30    InvalidHttpCacheMetadata {
31        name: String,
32        source: Arc<serde_json::Error>,
33    },
34    #[error("Offline HTTP source cache miss: {name}")]
35    OfflineHttpCacheMiss { name: String },
36    #[error("Too many HTTP redirects: {name}")]
37    TooManyHttpRedirects { name: String },
38    #[error("HTTP redirect has no valid Location: {current}")]
39    HttpRedirectNoLocation { current: String },
40    #[error("HTTP server returned {status} for {current}")]
41    HttpStatus { status: String, current: String },
42    #[error("HTTP server returned HTML instead of a module: {current}")]
43    HttpHtmlModule { current: String },
44    #[error("HTTP source is too large: {name}")]
45    HttpSourceTooLarge { name: String },
46    #[error("{detail}")]
47    HttpRequest { detail: String },
48    #[error("HTTP request timed out: {name}")]
49    HttpTimeout { name: String },
50    #[error("Remote final URL mismatch: {name}")]
51    RemoteFinalUrlMismatch { name: String },
52    #[error("{name}: Package integrity mismatch")]
53    VerifyMismatch { name: String },
54    #[error("{detail}")]
55    WorkerJoin { detail: String },
56    #[error("Module is not UTF-8: {name}")]
57    ModuleNotUtf8 { name: String },
58    #[error(transparent)]
59    Hyper(Arc<hyper::Error>),
60    #[error(transparent)]
61    Registry(Box<super::RegistryError>),
62    #[error(transparent)]
63    Caps(#[from] crate::caps::CapabilitiesError),
64    #[error(transparent)]
65    Url(#[from] url::ParseError),
66    #[error(transparent)]
67    Io(std::sync::Arc<std::io::Error>),
68    #[error(transparent)]
69    Json(std::sync::Arc<serde_json::Error>),
70}
71
72pub(super) fn http_url(name: &str) -> Result<Url, HttpError> {
73    let url = Url::parse(name)
74        .map_err(|source| HttpError::InvalidHttpModuleUrl { name: name.to_owned(), source })?;
75    if !matches!(url.scheme(), "http" | "https") || url.host().is_none() {
76        return Err(HttpError::ExpectedHttpModuleUrl { name: name.to_owned() });
77    }
78    if !url.username().is_empty() || url.password().is_some() {
79        return Err(HttpError::HttpModuleCredentials { name: name.to_owned() });
80    }
81    Ok(url)
82}
83
84pub(super) fn validate_remote_entry(entry: &RemoteEntry) -> Result<(), HttpError> {
85    if !valid_checksum(&entry.checksum) {
86        return Err(HttpError::InvalidRemoteChecksum);
87    }
88    http_url(&entry.url)?;
89    Ok(())
90}
91
92pub(super) fn remote_blob(cache: &Path, checksum: &str) -> PathBuf {
93    cache.join("http").join("blobs").join(checksum)
94}
95
96pub(super) fn remote_vendor_path(name: &str, base: &str) -> Result<PathBuf, HttpError> {
97    let url = http_url(base)?;
98    let extension = Path::new(url.path())
99        .extension()
100        .and_then(|ext| ext.to_str())
101        .filter(|ext| {
102            !ext.is_empty() && ext.len() <= 16 && ext.bytes().all(|ch| ch.is_ascii_alphanumeric())
103        })
104        .unwrap_or("js");
105    Ok(PathBuf::from("http").join(format!("{}.{extension}", artifact_key(name))))
106}
107
108pub(super) async fn acquire_remote(
109    context: AcquisitionContext,
110    name: String,
111    url: Url,
112    expected: Option<RemoteEntry>,
113    access: Access,
114) -> Result<RemoteArtifact, HttpError> {
115    let AcquisitionContext {
116        client,
117        options,
118        authority,
119        progress,
120    } = context;
121    access.check(authority.as_ref())?;
122    let key = artifact_key(&name);
123    let metadata_path = options.cache.join("http").join("metadata").join(&key);
124    let lock_path = options.cache.join("locks").join(format!("http-{key}"));
125    std::fs::create_dir_all(lock_path.parent().unwrap())?;
126    let _guard = tokio::task::spawn_blocking(move || {
127        let mut guard = fslock::LockFile::open(&lock_path)?;
128        guard.lock()?;
129        Ok::<_, HttpError>(guard)
130    })
131    .await
132    .map_err(|detail| HttpError::WorkerJoin { detail: detail.to_string() })??;
133    access.check(authority.as_ref())?;
134    let refresh = options.update && !options.offline;
135    let expected = if refresh { None } else { expected };
136    let mut entry = expected.clone();
137    if !refresh {
138        if entry.is_none()
139            && let Some(bytes) = read_artifact(&metadata_path)?
140        {
141            entry = Some(
142                serde_json::from_slice::<RemoteEntry>(&bytes).map_err(|source| {
143                    HttpError::InvalidHttpCacheMetadata {
144                        name: name.to_owned(),
145                        source: Arc::new(source),
146                    }
147                })?,
148            );
149        }
150        if let Some(entry) = &entry {
151            validate_remote_entry(entry)?;
152            let vendored = match &options.vendor {
153                Some(vendor) => {
154                    read_artifact(&vendor.join(remote_vendor_path(&name, &entry.url)?))?
155                }
156                None => None,
157            };
158            let bytes = match vendored {
159                Some(bytes) => Some(bytes),
160                None => read_artifact(&remote_blob(&options.cache, &entry.checksum))?,
161            };
162            if let Some(bytes) = bytes {
163                verify(&bytes, &entry.checksum)
164                    .map_err(|_| HttpError::VerifyMismatch { name: name.to_owned() })?;
165                access.check(authority.as_ref())?;
166                if let Some(authority) = authority.as_ref() {
167                    authority.check_remote_final(&entry.url)?;
168                }
169                return Ok(RemoteArtifact { bytes, entry: entry.clone() });
170            }
171        }
172    }
173    if options.offline {
174        return Err(HttpError::OfflineHttpCacheMiss { name: name.to_owned() });
175    }
176    let download = async {
177        let mut current = url;
178        // Execute keeps existing redirect semantics: only the requested URL
179        // needs authority. Read checks the network for every actual
180        // destination, including each redirect target.
181        let check_current = |current: &Url| -> Result<(), HttpError> {
182            access.check(authority.as_ref())?;
183            if let Some(authority) = authority.as_ref() {
184                authority.check_remote_final(current.as_str())?;
185            }
186            Ok(())
187        };
188        for redirects in 0..=10 {
189            http_url(current.as_str())?;
190            check_current(&current)?;
191            if let Some(sink) = &progress {
192                sink(&super::ProgressEvent::SourceDownload { url: current.to_string() });
193                check_current(&current)?;
194            }
195            let mut request_url = current.clone();
196            request_url.set_fragment(None);
197            let request = Request::get(request_url.as_str())
198                .header(
199                    "Accept",
200                    "application/typescript, application/javascript, */*",
201                )
202                .header("User-Agent", concat!("avril/", env!("CARGO_PKG_VERSION")))
203                .body(Empty::new())
204                .map_err(|detail| HttpError::HttpRequest { detail: detail.to_string() })?;
205            let response = client
206                .request(request)
207                .await
208                .map_err(|detail| HttpError::HttpRequest { detail: detail.to_string() })?;
209            check_current(&current)?;
210            if matches!(response.status().as_u16(), 301 | 302 | 303 | 307 | 308) {
211                if redirects == 10 {
212                    return Err(HttpError::TooManyHttpRedirects { name: name.to_owned() });
213                }
214                let location = response
215                    .headers()
216                    .get("location")
217                    .and_then(|value| value.to_str().ok())
218                    .ok_or_else(|| HttpError::HttpRedirectNoLocation {
219                        current: current.to_string(),
220                    })?;
221                current = current.join(location)?;
222                continue;
223            }
224            if !response.status().is_success() {
225                return Err(HttpError::HttpStatus {
226                    status: response.status().to_string(),
227                    current: current.to_string(),
228                });
229            }
230            if response
231                .headers()
232                .get("content-type")
233                .and_then(|value| value.to_str().ok())
234                .is_some_and(|value| {
235                    matches!(
236                        value
237                            .split(';')
238                            .next()
239                            .unwrap_or("")
240                            .trim()
241                            .to_ascii_lowercase()
242                            .as_str(),
243                        "text/html" | "application/xhtml+xml"
244                    )
245                })
246            {
247                return Err(HttpError::HttpHtmlModule { current: current.to_string() });
248            }
249            let mut body = response.into_body();
250            let mut bytes = Vec::new();
251            while let Some(frame) = body.frame().await {
252                if let Ok(data) = frame
253                    .map_err(|error| HttpError::Hyper(Arc::new(error)))?
254                    .into_data()
255                {
256                    if bytes.len() + data.len() > MAX_ARTIFACT_SIZE {
257                        return Err(HttpError::HttpSourceTooLarge { name: name.to_owned() });
258                    }
259                    bytes.extend_from_slice(&data);
260                }
261            }
262            let source = std::str::from_utf8(&bytes)
263                .map_err(|_| HttpError::ModuleNotUtf8 { name: name.to_owned() })?;
264            let prefix: String = source
265                .trim_start_matches(['\u{feff}', ' ', '\t', '\r', '\n'])
266                .chars()
267                .take(32)
268                .collect();
269            let prefix = prefix.to_ascii_lowercase();
270            if prefix.starts_with("<!doctype html") || prefix.starts_with("<html") {
271                return Err(HttpError::HttpHtmlModule { current: current.to_string() });
272            }
273            return Ok(RemoteArtifact {
274                entry: RemoteEntry {
275                    checksum: checksum(&bytes),
276                    url: current.to_string(),
277                },
278                bytes,
279            });
280        }
281        unreachable!()
282    };
283    let artifact = tokio::time::timeout(Duration::from_secs(30), download)
284        .await
285        .map_err(|_| HttpError::HttpTimeout { name: name.to_owned() })??;
286    if let Some(entry) = expected {
287        verify(&artifact.bytes, &entry.checksum)
288            .map_err(|_| HttpError::VerifyMismatch { name: name.to_owned() })?;
289        if artifact.entry.url != entry.url {
290            return Err(HttpError::RemoteFinalUrlMismatch { name: name.to_owned() });
291        }
292    }
293    access.check(authority.as_ref())?;
294    if let Some(authority) = authority.as_ref() {
295        authority.check_remote_final(&artifact.entry.url)?;
296    }
297    let blob = remote_blob(&options.cache, &artifact.entry.checksum);
298    std::fs::create_dir_all(blob.parent().unwrap())?;
299    atomic_write(&blob, &artifact.bytes)?;
300    std::fs::create_dir_all(metadata_path.parent().unwrap())?;
301    atomic_write(&metadata_path, &serde_json::to_vec(&artifact.entry)?)?;
302    Ok(artifact)
303}
304
305impl From<std::io::Error> for HttpError {
306    fn from(error: std::io::Error) -> Self {
307        Self::Io(std::sync::Arc::new(error))
308    }
309}
310
311impl From<serde_json::Error> for HttpError {
312    fn from(error: serde_json::Error) -> Self {
313        Self::Json(std::sync::Arc::new(error))
314    }
315}
316
317impl From<super::RegistryError> for HttpError {
318    fn from(error: super::RegistryError) -> Self {
319        Self::Registry(Box::new(error))
320    }
321}