use std::collections::HashMap; use std::convert::Infallible; use std::future::Future; use std::io; use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use avril_caps::Capabilities; use avril_core::serde_v8::{JsBuffer, ToJsBuffer}; use avril_core::{Extension, Op}; use bytes::Bytes; use futures::StreamExt; use http_body_util::{BodyExt, Full, StreamBody, combinators::UnsyncBoxBody}; use hyper::{ Method, Request, body::Frame, header::{HeaderMap, HeaderName, HeaderValue}, }; use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use hyper_util::{ client::legacy::{Client, connect::HttpConnector}, rt::TokioExecutor, }; use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; use tokio::sync::{Mutex as AsyncMutex, mpsc}; use tokio_util::{io::StreamReader, sync::CancellationToken}; type Error = Box; type Body = UnsyncBoxBody; type Reader = Pin>; /// Status, status text, headers, final URL, whether redirected, and whether a redirect /// abandoned the streamed request body. type Metadata = (u16, String, Vec<(String, String)>, String, bool, bool); type SendArgs = ( u64, String, String, Vec<(String, String)>, Option, bool, String, ); type Upload = mpsc::Sender, Error>>; const MAX_ID: u64 = 9_007_199_254_740_991; enum Redirect { Follow, Error, Manual, } impl Redirect { fn parse(mode: &str) -> Result { match mode { "follow" => Ok(Self::Follow), "error" => Ok(Self::Error), "manual" => Ok(Self::Manual), _ => Err("Invalid redirect mode".into()), } } } enum RequestBody { Empty, Bytes(Bytes), Upload(mpsc::Receiver, Error>>), } struct PendingRequest { url: String, method: String, headers: Vec<(String, String)>, body: RequestBody, redirect: String, } struct Resource { canceled: CancellationToken, body: AsyncMutex>, upload: Mutex>, /// Releases writes blocked on a request body that will no longer be sent. upload_closed: CancellationToken, } impl Resource { fn close_upload(&self) { self.upload.lock().unwrap().take(); self.upload_closed.cancel(); } } struct Fetch { runtime: tokio::runtime::Handle, client: Client, Body>, resources: Arc>>>, next: AtomicU64, caps: Arc, } impl Fetch { fn resource(&self, id: u64) -> Result, Error> { self.resources .lock() .unwrap() .get(&id) .cloned() .ok_or_else(|| "Fetch resource is closed".into()) } fn run> + Send + 'static>( &self, future: F, ) -> impl Future> + use { let handle = self.runtime.spawn(future); async move { // Dropping the handle detaches the task rather than aborting it. handle .await .map_err(|_| io::Error::other("Fetch worker stopped"))? } } fn cancel(&self, id: u64) { if let Some(resource) = self.resources.lock().unwrap().remove(&id) { resource.canceled.cancel(); resource.close_upload(); } } fn send( &self, (id, url, method, headers, bytes, streaming, redirect): SendArgs, ) -> impl Future> + use<> { let resource = self.resource(id); let client = self.client.clone(); let caps = self.caps.clone(); // Install the upload channel before returning the promise, so JS can submit its first chunk. let body = if streaming { let (send, receive) = mpsc::channel(1); if let Ok(resource) = &resource { *resource.upload.lock().unwrap() = Some(send); } RequestBody::Upload(receive) } else { bytes .map(|bytes| Bytes::copy_from_slice(&bytes)) .map(RequestBody::Bytes) .unwrap_or(RequestBody::Empty) }; let request = PendingRequest { url, method, headers, body, redirect, }; self.run(async move { let resource = resource?; let canceled = resource.canceled.clone(); let result = tokio::select! { _ = canceled.cancelled() => Err("Fetch aborted".into()), result = send_request(client, caps, &resource, request) => result, }; result }) } } async fn send_request( client: Client, Body>, caps: Arc, resource: &Resource, request: PendingRequest, ) -> Result { let PendingRequest { url, method, headers, body, redirect, } = request; let url = url::Url::parse(&url)?; let method = Method::from_bytes(method.as_bytes())?; if ["CONNECT", "TRACE", "TRACK"] .iter() .any(|name| method.as_str().eq_ignore_ascii_case(name)) { return Err("Forbidden HTTP method".into()); } let redirect = Redirect::parse(&redirect)?; if !matches!(body, RequestBody::Empty) && (method == Method::GET || method == Method::HEAD) { return Err("GET/HEAD requests cannot have a body".into()); } let mut list = HeaderMap::new(); for (name, value) in headers { let name = name.to_ascii_lowercase(); if [ "host", "content-length", "transfer-encoding", "connection", "upgrade", "te", "trailer", "keep-alive", "expect", ] .contains(&name.as_str()) || name.starts_with("proxy-") { return Err(format!("Forbidden request header: {name}").into()); } let value = value .chars() .map(|c| u8::try_from(c as u32)) .collect::, _>>()?; list.append( HeaderName::from_bytes(name.as_bytes())?, HeaderValue::from_bytes(&value)?, ); } if !list.contains_key("accept") { list.insert("accept", HeaderValue::from_static("*/*")); } if !list.contains_key("accept-encoding") { list.insert( "accept-encoding", HeaderValue::from_static("gzip, br, deflate"), ); } let streaming = matches!(body, RequestBody::Upload(_)); let (mut url, mut method, mut body) = (url, method, body); let mut abandoned = false; for hop in 0..=20 { if !["http", "https"].contains(&url.scheme()) { return Err("not yet implemented: non-HTTP fetch".into()); } if !url.username().is_empty() || url.password().is_some() { return Err("Fetch URLs cannot contain credentials".into()); } url.set_fragment(None); // No ambient proxies, automatic redirects, or snapshot permissions in the connector. caps.check_net( url.host_str().ok_or("Missing fetch host")?, url.port_or_known_default().ok_or("Missing fetch port")?, )?; let request_body = match std::mem::replace(&mut body, RequestBody::Empty) { RequestBody::Upload(receive) => { let stream = futures::stream::unfold(receive, |mut receive| async move { receive.recv().await.map(|frame| (frame, receive)) }); StreamBody::new(stream).boxed_unsync() } RequestBody::Bytes(bytes) => { let payload = Full::new(bytes.clone()) .map_err(|never| match never {}) .boxed_unsync(); body = RequestBody::Bytes(bytes); payload } RequestBody::Empty => Full::new(Bytes::new()) .map_err(|never| match never {}) .boxed_unsync(), }; let mut request = Request::builder() .method(method.clone()) .uri(url.as_str()) .body(request_body)?; *request.headers_mut() = list.clone(); let response = client.request(request).await?; let status = response.status(); if matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308) && !matches!(redirect, Redirect::Manual) && let Some(location) = response.headers().get("location") { if matches!(redirect, Redirect::Error) { return Err("Redirect encountered in redirect:error mode".into()); } if hop == 20 { return Err("Too many redirects".into()); } let location = String::from_utf8_lossy(location.as_bytes()); let next = url.join(&location)?; if url.origin() != next.origin() { for name in ["authorization", "proxy-authorization", "cookie", "host"] { list.remove(name); } } if ((status.as_u16() == 301 || status.as_u16() == 302) && method == Method::POST) || (status.as_u16() == 303 && method != Method::GET && method != Method::HEAD) { method = Method::GET; body = RequestBody::Empty; for name in [ "content-type", "content-length", "content-encoding", "content-language", "content-location", ] { list.remove(name); } resource.close_upload(); abandoned = streaming; } else if streaming && method != Method::GET && method != Method::HEAD { return Err( "not yet implemented: replaying streaming request bodies on redirects".into(), ); } url = next; continue; } let headers = response .headers() .iter() .map(|(key, value)| { ( key.to_string(), value.as_bytes().iter().map(|b| char::from(*b)).collect(), ) }) .collect(); let status_text = response .extensions() .get::() .map(|reason| { reason .as_bytes() .iter() .map(|byte| char::from(*byte)) .collect::() }) .unwrap_or_else(|| status.canonical_reason().unwrap_or("").into()); let encoding = response .headers() .get_all("content-encoding") .iter() .map(|value| value.to_str()) .collect::, _>>()? .join(","); let stream = response .into_body() .into_data_stream() .map(|result| result.map_err(io::Error::other)); let mut reader: Reader = Box::pin(StreamReader::new(stream)); if method != Method::HEAD && !matches!(status.as_u16(), 204 | 205 | 304) { for encoding in encoding.split(',').rev().map(str::trim) { reader = match encoding.to_ascii_lowercase().as_str() { "" | "identity" => reader, "gzip" => { let mut decoder = async_compression::tokio::bufread::GzipDecoder::new( BufReader::new(reader), ); decoder.multiple_members(true); Box::pin(decoder) } "br" => Box::pin(async_compression::tokio::bufread::BrotliDecoder::new( BufReader::new(reader), )), "deflate" => Box::pin(async_compression::tokio::bufread::ZlibDecoder::new( BufReader::new(reader), )), _ => return Err("not yet implemented: response content encoding".into()), }; } } *resource.body.lock().await = Some(reader); return Ok(( status.as_u16(), status_text, headers, url.into(), hop != 0, abandoned, )); } unreachable!() } pub fn init(runtime: tokio::runtime::Handle, caps: Arc, base: String) -> Extension { let connector = HttpsConnectorBuilder::new() .with_webpki_roots() .https_or_http() .enable_http1() .enable_http2() .build(); let client = Client::builder(TokioExecutor::new()).build(connector); let state = Arc::new(Fetch { runtime, client, resources: Arc::new(Mutex::new(HashMap::new())), next: AtomicU64::new(1), caps, }); let create = state.clone(); let send = state.clone(); let read = state.clone(); let collect = state.clone(); let write = state.clone(); Extension { name: "ext:fetch", dependencies: &[ "ext:encoding", "ext:http_types", "ext:timers", "ext:url", "ext:web", "ext:webstreams", ], ops: vec![ Op::new("op_fetch_base", move |(): ()| { Ok::<_, Infallible>(base.clone()) }), Op::new("op_fetch_create", move |_: ()| { let id = create .next .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| { (id <= MAX_ID).then_some(id + 1) }) .map_err(|_| io::Error::other("Fetch IDs exhausted"))?; create.resources.lock().unwrap().insert( id, Arc::new(Resource { canceled: CancellationToken::new(), body: AsyncMutex::new(None), upload: Mutex::new(None), upload_closed: CancellationToken::new(), }), ); Ok::<_, io::Error>(id) }), Op::asynchronous("op_fetch_send", move |args: SendArgs| { let pending = send.send(args); async move { pending.await.map_err(io::Error::other) } }), Op::asynchronous("op_fetch_read", move |id: u64| { let resource = read.resource(id); let resources = read.resources.clone(); let pending = read.run(async move { let resource = resource?; tokio::select! { _ = resource.canceled.cancelled() => Err("Fetch aborted".into()), result = async { let mut body = resource.body.lock().await; let mut bytes = vec![0; 64 * 1024]; let count = body.as_mut().ok_or("Response body is unavailable")?.read(&mut bytes).await?; bytes.truncate(count); if count == 0 { body.take(); resources.lock().unwrap().remove(&id); } Ok::<_, Error>(bytes) } => result, } }); async move { Ok::<_, io::Error>(match pending.await.map_err(io::Error::other)? { bytes if bytes.is_empty() => None, bytes => Some(ToJsBuffer::from(bytes)), }) } }), Op::asynchronous("op_fetch_read_all", move |id: u64| { let resource = collect.resource(id); let resources = collect.resources.clone(); let pending = collect.run(async move { let resource = resource?; let result = tokio::select! { biased; _ = resource.canceled.cancelled() => Err("Fetch aborted".into()), result = async { let mut reader = resource.body.lock().await.take() .ok_or("Response body is unavailable")?; let mut bytes = Vec::new(); reader.read_to_end(&mut bytes).await?; Ok::<_, Error>(bytes) } => result, }; resources.lock().unwrap().remove(&id); result }); async move { pending .await .map(ToJsBuffer::from) .map_err(io::Error::other) } }), Op::asynchronous( "op_fetch_write", move |(id, bytes): (u64, Option)| { let resource = write.resource(id); let bytes = bytes.map(|bytes| Bytes::copy_from_slice(&bytes)); let pending = write.run(async move { let Ok(resource) = resource else { return Ok(false); }; if let Some(bytes) = bytes { let Some(upload) = resource.upload.lock().unwrap().clone() else { return Ok(false); }; tokio::select! { _ = resource.canceled.cancelled() => Err("Fetch aborted".into()), _ = resource.upload_closed.cancelled() => Ok(false), result = upload.send(Ok(Frame::data(bytes))) => Ok(result.is_ok()), } } else { Ok(resource.upload.lock().unwrap().take().is_some()) } }); async move { pending.await.map_err(io::Error::other) } }, ), Op::new("op_fetch_cancel", move |id: u64| { state.cancel(id); Ok::<_, io::Error>(()) }), ], modules: &[ ( "ext:fetch/main.js", include_str!(concat!(env!("OUT_DIR"), "/main.js")), ), ( "ext:fetch/fetch.js", include_str!(concat!(env!("OUT_DIR"), "/fetch.js")), ), ( "ext:fetch/eventsource.js", include_str!(concat!(env!("OUT_DIR"), "/eventsource.js")), ), ], entry_point: Some("ext:fetch/main.js"), } }