import { inflate, oidBytes, parseCommit, parseLoose, parseTag, parseTree, toHex } from "./codec.ts"; import type { Pack } from "./pack.ts"; import * as j from "@char/justin"; import { type Commit, type GitInfo, GitInfoSchema, type GitObject, type TreeEntry, } from "./types.ts"; /** * fetch a repo-site-relative path (e.g. `.git/objects/info/packs`). * `range` is [start, end) into the resource, end=null meaning to-end; * `query` sends a QUERY request with an application/json body; * `total` reports the full resource size. resolves null on 404. */ export type Fetcher = ( path: string, range?: [number, number | null], query?: Uint8Array, ) => Promise<{ bytes: Uint8Array; total: number } | null>; export class HttpError extends Error { constructor(readonly status: number, path: string) { super(`fetching ${path}: HTTP ${status}`); } } export interface TransferProgress { id: number; path: string; loaded: number; total?: number; phase: "start" | "progress" | "done"; } export const httpFetcher = ( base: string, progress?: (transfer: TransferProgress) => void, signal?: () => AbortSignal, ): Fetcher => { let nextId = 0; return (path, range, query) => new Promise((resolve, reject) => { const active = signal?.(); if (active?.aborted) return reject(new Error(`fetching ${path}: aborted`)); const id = nextId++; const request = new XMLHttpRequest(); const onAbort = () => request.abort(); active?.addEventListener("abort", onAbort); request.onloadend = () => active?.removeEventListener("abort", onAbort); request.open(query ? "QUERY" : "GET", `${base}/${path}`); request.responseType = "arraybuffer"; if (query) { request.setRequestHeader("Content-Type", "application/json"); request.setRequestHeader("Accept", "application/x-git-object-bundle"); request.setRequestHeader("Cache-Control", "no-store"); } if (range) request.setRequestHeader("Range", `bytes=${range[0]}-${range[1] === null ? "" : range[1] - 1}`); const expected = range && range[1] !== null ? range[1] - range[0] : undefined; let loaded = 0; progress?.({ id, path, loaded, total: expected, phase: "start" }); request.onprogress = event => { loaded = event.loaded; progress?.({ id, path, loaded, total: event.lengthComputable ? event.total : expected, phase: "progress", }); }; request.onerror = () => { progress?.({ id, path, loaded, total: expected, phase: "done" }); reject(new Error(`fetching ${path}: network error`)); }; request.onabort = () => { progress?.({ id, path, loaded, total: expected, phase: "done" }); reject(new Error(`fetching ${path}: aborted`)); }; request.onload = () => { const bytes = new Uint8Array(request.response); progress?.({ id, path, loaded: bytes.length, total: bytes.length, phase: "done" }); if (request.status === 404) return resolve(null); if (request.status !== 200 && request.status !== 206) { return reject(new HttpError(request.status, path)); } const contentRange = request.getResponseHeader("content-range"); const total = contentRange ? Number(contentRange.split("/")[1]) : bytes.length; resolve({ bytes, total }); }; request.send(query ?? null); }); }; const validateGitInfo = j.validation.compile(GitInfoSchema); const BUNDLED_OBJECT_TYPES = [undefined, "commit", "tree", "blob", "tag"] as const; /** a payload-less frame naming a commit a path-history walk resumes from */ const FRONTIER_FRAME = 5; const OBJECT_QUERY_CHUNK_SIZE = 64; const OBJECT_QUERY_CONCURRENCY = 2; const OBJECT_FALLBACK_CONCURRENCY = 4; const PACK_PATH = /\/pack-[0-9a-f]+\.(?:idx|pack)$/; const PACK_CACHE_ORIGIN = "https://sorcery-pack-cache.invalid"; const OBJECT_CACHE_ORIGIN = "https://sorcery-object-cache.invalid"; export type ObjectQuery = | { depth: number } | { smart: "tree" | "commit-diff" } | { smart: "commit-pagination"; limit: number }; type PathHistoryQuery = { smart: "path-history"; limit: number; path: string[] }; interface ObjectBundle { /** in bundle order, which smart queries make meaningful */ objects: Map; frontier: string[]; } function parseObjectBundle(bytes: Uint8Array): ObjectBundle { if (bytes.length < 6 || new TextDecoder().decode(bytes.subarray(0, 4)) !== "SOBJ" || bytes[4] !== 1) { throw new Error("invalid object bundle"); } const objects = new Map(); const frontier: string[] = []; let position = 6; while (position < bytes.length) { const oidBytes = bytes[position++]; if ((oidBytes !== 20 && oidBytes !== 32) || position + oidBytes + 5 > bytes.length) { throw new Error("invalid object bundle frame"); } const oid = toHex(bytes.subarray(position, position + oidBytes)); position += oidBytes; const kind = bytes[position++]; const size = new DataView(bytes.buffer, bytes.byteOffset + position, 4).getUint32(0); position += 4; if (position + size > bytes.length) throw new Error("truncated object bundle"); if (kind === FRONTIER_FRAME) { frontier.push(oid); } else { const type = BUNDLED_OBJECT_TYPES[kind]; if (!type) throw new Error("invalid bundled object type"); objects.set(oid, { type, data: bytes.subarray(position, position + size) }); } position += size; } return { objects, frontier }; } export class GitRepo { #info?: Promise; #packs?: Promise; #objectQueryEndpoint = true; #storage: Promise; constructor(readonly fetch: Fetcher, cacheName?: string) { this.#storage = cacheName && "caches" in globalThis ? caches.open(cacheName).catch(() => null) : Promise.resolve(null); } info(): Promise { if (!this.#info) { this.#info = this.#cachedFetch("gitinfo.json").then(res => { if (!res) throw new Error("missing gitinfo.json"); const { value, errors } = validateGitInfo(JSON.parse(new TextDecoder().decode(res.bytes))); if (errors) throw new Error(`bad gitinfo.json: ${errors.map(e => `${e.path} ${e.msg}`).join(", ")}`); return value; }); this.#info.catch(() => (this.#info = undefined)); } return this.#info; } async object(oid: string): Promise { const cached = await this.#cachedObject(oid); if (cached) return cached; const bundled = (await this.#query([oid], { depth: 0 }))?.objects.get(oid); if (bundled) return bundled; const packs = await this.#allPacks(); const indexes = await Promise.all(packs.map(pack => pack.index())); let object: GitObject | null = null; for (let i = 0; i < packs.length; i++) { const offset = await indexes[i].lookup(oid); if (offset !== null) { object = await packs[i].readAt(offset, this); break; } } if (!object) { const loose = await this.#cachedFetch(`.git/objects/${oid.slice(0, 2)}/${oid.slice(2)}`); if (loose) object = parseLoose(await inflate(loose.bytes)); } if (!object) throw new Error(`object ${oid} not found`); await this.#storeObjects([[oid, object]]); return object; } async prefetch(oids: string[], objectQuery: ObjectQuery): Promise { const roots = [...new Set(oids)]; let cached: boolean[]; if ("smart" in objectQuery && objectQuery.smart === "commit-pagination") { cached = roots.map(() => false); } else { cached = await Promise.all(roots.map(oid => "smart" in objectQuery ? this.#hasSmart(objectQuery.smart, oid) : this.#hasObject(oid) )); } const pending = roots.filter((_, i) => !cached[i]); if (pending.length === 0) return; const chunks = Array.from( { length: Math.ceil(pending.length / OBJECT_QUERY_CHUNK_SIZE) }, (_, i) => pending.slice(i * OBJECT_QUERY_CHUNK_SIZE, (i + 1) * OBJECT_QUERY_CHUNK_SIZE), ); const loaded = new Set(); let nextChunk = 0; const query = async () => { while (nextChunk < chunks.length) { for (const oid of (await this.#query(chunks[nextChunk++], objectQuery))?.objects.keys() ?? []) loaded.add(oid); } }; await Promise.all(Array.from({ length: Math.min(OBJECT_QUERY_CONCURRENCY, chunks.length) }, query)); const present = await Promise.all(pending.map(oid => loaded.has(oid) || this.#hasObject(oid))); const missing = pending.filter((_, i) => !present[i]); let next = 0; const load = async () => { while (next < missing.length) await this.object(missing[next++]); }; await Promise.all(Array.from({ length: Math.min(OBJECT_FALLBACK_CONCURRENCY, missing.length) }, load)); } /** * a page of `git log -- path` computed by the daemon, resuming from * `frontier` (see `src/history.rs`). the commits' trees along `path` come * bundled too. null when the daemon's query endpoint is unavailable. */ async pathHistory( frontier: string[], path: string[], limit: number, ): Promise<{ commits: Commit[]; frontier: string[] } | null> { const bundle = await this.#query(frontier, { smart: "path-history", limit, path }); if (!bundle) return null; const commits = []; for (const [oid, object] of bundle.objects) { if (object.type === "commit") commits.push(parseCommit(oid, object.data)); } return { commits, frontier: bundle.frontier }; } /** * null means the server has no object query endpoint (404 or 501), which * is the only reason to read packfiles instead; any other failure is * reported, not silently downgraded */ async #query(oids: string[], query: ObjectQuery | PathHistoryQuery): Promise { if (!this.#objectQueryEndpoint) return null; const body = new TextEncoder().encode(JSON.stringify({ oids, ...query })); let result; try { result = await this.fetch("obj", undefined, body); } catch (err) { if (!(err instanceof HttpError && err.status === 501)) throw err; result = null; } if (!result) { this.#objectQueryEndpoint = false; return null; } const bundle = parseObjectBundle(result.bytes); await this.#storeObjects(bundle.objects); if ("smart" in query && (query.smart === "tree" || query.smart === "commit-diff")) { await this.#markSmart(query.smart, oids); } return bundle; } /** the pack reader is the fallback when the daemon's `/obj` is unavailable, so it loads lazily */ #allPacks(): Promise { if (!this.#packs) { const fetch: Fetcher = (path, range) => this.#cachedFetch(path, range); this.#packs = Promise.all([this.info(), import("./pack.ts")]) .then(([info, { Pack }]) => info.packs.map(stem => new Pack(fetch, stem))); this.#packs.catch(() => (this.#packs = undefined)); } return this.#packs; } async #cachedFetch( path: string, range?: [number, number | null], ): ReturnType { if (path === "gitinfo.json") { const result = await this.fetch(path, range); if (result) void this.#prunePacks(result.bytes); return result; } const cache = PACK_PATH.test(path) ? await this.#storage : null; if (!cache) return this.fetch(path, range); const key = `${PACK_CACHE_ORIGIN}/${path}?range=${range ? `${range[0]}-${range[1] ?? ""}` : "all"}`; const hit = await cache.match(key).catch(() => undefined); if (hit) { return { bytes: new Uint8Array(await hit.arrayBuffer()), total: Number(hit.headers.get("x-sorcery-total")), }; } const result = await this.fetch(path, range); if (result) { await cache.put(key, new Response(result.bytes.slice(), { headers: { "x-sorcery-total": String(result.total) }, })).catch(() => {}); } return result; } async #cachedObject(oid: string): Promise { const cache = await this.#storage; const hit = await cache?.match(`${OBJECT_CACHE_ORIGIN}/object/${oid}`).catch(() => undefined); if (!hit) return null; const type = hit.headers.get("x-sorcery-object-type"); if (type !== "commit" && type !== "tree" && type !== "blob" && type !== "tag") return null; return { type, data: new Uint8Array(await hit.arrayBuffer()) }; } async #hasObject(oid: string): Promise { const cache = await this.#storage; return !!await cache?.match(`${OBJECT_CACHE_ORIGIN}/object/${oid}`).catch(() => undefined); } async #storeObjects(objects: Iterable<[string, GitObject]>): Promise { const cache = await this.#storage; if (!cache) return; await Promise.all(Array.from(objects, ([oid, object]) => cache.put(`${OBJECT_CACHE_ORIGIN}/object/${oid}`, new Response(object.data.slice(), { headers: { "x-sorcery-object-type": object.type }, })).catch(() => {}) )); } async #hasSmart(smart: "tree" | "commit-diff", oid: string): Promise { const cache = await this.#storage; return !!await cache?.match(`${OBJECT_CACHE_ORIGIN}/smart-v1/${smart}/${oid}`).catch(() => undefined); } async #markSmart(smart: "tree" | "commit-diff", oids: string[]): Promise { const cache = await this.#storage; if (!cache) return; await Promise.all(oids.map(oid => cache.put(`${OBJECT_CACHE_ORIGIN}/smart-v1/${smart}/${oid}`, new Response()).catch(() => {}) )); } async #prunePacks(manifest: Uint8Array): Promise { const cache = await this.#storage; if (!cache) return; try { const packs = (JSON.parse(new TextDecoder().decode(manifest)) as { packs?: string[] }).packs; const live = new Set(packs ?? []); for (const request of await cache.keys()) { const stem = new URL(request.url).pathname.match(/(pack-[0-9a-f]+)\.[a-z]+$/)?.[1]; if (stem && !live.has(stem)) await cache.delete(request); } } catch { // a malformed manifest fails schema validation upstream; nothing to do here } } async commit(oid: string): Promise { const object = await this.object(oid); if (object.type === "tag") return this.commit(parseTag(object.data).object); if (object.type !== "commit") throw new Error(`${oid} is a ${object.type}, not a commit`); return parseCommit(oid, object.data); } async tree(oid: string): Promise { const object = await this.object(oid); if (object.type !== "tree") throw new Error(`${oid} is a ${object.type}, not a tree`); return parseTree(object.data, oidBytes(oid).length); } async blob(oid: string): Promise { const object = await this.object(oid); if (object.type !== "blob") throw new Error(`${oid} is a ${object.type}, not a blob`); return object.data; } }