char/sorcery

static-files based git repo viewer

git clone https://git.t4t.associates/char/sorcery

Charlotte Somexperiment: support sha-256 oids in git repos42f80d8

main
branchesmainall refs →
14.5 KiB392 linesraw
1import { inflate, oidBytes, parseCommit, parseLoose, parseTag, parseTree, toHex } from "./codec.ts";
2import type { Pack } from "./pack.ts";
3import * as j from "@char/justin";
4import {
5  type Commit,
6  type GitInfo,
7  GitInfoSchema,
8  type GitObject,
9  type TreeEntry,
10} from "./types.ts";
11
12/**
13 * fetch a repo-site-relative path (e.g. `.git/objects/info/packs`).
14 * `range` is [start, end) into the resource, end=null meaning to-end;
15 * `query` sends a QUERY request with an application/json body;
16 * `total` reports the full resource size. resolves null on 404.
17 */
18export type Fetcher = (
19  path: string,
20  range?: [number, number | null],
21  query?: Uint8Array<ArrayBuffer>,
22) => Promise<{ bytes: Uint8Array; total: number } | null>;
23
24export class HttpError extends Error {
25  constructor(readonly status: number, path: string) {
26    super(`fetching ${path}: HTTP ${status}`);
27  }
28}
29
30export interface TransferProgress {
31  id: number;
32  path: string;
33  loaded: number;
34  total?: number;
35  phase: "start" | "progress" | "done";
36}
37
38export const httpFetcher = (
39  base: string,
40  progress?: (transfer: TransferProgress) => void,
41  signal?: () => AbortSignal,
42): Fetcher => {
43  let nextId = 0;
44  return (path, range, query) => new Promise((resolve, reject) => {
45    const active = signal?.();
46    if (active?.aborted) return reject(new Error(`fetching ${path}: aborted`));
47    const id = nextId++;
48    const request = new XMLHttpRequest();
49    const onAbort = () => request.abort();
50    active?.addEventListener("abort", onAbort);
51    request.onloadend = () => active?.removeEventListener("abort", onAbort);
52    request.open(query ? "QUERY" : "GET", `${base}/${path}`);
53    request.responseType = "arraybuffer";
54    if (query) {
55      request.setRequestHeader("Content-Type", "application/json");
56      request.setRequestHeader("Accept", "application/x-git-object-bundle");
57      request.setRequestHeader("Cache-Control", "no-store");
58    }
59    if (range) request.setRequestHeader("Range", `bytes=${range[0]}-${range[1] === null ? "" : range[1] - 1}`);
60    const expected = range && range[1] !== null ? range[1] - range[0] : undefined;
61    let loaded = 0;
62    progress?.({ id, path, loaded, total: expected, phase: "start" });
63    request.onprogress = event => {
64      loaded = event.loaded;
65      progress?.({
66        id,
67        path,
68        loaded,
69        total: event.lengthComputable ? event.total : expected,
70        phase: "progress",
71      });
72    };
73    request.onerror = () => {
74      progress?.({ id, path, loaded, total: expected, phase: "done" });
75      reject(new Error(`fetching ${path}: network error`));
76    };
77    request.onabort = () => {
78      progress?.({ id, path, loaded, total: expected, phase: "done" });
79      reject(new Error(`fetching ${path}: aborted`));
80    };
81    request.onload = () => {
82      const bytes = new Uint8Array(request.response);
83      progress?.({ id, path, loaded: bytes.length, total: bytes.length, phase: "done" });
84      if (request.status === 404) return resolve(null);
85      if (request.status !== 200 && request.status !== 206) {
86        return reject(new HttpError(request.status, path));
87      }
88      const contentRange = request.getResponseHeader("content-range");
89      const total = contentRange ? Number(contentRange.split("/")[1]) : bytes.length;
90      resolve({ bytes, total });
91    };
92    request.send(query ?? null);
93  });
94};
95
96const validateGitInfo = j.validation.compile(GitInfoSchema);
97const BUNDLED_OBJECT_TYPES = [undefined, "commit", "tree", "blob", "tag"] as const;
98/** a payload-less frame naming a commit a path-history walk resumes from */
99const FRONTIER_FRAME = 5;
100const OBJECT_QUERY_CHUNK_SIZE = 64;
101const OBJECT_QUERY_CONCURRENCY = 2;
102const OBJECT_FALLBACK_CONCURRENCY = 4;
103const PACK_PATH = /\/pack-[0-9a-f]+\.(?:idx|pack)$/;
104const PACK_CACHE_ORIGIN = "https://sorcery-pack-cache.invalid";
105const OBJECT_CACHE_ORIGIN = "https://sorcery-object-cache.invalid";
106
107export type ObjectQuery =
108  | { depth: number }
109  | { smart: "tree" | "commit-diff" }
110  | { smart: "commit-pagination"; limit: number };
111
112type PathHistoryQuery = { smart: "path-history"; limit: number; path: string[] };
113
114interface ObjectBundle {
115  /** in bundle order, which smart queries make meaningful */
116  objects: Map<string, GitObject>;
117  frontier: string[];
118}
119
120function parseObjectBundle(bytes: Uint8Array): ObjectBundle {
121  if (bytes.length < 6 || new TextDecoder().decode(bytes.subarray(0, 4)) !== "SOBJ" || bytes[4] !== 1) {
122    throw new Error("invalid object bundle");
123  }
124  const objects = new Map<string, GitObject>();
125  const frontier: string[] = [];
126  let position = 6;
127  while (position < bytes.length) {
128    const oidBytes = bytes[position++];
129    if ((oidBytes !== 20 && oidBytes !== 32) || position + oidBytes + 5 > bytes.length) {
130      throw new Error("invalid object bundle frame");
131    }
132    const oid = toHex(bytes.subarray(position, position + oidBytes));
133    position += oidBytes;
134    const kind = bytes[position++];
135    const size = new DataView(bytes.buffer, bytes.byteOffset + position, 4).getUint32(0);
136    position += 4;
137    if (position + size > bytes.length) throw new Error("truncated object bundle");
138    if (kind === FRONTIER_FRAME) {
139      frontier.push(oid);
140    } else {
141      const type = BUNDLED_OBJECT_TYPES[kind];
142      if (!type) throw new Error("invalid bundled object type");
143      objects.set(oid, { type, data: bytes.subarray(position, position + size) });
144    }
145    position += size;
146  }
147  return { objects, frontier };
148}
149
150export class GitRepo {
151  #info?: Promise<GitInfo>;
152  #packs?: Promise<Pack[]>;
153  #objectQueryEndpoint = true;
154  #storage: Promise<Cache | null>;
155
156  constructor(readonly fetch: Fetcher, cacheName?: string) {
157    this.#storage = cacheName && "caches" in globalThis
158      ? caches.open(cacheName).catch(() => null)
159      : Promise.resolve(null);
160  }
161
162  info(): Promise<GitInfo> {
163    if (!this.#info) {
164      this.#info = this.#cachedFetch("gitinfo.json").then(res => {
165        if (!res) throw new Error("missing gitinfo.json");
166        const { value, errors } = validateGitInfo(JSON.parse(new TextDecoder().decode(res.bytes)));
167        if (errors) throw new Error(`bad gitinfo.json: ${errors.map(e => `${e.path} ${e.msg}`).join(", ")}`);
168        return value;
169      });
170      this.#info.catch(() => (this.#info = undefined));
171    }
172    return this.#info;
173  }
174
175  async object(oid: string): Promise<GitObject> {
176    const cached = await this.#cachedObject(oid);
177    if (cached) return cached;
178
179    const bundled = (await this.#query([oid], { depth: 0 }))?.objects.get(oid);
180    if (bundled) return bundled;
181
182    const packs = await this.#allPacks();
183    const indexes = await Promise.all(packs.map(pack => pack.index()));
184    let object: GitObject | null = null;
185    for (let i = 0; i < packs.length; i++) {
186      const offset = await indexes[i].lookup(oid);
187      if (offset !== null) {
188        object = await packs[i].readAt(offset, this);
189        break;
190      }
191    }
192    if (!object) {
193      const loose = await this.#cachedFetch(`.git/objects/${oid.slice(0, 2)}/${oid.slice(2)}`);
194      if (loose) object = parseLoose(await inflate(loose.bytes));
195    }
196    if (!object) throw new Error(`object ${oid} not found`);
197    await this.#storeObjects([[oid, object]]);
198    return object;
199  }
200
201  async prefetch(oids: string[], objectQuery: ObjectQuery): Promise<void> {
202    const roots = [...new Set(oids)];
203    let cached: boolean[];
204    if ("smart" in objectQuery && objectQuery.smart === "commit-pagination") {
205      cached = roots.map(() => false);
206    } else {
207      cached = await Promise.all(roots.map(oid =>
208        "smart" in objectQuery ? this.#hasSmart(objectQuery.smart, oid) : this.#hasObject(oid)
209      ));
210    }
211    const pending = roots.filter((_, i) => !cached[i]);
212    if (pending.length === 0) return;
213
214    const chunks = Array.from(
215      { length: Math.ceil(pending.length / OBJECT_QUERY_CHUNK_SIZE) },
216      (_, i) => pending.slice(i * OBJECT_QUERY_CHUNK_SIZE, (i + 1) * OBJECT_QUERY_CHUNK_SIZE),
217    );
218    const loaded = new Set<string>();
219    let nextChunk = 0;
220    const query = async () => {
221      while (nextChunk < chunks.length) {
222        for (const oid of (await this.#query(chunks[nextChunk++], objectQuery))?.objects.keys() ?? []) loaded.add(oid);
223      }
224    };
225    await Promise.all(Array.from({ length: Math.min(OBJECT_QUERY_CONCURRENCY, chunks.length) }, query));
226
227    const present = await Promise.all(pending.map(oid => loaded.has(oid) || this.#hasObject(oid)));
228    const missing = pending.filter((_, i) => !present[i]);
229    let next = 0;
230    const load = async () => {
231      while (next < missing.length) await this.object(missing[next++]);
232    };
233    await Promise.all(Array.from({ length: Math.min(OBJECT_FALLBACK_CONCURRENCY, missing.length) }, load));
234  }
235
236  /**
237   * a page of `git log -- path` computed by the daemon, resuming from
238   * `frontier` (see `src/history.rs`). the commits' trees along `path` come
239   * bundled too. null when the daemon's query endpoint is unavailable.
240   */
241  async pathHistory(
242    frontier: string[],
243    path: string[],
244    limit: number,
245  ): Promise<{ commits: Commit[]; frontier: string[] } | null> {
246    const bundle = await this.#query(frontier, { smart: "path-history", limit, path });
247    if (!bundle) return null;
248    const commits = [];
249    for (const [oid, object] of bundle.objects) {
250      if (object.type === "commit") commits.push(parseCommit(oid, object.data));
251    }
252    return { commits, frontier: bundle.frontier };
253  }
254
255  /**
256   * null means the server has no object query endpoint (404 or 501), which
257   * is the only reason to read packfiles instead; any other failure is
258   * reported, not silently downgraded
259   */
260  async #query(oids: string[], query: ObjectQuery | PathHistoryQuery): Promise<ObjectBundle | null> {
261    if (!this.#objectQueryEndpoint) return null;
262    const body = new TextEncoder().encode(JSON.stringify({ oids, ...query }));
263    let result;
264    try {
265      result = await this.fetch("obj", undefined, body);
266    } catch (err) {
267      if (!(err instanceof HttpError && err.status === 501)) throw err;
268      result = null;
269    }
270    if (!result) {
271      this.#objectQueryEndpoint = false;
272      return null;
273    }
274    const bundle = parseObjectBundle(result.bytes);
275    await this.#storeObjects(bundle.objects);
276    if ("smart" in query && (query.smart === "tree" || query.smart === "commit-diff")) {
277      await this.#markSmart(query.smart, oids);
278    }
279    return bundle;
280  }
281
282  /** the pack reader is the fallback when the daemon's `/obj` is unavailable, so it loads lazily */
283  #allPacks(): Promise<Pack[]> {
284    if (!this.#packs) {
285      const fetch: Fetcher = (path, range) => this.#cachedFetch(path, range);
286      this.#packs = Promise.all([this.info(), import("./pack.ts")])
287        .then(([info, { Pack }]) => info.packs.map(stem => new Pack(fetch, stem)));
288      this.#packs.catch(() => (this.#packs = undefined));
289    }
290    return this.#packs;
291  }
292
293  async #cachedFetch(
294    path: string,
295    range?: [number, number | null],
296  ): ReturnType<Fetcher> {
297    if (path === "gitinfo.json") {
298      const result = await this.fetch(path, range);
299      if (result) void this.#prunePacks(result.bytes);
300      return result;
301    }
302    const cache = PACK_PATH.test(path) ? await this.#storage : null;
303    if (!cache) return this.fetch(path, range);
304
305    const key = `${PACK_CACHE_ORIGIN}/${path}?range=${range ? `${range[0]}-${range[1] ?? ""}` : "all"}`;
306    const hit = await cache.match(key).catch(() => undefined);
307    if (hit) {
308      return {
309        bytes: new Uint8Array(await hit.arrayBuffer()),
310        total: Number(hit.headers.get("x-sorcery-total")),
311      };
312    }
313    const result = await this.fetch(path, range);
314    if (result) {
315      await cache.put(key, new Response(result.bytes.slice(), {
316        headers: { "x-sorcery-total": String(result.total) },
317      })).catch(() => {});
318    }
319    return result;
320  }
321
322  async #cachedObject(oid: string): Promise<GitObject | null> {
323    const cache = await this.#storage;
324    const hit = await cache?.match(`${OBJECT_CACHE_ORIGIN}/object/${oid}`).catch(() => undefined);
325    if (!hit) return null;
326    const type = hit.headers.get("x-sorcery-object-type");
327    if (type !== "commit" && type !== "tree" && type !== "blob" && type !== "tag") return null;
328    return { type, data: new Uint8Array(await hit.arrayBuffer()) };
329  }
330
331  async #hasObject(oid: string): Promise<boolean> {
332    const cache = await this.#storage;
333    return !!await cache?.match(`${OBJECT_CACHE_ORIGIN}/object/${oid}`).catch(() => undefined);
334  }
335
336  async #storeObjects(objects: Iterable<[string, GitObject]>): Promise<void> {
337    const cache = await this.#storage;
338    if (!cache) return;
339    await Promise.all(Array.from(objects, ([oid, object]) =>
340      cache.put(`${OBJECT_CACHE_ORIGIN}/object/${oid}`, new Response(object.data.slice(), {
341        headers: { "x-sorcery-object-type": object.type },
342      })).catch(() => {})
343    ));
344  }
345
346  async #hasSmart(smart: "tree" | "commit-diff", oid: string): Promise<boolean> {
347    const cache = await this.#storage;
348    return !!await cache?.match(`${OBJECT_CACHE_ORIGIN}/smart-v1/${smart}/${oid}`).catch(() => undefined);
349  }
350
351  async #markSmart(smart: "tree" | "commit-diff", oids: string[]): Promise<void> {
352    const cache = await this.#storage;
353    if (!cache) return;
354    await Promise.all(oids.map(oid =>
355      cache.put(`${OBJECT_CACHE_ORIGIN}/smart-v1/${smart}/${oid}`, new Response()).catch(() => {})
356    ));
357  }
358
359  async #prunePacks(manifest: Uint8Array): Promise<void> {
360    const cache = await this.#storage;
361    if (!cache) return;
362    try {
363      const packs = (JSON.parse(new TextDecoder().decode(manifest)) as { packs?: string[] }).packs;
364      const live = new Set(packs ?? []);
365      for (const request of await cache.keys()) {
366        const stem = new URL(request.url).pathname.match(/(pack-[0-9a-f]+)\.[a-z]+$/)?.[1];
367        if (stem && !live.has(stem)) await cache.delete(request);
368      }
369    } catch {
370      // a malformed manifest fails schema validation upstream; nothing to do here
371    }
372  }
373
374  async commit(oid: string): Promise<Commit> {
375    const object = await this.object(oid);
376    if (object.type === "tag") return this.commit(parseTag(object.data).object);
377    if (object.type !== "commit") throw new Error(`${oid} is a ${object.type}, not a commit`);
378    return parseCommit(oid, object.data);
379  }
380
381  async tree(oid: string): Promise<TreeEntry[]> {
382    const object = await this.object(oid);
383    if (object.type !== "tree") throw new Error(`${oid} is a ${object.type}, not a tree`);
384    return parseTree(object.data, oidBytes(oid).length);
385  }
386
387  async blob(oid: string): Promise<Uint8Array> {
388    const object = await this.object(oid);
389    if (object.type !== "blob") throw new Error(`${oid} is a ${object.type}, not a blob`);
390    return object.data;
391  }
392}