char/sorcery

git clone https://git.t4t.associates/char/sorcery

Charlotte Somexperiment: support sha-256 oids in git repos42f80d82026-09-04

import { assert, assertEquals, assertRejects } from "jsr:@std/assert@1";
import { type Fetcher, GitRepo } from "./repo.ts";
import { diffLines } from "./diff.ts";
import { log, treeDiff } from "./walk.ts";

async function sh(cwd: string, ...args: string[]): Promise<string> {
  const out = await new Deno.Command(args[0], {
    args: args.slice(1),
    cwd,
    env: {
      GIT_AUTHOR_NAME: "t",
      GIT_AUTHOR_EMAIL: "t@t",
      GIT_COMMITTER_NAME: "t",
      GIT_COMMITTER_EMAIL: "t@t",
    },
  }).output();
  if (!out.success) throw new Error(new TextDecoder().decode(out.stderr));
  return new TextDecoder().decode(out.stdout).trim();
}

/** reads from a bare repo dir like nginx would serve `.git/`, incl. ranges */
function fileFetcher(gitDir: string): Fetcher {
  return async (path, range) => {
    if (path === "gitinfo.json") {
      // stand-in for the server-generated manifest
      const packDir = `${gitDir}/objects/pack`;
      const packs: string[] = [];
      try {
        for await (const f of Deno.readDir(packDir)) {
          if (f.name.endsWith(".pack")) packs.push(f.name.slice(0, -5));
        }
      } catch { /* no packs yet */ }
      const head = (await sh(gitDir, "git", "symbolic-ref", "--short", "HEAD")) || null;
      const refs: unknown[] = [];
      for (const line of (await sh(gitDir, "git", "show-ref")).split("\n").filter(Boolean)) {
        const [oid, name] = line.split(" ");
        if (name.startsWith("refs/heads/")) {
          refs.push({ kind: "branch", name: name.slice(11), oid });
        }
      }
      const json = JSON.stringify({ head, refs, packs: packs.sort() });
      return { bytes: new TextEncoder().encode(json), total: json.length };
    }

    const filePath = `${gitDir}/${path.replace(/^\.git\//, "")}`;
    let bytes: Uint8Array;
    try {
      bytes = await Deno.readFile(filePath);
    } catch {
      return null;
    }
    const total = bytes.length;
    if (range) bytes = bytes.subarray(range[0], range[1] ?? undefined);
    return { bytes, total };
  };
}

function objectBundle(specs: Array<[string, number, Uint8Array]>): Uint8Array {
  const bundle = new Uint8Array(6 + specs.reduce((size, [oid, , data]) => size + oid.length / 2 + 6 + data.length, 0));
  bundle.set(new TextEncoder().encode("SOBJ"));
  bundle[4] = 1;
  let position = 6;
  for (const [oid, type, data] of specs) {
    const oidBytes = Uint8Array.from(oid.match(/../g)!, byte => parseInt(byte, 16));
    bundle[position++] = oidBytes.length;
    bundle.set(oidBytes, position);
    position += oidBytes.length;
    bundle[position++] = type;
    new DataView(bundle.buffer).setUint32(position, data.length);
    position += 4;
    bundle.set(data, position);
    position += data.length;
  }
  return bundle;
}

async function makeFixture(objectFormat: "sha1" | "sha256" = "sha1"): Promise<{ dir: string; repo: GitRepo }> {
  const dir = await Deno.makeTempDir({ prefix: "sgw-git-test" });
  const format = objectFormat === "sha256" ? ["--object-format=sha256"] : [];
  await sh(dir, "git", "init", "-q", "-b", "main", ...format, ".");
  await Deno.mkdir(`${dir}/src`);
  await Deno.writeTextFile(`${dir}/README.md`, "# fixture\n");
  await Deno.writeTextFile(`${dir}/src/lib.rs`, "fn one() {}\nfn two() {}\nfn three() {}\n");
  await sh(dir, "git", "add", "-A");
  await sh(dir, "git", "commit", "-q", "-m", "initial commit");
  await Deno.writeTextFile(`${dir}/src/lib.rs`, "fn one() {}\nfn two() { todo!() }\nfn three() {}\nfn four() {}\n");
  await Deno.writeTextFile(`${dir}/NOTES`, "hello\n");
  await sh(dir, "git", "rm", "-q", "README.md");
  await sh(dir, "git", "add", "-A");
  await sh(dir, "git", "commit", "-q", "-m", "second commit");
  return { dir, repo: new GitRepo(fileFetcher(`${dir}/.git`)) };
}

async function assertHistory(repo: GitRepo) {
  const info = await repo.info();
  assertEquals(info.head, "main");
  const main = info.refs.find(r => r.name === "main")!;

  const commits = [];
  for await (const commit of log(repo, [main.oid], 10)) commits.push(commit);
  assertEquals(commits.map(c => c.message.trim()), ["second commit", "initial commit"]);
  assertEquals(commits[0].parents, [commits[1].oid]);
  assertEquals(commits[0].author.name, "t");

  const changes = await treeDiff(repo, commits[1].tree, commits[0].tree);
  assertEquals(
    changes.map(c => `${c.status} ${c.path}`),
    ["added NOTES", "deleted README.md", "modified src/lib.rs"],
  );

  const modified = changes.find(c => c.path === "src/lib.rs")!;
  const oldData = await repo.blob(modified.oldOid!);
  const newData = await repo.blob(modified.newOid!);
  const oldText = new TextDecoder().decode(oldData);
  const newText = new TextDecoder().decode(newData);
  const hunks = diffLines(oldText, newText);
  assertEquals(hunks.length, 1);
  assertEquals(
    hunks[0].lines.map(l => l.sign + l.text),
    [
      " fn one() {}",
      "-fn two() {}",
      "+fn two() { todo!() }",
      " fn three() {}",
      "+fn four() {}",
    ],
  );
}

Deno.test("loose objects", async () => {
  const { repo } = await makeFixture();
  await assertHistory(repo);
});

Deno.test("object queries seed the repository cache", async () => {
  const { dir } = await makeFixture();
  const oid = await sh(dir, "git", "rev-parse", "HEAD");
  const treeOid = await sh(dir, "git", "show", "-s", "--format=%T", "HEAD");
  const commit = await new Deno.Command("git", { args: ["cat-file", "commit", oid], cwd: dir }).output();
  const tree = await new Deno.Command("git", { args: ["cat-file", "tree", treeOid], cwd: dir }).output();
  assert(commit.success && tree.success);

  const bundle = objectBundle([[oid, 1, commit.stdout], [treeOid, 2, tree.stdout]]);

  let rawReads = 0;
  const queries: unknown[] = [];
  const files = fileFetcher(`${dir}/.git`);
  const fetch: Fetcher = (path, range, query) => {
    if (path === "obj" && query) {
      queries.push(JSON.parse(new TextDecoder().decode(query)));
      return Promise.resolve({ bytes: bundle, total: bundle.length });
    }
    if (path.startsWith("obj/") || path.startsWith(".git/objects/")) rawReads++;
    return files(path, range);
  };
  const name = `sorcery-test-${crypto.randomUUID()}`;
  try {
    const repo = new GitRepo(fetch, name);
    await repo.prefetch([oid], { depth: 2 });
    await repo.prefetch([oid], { smart: "tree" });
    const loaded = await repo.commit(oid);
    await repo.tree(loaded.tree);

    const warm = new GitRepo(fetch, name);
    await warm.prefetch([oid], { smart: "tree" });
    await warm.tree((await warm.commit(oid)).tree);
    assertEquals(queries, [
      { oids: [oid], depth: 2 },
      { oids: [oid], smart: "tree" },
    ]);
    assertEquals(rawReads, 0);
  } finally {
    await caches.delete(name);
  }
});

Deno.test("commit logs prefetch one page at a time", async () => {
  const { dir } = await makeFixture();
  for (let i = 3; i <= 12; i++) {
    await Deno.writeTextFile(`${dir}/n`, `${i}\n`);
    await sh(dir, "git", "add", "n");
    await sh(dir, "git", "commit", "-q", "-m", `commit ${i}`);
  }
  const oids = (await sh(dir, "git", "rev-list", "HEAD")).split("\n");
  const commits = new Map<string, Uint8Array>();
  for (const oid of oids) {
    const result = await new Deno.Command("git", { args: ["cat-file", "commit", oid], cwd: dir }).output();
    assert(result.success);
    commits.set(oid, result.stdout);
  }

  const queries: unknown[] = [];
  let rawReads = 0;
  const files = fileFetcher(`${dir}/.git`);
  const fetch: Fetcher = (path, range, query) => {
    if (path === "obj" && query) {
      const body = JSON.parse(new TextDecoder().decode(query)) as {
        oids: string[];
        smart: string;
        limit: number;
      };
      queries.push(body);
      const start = oids.indexOf(body.oids[0]);
      const specs = oids.slice(start, start + body.limit).map(oid => [oid, 1, commits.get(oid)!] as [string, number, Uint8Array]);
      const bundle = objectBundle(specs);
      return Promise.resolve({ bytes: bundle, total: bundle.length });
    }
    if (path.startsWith(".git/objects/")) rawReads++;
    return files(path, range);
  };
  const name = `sorcery-test-${crypto.randomUUID()}`;
  try {
    const loaded = [];
    for await (const commit of log(new GitRepo(fetch, name), oids, 10)) loaded.push(commit.oid);
    assertEquals(loaded, oids);
    assertEquals(queries, [
      { oids: oids.slice(0, 10), smart: "commit-pagination", limit: 10 },
      { oids: oids.slice(10), smart: "commit-pagination", limit: 10 },
    ]);
    assertEquals(rawReads, 0);
  } finally {
    await caches.delete(name);
  }
});

Deno.test("packed objects (with deltas)", async () => {
  const { dir } = await makeFixture();
  // window/depth defaults keep our similar blobs as deltas; prune loose ones
  await sh(dir, "git", "-c", "gc.pruneExpire=now", "gc", "-q", "--aggressive", "--prune=now");
  const repo = new GitRepo(fileFetcher(`${dir}/.git`));
  const info = await repo.info();
  assert(info.packs.length > 0, "expected a pack after gc");
  await assertHistory(repo);
});

Deno.test("SHA-256 loose and packed objects", async () => {
  const { dir, repo } = await makeFixture("sha256");
  const info = await repo.info();
  assert(info.refs.every(ref => ref.oid.length === 64));
  await assertHistory(repo);

  await sh(dir, "git", "-c", "gc.pruneExpire=now", "gc", "-q", "--aggressive", "--prune=now");
  const packed = new GitRepo(fileFetcher(`${dir}/.git`));
  const packs = (await packed.info()).packs;
  assert(packs.length > 0 && packs.every(stem => stem.length === 69));
  await assertHistory(packed);
});

Deno.test("pack ranges persist in the cache; stale packs are pruned", async () => {
  const { dir } = await makeFixture();
  await sh(dir, "git", "-c", "gc.pruneExpire=now", "gc", "-q", "--aggressive", "--prune=now");
  const files = fileFetcher(`${dir}/.git`);
  let packFetches = 0;
  const counting: Fetcher = (path, range) => {
    if (path.includes("/pack/")) packFetches++;
    return files(path, range);
  };
  const name = `sorcery-test-${crypto.randomUUID()}`;
  try {
    const stale = "https://sorcery-pack-cache.invalid/x/pack-dead.idx?range=all";
    await (await caches.open(name)).put(stale, new Response("junk"));

    await assertHistory(new GitRepo(counting, name));
    assert(packFetches > 0, "expected cold-cache pack fetches");

    packFetches = 0;
    await assertHistory(new GitRepo(counting, name));
    assertEquals(packFetches, 0, "warm cache should serve all pack ranges");

    // pruning is fire-and-forget off the gitinfo fetch, so poll briefly
    const cache = await caches.open(name);
    for (let i = 0; i < 20 && (await cache.match(stale)); i++) {
      await new Promise(resolve => setTimeout(resolve, 50));
    }
    assertEquals(await cache.match(stale), undefined, "stale pack entry should be pruned");
  } finally {
    await caches.delete(name);
  }
});

Deno.test("failed fetches are retried, not cached", async () => {
  const { dir } = await makeFixture();
  await sh(dir, "git", "-c", "gc.pruneExpire=now", "gc", "-q", "--aggressive", "--prune=now");
  const files = fileFetcher(`${dir}/.git`);
  let failures = 1;
  const fetch: Fetcher = (path, range) => {
    if (path.endsWith(".pack") && failures-- > 0) return Promise.reject(new Error("aborted"));
    return files(path, range);
  };
  const repo = new GitRepo(fetch);
  const info = await repo.info();
  const main = info.refs.find(r => r.name === "main")!;
  await assertRejects(() => repo.commit(main.oid), Error, "aborted");
  assertEquals((await repo.commit(main.oid)).message.trim(), "second commit");
});

Deno.test("myers diff edge cases", () => {
  assertEquals(diffLines("", ""), []);
  assertEquals(diffLines("a\n", "a\n"), []);
  const addOnly = diffLines("", "a\nb\n");
  assertEquals(addOnly[0].lines.map(l => l.sign + l.text), ["+a", "+b"]);
  const delOnly = diffLines("a\nb\n", "");
  assertEquals(delOnly[0].lines.map(l => l.sign + l.text), ["-a", "-b"]);
  // no trailing newline handling
  const noEol = diffLines("a", "b");
  assertEquals(noEol[0].lines.map(l => l.sign + l.text), ["-a", "+b"]);
});