char-slop/berry

git clone https://git.t4t.associates/char-slop/berry

Charlotte Somperf: abort subprocesses instantly on restartba8ed3c2026-10-22

// Extensions come from configured directories: loaded at startup, and again whenever their files change.

import { existsSync, mkdirSync } from "node:fs";
import { copy } from "@std/fs/copy";
import { fromFileUrl, join, relative, resolve, toFileUrl } from "@std/path";
import type { TObject } from "typebox";
import type { ExtensionStartup, FrontendSource } from "@berry/api/extension-startup.ts";
import type { Berry, ExtensionStatus } from "./berry.ts";
import { denoExecutable } from "./deno.ts";
import { published } from "./pubsub.ts";
import type { ExtensionWeb, WebEntry } from "./web.ts";

/** An extension as the ui sees it: how its last load went, whether a version of it runs, and its frontend. */
export type ExtensionEntry = ExtensionStatus & {
  readonly running: boolean;
  /** The source directory, or null for a builtin or an unresolved name collision. */
  readonly directory: string | null;
  /** The harness's own: always there, nothing on disk to reload. */
  readonly builtin: boolean;
  readonly alwaysOn: boolean;
  /** Whether a new conversation starts with it, until one is chosen from before. */
  readonly defaultEnabled: boolean;
  /** Where its configuration sorts among the others': lower first. */
  readonly sortOrder: number | null;
  readonly web: WebEntry | null;
  /** Names it went by before, which old history still calls it. */
  readonly formerly: readonly string[];
  /** What a new conversation with it is asked, and what an existing one may change, as a json schema. */
  readonly configuration: TObject | null;
  /** Its global editable prose default, when it registers any. */
  readonly promptProse: { default: string } | null;
  readonly startup: ExtensionStartup | null;
};

const SETTLE_MS = 300;

/**
 * What `deno check` says of `files`, or `null` when they pass; a signal kills the check and rejects with its reason.
 */
export async function denoCheck(
  root: string,
  files: readonly string[],
  signal?: AbortSignal,
): Promise<string | null> {
  const { success, stderr } = await new Deno.Command(denoExecutable(), {
    args: [
      "check",
      "--quiet",
      "--config",
      fromFileUrl(new URL("../deno.json", import.meta.url)),
      ...files,
    ],
    cwd: root,
    env: { NO_COLOR: "1" },
    stdout: "null",
    stderr: "piped",
    signal,
  }).output();
  // a killed check exits by signal and writes nothing, which is not a verdict on the code
  signal?.throwIfAborted();
  return success ? null : new TextDecoder().decode(stderr).trim();
}

/** The extensions as the ui lists them: by `sortOrder`, lower first, and those without one after them by name. */
function ranked(entries: readonly ExtensionEntry[]): readonly ExtensionEntry[] {
  const at = (entry: ExtensionEntry) => entry.sortOrder ?? Number.MAX_SAFE_INTEGER;
  return [...entries].sort((a, b) => at(a) - at(b) || a.name.localeCompare(b.name));
}

export function extensionLoader(
  { berry, web, root, directories = [join(root, "extensions")] }: {
    berry: Berry;
    web: ExtensionWeb;
    root: string;
    directories?: readonly string[];
  },
) {
  const roots = [...new Set(directories.map((directory) => resolve(directory)))];
  const entries = published<readonly ExtensionEntry[]>([]);
  const builtins: readonly ExtensionEntry[] = berry.builtins().map((extension) => ({
    name: extension.name,
    description: extension.description ?? null,
    status: "loaded",
    error: null,
    running: true,
    directory: null,
    builtin: true,
    alwaysOn: extension.alwaysOn ?? false,
    defaultEnabled: extension.defaultEnabled ?? false,
    sortOrder: extension.sortOrder ?? null,
    web: null,
    formerly: extension.formerly ?? [],
    configuration: berry.configurationParameters(extension.name),
    promptProse: berry.promptProseDefault(extension.name) === null
      ? null
      : { default: berry.promptProseDefault(extension.name)! },
    startup: null,
  }));
  /** What the ui lists: the extensions by their configuration order. */
  const listed = published<readonly ExtensionEntry[]>(ranked(builtins));
  const copies = new Map<string, string>();
  const queues = new Map<string, Promise<unknown>>();
  // a restart loads every extension again, so a reload still being checked is only holding the process up
  const checks = new AbortController();
  let watcher: Deno.FsWatcher | undefined;
  let watching: Promise<void> | undefined;

  function sources() {
    const found = new Map<string, string[]>();
    for (const directory of roots) {
      if (!existsSync(directory)) continue;
      for (const entry of Deno.readDirSync(directory)) {
        if (!entry.isDirectory || entry.name.startsWith(".")) continue;
        const paths = found.get(entry.name) ?? [];
        paths.push(join(directory, entry.name));
        found.set(entry.name, paths);
      }
    }
    return found;
  }

  function record(entry: ExtensionEntry) {
    const rest = entries.value.filter((e) => e.name !== entry.name);
    entries.set([...rest, entry].sort((a, b) => a.name.localeCompare(b.name)));
    listed.set(ranked([...builtins, ...entries.value]));
    return entry;
  }

  /**
   * Load an extension's files. A reload checks them first, and imports a fresh copy of the directory, so that its
   * relative imports load anew too; deno would otherwise reuse what it has for their unchanged urls.
   */
  async function load(name: string, reload: boolean): Promise<ExtensionStatus> {
    const builtin = builtins.find((extension) => extension.name === name);
    if (builtin) {
      return {
        name,
        description: builtin.description,
        status: "failed",
        error: `${name} is the harness's own; restart berry to reload it`,
      };
    }
    const started = performance.now();
    const phases: ExtensionStartup["phases"][number][] = [];
    let frontendSource: FrontendSource | undefined;
    const candidates = sources().get(name) ?? [];
    const previous = entries.value.find((e) => e.name === name);

    async function trace<T>(
      phase: ExtensionStartup["phases"][number]["name"],
      run: () => Promise<T>,
    ) {
      if (reload) return await run();
      const started = performance.now();
      try {
        return await run();
      } finally {
        const durationMs = performance.now() - started;
        phases.push({
          name: phase,
          durationMs,
          ...(phase === "frontend" && frontendSource ? { source: frontendSource } : {}),
        });
      }
    }

    function finish(entry: Omit<ExtensionEntry, "startup">) {
      return record({
        ...entry,
        startup: reload ? previous?.startup ?? null : {
          status: entry.status,
          totalMs: performance.now() - started,
          phases,
        },
      });
    }
    const source = candidates.length === 1 ? candidates[0] : undefined;
    let copied: string | undefined;
    // its directory gone: so is the extension
    if (candidates.length === 0) {
      berry.uninstall(name);
      entries.set(entries.value.filter((e) => e.name !== name));
      listed.set(ranked([...builtins, ...entries.value]));
      return { name, description: null, status: "failed", error: "removed" };
    }
    try {
      if (source === undefined) {
        throw new Error(`duplicate extension ${name}:\n${candidates.join("\n")}`);
      }
      const frontend = join(source, "web.tsx");
      if (!existsSync(join(source, "mod.ts"))) throw new Error(`${source} has no mod.ts`);
      const files = [join(source, "mod.ts"), ...(existsSync(frontend) ? [frontend] : [])];
      if (reload) {
        const failures = await denoCheck(
          root,
          files.map((file) => relative(root, file)),
          checks.signal,
        );
        if (failures !== null) throw new Error(failures);
        copied = await Deno.makeTempDir({ prefix: `berry-${name}-` });
        await copy(source, copied, { overwrite: true });
      }
      const module = await trace(
        "import",
        () => import(toFileUrl(join(copied ?? source, "mod.ts")).href),
      );
      const extension = module.default;
      // its frontend is served, and its state kept, under one name
      if (extension?.name !== name) {
        throw new Error(`${source} names itself ${extension?.name}`);
      }
      // the frontend first: a build that fails leaves both halves as they were
      const built = existsSync(frontend)
        ? await trace("frontend", () =>
          web.build(name, frontend, (source) => {
            frontendSource = source;
          }))
        : null;
      await trace("install", () => berry.install(extension));
      if (copied !== undefined) {
        const old = copies.get(name);
        copies.set(name, copied);
        if (old) await Deno.remove(old, { recursive: true }).catch(() => {});
      }
      return finish({
        name,
        description: extension.description ?? null,
        status: "loaded",
        error: null,
        running: true,
        directory: source,
        builtin: false,
        alwaysOn: extension.alwaysOn ?? false,
        defaultEnabled: extension.defaultEnabled ?? false,
        sortOrder: extension.sortOrder ?? null,
        web: built,
        formerly: extension.formerly ?? [],
        configuration: berry.configurationParameters(name),
        promptProse: berry.promptProseDefault(name) === null
          ? null
          : { default: berry.promptProseDefault(name)! },
      });
    } catch (error) {
      if (copied !== undefined) await Deno.remove(copied, { recursive: true }).catch(() => {});
      // a cancelled reload says nothing about the extension, which the restart loads anew
      if (checks.signal.aborted) {
        return previous ?? {
          name,
          description: null,
          status: "failed",
          error: `${name} did not finish loading`,
        };
      }
      const message = error instanceof Error ? error.message : String(error);
      console.error(`[berry] ${name} did not ${reload ? "reload" : "load"}:\n${message}`);
      return finish({
        name,
        description: previous?.description ?? null,
        status: "failed",
        error: message,
        running: previous?.running ?? false,
        directory: previous?.running ? previous.directory : source ?? null,
        builtin: false,
        alwaysOn: previous?.alwaysOn ?? false,
        defaultEnabled: previous?.defaultEnabled ?? false,
        sortOrder: previous?.sortOrder ?? null,
        web: previous?.web ?? null,
        formerly: previous?.formerly ?? [],
        configuration: previous?.configuration ?? null,
        promptProse: previous?.promptProse ?? null,
      });
    }
  }

  /** One load of an extension at a time; a change while one runs loads again after it. */
  function queue(name: string, reload: boolean): Promise<ExtensionStatus> {
    const next = (queues.get(name) ?? Promise.resolve()).then(() => load(name, reload));
    queues.set(name, next.catch(() => {}));
    return next;
  }

  return {
    directories: roots as readonly string[],

    files: () =>
      [...sources().values()].flat().flatMap((directory) => [
        join(directory, "mod.ts"),
        ...(existsSync(join(directory, "web.tsx")) ? [join(directory, "web.tsx")] : []),
      ]),

    async loadAll() {
      for (const name of sources().keys()) await queue(name, false);
    },

    reload: (name: string) => queue(name, true),

    entries: (): readonly ExtensionEntry[] => entries.value,

    feed: () => listed.feed(),

    /** Reload each extension whose files change, once they settle. */
    watch() {
      if (watcher || roots.length === 0) return;
      for (const directory of roots) mkdirSync(directory, { recursive: true, mode: 0o700 });
      watcher = Deno.watchFs(roots, { recursive: true });
      const timers = new Map<string, ReturnType<typeof setTimeout>>();
      watching = (async () => {
        for await (const event of watcher!) {
          const affected = new Set<string>();
          for (const path of event.paths) {
            for (const directory of roots) {
              const name = relative(directory, path).split(/[/\\]/)[0];
              // dot directories, and anything outside, which relative() starts with ".."
              if (name.startsWith(".")) continue;
              if (name) affected.add(name);
              else {
                for (const name of sources().keys()) affected.add(name);
                for (const entry of entries.value) affected.add(entry.name);
              }
            }
          }
          for (const name of affected) {
            clearTimeout(timers.get(name));
            timers.set(name, setTimeout(() => void queue(name, true), SETTLE_MS));
          }
        }
        for (const timer of timers.values()) clearTimeout(timer);
      })();
    },

    async close() {
      checks.abort(new Error("berry is stopping"));
      watcher?.close();
      await watching;
      await Promise.all(queues.values());
      for (const copied of copies.values()) {
        await Deno.remove(copied, { recursive: true }).catch(() => {});
      }
    },
  };
}

export type Loader = ReturnType<typeof extensionLoader>;