char-slop/berry

git clone https://git.t4t.associates/char-slop/berry

Charlotte Somcodemode: flatten the generated tool catalogf4dfe652026-10-17

import { assert, assertEquals, assertMatch, assertStringIncludes } from "@std/assert";
import { Type } from "@earendil-works/pi-ai";
import { renderCatalog } from "./codemode.ts";
import { lastToolResult, script, sectionsSeen, setup, turn } from "./fixture.ts";

Deno.test("scripts call tools in their own extension's context and see its emits", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`return [await tools.pwd({}), await tools.cd({ path: "/x" }), await tools.pwd({})]`),
  ]);
  assertEquals(lastToolResult(seen[1]).text, `Script completed\n["/","now in /x","/x"]`);
  assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/x" });
});

Deno.test("the catalog declares only the selected extensions' codemode tools", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const [seen] = await turn(harness, tid, []);
  const catalog = sectionsSeen(seen).at(-1)?.codemode ?? "";
  assertStringIncludes(catalog, "print the working directory");
  assertStringIncludes(catalog, "cd: string");
  assert(
    !catalog.includes("path:"),
    "direct tools' arguments are supplied by their native schemas",
  );
  assertStringIncludes(catalog, "text: string");
  assert(!catalog.includes("change directory"), "direct tools are described only once");
  assert(!catalog.includes("shout"), "model-only tools are not declared to scripts");
  assert(!catalog.includes("hold"), "unselected extensions are not declared");
});

Deno.test("the catalog retains tools and return types across extensions", () => {
  const catalog = renderCatalog([
    {
      name: "run",
      extension: "shell",
      description: "execute a command",
      parameters: Type.Object({ command: Type.String() }),
      returns: Type.Object({
        exitCode: Type.Integer(),
        output: Type.String(),
        fullOutputPath: Type.Optional(Type.String()),
      }),
      direct: true,
    },
    {
      name: "find",
      extension: "search",
      description: "search indexed documents",
      parameters: Type.Object({ query: Type.String() }),
      returns: Type.Array(Type.String()),
    },
  ])!;
  assertStringIncludes(catalog, "run:");
  assertStringIncludes(catalog, "exitCode: number");
  assertStringIncludes(catalog, "output: string");
  assertStringIncludes(catalog, "fullOutputPath?: string");
  assertStringIncludes(catalog, "find(args:");
  assertStringIncludes(catalog, "search indexed documents");
  assertStringIncludes(catalog, "query: string");
  assertStringIncludes(catalog, "Promise<Array<string>>");
});

Deno.test("tools of unselected extensions are unreachable", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { gate: {} }, model: harness.model });
  const seen = await turn(harness, tid, [script(`return await tools.pwd({})`)]);
  const result = lastToolResult(seen[1]);
  assert(result.isError);
  assertMatch(result.text, /^Script failed/);
});

Deno.test("nested arguments are validated like the model's, before the tool runs", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`try { await tools.cd({ nowhere: 1 }) } catch (error) { return error.message }`),
  ]);
  assertStringIncludes(lastToolResult(seen[1]).text, "path");
  assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/" });
});

Deno.test("nested calls are recorded for display, never for the model", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`await tools.pwd({}); try { await tools.cd({}) } catch {} ; return "ok"`),
  ]);
  const { text, details } = lastToolResult(seen[1]);
  assertEquals(text, "Script completed\nok");
  const calls = (details as { calls: Record<string, unknown>[] }).calls;
  assertEquals(
    calls.map(({ name, extension, status, value }) => ({ name, extension, status, value })),
    [
      { name: "pwd", extension: "dir", status: "ok", value: "/" },
      { name: "cd", extension: "dir", status: "error", value: undefined },
    ],
  );
  assert(typeof calls[1].error === "string");
});

Deno.test("nested running output streams into the call's record", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const seen = await turn(harness, tid, [script(`return await tools.say({ text: "one two" })`)]);
  const { text, details } = lastToolResult(seen[1]);
  assertEquals(text, "Script completed\n7");
  assertEquals((details as { calls: { output?: string }[] }).calls[0].output, "one\ntwo\n");
});

Deno.test("the store keeps successful scripts' writes across calls", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: {}, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`store("n", 1)`),
    script(`store("n", load("n") + 1); throw new Error("no")`),
    script(`return load("n")`),
  ]);
  assert(lastToolResult(seen[2]).isError);
  assertEquals(lastToolResult(seen[3]).text, "Script completed\n1");
  assertEquals(await harness.berry.state(tid, "berry"), { n: 1 });
});

Deno.test("failures report the calls already made, which stand", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`await tools.cd({ path: "/y" }); throw new Error("boom")`),
  ]);
  const { text, isError } = lastToolResult(seen[1]);
  assert(isError);
  assertStringIncludes(text, "Error: boom");
  assertStringIncludes(text, "cd (ok)");
  assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/y" });
});

Deno.test("output over budget keeps its start and end and spills the whole of it", async () => {
  await using harness = await setup();
  const tid = await harness.berry.create({ extensions: {}, model: harness.model });
  const seen = await turn(harness, tid, [
    script(`// @options: {"max_output_tokens": 10}\ntext("a".repeat(100) + "b".repeat(100))`),
  ]);
  const { text } = lastToolResult(seen[1]);
  assertMatch(text, /^Script completed\na{20}…\d+ tokens truncated…b{20}/);
  const path = text.match(/\[Full output: (.+)\]/)![1];
  assertEquals(await Deno.readTextFile(path), "a".repeat(100) + "b".repeat(100));
  await Deno.remove(path);
});