import { assert, assertEquals, assertMatch, assertStringIncludes } from "@std/assert"; import { Type } from "@earendil-works/pi-ai"; import { renderCatalog } from "./codemode.ts"; import { lastToolResult, script, sectionsSeen, setup, turn } from "./fixture.ts"; Deno.test("scripts call tools in their own extension's context and see its emits", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const seen = await turn(harness, tid, [ script(`return [await tools.pwd({}), await tools.cd({ path: "/x" }), await tools.pwd({})]`), ]); assertEquals(lastToolResult(seen[1]).text, `Script completed\n["/","now in /x","/x"]`); assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/x" }); }); Deno.test("the catalog declares only the selected extensions' codemode tools", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const [seen] = await turn(harness, tid, []); const catalog = sectionsSeen(seen).at(-1)?.codemode ?? ""; assertStringIncludes(catalog, "print the working directory"); assertStringIncludes(catalog, "cd: string"); assert( !catalog.includes("path:"), "direct tools' arguments are supplied by their native schemas", ); assertStringIncludes(catalog, "text: string"); assert(!catalog.includes("change directory"), "direct tools are described only once"); assert(!catalog.includes("shout"), "model-only tools are not declared to scripts"); assert(!catalog.includes("hold"), "unselected extensions are not declared"); }); Deno.test("the catalog retains tools and return types across extensions", () => { const catalog = renderCatalog([ { name: "run", extension: "shell", description: "execute a command", parameters: Type.Object({ command: Type.String() }), returns: Type.Object({ exitCode: Type.Integer(), output: Type.String(), fullOutputPath: Type.Optional(Type.String()), }), direct: true, }, { name: "find", extension: "search", description: "search indexed documents", parameters: Type.Object({ query: Type.String() }), returns: Type.Array(Type.String()), }, ])!; assertStringIncludes(catalog, "run:"); assertStringIncludes(catalog, "exitCode: number"); assertStringIncludes(catalog, "output: string"); assertStringIncludes(catalog, "fullOutputPath?: string"); assertStringIncludes(catalog, "find(args:"); assertStringIncludes(catalog, "search indexed documents"); assertStringIncludes(catalog, "query: string"); assertStringIncludes(catalog, "Promise>"); }); Deno.test("tools of unselected extensions are unreachable", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { gate: {} }, model: harness.model }); const seen = await turn(harness, tid, [script(`return await tools.pwd({})`)]); const result = lastToolResult(seen[1]); assert(result.isError); assertMatch(result.text, /^Script failed/); }); Deno.test("nested arguments are validated like the model's, before the tool runs", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const seen = await turn(harness, tid, [ script(`try { await tools.cd({ nowhere: 1 }) } catch (error) { return error.message }`), ]); assertStringIncludes(lastToolResult(seen[1]).text, "path"); assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/" }); }); Deno.test("nested calls are recorded for display, never for the model", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const seen = await turn(harness, tid, [ script(`await tools.pwd({}); try { await tools.cd({}) } catch {} ; return "ok"`), ]); const { text, details } = lastToolResult(seen[1]); assertEquals(text, "Script completed\nok"); const calls = (details as { calls: Record[] }).calls; assertEquals( calls.map(({ name, extension, status, value }) => ({ name, extension, status, value })), [ { name: "pwd", extension: "dir", status: "ok", value: "/" }, { name: "cd", extension: "dir", status: "error", value: undefined }, ], ); assert(typeof calls[1].error === "string"); }); Deno.test("nested running output streams into the call's record", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const seen = await turn(harness, tid, [script(`return await tools.say({ text: "one two" })`)]); const { text, details } = lastToolResult(seen[1]); assertEquals(text, "Script completed\n7"); assertEquals((details as { calls: { output?: string }[] }).calls[0].output, "one\ntwo\n"); }); Deno.test("the store keeps successful scripts' writes across calls", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: {}, model: harness.model }); const seen = await turn(harness, tid, [ script(`store("n", 1)`), script(`store("n", load("n") + 1); throw new Error("no")`), script(`return load("n")`), ]); assert(lastToolResult(seen[2]).isError); assertEquals(lastToolResult(seen[3]).text, "Script completed\n1"); assertEquals(await harness.berry.state(tid, "berry"), { n: 1 }); }); Deno.test("failures report the calls already made, which stand", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: { dir: {} }, model: harness.model }); const seen = await turn(harness, tid, [ script(`await tools.cd({ path: "/y" }); throw new Error("boom")`), ]); const { text, isError } = lastToolResult(seen[1]); assert(isError); assertStringIncludes(text, "Error: boom"); assertStringIncludes(text, "cd (ok)"); assertEquals(await harness.berry.state(tid, "dir"), { cwd: "/y" }); }); Deno.test("output over budget keeps its start and end and spills the whole of it", async () => { await using harness = await setup(); const tid = await harness.berry.create({ extensions: {}, model: harness.model }); const seen = await turn(harness, tid, [ script(`// @options: {"max_output_tokens": 10}\ntext("a".repeat(100) + "b".repeat(100))`), ]); const { text } = lastToolResult(seen[1]); assertMatch(text, /^Script completed\na{20}…\d+ tokens truncated…b{20}/); const path = text.match(/\[Full output: (.+)\]/)![1]; assertEquals(await Deno.readTextFile(path), "a".repeat(100) + "b".repeat(100)); await Deno.remove(path); });