char-slop/berry

multipurpose self-modifiable agentic harness

git clone https://git.t4t.associates/char-slop/berry

Charlotte Somtests: cut test sprawl1c0a335

main
branchesmainall refs →
3.9 KiB91 linesraw
1import { assert, assertEquals, assertStringIncludes } from "@std/assert";
2import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context";
3import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
4import { fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai/providers/faux";
5import { createModels } from "@earendil-works/pi-ai/models";
6import { MemoryStorage } from "@earendil-works/pi-durable";
7import { openBerry } from "./berry.ts";
8import { setup, turn } from "./fixture.ts";
9
10Deno.test("a connection body read failure retries generation", async () => {
11  await using harness = await setup();
12  const { berry, faux, model } = harness;
13  const tid = await berry.create({ extensions: {}, model });
14  faux.setResponses([
15    fauxAssistantMessage("", {
16      stopReason: "error",
17      errorMessage: "error reading a body from connection",
18    }),
19    fauxAssistantMessage("recovered"),
20  ]);
21  const conversation = (await berry.conversation(tid))!;
22  const settled = await (await conversation.submit({ type: "input", content: "go" }, context))
23    .wait(context);
24  await conversation.waitForIdle(context);
25  assertEquals(settled.status, "done");
26  assertEquals(faux.state.callCount, 2);
27  const stored = await conversation.context(context);
28  assertEquals(stored.messages.at(-1)?.content, [{ type: "text", text: "recovered" }]);
29});
30
31for (const delivery of ["a setup exception", "a result-only stream"] as const) {
32  Deno.test(`generation retries EOF delivered through ${delivery}`, async () => {
33    const faux = fauxProvider();
34    const models = createModels();
35    models.setProvider(faux.provider);
36    const original = models.streamSimple.bind(models);
37    let attempts = 0;
38    models.streamSimple = (...args) => {
39      if (++attempts === 1) {
40        if (delivery === "a setup exception") throw new Error("Unexpected EOF");
41        const stream = createAssistantMessageEventStream();
42        stream.end(
43          fauxAssistantMessage("", { stopReason: "error", errorMessage: "Unexpected EOF" }),
44        );
45        return stream;
46      }
47      return original(...args);
48    };
49    const berry = await openBerry(new MemoryStorage(), models);
50    try {
51      const model = { provider: "faux", modelId: faux.getModel().id };
52      await berry.enableModels([model]);
53      const tid = await berry.create({ extensions: {}, model });
54      faux.setResponses([fauxAssistantMessage("recovered")]);
55      const conversation = (await berry.conversation(tid))!;
56      const settled = await (await conversation.submit({ type: "input", content: "go" }, context))
57        .wait(context);
58      assertEquals(settled.status, "done");
59      assertEquals(attempts, 2);
60    } finally {
61      await berry.close();
62    }
63  });
64}
65
66Deno.test("compaction also retries EOF and places the recovered summary", async () => {
67  await using harness = await setup();
68  const { berry, faux, model } = harness;
69  const tid = await berry.create({ extensions: {}, model });
70  await turn(harness, tid, []);
71  const conversation = (await berry.conversation(tid))!;
72  faux.setResponses([fauxAssistantMessage("done")]);
73  await (await conversation.submit({ type: "input", content: "x".repeat(100_000) }, context))
74    .wait(context);
75  await conversation.waitForIdle(context);
76  faux.setResponses([
77    fauxAssistantMessage("", { stopReason: "error", errorMessage: "Unexpected EOF" }),
78    fauxAssistantMessage("Recovered summary"),
79  ]);
80  const task = await conversation.compact(undefined, context);
81  const { outcome } = (await berry.harness.waitForTask(task, context)).state;
82  assert(outcome.status === "completed");
83  await conversation.waitForIdle(context);
84  assertEquals(faux.state.callCount, 4);
85  const entries =
86    (await conversation.entries({ order: "ascending" }, 100, undefined, context)).items;
87  const summary = entries.find((entry) => entry.kind === "pi.compaction");
88  assert(summary);
89  const text = JSON.stringify(summary.model);
90  assertStringIncludes(text, "Recovered summary");
91});