char-slop/berry

multipurpose self-modifiable agentic harness

git clone https://git.t4t.associates/char-slop/berry

Charlotte Sommisc: add /context-cap command for forcing short context on 1M modelsecb3f3b

main
branchesmainall refs →
3.6 KiB107 linesraw
1import { assertEquals } from "@std/assert";
2import { type CompactionCap, defineExtension } from "@berry/api";
3import { envApiKeyAuth, type Model, type Provider } from "@earendil-works/pi-ai";
4import { setup } from "./fixture.ts";
5
6const model = (id: string, contextWindow: number): Model<"openai-completions"> => ({
7  id,
8  name: id,
9  api: "openai-completions",
10  provider: "sized",
11  baseUrl: "https://example.invalid",
12  input: ["text"],
13  reasoning: false,
14  contextWindow,
15  maxTokens: 4096,
16  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
17});
18
19const provider: Provider = {
20  id: "sized",
21  name: "sized",
22  auth: { apiKey: envApiKeyAuth("sized key", ["SIZED_KEY"]) },
23  getModels: () => [model("big", 1_048_576), model("small", 20_000)],
24  stream: () => {
25    throw new Error("unused");
26  },
27  streamSimple: () => {
28    throw new Error("unused");
29  },
30};
31
32const sized = () =>
33  defineExtension({
34    name: "sized",
35    initial: () => null,
36    setup(ext) {
37      ext.provider(provider);
38    },
39  });
40
41const cap = (name: string, value: CompactionCap | undefined) =>
42  defineExtension({
43    name,
44    initial: () => null,
45    setup(ext) {
46      ext.compactionCap(() => value);
47    },
48  });
49
50Deno.test("generation sees windows capped at the threshold plus reserve, smaller ones alone", async () => {
51  await using harness = await setup();
52  const { berry, faux, models } = harness;
53  await berry.install(sized());
54  await berry.install(cap("cap", { maxContextTokens: 262_144, backgroundTokens: 32_768 }));
55  const seen = berry.effectiveModels;
56  assertEquals(seen.getModel("sized", "big")?.contextWindow, 262_144 + 16_384);
57  assertEquals(seen.getModel("sized", "small")?.contextWindow, 20_000);
58  assertEquals(seen.getModel("faux", faux.getModel().id)?.contextWindow, 128_000);
59  // the stored catalog keeps its own windows
60  assertEquals(models.getModel("sized", "big")?.contextWindow, 1_048_576);
61  assertEquals(
62    berry.compactionCap(),
63    { maxContextTokens: 262_144, backgroundTokens: 32_768 },
64  );
65});
66
67Deno.test("the tightest cap wins and lapses on uninstall", async () => {
68  await using harness = await setup();
69  const { berry } = harness;
70  const seen = berry.effectiveModels;
71  await berry.install(sized());
72  await berry.install(cap("loose", { maxContextTokens: 500_000, backgroundTokens: 1_000 }));
73  await berry.install(cap("tight", { maxContextTokens: 100_000, backgroundTokens: 2_000 }));
74  assertEquals(
75    berry.compactionCap(),
76    { maxContextTokens: 100_000, backgroundTokens: 2_000 },
77  );
78  assertEquals(seen.getModel("sized", "big")?.contextWindow, 100_000 + 16_384);
79  berry.uninstall("tight");
80  assertEquals(
81    berry.compactionCap(),
82    { maxContextTokens: 500_000, backgroundTokens: 1_000 },
83  );
84  assertEquals(seen.getModel("sized", "big")?.contextWindow, 500_000 + 16_384);
85  berry.uninstall("loose");
86  assertEquals(berry.compactionCap(), undefined);
87  assertEquals(seen.getModel("sized", "big")?.contextWindow, 1_048_576);
88});
89
90Deno.test("invalid and failing caps read as no cap", async () => {
91  await using harness = await setup();
92  const { berry } = harness;
93  await berry.install(sized());
94  await berry.install(cap("zero", { maxContextTokens: 0, backgroundTokens: 0 }));
95  await berry.install(cap("negative", { maxContextTokens: 100_000, backgroundTokens: -1 }));
96  await berry.install(defineExtension({
97    name: "failing",
98    initial: () => null,
99    setup(ext) {
100      ext.compactionCap(() => {
101        throw new Error("no cap here");
102      });
103    },
104  }));
105  assertEquals(berry.compactionCap(), undefined);
106  assertEquals(berry.effectiveModels.getModel("sized", "big")?.contextWindow, 1_048_576);
107});