char-slop/berry

multipurpose self-modifiable agentic harness

git clone https://git.t4t.associates/char-slop/berry

Charlotte Somadd skills extension for loading skillsa4e772a

main
branchesmainall refs →
10.3 KiB269 linesraw
1import { createHash } from "node:crypto";
2import { isAbsolute, relative, sep } from "node:path";
3import type { Context } from "@earendil-works/chord";
4import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
5import { getOrThrow } from "@earendil-works/pi-durable/env";
6import type { Environment } from "@berry/api";
7import { MAX_SKILL_BYTES } from "./catalog.ts";
8import { userDirectory } from "./paths.ts";
9
10export const MAX_BUNDLE_BYTES = 32 * 1024 * 1024;
11const MAX_ENTRIES = 512;
12const MAX_DEPTH = 16;
13
14type Entry = { path: string; kind: "directory" } | {
15  path: string;
16  kind: "file";
17  executable: boolean;
18  data: Uint8Array;
19};
20type ManifestEntry =
21  | Omit<Extract<Entry, { kind: "file" }>, "data">
22  | Extract<Entry, { kind: "directory" }>;
23type Manifest = { id: string; entries: ManifestEntry[] };
24
25function portablePath(path: string): boolean {
26  return !/[\\\0]/.test(path) &&
27    path.split("/").every((part) => part && part !== "." && part !== "..");
28}
29
30function contained(root: string, path: string): boolean {
31  const fromRoot = relative(root, path);
32  return fromRoot !== ".." && !fromRoot.startsWith(`..${sep}`) && !isAbsolute(fromRoot);
33}
34
35function digest(entries: Entry[]): string {
36  const hash = createHash("sha256");
37  for (const entry of entries) {
38    hash.update(
39      JSON.stringify(
40        entry.kind === "file"
41          ? [entry.path, entry.kind, entry.executable, entry.data.length]
42          : [entry.path, entry.kind],
43      ) + "\n",
44    );
45    if (entry.kind === "file") hash.update(entry.data);
46  }
47  return hash.digest("hex");
48}
49
50async function readBytes(env: Environment, path: string, limit: number, context: Context) {
51  const reader = getOrThrow(await env.openBinaryReader(path, { noFollow: true }, context));
52  try {
53    const { size } = getOrThrow(await reader.info(context));
54    if (size > limit) throw new Error("Skill bundle exceeds its size limit");
55    const bytes = getOrThrow(await reader.read(0, size + 1, context));
56    if (bytes.length !== size) {
57      throw new Error("Skill resource changed while being copied; reload it again");
58    }
59    return bytes;
60  } finally {
61    await reader.close(context);
62  }
63}
64
65async function ready(
66  env: Environment,
67  version: string,
68  id: string,
69  context: Context,
70): Promise<boolean> {
71  const path = getOrThrow(await env.joinPath([version, "manifest.json"], context));
72  const text = await env.readTextFile(path, context);
73  if (!text.ok) {
74    if (text.error.code === "not_found") return false;
75    throw text.error;
76  }
77  if (JSON.parse(text.value).id !== id) throw new Error(`Invalid skill bundle cache at ${version}`);
78  return true;
79}
80
81/** Publish only complete versions, so interrupted or concurrent transfers cannot expose partial bundles. */
82async function publish(
83  env: Environment,
84  version: string,
85  id: string,
86  entries: Entry[],
87  context: Context,
88) {
89  const directory = getOrThrow(await env.joinPath([version, "skill"], context));
90  if (await ready(env, version, id, context)) return directory;
91  const stage = `${version}.${crypto.randomUUID()}`;
92  try {
93    const root = getOrThrow(await env.joinPath([stage, "skill"], context));
94    getOrThrow(await env.createDir(root, { recursive: true }, context));
95    if (env.platform !== "win32") {
96      const result = getOrThrow(
97        await env.exec(["chmod", "700", "--", stage], { timeout: 10 }, context),
98      );
99      if (result.exitCode !== 0) throw new Error("Could not make the skill bundle private");
100    }
101    const executable: string[] = [];
102    const manifest: Manifest = { id, entries: [] };
103    for (const entry of entries) {
104      const path = getOrThrow(await env.joinPath([root, ...entry.path.split("/")], context));
105      if (entry.kind === "directory") {
106        getOrThrow(await env.createDir(path, { recursive: true }, context));
107        manifest.entries.push(entry);
108      } else {
109        getOrThrow(await env.writeFile(path, entry.data, context));
110        manifest.entries.push({ path: entry.path, kind: entry.kind, executable: entry.executable });
111        if (entry.executable) executable.push(path);
112      }
113    }
114    if (env.platform !== "win32") {
115      for (let i = 0; i < executable.length; i += 64) {
116        const result = getOrThrow(
117          await env.exec(
118            ["chmod", "a+x", "--", ...executable.slice(i, i + 64)],
119            { timeout: 10 },
120            context,
121          ),
122        );
123        if (result.exitCode !== 0) {
124          throw new Error("Could not preserve skill script executable permissions");
125        }
126      }
127    }
128    getOrThrow(
129      await env.writeFile(
130        getOrThrow(await env.joinPath([stage, "manifest.json"], context)),
131        JSON.stringify(manifest),
132        context,
133      ),
134    );
135    const renamed = await env.renameFile(stage, version, context);
136    if (!renamed.ok && !await ready(env, version, id, context)) throw renamed.error;
137    return directory;
138  } finally {
139    await env.remove(stage, { recursive: true, force: true }, BACKGROUND_CONTEXT);
140  }
141}
142
143/** Host-side bundle snapshots keep resource versions available to historical conversations and new targets. */
144export class Bundles {
145  constructor(private readonly host: Environment) {}
146
147  private async version(id: string, context: Context): Promise<string> {
148    if (!/^[a-f0-9]{64}$/.test(id)) throw new Error("Invalid skill bundle identity");
149    const data = await userDirectory(this.host, "XDG_DATA_HOME", [".local", "share"], context);
150    return getOrThrow(await this.host.joinPath([data, "berry", "skills", "bundles", id], context));
151  }
152
153  async capture(directory: string, context: Context): Promise<{ id: string; content: string }> {
154    const root = await Deno.realPath(directory);
155    const host = this.host;
156    const entries: Entry[] = [];
157    let total = 0;
158    async function visit(path: string, prefix: string, ancestors: Set<string>) {
159      context.abortSignal?.throwIfAborted();
160      const canonical = await Deno.realPath(path);
161      if (!contained(root, canonical)) {
162        throw new Error(`Skill symlink escapes its directory: ${prefix}`);
163      }
164      const info = await Deno.stat(canonical);
165      if (prefix) {
166        if (!portablePath(prefix)) {
167          throw new Error(`Skill resource path is not portable: ${prefix}`);
168        }
169        if (entries.length === MAX_ENTRIES) throw new Error("Skill bundle has too many entries");
170      }
171      if (info.isDirectory) {
172        if (prefix) entries.push({ path: prefix, kind: "directory" });
173        if (ancestors.has(canonical)) throw new Error(`Skill symlink cycle: ${prefix}`);
174        if (ancestors.size > MAX_DEPTH) throw new Error("Skill bundle is too deeply nested");
175        const next = new Set(ancestors).add(canonical);
176        const children: string[] = [];
177        for await (const entry of Deno.readDir(canonical)) {
178          if (children.length + entries.length >= MAX_ENTRIES) {
179            throw new Error("Skill bundle has too many entries");
180          }
181          children.push(entry.name);
182        }
183        for (const name of children.sort()) {
184          const child = prefix ? `${prefix}/${name}` : name;
185          await visit(getOrThrow(await host.joinPath([canonical, name], context)), child, next);
186        }
187      } else if (info.isFile) {
188        const limit = prefix === "SKILL.md"
189          ? Math.min(MAX_SKILL_BYTES, MAX_BUNDLE_BYTES - total)
190          : MAX_BUNDLE_BYTES - total;
191        const data = await readBytes(host, canonical, limit, context);
192        total += data.length;
193        entries.push({
194          path: prefix,
195          kind: "file",
196          executable: Boolean((info.mode ?? 0) & 0o111),
197          data,
198        });
199      } else {
200        throw new Error(`Skill resource is not a regular file or directory: ${prefix}`);
201      }
202    }
203    await visit(root, "", new Set());
204    const main = entries.find((entry) => entry.path === "SKILL.md");
205    if (main?.kind !== "file") throw new Error("Skill bundle has no regular SKILL.md");
206    const content = new TextDecoder("utf-8", { fatal: true }).decode(main.data);
207    const id = digest(entries);
208    await publish(this.host, await this.version(id, context), id, entries, context);
209    return { id, content };
210  }
211
212  async materialize(id: string, env: Environment, context: Context): Promise<string> {
213    const source = await this.version(id, context);
214    const sourceRoot = getOrThrow(await this.host.joinPath([source, "skill"], context));
215    if (env.id === this.host.id) {
216      if (!await ready(this.host, source, id, context)) {
217        throw new Error("Host skill bundle snapshot is missing; reload the skill");
218      }
219      return sourceRoot;
220    }
221    const cache = await userDirectory(env, "XDG_CACHE_HOME", [".cache"], context);
222    const version = getOrThrow(await env.joinPath([cache, "berry", "skills", id], context));
223    if (await ready(env, version, id, context)) {
224      return getOrThrow(await env.joinPath([version, "skill"], context));
225    }
226    const text = getOrThrow(
227      await this.host.readTextFile(
228        getOrThrow(await this.host.joinPath([source, "manifest.json"], context)),
229        context,
230      ),
231    );
232    const manifest: Manifest = JSON.parse(text);
233    if (
234      manifest.id !== id || !Array.isArray(manifest.entries) ||
235      manifest.entries.length > MAX_ENTRIES
236    ) {
237      throw new Error("Invalid host skill bundle manifest");
238    }
239    const root = await Deno.realPath(sourceRoot);
240    const entries: Entry[] = [];
241    let total = 0;
242    for (const entry of manifest.entries) {
243      if (
244        typeof entry?.path !== "string" || !portablePath(entry.path) ||
245        (entry.kind !== "directory" &&
246          (entry.kind !== "file" || typeof entry.executable !== "boolean"))
247      ) {
248        throw new Error("Invalid host skill bundle entry");
249      }
250      if (entry.kind === "directory") entries.push(entry);
251      else {
252        const path = getOrThrow(
253          await this.host.joinPath([sourceRoot, ...entry.path.split("/")], context),
254        );
255        const canonical = await Deno.realPath(path);
256        if (!contained(root, canonical)) {
257          throw new Error("Host skill bundle cache contains an escaping symlink");
258        }
259        const data = await readBytes(this.host, canonical, MAX_BUNDLE_BYTES - total, context);
260        total += data.length;
261        entries.push({ ...entry, data });
262      }
263    }
264    if (digest(entries) !== id) {
265      throw new Error("Host skill bundle snapshot was modified; reload the skill");
266    }
267    return await publish(env, version, id, entries, context);
268  }
269}