import assert from "node:assert/strict"; import childProcess from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test, type TestContext } from "node:test"; import type { ContextEvent, ContextEventResult, ExtensionAPI, ExtensionToolContext, SessionEntry, ToolDefinition, } from "@earendil-works/pi-coding-agent"; import computerUse from "../computer-use.ts"; import { parameters, type Action } from "../computer-use/backend.ts"; import { createWindowsBackend } from "../computer-use/windows.ts"; const png = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aV1kAAAAASUVORK5CYII=", "base64", ); async function fixture(t: TestContext, failStartup = false) { const directory = await mkdtemp(join(tmpdir(), "pi-desktop-test-")); const log = join(directory, "actions.jsonl"); const spawn = childProcess.spawn; let processes = 0; t.mock.method(childProcess, "spawn", () => { processes++; return spawn(process.execPath, ["-e", failStartup ? ` console.error("Desktop initialization failed"); process.exit(1); ` : ` const fs = require("node:fs"); const rl = require("node:readline").createInterface({ input: process.stdin }); const record = (value) => fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify(value) + "\\n"); rl.on("line", (line) => { const action = JSON.parse(line); record(action); if (action.key === "Wait") return; const response = action.key === "Unknown" ? { error: "Unsupported key name: Unknown" } : action.action === "screenshot" ? { png: ${JSON.stringify(png.toString("base64"))} } : {}; console.log(JSON.stringify(response)); }); rl.on("close", () => record("released")); `], { stdio: ["pipe", "pipe", "pipe"] }); }); syncBuiltinESMExports(); t.after(async () => { t.mock.restoreAll(); syncBuiltinESMExports(); await rm(directory, { recursive: true, force: true }); }); let tool: ToolDefinition | undefined; let context: ((event: ContextEvent) => ContextEventResult) | undefined; let agentEnd: (() => void) | undefined; const platform = Object.getOwnPropertyDescriptor(process, "platform")!; try { Object.defineProperty(process, "platform", { value: "win32" }); computerUse({ on(name: string, handler: unknown) { if (name === "context") context = handler as typeof context; if (name === "agent_end") agentEnd = handler as typeof agentEnd; }, registerTool(definition: ToolDefinition) { tool = definition; }, } as ExtensionAPI); } finally { Object.defineProperty(process, "platform", platform); } assert.ok(tool); assert.ok(context); assert.ok(agentEnd); const branch: SessionEntry[] = []; return { tool, context, agentEnd, branch, processes: () => processes, records: async () => (await readFile(log, "utf8")).trim().split("\n").map((line) => JSON.parse(line)), execute(actions: Action[], signal?: AbortSignal) { return tool!.execute("call-1", { actions }, signal, undefined, { sessionManager: { getBranch: () => branch }, } as ExtensionToolContext); }, }; } test("Windows actions share one helper and preserve drag state and Unicode payloads", async (t) => { const f = await fixture(t); const actions: Action[] = [ { action: "pointer", x: 100, y: 200, buttons: ["left"] }, { action: "pointer", x: 700, y: 400, buttons: ["left"] }, { action: "scroll", x: 700, y: 400, direction: "right", ticks: 3, buttons: ["left"] }, { action: "pointer", x: 700, y: 400, buttons: [] }, { action: "type", text: 'é中😀\n\t"\\' }, { action: "key", key: "a", modifiers: ["ctrl"] }, { action: "new_browser_tab", url: "https://example.org/" }, { action: "screenshot" }, ]; const result = await f.execute(actions); assert.equal(f.processes(), 1); assert.deepEqual(await f.records(), [...actions, "released"]); assert.deepEqual(result.content.filter((part) => part.type === "image"), [ { type: "image", data: png.toString("base64"), mimeType: "image/png" }, ]); }); test("screenshot visibility and recall remain shared across backends", async (t) => { const f = await fixture(t); const result = await f.execute([{ action: "screenshot" }]); const { screenshotIds: [id] } = result.details as { screenshotIds: string[] }; const message = { role: "toolResult" as const, toolName: "computer_use", toolCallId: "call-1", content: result.content, details: result.details, isError: false, timestamp: Date.now(), }; f.branch.push({ type: "message", id: "entry-1", parentId: null, timestamp: new Date().toISOString(), message }); const event: ContextEvent = { type: "context", messages: [message] }; assert.deepEqual(f.context(event).messages, [message]); f.agentEnd(); assert.deepEqual(f.context(event).messages, [{ ...message, content: message.content.filter((part) => part.type !== "image"), }]); const recalled = await f.execute([{ action: "recall", id }]); assert.deepEqual(recalled.content.filter((part) => part.type === "image"), result.content.filter((part) => part.type === "image")); assert.equal(f.processes(), 1); assert.deepEqual(f.context(event).messages, [message]); }); test("native errors still close the helper and release inputs", async (t) => { const f = await fixture(t); await assert.rejects(f.execute([{ action: "key", key: "Unknown" }]), /Unsupported key name/); assert.equal((await f.records()).at(-1), "released"); }); test("cancellation closes stdin gracefully instead of leaving held inputs", async (t) => { const f = await fixture(t); const controller = new AbortController(); const result = f.execute([{ action: "key", key: "Wait" }], controller.signal); controller.abort(); await assert.rejects(result, /Operation aborted/); assert.equal((await f.records()).at(-1), "released"); }); test("helper startup diagnostics are surfaced", async (t) => { const f = await fixture(t, true); await assert.rejects(f.execute([{ action: "screenshot" }]), /Desktop initialization failed/); }); test("unsafe browser URLs fail before starting a desktop helper", async (t) => { const f = await fixture(t); await assert.rejects(f.execute([{ action: "new_browser_tab", url: "file:///C:/Windows" }]), /Only HTTP and HTTPS/); assert.equal(f.processes(), 0); }); test("native Windows screen capture returns a 1920×1080 PNG", { skip: process.platform !== "win32" || process.env.PI_TEST_WINDOWS_DESKTOP !== "1", }, async () => { const desktop = createWindowsBackend(); try { const image = await desktop.execute({ action: "screenshot" }); assert.ok(image); assert.equal(image.subarray(1, 4).toString(), "PNG"); assert.equal(image.readUInt32BE(16), 1920); assert.equal(image.readUInt32BE(20), 1080); } finally { await desktop.close(); } });