char-slop/ai-dots

git clone https://git.t4t.associates/char-slop/ai-dots

Charlotte Somcomputer use for windowsdc2c12a2026-10-02

import assert from "node:assert/strict";
import childProcess from "node:child_process";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import type {
  ContextEvent, ContextEventResult, ExtensionAPI, ExtensionToolContext,
  SessionEntry, ToolDefinition,
} from "@earendil-works/pi-coding-agent";
import computerUse from "../computer-use.ts";
import { parameters, type Action } from "../computer-use/backend.ts";
import { createWindowsBackend } from "../computer-use/windows.ts";

const png = Buffer.from(
  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aV1kAAAAASUVORK5CYII=",
  "base64",
);

async function fixture(t: TestContext, failStartup = false) {
  const directory = await mkdtemp(join(tmpdir(), "pi-desktop-test-"));
  const log = join(directory, "actions.jsonl");
  const spawn = childProcess.spawn;
  let processes = 0;
  t.mock.method(childProcess, "spawn", () => {
    processes++;
    return spawn(process.execPath, ["-e", failStartup ? `
      console.error("Desktop initialization failed");
      process.exit(1);
    ` : `
      const fs = require("node:fs");
      const rl = require("node:readline").createInterface({ input: process.stdin });
      const record = (value) => fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify(value) + "\\n");
      rl.on("line", (line) => {
        const action = JSON.parse(line);
        record(action);
        if (action.key === "Wait") return;
        const response = action.key === "Unknown"
          ? { error: "Unsupported key name: Unknown" }
          : action.action === "screenshot" ? { png: ${JSON.stringify(png.toString("base64"))} } : {};
        console.log(JSON.stringify(response));
      });
      rl.on("close", () => record("released"));
    `], { stdio: ["pipe", "pipe", "pipe"] });
  });
  syncBuiltinESMExports();
  t.after(async () => {
    t.mock.restoreAll();
    syncBuiltinESMExports();
    await rm(directory, { recursive: true, force: true });
  });

  let tool: ToolDefinition<typeof parameters, { screenshotIds: string[] }> | undefined;
  let context: ((event: ContextEvent) => ContextEventResult) | undefined;
  let agentEnd: (() => void) | undefined;
  const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
  try {
    Object.defineProperty(process, "platform", { value: "win32" });
    computerUse({
      on(name: string, handler: unknown) {
        if (name === "context") context = handler as typeof context;
        if (name === "agent_end") agentEnd = handler as typeof agentEnd;
      },
      registerTool(definition: ToolDefinition<typeof parameters, { screenshotIds: string[] }>) { tool = definition; },
    } as ExtensionAPI);
  } finally {
    Object.defineProperty(process, "platform", platform);
  }
  assert.ok(tool);
  assert.ok(context);
  assert.ok(agentEnd);
  const branch: SessionEntry[] = [];
  return {
    tool,
    context,
    agentEnd,
    branch,
    processes: () => processes,
    records: async () => (await readFile(log, "utf8")).trim().split("\n").map((line) => JSON.parse(line)),
    execute(actions: Action[], signal?: AbortSignal) {
      return tool!.execute("call-1", { actions }, signal, undefined, {
        sessionManager: { getBranch: () => branch },
      } as ExtensionToolContext);
    },
  };
}

test("Windows actions share one helper and preserve drag state and Unicode payloads", async (t) => {
  const f = await fixture(t);
  const actions: Action[] = [
    { action: "pointer", x: 100, y: 200, buttons: ["left"] },
    { action: "pointer", x: 700, y: 400, buttons: ["left"] },
    { action: "scroll", x: 700, y: 400, direction: "right", ticks: 3, buttons: ["left"] },
    { action: "pointer", x: 700, y: 400, buttons: [] },
    { action: "type", text: 'é中😀\n\t"\\' },
    { action: "key", key: "a", modifiers: ["ctrl"] },
    { action: "new_browser_tab", url: "https://example.org/" },
    { action: "screenshot" },
  ];
  const result = await f.execute(actions);
  assert.equal(f.processes(), 1);
  assert.deepEqual(await f.records(), [...actions, "released"]);
  assert.deepEqual(result.content.filter((part) => part.type === "image"), [
    { type: "image", data: png.toString("base64"), mimeType: "image/png" },
  ]);
});

test("screenshot visibility and recall remain shared across backends", async (t) => {
  const f = await fixture(t);
  const result = await f.execute([{ action: "screenshot" }]);
  const { screenshotIds: [id] } = result.details as { screenshotIds: string[] };
  const message = {
    role: "toolResult" as const, toolName: "computer_use", toolCallId: "call-1",
    content: result.content, details: result.details, isError: false, timestamp: Date.now(),
  };
  f.branch.push({ type: "message", id: "entry-1", parentId: null, timestamp: new Date().toISOString(), message });
  const event: ContextEvent = { type: "context", messages: [message] };
  assert.deepEqual(f.context(event).messages, [message]);
  f.agentEnd();
  assert.deepEqual(f.context(event).messages, [{
    ...message, content: message.content.filter((part) => part.type !== "image"),
  }]);
  const recalled = await f.execute([{ action: "recall", id }]);
  assert.deepEqual(recalled.content.filter((part) => part.type === "image"), result.content.filter((part) => part.type === "image"));
  assert.equal(f.processes(), 1);
  assert.deepEqual(f.context(event).messages, [message]);
});

test("native errors still close the helper and release inputs", async (t) => {
  const f = await fixture(t);
  await assert.rejects(f.execute([{ action: "key", key: "Unknown" }]), /Unsupported key name/);
  assert.equal((await f.records()).at(-1), "released");
});

test("cancellation closes stdin gracefully instead of leaving held inputs", async (t) => {
  const f = await fixture(t);
  const controller = new AbortController();
  const result = f.execute([{ action: "key", key: "Wait" }], controller.signal);
  controller.abort();
  await assert.rejects(result, /Operation aborted/);
  assert.equal((await f.records()).at(-1), "released");
});

test("helper startup diagnostics are surfaced", async (t) => {
  const f = await fixture(t, true);
  await assert.rejects(f.execute([{ action: "screenshot" }]), /Desktop initialization failed/);
});

test("unsafe browser URLs fail before starting a desktop helper", async (t) => {
  const f = await fixture(t);
  await assert.rejects(f.execute([{ action: "new_browser_tab", url: "file:///C:/Windows" }]), /Only HTTP and HTTPS/);
  assert.equal(f.processes(), 0);
});

test("native Windows screen capture returns a 1920×1080 PNG", {
  skip: process.platform !== "win32" || process.env.PI_TEST_WINDOWS_DESKTOP !== "1",
}, async () => {
  const desktop = createWindowsBackend();
  try {
    const image = await desktop.execute({ action: "screenshot" });
    assert.ok(image);
    assert.equal(image.subarray(1, 4).toString(), "PNG");
    assert.equal(image.readUInt32BE(16), 1920);
    assert.equal(image.readUInt32BE(20), 1080);
  } finally {
    await desktop.close();
  }
});