import type { Environment } from "@berry/api"; import { observe, resolveElement } from "./page.ts"; import { launch, type Session } from "./transport.ts"; import type { Action, Crop, Image, Snapshot } from "./types.ts"; type Tab = { context: string; key: string; prefix: string; document?: string; snapshot: Snapshot; image?: Image; }; type Evaluation = | { type: "success"; result: { type: string; value?: string; sharedId?: string } } | { type: "exception"; exceptionDetails: { text: string } }; const KEYS: Record = { Enter: "\uE007", Tab: "\uE004", Escape: "\uE00C", Backspace: "\uE003", Delete: "\uE017", ArrowLeft: "\uE012", ArrowUp: "\uE013", ArrowRight: "\uE014", ArrowDown: "\uE015", Home: "\uE011", End: "\uE010", PageUp: "\uE00E", PageDown: "\uE00F", Space: " ", Control: "\uE009", Shift: "\uE008", Alt: "\uE00A", Meta: "\uE03D", }; /** How long a session may go without a call before it closes. */ export const IDLE_MS = 15 * 60_000; /** One conversation's temporary session; operations are serialized to keep references meaningful. */ export class Browser { #session?: Promise; #tabs = new Map(); #bootstrap = new Set(); #diagnostics = new Map(); #navigation = new Map(); #nextTab = 0; #queue: Promise = Promise.resolve(); #signal?: AbortSignal; #disposed = false; #lifetime = new AbortController(); #stopped?: AbortController; #idleMs: number; #idle?: ReturnType; #calls = 0; constructor(readonly env: Environment, { idleMs = IDLE_MS } = {}) { this.#idleMs = idleMs; env.lifetime?.addEventListener("abort", this.#disconnected, { once: true }); } #disconnected = () => void this.dispose(); get active(): boolean { return this.#session !== undefined; } get snapshots(): Snapshot[] { return [...this.#tabs.values()].map((tab) => tab.snapshot); } get screenshots(): Record { return Object.fromEntries( [...this.#tabs].flatMap(([id, tab]) => tab.image ? [[id, tab.image]] : []), ); } async exclusive(work: () => Promise, signal: AbortSignal): Promise { this.#calls++; clearTimeout(this.#idle); const result = this.#queue.catch(() => {}).then(async () => { const stopped = this.#stopped = new AbortController(); signal = AbortSignal.any([ signal, this.#lifetime.signal, stopped.signal, ...(this.env.lifetime ? [this.env.lifetime] : []), ]); signal.throwIfAborted(); if (this.#disposed) throw new Error("Browser extension has been unloaded."); this.#signal = signal; let closing: Promise | undefined; let abort!: () => void; const cancelled = new Promise((_, reject) => { abort = () => { // stop abandoned input/navigation before another call uses the session closing = this.close(); reject(signal.reason); }; signal.addEventListener("abort", abort, { once: true }); }); const running = work(); try { return await Promise.race([running, cancelled]); } finally { signal.removeEventListener("abort", abort); if (signal.aborted) { await running.catch(() => {}); await closing; } this.#signal = undefined; this.#stopped = undefined; } }); this.#queue = result; try { return await result; } finally { if (--this.#calls === 0 && this.#session) { this.#idle = setTimeout(() => void this.close(), this.#idleMs); Deno.unrefTimer(this.#idle); } } } /** End the session now, interrupting the call using it. */ async stop(): Promise { this.#stopped?.abort(new Error("The user closed the browser session.")); await this.close(); } async dispose(): Promise { this.env.lifetime?.removeEventListener("abort", this.#disconnected); this.#disposed = true; this.#lifetime.abort(new Error("Browser extension has been unloaded.")); await this.close(); await this.#queue.catch(() => {}); } async close(): Promise { clearTimeout(this.#idle); const session = this.#session; this.#session = undefined; this.#tabs.clear(); this.#bootstrap.clear(); this.#diagnostics.clear(); this.#navigation.clear(); await (await session?.catch(() => undefined))?.close(); } #event = (method: string, params: Record) => { const source = params.source as { context?: string } | undefined; const context = String(params.context ?? source?.context ?? ""); if (method === "browsingContext.navigationStarted") { const tab = [...this.#tabs.values()].find((tab) => tab.context === context); this.#navigation.set(context, { id: params.navigation, url: String(params.url), document: tab?.document, }); } else if ( method.startsWith("browsingContext.") && this.#navigation.get(context)?.id === params.navigation ) { this.#navigation.delete(context); } let message = ""; if (method === "log.entryAdded" && (params.level === "error" || params.level === "warn")) { message = `${params.level}: ${params.text ?? "Console error"}`; } else if (method === "network.fetchError") { const request = params.request as { url?: string } | undefined; message = `Network: ${params.errorText} ${request?.url ?? ""}`; } else if (method === "network.responseCompleted") { const response = params.response as { status?: number; url?: string } | undefined; if ((response?.status ?? 0) >= 400) message = `HTTP ${response!.status}: ${response!.url}`; } if (message && context) { this.#diagnostics.set( context, [...(this.#diagnostics.get(context) ?? []), message.slice(0, 1_000)].slice(-20), ); } }; async #getSession(): Promise { this.#signal?.throwIfAborted(); if (this.#disposed) throw new Error("Browser extension has been unloaded."); if (!this.#session) { const started = launch(this.env, this.#event, this.#signal).then(async (session) => { try { const { contexts } = await session.request<{ contexts: { context: string }[] }>( "browsingContext.getTree", {}, ); this.#bootstrap = new Set(contexts.map((context) => context.context)); session.closed.then(() => { if (this.#session === started) { this.#session = undefined; this.#tabs.clear(); this.#diagnostics.clear(); } }); return session; } catch (error) { await session.close(); throw error; } }); this.#session = started; started.catch(() => { if (this.#session === started) this.#session = undefined; }); } const session = await this.#session; this.#signal?.throwIfAborted(); if (this.#disposed) throw new Error("Browser extension has been unloaded."); return session; } #adopt(context: string, url = "about:blank"): string { const id = `t${++this.#nextTab}`; this.#tabs.set(id, { context, key: `berry_${crypto.randomUUID().replaceAll("-", "")}`, prefix: "", snapshot: { tab: id, url, title: "", text: "", elements: [], truncated: false, frames: [], diagnostics: [], }, }); return id; } async tabs(): Promise<{ tab: string; url: string; title: string }[]> { if (!this.#session) return []; const session = await this.#getSession(); const { contexts } = await session.request<{ contexts: { context: string; url: string }[] }>( "browsingContext.getTree", {}, ); for (const [id, tab] of this.#tabs) { if (!contexts.some((context) => context.context === tab.context)) this.#tabs.delete(id); } for (const context of contexts) { if (this.#bootstrap.has(context.context)) continue; const tab = [...this.#tabs.values()].find((tab) => tab.context === context.context); if (tab) tab.snapshot = { ...tab.snapshot, url: context.url }; else this.#adopt(context.context, context.url); } return this.snapshots.map(({ tab, url, title }) => ({ tab, url, title })); } #tab(id: string): Tab { const tab = this.#tabs.get(id); if (!tab) { throw new Error( `No live tab ${id}. Use browser with action "tabs"; sessions end when idle, closed, reloaded, or reset.`, ); } return tab; } async #evaluate(tab: Tab, expression: string): Promise { const session = await this.#getSession(); const result = await session.request("script.evaluate", { expression, target: { context: tab.context, sandbox: "berry" }, awaitPromise: true, resultOwnership: "none", }); if (result.type === "exception") throw new Error(result.exceptionDetails.text); return result; } #resolve(tab: Tab, ref: string): string { if (!tab.prefix || !tab.snapshot.elements.some((element) => element.ref === ref)) { throw new Error("Stale or unknown element reference; take a new snapshot."); } return `(${resolveElement.toString()})(${JSON.stringify(tab.key)}, ${JSON.stringify(ref)})`; } async #keys(tab: Tab, actions: { type: string; value: string }[]): Promise { const session = await this.#getSession(); try { await session.request("input.performActions", { context: tab.context, actions: [{ type: "key", id: "keyboard", actions }], }); } finally { await session.request("input.releaseActions", { context: tab.context }).catch(() => {}); } } async snapshot(id: string): Promise { const tab = this.#tab(id); const prefix = `${crypto.randomUUID().slice(0, 8)}-`; const expression = `(${observe.toString()})(${JSON.stringify(tab.key)}, ${ JSON.stringify(prefix) }).then(JSON.stringify)`; const deadline = Date.now() + 3_000; while (true) { await this.#waitForNavigation(tab.context); try { const { result } = await this.#evaluate(tab, expression); if (result.type !== "string") throw new Error("Browser observation was not text."); const { documentId, ...observation } = JSON.parse(result.value!) as & Omit & { documentId: string }; tab.document = documentId; tab.snapshot = { ...observation, tab: id, diagnostics: this.#diagnostics.get(tab.context) ?? [], }; tab.prefix = prefix; tab.image = undefined; return tab.snapshot; } catch (error) { if ( Date.now() >= deadline || !/realm|document.*unload|execution context/i.test(String(error)) ) throw error; await this.#pause(100); } } } async screenshot(id: string, crop?: Crop): Promise { const tab = this.#tab(id); const session = await this.#getSession(); const { data } = await session.request<{ data: string }>("browsingContext.captureScreenshot", { context: tab.context, origin: "viewport", ...(crop ? { clip: { type: "box", ...crop } } : {}), }); return tab.image = { type: "image", mimeType: "image/png", data }; } async act(action: Action): Promise { if (action.action === "reset") { await this.close(); return null; } if (action.action === "tabs") { await this.tabs(); return null; } if (action.action === "open") { this.#url(action.url); if (this.#tabs.size >= 12) throw new Error("Close a tab before opening more (limit: 12)."); const session = await this.#getSession(); const { context } = await session.request<{ context: string }>("browsingContext.create", { type: "tab", }); const id = this.#adopt(context); await session.request("browsingContext.setViewport", { context, viewport: { width: 1280, height: 900 }, devicePixelRatio: 1, }); await session.request("browsingContext.navigate", { context, url: action.url, wait: "interactive", }); return await this.snapshot(id); } const tab = this.#tab(action.tab); const session = await this.#getSession(); const context = tab.context; switch (action.action) { case "snapshot": break; case "close": await session.request("browsingContext.close", { context }); this.#tabs.delete(action.tab); this.#diagnostics.delete(context); return null; case "navigate": this.#url(action.url); await session.request("browsingContext.navigate", { context, url: action.url, wait: "interactive", }); break; case "reload": await session.request("browsingContext.reload", { context, wait: "interactive" }); break; case "back": case "forward": await session.request("browsingContext.traverseHistory", { context, delta: action.action === "back" ? -1 : 1, }); break; case "click": { const { result } = await this.#evaluate( tab, `(() => { const element = ${this.#resolve(tab, action.ref)}; const rect = element.getBoundingClientRect(); const x = (Math.max(0, rect.left) + Math.min(innerWidth, rect.right)) / 2; const y = (Math.max(0, rect.top) + Math.min(innerHeight, rect.bottom)) / 2; let hit = document.elementFromPoint(x, y); while (hit?.shadowRoot?.elementFromPoint(x, y)) { const next = hit.shadowRoot.elementFromPoint(x, y); if (next === hit) break; hit = next; } for (let current = hit; current; current = current.parentElement || current.getRootNode().host) { if (current === element) return element; } throw new Error("Element is obscured; take a new snapshot or screenshot."); })()`, ); if (!result.sharedId) throw new Error("Could not resolve the element."); try { await session.request("input.performActions", { context, actions: [{ type: "pointer", id: "mouse", parameters: { pointerType: "mouse" }, actions: [ { type: "pointerMove", x: 0, y: 0, origin: { type: "element", element: { sharedId: result.sharedId } }, }, { type: "pointerDown", button: 0 }, { type: "pointerUp", button: 0 }, ], }], }); } finally { await session.request("input.releaseActions", { context }).catch(() => {}); } break; } case "fill": { await this.#evaluate( tab, `(() => { const element = ${this.#resolve(tab, action.ref)}; if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || element.isContentEditable) || element.readOnly) throw new Error("Element is not an editable text field."); element.focus(); if (document.activeElement !== element && element.getRootNode().activeElement !== element) throw new Error("Could not focus the field."); })()`, ); const control = KEYS.Control; await this.#keys(tab, [ { type: "keyDown", value: control }, { type: "keyDown", value: "a" }, { type: "keyUp", value: "a" }, { type: "keyUp", value: control }, { type: "keyDown", value: KEYS.Backspace }, { type: "keyUp", value: KEYS.Backspace }, ...[...action.text].flatMap(( value, ) => [{ type: "keyDown", value }, { type: "keyUp", value }]), ]); break; } case "select": await this.#evaluate( tab, `(() => { const element = ${this.#resolve(tab, action.ref)}; if (!(element instanceof HTMLSelectElement)) throw new Error("Element is not a select."); const value = ${JSON.stringify(action.value)}; const option = [...element.options].find(option => option.value === value); if (!option || option.disabled) throw new Error("No enabled option with that value."); element.value = value; element.dispatchEvent(new Event("input", { bubbles: true })); element.dispatchEvent(new Event("change", { bubbles: true })); })()`, ); break; case "press": { if (action.ref) await this.#evaluate(tab, `(${this.#resolve(tab, action.ref)}).focus()`); const parts = action.key.split("+"); const key = parts.pop()!; if ( parts.some((part) => !["Control", "Shift", "Alt", "Meta"].includes(part)) || (!KEYS[key] && [...key].length !== 1) ) { throw new Error( "Use a named key or character, optionally prefixed with Control+, Shift+, Alt+, or Meta+.", ); } const values = [...parts.map((part) => KEYS[part]), KEYS[key] ?? key]; await this.#keys(tab, [ ...values.map((value) => ({ type: "keyDown", value })), ...values.toReversed().map((value) => ({ type: "keyUp", value })), ]); break; } case "scroll": await this.#evaluate(tab, `window.scrollBy(${action.x}, ${action.y})`); break; case "wait": await this.#pause(action.ms); break; } return await this.snapshot(action.tab); } async #waitForNavigation(context: string): Promise { const tab = [...this.#tabs.values()].find((tab) => tab.context === context)!; const deadline = Date.now() + 25_000; while (this.#navigation.has(context)) { const navigation = this.#navigation.get(context)!; try { // firefox does not emit domContentLoaded when restoring from the back/forward cache const { result } = await this.#evaluate( tab, `JSON.stringify(document.readyState !== "loading" && location.href === ${ JSON.stringify(navigation.url) } && (${JSON.stringify(navigation.document ?? null)} === null || globalThis[${ JSON.stringify(tab.key + "_document") }] !== ${JSON.stringify(navigation.document)} || location.href !== ${ JSON.stringify(tab.snapshot.url) }))`, ); if (result.value === "true") { if (this.#navigation.get(context) === navigation) this.#navigation.delete(context); continue; } } catch (error) { if (!/realm|document.*unload|execution context/i.test(String(error))) throw error; } if (Date.now() >= deadline) { throw new Error("Navigation did not finish; inspect the tab again."); } await this.#pause(50); } } #pause(ms: number): Promise { const signal = this.#signal; signal?.throwIfAborted(); return new Promise((resolve, reject) => { const finish = () => { clearTimeout(timer); signal?.removeEventListener("abort", abort); resolve(); }; const abort = () => { clearTimeout(timer); reject(signal!.reason); }; const timer = setTimeout(finish, ms); signal?.addEventListener("abort", abort, { once: true }); }); } #url(url: string) { if (url !== "about:blank" && !["http:", "https:"].includes(new URL(url).protocol)) { throw new Error("Only HTTP/HTTPS URLs and about:blank are supported."); } } }