// Functions that run inside the loaded page: serialized with toString(), so each is self-contained. export type SearchResult = { title: string; url: string; snippet: string }; /** The page's own value of `fn()`, as json text for the browser to hand back. */ export function expressionFor(fn: () => unknown): string { return `Promise.resolve((${fn.toString()})()).then((result) => JSON.stringify(result))`; } /** Results of a duckduckgo search page, without its ads. */ export async function searchPage(): Promise { const results = () => document.querySelectorAll('article[data-testid="result"]'); // the page arrives empty and its own script fills in the results, so wait for them; a query // that matches nothing renders a message instead of any const deadline = Date.now() + 15_000; while ( Date.now() < deadline && results().length === 0 && !/no results found for/i.test(document.body.textContent ?? "") ) { await new Promise((resolve) => setTimeout(resolve, 100)); } if (document.querySelector('[data-area="mainline"]') === null) { throw new Error("DuckDuckGo did not render a results page"); } // innerText glues adjacent spans, such as a result's date, to the text after them const text = (element: HTMLElement): string => [...element.childNodes] .map((node) => (node instanceof HTMLElement ? text(node) : node.textContent ?? "")) .join(element.children.length > 1 ? " " : "") .replace(/\s+/g, " ") .trim(); return [...results()] .map((result) => { const anchor = result.querySelector('a[data-testid="result-title-a"]'); if (!anchor) return undefined; const redirect = new URL(anchor.href); const url = redirect.searchParams.get("uddg") ?? anchor.href; const snippet = result.querySelector('[data-result="snippet"]'); return { title: text(anchor), url, snippet: snippet ? text(snippet) : "" }; }) .filter((result): result is SearchResult => Boolean(result?.title && result.url)); } /** The response body as served, whatever its type. */ export async function pageAsRaw(): Promise { document.querySelector("#rawdata-tab")?.click(); const rawJson = document.querySelector("#rawdata-panel .data")?.textContent; if (rawJson !== undefined && rawJson !== null) return rawJson; try { return await (await fetch(location.href)).text(); } catch { return document.querySelector("body > pre")?.textContent ?? document.documentElement.outerHTML; } } /** The page's main content as markdown; other documents as their text. */ export async function pageAsMarkdown(): Promise { if (!/^(?:text\/html|application\/xhtml\+xml)$/i.test(document.contentType)) { document.querySelector("#rawdata-tab")?.click(); const rawJson = document.querySelector("#rawdata-panel .data")?.textContent; if (rawJson !== undefined && rawJson !== null) return rawJson; try { return await (await fetch(location.href)).text(); } catch { return document.querySelector("body > pre")?.textContent ?? document.documentElement.textContent ?? ""; } } const source = document.querySelector("article") ?? document.querySelector("main, [role=main]") ?? document.body; const root = source.cloneNode(true) as HTMLElement; const sourceElements = source.querySelectorAll("*"); const clonedElements = root.querySelectorAll("*"); sourceElements.forEach((element, index) => { const style = getComputedStyle(element); if (style.display === "none" || style.contentVisibility === "hidden") { clonedElements[index]?.remove(); } }); root .querySelectorAll( "script, style, noscript, template, svg, canvas, nav, header, footer, form, button, input, select, textarea, dialog, [hidden], [aria-hidden=true]", ) .forEach((element) => element.remove()); const escapeText = (text: string) => text.replace(/\s+/g, " ").replace(/([\\`*_[\]])/g, "\\$1"); const children = (element: Element) => [...element.childNodes].map(render).join(""); const block = (text: string) => `\n\n${text.trim()}\n\n`; function render(node: Node): string { if (node.nodeType === Node.TEXT_NODE) return escapeText(node.textContent ?? ""); if (!(node instanceof Element)) return ""; const tag = node.tagName.toLowerCase(); if (tag === "br") return "\n"; if (tag === "hr") return "\n\n---\n\n"; if (/^h[1-6]$/.test(tag)) { return block(`${"#".repeat(Number(tag[1]))} ${children(node).trim()}`); } if (tag === "p") return block(children(node)); if (tag === "strong" || tag === "b") return `**${children(node).trim()}**`; if (tag === "em" || tag === "i") return `*${children(node).trim()}*`; if (tag === "del" || tag === "s") return `~~${children(node).trim()}~~`; if (tag === "code" && node.parentElement?.tagName.toLowerCase() !== "pre") { const text = node.textContent ?? ""; const fence = text.includes("`") ? "``" : "`"; return `${fence}${text}${fence}`; } if (tag === "pre") { return `\n\n\`\`\`\n${node.textContent?.replace(/^\n|\n$/g, "") ?? ""}\n\`\`\`\n\n`; } if (tag === "a") { const text = children(node).trim(); const href = (node as HTMLAnchorElement).href; if (!href || href.startsWith("javascript:")) return text; return text ? `[${text}](<${href.replace(/>/g, "%3E")}>)` : `<${href}>`; } if (tag === "img") { const image = node as HTMLImageElement; if (!image.src || image.src.startsWith("data:")) return ""; return `![${escapeText(image.alt)}](<${image.src.replace(/>/g, "%3E")}>)`; } if (tag === "blockquote") { return block(children(node).trim().split("\n").map((line) => `> ${line}`).join("\n")); } if (tag === "ul" || tag === "ol") { const items = [...node.children] .filter((child) => child.tagName.toLowerCase() === "li") .map((item, index) => `${tag === "ol" ? `${index + 1}.` : "-"} ${children(item).trim().replace(/\n/g, "\n ")}` ); return block(items.join("\n")); } if (tag === "table") { const rows = [...node.querySelectorAll("tr")].map((row) => [...row.querySelectorAll(":scope > th, :scope > td")].map((cell) => children(cell).trim().replace(/\|/g, "\\|").replace(/\n+/g, " ") ) ); if (!rows.length) return ""; const width = Math.max(...rows.map((row) => row.length)); const line = (row: string[]) => `| ${[...row, ...Array(width - row.length).fill("")].join(" | ")} |`; return block( [line(rows[0]), line(Array(width).fill("---")), ...rows.slice(1).map(line)].join("\n"), ); } if (tag === "dt") return block(`**${children(node).trim()}**`); if (tag === "dd") return block(children(node)); if ( ["article", "aside", "details", "div", "figcaption", "figure", "main", "section", "summary"] .includes(tag) ) { return block(children(node)); } return children(node); } const markdown = render(root) .replace(/[ \t]+\n/g, "\n") .replace(/\n[ \t]+/g, "\n") .replace(/\n{3,}/g, "\n\n") .trim(); return markdown || document.body.innerText.trim(); }