char-slop/berry

git clone https://git.t4t.associates/char-slop/berry

Charlotte Somweb: use regular duckduckgo for search queriesac590092026-10-25

// Functions that run inside the loaded page: serialized with toString(), so each is self-contained.

export type SearchResult = { title: string; url: string; snippet: string };

/** The page's own value of `fn()`, as json text for the browser to hand back. */
export function expressionFor(fn: () => unknown): string {
  return `Promise.resolve((${fn.toString()})()).then((result) => JSON.stringify(result))`;
}

/** Results of a duckduckgo search page, without its ads. */
export async function searchPage(): Promise<SearchResult[]> {
  const results = () => document.querySelectorAll<HTMLElement>('article[data-testid="result"]');
  // the page arrives empty and its own script fills in the results, so wait for them; a query
  // that matches nothing renders a message instead of any
  const deadline = Date.now() + 15_000;
  while (
    Date.now() < deadline &&
    results().length === 0 &&
    !/no results found for/i.test(document.body.textContent ?? "")
  ) {
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
  if (document.querySelector('[data-area="mainline"]') === null) {
    throw new Error("DuckDuckGo did not render a results page");
  }

  // innerText glues adjacent spans, such as a result's date, to the text after them
  const text = (element: HTMLElement): string =>
    [...element.childNodes]
      .map((node) => (node instanceof HTMLElement ? text(node) : node.textContent ?? ""))
      .join(element.children.length > 1 ? " " : "")
      .replace(/\s+/g, " ")
      .trim();

  return [...results()]
    .map((result) => {
      const anchor = result.querySelector<HTMLAnchorElement>('a[data-testid="result-title-a"]');
      if (!anchor) return undefined;
      const redirect = new URL(anchor.href);
      const url = redirect.searchParams.get("uddg") ?? anchor.href;
      const snippet = result.querySelector<HTMLElement>('[data-result="snippet"]');
      return { title: text(anchor), url, snippet: snippet ? text(snippet) : "" };
    })
    .filter((result): result is SearchResult => Boolean(result?.title && result.url));
}

/** The response body as served, whatever its type. */
export async function pageAsRaw(): Promise<string> {
  document.querySelector<HTMLElement>("#rawdata-tab")?.click();
  const rawJson = document.querySelector<HTMLElement>("#rawdata-panel .data")?.textContent;
  if (rawJson !== undefined && rawJson !== null) return rawJson;
  try {
    return await (await fetch(location.href)).text();
  } catch {
    return document.querySelector<HTMLElement>("body > pre")?.textContent ??
      document.documentElement.outerHTML;
  }
}

/** The page's main content as markdown; other documents as their text. */
export async function pageAsMarkdown(): Promise<string> {
  if (!/^(?:text\/html|application\/xhtml\+xml)$/i.test(document.contentType)) {
    document.querySelector<HTMLElement>("#rawdata-tab")?.click();
    const rawJson = document.querySelector<HTMLElement>("#rawdata-panel .data")?.textContent;
    if (rawJson !== undefined && rawJson !== null) return rawJson;
    try {
      return await (await fetch(location.href)).text();
    } catch {
      return document.querySelector<HTMLElement>("body > pre")?.textContent ??
        document.documentElement.textContent ?? "";
    }
  }

  const source = document.querySelector<HTMLElement>("article") ??
    document.querySelector<HTMLElement>("main, [role=main]") ??
    document.body;
  const root = source.cloneNode(true) as HTMLElement;
  const sourceElements = source.querySelectorAll("*");
  const clonedElements = root.querySelectorAll("*");
  sourceElements.forEach((element, index) => {
    const style = getComputedStyle(element);
    if (style.display === "none" || style.contentVisibility === "hidden") {
      clonedElements[index]?.remove();
    }
  });
  root
    .querySelectorAll(
      "script, style, noscript, template, svg, canvas, nav, header, footer, form, button, input, select, textarea, dialog, [hidden], [aria-hidden=true]",
    )
    .forEach((element) => element.remove());

  const escapeText = (text: string) => text.replace(/\s+/g, " ").replace(/([\\`*_[\]])/g, "\\$1");
  const children = (element: Element) => [...element.childNodes].map(render).join("");
  const block = (text: string) => `\n\n${text.trim()}\n\n`;

  function render(node: Node): string {
    if (node.nodeType === Node.TEXT_NODE) return escapeText(node.textContent ?? "");
    if (!(node instanceof Element)) return "";

    const tag = node.tagName.toLowerCase();
    if (tag === "br") return "\n";
    if (tag === "hr") return "\n\n---\n\n";
    if (/^h[1-6]$/.test(tag)) {
      return block(`${"#".repeat(Number(tag[1]))} ${children(node).trim()}`);
    }
    if (tag === "p") return block(children(node));
    if (tag === "strong" || tag === "b") return `**${children(node).trim()}**`;
    if (tag === "em" || tag === "i") return `*${children(node).trim()}*`;
    if (tag === "del" || tag === "s") return `~~${children(node).trim()}~~`;
    if (tag === "code" && node.parentElement?.tagName.toLowerCase() !== "pre") {
      const text = node.textContent ?? "";
      const fence = text.includes("`") ? "``" : "`";
      return `${fence}${text}${fence}`;
    }
    if (tag === "pre") {
      return `\n\n\`\`\`\n${node.textContent?.replace(/^\n|\n$/g, "") ?? ""}\n\`\`\`\n\n`;
    }
    if (tag === "a") {
      const text = children(node).trim();
      const href = (node as HTMLAnchorElement).href;
      if (!href || href.startsWith("javascript:")) return text;
      return text ? `[${text}](<${href.replace(/>/g, "%3E")}>)` : `<${href}>`;
    }
    if (tag === "img") {
      const image = node as HTMLImageElement;
      if (!image.src || image.src.startsWith("data:")) return "";
      return `![${escapeText(image.alt)}](<${image.src.replace(/>/g, "%3E")}>)`;
    }
    if (tag === "blockquote") {
      return block(children(node).trim().split("\n").map((line) => `> ${line}`).join("\n"));
    }
    if (tag === "ul" || tag === "ol") {
      const items = [...node.children]
        .filter((child) => child.tagName.toLowerCase() === "li")
        .map((item, index) =>
          `${tag === "ol" ? `${index + 1}.` : "-"} ${children(item).trim().replace(/\n/g, "\n  ")}`
        );
      return block(items.join("\n"));
    }
    if (tag === "table") {
      const rows = [...node.querySelectorAll("tr")].map((row) =>
        [...row.querySelectorAll(":scope > th, :scope > td")].map((cell) =>
          children(cell).trim().replace(/\|/g, "\\|").replace(/\n+/g, " ")
        )
      );
      if (!rows.length) return "";
      const width = Math.max(...rows.map((row) => row.length));
      const line = (row: string[]) =>
        `| ${[...row, ...Array(width - row.length).fill("")].join(" | ")} |`;
      return block(
        [line(rows[0]), line(Array(width).fill("---")), ...rows.slice(1).map(line)].join("\n"),
      );
    }
    if (tag === "dt") return block(`**${children(node).trim()}**`);
    if (tag === "dd") return block(children(node));
    if (
      ["article", "aside", "details", "div", "figcaption", "figure", "main", "section", "summary"]
        .includes(tag)
    ) {
      return block(children(node));
    }
    return children(node);
  }

  const markdown = render(root)
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n[ \t]+/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
  return markdown || document.body.innerText.trim();
}