char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

parabelboiui: add read_media tool (#25877)4dd1275842026-08-12

// Microbenchmarks for the functions on the agentic-thread streaming hot path.
//
// Every function here is invoked from a `$derived` that is invalidated on each
// streamed token, for every tool-call section in the message. These numbers give
// the per-call cost; multiply by (tokens x sections) for the real damage.
//
// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts

import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic';
import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code';
import { computeLineDiff } from '$lib/utils/compute-line-diff';
import { preprocessLaTeX } from '$lib/utils/latex-protection';
import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args';
import { extractSearchQuery, extractSearchResults } from '$lib/utils/search-results';
import { all as lowlightAll } from 'lowlight';
import rehypeHighlight from 'rehype-highlight';
import rehypeKatex from 'rehype-katex';
import rehypeStringify from 'rehype-stringify';
import { remark } from 'remark';
import remarkBreaks from 'remark-breaks';
import remarkGfm from 'remark-gfm';
import remarkMath from 'remark-math';
import remarkRehype from 'remark-rehype';
import { bench, describe } from 'vitest';

// --- fixtures -------------------------------------------------------------

function lines(n: number, seed: string): string {
	const out: string[] = [];

	for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`);

	return out.join('\n');
}

const SHELL_OUTPUT_1KB = lines(12, 'out');
const SHELL_OUTPUT_200KB = lines(2600, 'out');
const SHELL_OUTPUT_2MB = lines(26000, 'out');
// Realistic exec_shell_command result: the exit-code marker is the final line,
// which is what the un-anchored EXIT_CODE regex has to scan the whole blob for.
const SHELL_2MB_WITH_EXIT = `${SHELL_OUTPUT_2MB}\n[exit code: 0]`;
const EDIT_OLD_400 = lines(400, 'old');
const EDIT_NEW_400 = lines(400, 'new');
const EDIT_OLD_50 = lines(50, 'old');
const EDIT_NEW_50 = lines(50, 'new');
const WRITE_FILE_ARGS = JSON.stringify({
	content: lines(1500, 'src'),
	path: '/src/lib/thing.ts'
});
const MARKDOWN_50KB = Array.from(
	{ length: 400 },
	(_, i) =>
		`## Section ${i}\n\nSome **bold** prose with a [link](https://example.com) and \`inline\` code.\n\n- bullet one\n- bullet two\n\n\`\`\`ts\nconst x${i} = ${i};\n\`\`\`\n`
).join('\n');
const CODE_BLOCK_5KB = lines(60, 'code');

// --- A: computeLineDiff (O(m*n) LCS, allocates a full matrix) --------------

describe('computeLineDiff', () => {
	bench('50 x 50 lines', () => {
		computeLineDiff(EDIT_OLD_50, EDIT_NEW_50);
	});

	bench('400 x 400 lines', () => {
		computeLineDiff(EDIT_OLD_400, EDIT_NEW_400);
	});
});

// --- B: the unified processor, rebuilt per call ---------------------------
// Mirrors MarkdownContent.svelte:144-176. The local rehype/remark plugins are
// omitted (they pull in browser-only modules); rehypeHighlight + lowlightAll is
// the dominant term, so this is a lower bound on the real cost.

function buildProcessor() {
	// eslint-disable-next-line @typescript-eslint/no-explicit-any
	let proc: any = remark().use(remarkGfm);

	proc = proc.use(remarkMath).use(remarkBreaks).use(remarkRehype).use(rehypeKatex);

	return proc.use(rehypeHighlight, { languages: lowlightAll }).use(rehypeStringify, {
		allowDangerousHtml: true
	});
}

describe('markdown processor', () => {
	bench('build processor (per processMarkdown call, x2)', () => {
		buildProcessor();
	});

	const prebuilt = buildProcessor();

	bench('parse 50KB markdown with a prebuilt processor', () => {
		prebuilt.parse(MARKDOWN_50KB);
	});
});

// Isolates the O(n^2) term measured in Tier 1: processMarkdown re-parses the
// WHOLE accumulated string every frame, while only the last block can have
// changed. If parse() dominates at these sizes, incremental parsing (re-parsing
// only the tail after the last stable block) is the fix; if not, the cost is
// downstream in transform/stringify/DOM.
describe('markdown parse scaling (whole-string reparse per frame)', () => {
	const proc = buildProcessor();
	const prose = (kb: number) =>
		Array.from(
			{ length: Math.ceil((kb * 1024) / 64) },
			(_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`
		).join(' ');
	const MD_3KB = prose(3);
	const MD_11KB = prose(11);
	const MD_26KB = prose(26);

	bench('parse 3KB', () => {
		proc.parse(MD_3KB);
	});

	bench('parse 11KB', () => {
		proc.parse(MD_11KB);
	});

	bench('parse 26KB', () => {
		proc.parse(MD_26KB);
	});

	bench('parse only a 200-char tail (proposed incremental)', () => {
		proc.parse(MD_26KB.slice(-200));
	});
});

// processMarkdown runs these over the WHOLE accumulated string every frame too,
// before parse() even starts. Measure them before assuming parse is the target.
describe('other whole-string passes per frame', () => {
	const prose = (kb: number) =>
		Array.from(
			{ length: Math.ceil((kb * 1024) / 64) },
			(_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`
		).join(' ');
	const MD_3KB = prose(3);
	const MD_11KB = prose(11);
	const MD_26KB = prose(26);
	const MD_26KB_LATEX = `${MD_26KB} and some math $x^2 + y^2 = z^2$ inline.`;

	bench('preprocessLaTeX 3KB (no latex present)', () => {
		preprocessLaTeX(MD_3KB);
	});

	bench('preprocessLaTeX 11KB (no latex present)', () => {
		preprocessLaTeX(MD_11KB);
	});

	bench('preprocessLaTeX 26KB (no latex present)', () => {
		preprocessLaTeX(MD_26KB);
	});

	bench('preprocessLaTeX 26KB (latex present)', () => {
		preprocessLaTeX(MD_26KB_LATEX);
	});

	bench('detectIncompleteCodeBlock 26KB', () => {
		detectIncompleteCodeBlock(MD_26KB);
	});
});

// --- C: extractSearchResults, run for EVERY tool of every type ------------

describe('extractSearchResults (only .length > 0 is consumed)', () => {
	bench('1KB non-search tool result', () => {
		extractSearchResults(SHELL_OUTPUT_1KB);
	});

	bench('200KB non-search tool result', () => {
		extractSearchResults(SHELL_OUTPUT_200KB);
	});

	bench('2MB non-search tool result', () => {
		extractSearchResults(SHELL_OUTPUT_2MB);
	});
});

describe('extractSearchQuery', () => {
	bench('write_file args (throws on the JSON.parse path)', () => {
		extractSearchQuery(WRITE_FILE_ARGS);
	});
});

// --- E: the un-anchored exit-code regex -----------------------------------
// Reproduced inline so the bench is independent of the current call site.

const EXIT_CODE_TAIL = /\[exit code: (-?\d+)\]\s*$/;

describe('exit-code regex', () => {
	bench('whole 2MB blob (current behaviour)', () => {
		SHELL_2MB_WITH_EXIT.match(EXIT_CODE_TAIL);
	});

	bench('last 64 chars only (proposed)', () => {
		SHELL_2MB_WITH_EXIT.slice(-64).match(EXIT_CODE_TAIL);
	});
});

// --- per-line result parsers ----------------------------------------------

describe('parseToolResultWithMedia', () => {
	bench('1KB', () => {
		parseToolResultWithMedia(SHELL_OUTPUT_1KB, []);
	});

	bench('200KB', () => {
		parseToolResultWithMedia(SHELL_OUTPUT_200KB, []);
	});

	bench('2MB', () => {
		parseToolResultWithMedia(SHELL_OUTPUT_2MB, []);
	});
});

describe('classifyToolResult', () => {
	bench('200KB plain text (falls through to looksLikeMarkdown)', () => {
		classifyToolResult(SHELL_OUTPUT_200KB);
	});
});

describe('parsePartialJsonArgs', () => {
	bench('write_file args, ~60KB (char-by-char scan)', () => {
		parsePartialJsonArgs(WRITE_FILE_ARGS);
	});
});

// --- F / I: highlight.js ---------------------------------------------------

describe('highlightCode', () => {
	bench('short bash command (title row, per token)', () => {
		highlightCode('grep -rn "foo" src/ | head -50', 'bash');
	});

	bench('5KB known language', () => {
		highlightCode(CODE_BLOCK_5KB, 'typescript');
	});

	bench('5KB unknown language -> highlightAuto', () => {
		highlightCode(CODE_BLOCK_5KB, 'not-a-language');
	});
});