char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aleksander Grygierui: Refactor Built-In Tools naming (Server/Browser) (#27271)0021a77de

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB66 linesraw
1import {
2	BuiltInTool,
3	JsonSchemaType,
4	MimeTypeAudio,
5	MimeTypeImage,
6	ToolCallType
7} from '$lib/enums';
8import type { OpenAIToolDefinition } from '$lib/types';
9
10export const READ_MEDIA_TOOL_NAME = BuiltInTool.BROWSER_READ_MEDIA;
11
12// header lines of the tool result, parsed back by the read_media renderer
13export const PREFIX_FILE = 'File: ';
14export const PREFIX_SIZE = 'Size: ';
15export const PREFIX_MIME = 'MIME: ';
16
17/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */
18export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`);
19
20/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */
21export const READ_MEDIA_IMAGE_MIME: Record<string, string> = {
22	gif: MimeTypeImage.GIF,
23	jpeg: MimeTypeImage.JPEG,
24	jpg: MimeTypeImage.JPEG,
25	png: MimeTypeImage.PNG
26} as const;
27
28/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */
29export const READ_MEDIA_AUDIO_MIME: Record<string, string> = {
30	mp3: MimeTypeAudio.MP3_MPEG,
31	wav: MimeTypeAudio.WAV
32} as const;
33
34/**
35 * Build the read_media tool definition for the modalities the active model has.
36 * At least one of the two flags must be true, otherwise the tool is not offered
37 * at all - a model that cannot see or hear has nothing to do with the bytes.
38 */
39export function buildReadMediaToolDefinition(
40	supportsVision: boolean,
41	supportsAudio: boolean
42): OpenAIToolDefinition {
43	const kinds: string[] = [];
44
45	if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`);
46
47	if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`);
48
49	return {
50		function: {
51			description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`,
52			name: READ_MEDIA_TOOL_NAME,
53			parameters: {
54				properties: {
55					path: {
56						description: 'Path to the media file',
57						type: JsonSchemaType.STRING
58					}
59				},
60				required: ['path'],
61				type: JsonSchemaType.OBJECT
62			}
63		},
64		type: ToolCallType.FUNCTION
65	};
66}