char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aleksander Grygierui: Linting & Formatting scripts (#26819)92d1bb0c9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.8 KiB95 linesraw
1/**
2 * Text file processing utilities
3 * Handles text file detection, reading, and validation
4 */
5
6import { DEFAULT_BINARY_DETECTION_OPTIONS } from '$lib/constants';
7import { FileExtensionText } from '$lib/enums';
8import type { BinaryDetectionOptions } from '$lib/types';
9
10/**
11 * Check if a filename indicates a text file based on its extension
12 * @param filename - The filename to check
13 * @returns True if the filename has a recognized text file extension
14 */
15export function isTextFileByName(filename: string): boolean {
16	const textExtensions = Object.values(FileExtensionText);
17
18	return textExtensions.some((ext: FileExtensionText) => filename.toLowerCase().endsWith(ext));
19}
20
21/**
22 * Read a file's content as text
23 * @param file - The file to read
24 * @returns Promise resolving to the file's text content
25 */
26export async function readFileAsText(file: File): Promise<string> {
27	return new Promise((resolve, reject) => {
28		const reader = new FileReader();
29
30		reader.onload = (event) => {
31			if (event.target?.result !== null && event.target?.result !== undefined) {
32				resolve(event.target.result as string);
33			} else {
34				reject(new Error('Failed to read file'));
35			}
36		};
37
38		reader.onerror = () => reject(new Error('File reading error'));
39
40		reader.readAsText(file);
41	});
42}
43
44/**
45 * Heuristic check to determine if content is likely from a text file
46 * Detects binary files by counting suspicious characters and null bytes
47 * @param content - The file content to analyze
48 * @param options - Optional configuration for detection parameters
49 * @returns True if the content appears to be text-based
50 */
51export function isLikelyTextFile(
52	content: string,
53	options: Partial<BinaryDetectionOptions> = {}
54): boolean {
55	if (!content) return true;
56
57	const config = { ...DEFAULT_BINARY_DETECTION_OPTIONS, ...options };
58	const sample = content.substring(0, config.prefixLength);
59
60	let nullCount = 0;
61	let suspiciousControlCount = 0;
62
63	for (let i = 0; i < sample.length; i++) {
64		const charCode = sample.charCodeAt(i);
65
66		// Count null bytes - these are strong indicators of binary files
67		if (charCode === 0) {
68			nullCount++;
69
70			continue;
71		}
72
73		// Count suspicious control characters
74		// Allow common whitespace characters: tab (9), newline (10), carriage return (13)
75		if (charCode < 32 && charCode !== 9 && charCode !== 10 && charCode !== 13) {
76			// Count most suspicious control characters
77			if (charCode < 8 || (charCode > 13 && charCode < 27)) {
78				suspiciousControlCount++;
79			}
80		}
81
82		// Count replacement characters (indicates encoding issues)
83		if (charCode === 0xfffd) {
84			suspiciousControlCount++;
85		}
86	}
87
88	// Reject if too many null bytes
89	if (nullCount > config.maxAbsoluteNullBytes) return false;
90
91	// Reject if too many suspicious characters
92	if (suspiciousControlCount / sample.length > config.suspiciousCharThresholdRatio) return false;
93
94	return true;
95}