char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aleksander Grygierui: Stores split refactor (#27240)521a64cd0

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.3 KiB150 linesraw
1/**
2 * View layer over contextStatsStore for the context usage gauge: adds
3 * color levels, transient detail formatting, on-demand /props fetching
4 * and model loading on top of the store's token stats.
5 */
6
7import { useProcessingState } from './use-processing-state.svelte';
8import { colorLevelFromPercent } from '$lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge';
9import { STATS_UNITS } from '$lib/constants';
10import { ColorLevel } from '$lib/enums';
11import { contextStatsStore, modelsStore } from '$lib/stores';
12
13export interface UseContextGaugeReturn {
14	readonly activeModelId: string | null;
15	readonly isActiveModelLoaded: boolean;
16	readonly isActiveModelLoading: boolean;
17	readonly contextTotal: number | null;
18	readonly contextUsed: number;
19	readonly contextAvailable: number | null;
20	readonly currentRead: number;
21	readonly currentFresh: number;
22	readonly currentCache: number;
23	readonly currentOutput: number;
24	readonly kvTotal: number;
25	readonly cumulativeRead: number;
26	readonly cumulativeOutput: number;
27	readonly cumulativeCacheTotal: number;
28	readonly averageTokensPerSecond: number | null;
29	readonly contextPercent: number | null;
30	readonly colorLevel: ColorLevel;
31	readonly transientDetails: string[];
32	readonly hasAnyUsage: boolean;
33	loadModel(): Promise<void>;
34	startMonitoring(): void;
35}
36
37const TRANSIENT_DETAILS_EXCLUDED_PREFIXES = ['Context:', 'Output:'];
38
39function filterTransientDetails(raw: string[]): string[] {
40	return raw.filter((detail) => {
41		if (TRANSIENT_DETAILS_EXCLUDED_PREFIXES.some((prefix) => detail.startsWith(prefix))) {
42			return false;
43		}
44
45		return !detail.includes(STATS_UNITS.TOKENS_PER_SECOND);
46	});
47}
48
49export function useContextGauge(): UseContextGaugeReturn {
50	const processingState = useProcessingState();
51
52	// Pull /props on demand so n_ctx surfaces before the first chat request.
53	$effect(() => {
54		const modelId = contextStatsStore.activeModelId;
55
56		if (modelId && contextStatsStore.isActiveModelLoaded) {
57			const cached = modelsStore.props.getModelProps(modelId);
58
59			if (!cached) {
60				void modelsStore.props.fetchModelProps(modelId);
61			}
62		}
63	});
64
65	const colorLevel = $derived(colorLevelFromPercent(contextStatsStore.contextPercent));
66	// Drop lines the surrounding Context / Output / speed rows already render.
67	const transientDetails = $derived(filterTransientDetails(processingState.getTechnicalDetails()));
68	const hasAnyUsage = $derived(
69		contextStatsStore.cumulativeRead > 0 ||
70			contextStatsStore.cumulativeOutput > 0 ||
71			contextStatsStore.currentRead > 0 ||
72			contextStatsStore.currentOutput > 0 ||
73			contextStatsStore.averageTokensPerSecond !== null ||
74			transientDetails.length > 0
75	);
76
77	async function loadModel() {
78		const modelId = contextStatsStore.activeModelId;
79
80		if (!modelId || contextStatsStore.isActiveModelLoading) return;
81
82		try {
83			await modelsStore.status.load(modelId);
84		} catch {
85			// toast already surfaced by modelsStore.status.load
86		}
87	}
88
89	return {
90		get activeModelId() {
91			return contextStatsStore.activeModelId;
92		},
93		get averageTokensPerSecond() {
94			return contextStatsStore.averageTokensPerSecond;
95		},
96		get colorLevel() {
97			return colorLevel;
98		},
99		get contextAvailable() {
100			return contextStatsStore.contextAvailable;
101		},
102		get contextPercent() {
103			return contextStatsStore.contextPercent;
104		},
105		get contextTotal() {
106			return contextStatsStore.contextTotal;
107		},
108		get contextUsed() {
109			return contextStatsStore.contextUsed;
110		},
111		get cumulativeCacheTotal() {
112			return contextStatsStore.cumulativeCacheTotal;
113		},
114		get cumulativeOutput() {
115			return contextStatsStore.cumulativeOutput;
116		},
117		get cumulativeRead() {
118			return contextStatsStore.cumulativeRead;
119		},
120		get currentCache() {
121			return contextStatsStore.currentCache;
122		},
123		get currentFresh() {
124			return contextStatsStore.currentFresh;
125		},
126		get currentOutput() {
127			return contextStatsStore.currentOutput;
128		},
129		get currentRead() {
130			return contextStatsStore.currentRead;
131		},
132		get hasAnyUsage() {
133			return hasAnyUsage;
134		},
135		get isActiveModelLoaded() {
136			return contextStatsStore.isActiveModelLoaded;
137		},
138		get isActiveModelLoading() {
139			return contextStatsStore.isActiveModelLoading;
140		},
141		get kvTotal() {
142			return contextStatsStore.kvTotal;
143		},
144		loadModel,
145		startMonitoring: () => processingState.startMonitoring(),
146		get transientDetails() {
147			return transientDetails;
148		}
149	};
150}