char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aleksander Grygierui: Stores split refactor (#27240)521a64cd0

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.5 KiB131 linesraw
1/**
2 * serverStore - Server connection state, configuration and role detection
3 *
4 * Owns the connection state and properties fetched from /props, plus MODEL
5 * vs ROUTER role detection and server-wide generation defaults. Uses
6 * PropsService for the /props fetch.
7 */
8
9import { ServerRole } from '$lib/enums';
10import { PropsService } from '$lib/services/props.service';
11import { ApiError } from '$lib/utils';
12
13const LOADING_RETRY_INTERVAL_MS = 1000;
14
15class ServerStore {
16	error = $state<string | null>(null);
17	loading = $state(false);
18	props = $state<ApiLlamaCppServerProps | null>(null);
19	role = $state<ServerRole | null>(null);
20	status = $state<number | null>(null);
21	private fetchPromise: Promise<void> | null = null;
22	private retryTimer: ReturnType<typeof setTimeout> | null = null;
23
24	get contextSize(): number | null {
25		const nCtx = this.props?.default_generation_settings?.n_ctx;
26
27		return typeof nCtx === 'number' ? nCtx : null;
28	}
29
30	get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null {
31		return this.props?.default_generation_settings?.params || null;
32	}
33
34	get isModelMode(): boolean {
35		return this.role === ServerRole.MODEL;
36	}
37
38	get isRouterMode(): boolean {
39		return this.role === ServerRole.ROUTER;
40	}
41
42	get uiSettings(): Record<string, string | number | boolean> | undefined {
43		return this.props?.ui_settings ?? this.props?.webui_settings;
44	}
45
46	clear(): void {
47		this.clearRetryTimer();
48		this.props = null;
49		this.error = null;
50		this.status = null;
51		this.loading = false;
52		this.role = null;
53		this.fetchPromise = null;
54	}
55
56	/**
57	 * @param background - Set by the automatic "still loading" poll. Skips the
58	 * `loading` flag flip so the UI doesn't bounce between the full loading
59	 * splash and the chat screen every retry tick.
60	 */
61	async fetch({ background = false }: { background?: boolean } = {}): Promise<void> {
62		if (this.fetchPromise) return this.fetchPromise;
63
64		this.clearRetryTimer();
65
66		if (!background) {
67			this.loading = true;
68		}
69
70		// Don't clear an existing "still loading" error before a retry -
71		// doing so would unmount/remount the error banner every second.
72		if (this.status !== 503) {
73			this.error = null;
74		}
75
76		const fetchPromise = (async () => {
77			try {
78				const props = await PropsService.fetch();
79
80				this.props = props;
81				this.error = null;
82				this.status = null;
83				this.detectRole(props);
84			} catch (error: unknown) {
85				this.error = error instanceof Error ? error.message : String(error);
86				this.status = error instanceof ApiError ? error.status : null;
87				console.error('Error fetching server properties:', error);
88
89				if (this.status === 503) {
90					this.scheduleRetry();
91				}
92			} finally {
93				if (!background) {
94					this.loading = false;
95				}
96
97				this.fetchPromise = null;
98			}
99		})();
100
101		this.fetchPromise = fetchPromise;
102		await fetchPromise;
103	}
104
105	private clearRetryTimer(): void {
106		if (this.retryTimer) {
107			clearTimeout(this.retryTimer);
108			this.retryTimer = null;
109		}
110	}
111
112	private detectRole(props: ApiLlamaCppServerProps): void {
113		const newRole = props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL;
114
115		if (this.role !== newRole) {
116			this.role = newRole;
117			console.info(`Server running in ${newRole === ServerRole.ROUTER ? 'ROUTER' : 'MODEL'} mode`);
118		}
119	}
120
121	private scheduleRetry(): void {
122		if (this.retryTimer) return;
123
124		this.retryTimer = setTimeout(() => {
125			this.retryTimer = null;
126			this.fetch({ background: true });
127		}, LOADING_RETRY_INTERVAL_MS);
128	}
129}
130
131export const serverStore = new ServerStore();