char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenserver: refactor + correctness fixes for metrics (#26920)decaf508b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.6 KiB227 linesraw
1import pytest
2from utils import *
3
4server = ServerPreset.tinyllama2()
5
6
7@pytest.fixture(autouse=True)
8def create_server():
9    global server
10    server = ServerPreset.tinyllama2()
11    server.server_metrics = True
12
13
14def fetch_metrics(server: ServerProcess) -> str:
15    """get /metrics as raw prometheus text"""
16    res = server.make_request("GET", "/metrics")
17    assert res.status_code == 200
18    assert "Process-Start-Time-Unix" in res.headers
19    assert isinstance(res.body, str)
20    return res.body
21
22
23def parse_metrics(text: str) -> dict:
24    """parse the prometheus text format into {name: (type, value)}"""
25    out = {}
26    types = {}
27    for line in text.splitlines():
28        if line.startswith("# TYPE "):
29            _, _, name, kind = line.split(" ", 3)
30            types[name] = kind
31        elif line.startswith("llamacpp:") and "{" not in line:
32            name, value = line.split(" ", 1)
33            assert name in types, f"{name} has no # TYPE line"
34            out[name] = (types[name], float(value))
35    return out
36
37
38def test_metrics_disabled():
39    global server
40    server.server_metrics = False
41    server.start()
42    res = server.make_request("GET", "/metrics")
43    assert res.status_code == 501  # ERROR_TYPE_NOT_SUPPORTED
44
45
46def test_metrics_prometheus_format():
47    global server
48    server.start()
49    server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
50
51    text = fetch_metrics(server)
52    metrics = parse_metrics(text)
53
54    expected_counters = [
55        "llamacpp:prompt_tokens_total",
56        "llamacpp:prompt_tokens_cached_total",
57        "llamacpp:prompt_seconds_total",
58        "llamacpp:tokens_predicted_total",
59        "llamacpp:tokens_predicted_seconds_total",
60        "llamacpp:n_decode_total",
61        "llamacpp:n_tokens_max",
62        "llamacpp:spec_decode_num_draft_tokens_total",
63        "llamacpp:spec_decode_num_accepted_tokens_total",
64        "llamacpp:spec_decode_num_drafts_total",
65    ]
66    expected_gauges = [
67        "llamacpp:prompt_tokens_seconds",
68        "llamacpp:predicted_tokens_seconds",
69        "llamacpp:requests_processing",
70        "llamacpp:requests_deferred",
71        "llamacpp:n_busy_slots_per_decode",
72    ]
73
74    for name in expected_counters:
75        assert metrics[name][0] == "counter"
76    for name in expected_gauges:
77        assert metrics[name][0] == "gauge"
78
79    # every metric must carry a help line
80    for name in expected_counters + expected_gauges:
81        assert f"# HELP {name} " in text
82
83    assert metrics["llamacpp:n_decode_total"][1] > 0
84    assert metrics["llamacpp:requests_processing"][1] == 0
85
86
87def test_metrics_prompt_processed_and_cached():
88    global server
89    server.n_slots = 1  # keep the prompt cache on a single slot
90    server.start()
91
92    prompt = "the quick brown fox jumps over the lazy dog"
93
94    n_processed = 0
95    n_cached = 0
96    for _ in range(2):
97        res = server.make_request("POST", "/completion", data={"prompt": prompt, "n_predict": 4})
98        assert res.status_code == 200
99        n_processed += res.body["timings"]["prompt_n"]
100        n_cached += res.body["timings"]["cache_n"]
101
102    # the second request must reuse the prompt of the first one
103    assert n_cached > 0
104
105    metrics = parse_metrics(fetch_metrics(server))
106
107    # cached tokens are counted apart, they cost no decode
108    assert metrics["llamacpp:prompt_tokens_total"][1] == n_processed
109    assert metrics["llamacpp:prompt_tokens_cached_total"][1] == n_cached
110
111
112def test_metrics_predicted_total_matches_requests():
113    global server
114    server.start()
115
116    n_predicted = 0
117    for n_predict in [1, 4, 16]:
118        res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
119        assert res.status_code == 200
120        n_predicted += res.body["timings"]["predicted_n"]
121
122    metrics = parse_metrics(fetch_metrics(server))
123    assert metrics["llamacpp:tokens_predicted_total"][1] == n_predicted
124
125
126def test_metrics_generation_rate_excludes_first_token():
127    global server
128    server.start()
129
130    # the first token comes from the logits of the last prompt batch, so it costs no decode step
131    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 1})
132    timings = res.body["timings"]
133    assert timings["predicted_n"] == 1
134    assert timings["predicted_per_second"] == 0.0
135    assert timings["predicted_per_token_ms"] == 0.0
136
137    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 16})
138    timings = res.body["timings"]
139    assert timings["predicted_n"] == 16
140    # the rate is over 15 decode steps, not 16 tokens
141    expected = 1e3 / timings["predicted_ms"] * 15
142    assert abs(timings["predicted_per_second"] - expected) < 1e-6
143
144
145@pytest.mark.parametrize("n_predict", [1, 8])
146def test_metrics_timings_are_finite(n_predict: int):
147    global server
148    server.start()
149    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
150    timings = res.body["timings"]
151
152    # a null here means the server produced inf or nan
153    for key, value in timings.items():
154        assert value is not None, f"{key} is null"
155        assert value >= 0, f"{key} is negative"
156
157    assert timings["prompt_ms"] > 0
158    assert timings["prompt_per_token_ms"] > 0
159
160
161def test_metrics_timings_on_prompt_progress():
162    global server
163    server.start()
164
165    # a long prompt so that it is split over several batches (n_batch = 32)
166    prompt = "the quick brown fox jumps over the lazy dog " * 8
167    chunks = list(server.make_stream_request("POST", "/completion", data={
168        "prompt": prompt,
169        "n_predict": 4,
170        "stream": True,
171        "timings_per_token": True,
172        "return_progress": True,
173    }))
174
175    progress = [c for c in chunks if "prompt_progress" in c]
176    assert len(progress) > 1  # the prompt did not fit in a single batch
177
178    # the very first update is sent before any prompt token is decoded
179    first = progress[0]["timings"]
180    assert first["prompt_n"] == 0
181    assert first["prompt_ms"] == 0.0
182    assert first["predicted_n"] == 0
183    assert first["predicted_ms"] == 0.0
184
185    # timings must never go backwards, nor report bogus values
186    prompt_ms = 0.0
187    for chunk in progress:
188        timings = chunk["timings"]
189        for key, value in timings.items():
190            assert value is not None, f"{key} is null"
191            assert value >= 0, f"{key} is negative"
192        assert timings["prompt_ms"] >= prompt_ms
193        prompt_ms = timings["prompt_ms"]
194
195    assert prompt_ms > 0
196
197
198def test_metrics_slots_idle_after_completion():
199    global server
200    server.server_slots = True
201    server.start()
202    server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
203
204    res = server.make_request("GET", "/slots")
205    assert res.status_code == 200
206    for slot in res.body:
207        assert slot["is_processing"] is False
208        if "next_token" in slot:
209            # the budget of the finished task must not leak into the idle slot
210            assert slot["next_token"][0]["n_remain"] == -1
211            assert slot["next_token"][0]["n_decoded"] == 0
212
213
214def test_metrics_embedding_prompt_is_counted():
215    global server
216    server = ServerPreset.bert_bge_small()
217    server.server_metrics = True
218    server.start()
219
220    res = server.make_request("POST", "/v1/embeddings", data={"input": ["hello world", "goodbye world"]})
221    assert res.status_code == 200
222
223    # embedding tasks never sample a token, but their prompt still costs a decode
224    metrics = parse_metrics(fetch_metrics(server))
225    assert metrics["llamacpp:prompt_tokens_total"][1] > 0
226    assert metrics["llamacpp:n_decode_total"][1] > 0
227    assert metrics["llamacpp:tokens_predicted_total"][1] == 0