char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Daniel Beveniusserver : use pytest-xdist for server tests (#28298)42f0225fe

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
34.8 KiB644 linesraw
1#!/usr/bin/env python
2import pytest
3
4# ensure grandparent path is in sys.path
5from pathlib import Path
6import sys
7path = Path(__file__).resolve().parents[1]
8sys.path.insert(0, str(path))
9
10from utils import *
11from enum import Enum
12from typing import TypedDict
13
14server: ServerProcess
15
16TIMEOUT_START_SLOW = 15 * 60 # this is needed for real model tests
17TIMEOUT_HTTP_REQUEST = 60
18
19@pytest.fixture(autouse=True)
20def create_server():
21    global server
22    server = ServerPreset.tinyllama2()
23    server.model_alias = "tinyllama-2-tool-call"
24    server.n_slots = 1
25    server.n_ctx = 8192
26    server.n_batch = 2048
27
28class CompletionMode(Enum):
29    NORMAL = "normal"
30    STREAMED = "streamed"
31
32class ToolParameters(TypedDict):
33    type: str
34    properties: dict[str, dict]
35    required: list[str]
36
37class ToolFunction(TypedDict):
38    name: str
39    description: str
40    parameters: ToolParameters
41
42class ToolDefinition(TypedDict):
43    type: str
44    function: ToolFunction
45
46TEST_TOOL = ToolDefinition(
47    type = "function",
48    function = ToolFunction(
49        name = "test",
50        description = "",
51        parameters = ToolParameters(
52            type = "object",
53            properties = {
54                "success": {
55                    "type": "boolean",
56                    "const": True,
57                },
58            },
59            required = ["success"],
60        ),
61    ),
62)
63
64PYTHON_TOOL = ToolDefinition(
65    type = "function",
66    function = ToolFunction(
67        name = "python",
68        description = "Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.",
69        parameters = ToolParameters(
70            type = "object",
71            properties = {
72                "code": {
73                    "type": "string",
74                    "description": "The code to run in the ipython interpreter.",
75                },
76            },
77            required = ["code"],
78        ),
79    ),
80)
81
82WEATHER_TOOL = ToolDefinition(
83    type = "function",
84    function = ToolFunction(
85        name = "get_current_weather",
86        description = "Get the current weather in a given location",
87        parameters = ToolParameters(
88            type = "object",
89            properties = {
90                "location": {
91                    "type": "string",
92                    "description": "The city and country/state, e.g. 'San Francisco, CA', or 'Paris, France'",
93                },
94            },
95            required = ["location"],
96        ),
97    ),
98)
99
100def do_test_completion_with_required_tool_tiny(server: ServerProcess, tool: dict, argument_key: str | None, n_predict, **kwargs):
101    body = server.make_any_request("POST", "/v1/chat/completions", data={
102        "max_tokens": n_predict,
103        "messages": [
104            {"role": "system", "content": "You are a coding assistant."},
105            {"role": "user", "content": "Write an example"},
106        ],
107        "tool_choice": "required",
108        "tools": [tool],
109        "parallel_tool_calls": False,
110        **kwargs,
111    })
112    # assert res.status_code == 200, f"Expected status code 200, got {res.status_code}"
113    choice = body["choices"][0]
114    tool_calls = choice["message"].get("tool_calls")
115    assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'
116    tool_call = tool_calls[0]
117    assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'
118    # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'
119    expected_function_name = "python" if tool["type"] == "code_interpreter" else tool["function"]["name"]
120    assert expected_function_name == tool_call["function"]["name"], f'Expected tool name to be {tool_call["function"]["name"]} in {choice["message"]}'
121    actual_arguments = tool_call["function"]["arguments"]
122    assert isinstance(actual_arguments, dict) or isinstance(actual_arguments, str), f'Expected arguments to be a dict or str, got: {actual_arguments}'
123    if argument_key is not None:
124        if (isinstance(actual_arguments, str)):
125            actual_arguments = json.loads(actual_arguments)
126        assert argument_key in actual_arguments, f"tool arguments: {actual_arguments}, expected: {argument_key}"
127
128# PR #22654: commented out since we're now allowing content before tool calls in tool_call: required, so we can't force this
129# in the tiny model just by using the grammar
130#
131# @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
132# @pytest.mark.parametrize("template_name,tool,argument_key", [
133#     ("Qwen3-Coder",                                   TEST_TOOL,            "success"),
134#     ("Qwen3-Coder",                                   TEST_TOOL,            "success"),
135#     ("meta-llama-Llama-3.3-70B-Instruct",             TEST_TOOL,            "success"),
136#     ("meta-llama-Llama-3.3-70B-Instruct",             TEST_TOOL,            "success"),
137#     ("meta-llama-Llama-3.3-70B-Instruct",             PYTHON_TOOL,          "code"),
138#     ("meta-llama-Llama-3.3-70B-Instruct",             PYTHON_TOOL,          "code"),
139# ])
140# def test_completion_with_required_tool_tiny_fast(template_name: str, tool: dict, argument_key: str | None, stream: CompletionMode):
141#     global server
142#     n_predict = 1024
143#     # server = ServerPreset.stories15m_moe()
144#     server.jinja = True
145#     server.n_predict = n_predict
146#     server.chat_template_file = f'../../../models/templates/{template_name}.jinja'
147#     server.start()
148#     do_test_completion_with_required_tool_tiny(server, tool, argument_key, n_predict, stream=stream == CompletionMode.STREAMED, temperature=0.0, top_k=1, top_p=1.0)
149
150# @pytest.mark.slow
151# @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
152# @pytest.mark.parametrize("template_name,tool,argument_key", [
153#     ("meta-llama-Llama-3.1-8B-Instruct",              TEST_TOOL,            "success"),
154#     ("meta-llama-Llama-3.1-8B-Instruct",              PYTHON_TOOL,          "code"),
155
156#     ("meetkai-functionary-medium-v3.1",               TEST_TOOL,            "success"),
157#     ("meetkai-functionary-medium-v3.1",               PYTHON_TOOL,          "code"),
158
159#     ("meetkai-functionary-medium-v3.2",               TEST_TOOL,            "success"),
160#     # Functionary v3.2 format supports raw python content, which w/ a dummy stories model will never end on its own.
161#     # ("meetkai-functionary-medium-v3.2",               PYTHON_TOOL,          "code"),
162
163#     ("NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use", TEST_TOOL,            "success"),
164#     ("NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use", PYTHON_TOOL,          "code"),
165
166#     ("meta-llama-Llama-3.2-3B-Instruct",              TEST_TOOL,            "success"),
167#     ("meta-llama-Llama-3.2-3B-Instruct",              PYTHON_TOOL,          "code"),
168
169#     ("mistralai-Mistral-Nemo-Instruct-2407",          TEST_TOOL,            "success"),
170#     ("mistralai-Mistral-Nemo-Instruct-2407",          PYTHON_TOOL,          "code"),
171
172#     ("NousResearch-Hermes-3-Llama-3.1-8B-tool_use",   TEST_TOOL,            "success"),
173#     ("NousResearch-Hermes-3-Llama-3.1-8B-tool_use",   PYTHON_TOOL,          "code"),
174
175#     ("deepseek-ai-DeepSeek-R1-Distill-Llama-8B",      TEST_TOOL,            "success"),
176#     ("deepseek-ai-DeepSeek-R1-Distill-Llama-8B",      PYTHON_TOOL,          "code"),
177
178#     ("fireworks-ai-llama-3-firefunction-v2",          TEST_TOOL,            "success"),
179#     # ("fireworks-ai-llama-3-firefunction-v2",          PYTHON_TOOL,          "codeFalse), True),
180#     # ("fireworks-ai-llama-3-firefunction-v2",          PYTHON_TOOL,          "code"),
181
182# ])
183# def test_completion_with_required_tool_tiny_slow(template_name: str, tool: dict, argument_key: str | None, stream: CompletionMode):
184#     global server
185#     n_predict = 512
186#     # server = ServerPreset.stories15m_moe()
187#     server.jinja = True
188#     server.n_predict = n_predict
189#     server.chat_template_file = f'../../../models/templates/{template_name}.jinja'
190#     server.start(timeout_seconds=TIMEOUT_START_SLOW)
191#     do_test_completion_with_required_tool_tiny(server, tool, argument_key, n_predict, stream=stream == CompletionMode.STREAMED)
192
193
194@pytest.mark.slow
195@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
196@pytest.mark.parametrize("tool,argument_key,hf_repo,template_override", [
197    (TEST_TOOL,    "success",  "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
198    (PYTHON_TOOL,  "code",     "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
199    (PYTHON_TOOL,  "code",     "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
200
201    (TEST_TOOL,    "success",  "bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              None),
202    (PYTHON_TOOL,  "code",     "bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              None),
203    (PYTHON_TOOL,  "code",     "bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              "chatml"),
204
205    (TEST_TOOL,    "success",  "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      None),
206    (PYTHON_TOOL,  "code",     "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      None),
207    (PYTHON_TOOL,  "code",     "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      "chatml"),
208
209    (TEST_TOOL,    "success",  "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M",      None),
210    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M",      None),
211    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M",      "chatml"),
212
213    (TEST_TOOL,    "success",  "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M",      None),
214    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M",      None),
215    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M",      "chatml"),
216
217    (TEST_TOOL,    "success",  "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        None),
218    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        None),
219    (PYTHON_TOOL,  "code",     "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        "chatml"),
220
221    (TEST_TOOL,    "success",  "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
222    (PYTHON_TOOL,  "code",     "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
223    (PYTHON_TOOL,  "code",     "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
224
225    (TEST_TOOL,    "success",  "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",   ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
226    (PYTHON_TOOL,  "code",     "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",   ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
227    (PYTHON_TOOL,  "code",     "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",   "chatml"),
228
229    # (TEST_TOOL,    "success",  "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
230    # (PYTHON_TOOL,  "code",     "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
231    # (PYTHON_TOOL,  "code",     "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
232
233    (TEST_TOOL,    "success",  "bartowski/functionary-small-v3.2-GGUF:Q4_K_M",       ("meetkai/functionary-medium-v3.2", None)),
234    (PYTHON_TOOL,  "code",     "bartowski/functionary-small-v3.2-GGUF:Q4_K_M",       ("meetkai/functionary-medium-v3.2", None)),
235    (PYTHON_TOOL,  "code",     "bartowski/functionary-small-v3.2-GGUF:Q4_K_M",       "chatml"),
236
237    (TEST_TOOL,    "success",  "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      ("meta-llama/Llama-3.2-3B-Instruct", None)),
238    (PYTHON_TOOL,  "code",     "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      ("meta-llama/Llama-3.2-3B-Instruct", None)),
239    (PYTHON_TOOL,  "code",     "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      "chatml"),
240
241    (TEST_TOOL,    "success",  "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",      ("meta-llama/Llama-3.2-3B-Instruct", None)),
242    (PYTHON_TOOL,  "code",     "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",      ("meta-llama/Llama-3.2-3B-Instruct", None)),
243    (PYTHON_TOOL,  "code",     "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",      "chatml"),
244
245    (TEST_TOOL,    "success",  "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
246    (PYTHON_TOOL,  "code",     "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
247])
248def test_completion_with_required_tool_real_model(tool: dict, argument_key: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):
249    global server
250    n_predict = 512
251    server.jinja = True
252    server.n_ctx = 8192
253    server.n_predict = n_predict
254    server.model_hf_repo = hf_repo
255    server.model_hf_file = None
256    if isinstance(template_override, tuple):
257        (template_hf_repo, template_variant) = template_override
258        server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
259        assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
260    elif isinstance(template_override, str):
261        server.chat_template = template_override
262    server.start(timeout_seconds=TIMEOUT_START_SLOW)
263    body = server.make_any_request("POST", "/v1/chat/completions", data={
264        "max_tokens": n_predict,
265        "messages": [
266            {"role": "system", "content": "You are a coding assistant."},
267            {"role": "user", "content": "Write an example"},
268        ],
269        "tool_choice": "required",
270        "tools": [tool],
271        "parallel_tool_calls": False,
272        "stream": stream == CompletionMode.STREAMED,
273        "temperature": 0.0,
274        "top_k": 1,
275        "top_p": 1.0,
276    }, timeout=TIMEOUT_HTTP_REQUEST)
277    choice = body["choices"][0]
278    tool_calls = choice["message"].get("tool_calls")
279    assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'
280    tool_call = tool_calls[0]
281    # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'
282    expected_function_name = "python" if tool["type"] == "code_interpreter" else tool["function"]["name"]
283    assert expected_function_name == tool_call["function"]["name"]
284    actual_arguments = tool_call["function"]["arguments"]
285    assert isinstance(actual_arguments, str)
286    if argument_key is not None:
287        actual_arguments = json.loads(actual_arguments)
288        assert argument_key in actual_arguments, f"tool arguments: {json.dumps(actual_arguments)}, expected: {argument_key}"
289
290
291def do_test_completion_without_tool_call(server: ServerProcess, n_predict: int, tools: list[dict], tool_choice: str | None, **kwargs):
292    body = server.make_any_request("POST", "/v1/chat/completions", data={
293        "max_tokens": n_predict,
294        "messages": [
295            {"role": "system", "content": "You are a coding assistant."},
296            {"role": "user", "content": "say hello world with python"},
297        ],
298        "tools": tools if tools else None,
299        "tool_choice": tool_choice,
300        **kwargs,
301    }, timeout=TIMEOUT_HTTP_REQUEST)
302    choice = body["choices"][0]
303    assert choice["message"].get("tool_calls") is None, f'Expected no tool call in {choice["message"]}'
304
305
306@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
307@pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [
308    ("meta-llama-Llama-3.3-70B-Instruct",         128, [],            None),
309    ("meta-llama-Llama-3.3-70B-Instruct",         128, [TEST_TOOL],   None),
310    ("meta-llama-Llama-3.3-70B-Instruct",         128, [PYTHON_TOOL], 'none'),
311])
312def test_completion_without_tool_call_fast(template_name: str, n_predict: int, tools: list[dict], tool_choice: str | None, stream: CompletionMode):
313    global server
314    server.n_predict = n_predict
315    server.jinja = True
316    server.chat_template_file = f'../../../models/templates/{template_name}.jinja'
317    server.start()
318    do_test_completion_without_tool_call(server, n_predict, tools, tool_choice, stream=stream == CompletionMode.STREAMED)
319
320
321@pytest.mark.slow
322@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
323@pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [
324    ("meetkai-functionary-medium-v3.2",               256, [],            None),
325    ("meetkai-functionary-medium-v3.2",               256, [TEST_TOOL],   None),
326    ("meetkai-functionary-medium-v3.2",               256, [PYTHON_TOOL], 'none'),
327    ("meetkai-functionary-medium-v3.1",               256, [],            None),
328    ("meetkai-functionary-medium-v3.1",               256, [TEST_TOOL],   None),
329    ("meetkai-functionary-medium-v3.1",               256, [PYTHON_TOOL], 'none'),
330    ("meta-llama-Llama-3.2-3B-Instruct",              256, [],            None),
331    ("meta-llama-Llama-3.2-3B-Instruct",              256, [TEST_TOOL],   None),
332    ("meta-llama-Llama-3.2-3B-Instruct",              256, [PYTHON_TOOL], 'none'),
333])
334def test_completion_without_tool_call_slow(template_name: str, n_predict: int, tools: list[dict], tool_choice: str | None, stream: CompletionMode):
335    global server
336    server.n_predict = n_predict
337    server.jinja = True
338    server.chat_template_file = f'../../../models/templates/{template_name}.jinja'
339    server.start(timeout_seconds=TIMEOUT_START_SLOW)
340    do_test_completion_without_tool_call(server, n_predict, tools, tool_choice, stream=stream == CompletionMode.STREAMED)
341
342
343@pytest.mark.slow
344@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
345@pytest.mark.parametrize("hf_repo,template_override", [
346    ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
347    ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
348
349    ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      None),
350    ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      "chatml"),
351
352    ("bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M",      None),
353    ("bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M",      "chatml"),
354
355    ("bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M",      None),
356    ("bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M",      "chatml"),
357
358    ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        None),
359    ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        "chatml"),
360
361    ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M",    ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
362    ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M",    "chatml"),
363
364    ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",      ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
365    ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",      "chatml"),
366
367    # ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
368    # ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
369
370    # ("bartowski/functionary-small-v3.2-GGUF:Q8_0",       ("meetkai/functionary-medium-v3.2", None)),
371    # ("bartowski/functionary-small-v3.2-GGUF:Q8_0",       "chatml"),
372
373    ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      ("meta-llama/Llama-3.2-3B-Instruct", None)),
374    ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      "chatml"),
375
376    ("bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L",   ("CohereForAI/c4ai-command-r7b-12-2024", "tool_use")),
377
378    ("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
379
380    # Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
381    ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              None),
382
383    # ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
384])
385def test_weather(hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):
386    global server
387    n_predict = 512
388    server.jinja = True
389    server.n_ctx = 8192
390    server.n_predict = n_predict
391    server.model_hf_repo = hf_repo
392    server.model_hf_file = None
393    if isinstance(template_override, tuple):
394        (template_hf_repo, template_variant) = template_override
395        server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
396        assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
397    elif isinstance(template_override, str):
398        server.chat_template = template_override
399    server.start()
400    do_test_weather(server, stream=stream == CompletionMode.STREAMED, max_tokens=n_predict)
401
402
403def do_test_weather(server: ServerProcess, **kwargs):
404    body = server.make_any_request("POST", "/v1/chat/completions", data={
405        "messages": [
406            {"role": "system", "content": "You are a chatbot that uses tools/functions. Dont overthink things."},
407            {"role": "user", "content": "What is the weather in Istanbul?"},
408        ],
409        "tools": [WEATHER_TOOL],
410        **kwargs,
411    }, timeout=TIMEOUT_HTTP_REQUEST)
412    choice = body["choices"][0]
413    tool_calls = choice["message"].get("tool_calls")
414    assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'
415    tool_call = tool_calls[0]
416    # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'
417    assert tool_call["function"]["name"] == WEATHER_TOOL["function"]["name"], f'Expected weather tool call, got {tool_call["function"]["name"]}'
418    # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'
419    actual_arguments = json.loads(tool_call["function"]["arguments"])
420    assert 'location' in actual_arguments, f"location not found in {json.dumps(actual_arguments)}"
421    location = actual_arguments["location"]
422    assert isinstance(location, str), f"Expected location to be a string, got {type(location)}: {json.dumps(location)}"
423    assert re.match('^Istanbul(( |, ?)(TR|Turkey|Türkiye))?$', location), f'Expected Istanbul for location, got {location}'
424
425
426@pytest.mark.slow
427@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
428@pytest.mark.parametrize("result_override,n_predict,hf_repo,template_override", [
429    (None,                                           128,  "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",       "chatml"),
430    (None,                                           128,  "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", None),
431    (None,                                           128,  "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", "chatml"),
432    (None,                                           128,  "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",         "chatml"),
433    (None,                                           128,  "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M",     ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
434    (None,                                           128,  "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",       ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
435    (None,                                           128,  "bartowski/functionary-small-v3.2-GGUF:Q8_0",        ("meetkai/functionary-medium-v3.2", None)),
436    (None,                                           128,  "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M",  None),
437    (None,                                           128,  "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M",  "chatml"),
438    (None,                                           128,  "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",       None),
439    ("[\\s\\S]*?\\*\\*\\s*0.5($|\\*\\*)",            8192, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),
440
441    # TODO: fix these (wrong results, either didn't respect decimal instruction or got wrong value)
442    # (None,                                           128,  "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M",  None),
443    # ("[\\s\\S]*?\\*\\*\\s*0.5($|\\*\\*)",            8192, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
444])
445def test_calc_result(result_override: str | None, n_predict: int, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):
446    global server
447    server.jinja = True
448    server.n_ctx = 8192 * 2
449    server.n_predict = n_predict
450    server.model_hf_repo = hf_repo
451    server.model_hf_file = None
452    if isinstance(template_override, tuple):
453        (template_hf_repo, template_variant) = template_override
454        server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
455        assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
456    elif isinstance(template_override, str):
457        server.chat_template = template_override
458    server.start(timeout_seconds=TIMEOUT_START_SLOW)
459    do_test_calc_result(server, result_override, n_predict, stream=stream == CompletionMode.STREAMED)
460
461
462def do_test_calc_result(server: ServerProcess, result_override: str | None, n_predict: int, **kwargs):
463    body = server.make_any_request("POST", "/v1/chat/completions", data={
464        "max_tokens": n_predict,
465        "messages": [
466            {"role": "system", "content": "You are a tools-calling assistant. You express numerical values with at most two decimals."},
467            {"role": "user", "content": "What's the y coordinate of a point on the unit sphere at angle 30 degrees?"},
468            {
469                "role": "assistant",
470                "content": None,
471                "tool_calls": [
472                    {
473                        "id": "call_6789",
474                        "type": "function",
475                        "function": {
476                            "name": "calculate",
477                            "arguments": "{\"expression\":\"sin(30 * pi / 180)\"}"
478                        }
479                    }
480                ]
481            },
482            {
483                "role": "tool",
484                "name": "calculate",
485                "content": "0.55644242476",
486                "tool_call_id": "call_6789"
487            }
488        ],
489        "tools": [
490            {
491                "type":"function",
492                "function":{
493                    "name":"calculate",
494                    "description":"A calculator function that computes values of arithmetic expressions in the Python syntax",
495                    "parameters":{
496                        "type":"object",
497                        "properties":{
498                            "expression":{
499                            "type":"string",
500                            "description":"An arithmetic expression to compute the value of (Python syntad, assuming all floats)"
501                            }
502                        },
503                        "required":["expression"]
504                    }
505                }
506            }
507        ],
508        **kwargs,
509    }, timeout=TIMEOUT_HTTP_REQUEST)
510    choice = body["choices"][0]
511    tool_calls = choice["message"].get("tool_calls")
512    assert tool_calls is None, f'Expected no tool call in {choice["message"]}'
513    content = choice["message"].get("content")
514    assert content is not None, f'Expected content in {choice["message"]}'
515    if result_override is not None:
516        assert re.match(result_override, content), f'Expected {result_override}, got {content}'
517    else:
518        assert re.match('^[\\s\\S]*?((That\'s|\\bis) (approximately )?)?\\b0\\.(5\\b|56\\b|556)', content), \
519            f'Expected something like "The y coordinate is 0.56.", got {content}'
520
521
522@pytest.mark.slow
523@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
524@pytest.mark.parametrize("n_predict,reasoning_format,expect_reasoning_content,expect_content,hf_repo,template_override", [
525    (128, 'deepseek',   None, "^The sum of 102 and 7 is 109[\\s\\S]*",                                       "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",       None),
526    (128,  None,        None, "^The sum of 102 and 7 is 109[\\s\\S]*",                                       "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",       None),
527    (1024, 'deepseek',  "I need to calculate the sum of 102 and 7[\\s\\S]*", "To find the sum of[\\s\\S]*",  "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
528    (1024, 'deepseek',  "First, I [\\s\\S]*", "To find the sum of[\\s\\S]*",                                 "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),
529    # (1024, 'none',      CompletionMode.NORMAL,   None, "^(<think>\\s*)?I need[\\s\\S]*?</think>\\s*To find[\\s\\S]*",                 "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
530    # (128,  'deepseek',  None, "^Okay, let me figure out the sum of 102 and 7[\\s\\S]*",                      "bartowski/Qwen_QwQ-32B-GGUF:Q4_K_M",                None),
531])
532def test_thoughts(n_predict: int, reasoning_format: Literal['deepseek', 'none'] | None, expect_content: str | None, expect_reasoning_content: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):
533    global server
534    server.reasoning_format = reasoning_format
535    server.jinja = True
536    server.n_ctx = 8192 * 2
537    server.n_predict = n_predict
538    server.model_hf_repo = hf_repo
539    server.model_hf_file = None
540    if isinstance(template_override, tuple):
541        (template_hf_repo, template_variant) = template_override
542        server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
543        assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
544    elif isinstance(template_override, str):
545        server.chat_template = template_override
546    server.start()
547    body = server.make_any_request("POST", "/v1/chat/completions", data={
548        "max_tokens": n_predict,
549        "messages": [
550            {"role": "user", "content": "What's the sum of 102 and 7?"},
551        ],
552        "stream": stream == CompletionMode.STREAMED,
553    }, timeout=TIMEOUT_HTTP_REQUEST)
554    choice = body["choices"][0]
555    assert choice["message"].get("tool_calls") is None, f'Expected no tool call in {choice["message"]}'
556
557    content = choice["message"].get("content")
558    if expect_content is None:
559        assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'
560    else:
561        assert re.match(expect_content, content), f'Expected {expect_content}, got {content}'
562
563    reasoning_content = choice["message"].get("reasoning_content")
564    if expect_reasoning_content is None:
565        assert reasoning_content is None, f'Expected no reasoning content in {choice["message"]}'
566    else:
567        assert re.match(expect_reasoning_content, reasoning_content), f'Expected {expect_reasoning_content}, got {reasoning_content}'
568
569
570@pytest.mark.slow
571@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])
572@pytest.mark.parametrize("hf_repo,template_override", [
573    ("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
574
575    ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      None),
576    ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M",      "chatml"),
577
578    ("bartowski/functionary-small-v3.2-GGUF:Q8_0",       ("meetkai-functionary-medium-v3.2", None)),
579    ("bartowski/functionary-small-v3.2-GGUF:Q8_0",       "chatml"),
580
581    # ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
582    ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
583
584    ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",      ("meta-llama-Llama-3.2-3B-Instruct", None)),
585    ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",      None),
586
587    ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      ("meta-llama-Llama-3.2-3B-Instruct", None)),
588    ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",      None),
589
590    ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        None),
591    ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M",        "chatml"),
592
593    ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M",    ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
594    ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M",    "chatml"),
595
596    ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",      ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),
597    ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M",      "chatml"),
598
599    ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
600    ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
601
602    ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              None),
603    ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M",              "chatml"),
604])
605def test_hello_world(hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):
606    global server
607    n_predict = 512 # High because of DeepSeek R1
608    server.jinja = True
609    server.n_ctx = 8192
610    server.n_predict = n_predict
611    server.model_hf_repo = hf_repo
612    server.model_hf_file = None
613    if isinstance(template_override, tuple):
614        (template_hf_repo, template_variant) = template_override
615        server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
616        assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
617    elif isinstance(template_override, str):
618        server.chat_template = template_override
619    server.start(timeout_seconds=TIMEOUT_START_SLOW)
620
621    do_test_hello_world(server, stream=stream == CompletionMode.STREAMED, max_tokens=n_predict)
622
623
624def do_test_hello_world(server: ServerProcess, **kwargs):
625    body = server.make_any_request("POST", "/v1/chat/completions", data={
626        "messages": [
627            {"role": "system", "content": "You are a tool-calling agent."},
628            {"role": "user", "content": "say hello world with python"},
629        ],
630        "tools": [PYTHON_TOOL],
631        **kwargs,
632    }, timeout=TIMEOUT_HTTP_REQUEST)
633    choice = body["choices"][0]
634    tool_calls = choice["message"].get("tool_calls")
635    assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'
636    tool_call = tool_calls[0]
637    # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'
638    assert tool_call["function"]["name"] == PYTHON_TOOL["function"]["name"]
639    # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'
640    actual_arguments = json.loads(tool_call["function"]["arguments"])
641    assert 'code' in actual_arguments, f"code not found in {json.dumps(actual_arguments)}"
642    code = actual_arguments["code"]
643    assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}"
644    assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', re.sub(r'#.*\n?', '', code)), f'Expected hello world, got {code}'