char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenserver: support Vertex AI compatible API (#22545)29debb3a6

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.8 KiB60 linesraw
1import pytest
2from utils import *
3
4server: ServerProcess
5
6
7@pytest.fixture(autouse=True)
8def create_server():
9    global server
10    server = ServerPreset.tinyllama2()
11    server.gcp_compat = True
12
13
14def test_gcp_predict_camel_case():
15    global server
16    server.start()
17    res = server.make_request("POST", "/predict", data={
18        "instances": [
19            {
20                "@requestFormat": "chatCompletions",
21                "max_tokens": 8,
22                "messages": [
23                    {"role": "user", "content": "What is the meaning of life?"},
24                ],
25            }
26        ],
27    })
28    assert res.status_code == 200
29    assert "predictions" in res.body
30    assert len(res.body["predictions"]) == 1
31    prediction = res.body["predictions"][0]
32    assert "choices" in prediction
33    assert len(prediction["choices"]) == 1
34    assert prediction["choices"][0]["message"]["role"] == "assistant"
35    assert len(prediction["choices"][0]["message"]["content"]) > 0
36
37
38def test_gcp_predict_multiple_instances():
39    global server
40    server.n_slots = 2
41    server.start()
42    res = server.make_request("POST", "/predict", data={
43        "instances": [
44            {
45                "@requestFormat": "chatCompletions",
46                "max_tokens": 8,
47                "messages": [{"role": "user", "content": "Say hello"}],
48            },
49            {
50                "@requestFormat": "chatCompletions",
51                "max_tokens": 8,
52                "messages": [{"role": "user", "content": "Say world"}],
53            },
54        ],
55    })
56    assert res.status_code == 200
57    assert len(res.body["predictions"]) == 2
58    for prediction in res.body["predictions"]:
59        assert "choices" in prediction
60        assert len(prediction["choices"][0]["message"]["content"]) > 0