char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencli : move to HTTP-based implementation (#24948)c264f65ff

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.9 KiB130 linesraw
1#include "cli-client.h"
2
3#include "http.h"
4
5#include <algorithm>
6#include <chrono>
7#include <thread>
8
9// generation can stall for a long time during prompt processing, so the
10// read timeout must be generous
11static constexpr time_t CLI_HTTP_READ_TIMEOUT_SEC = 3600;
12
13// upper bound for the accumulated response body kept for error reporting
14static constexpr size_t CLI_HTTP_MAX_ERROR_BODY = 1024 * 1024;
15
16// returns the path with the base url's path prefix prepended (if any)
17static std::string join_path(const common_http_url & parts, const std::string & path) {
18    if (parts.path.empty() || parts.path == "/") {
19        return path;
20    }
21    std::string prefix = parts.path;
22    if (prefix.back() == '/') {
23        prefix.pop_back();
24    }
25    return prefix + path;
26}
27
28std::string cli_client::get(const std::string & path) {
29    auto [cli, parts] = common_http_client(server_base);
30    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
31    auto path_with_model = path + (model.empty() ? "" : ("?model=" + model));
32    auto res = cli.Get(join_path(parts, path_with_model));
33    if (!res) {
34        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
35    }
36    if (res->status < 200 || res->status >= 300) {
37        throw std::runtime_error("GET " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
38    }
39    return res->body;
40}
41
42std::string cli_client::post(const std::string & path, const std::string & body) {
43    auto [cli, parts] = common_http_client(server_base);
44    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
45    auto res = cli.Post(join_path(parts, path), body, "application/json");
46    if (!res) {
47        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
48    }
49    if (res->status < 200 || res->status >= 300) {
50        throw std::runtime_error("POST " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
51    }
52    return res->body;
53}
54
55std::string cli_client::post_sse(const std::string & path,
56                                  const std::string & body,
57                                  const std::function<bool()> & should_stop,
58                                  const std::function<void(const std::string &)> & on_data) {
59    auto [cli, parts] = common_http_client(server_base);
60    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
61
62    std::string pending;  // buffer for incomplete SSE lines
63    std::string raw_body; // accumulated body, used only for error reporting
64
65    auto receiver = [&](const char * data, size_t len) -> bool {
66        if (should_stop()) {
67            return false; // aborts the request
68        }
69        if (raw_body.size() < CLI_HTTP_MAX_ERROR_BODY) {
70            raw_body.append(data, std::min(len, CLI_HTTP_MAX_ERROR_BODY - raw_body.size()));
71        }
72        pending.append(data, len);
73        size_t pos;
74        while ((pos = pending.find('\n')) != std::string::npos) {
75            std::string line = pending.substr(0, pos);
76            pending.erase(0, pos + 1);
77            if (!line.empty() && line.back() == '\r') {
78                line.pop_back();
79            }
80            if (line.rfind("data: ", 0) != 0) {
81                continue;
82            }
83            std::string payload = line.substr(6);
84            if (payload == "[DONE]") {
85                continue;
86            }
87            on_data(payload);
88        }
89        return true;
90    };
91
92    httplib::Headers headers = {{"Accept", "text/event-stream"}};
93    auto res = cli.Post(join_path(parts, path), headers, body, "application/json", receiver);
94
95    if (!res) {
96        if (res.error() == httplib::Error::Canceled && should_stop()) {
97            return ""; // cancelled by the user
98        }
99        return "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
100    }
101    if (res->status < 200 || res->status >= 300) {
102        if (!raw_body.empty()) {
103            return raw_body;
104        }
105        return "request failed with status " + std::to_string(res->status);
106    }
107    return "";
108}
109
110bool cli_client::wait_health(const std::function<bool()> & is_aborted) {
111    int connect_attempts = 0;
112    while (!is_aborted()) {
113        auto [cli, parts] = common_http_client(server_base);
114        cli.set_connection_timeout(1, 0);
115        auto res = cli.Get(join_path(parts, "/health"));
116        if (res) {
117            if (res->status == 200) {
118                return true;
119            }
120            // any other status means the server is up but not ready yet
121            // (e.g. 503 while the model is still loading)
122        } else if (++connect_attempts >= 10) {
123            last_error = "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
124            return false;
125        }
126        std::this_thread::sleep_for(std::chrono::milliseconds(300));
127    }
128    last_error = "aborted while waiting for the server to become ready";
129    return false;
130}