char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencli : move to HTTP-based implementation (#24948)c264f65ff

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.4 KiB33 linesraw
1#pragma once
2
3#include <functional>
4#include <string>
5
6// openai-like client for CLI
7struct cli_client {
8    std::string server_base; // base url, for example "http://127.0.0.1:8080"
9    std::string last_error;  // set when wait_health() fails
10
11    std::string model; // optional, set when the server has multiple models (router mode)
12
13    // simple GET request, returns the raw response body
14    // throws std::runtime_error on transport error or non-2xx status
15    std::string get(const std::string & path);
16
17    // simple POST request, returns the raw response body
18    // throws std::runtime_error on transport error or non-2xx status
19    std::string post(const std::string & path, const std::string & body);
20
21    // POST request with an SSE streaming response
22    // on_data is invoked per "data:" event with the raw event payload
23    // returns after the stream is finished (empty string on graceful exit)
24    // otherwise, the raw error response body
25    std::string post_sse(const std::string & path,
26                          const std::string & body,
27                          const std::function<bool()> & should_stop,
28                          const std::function<void(const std::string &)> & on_data);
29
30    // poll /health until the server is ready to accept requests
31    // returns false if is_aborted returned true or the server is unreachable
32    bool wait_health(const std::function<bool()> & is_aborted);
33};