char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenserver: refactor server_stream (#25541)ea1f7bbb5

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.1 KiB100 linesraw
1#pragma once
2
3#include <atomic>
4#include <functional>
5#include <map>
6#include <memory>
7#include <string>
8#include <thread>
9#include <vector>
10#include <cstdint>
11#include <unordered_map>
12
13struct common_params;
14
15// generator-like API for HTTP response generation
16// this object response with one of the 2 modes:
17// 1) normal response: `data` contains the full response body
18// 2) streaming response: each call to next(output) generates the next chunk
19//    when next(output) returns false, no more data after the current chunk
20//    note: some chunks can be empty, in which case no data is sent for that chunk
21struct server_http_res {
22    std::string content_type = "application/json; charset=utf-8";
23    int status = 200;
24    std::string data;
25    std::map<std::string, std::string> headers;
26
27    std::function<bool(std::string &)> next = nullptr;
28    bool is_stream() const {
29        return next != nullptr;
30    }
31
32    // fired before req and res are destroyed
33    virtual void on_complete() {}
34
35    virtual ~server_http_res() = default;
36};
37
38// unique pointer, used by set_chunked_content_provider
39// httplib requires the stream provider to be stored in heap
40using server_http_res_ptr = std::unique_ptr<server_http_res>;
41using raw_buffer = std::vector<uint8_t>;
42
43struct uploaded_file {
44    raw_buffer data;
45    std::string filename;
46    std::string content_type;
47};
48
49struct server_http_req {
50    std::map<std::string, std::string> params; // path_params + query_params
51    std::map<std::string, std::string> headers; // used by MCP proxy
52    std::string path;
53    std::string query_string; // query parameters string (e.g. "action=save")
54    std::string body;
55    std::map<std::string, uploaded_file> files; // used for file uploads (form data)
56    const std::function<bool()> & should_stop;
57
58    std::string get_param(const std::string & key, const std::string & def = "") const {
59        auto it = params.find(key);
60        if (it != params.end()) {
61            return it->second;
62        }
63        return def;
64    }
65};
66
67struct server_http_context {
68    class Impl;
69    std::unique_ptr<Impl> pimpl;
70
71    std::thread thread; // server thread
72    std::atomic<bool> is_ready = false;
73
74    // note: the handler should never throw exceptions
75    using handler_t = std::function<server_http_res_ptr(const server_http_req & req)>;
76    mutable std::unordered_map<std::string, handler_t> handlers;
77
78    std::string path_prefix;
79    std::string hostname;
80    int port    = 8080;
81    bool is_ssl = false;
82
83    server_http_context();
84    ~server_http_context();
85
86    bool init(const common_params & params);
87    bool start();
88    void stop() const;
89
90    void get(const std::string & path, const handler_t & handler) const;
91    void post(const std::string & path, const handler_t & handler) const;
92    void del(const std::string & path, const handler_t & handler) const;
93
94    // Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict)
95    // Must be called AFTER all other API routes are registered
96    void register_gcp_compat() const;
97
98    // for debugging
99    std::string listening_address;
100};