char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenargs: add --video-* CLI arguments (#24318)f29551215

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.8 KiB193 linesraw
1#pragma once
2
3#include "server-http.h"
4#include "server-task.h"
5#include "server-queue.h"
6
7#include "json.h"
8
9#include <cstddef>
10#include <memory>
11#include <mutex>
12#include <set>
13
14struct server_context_impl; // private implementation
15
16struct server_context_meta {
17    std::string build_info;
18    std::string model_name;
19    std::set<std::string> model_aliases;
20    std::set<std::string> model_tags;
21    std::string model_path;
22    bool has_mtmd;
23    bool has_inp_image;
24    bool has_inp_audio;
25    bool has_inp_video;
26    json json_ui_settings;
27    int slot_n_ctx;
28    enum llama_pooling_type pooling_type;
29
30    // chat params
31    server_chat_params & chat_params;
32    std::map<std::string, bool> chat_template_caps;
33
34    // tokens
35    std::string bos_token_str;
36    std::string eos_token_str;
37    llama_token fim_pre_token;
38    llama_token fim_sub_token;
39    llama_token fim_mid_token;
40    llama_token fim_pad_token;
41    llama_token fim_rep_token;
42    llama_token fim_sep_token;
43
44    // sampling
45    std::vector<llama_logit_bias> logit_bias_eog;
46
47    // model meta
48    enum llama_vocab_type model_vocab_type;
49    int32_t model_vocab_n_tokens;
50    int32_t model_n_ctx_train;
51    int32_t model_n_embd_inp;
52    uint64_t model_n_params;
53    uint64_t model_size;
54    std::string model_ftype;
55};
56
57enum server_state {
58    SERVER_STATE_DOWNLOADING,
59    SERVER_STATE_LOADING,
60    SERVER_STATE_READY,
61    SERVER_STATE_SLEEPING,
62};
63
64static std::string server_state_to_str(server_state state) {
65    switch (state) {
66        case SERVER_STATE_DOWNLOADING: return "downloading";
67        case SERVER_STATE_LOADING:     return "loading";
68        case SERVER_STATE_READY:       return "ready";
69        case SERVER_STATE_SLEEPING:    return "sleeping";
70        default: GGML_ASSERT(false && "invalid server_state");
71    }
72}
73
74static server_state server_state_from_str(const std::string & str) {
75    if (str == "downloading") return SERVER_STATE_DOWNLOADING;
76    if (str == "loading")     return SERVER_STATE_LOADING;
77    if (str == "ready")       return SERVER_STATE_READY;
78    if (str == "sleeping")    return SERVER_STATE_SLEEPING;
79    GGML_ASSERT(false && "invalid server_state string");
80}
81
82using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
83
84struct server_context {
85    std::unique_ptr<server_context_impl> impl;
86
87    server_context();
88    ~server_context();
89
90    // load the model and initialize llama_context
91    // returns true on success
92    bool load_model(common_params & params);
93
94    // this function will block main thread until termination
95    void start_loop();
96
97    // terminate main loop (will unblock start_loop)
98    void terminate();
99
100    // get the underlaying llama_context, can return nullptr if sleeping
101    // not thread-safe, should only be used from the main thread
102    llama_context * get_llama_context() const;
103
104    // get a new response reader, used by CLI application
105    server_response_reader get_response_reader();
106
107    // get server metadata (read-only), can only be called after load_model()
108    // not thread-safe, should only be used from the main thread
109    server_context_meta get_meta() const;
110
111    // note: must be set before load_model() is called
112    void set_state_callback(server_state_callback_t callback);
113};
114
115
116// forward declarations
117struct server_res_generator;
118
119struct server_routes {
120    server_routes(const common_params & params, server_context & ctx_server);
121
122    void init_routes();
123
124    // note: this is not thread-safe and can only when ctx_http.is_ready is false
125    void update_meta(const server_context & ctx_server) {
126        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
127    }
128
129    // handlers using lambda function, so that they can capture `this` without `std::bind`
130    // they won't be called until ctx_http.is_ready is set to true
131    server_http_context::handler_t get_health;
132    server_http_context::handler_t get_metrics;
133    server_http_context::handler_t get_slots;
134    server_http_context::handler_t post_slots;
135    server_http_context::handler_t get_props;
136    server_http_context::handler_t post_props;
137    server_http_context::handler_t post_infill;
138    server_http_context::handler_t post_completions;
139    server_http_context::handler_t post_completions_oai;
140    server_http_context::handler_t post_chat_completions;
141    server_http_context::handler_t post_chat_completions_tok;
142    server_http_context::handler_t post_control;
143    server_http_context::handler_t post_responses_oai;
144    server_http_context::handler_t post_responses_tok_oai;
145    server_http_context::handler_t post_transcriptions_oai;
146    server_http_context::handler_t post_anthropic_messages;
147    server_http_context::handler_t post_anthropic_count_tokens;
148    server_http_context::handler_t post_apply_template;
149    server_http_context::handler_t get_models;
150    server_http_context::handler_t post_tokenize;
151    server_http_context::handler_t post_detokenize;
152    server_http_context::handler_t post_embeddings;
153    server_http_context::handler_t post_embeddings_oai;
154    server_http_context::handler_t post_rerank;
155    server_http_context::handler_t get_lora_adapters;
156    server_http_context::handler_t post_lora_adapters;
157
158    // to be used in router mode
159    json get_model_info() const;
160
161private:
162    std::unique_ptr<server_res_generator> handle_completions_impl(
163            const server_http_req & req,
164            server_task_type type,
165            const json & data,
166            const std::vector<raw_buffer> & files,
167            task_response_type res_type);
168    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
169    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
170    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
171    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
172    std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type);
173
174    // using unique_ptr to allow late initialization of const
175    std::unique_ptr<const server_context_meta> meta;
176
177    const common_params & params;
178    server_context_impl & ctx_server;
179
180    server_queue & queue_tasks;
181    server_response & queue_results;
182    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
183
184    // cached responses, to be used during sleep
185    std::mutex     mutex_cache;
186    json           cached_models  = nullptr;
187    json           cached_props   = nullptr;
188    server_metrics cached_metrics;
189    // set when a scrape during sleep already reported the throughput buckets
190    bool           should_reset_buckets = false;
191    // call right before sleep to update the cached responses
192    void update_cached_responses(bool is_sleeping);
193};