char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencli : move to HTTP-based implementation (#24948)c264f65ff

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB89 linesraw
1#pragma once
2
3#include <thread>
4
5#include "http.h"
6
7// llama_server will be available as a dynamic library symbol
8int llama_server(common_params & params, int argc, char ** argv);
9void llama_server_terminate();
10
11struct cli_server {
12    std::thread th;
13    int port = -1;
14    std::atomic<bool> is_alive = false;
15    std::atomic<bool> is_stopping = false;
16
17    ~cli_server() {
18        stop();
19    }
20
21    void stop() {
22        if (is_stopping.exchange(true)) {
23            return;
24        }
25        if (alive()) {
26            llama_server_terminate();
27        }
28        if (th.joinable()) {
29            th.join();
30        }
31    }
32
33    // spawn llama-server in a thread and interact with it via a random port
34    bool start(common_params & params) {
35        port = common_http_get_free_port();
36        if (port <= 0) {
37            fprintf(stderr, "failed to get a free port\n");
38            exit(1);
39        }
40
41        is_alive.store(true, std::memory_order_release);
42
43        common_params server_params = params; // copy
44        server_params.port = port;
45
46        th = std::thread([this, server_params]() mutable {
47            // argc / argv are only used in router mode, we can skip them for now
48            int res = llama_server(server_params, 0, nullptr);
49            if (res != 0) {
50                fprintf(stderr, "llama_server exited with code %d\n", res);
51            }
52            is_alive.store(false, std::memory_order_release);
53        });
54
55        return true;
56    }
57
58    std::string address() const {
59        return "http://127.0.0.1:" + std::to_string(port);
60    }
61
62    bool wait_ready(std::function<bool()> should_stop) {
63        if (!alive()) {
64            return false;
65        }
66        while (!should_stop()) {
67            auto [cli, parts] = common_http_client(address());
68            cli.set_connection_timeout(1, 0);
69            auto res = cli.Get("/health");
70            if (res) {
71                if (res->status == 200) {
72                    return true;
73                }
74                // any other status means the server is up but not ready yet
75                // (e.g. 503 while the model is still loading)
76            }
77            if (!alive()) {
78                // in case server die permanently
79                return false;
80            }
81            std::this_thread::sleep_for(std::chrono::milliseconds(200));
82        }
83        return true;
84    }
85
86    bool alive() const {
87        return is_alive.load(std::memory_order_acquire);
88    }
89};