char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencli: add --output option (#25484)3de7dd4c8

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB76 linesraw
1#pragma once
2
3#include "common.h"
4
5#include "cli-client.h"
6#include "cli-server.h"
7
8#include <atomic>
9#include <memory>
10#include <optional>
11#include <string>
12#include <fstream>
13
14struct cli_timings {
15    double prompt_per_second    = 0.0;
16    double predicted_per_second = 0.0;
17};
18
19struct cli_context_impl;
20
21struct cli_context {
22    common_params params;
23
24    cli_client client;                // always initialized
25    std::optional<cli_server> server; // only set when no --server-base is given
26
27    // properties of the connected server
28    // will be populated by fetch_server_props()
29    std::string model_name;
30    std::string model_ftype;
31    std::string build_info;
32    bool has_vision = false;
33    bool has_audio  = false;
34    bool has_video  = false;
35
36    std::optional<std::ofstream> output_file;
37
38    cli_context(const common_params & params);
39    ~cli_context();
40
41    // connect to --server-base or spawn a local llama-server child;
42    // argc/argv are needed to forward the server-relevant args to the child
43    bool init();
44
45    // run the interactive chat loop, returns the process exit code
46    int run();
47
48    // stop the local server child (if any)
49    void shutdown();
50
51    // set by the SIGINT handler; cleared once the interrupt has been handled
52    static std::atomic<bool> & interrupted();
53
54private:
55    struct generated_content {
56        std::string reasoning;
57        std::string content;
58    };
59    bool generate_completion(generated_content & content_out, cli_timings & timings);
60    void fetch_server_props();
61    void add_system_prompt();
62    void push_user_message(const std::string & text);
63
64    // check if server have multiple models (router mode)
65    // if yes, list them then ask; do nothing otherwise
66    bool list_and_ask_models();
67
68    // read a file and stage it as a multimodal content part; type is one of
69    // "image", "audio", "video"; returns false if the file cannot be read
70    bool stage_media_file(const std::string & fname, const std::string & type);
71
72    // no-op if output file is not set
73    void write_output_file(const std::string & content);
74
75    std::unique_ptr<cli_context_impl> impl;
76};