char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencommon: add json.h abstraction (#27511)d9f918d2d

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
22.0 KiB677 linesraw
1#include "cli-context.h"
2#include "cli-ui.h"
3
4#include "arg.h"
5#include "base64.hpp"
6#include "log.h"
7#include "console.h"
8
9#include "json.h"
10
11#include <algorithm>
12#include <cctype>
13#include <filesystem>
14#include <fstream>
15#include <map>
16#include <set>
17
18using json = common_json;
19
20struct cli_context_impl {
21    json messages      = json::array();
22    json pending_media = json::array(); // staged multimodal content parts
23};
24
25cli_context::cli_context(const common_params & params) : params(params), impl(new cli_context_impl()) {}
26
27cli_context::~cli_context() {
28    shutdown();
29}
30
31std::atomic<bool> & cli_context::interrupted() {
32    static std::atomic<bool> flag = false;
33    return flag;
34}
35
36static bool should_stop() {
37    return cli_context::interrupted().load();
38}
39
40static constexpr size_t FILE_GLOB_MAX_RESULTS = 100;
41
42const char * LLAMA_ASCII_LOGO = R"(
43▄▄ ▄▄
44██ ██
45██ ██  ▀▀█▄ ███▄███▄  ▀▀█▄    ▄████ ████▄ ████▄
46██ ██ ▄█▀██ ██ ██ ██ ▄█▀██    ██    ██ ██ ██ ██
47██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
48                                    ██    ██
49                                    ▀▀    ▀▀
50)";
51
52// number of values an arg consumes on the command line
53static int arg_num_values(const common_arg & opt) {
54    if (opt.value_hint_2 != nullptr) {
55        return 2;
56    }
57    if (opt.value_hint != nullptr) {
58        return 1;
59    }
60    return 0;
61}
62
63static std::string format_error_message(const json & err) {
64    if (err.contains("error") && err.at("error").is_object()) {
65        const auto & e = err.at("error");
66        if (e.contains("message") && e.at("message").is_string()) {
67            return e.at("message").get<std::string>();
68        }
69    }
70    return err.dump();
71}
72
73// err is the raw response body of a failed request; it may or may not be JSON
74static std::string format_error_message(const std::string & err) {
75    json parsed = json::parse_no_throw(err);
76    if (!parsed.is_discarded()) {
77        return format_error_message(parsed);
78    }
79    return err;
80}
81
82static std::string media_type_from_ext(const std::string & fname) {
83    std::string ext = std::filesystem::path(fname).extension().string();
84    std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
85    if (ext == ".wav" || ext == ".mp3") {
86        return "audio";
87    }
88    if (ext == ".mp4" || ext == ".avi" || ext == ".mkv" || ext == ".mov" || ext == ".webm") {
89        return "video";
90    }
91    return "image";
92}
93
94bool cli_context::init() {
95    ui::init(params);
96
97    std::optional<ui::spinner> spinner;
98
99    bool use_external_server = !params.server_base.empty();
100    if (use_external_server) {
101        std::string base = params.server_base;
102        while (!base.empty() && base.back() == '/') {
103            base.pop_back();
104        }
105        client.server_base = base;
106
107        spinner.emplace("Connecting to server at " + base);
108    } else {
109        if (params.model.path.empty() && params.model.url.empty() &&
110                params.model.hf_repo.empty() && params.model.docker_repo.empty()) {
111            ui::show_error(
112                "no model specified",
113                "use -m <file.gguf> or -hf <user/repo> to run a local model,\n"
114                "or --server-base <url> to connect to a running llama-server"
115            );
116            return false;
117        }
118
119        spinner.emplace("\n\nLoading model...");
120
121        server.emplace();
122        if (!server->start(params)) {
123            ui::show_error("server start failed");
124            return false;
125        }
126        if (!server->wait_ready(should_stop)) {
127            if (!should_stop()) {
128                ui::show_error("the server exited before becoming ready");
129            }
130            return false;
131        }
132        client.server_base = server->address();
133    }
134
135    // for --server-base this is the main availability check; for a spawned
136    // server it is a cheap sanity check on top of the ready signal
137    auto is_aborted = [this]() {
138        return should_stop() || (server && !server->alive());
139    };
140    bool healthy = false;
141    try {
142        healthy = client.wait_health(is_aborted);
143    } catch (const std::exception & e) {
144        client.last_error = e.what();
145    }
146    if (!healthy) {
147        if (!should_stop()) {
148            ui::show_error(client.last_error);
149        }
150        return false;
151    }
152
153    if (use_external_server) {
154        spinner.reset();
155        try {
156            if (!list_and_ask_models()) {
157                return false;
158            }
159        } catch (const common_json_error & e) {
160            ui::show_error(e.what());
161            ui::show_message("This might be caused by an incorrect server-base endpoint URL");
162            return false;
163        } catch (const std::exception & e) {
164            ui::show_error(e.what());
165            return false;
166        }
167
168        // restore the spinner for the next step
169        spinner.emplace("Waiting for server...");
170    }
171
172    fetch_server_props();
173
174    if (!params.out_file.empty()) {
175        output_file.emplace(params.out_file);
176        if (!output_file->is_open()) {
177            ui::show_error(string_format("failed to open output file '%s'", params.out_file.c_str()));
178            return false;
179        }
180    }
181
182    return true;
183}
184
185void cli_context::fetch_server_props() {
186    try {
187        json props = json::parse(client.get("/props"));
188        model_name = props.value("model_alias", "");
189        if (model_name.empty()) {
190            const std::string path = props.value("model_path", "");
191            if (!path.empty()) {
192                model_name = std::filesystem::path(path).filename().string();
193            }
194        }
195        model_ftype = props.value("model_ftype", "");
196        build_info = props.value("build_info", "");
197        if (props.contains("modalities") && props.at("modalities").is_object()) {
198            const auto & modalities = props.at("modalities");
199            has_vision = modalities.value("vision", false);
200            has_audio  = modalities.value("audio", false);
201            has_video  = modalities.value("video", false);
202        }
203    } catch (const std::exception & e) {
204        // /props can be disabled on remote servers; not fatal
205        LOG_DBG("failed to fetch /props: %s\n", e.what());
206    }
207}
208
209bool cli_context::list_and_ask_models() {
210    json resp = json::parse(client.get("/v1/models"));
211    if (!resp.contains("data") || !resp.at("data").is_array()) {
212        throw std::runtime_error("invalid response from /v1/models");
213    }
214    std::vector<std::string> models;
215    std::vector<std::string> models_display;
216    for (const auto & m : resp.at("data")) {
217        if (!m.contains("id") || !m.at("id").is_string()) {
218            continue;
219        }
220        std::string name = m.at("id").get<std::string>();
221        std::string display = name;
222        if (m.contains("aliases") && m.at("aliases").is_array()) {
223            std::vector<std::string> aliases;
224            for (const auto & a : m.at("aliases")) {
225                if (a.is_string()) {
226                    aliases.push_back(a.get<std::string>());
227                }
228            }
229            if (!aliases.empty()) {
230                display += " (" + string_join(aliases, ", ") + ")";
231            }
232        }
233        models.push_back(name);
234        models_display.push_back(display);
235    }
236
237    // only one model: use it without asking
238    if (models.size() == 1) {
239        model_name = models[0];
240        client.model = model_name;
241        return true;
242    }
243
244    std::string message = "\nAvailable models:";
245    for (size_t i = 0; i < models_display.size(); ++i) {
246        message += "\n  " + std::to_string(i + 1) + ". " + models_display[i];
247    }
248    message += "\n";
249    ui::show_message(message);
250    std::string selection;
251    while (selection.empty()) {
252        if (should_stop()) {
253            return false;
254        }
255        ui::user_turn user_turn;
256        selection = user_turn.read_input(false, "Select model by number: ");
257        if (selection.empty()) {
258            continue;
259        }
260        try {
261            size_t idx = std::stoul(selection);
262            if (idx > 0 && idx <= models.size()) {
263                model_name = models[idx - 1];
264                client.model = model_name;
265                ui::show_message("Selected model: " + model_name);
266                break;
267            }
268        } catch (...) {
269            // ignore
270        }
271        ui::show_error("Invalid selection. Please enter a valid number.");
272        selection.clear();
273        continue;
274    }
275    return true;
276}
277
278void cli_context::add_system_prompt() {
279    if (!params.system_prompt.empty()) {
280        impl->messages.push_back({
281            {"role",    "system"},
282            {"content", params.system_prompt}
283        });
284    }
285}
286
287void cli_context::push_user_message(const std::string & text) {
288    json content;
289    if (impl->pending_media.empty()) {
290        content = text;
291    } else {
292        // multimodal message: media parts first, then the text
293        content = impl->pending_media;
294        content.push_back({
295            {"type", "text"},
296            {"text", text}
297        });
298        impl->pending_media = json::array();
299    }
300    impl->messages.push_back({
301        {"role",    "user"},
302        {"content", content}
303    });
304}
305
306bool cli_context::stage_media_file(const std::string & fname, const std::string & type) {
307    std::ifstream file(fname, std::ios::binary);
308    if (!file) {
309        return false;
310    }
311    std::string data((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
312    std::string encoded = base64::encode(data);
313
314    if (type == "audio") {
315        std::string ext = std::filesystem::path(fname).extension().string();
316        std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
317        impl->pending_media.push_back({
318            {"type", "input_audio"},
319            {"input_audio", {
320                {"data",   encoded},
321                {"format", ext == ".mp3" ? "mp3" : "wav"}
322            }}
323        });
324    } else if (type == "video") {
325        impl->pending_media.push_back({
326            {"type", "input_video"},
327            {"input_video", {
328                {"data", encoded}
329            }}
330        });
331    } else {
332        // the server detects the actual image type from the data
333        impl->pending_media.push_back({
334            {"type", "image_url"},
335            {"image_url", {
336                {"url", "data:image/unknown;base64," + encoded}
337            }}
338        });
339    }
340    return true;
341}
342
343void cli_context::write_output_file(const std::string & content) {
344    if (output_file) {
345        (*output_file) << content;
346        output_file->flush();
347    }
348}
349
350bool cli_context::generate_completion(generated_content & content_out, cli_timings & timings) {
351    json body = {
352        {"messages",          impl->messages},
353        {"stream",            true},
354        // in order to get timings even when we cancel mid-way
355        {"timings_per_token", true},
356    };
357    if (!client.model.empty()) {
358        body["model"] = client.model;
359    }
360
361    bool stream_error = false;
362
363    ui::assistant_turn a;
364
365    std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) {
366        json chunk = json::parse_no_throw(payload);
367        if (chunk.is_discarded()) {
368            return;
369        }
370        if (chunk.contains("error")) {
371            stream_error = true;
372            ui::show_error(format_error_message(chunk));
373            return;
374        }
375        if (chunk.contains("timings")) {
376            const auto & t = chunk.at("timings");
377            timings.prompt_per_second    = t.value("prompt_per_second",    0.0);
378            timings.predicted_per_second = t.value("predicted_per_second", 0.0);
379        }
380        if (!chunk.contains("choices") || !chunk.at("choices").is_array() || chunk.at("choices").empty()) {
381            return;
382        }
383        const auto & choice = chunk.at("choices").at(0);
384        if (!choice.contains("delta")) {
385            return;
386        }
387        const auto & delta = choice.at("delta");
388        if (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) {
389            const std::string text = delta.at("reasoning_content").get<std::string>();
390            if (!text.empty()) {
391                content_out.reasoning += text;
392                a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text);
393            }
394        }
395        if (delta.contains("content") && delta.at("content").is_string()) {
396            const std::string text = delta.at("content").get<std::string>();
397            if (!text.empty()) {
398                content_out.content += text;
399                a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text);
400            }
401        }
402    });
403
404    cli_context::interrupted().store(false);
405
406    if (!err.empty()) {
407        ui::show_error(format_error_message(err));
408        return false;
409    }
410    return !stream_error;
411}
412
413int cli_context::run() {
414    add_system_prompt();
415
416    std::string modalities = "text";
417    if (has_vision) {
418        modalities += ", vision";
419    }
420    if (has_audio) {
421        modalities += ", audio";
422    }
423    if (has_video) {
424        modalities += ", video";
425    }
426
427    std::string banner;
428    banner += "\n";
429    banner += LLAMA_ASCII_LOGO;
430    banner += "\n";
431    banner += "build      : " + build_info + "\n";
432    banner += "model      : " + model_name + "\n";
433    if (!model_ftype.empty()) {
434        banner += "ftype      : " + model_ftype + "\n";
435    }
436    banner += "modalities : " + modalities + "\n";
437    if (!params.system_prompt.empty()) {
438        banner += "using custom system prompt\n";
439    }
440    banner += "\n";
441    banner += "available commands:\n";
442    banner += "  /exit or Ctrl+C     stop or exit\n";
443    banner += "  /regen              regenerate the last response\n";
444    banner += "  /clear              clear the chat history\n";
445    banner += "  /read <file>        add a text file\n";
446    banner += "  /glob <pattern>     add text files using globbing pattern\n";
447    if (has_vision) {
448        banner += "  /image <file>       add an image file\n";
449    }
450    if (has_audio) {
451        banner += "  /audio <file>       add an audio file\n";
452    }
453    if (has_video) {
454        banner += "  /video <file>       add a video file\n";
455    }
456    banner += "\n";
457
458    ui::show_message(banner);
459
460    // interactive loop
461    std::string cur_msg;
462
463    auto add_text_file = [&](const std::string & fname) -> bool {
464        std::ifstream file(fname, std::ios::binary);
465        if (!file) {
466            ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
467            return false;
468        }
469        std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
470        cur_msg += "--- File: ";
471        cur_msg += fname;
472        cur_msg += " ---\n";
473        cur_msg += content;
474        ui::show_message(string_format("Loaded text from '%s'", fname.c_str()));
475        return true;
476    };
477
478    while (true) {
479        std::string buffer;
480        {
481            ui::user_turn user_turn;
482
483            if (params.prompt.empty()) {
484                buffer = user_turn.read_input(params.multiline_input);
485            } else {
486                // process input prompt from args
487                for (auto & fname : params.image) {
488                    if (!stage_media_file(fname, media_type_from_ext(fname))) {
489                        ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
490                        break;
491                    }
492                    ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
493                }
494                buffer = params.prompt;
495                user_turn.echo(buffer);
496                params.prompt.clear(); // only use it once
497            }
498        }
499
500        if (should_stop()) {
501            cli_context::interrupted().store(false);
502            break;
503        }
504
505        // remove trailing newline
506        if (!buffer.empty() && buffer.back() == '\n') {
507            buffer.pop_back();
508        }
509
510        // skip empty messages
511        if (buffer.empty()) {
512            continue;
513        }
514
515        bool add_user_msg = true;
516
517        // process commands
518        if (string_starts_with(buffer, "/exit")) {
519            break;
520        } else if (string_starts_with(buffer, "/regen")) {
521            if (impl->messages.size() >= 2) {
522                size_t last_idx = impl->messages.size() - 1;
523                impl->messages.erase(last_idx);
524                add_user_msg = false;
525            } else {
526                ui::show_error("No message to regenerate.");
527                continue;
528            }
529        } else if (string_starts_with(buffer, "/clear")) {
530            impl->messages.clear();
531            add_system_prompt();
532
533            impl->pending_media = json::array();
534            ui::show_message("Chat history cleared.");
535            continue;
536        } else if (
537                (string_starts_with(buffer, "/image ") && has_vision) ||
538                (string_starts_with(buffer, "/audio ") && has_audio) ||
539                (string_starts_with(buffer, "/video ") && has_video)) {
540            std::string type = buffer.substr(1, 5);
541            // just in case (bad copy-paste for example), we strip all trailing/leading spaces
542            std::string fname = string_strip(buffer.substr(7));
543            if (!stage_media_file(fname, type)) {
544                ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));
545                continue;
546            }
547            ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));
548            write_output_file(string_format("User: Added media: %s\n", fname.c_str()));
549            continue;
550        } else if (string_starts_with(buffer, "/read ")) {
551            std::string fname = string_strip(buffer.substr(6));
552            add_text_file(fname);
553            write_output_file(string_format("User: Added text file: %s\n", fname.c_str()));
554            continue;
555        } else if (string_starts_with(buffer, "/glob ")) {
556            std::error_code ec;
557            size_t count = 0;
558            auto curdir = std::filesystem::current_path();
559            std::string pattern = string_strip(buffer.substr(6));
560            std::filesystem::path rel_path;
561
562            auto startglob = pattern.find_first_of("![*?");
563            if (startglob != std::string::npos && startglob != 0) {
564                auto endpath = pattern.substr(0, startglob).find_last_of('/');
565                if (endpath != std::string::npos) {
566                    std::string rel_pattern = pattern.substr(0, endpath);
567#if !defined(_WIN32)
568                    if (string_starts_with(rel_pattern, '~')) {
569                        const char * home = std::getenv("HOME");
570                        if (home && home[0]) {
571                            rel_pattern = home + rel_pattern.substr(1);
572                        }
573                    }
574#endif
575                    rel_path = rel_pattern;
576                    pattern.erase(0, endpath + 1);
577                    curdir /= rel_path;
578                }
579            }
580
581            for (const auto & entry : std::filesystem::recursive_directory_iterator(curdir,
582                    std::filesystem::directory_options::skip_permission_denied, ec)) {
583                if (!entry.is_regular_file()) {
584                    continue;
585                }
586
587                std::string rel = std::filesystem::relative(entry.path(), curdir, ec).string();
588                if (ec) {
589                    ec.clear();
590                    continue;
591                }
592                std::replace(rel.begin(), rel.end(), '\\', '/');
593
594                if (!glob_match(pattern, rel)) {
595                    continue;
596                }
597
598                const std::string full_path = (curdir / rel).string();
599                if (!add_text_file(full_path)) {
600                    continue;
601                }
602                write_output_file(string_format("User: Added text file: %s\n", full_path.c_str()));
603
604                if (++count >= FILE_GLOB_MAX_RESULTS) {
605                    ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS));
606                    break;
607                }
608            }
609            continue;
610        } else {
611            // not a command
612            cur_msg += buffer;
613        }
614
615        // generate response
616        if (add_user_msg) {
617            push_user_message(cur_msg);
618            write_output_file(string_format("User:\n%s\n\n", cur_msg.c_str()));
619            cur_msg.clear();
620        }
621
622        cli_timings timings;
623        generated_content content;
624        generate_completion(content, timings);
625
626        json assistant_msg = {
627            {"role",    "assistant"},
628            {"content", content.content}
629        };
630        if (!content.reasoning.empty()) {
631            assistant_msg["reasoning_content"] = content.reasoning;
632        }
633        impl->messages.push_back(std::move(assistant_msg));
634
635        if (output_file) {
636            std::string out_content = "Assistant:\n";
637            if (!content.reasoning.empty()) {
638                out_content += "[Start thinking]\n\n";
639                out_content += content.reasoning;
640                out_content += "[End thinking]\n\n";
641            }
642            out_content += content.content;
643            if (!out_content.empty() && out_content.back() != '\n') {
644                out_content += "\n";
645            }
646            out_content += "\n";
647            write_output_file(out_content);
648        }
649
650        if (params.show_timings) {
651            ui::show_info(string_format(
652                "\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]",
653                timings.prompt_per_second,
654                timings.predicted_per_second
655            ));
656        }
657
658        if (params.single_turn) {
659            break;
660        }
661    }
662
663    ui::show_message("\n\nExiting...");
664
665    return 0;
666}
667
668void cli_context::shutdown() {
669    if (server) {
670        server->stop();
671        server.reset();
672    }
673    if (output_file) {
674        output_file->close();
675        output_file.reset();
676    }
677}