char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: add mtmd_tokenize_from_parts() (#28250)733905474

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
20.4 KiB587 linesraw
1#include "arg.h"
2#include "debug.h"
3#include "log.h"
4#include "common.h"
5#include "sampling.h"
6#include "llama.h"
7#include "ggml.h"
8#include "console.h"
9#include "chat.h"
10#include "mtmd.h"
11#include "mtmd-helper.h"
12
13#include <vector>
14#include <limits.h>
15#include <cinttypes>
16#include <clocale>
17
18#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
19#include <signal.h>
20#include <unistd.h>
21#elif defined (_WIN32)
22#define WIN32_LEAN_AND_MEAN
23#ifndef NOMINMAX
24#define NOMINMAX
25#endif
26#include <windows.h>
27#include <signal.h>
28#endif
29
30// volatile, because of signal being an interrupt
31static volatile bool g_is_generating = false;
32static volatile bool g_is_interrupted = false;
33
34/**
35 * Please note that this is NOT a production-ready binary.
36 * It is a playground for trying multimodal support in llama.cpp.
37 * For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing multimodal support.
38 */
39
40static void show_additional_info(int /*argc*/, char ** argv) {
41    LOG(
42        "Experimental CLI for multimodal\n\n"
43        "Usage: %s [options] -m <model> --mmproj <mmproj> --image <image> --audio <audio> -p <prompt>\n\n"
44        "  -m and --mmproj are required\n"
45        "  -hf user/repo can replace both -m and --mmproj in most cases\n"
46        "  --image, --audio and -p are optional, if NOT provided, the CLI will run in chat mode\n"
47        "  to disable using GPU for mmproj model, add --no-mmproj-offload\n",
48        argv[0]
49    );
50}
51
52#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
53static void sigint_handler(int signo) {
54    if (signo == SIGINT) {
55        if (g_is_generating) {
56            g_is_generating = false;
57        } else {
58            console::cleanup();
59            if (g_is_interrupted) {
60                _exit(1);
61            }
62            g_is_interrupted = true;
63        }
64    }
65}
66#endif
67
68// this is only used by tests.sh to capture the response ; it's not meant to be used in production
69static void inject_test_response_marker() {
70    const char * env = std::getenv("MTMD_TEST_RESPONSE_MARKER");
71    if (env) {
72        LOG("%s\n", env);
73    }
74}
75
76struct mtmd_cli_context {
77    mtmd::context_ptr ctx_vision;
78    common_init_result_ptr llama_init;
79
80    llama_model       * model;
81    llama_context     * lctx;
82    const llama_vocab * vocab;
83    common_sampler    * smpl;
84    llama_batch         batch;
85    int                 n_batch;
86
87    mtmd::bitmaps bitmaps;
88    std::vector<mtmd_helper::video_ptr> videos;
89
90    mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
91    std::string video_ffmpeg_bin_dir;
92
93    mtmd::batch_ptr mbatch;
94
95    // chat template
96    common_chat_templates_ptr tmpls;
97    std::vector<common_chat_msg> chat_history;
98    bool use_jinja = false;
99    // TODO: support for --system-prompt with /clear command
100
101    // support for legacy templates (models not having EOT token)
102    llama_tokens antiprompt_tokens;
103
104    int n_threads    = 1;
105    llama_pos n_past = 0;
106
107    common_debug_cb_user_data cb_data;
108
109    mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {
110        model = llama_init->model();
111        lctx = llama_init->context();
112        if (!model || !lctx) {
113            exit(1);
114        }
115        vocab = llama_model_get_vocab(model);
116        smpl = common_sampler_init(model, params.sampling);
117        n_threads = params.cpuparams.n_threads;
118        batch = llama_batch_init(1, 0, 1); // batch for next token generation
119        n_batch = params.n_batch;
120
121        init_vision_context(params);
122
123        if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
124            LOG_ERR("Model does not support chat mode\n");
125            LOG_ERR("Hint: for TTS models, please use llama-tts\n");
126            exit(1);
127        }
128
129        if (!llama_model_chat_template(model, nullptr) && params.chat_template.empty()) {
130            LOG_ERR("Model does not have chat template.\n");
131            LOG_ERR("  For old llava models, you may need to use '--chat-template vicuna'\n");
132            LOG_ERR("  For MobileVLM models, use '--chat-template deepseek'\n");
133            LOG_ERR("  For Mistral Small 3.1, use '--chat-template mistral-v7'\n");
134            exit(1);
135        }
136
137        tmpls = common_chat_templates_init(model, params.chat_template);
138        use_jinja = params.use_jinja;
139        chat_history.clear();
140        LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str());
141
142        // load antiprompt tokens for legacy templates
143        if (params.chat_template == "vicuna") {
144            antiprompt_tokens = common_tokenize(lctx, "ASSISTANT:", false, true);
145        } else if (params.chat_template == "deepseek") {
146            antiprompt_tokens = common_tokenize(lctx, "###", false, true);
147        }
148    }
149
150    ~mtmd_cli_context() {
151        llama_batch_free(batch);
152        common_sampler_free(smpl);
153    }
154
155    void init_vision_context(common_params & params) {
156        const char * clip_path = params.mmproj.path.c_str();
157        mtmd_context_params mparams = mtmd_context_params_default();
158        mparams.use_gpu          = params.mmproj_use_gpu;
159        mparams.device           = params.mmproj_device;
160        mparams.print_timings    = true;
161        mparams.n_threads        = params.cpuparams.n_threads;
162        mparams.flash_attn_type  = params.flash_attn_type;
163        mparams.warmup           = params.warmup;
164        mparams.image_min_tokens = params.image_min_tokens;
165        mparams.image_max_tokens = params.image_max_tokens;
166        if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {
167            mparams.cb_eval_user_data = &cb_data;
168            mparams.cb_eval = common_debug_cb_eval;
169        }
170        ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams));
171        if (!ctx_vision.get()) {
172            LOG_ERR("Failed to load vision model from %s\n", clip_path);
173            exit(1);
174        }
175
176        video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;
177        init_opt.video_params.fps_target = params.video_fps;
178        init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;
179        init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()
180                            ? nullptr : video_ffmpeg_bin_dir.c_str();
181    }
182
183    bool check_antiprompt(const llama_tokens & generated_tokens) {
184        if (antiprompt_tokens.empty() || generated_tokens.size() < antiprompt_tokens.size()) {
185            return false;
186        }
187        return std::equal(
188            generated_tokens.end() - antiprompt_tokens.size(),
189            generated_tokens.end(),
190            antiprompt_tokens.begin()
191        );
192    }
193
194    bool load_media(const std::string & fname) {
195        auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);
196        if (!res.bitmap) {
197            return false;
198        }
199        bitmaps.entries.emplace_back(res.bitmap);
200        if (res.video_ctx) {
201            videos.emplace_back(res.video_ctx);
202        }
203        return true;
204    }
205};
206
207static int generate_response(mtmd_cli_context & ctx, int n_predict) {
208    llama_tokens generated_tokens;
209    for (int i = 0; i < n_predict; i++) {
210        if (i > n_predict || !g_is_generating || g_is_interrupted) {
211            LOG("\n");
212            break;
213        }
214
215        llama_token token_id = common_sampler_sample(ctx.smpl, ctx.lctx, -1);
216        generated_tokens.push_back(token_id);
217        common_sampler_accept(ctx.smpl, token_id, true);
218
219        if (llama_vocab_is_eog(ctx.vocab, token_id) || ctx.check_antiprompt(generated_tokens)) {
220            LOG("\n");
221            break; // end of generation
222        }
223
224        LOG("%s", common_token_to_piece(ctx.lctx, token_id).c_str());
225        fflush(stdout);
226
227        if (g_is_interrupted) {
228            LOG("\n");
229            break;
230        }
231
232        // eval the token
233        common_batch_clear(ctx.batch);
234        common_batch_add(ctx.batch, token_id, ctx.n_past++, {0}, true);
235        if (llama_decode(ctx.lctx, ctx.batch)) {
236            LOG_ERR("failed to decode token\n");
237            return 1;
238        }
239    }
240
241    std::string generated_text = common_detokenize(ctx.lctx, generated_tokens);
242    common_chat_msg msg;
243    msg.role    = "assistant";
244    msg.content = generated_text;
245    ctx.chat_history.push_back(std::move(msg));
246
247    return 0;
248}
249
250static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg & new_msg) {
251    LOG_DBG("chat_add_and_format: new_msg.role='%s', new_msg.content='%s'\n",
252        new_msg.role.c_str(), new_msg.content.c_str());
253    auto formatted = common_chat_format_single(ctx.tmpls.get(), ctx.chat_history,
254        new_msg, new_msg.role == "user",
255        ctx.use_jinja);
256    ctx.chat_history.push_back(new_msg);
257    return formatted;
258}
259
260static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
261    inject_test_response_marker();
262
263    bool add_bos = ctx.chat_history.empty();
264    auto formatted_chat = chat_add_and_format(ctx, msg);
265    LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
266
267    if (g_is_interrupted) return 0;
268
269    // note: we replace the marker here instead of letting mtmd_tokenize() to do that
270    //       because we want to demonstrate how to use mtmd_tokenize_from_parts()
271
272    // split the formatted chat on the media marker to get text segments
273    const std::string marker = mtmd_default_marker();
274    std::vector<std::string> segments;
275    size_t start = 0;
276    size_t pos;
277    while ((pos = formatted_chat.find(marker, start)) != std::string::npos) {
278        segments.push_back(formatted_chat.substr(start, pos - start));
279        start = pos + marker.size();
280    }
281    segments.push_back(formatted_chat.substr(start));
282
283    auto bitmaps_c_ptr = ctx.bitmaps.c_ptr();
284    if (segments.size() - 1 != bitmaps_c_ptr.size()) {
285        LOG_ERR("Number of media markers (%zu) does not match number of loaded media (%zu)\n",
286                segments.size() - 1, bitmaps_c_ptr.size());
287        return 1;
288    }
289
290    // interleave text and media parts
291    std::vector<mtmd_input_text> texts(segments.size());
292    std::vector<mtmd_input_part> parts;
293    for (size_t i = 0; i < segments.size(); i++) {
294        texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true};
295        parts.push_back({&texts[i], nullptr});
296        if (i < bitmaps_c_ptr.size()) {
297            parts.push_back({nullptr, bitmaps_c_ptr[i]});
298        }
299    }
300    std::vector<const mtmd_input_part *> parts_ptr;
301    for (const auto & p : parts) {
302        parts_ptr.push_back(&p);
303    }
304
305    mtmd::input_chunks chunks(mtmd_input_chunks_init());
306    int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(),
307                        chunks.ptr.get(), // output
308                        parts_ptr.data(),
309                        parts_ptr.size(),
310                        add_bos);
311    if (res != 0) {
312        LOG_ERR("Unable to tokenize prompt, res = %d\n", res);
313        return 1;
314    }
315
316    ctx.bitmaps.entries.clear();
317    ctx.videos.clear();
318
319    // batch encode all media chunks, then decode each
320    size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get());
321    for (size_t i = 0; i < n_chunks; i++) {
322        auto chunk = mtmd_input_chunks_get(chunks.ptr.get(), i);
323        auto chunk_type = mtmd_input_chunk_get_type(chunk);
324
325        if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
326            // decode text chunk
327            llama_pos new_n_past = ctx.n_past;
328            res = mtmd_helper_eval_chunk_single(ctx.ctx_vision.get(),
329                        ctx.lctx,
330                        chunk,
331                        ctx.n_past,
332                        0, // seq_id
333                        ctx.n_batch,
334                        i == n_chunks - 1, // logits_last
335                        &new_n_past);
336            if (res != 0) {
337                LOG_ERR("Unable to eval text chunk %zu\n", i);
338                return 1;
339            }
340            ctx.n_past = new_n_past;
341        } else {
342            // media chunk: try to get embd from existing batch, or create a new batch
343            float * embd = nullptr;
344            if (ctx.mbatch) {
345                embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
346
347                if (embd) {
348                    LOG_DBG("found embd for media chunk %zu in existing batch\n", i);
349                } else {
350                    LOG_DBG("media chunk %zu not found in existing batch, creating new batch\n", i);
351                }
352            }
353
354            if (!embd) {
355                // create and encode a new batch with as many media chunks as possible
356                ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get()));
357                res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk);
358                GGML_ASSERT(res == 0); // first chunk must always succeed
359
360                int n_added = 1;
361                // add as many subsequent media chunks as possible
362                for (size_t j = i + 1; j < n_chunks; j++) {
363                    auto next_chunk = mtmd_input_chunks_get(chunks.ptr.get(), j);
364                    auto next_type = mtmd_input_chunk_get_type(next_chunk);
365                    if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
366                        break; // text chunk splits the batch
367                    }
368                    res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk);
369                    if (res != 0) {
370                        break; // batch full or incompatible
371                    }
372                    n_added++;
373                }
374
375                int64_t time_start = ggml_time_ms();
376                LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks);
377                res = mtmd_batch_encode(ctx.mbatch.get());
378                if (res != 0) {
379                    LOG_ERR("Failed to encode mtmd batch, res = %d\n", res);
380                    return 1;
381                }
382                LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start));
383
384                embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
385            }
386
387            GGML_ASSERT(embd != nullptr);
388
389            llama_pos new_n_past = ctx.n_past;
390            res = mtmd_helper_decode_image_chunk(ctx.ctx_vision.get(),
391                        ctx.lctx,
392                        chunk,
393                        embd,
394                        ctx.n_past,
395                        0, // seq_id
396                        ctx.n_batch,
397                        &new_n_past,
398                        nullptr, // callback
399                        nullptr  // user_data
400                    );
401            if (res != 0) {
402                LOG_ERR("Unable to decode media chunk %zu\n", i);
403                return 1;
404            }
405            ctx.n_past = new_n_past;
406        }
407    }
408
409    LOG("\n");
410
411    return 0;
412}
413
414int main(int argc, char ** argv) {
415    std::setlocale(LC_NUMERIC, "C");
416
417    ggml_time_init();
418
419    common_params params;
420
421    common_init();
422
423    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {
424        return 1;
425    }
426
427    mtmd_helper_log_set(common_log_default_callback, nullptr);
428
429    if (params.mmproj.path.empty()) {
430        show_additional_info(argc, argv);
431        LOG_ERR("ERR: Missing --mmproj argument\n");
432        return 1;
433    }
434
435    ggml_backend_load_all();
436
437    mtmd_cli_context ctx(params);
438    LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());
439
440    bool is_single_turn = !params.prompt.empty() && !params.image.empty();
441
442    int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict;
443
444    console::init(params.simple_io, params.use_color);
445    atexit([]() { console::cleanup(); });
446
447    // Ctrl+C handling
448    {
449#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
450        struct sigaction sigint_action;
451        sigint_action.sa_handler = sigint_handler;
452        sigemptyset (&sigint_action.sa_mask);
453        sigint_action.sa_flags = 0;
454        sigaction(SIGINT, &sigint_action, NULL);
455#elif defined (_WIN32)
456        auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
457            return (ctrl_type == CTRL_C_EVENT) ? (sigint_handler(SIGINT), true) : false;
458        };
459        SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
460#endif
461    }
462
463    if (g_is_interrupted) return 130;
464
465    auto eval_system_prompt_if_present = [&] {
466        if (params.system_prompt.empty()) {
467            return 0;
468        }
469
470        common_chat_msg msg;
471        msg.role = "system";
472        msg.content = params.system_prompt;
473        return eval_message(ctx, msg);
474    };
475
476    LOG_WRN("WARN: This is an experimental CLI for testing multimodal capability.\n");
477    LOG_WRN("      For normal use cases, please use the standard llama-cli\n");
478
479    if (eval_system_prompt_if_present()) {
480        return 1;
481    }
482
483    if (is_single_turn) {
484        g_is_generating = true;
485        if (params.prompt.find(mtmd_default_marker()) == std::string::npos) {
486            for (size_t i = 0; i < params.image.size(); i++) {
487                // most models require the marker before each image
488                // ref: https://github.com/ggml-org/llama.cpp/pull/17616
489                params.prompt = mtmd_default_marker() + params.prompt;
490            }
491        }
492
493        common_chat_msg msg;
494        msg.role = "user";
495        msg.content = params.prompt;
496        for (const auto & image : params.image) {
497            if (!ctx.load_media(image)) {
498                return 1; // error is already printed by libmtmd
499            }
500        }
501        if (eval_message(ctx, msg)) {
502            return 1;
503        }
504        if (!g_is_interrupted && generate_response(ctx, n_predict)) {
505            return 1;
506        }
507
508    } else {
509        LOG("\n Running in chat mode, available commands:");
510        if (mtmd_support_vision(ctx.ctx_vision.get())) {
511            LOG("\n   /image <path>    load an image");
512        }
513        if (mtmd_support_audio(ctx.ctx_vision.get())) {
514            LOG("\n   /audio <path>    load an audio");
515        }
516        if (mtmd_helper_support_video(ctx.ctx_vision.get())) {
517            LOG("\n   /video <path>    load a video");
518        }
519        LOG("\n   /clear           clear the chat history");
520        LOG("\n   /quit or /exit   exit the program");
521        LOG("\n");
522
523        std::string content;
524
525        while (!g_is_interrupted) {
526            g_is_generating = false;
527            LOG("\n> ");
528            console::set_display(DISPLAY_TYPE_USER_INPUT);
529            std::string line;
530            console::readline(line, false);
531            if (g_is_interrupted) break;
532            console::set_display(DISPLAY_TYPE_RESET);
533            line = string_strip(line);
534            if (line.empty()) {
535                continue;
536            }
537            if (line == "/quit" || line == "/exit") {
538                break;
539            }
540            if (line == "/clear") {
541                ctx.n_past = 0;
542                ctx.chat_history.clear();
543                llama_memory_clear(llama_get_memory(ctx.lctx), true);
544                if (eval_system_prompt_if_present()) {
545                    return 1;
546                }
547                LOG("Chat history cleared\n\n");
548                continue;
549            }
550            g_is_generating = true;
551            bool is_image = line == "/image" || line.find("/image ") == 0;
552            bool is_audio = line == "/audio" || line.find("/audio ") == 0;
553            bool is_video = line == "/video" || line.find("/video ") == 0;
554            if (is_image || is_audio || is_video) {
555                if (line.size() < 8) {
556                    LOG_ERR("ERR: Missing media filename\n");
557                    continue;
558                }
559                std::string media_path = line.substr(7);
560                if (ctx.load_media(media_path)) {
561                    LOG("%s %s loaded\n", media_path.c_str(), is_image ? "image" : is_audio ? "audio" : "video");
562                    content += mtmd_default_marker();
563                }
564                // else, error is already printed by libmtmd
565                continue;
566            } else {
567                content += line;
568            }
569            common_chat_msg msg;
570            msg.role = "user";
571            msg.content = content;
572            int ret = eval_message(ctx, msg);
573            if (ret) {
574                return 1;
575            }
576            if (g_is_interrupted) break;
577            if (generate_response(ctx, n_predict)) {
578                return 1;
579            }
580            content.clear();
581        }
582    }
583    if (g_is_interrupted) LOG("\nInterrupted by user\n");
584    LOG("\n\n");
585    llama_perf_context_print(ctx.lctx);
586    return g_is_interrupted ? 130 : 0;
587}