char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

John-Henry Limmtmd: add --mmproj-device argument (#23255)681c29d36

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.3 KiB294 linesraw
1#include "mtmd-debug.h"
2
3#include "arg.h"
4#include "debug.h"
5#include "log.h"
6#include "common.h"
7#include "llama.h"
8#include "ggml.h"
9#include "mtmd.h"
10#include "mtmd-helper.h"
11
12#include <vector>
13#include <cmath>
14#include <limits.h>
15#include <cinttypes>
16#include <clocale>
17
18// INTERNAL TOOL FOR DEBUGGING PURPOSES ONLY
19// NOT INTENDED FOR PUBLIC USE
20
21static void show_additional_info(int /*argc*/, char ** argv) {
22    LOG(
23        "Internal debugging tool for mtmd; See mtmd-debug.md for the pytorch equivalent code\n"
24        "Note: we repurpose some args from other examples, they will have different meaning here\n"
25        "\n"
26        "Usage: %s -m <model> --mmproj <mmproj> -p <mode> -n <size> --image <image> --audio <audio>\n"
27        "\n"
28        "    -n <size>: number of pixels per edge for image (always square image), or number of samples for audio\n"
29        "\n"
30        "    -p \"encode\" (debugging encode pass, default case):\n"
31        "        --image can be:\n"
32        "          \"white\", \"black\", \"gray\": filled 1.0f, 0.0f and 0.5f respectively\n"
33        "          \"red\", \"green\", \"blue\": filled with respective colors\n"
34        "          \"cb\": checkerboard pattern, alternate 1.0f and 0.0f\n"
35        "          \"rainbow\": raspberry-pi-like rainbow pattern\n"
36        "        --audio can be:\n"
37        "          \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
38        "          \"1010\": checkerboard pattern, alternate 1.0f and 0.0f\n"
39        "\n"
40        "    -p \"preproc\" (debugging preprocessing pass):\n"
41        "        --image can be:\n"
42        "          \"white\", \"black\", \"gray\": filled image with respective colors\n"
43        "          \"cb\": checkerboard pattern\n"
44        "        --audio can be:\n"
45        "          \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
46        "          \"440\": sine wave with 440 Hz frequency\n"
47        "\n",
48        argv[0]
49    );
50}
51
52int main(int argc, char ** argv) {
53    std::setlocale(LC_NUMERIC, "C");
54
55    ggml_time_init();
56
57    common_params params;
58
59    common_init();
60
61    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {
62        return 1;
63    }
64
65    mtmd_helper_log_set(common_log_default_callback, nullptr);
66
67    if (params.mmproj.path.empty()) {
68        show_additional_info(argc, argv);
69        LOG_ERR("ERR: Missing --mmproj argument\n");
70        return 1;
71    }
72
73    ggml_backend_load_all();
74
75    LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());
76
77    mtmd::context_ptr ctx_mtmd;
78    common_init_result_ptr llama_init;
79    common_debug_cb_user_data cb_data;
80
81    llama_init = common_init_from_params(params);
82    {
83        auto * model = llama_init->model();
84        const char * clip_path = params.mmproj.path.c_str();
85        mtmd_context_params mparams = mtmd_context_params_default();
86        mparams.use_gpu          = params.mmproj_use_gpu;
87        mparams.device           = params.mmproj_device;
88        mparams.print_timings    = true;
89        mparams.n_threads        = params.cpuparams.n_threads;
90        mparams.flash_attn_type  = params.flash_attn_type;
91        mparams.warmup           = params.warmup;
92        mparams.image_min_tokens = params.image_min_tokens;
93        mparams.image_max_tokens = params.image_max_tokens;
94        {
95            // always enable debug callback
96            mparams.cb_eval_user_data = &cb_data;
97            mparams.cb_eval = common_debug_cb_eval;
98        }
99        ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams));
100        if (!ctx_mtmd.get()) {
101            LOG_ERR("Failed to load vision model from %s\n", clip_path);
102            exit(1);
103        }
104    }
105
106    std::string input;
107    int32_t inp_size = params.n_predict;
108    if (params.image.empty()) {
109        LOG_ERR("ERR: At least one of --image or --audio must be specified\n");
110        return 1;
111    }
112    if (inp_size <= 0) {
113        LOG_ERR("ERR: Invalid size specified with -n, must be greater than 0\n");
114        return 1;
115    }
116    input = params.image[0];
117
118    if (params.prompt.empty() || params.prompt == "encode") {
119        std::vector<std::vector<float>> image;
120        std::vector<float> samples;
121
122        if (input == "black") {
123            for (int i = 0; i < inp_size; ++i) {
124                auto row = std::vector<float>(inp_size * 3, 0.0f);
125                image.push_back(row);
126            }
127        } else if (input == "white") {
128            for (int i = 0; i < inp_size; ++i) {
129                auto row = std::vector<float>(inp_size * 3, 1.0f);
130                image.push_back(row);
131            }
132        } else if (input == "gray") {
133            for (int i = 0; i < inp_size; ++i) {
134                auto row = std::vector<float>(inp_size * 3, 0.5f);
135                image.push_back(row);
136            }
137        } else if (input == "cb") {
138            for (int i = 0; i < inp_size; ++i) {
139                auto row = std::vector<float>(inp_size * 3, 0.0f);
140                image.push_back(row);
141            }
142            for (int y = 0; y < inp_size; ++y) {
143                for (int x = 0; x < inp_size; ++x) {
144                    float v = ((x + y) % 2) ? 0.0f : 1.0f;
145                    image[y][x * 3 + 0] = v;
146                    image[y][x * 3 + 1] = v;
147                    image[y][x * 3 + 2] = v;
148                }
149            }
150        } else if (input == "red") {
151            for (int i = 0; i < inp_size; ++i) {
152                auto row = std::vector<float>(inp_size * 3, 0.0f);
153                for (int j = 0; j < inp_size; ++j) {
154                    row[j * 3 + 0] = 1.0f;  // R channel
155                }
156                image.push_back(row);
157            }
158        } else if (input == "green") {
159            for (int i = 0; i < inp_size; ++i) {
160                auto row = std::vector<float>(inp_size * 3, 0.0f);
161                for (int j = 0; j < inp_size; ++j) {
162                    row[j * 3 + 1] = 1.0f;  // G channel
163                }
164                image.push_back(row);
165            }
166        } else if (input == "blue") {
167            for (int i = 0; i < inp_size; ++i) {
168                auto row = std::vector<float>(inp_size * 3, 0.0f);
169                for (int j = 0; j < inp_size; ++j) {
170                    row[j * 3 + 2] = 1.0f;  // B channel
171                }
172                image.push_back(row);
173            }
174        } else if (input == "rainbow") {
175            for (int i = 0; i < inp_size; ++i) {
176                image.push_back(std::vector<float>(inp_size * 3, 0.0f));
177            }
178            float cx = inp_size / 2.0f;
179            float cy = inp_size / 2.0f;
180            float max_dist = std::sqrt(cx * cx + cy * cy);
181            for (int y = 0; y < inp_size; ++y) {
182                for (int x = 0; x < inp_size; ++x) {
183                    float dx = x - cx;
184                    float dy = y - cy;
185                    float hue = std::atan2(dy, dx) / (2.0f * 3.14159265f);
186                    if (hue < 0) hue += 1.0f;
187                    float sat = std::sqrt(dx * dx + dy * dy) / max_dist;
188                    if (sat > 1.0f) sat = 1.0f;
189                    float h6 = hue * 6.0f;
190                    int i6 = (int)h6;
191                    float f = h6 - i6;
192                    float p = 1.0f - sat;
193                    float q = 1.0f - sat * f;
194                    float t = 1.0f - sat * (1.0f - f);
195                    float r, g, b;
196                    switch (i6 % 6) {
197                        case 0: r=1; g=t; b=p; break;
198                        case 1: r=q; g=1; b=p; break;
199                        case 2: r=p; g=1; b=t; break;
200                        case 3: r=p; g=q; b=1; break;
201                        case 4: r=t; g=p; b=1; break;
202                        default: r=1; g=p; b=q; break;
203                    }
204                    image[y][x * 3 + 0] = r;
205                    image[y][x * 3 + 1] = g;
206                    image[y][x * 3 + 2] = b;
207                }
208            }
209        } else if (input == "one") {
210            samples = std::vector<float>(inp_size, 1.0f);
211        } else if (input == "zero") {
212            samples = std::vector<float>(inp_size, 0.0f);
213        } else if (input == "half") {
214            samples = std::vector<float>(inp_size, 0.5f);
215        } else if (input == "1010") {
216            samples.resize(inp_size);
217            for (int i = 0; i < inp_size; ++i) {
218                samples[i] = (i % 2) ? 0.0f : 1.0f;
219            }
220        } else {
221            LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
222            show_additional_info(argc, argv);
223            return 1;
224        }
225
226        // run encode pass
227        LOG_INF("Running encode pass for input type: %s\n", input.c_str());
228        if (samples.size() > 0) {
229            LOG_INF("Input audio with %zu samples, type: %s\n", samples.size(), input.c_str());
230            mtmd_debug_encode_audio(ctx_mtmd.get(), samples);
231        } else {
232            LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
233            mtmd_debug_encode_image(ctx_mtmd.get(), image);
234        }
235
236    } else if (params.prompt == "preproc") {
237        std::vector<uint8_t> rgb_values;
238        std::vector<float> pcm_samples;
239
240        if (input == "black") {
241            rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 0);
242        } else if (input == "white") {
243            rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 255);
244        } else if (input == "gray") {
245            rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 128);
246        } else if (input == "cb") {
247            rgb_values.resize(inp_size * inp_size * 3);
248            for (int y = 0; y < inp_size; ++y) {
249                for (int x = 0; x < inp_size; ++x) {
250                    uint8_t v = ((x + y) % 2) ? 0 : 255;
251                    rgb_values[(y * inp_size + x) * 3 + 0] = v;
252                    rgb_values[(y * inp_size + x) * 3 + 1] = v;
253                    rgb_values[(y * inp_size + x) * 3 + 2] = v;
254                }
255            }
256        } else if (input == "one") {
257            pcm_samples = std::vector<float>(inp_size, 1.0f);
258        } else if (input == "zero") {
259            pcm_samples = std::vector<float>(inp_size, 0.0f);
260        } else if (input == "half") {
261            pcm_samples = std::vector<float>(inp_size, 0.5f);
262        } else if (input == "440") {
263            pcm_samples.resize(inp_size);
264            float freq = 440.0f;
265            float sample_rate = mtmd_get_audio_sample_rate(ctx_mtmd.get());
266            float pi = 3.14159265f;
267            for (int i = 0; i < inp_size; ++i) {
268                pcm_samples[i] = sinf(2 * pi * freq * i / sample_rate);
269            }
270        } else {
271            LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
272            show_additional_info(argc, argv);
273            return 1;
274        }
275
276        // run preprocessing pass
277        LOG_INF("Running preprocessing pass for input type: %s\n", input.c_str());
278        if (pcm_samples.size() > 0) {
279            LOG_INF("Input audio with %zu samples, type: %s\n", pcm_samples.size(), input.c_str());
280            mtmd_debug_preprocess_audio(ctx_mtmd.get(), pcm_samples);
281        } else {
282            LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
283            mtmd_debug_preprocess_image(ctx_mtmd.get(), rgb_values, inp_size, inp_size);
284        }
285
286    } else {
287        LOG_ERR("ERR: Invalid mode specified with -p\n");
288        show_additional_info(argc, argv);
289        return 1;
290    }
291
292    return 0;
293}
294