char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovggml : update ggml_prec specification (#26675)5a6caa05f

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
311.2 KiB6071 linesraw
1#include "clip.h"
2#include "clip-impl.h"
3#include "clip-model.h"
4#include "clip-graph.h"
5#include "models/models.h"
6
7#include "ggml.h"
8#include "ggml-cpp.h"
9#include "ggml-alloc.h"
10#include "ggml-backend.h"
11#include "gguf.h"
12
13#include <algorithm>
14#include <cassert>
15#include <cmath>
16#include <cstdlib>
17#include <cstring>
18#include <fstream>
19#include <map>
20#include <random>
21#include <stdexcept>
22#include <unordered_set>
23#include <vector>
24#include <cinttypes>
25#include <limits>
26#include <array>
27#include <functional>
28#include <float.h>
29
30struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};
31
32//#define CLIP_DEBUG_FUNCTIONS
33
34#ifdef CLIP_DEBUG_FUNCTIONS
35static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) {
36    std::ofstream file(filename, std::ios::binary);
37    if (!file.is_open()) {
38        LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
39        return;
40    }
41
42    // PPM header: P6 format, width, height, and max color value
43    const auto ppm_size = img.get_size();
44    file << "P6\n" << ppm_size.width << " " << ppm_size.height << "\n255\n";
45
46    // Write pixel data
47    const auto & ppm_buf = img.get_ro_buf();
48    for (size_t i = 0; i < ppm_buf.size(); i += 3) {
49        // PPM expects binary data in RGB format, which matches our image buffer
50        file.write(reinterpret_cast<const char*>(&ppm_buf[i]), 3);
51    }
52
53    file.close();
54}
55
56static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) {
57    std::ofstream file(filename, std::ios::binary);
58    if (!file.is_open()) {
59        LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());
60        return;
61    }
62
63    const auto bmp_size = img.get_size();
64    int fileSize = 54 + 3 * bmp_size.width * bmp_size.height; // File header + info header + pixel data
65    int bytesPerPixel = 3;
66    int widthInBytes = bmp_size.width * bytesPerPixel;
67    int paddingAmount = (4 - (widthInBytes % 4)) % 4;
68    int stride = widthInBytes + paddingAmount;
69
70    // Bitmap file header
71    unsigned char fileHeader[14] = {
72        'B','M',     // Signature
73        0,0,0,0,    // Image file size in bytes
74        0,0,0,0,    // Reserved
75        54,0,0,0    // Start of pixel array
76    };
77
78    // Total file size
79    fileSize = 54 + (stride * bmp_size.height);
80    fileHeader[2] = (unsigned char)(fileSize);
81    fileHeader[3] = (unsigned char)(fileSize >> 8);
82    fileHeader[4] = (unsigned char)(fileSize >> 16);
83    fileHeader[5] = (unsigned char)(fileSize >> 24);
84
85    // Bitmap information header (BITMAPINFOHEADER)
86    unsigned char infoHeader[40] = {
87        40,0,0,0,   // Size of this header (40 bytes)
88        0,0,0,0,    // Image width
89        0,0,0,0,    // Image height
90        1,0,        // Number of color planes
91        24,0,       // Bits per pixel
92        0,0,0,0,    // No compression
93        0,0,0,0,    // Image size (can be 0 for no compression)
94        0,0,0,0,    // X pixels per meter (not specified)
95        0,0,0,0,    // Y pixels per meter (not specified)
96        0,0,0,0,    // Total colors (color table not used)
97        0,0,0,0     // Important colors (all are important)
98    };
99
100    // Width and height in the information header
101    infoHeader[4] = (unsigned char)(bmp_size.width);
102    infoHeader[5] = (unsigned char)(bmp_size.width >> 8);
103    infoHeader[6] = (unsigned char)(bmp_size.width >> 16);
104    infoHeader[7] = (unsigned char)(bmp_size.width >> 24);
105    infoHeader[8] = (unsigned char)(bmp_size.height);
106    infoHeader[9] = (unsigned char)(bmp_size.height >> 8);
107    infoHeader[10] = (unsigned char)(bmp_size.height >> 16);
108    infoHeader[11] = (unsigned char)(bmp_size.height >> 24);
109
110    // Write file headers
111    file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));
112    file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));
113
114    // Pixel data
115    std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row
116    for (int y = bmp_size.height - 1; y >= 0; --y) { // BMP files are stored bottom-to-top
117        for (int x = 0; x < bmp_size.width; ++x) {
118            // Each pixel
119            const auto px = img.get_pixel(x, y);
120            unsigned char pixel[3] = {
121                px[2], // BMP stores pixels in BGR format
122                px[1],
123                px[0]
124            };
125            file.write(reinterpret_cast<char*>(pixel), 3);
126        }
127        // Write padding for the row
128        file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);
129    }
130
131    file.close();
132}
133
134// debug function to convert f32 to u8
135static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) {
136    dst.set_size(src.get_size(), false);
137    const auto & src_buf = src.get_ro_buf();
138    std::vector<uint8_t> dst_buf(src.n_elements());
139    for (size_t i = 0; i < src.n_elements(); ++i) {
140        dst_buf[i] = static_cast<uint8_t>(std::min(std::max(int(src_buf[i] * 255.0f), 0), 255));
141    }
142    dst.cpy_buf(dst_buf);
143}
144#endif
145
146
147struct clip_ctx {
148    clip_model model;
149
150    gguf_context_ptr ctx_gguf;
151    ggml_context_ptr ctx_data;
152
153    std::vector<uint8_t> buf_compute_meta;
154
155    std::vector<ggml_backend_t> backend_ptrs;
156    std::vector<ggml_backend_buffer_type_t> backend_buft;
157
158    ggml_backend_t backend = nullptr;
159    ggml_backend_t backend_cpu = nullptr;
160    ggml_backend_buffer_ptr buf;
161
162
163    int max_nodes = 8192;
164    ggml_backend_sched_ptr sched;
165    clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO;
166    bool is_allocated = false;
167
168    bool debug_output_embeddings = false;
169
170    // for measuring memory usage
171    bool no_alloc = false;
172    std::map<ggml_backend_dev_t, size_t> mem_usage;
173    std::map<ggml_backend_dev_t, size_t> mem_compute;
174
175    bool support_batch = false;
176
177    // for audio gen, reseeded only when the caller asks for another seed
178    std::mt19937 rng{std::random_device{}()};
179    uint32_t rng_seed = UINT32_MAX;
180
181    clip_ctx(clip_context_params & ctx_params) {
182        flash_attn_type = ctx_params.flash_attn_type;
183        no_alloc = ctx_params.no_alloc;
184        backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
185        if (!backend_cpu) {
186            throw std::runtime_error("failed to initialize CPU backend");
187        }
188        if (ctx_params.use_gpu) {
189            if (ctx_params.device != nullptr) {
190                backend = ggml_backend_dev_init(ctx_params.device, nullptr);
191                if (!backend) {
192                    throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
193                                                           __func__, ggml_backend_dev_name(ctx_params.device)));
194                }
195            } else {
196                backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);
197                backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);
198            }
199        }
200
201        if (backend) {
202            LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend));
203            backend_ptrs.push_back(backend);
204            backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
205        } else {
206            backend = backend_cpu;
207            LOG_INF("%s: CLIP using CPU backend\n", __func__);
208        }
209
210        if (ctx_params.image_min_tokens > 0) {
211            model.hparams.custom_image_min_tokens = ctx_params.image_min_tokens;
212        }
213        if (ctx_params.image_max_tokens > 0) {
214            model.hparams.custom_image_max_tokens = ctx_params.image_max_tokens;
215        }
216
217        backend_ptrs.push_back(backend_cpu);
218        backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
219
220        sched.reset(
221            ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true)
222        );
223
224        if (ctx_params.cb_eval != nullptr) {
225            ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data);
226        }
227
228        debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
229    }
230
231    ~clip_ctx() {
232        ggml_backend_free(backend);
233        if (backend != backend_cpu) {
234            ggml_backend_free(backend_cpu);
235        }
236    }
237
238    // this function is added so that we don't change too much of the existing code
239    projector_type proj_type() const {
240        return model.proj_type;
241    }
242};
243
244//
245// clip_graph
246//
247
248clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
249        model(ctx->model),
250        hparams(model.hparams),
251        proj_type(ctx->proj_type()),
252        img(img),
253        patch_size(hparams.patch_size),
254        n_patches_x(img.nx() / patch_size),
255        n_patches_y(img.ny() / patch_size),
256        n_patches(n_patches_x * n_patches_y),
257        n_embd(hparams.n_embd),
258        n_head(hparams.n_head),
259        n_head_kv(hparams.n_head_kv),
260        d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)),
261        n_layer(hparams.n_layer),
262        n_mmproj_embd(clip_n_mmproj_embd(ctx)),
263        eps(hparams.eps),
264        kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f),
265        flash_attn_type(ctx->flash_attn_type) {
266    struct ggml_init_params params = {
267        /*.mem_size   =*/ ctx->buf_compute_meta.size(),
268        /*.mem_buffer =*/ ctx->buf_compute_meta.data(),
269        /*.no_alloc   =*/ true,
270    };
271    ctx0_ptr.reset(ggml_init(params));
272    ctx0 = ctx0_ptr.get();
273    gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false);
274}
275
276clip_graph::clip_graph(const clip_graph & parent) :
277        model(parent.model),
278        hparams(parent.hparams),
279        proj_type(parent.proj_type),
280        img(parent.img),
281        patch_size(parent.patch_size),
282        n_patches_x(parent.n_patches_x),
283        n_patches_y(parent.n_patches_y),
284        n_patches(parent.n_patches),
285        n_embd(parent.n_embd),
286        n_head(parent.n_head),
287        n_head_kv(parent.n_head_kv),
288        d_head(parent.d_head),
289        n_layer(parent.n_layer),
290        n_mmproj_embd(parent.n_mmproj_embd),
291        eps(parent.eps),
292        kq_scale(parent.kq_scale),
293        flash_attn_type(parent.flash_attn_type) {
294    // reuse from parent
295    ctx0 = parent.ctx0;
296    gf   = parent.gf;
297}
298
299ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const {
300    return ggml_mul_mat(ctx0, w, x);
301}
302
303void clip_graph::cb(ggml_tensor * cur, const char * name, int il) const {
304    if (il >= 0) {
305        ggml_format_name(cur, "%s-%d", name, il);
306    } else {
307        ggml_set_name(cur, name);
308    }
309}
310
311// siglip2 naflex
312ggml_tensor * clip_graph::resize_position_embeddings(uint32_t interpolation_mode) {
313    ggml_tensor * pos_embd = model.position_embeddings;
314    const int height       = img.ny() / patch_size;
315    const int width        = img.nx() / patch_size;
316    const uint32_t mode    = interpolation_mode;
317    const int n_per_side   = (int)std::sqrt(pos_embd->ne[1]);
318
319    GGML_ASSERT(pos_embd);
320
321    if (height == n_per_side && width == n_per_side) {
322        return pos_embd;
323    }
324
325    pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side);  // -> (n_embd, n_per_side, n_per_side)
326    pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3);                         // -> (n_per_side, n_per_side, n_embd)
327    pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd)
328    pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3);                         // -> (n_embd, width, height)
329    pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);             // -> (n_embd, width * height)
330
331    return pos_embd;
332}
333
334// build vision transformer (ViT) cgraph
335// this function should cover most of the models
336// if your model has specific features, you should probably duplicate this function
337ggml_tensor * clip_graph::build_vit(
338            ggml_tensor * inp,
339            int64_t n_pos,
340            norm_type norm_t,
341            ffn_op_type ffn_t,
342            ggml_tensor * learned_pos_embd,
343            std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
344            const build_vit_opts & opts
345        ) {
346    // batch dim: inp is [n_embd, n_pos, B]
347    const int64_t B = inp->ne[2];
348
349    if (learned_pos_embd) {
350        inp = ggml_add(ctx0, inp, learned_pos_embd);
351        cb(inp, "pos_embed", -1);
352    }
353
354    // flatten batch; unflatten again in attention
355    inp = ggml_reshape_2d(ctx0, inp, n_embd, n_pos * B);
356
357    ggml_tensor * inpL = inp;
358
359    // pre-layernorm
360    if (model.pre_ln_w) {
361        inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
362        cb(inpL, "pre_ln", -1);
363    }
364
365    // loop over layers
366    for (int il = 0; il < n_layer; il++) {
367        auto & layer = model.layers[il];
368        ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
369
370        ggml_tensor * attn_mask = opts.attn_mask;
371        if (opts.attn_mask_layers.size() > (size_t) il) {
372            attn_mask = opts.attn_mask_layers[il];
373        }
374
375        // layernorm1
376        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
377        cb(cur, "layer_inp_normed", il);
378
379        // self-attention
380        {
381            ggml_tensor * Qcur = nullptr;
382            ggml_tensor * Kcur = nullptr;
383            ggml_tensor * Vcur = nullptr;
384            if (layer.qkv_w != nullptr) {
385                // fused qkv
386                cur = build_mm(layer.qkv_w, cur);
387                if (layer.qkv_b != nullptr) {
388                    cur = ggml_add(ctx0, cur, layer.qkv_b);
389                }
390
391                // Q/K/V as [d_head, n_head, n_pos, B], the batch stride is cur->nb[1]*n_pos.
392                Qcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
393                /* nb1    */ ggml_row_size(cur->type, d_head),
394                /* nb2    */ cur->nb[1],
395                /* nb3    */ cur->nb[1] * n_pos,
396                /* offset */ 0);
397
398                Kcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
399                /* nb1    */ ggml_row_size(cur->type, d_head),
400                /* nb2    */ cur->nb[1],
401                /* nb3    */ cur->nb[1] * n_pos,
402                /* offset */ ggml_row_size(cur->type, n_head * d_head));
403
404                Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
405                /* nb1    */ ggml_row_size(cur->type, d_head),
406                /* nb2    */ cur->nb[1],
407                /* nb3    */ cur->nb[1] * n_pos,
408                /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head));
409
410                if (layer.q_norm) {
411                    GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
412                    Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
413                    cb(Qcur, "Qcur_norm", il);
414                }
415
416                if (layer.k_norm) {
417                    GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
418                    Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
419                    cb(Kcur, "Kcur_norm", il);
420                }
421
422            } else {
423                // separate q, k, v
424                Qcur = build_mm(layer.q_w, cur);
425                if (layer.q_b) {
426                    Qcur = ggml_add(ctx0, Qcur, layer.q_b);
427                }
428
429                Kcur = build_mm(layer.k_w, cur);
430                if (layer.k_b) {
431                    Kcur = ggml_add(ctx0, Kcur, layer.k_b);
432                }
433
434                Vcur = build_mm(layer.v_w, cur);
435                if (layer.v_b) {
436                    Vcur = ggml_add(ctx0, Vcur, layer.v_b);
437                }
438
439                // if true, norm must be applied after reshaping to (d_head, n_head, n_pos)
440                bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head;
441
442                if (!norm_per_head) {
443                    if (layer.q_norm) {
444                        Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
445                        cb(Qcur, "Qcur_norm", il);
446                    }
447                    if (layer.k_norm) {
448                        Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
449                        cb(Kcur, "Kcur_norm", il);
450                    }
451                }
452
453                Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head,    n_pos, B);
454                Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head_kv, n_pos, B);
455                Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head_kv, n_pos, B);
456
457                if (norm_per_head) {
458                    if (layer.q_norm) {
459                        Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
460                        cb(Qcur, "Qcur_norm_per_head", il);
461                    }
462                    if (layer.k_norm) {
463                        Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
464                        cb(Kcur, "Kcur_norm_per_head", il);
465                    }
466                }
467            }
468
469            cb(Qcur, "Qcur", il);
470            cb(Kcur, "Kcur", il);
471            cb(Vcur, "Vcur", il);
472
473            if (add_pos) {
474                Qcur = add_pos(Qcur, layer);
475                Kcur = add_pos(Kcur, layer);
476                cb(Qcur, "Qcur_pos", il);
477                cb(Kcur, "Kcur_pos", il);
478            }
479
480            if (proj_type == PROJECTOR_TYPE_GEMMA4V) {
481                Vcur = ggml_rms_norm(ctx0, Vcur, eps);
482                cb(Vcur, "Vcur_normed", il);
483            }
484
485            // build_attn returns a flat 2D [n_embd, n_pos*B]
486            cur = build_attn(layer.o_w, layer.o_b,
487                Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
488            cb(cur, "attn_out", il);
489        }
490
491        if (layer.ls_1_w) {
492            cur = ggml_mul(ctx0, cur, layer.ls_1_w);
493            cb(cur, "attn_out_scaled", il);
494        }
495
496        if (layer.attn_post_norm_w) {
497            cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il);
498            cb(cur, "attn_post_normed", il);
499        }
500
501        // re-add the layer input, e.g., residual
502        cur = ggml_add(ctx0, cur, inpL);
503
504        inpL = cur; // inpL = residual, cur = hidden_states
505
506        if (opts.callback_layer_out) {
507            opts.callback_layer_out(cur, il);
508        }
509
510        cb(cur, "ffn_inp", il);
511
512        // layernorm2 (pre-ffn norm)
513        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
514        cb(cur, "ffn_inp_normed", il);
515
516        // ffn
517        cur = layer.ff_gate_exps_w
518            ? build_moe_ffn(cur, layer, ffn_t, il)
519            : build_ffn(cur,
520                layer.ff_up_w, layer.ff_up_b,
521                layer.ff_gate_w, layer.ff_gate_b,
522                layer.ff_down_w, layer.ff_down_b,
523                ffn_t, il);
524
525        cb(cur, "ffn_out", il);
526
527        if (layer.ff_post_norm_w) {
528            cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il);
529            cb(cur, "ffn_post_normed", il);
530        }
531
532        if (layer.ls_2_w) {
533            cur = ggml_mul(ctx0, cur, layer.ls_2_w);
534            cb(cur, "ffn_out_scaled", il);
535        }
536
537        // residual 2
538        cur = ggml_add(ctx0, inpL, cur);
539        cb(cur, "layer_out", il);
540
541        if (layer.ls_out_w) {
542            cur = ggml_mul(ctx0, cur, layer.ls_out_w);
543            cb(cur, "layer_out_scaled", il);
544        }
545
546        inpL = cur;
547    }
548
549    if (model.audio_has_avgpool()) {
550        ggml_tensor * cur = inpL;
551        cur = ggml_transpose(ctx0, cur);
552        cur = ggml_cont(ctx0, cur);
553        cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0);
554        cur = ggml_transpose(ctx0, cur);
555        cur = ggml_cont(ctx0, cur);
556        inpL = cur;
557    }
558
559    // post-layernorm
560    if (model.post_ln_w && !opts.skip_post_ln) {
561        inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);
562    }
563
564    // restore the batch dim
565    GGML_ASSERT(inpL->ne[1] % B == 0);
566    inpL = ggml_reshape_3d(ctx0, inpL, n_embd, inpL->ne[1] / B, B);
567    return inpL;
568}
569
570// build the input after conv2d (inp_raw --> patches)
571// returns tensor with shape [n_embd, n_patches]
572ggml_tensor * clip_graph::build_inp() {
573    ggml_tensor * inp_raw = build_inp_raw();
574    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
575    inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
576    inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
577    if (model.patch_bias) {
578        inp = ggml_add(ctx0, inp, model.patch_bias);
579        cb(inp, "patch_bias", -1);
580    }
581    return inp;
582}
583
584ggml_tensor * clip_graph::build_inp_raw(int channels) {
585    ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch);
586    ggml_set_name(inp_raw, "inp_raw");
587    ggml_set_input(inp_raw);
588    return inp_raw;
589}
590
591ggml_tensor * clip_graph::build_norm(
592        ggml_tensor * cur,
593        ggml_tensor * mw,
594        ggml_tensor * mb,
595        norm_type type,
596        float norm_eps,
597        int il) const {
598
599    cur = type == NORM_TYPE_RMS
600        ? ggml_rms_norm(ctx0, cur, norm_eps)
601        : ggml_norm(ctx0, cur, norm_eps);
602
603    if (mw) {
604        cur = ggml_mul(ctx0, cur, mw);
605        cb(cur, "norm_w", il);
606    }
607
608    if (mb) {
609        cur = ggml_add(ctx0, cur, mb);
610        cb(cur, "norm_b", il);
611    }
612
613    return cur;
614}
615
616ggml_tensor * clip_graph::build_ffn(
617        ggml_tensor * cur,
618        ggml_tensor * up,
619        ggml_tensor * up_b,
620        ggml_tensor * gate,
621        ggml_tensor * gate_b,
622        ggml_tensor * down,
623        ggml_tensor * down_b,
624        ffn_op_type type_op,
625        int il) const {
626
627    ggml_tensor * tmp = up ? build_mm(up, cur) : cur;
628    cb(tmp, "ffn_up", il);
629
630    if (up_b) {
631        tmp = ggml_add(ctx0, tmp, up_b);
632        cb(tmp, "ffn_up_b", il);
633    }
634
635    if (gate) {
636        cur = build_mm(gate, cur);
637        cb(cur, "ffn_gate", il);
638
639        if (gate_b) {
640            cur = ggml_add(ctx0, cur, gate_b);
641            cb(cur, "ffn_gate_b", il);
642        }
643    } else {
644        cur = tmp;
645    }
646
647    // we only support parallel ffn for now
648    switch (type_op) {
649        case FFN_SILU:
650            if (gate) {
651                cur = ggml_swiglu_split(ctx0, cur, tmp);
652                cb(cur, "ffn_swiglu", il);
653            } else {
654                cur = ggml_silu(ctx0, cur);
655                cb(cur, "ffn_silu", il);
656            } break;
657        case FFN_GELU:
658            if (gate) {
659                cur = ggml_geglu_split(ctx0, cur, tmp);
660                cb(cur, "ffn_geglu", il);
661            } else {
662                cur = ggml_gelu(ctx0, cur);
663                cb(cur, "ffn_gelu", il);
664            } break;
665        case FFN_GELU_ERF:
666            if (gate) {
667                cur = ggml_geglu_erf_split(ctx0, cur, tmp);
668                cb(cur, "ffn_geglu_erf", il);
669            } else {
670                cur = ggml_gelu_erf(ctx0, cur);
671                cb(cur, "ffn_gelu_erf", il);
672            } break;
673        case FFN_GELU_QUICK:
674            if (gate) {
675                cur = ggml_geglu_quick_split(ctx0, cur, tmp);
676                cb(cur, "ffn_geglu_quick", il);
677            } else {
678                cur = ggml_gelu_quick(ctx0, cur);
679                cb(cur, "ffn_gelu_quick", il);
680            } break;
681        case FFN_RELU_SQR:
682            {
683                cur = ggml_relu(ctx0, cur);
684                cur = ggml_sqr(ctx0, cur);
685                cb(cur, "ffn_relu_sqr", il);
686            } break;
687    }
688
689    if (down) {
690        cur = build_mm(down, cur);
691    }
692
693    if (down_b) {
694        cb(cur, "ffn_down", il);
695    }
696
697    if (down_b) {
698        cur = ggml_add(ctx0, cur, down_b);
699    }
700
701    return cur;
702}
703
704// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
705// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
706ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const {
707    const int64_t n_tokens      = cur->ne[1];
708    const int64_t n_expert      = layer.ff_gate_exps_w->ne[2];
709    const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
710    GGML_ASSERT(n_expert_used > 0);
711    GGML_ASSERT(type_op == FFN_SILU);
712
713    ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
714    cb(probs, "ffn_moe_probs", il);
715
716    ggml_tensor * sel = layer.ff_exp_probs_b
717        ? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
718        : probs;
719    ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
720
721    ggml_tensor * weights = ggml_get_rows(ctx0,
722        ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
723    weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
724    weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
725    weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
726    cb(weights, "ffn_moe_weights", il);
727
728    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
729    ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
730    ggml_tensor * up   = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w,   cur, selected);
731    cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
732    cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
733    cur = ggml_mul(ctx0, cur, weights);
734
735    // sum over the selected experts
736    ggml_tensor * out = nullptr;
737    for (int64_t i = 0; i < n_expert_used; i++) {
738        ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
739        out = out ? ggml_add(ctx0, out, v) : v;
740    }
741    if (n_expert_used == 1) {
742        out = ggml_cont(ctx0, out);
743    }
744    cb(out, "ffn_moe_out", il);
745    return out;
746}
747
748ggml_tensor * clip_graph::build_attn(
749        ggml_tensor * wo,
750        ggml_tensor * wo_b,
751        ggml_tensor * q_cur,
752        ggml_tensor * k_cur,
753        ggml_tensor * v_cur,
754        ggml_tensor * kq_mask,
755        float kq_scale,
756        int il,
757        ggml_tensor * sinks) const {
758    // these nodes are added to the graph together so that they are not reordered
759    // by doing so, the number of splits in the graph is reduced
760    // the order is fixed without the compute flag, so an unselected branch stays out of the compute set
761    ggml_build_forward_order(gf, q_cur);
762    ggml_build_forward_order(gf, k_cur);
763    ggml_build_forward_order(gf, v_cur);
764
765    ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);
766    //cb(q, "q", il);
767
768    ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3);
769    //cb(k, "k", il);
770
771    ggml_tensor * cur;
772
773    if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
774        ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3);
775
776        k = ggml_cast(ctx0, k, GGML_TYPE_F16);
777        v = ggml_cast(ctx0, v, GGML_TYPE_F16);
778        if (kq_mask) {
779            kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
780        }
781
782        cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
783        ggml_prec_set_acc(cur, GGML_PREC_F32);
784        if (sinks != nullptr) {
785            ggml_flash_attn_ext_add_sinks(cur, sinks);
786        }
787
788        cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
789
790    } else {
791        ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3);
792        v = ggml_cont(ctx0, v);
793
794        ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
795        // F32 may not needed for vision encoders?
796        // ggml_prec_set_acc(kq, GGML_PREC_F32);
797
798        kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
799        if (sinks != nullptr) {
800            ggml_soft_max_add_sinks(kq, sinks);
801        }
802
803        ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
804        cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
805        cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]);
806    }
807
808    cb(cur, "kqv_out", il);
809
810    if (wo) {
811        cur = build_mm(wo, cur);
812    }
813
814    if (wo_b) {
815        cur = ggml_add(ctx0, cur, wo_b);
816    }
817
818    return cur;
819}
820
821// implementation of the 2D RoPE without adding a new op in ggml
822ggml_tensor * clip_graph::build_rope_2d(
823    ggml_context * ctx0,
824    ggml_tensor * cur,
825    ggml_tensor * pos_a, // first half
826    ggml_tensor * pos_b, // second half
827    const float freq_base,
828    const bool interleave_freq
829) {
830    const int64_t n_dim = cur->ne[0];
831
832    // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)
833    // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3
834    // first half of cur will use 1e-0, 1e-2 (even)
835    // second half of cur will use 1e-1, 1e-3 (odd)
836    // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even
837    //  ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2)
838    // then for the second half, we use freq_scale to shift the inv_freq
839    //  ^ why? replace (2i) with (2i+1) in the above equation
840    const float freq_scale_odd = interleave_freq
841                                ? std::pow(freq_base, (float)-2/n_dim)
842                                : 1.0;
843
844    // first half, dims [0, n_dim/2)
845    cur = ggml_rope_ext(
846        ctx0,
847        cur,
848        pos_a,      // positions
849        nullptr,    // freq factors
850        n_dim/2,    // n_dims
851        0, 0, freq_base,
852        1.0f, 0.0f, 1.0f, 0.0f, 0.0f
853    );
854
855    // second half, dims [n_dim/2, n_dim)
856    cur = ggml_rope_ext(
857        ctx0,
858        cur,
859        pos_b,      // positions
860        nullptr,    // freq factors
861        n_dim/2,    // n_dims
862        0, 0, freq_base,
863        freq_scale_odd,
864        0.0f, 1.0f, 0.0f, 0.0f
865    );
866    cur = ggml_rope_set_offset(cur, n_dim/2);
867
868    return cur;
869}
870
871// Generic function to stack frames for audio processing
872// Abstracts out the StackAudioFrames logic used by ultravox
873ggml_tensor * clip_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) {
874    if (stack_factor <= 1) {
875        return cur;
876    }
877
878    int64_t total_elements = ggml_nelements(cur);
879    int64_t stride = n_embed * stack_factor;
880
881    // Calculate padded length
882    int64_t padded_len = GGML_PAD(total_elements, stride);
883    int64_t pad = padded_len - total_elements;
884
885    if (pad > 0) {
886        // Pad the tensor to make it divisible by stride
887        cur = ggml_view_1d(ctx0, cur, total_elements, 0);
888        cur = ggml_pad(ctx0, cur, pad, 0, 0, 0);
889    }
890
891    // Reshape to [stride, padded_len / stride]
892    cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride,
893                        ggml_row_size(cur->type, stride), 0);
894    return cur;
895}
896
897// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
898// support dynamic resolution
899ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) {
900    GGML_ASSERT(scale_factor > 1);
901
902    const int n_embd = cur->ne[0];
903    int width  = img.nx() / patch_size;
904    int height = img.ny() / patch_size;
905
906    // pad width and height to factor
907    const int64_t pad_width  = CLIP_ALIGN(width,  scale_factor) - width;
908    const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height;
909    cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
910    if (pad_width || pad_height) {
911        cur     = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
912        width  += pad_width;
913        height += pad_height;
914    }
915
916    // unshuffle h
917    cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);
918    cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
919
920    // unshuffle w
921    cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
922    cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
923
924    cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
925    cb(cur, "pixel_shuffle", -1);
926
927    return cur;
928}
929
930static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs,
931                                                            const clip_encode_params * params = nullptr) {
932    const clip_image_f32 & img = imgs.entries[0];
933    std::unique_ptr<clip_graph> builder;
934
935    switch (ctx->proj_type()) {
936        case PROJECTOR_TYPE_GEMMA3:
937        case PROJECTOR_TYPE_IDEFICS3:
938        case PROJECTOR_TYPE_LFM2:
939        case PROJECTOR_TYPE_JANUS_PRO:
940        case PROJECTOR_TYPE_PHI4:
941            {
942                builder = std::make_unique<clip_graph_siglip>(ctx, img);
943            } break;
944        case PROJECTOR_TYPE_GEMMA3NV:
945            {
946                builder = std::make_unique<clip_graph_mobilenetv5>(ctx, img);
947            } break;
948        case PROJECTOR_TYPE_GEMMA4V:
949            {
950                builder = std::make_unique<clip_graph_gemma4v>(ctx, img);
951            } break;
952        case PROJECTOR_TYPE_GEMMA4UV:
953            {
954                builder = std::make_unique<clip_graph_gemma4uv>(ctx, img);
955            } break;
956        case PROJECTOR_TYPE_PIXTRAL:
957        case PROJECTOR_TYPE_LIGHTONOCR:
958            {
959                builder = std::make_unique<clip_graph_pixtral>(ctx, img);
960            } break;
961        case PROJECTOR_TYPE_DOTS_OCR:
962        case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
963            {
964                builder = std::make_unique<clip_graph_dotsocr>(ctx, img);
965            } break;
966        case PROJECTOR_TYPE_DOTS3NOTE_A:
967            {
968                builder = std::make_unique<clip_graph_dots3note_a>(ctx, img);
969            } break;
970        case PROJECTOR_TYPE_QWEN2VL:
971        case PROJECTOR_TYPE_QWEN25VL:
972            {
973                builder = std::make_unique<clip_graph_qwen2vl>(ctx, img);
974            } break;
975        case PROJECTOR_TYPE_QWEN3VL:
976            {
977                builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
978            } break;
979        case PROJECTOR_TYPE_EXAONE4_5:
980            {
981                builder = std::make_unique<clip_graph_exaone4_5>(ctx, img);
982            } break;
983        case PROJECTOR_TYPE_MIMOVL:
984            {
985                builder = std::make_unique<clip_graph_mimovl>(ctx, img);
986            } break;
987        case PROJECTOR_TYPE_MINIMAX_M3:
988            {
989                builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
990            } break;
991        case PROJECTOR_TYPE_MUSE_GLIMMER:
992            {
993                builder = std::make_unique<clip_graph_muse_glimmer>(ctx, img);
994            } break;
995        case PROJECTOR_TYPE_STEP3VL:
996            {
997                builder = std::make_unique<clip_graph_step3vl>(ctx, img);
998            } break;
999        case PROJECTOR_TYPE_MINICPMV:
1000            {
1001                builder = std::make_unique<clip_graph_minicpmv>(ctx, img);
1002            } break;
1003        case PROJECTOR_TYPE_MINICPMV4_6:
1004            {
1005                builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
1006            } break;
1007        case PROJECTOR_TYPE_INTERNVL:
1008            {
1009                builder = std::make_unique<clip_graph_internvl>(ctx, img);
1010            } break;
1011        case PROJECTOR_TYPE_NEMOTRON_V2_VL:
1012            {
1013                builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);
1014            } break;
1015        case PROJECTOR_TYPE_LLAMA4:
1016            {
1017                builder = std::make_unique<clip_graph_llama4>(ctx, img);
1018            } break;
1019        case PROJECTOR_TYPE_ULTRAVOX:
1020        case PROJECTOR_TYPE_VOXTRAL:
1021        case PROJECTOR_TYPE_QWEN2A:
1022        case PROJECTOR_TYPE_GLMA:
1023        case PROJECTOR_TYPE_MERALION:
1024        case PROJECTOR_TYPE_MUSIC_FLAMINGO:
1025            {
1026                builder = std::make_unique<clip_graph_whisper_enc>(ctx, img);
1027            } break;
1028        case PROJECTOR_TYPE_KIMIVL:
1029            {
1030                builder = std::make_unique<clip_graph_kimivl>(ctx, img);
1031            } break;
1032        case PROJECTOR_TYPE_PADDLEOCR:
1033            {
1034                builder = std::make_unique<clip_graph_paddleocr>(ctx, img);
1035            } break;
1036        case PROJECTOR_TYPE_KIMIK25:
1037            {
1038                builder = std::make_unique<clip_graph_kimik25>(ctx, img);
1039            } break;
1040        case PROJECTOR_TYPE_DEEPSEEK4V:
1041            {
1042                builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
1043            } break;
1044        case PROJECTOR_TYPE_COGVLM:
1045            {
1046                builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
1047            } break;
1048        case PROJECTOR_TYPE_HUNYUANVL:
1049            {
1050                builder = std::make_unique<clip_graph_hunyuanvl>(ctx, img);
1051            } break;
1052        case PROJECTOR_TYPE_MLP:
1053        case PROJECTOR_TYPE_MLP_NORM:
1054        case PROJECTOR_TYPE_LDP:
1055        case PROJECTOR_TYPE_LDPV2:
1056        case PROJECTOR_TYPE_GLM_EDGE:
1057            {
1058                builder = std::make_unique<clip_graph_llava>(ctx, img);
1059            } break;
1060        case PROJECTOR_TYPE_DEEPSEEKOCR:
1061            {
1062                builder = std::make_unique<clip_graph_deepseekocr>(ctx, img);
1063            } break;
1064        case PROJECTOR_TYPE_DEEPSEEKOCR2:
1065             {
1066                builder = std::make_unique<clip_graph_deepseekocr2>(ctx, img);
1067            } break;
1068        case PROJECTOR_TYPE_LFM2A:
1069            {
1070                builder = std::make_unique<clip_graph_conformer>(ctx, img);
1071            } break;
1072        case PROJECTOR_TYPE_GEMMA4A:
1073            {
1074                builder = std::make_unique<clip_graph_gemma4a>(ctx, img);
1075            } break;
1076        case PROJECTOR_TYPE_GEMMA4UA:
1077            {
1078                builder = std::make_unique<clip_graph_gemma4ua>(ctx, img);
1079            } break;
1080        case PROJECTOR_TYPE_GRANITE_SPEECH:
1081            {
1082                builder = std::make_unique<clip_graph_granite_speech>(ctx, img);
1083            } break;
1084        case PROJECTOR_TYPE_GLM4V:
1085            {
1086                builder = std::make_unique<clip_graph_glm4v>(ctx, img);
1087            } break;
1088        case PROJECTOR_TYPE_QWEN3A:
1089            {
1090                builder = std::make_unique<clip_graph_qwen3a>(ctx, img);
1091            } break;
1092        case PROJECTOR_TYPE_MIMO_AUDIO:
1093            {
1094                builder = std::make_unique<clip_graph_mimo_audio>(ctx, img);
1095            } break;
1096        case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
1097            {
1098                builder = std::make_unique<clip_graph_qwen3tts_spkenc>(ctx, img);
1099            } break;
1100        case PROJECTOR_TYPE_POCKETTTS_SPKENC:
1101            {
1102                builder = std::make_unique<clip_graph_pockettts_spkenc>(ctx, img);
1103            } break;
1104        case PROJECTOR_TYPE_POCKETTTS_GEN:
1105            {
1106                const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
1107                const int  n_step = ctx->model.hparams.flow_n_step;
1108                const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0];
1109                GGML_ASSERT(n_step > 0);
1110                GGML_ASSERT(n_latent > 0);
1111                // "inp_feats" takes the caller's buffer as-is, the graph must consume all of it
1112                if (params && params->feats) {
1113                    GGML_ASSERT(params->feats->size() % (size_t) n_latent == 0);
1114                    GGML_ASSERT(params->feats->size() >= (size_t) n_latent);
1115                }
1116                const int  n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1;
1117                builder = std::make_unique<clip_graph_pockettts_gen>(ctx, img, gen_process, n_step, n_frames);
1118            } break;
1119        case PROJECTOR_TYPE_QWEN3TTS_GEN:
1120            {
1121                const auto  gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
1122                const int   top_k = params ? params->top_k : 50;
1123                const float top_p = params ? params->top_p : 1.0f;
1124                builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
1125            } break;
1126        case PROJECTOR_TYPE_YOUTUVL:
1127            {
1128                builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
1129            } break;
1130        case PROJECTOR_TYPE_YASA2:
1131            {
1132                builder = std::make_unique<clip_graph_yasa2>(ctx, img);
1133            } break;
1134        case PROJECTOR_TYPE_PARAKEET:
1135            {
1136                builder = std::make_unique<clip_graph_parakeet>(ctx, img);
1137            } break;
1138        case PROJECTOR_TYPE_GRANITE4_VISION:
1139            {
1140                builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
1141            } break;
1142        default:
1143            GGML_ABORT("missing cgraph builder");
1144    }
1145
1146    builder->img_batch = &imgs;
1147
1148    // TODO [QWEN_VIDEO]: improve this in the future
1149    builder->n_batch = imgs.entries.size();
1150
1151    return builder;
1152}
1153
1154//
1155// clip_model_loader
1156//
1157
1158struct clip_model_loader {
1159    ggml_context_ptr ctx_meta;
1160    gguf_context_ptr ctx_gguf;
1161
1162    std::string fname;
1163
1164    size_t model_size = 0; // in bytes
1165
1166    bool has_vision    = false;
1167    bool has_audio     = false;
1168    bool has_gen_audio = false;
1169
1170    mtmd_progress_callback progress_callback = nullptr;
1171    void * progress_callback_user_data = nullptr;
1172
1173    // TODO @ngxson : we should not pass clip_ctx here, it should be clip_model
1174    clip_model_loader(const char * fname,
1175            bool skip_tensors = false,
1176            mtmd_progress_callback progress_cb = nullptr,
1177            void * progress_user_data = nullptr)
1178        : fname(fname),
1179          progress_callback(progress_cb),
1180          progress_callback_user_data(progress_user_data) {
1181        struct ggml_context * meta = nullptr;
1182
1183        struct gguf_init_params params = {
1184            /*.no_alloc = */ true,
1185            /*.ctx      = */ &meta,
1186        };
1187
1188        ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params));
1189        if (!ctx_gguf.get()) {
1190            throw std::runtime_error(string_format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname));
1191        }
1192
1193        ctx_meta.reset(meta);
1194
1195        const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
1196
1197        // print gguf info
1198        {
1199            std::string name;
1200            get_string(KEY_NAME, name, false);
1201            std::string description;
1202            get_string(KEY_DESCRIPTION, description, false);
1203            LOG_INF("%s: model name:   %s\n",  __func__, name.c_str());
1204            LOG_INF("%s: description:  %s\n",  __func__, description.c_str());
1205            LOG_INF("%s: GGUF version: %d\n",  __func__, gguf_get_version(ctx_gguf.get()));
1206            LOG_INF("%s: alignment:    %zu\n", __func__, gguf_get_alignment(ctx_gguf.get()));
1207            LOG_INF("%s: n_tensors:    %d\n",  __func__, n_tensors);
1208            LOG_INF("%s: n_kv:         %d\n",  __func__, (int)gguf_get_n_kv(ctx_gguf.get()));
1209            LOG_INF("\n");
1210        }
1211
1212        // modalities
1213        {
1214            get_bool(KEY_HAS_VISION_ENC,    has_vision,    false);
1215            get_bool(KEY_HAS_AUDIO_ENC,     has_audio,     false);
1216            get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
1217
1218            if (has_vision) {
1219                LOG_INF("%s: has vision encoder\n", __func__);
1220            }
1221            if (has_audio) {
1222                LOG_INF("%s: has audio encoder\n", __func__);
1223            }
1224            if (has_gen_audio) {
1225                LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
1226            }
1227        }
1228
1229        // tensors
1230        if (!skip_tensors) {
1231            for (int i = 0; i < n_tensors; ++i) {
1232                const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
1233                const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
1234                enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i);
1235                ggml_tensor * cur = ggml_get_tensor(meta, name);
1236                size_t tensor_size = ggml_nbytes(cur);
1237                model_size += tensor_size;
1238                LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",
1239                    __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));
1240            }
1241        }
1242    }
1243
1244    void load_hparams(clip_model & model, clip_modality modality) {
1245        auto & hparams = model.hparams;
1246        std::string log_ffn_op; // for logging
1247
1248        // sanity check
1249        if (modality == CLIP_MODALITY_VISION) {
1250            GGML_ASSERT(has_vision);
1251        } else if (modality == CLIP_MODALITY_AUDIO) {
1252            GGML_ASSERT(has_audio);
1253        } else if (modality == CLIP_MODALITY_GEN_AUDIO) {
1254            GGML_ASSERT(has_gen_audio);
1255        }
1256        model.modality = modality;
1257
1258
1259        // projector type
1260        std::string proj_type;
1261        {
1262            // default key
1263            get_string(KEY_PROJ_TYPE, proj_type, false);
1264
1265            // for models with mixed modalities
1266            if (proj_type.empty()) {
1267                if (modality == CLIP_MODALITY_VISION) {
1268                    get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
1269                } else if (modality == CLIP_MODALITY_AUDIO) {
1270                    get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
1271                } else if (modality == CLIP_MODALITY_GEN_AUDIO) {
1272                    get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
1273                } else {
1274                    GGML_ABORT("unknown modality");
1275                }
1276            }
1277
1278            model.proj_type = clip_projector_type_from_string(proj_type);
1279
1280            if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
1281                throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
1282            }
1283
1284            // correct arch for multimodal models (legacy method)
1285            if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
1286                model.proj_type = modality == CLIP_MODALITY_VISION
1287                                    ? PROJECTOR_TYPE_QWEN25VL
1288                                    : PROJECTOR_TYPE_QWEN2A;
1289            }
1290        }
1291
1292        const bool is_vision    = model.modality == CLIP_MODALITY_VISION;
1293        const bool is_audio     = model.modality == CLIP_MODALITY_AUDIO;
1294        const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO;
1295
1296        // other hparams
1297        {
1298            const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
1299            get_u32(string_format(KEY_N_EMBD,         prefix), hparams.n_embd);
1300            get_u32(string_format(KEY_N_HEAD,         prefix), hparams.n_head);
1301            get_u32(string_format(KEY_N_EMBD_HEAD,    prefix), hparams.n_embd_head, false);
1302            get_u32(string_format(KEY_N_FF,           prefix), hparams.n_ff);
1303            get_u32(string_format(KEY_N_BLOCK,        prefix), hparams.n_layer);
1304            get_u32(string_format(KEY_PROJ_DIM,       prefix), hparams.projection_dim);
1305            get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);
1306
1307            // n_head_kv is optional (for GQA), default to n_head
1308            hparams.n_head_kv = hparams.n_head;
1309            get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false);
1310
1311            if (is_vision) {
1312                get_u32(KEY_IMAGE_SIZE, hparams.image_size);
1313                get_u32(KEY_PATCH_SIZE, hparams.patch_size);
1314                get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy
1315                get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false);
1316                if (hparams.minicpmv_query_num == 0) {
1317                    // Fallback to hardcoded values for legacy models
1318                    if (hparams.minicpmv_version == 3) {
1319                        hparams.minicpmv_query_num = 64;
1320                    } else if (hparams.minicpmv_version == 4) {
1321                        hparams.minicpmv_query_num = 64;
1322                    } else if (hparams.minicpmv_version == 5) {
1323                        hparams.minicpmv_query_num = 64;
1324                    } else if (hparams.minicpmv_version == 6) {
1325                        hparams.minicpmv_query_num = 64;
1326                    } else if (hparams.minicpmv_version == 100045) {
1327                        hparams.minicpmv_query_num = 64;
1328                    } else {
1329                        hparams.minicpmv_query_num = 96;
1330                    }
1331                }
1332            } else if (is_audio) {
1333                get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins);
1334                // some hparams are unused, but still need to set to avoid issues
1335                hparams.image_size = 0;
1336                hparams.patch_size = 1;
1337
1338            } else if (is_gen_audio) {
1339                // these are unused, but still need to be set to avoid issues
1340                hparams.image_size = 0;
1341                hparams.patch_size = 1;
1342                get_string(KEY_GEN_AUDIO_VARIANT, hparams.gen_model_variant, false);
1343
1344            } else {
1345                GGML_ASSERT(false && "unknown modality");
1346            }
1347
1348            // for pinpoints, we need to convert it into a list of resolution candidates
1349            {
1350                std::vector<int> pinpoints;
1351                get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false);
1352                if (pinpoints.size() % 2 != 0) {
1353                    throw std::runtime_error(string_format("%s: image_grid_pinpoints must have an even number of elements, got %zu\n", __func__, pinpoints.size()));
1354                }
1355                if (!pinpoints.empty()) {
1356                    for (size_t i = 0; i < pinpoints.size(); i += 2) {
1357                        hparams.image_res_candidates.push_back({
1358                            pinpoints[i],
1359                            pinpoints[i+1],
1360                        });
1361                    }
1362                }
1363            }
1364
1365            // default warmup value
1366            hparams.warmup_image_size = hparams.image_size;
1367
1368            {
1369                bool use_gelu = false;
1370                bool use_silu = false;
1371                get_bool(KEY_USE_GELU, use_gelu, false);
1372                get_bool(KEY_USE_SILU, use_silu, false);
1373                if (use_gelu && use_silu) {
1374                    throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));
1375                }
1376                if (use_gelu) {
1377                    hparams.ffn_op = FFN_GELU;
1378                    log_ffn_op = "gelu";
1379                } else if (use_silu) {
1380                    hparams.ffn_op = FFN_SILU;
1381                    log_ffn_op = "silu";
1382                } else {
1383                    hparams.ffn_op = FFN_GELU_QUICK;
1384                    log_ffn_op = "gelu_quick";
1385                }
1386            }
1387
1388            {
1389                std::string mm_patch_merge_type;
1390                get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false);
1391                if (mm_patch_merge_type == "spatial_unpad") {
1392                    hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD;
1393                }
1394            }
1395
1396            if (is_vision) {
1397                std::vector<float> image_mean;
1398                std::vector<float> image_std;
1399                get_arr_f32(KEY_IMAGE_MEAN, image_mean, false);
1400                get_arr_f32(KEY_IMAGE_STD , image_std, false);
1401                if (image_mean.size() < 3 || image_std.size() < 3) {
1402                    throw std::runtime_error(string_format("%s: image_mean/image_std arrays must have at least 3 elements, got %zu and %zu\n", __func__, image_mean.size(), image_std.size()));
1403                }
1404                for (int i = 0; i < 3; ++i) {
1405                    hparams.image_mean[i] = image_mean[i];
1406                    hparams.image_std[i]  = image_std[i];
1407                }
1408            }
1409
1410            // Load the vision/audio feature layer indices if they are explicitly provided
1411            // NOTE: gguf conversions should standardize the values of the vision feature layer to
1412            // be non-negative, since we use -1 to mark values as unset here.
1413            get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false);
1414
1415            // model-specific params
1416            switch (model.proj_type) {
1417                case PROJECTOR_TYPE_MLP:
1418                case PROJECTOR_TYPE_MLP_NORM:
1419                case PROJECTOR_TYPE_LDP:
1420                case PROJECTOR_TYPE_LDPV2:
1421                case PROJECTOR_TYPE_COGVLM:
1422                    {
1423                        hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM;
1424                        hparams.image_pad_color     = {122, 116, 104};
1425                        if (!hparams.image_res_candidates.empty()) {
1426                            hparams.image_resize_pad  = PAD_CEIL;
1427                            hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1428                        } else {
1429                            // llava-1.6 default params
1430                            hparams.image_pad_ov         = PAD_NONE;
1431                            hparams.image_pad_rf         = PAD_CEIL;
1432                            hparams.image_pad_color_rf   = {122, 116, 104};
1433                        }
1434                    } break;
1435                case PROJECTOR_TYPE_GLM_EDGE:
1436                    {
1437                        hparams.image_resize_pad  = PAD_CEIL;
1438                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1439                    } break;
1440                case PROJECTOR_TYPE_MINICPMV:
1441                    {
1442                        // use default llava-uhd preprocessing params
1443                        if (hparams.minicpmv_version == 0) {
1444                            hparams.minicpmv_version = 2; // default to 2 if not set
1445                        }
1446                    } break;
1447                case PROJECTOR_TYPE_MINICPMV4_6:
1448                    {
1449                        // MiniCPM-V 4.6 unified merger projector
1450                        // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
1451                        hparams.n_merge = 4;
1452                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1453                        GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4);
1454
1455                        // borrow wa_layer_indexes for vit_merger insertion point
1456                        std::vector<int> wa_layer_indexes_vec;
1457                        get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false);
1458                        if (!wa_layer_indexes_vec.empty()) {
1459                            hparams.insert_layer_id = wa_layer_indexes_vec[0];
1460                        }
1461                    } break;
1462                case PROJECTOR_TYPE_INTERNVL:
1463                    {
1464                        // use default llava-uhd preprocessing params
1465                        // older version of internvl doesn't have min/max tiles, we need to provide default values for them to avoid issues
1466                        hparams.preproc_min_tiles = 1;
1467                        hparams.preproc_max_tiles = 12;
1468                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1469                        get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
1470                        get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
1471                        GGML_ASSERT(hparams.preproc_min_tiles <= hparams.preproc_max_tiles && hparams.preproc_max_tiles < INT32_MAX);
1472                        set_internvl_dhr_res_candidates(model);
1473                    } break;
1474                case PROJECTOR_TYPE_NEMOTRON_V2_VL:
1475                    {
1476                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1477                    } break;
1478                case PROJECTOR_TYPE_PARAKEET:
1479                    {
1480                        get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
1481                        GGML_ASSERT(hparams.subsampling_factor == 8 &&
1482                            "subsampling_factor must match the conv strides in clip_graph_parakeet::build()");
1483                        get_u32(KEY_A_CONV_KERNEL_SIZE,       hparams.audio_conv_kernel_size);
1484                        GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 &&
1485                            "audio_conv_kernel_size must be a positive odd integer");
1486                        hparams.audio_chunk_len    = 0;
1487                        hparams.audio_sample_rate  = 16000;
1488                        hparams.audio_n_fft        = 512;
1489                        hparams.audio_window_len   = 400;
1490                        hparams.audio_hop_len      = 160;
1491                    } break;
1492                case PROJECTOR_TYPE_IDEFICS3:
1493                    {
1494                        // use default llava-uhd preprocessing params
1495                        hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
1496                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1497                        get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
1498                        hparams.set_limit_image_tokens();
1499                    } break;
1500                case PROJECTOR_TYPE_LFM2:
1501                    {
1502                        hparams.image_resize_algo    = RESIZE_ALGO_BILINEAR;
1503                        hparams.image_resize_algo_rf = RESIZE_ALGO_BILINEAR;
1504                        hparams.image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
1505                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1506                        // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
1507                        hparams.set_limit_image_tokens(64, 256);
1508                    } break;
1509                case PROJECTOR_TYPE_PHI4:
1510                    {
1511                        hparams.n_merge = 1;
1512                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
1513                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1514                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1515                        hparams.set_warmup_n_tokens(16*16);
1516                    } break;
1517                case PROJECTOR_TYPE_PIXTRAL:
1518                    {
1519                        // ref: https://huggingface.co/mistral-community/pixtral-12b/blob/main/preprocessor_config.json
1520                        // TODO: verify the image_min_tokens
1521                        hparams.n_merge = 1; // the original pixtral does not use patch merging
1522                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1523                        hparams.rope_theta = 10000.0f;
1524                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1525                        hparams.set_limit_image_tokens(8, 1024);
1526                        hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
1527                    } break;
1528                case PROJECTOR_TYPE_LIGHTONOCR:
1529                    {
1530                        hparams.n_merge = 1;
1531                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1532                        hparams.rope_theta = 10000.0f;
1533                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1534                        hparams.image_longest_edge = hparams.image_size;
1535                        get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
1536                        hparams.set_limit_image_tokens();
1537                        hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
1538                    } break;
1539                case PROJECTOR_TYPE_DOTS_OCR:
1540                    {
1541                        hparams.rope_theta = 10000.0f;
1542                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
1543                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1544                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1545                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
1546                    } break;
1547                case PROJECTOR_TYPE_DOTS3NOTE_V:
1548                    {
1549                        hparams.rope_theta = 10000.0f;
1550                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1551                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
1552                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1553                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1554                        get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used);
1555                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
1556                    } break;
1557                case PROJECTOR_TYPE_DOTS3NOTE_A:
1558                    {
1559                        hparams.rope_theta = 10000.0f;
1560                        hparams.audio_chunk_len   = 60; // in seconds
1561                        hparams.audio_sample_rate = 16000;
1562                        hparams.audio_n_fft       = 400;
1563                        hparams.audio_window_len  = 400;
1564                        hparams.audio_hop_len     = 160;
1565                    } break;
1566                case PROJECTOR_TYPE_KIMIVL:
1567                    {
1568                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1569                        hparams.rope_theta = 10000.0f;
1570                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1571                        // TODO: check kimivl preprocessor for exact values
1572                        hparams.set_limit_image_tokens(8, 1024);
1573                        hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
1574                    } break;
1575                case PROJECTOR_TYPE_KIMIK25:
1576                    {
1577                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1578                        hparams.rope_theta = 10000.0f;
1579                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1580
1581                        int min_pixels = 0, max_pixels = 0;
1582                        get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false);
1583                        get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false);
1584                        if (min_pixels > 0 && max_pixels > 0) {
1585                            hparams.image_min_pixels = min_pixels;
1586                            hparams.image_max_pixels = max_pixels;
1587                            hparams.warmup_image_size = static_cast<int>(std::sqrt(max_pixels));
1588                        } else {
1589                            hparams.set_limit_image_tokens(2, 4096);
1590                        }
1591                    } break;
1592                case PROJECTOR_TYPE_DEEPSEEK4V:
1593                    {
1594                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1595                        hparams.image_pad_color   = {127, 127, 127};
1596                        hparams.rope_theta = 10000.0f;
1597                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
1598                        get_u32(KEY_IMAGE_MIN_PIXELS,  hparams.image_min_pixels);
1599                        hparams.dsv4_max_n_token  = 384;
1600                        hparams.dsv4_max_wh_ratio = 8;
1601                        const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
1602                        // handle min/max token counts from CLI
1603                        if (hparams.custom_image_min_tokens > 0) {
1604                            hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
1605                        }
1606                        if (hparams.custom_image_max_tokens > 0) {
1607                            // the cap is on the whole token block, keep some room for the resize solver
1608                            hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
1609                        }
1610                        hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
1611                        // a small custom max token count also lowers the min-pixel upscale threshold
1612                        hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
1613                        // avoid OOM on warmup
1614                        const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
1615                        hparams.set_warmup_n_tokens(warmup_side * warmup_side);
1616                    } break;
1617                case PROJECTOR_TYPE_GEMMA3:
1618                    {
1619                        // default value (used by all model sizes in gemma 3 family)
1620                        // number of patches for each **side** is reduced by a factor of 4
1621                        hparams.n_merge = 4;
1622                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
1623                        // test model (tinygemma3) has a different value, we optionally read it
1624                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1625                    } break;
1626
1627                case PROJECTOR_TYPE_GEMMA4V:
1628                case PROJECTOR_TYPE_GEMMA4UV:
1629                    {
1630                        hparams.rope_theta = 100.0f;
1631                        hparams.n_merge = 3; // pooling_kernel_size
1632                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1633                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1634                        if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) {
1635                            // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer
1636                            hparams.patch_size = hparams.patch_size * hparams.n_merge;
1637                            hparams.n_merge = 1;
1638                        }
1639                        hparams.set_limit_image_tokens(70, 1120);
1640                        hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
1641                    } break;
1642
1643                case PROJECTOR_TYPE_GEMMA3NV:
1644                    {
1645                        // Gemma3n uses MobileNetV5 which produces 256 tokens (16x16)
1646                        // Similar configuration to Gemma3
1647                        hparams.n_merge = 1;  // MobileNetV5 handles resizing internally
1648                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
1649                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1650                    } break;
1651                case PROJECTOR_TYPE_QWEN2VL:
1652                case PROJECTOR_TYPE_QWEN25VL:
1653                case PROJECTOR_TYPE_QWEN3VL:
1654                    {
1655                        hparams.n_merge = 2; // default value for Qwen 2 and 2.5
1656                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1657                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1658                        get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it
1659                        // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
1660                        hparams.set_limit_image_tokens(8, 4096);
1661                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
1662                        const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size;
1663                        if (hparams.image_min_pixels < warn_min_pixels) {
1664                            LOG_WRN("%s: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks\n", __func__);
1665                            LOG_WRN("%s: if you encounter problems with accuracy, try adding --image-min-tokens 1024\n", __func__);
1666                            LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
1667                        }
1668                    } break;
1669                case PROJECTOR_TYPE_MINIMAX_M3:
1670                    {
1671                        hparams.n_merge = 2; // spatial_merge_size
1672                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1673                        hparams.image_resize_pad  = PAD_NONE;
1674                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1675                        // n_merge is used as a divisor in clip_image_batch_encode
1676                        // (gh / n_merge); reject 0 to avoid int div-by-zero (DoS).
1677                        GGML_ASSERT(hparams.n_merge > 0);
1678                        hparams.rope_theta = 10000.0f; // vision_config.rope_theta
1679                        // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
1680                        hparams.set_limit_image_tokens(8, 576);
1681                        hparams.set_warmup_n_tokens(16*16);
1682                    } break;
1683                case PROJECTOR_TYPE_MUSE_GLIMMER:
1684                    {
1685                        hparams.n_merge = 2; // pixel-shuffle downsample after the ViT
1686                        hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
1687                        hparams.rope_theta = 10000.0f;
1688                        hparams.muse_glimmer_patch_temporal = 2;
1689                        hparams.muse_glimmer_sparse_factor  = 4; // 3 sparse layers + 1 global, repeating
1690                        get_u32(KEY_SPATIAL_MERGE_SIZE,  hparams.n_merge,             false);
1691                        hparams.set_limit_image_tokens(1, 4096);
1692                        hparams.set_warmup_n_tokens(32*32);
1693                    } break;
1694                case PROJECTOR_TYPE_MIMOVL:
1695                    {
1696                        hparams.n_merge = 2; // spatial_merge_size
1697                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1698                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1699                        get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
1700                        // 1D banded sliding-window radius (visual_token_window_size); required
1701                        get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size);
1702                        std::vector<int> pat;
1703                        get_arr_int(KEY_WA_PATTERN_MODE, pat, true);
1704                        GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer");
1705                        hparams.wa_pattern_mode.assign(pat.begin(), pat.end());
1706                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1707                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1708                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
1709                    } break;
1710                case PROJECTOR_TYPE_STEP3VL:
1711                    {
1712                        hparams.n_merge = 4; // two stride-2 downsamplers after patching
1713                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1714                        hparams.rope_theta = 10000.0f;
1715                        get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
1716                        if (hparams.image_longest_edge == 0) {
1717                            hparams.image_longest_edge = 3024;
1718                        }
1719                        // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens
1720                        hparams.warmup_image_size = hparams.image_size;
1721                    } break;
1722                case PROJECTOR_TYPE_YOUTUVL:
1723                    {
1724                        hparams.n_merge = 2;
1725                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
1726                        hparams.image_resize_pad  = PAD_NONE;
1727                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1728                        get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
1729                        std::vector<int> wa_layer_indexes_vec;
1730                        get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, true);
1731                        for (auto & layer : wa_layer_indexes_vec) {
1732                            hparams.wa_layer_indexes.insert(layer);
1733                        }
1734                        // support max_height * max_width = 8000 * 8000. 8000/16/2 = 250 image tokens
1735                        hparams.set_limit_image_tokens(1, 62500);
1736                        hparams.set_warmup_n_tokens(16*16); // avoid OOM on warmup
1737                    } break;
1738                case PROJECTOR_TYPE_YASA2:
1739                    {
1740                        hparams.ffn_op = FFN_GELU_ERF;
1741                        log_ffn_op = "gelu_erf";
1742                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1743
1744                        // reka model performs better when the image is stretched to fit
1745                        // fixed square size (no padding)
1746                        hparams.image_resize_pad = PAD_NONE;
1747                    } break;
1748                case PROJECTOR_TYPE_GLM4V:
1749                    {
1750                        hparams.rope_theta = 10000.0f;
1751                        hparams.n_merge = 2; // default value for GLM4-V
1752                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1753                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1754                        hparams.set_limit_image_tokens(8, 4096);
1755                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
1756                    } break;
1757                case PROJECTOR_TYPE_LLAMA4:
1758                    {
1759                        hparams.rope_theta = 10000.0f;
1760                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
1761                        get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
1762                        set_llava_uhd_res_candidates(model, 3);
1763                    } break;
1764                case PROJECTOR_TYPE_ULTRAVOX:
1765                case PROJECTOR_TYPE_QWEN2A:
1766                case PROJECTOR_TYPE_QWEN3A:
1767                case PROJECTOR_TYPE_GLMA:
1768                case PROJECTOR_TYPE_VOXTRAL:
1769                case PROJECTOR_TYPE_MERALION:
1770                case PROJECTOR_TYPE_MUSIC_FLAMINGO:
1771                    {
1772                        bool require_stack = model.proj_type == PROJECTOR_TYPE_ULTRAVOX ||
1773                                             model.proj_type == PROJECTOR_TYPE_VOXTRAL ||
1774                                             model.proj_type == PROJECTOR_TYPE_MERALION ||
1775                                             model.proj_type == PROJECTOR_TYPE_GLMA;
1776                        get_u32(KEY_A_PROJ_STACK_FACTOR, hparams.proj_stack_factor, require_stack);
1777                        hparams.ffn_op = FFN_GELU_ERF;
1778                        log_ffn_op = "gelu_erf"; // temporary solution for logging
1779
1780                        // audio preprocessing params
1781                        hparams.audio_chunk_len    = 30; // in seconds
1782                        hparams.audio_sample_rate  = 16000;
1783                        hparams.audio_n_fft        = 400;
1784                        hparams.audio_window_len   = 400;
1785                        hparams.audio_hop_len      = 160;
1786                    } break;
1787                case PROJECTOR_TYPE_MIMO_AUDIO:
1788                    {
1789                        get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false);
1790                        get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false);
1791                        if (hparams.rvq_num_quantizers <= 0) {
1792                            throw std::runtime_error(string_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS));
1793                        }
1794                        if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) {
1795                            throw std::runtime_error(string_format(
1796                                "%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__,
1797                                KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(),
1798                                KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers));
1799                        }
1800                        hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none")
1801                        hparams.rope_theta = 10000.0f;
1802
1803                        // audio preprocessing params (mel spectrogram)
1804                        hparams.audio_sample_rate = 24000;
1805                        hparams.audio_n_fft       = 960;
1806                        hparams.audio_window_len  = 960;
1807                        hparams.audio_hop_len     = 240;
1808
1809                        get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
1810                        std::vector<int> wa_pattern;
1811                        get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true);
1812                        if ((int) wa_pattern.size() != hparams.n_layer) {
1813                            throw std::runtime_error(string_format(
1814                                "%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__,
1815                                KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer));
1816                        }
1817                        hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end());
1818
1819                        get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer);
1820                        get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size);
1821                        if (hparams.audio_local_group_size <= 0) {
1822                            throw std::runtime_error(string_format(
1823                                "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
1824                        }
1825                    } break;
1826                case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
1827                    {
1828                        // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2)
1829                        hparams.audio_sample_rate = 24000;
1830                        hparams.audio_n_fft       = 1024;
1831                        hparams.audio_window_len  = 1024;
1832                        hparams.audio_hop_len     = 256;
1833                    } break;
1834                case PROJECTOR_TYPE_QWEN3TTS_GEN:
1835                    {
1836                        // TODO: hardcoded for now, read from code_predictor_config instead
1837                        hparams.rope_theta = 1000000.0f;
1838
1839                        // code2wav params
1840                        hparams.wav_tfm_n_layer      = 8;
1841                        hparams.wav_tfm_n_embd       = 512;
1842                        hparams.wav_tfm_n_ff         = 1024;
1843                        hparams.wav_tfm_n_head       = 16;
1844                        hparams.wav_tfm_n_head_kv    = 16;
1845                        hparams.wav_tfm_eps          = 1e-5f;
1846                        hparams.wav_tfm_rope_theta   = 10000.0f;
1847                        hparams.wav_upsample_n_block = 2;
1848                        hparams.wav_dac_n_block      = 4;
1849                        hparams.wav_dac_n_res        = 3;
1850                        // matches the reference decoder's sliding_window (speech_tokenizer/config.json)
1851                        hparams.wav_tfm_swa = 72;
1852                    } break;
1853                case PROJECTOR_TYPE_POCKETTTS_SPKENC:
1854                case PROJECTOR_TYPE_POCKETTTS_GEN:
1855                    {
1856                        // mimi front-end takes the raw waveform, no mel
1857                        hparams.audio_sample_rate = 24000;
1858                        // seanet ratios are [6,5,4] in the config, the encoder reverses them
1859                        hparams.seanet_ratios   = { 4, 5, 6 };
1860                        hparams.seanet_n_stage  = (int32_t) hparams.seanet_ratios.size();
1861                        hparams.mimi_downsample = 16;
1862                        // matches the reference transformer's "context"
1863                        hparams.mimi_tfm_context = 250;
1864                        hparams.rope_theta       = 10000.0f;
1865                        // flow_lm defaults, see pocket_tts/default_parameters.py
1866                        hparams.flow_n_step       = 1;
1867                        hparams.gen_eos_threshold = -4.0f;
1868                    } break;
1869                case PROJECTOR_TYPE_PADDLEOCR:
1870                    {
1871                        hparams.n_merge = 2;
1872                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1873                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1874                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1875
1876                        hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup
1877                    } break;
1878                case PROJECTOR_TYPE_DEEPSEEKOCR:
1879                case PROJECTOR_TYPE_DEEPSEEKOCR2:
1880                    {
1881                        hparams.patch_size = 16;
1882                        hparams.image_size = 1024;
1883                        hparams.warmup_image_size = 1024;
1884                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1885                        hparams.image_pad_color = {127, 127, 127};
1886
1887                        get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true);
1888                        get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
1889                        get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
1890                        get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
1891                        hparams.preproc_min_tiles = 2;
1892                        if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
1893                            hparams.preproc_max_tiles = 9;
1894                            hparams.preproc_tile_size = 640;
1895                            // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
1896                            hparams.eps = 1e-5f;
1897                        }
1898                        if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
1899                            hparams.preproc_max_tiles = 6;
1900                            hparams.preproc_tile_size = 768;
1901                            // qwen2 encoder is GQA, requires KEY_N_HEAD_KV
1902                            get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
1903                        }
1904                        // unlimited-ocr shares the v1 projector but tiles up to 32
1905                        get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
1906                        get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
1907                        GGML_ASSERT(hparams.preproc_min_tiles >= 0
1908                                    && hparams.preproc_min_tiles <= hparams.preproc_max_tiles
1909                                    && hparams.preproc_max_tiles <= 256);
1910                     } break;
1911                case PROJECTOR_TYPE_HUNYUANVL:
1912                    {
1913                        hparams.n_merge = 2;
1914                        hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
1915                        hparams.image_resize_pad = PAD_NONE;
1916                        hparams.ffn_op = FFN_GELU;
1917                        hparams.set_limit_image_tokens(256, 16384);
1918                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1919                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false);
1920                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false);
1921                        hparams.set_warmup_n_tokens(32*32);
1922                    } break;
1923                case PROJECTOR_TYPE_LFM2A:
1924                    {
1925                        // audio preprocessing params
1926                        hparams.audio_chunk_len        = 1; // in seconds
1927                        hparams.audio_sample_rate      = 16000;
1928                        hparams.audio_n_fft            = 512;
1929                        hparams.audio_window_len       = 400;
1930                        hparams.audio_hop_len          = 160;
1931                    } break;
1932                case PROJECTOR_TYPE_EXAONE4_5:
1933                    {
1934                        hparams.n_merge = 2;
1935                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
1936                        get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
1937                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
1938                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
1939                        hparams.set_warmup_n_tokens(46 * 46);
1940                        if (hparams.rope_theta <= 0.0f) {
1941                            hparams.rope_theta = 10000.0f;
1942                        }
1943                        get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
1944                    } break;
1945                case PROJECTOR_TYPE_GEMMA4A:
1946                    {
1947                        // Gemma4 feature_extraction_gemma4.py:
1948                        // frame_length_ms=20 -> 320 samples, n_fft=512, hop=10ms -> 160
1949                        hparams.audio_chunk_len        = 0;  // no fixed-length padding
1950                        hparams.audio_sample_rate      = 16000;
1951                        hparams.audio_n_fft            = 512;
1952                        hparams.audio_window_len       = 320;  // 20ms frame (NOT 25ms/400)
1953                        hparams.audio_hop_len          = 160;
1954                        // due to a mistake in the original conversion code, rms_norm_eps is set to a wrong value
1955                        // since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion
1956                        hparams.eps = 1e-6f;
1957                    } break;
1958                case PROJECTOR_TYPE_GEMMA4UA:
1959                    {
1960                        // Encoder-free: raw 16 kHz waveform chunked into 640-sample frames.
1961                        hparams.audio_chunk_len   = 0;
1962                        hparams.audio_sample_rate = 16000;
1963                        hparams.eps               = 1e-6f;
1964                        hparams.n_mel_bins        = 640;
1965                    } break;
1966                case PROJECTOR_TYPE_GRANITE_SPEECH:
1967                    {
1968                        hparams.audio_chunk_len        = 0;
1969                        hparams.audio_sample_rate      = 16000;
1970                        hparams.audio_n_fft            = 512;
1971                        hparams.audio_window_len       = 400;
1972                        hparams.audio_hop_len          = 160;
1973                        get_u32(KEY_A_CHUNK_SIZE,           hparams.audio_chunk_size);
1974                        // context_size is squared for the attn_dists/mask buffers; cap to prevent int32 overflow
1975                        // (legitimate values are small, e.g. 12-200; 8192^2 = 67M still fits int32)
1976                        GGML_ASSERT(hparams.audio_chunk_size > 0 && hparams.audio_chunk_size <= 8192);
1977                        get_u32(KEY_A_CONV_KERNEL_SIZE,     hparams.audio_conv_kernel_size);
1978                        get_u32(KEY_A_MAX_POS_EMB,          hparams.audio_max_pos_emb);
1979                        get_u32(KEY_A_PROJ_WINDOW_SIZE,     hparams.audio_proj_window_size);
1980                        get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate);
1981                        get_u32(KEY_A_PROJ_HEAD_COUNT,      hparams.audio_proj_head_count);
1982                        // NOTE: feature layers loaded above in common path
1983                    } break;
1984                case PROJECTOR_TYPE_JANUS_PRO:
1985                    {
1986                        hparams.image_pad_color   = {127, 127, 127};
1987                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1988                    } break;
1989                case PROJECTOR_TYPE_GRANITE4_VISION:
1990                    {
1991                        // SigLIP tower.
1992                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
1993                        hparams.image_resize_pad = PAD_CEIL;
1994
1995                        // NOTE: feature_layers loaded in common path as optional
1996                        get_arr_int(KEY_PROJ_SPATIAL_OFFSETS, hparams.proj_spatial_offsets);
1997                        if (hparams.feature_layers.size() != hparams.proj_spatial_offsets.size()) {
1998                            throw std::runtime_error(string_format("%s: feature_layers.size() %d != proj_spatial_offsets.size() %d",
1999                                                                   hparams.feature_layers.size(), hparams.proj_spatial_offsets.size()));
2000                        }
2001
2002                        get_u32(KEY_PROJ_SAMPLE_QUERY_SIDE,  hparams.downsample_query_side);
2003                        get_u32(KEY_PROJ_SAMPLE_WINDOW_SIDE, hparams.downsample_window_side);
2004                        hparams.warmup_image_size = hparams.image_size;
2005                    } break;
2006                default:
2007                    throw std::runtime_error(string_format("%s: unknown vision projector type %s\n", __func__, proj_type.c_str()));
2008            }
2009
2010            // sanity check
2011            {
2012                if (hparams.image_size < 0) {
2013                    // note: some models having hparams.image_size == 0, which means the image size is dynamic
2014                    throw std::runtime_error(string_format("%s: image_size (%d) cannot be negative\n", __func__, hparams.image_size));
2015                }
2016                if (hparams.image_size > 8192) {
2017                    // cap prevents int32 overflow in n_patches = (image_size/patch_size)^2
2018                    throw std::runtime_error(string_format("%s: image_size (%d) is too large (max 8192)\n", __func__, hparams.image_size));
2019                }
2020                if (hparams.patch_size <= 0 || hparams.patch_size >= 65536) {
2021                    throw std::runtime_error(string_format("%s: patch_size (%d) must be positive and less than 65536\n", __func__, hparams.patch_size));
2022                }
2023                if (hparams.n_embd <= 0) {
2024                    throw std::runtime_error(string_format("%s: n_embd (%d) must be greater than 0\n", __func__, hparams.n_embd));
2025                }
2026                if (hparams.image_max_pixels < hparams.image_min_pixels) {
2027                    throw std::runtime_error(string_format("%s: image_max_pixels (%d) is less than image_min_pixels (%d)\n", __func__, hparams.image_max_pixels, hparams.image_min_pixels));
2028                }
2029                if (hparams.n_merge <= 0 || hparams.n_merge >= 65536) {
2030                    throw std::runtime_error(string_format("%s: n_merge (%d) must be greater than 0 and less than 65536\n", __func__, hparams.n_merge));
2031                }
2032                if (hparams.attn_window_size > 4096) {
2033                    throw std::runtime_error(string_format("%s: attn_window_size (%d) is too large (max 4096)\n", __func__, hparams.attn_window_size));
2034                }
2035            }
2036
2037            LOG_INF("%s: projector:          %s\n", __func__, proj_type.c_str());
2038            LOG_INF("%s: n_embd:             %d\n", __func__, hparams.n_embd);
2039            LOG_INF("%s: n_head:             %d\n", __func__, hparams.n_head);
2040            LOG_INF("%s: n_ff:               %d\n", __func__, hparams.n_ff);
2041            LOG_INF("%s: n_layer:            %d\n", __func__, hparams.n_layer);
2042            LOG_INF("%s: ffn_op:             %s\n", __func__, log_ffn_op.c_str());
2043            LOG_INF("%s: projection_dim:     %d\n", __func__, hparams.projection_dim);
2044            if (is_vision) {
2045                LOG_INF("\n--- vision hparams ---\n");
2046                LOG_INF("%s: image_size:         %d\n", __func__, hparams.image_size);
2047                LOG_INF("%s: patch_size:         %d\n", __func__, hparams.patch_size);
2048                LOG_INF("%s: has_llava_proj:     %d\n", __func__, hparams.has_llava_projector);
2049                LOG_INF("%s: minicpmv_version:   %d\n", __func__, hparams.minicpmv_version);
2050                LOG_INF("%s: n_merge:            %d\n", __func__, hparams.n_merge);
2051                LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern);
2052                if (!hparams.wa_layer_indexes.empty()) {
2053                    LOG_INF("%s: wa_layer_indexes:  ", __func__);
2054                    for (auto & layer : hparams.wa_layer_indexes) {
2055                        LOG_INF("%d ", layer);
2056                    }
2057                    LOG_INF("\n");
2058                }
2059                if (hparams.image_min_pixels > 0) {
2060                    LOG_INF("%s: image_min_pixels:   %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : "");
2061                }
2062                if (hparams.image_max_pixels > 0) {
2063                    LOG_INF("%s: image_max_pixels:   %d%s\n", __func__, hparams.image_max_pixels, hparams.custom_image_max_tokens > 0 ? " (custom value)" : "");
2064                }
2065                if (hparams.preproc_max_tiles > 0) {
2066                    LOG_INF("%s: preproc_tiles:      %d - %d\n", __func__, hparams.preproc_min_tiles, hparams.preproc_max_tiles);
2067                }
2068            } else if (is_audio) {
2069                LOG_INF("\n--- audio hparams ---\n");
2070                LOG_INF("%s: n_mel_bins:         %d\n", __func__, hparams.n_mel_bins);
2071                LOG_INF("%s: proj_stack_factor:  %d\n", __func__, hparams.proj_stack_factor);
2072                LOG_INF("%s: audio_chunk_len:    %d\n", __func__, hparams.audio_chunk_len);
2073                LOG_INF("%s: audio_sample_rate:  %d\n", __func__, hparams.audio_sample_rate);
2074                LOG_INF("%s: audio_n_fft:        %d\n", __func__, hparams.audio_n_fft);
2075                LOG_INF("%s: audio_window_len:   %d\n", __func__, hparams.audio_window_len);
2076                LOG_INF("%s: audio_hop_len:      %d\n", __func__, hparams.audio_hop_len);
2077
2078                // GEMMA4UA is encoder-free: it uses n_mel_bins as a raw-waveform frame size (640) and has no FFT/filterbank, so the mel-range and FFT
2079                // checks below do not apply to it.
2080                // pocket-tts is encoder-free in the same sense: mimi convolves the raw waveform
2081                const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
2082                                       model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
2083
2084                // Validate audio hparams loaded from GGUF metadata
2085                if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
2086                    throw std::runtime_error(string_format("%s: n_mel_bins (%d) must be in range [1, 256]\n", __func__, hparams.n_mel_bins));
2087                }
2088                if (fft_based && (hparams.audio_sample_rate <= 0 || hparams.audio_n_fft <= 0 || hparams.audio_hop_len <= 0 || hparams.audio_window_len <= 0)) {
2089                    throw std::runtime_error(string_format("%s: audio hparams invalid: sample_rate=%d n_fft=%d window_len=%d hop_len=%d\n",
2090                        __func__, hparams.audio_sample_rate, hparams.audio_n_fft, hparams.audio_window_len, hparams.audio_hop_len));
2091                }
2092            }
2093            LOG_INF("\n");
2094            LOG_INF("%s: model size:         %.2f MiB\n", __func__, model_size / 1024.0 / 1024.0);
2095            LOG_INF("%s: metadata size:      %.2f MiB\n", __func__, ggml_get_mem_size(ctx_meta.get()) / 1024.0 / 1024.0);
2096        }
2097    }
2098
2099    void load_tensors(clip_ctx & ctx_clip) {
2100        auto & model = ctx_clip.model;
2101        auto & hparams = model.hparams;
2102        std::map<std::string, size_t> tensor_offset;
2103        std::vector<ggml_tensor *> tensors_to_load;
2104
2105        auto fin = open_ifstream_binary(fname);
2106        if (!fin) {
2107            throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str()));
2108        }
2109
2110        // TODO @ngxson : support both audio and video in the future
2111        const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a"
2112                             : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code"
2113                             : "v";
2114
2115        // get offsets
2116        for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
2117            const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
2118            tensor_offset[name] = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), i);
2119        }
2120
2121        // create data context
2122        struct ggml_init_params params = {
2123            /*.mem_size =*/ static_cast<size_t>(gguf_get_n_tensors(ctx_gguf.get()) + 1) * ggml_tensor_overhead(),
2124            /*.mem_buffer =*/ NULL,
2125            /*.no_alloc =*/ true,
2126        };
2127        ctx_clip.ctx_data.reset(ggml_init(params));
2128        if (!ctx_clip.ctx_data) {
2129            throw std::runtime_error(string_format("%s: failed to init ggml context\n", __func__));
2130        }
2131
2132        // helper function
2133        std::unordered_set<std::string> loaded_tensor_names;
2134        auto get_tensor = [&](const std::string & name, bool required = true) {
2135            // Each tensor should only be loaded once; duplicates indicate a bug
2136            if (loaded_tensor_names.count(name)) {
2137                throw std::runtime_error(string_format("%s: tensor already loaded: %s\n", __func__, name.c_str()));
2138            }
2139            ggml_tensor * cur = ggml_get_tensor(ctx_meta.get(), name.c_str());
2140            if (!cur && required) {
2141                throw std::runtime_error(string_format("%s: unable to find tensor %s\n", __func__, name.c_str()));
2142            }
2143            if (cur) {
2144                tensors_to_load.push_back(cur);
2145                ggml_tensor * data_tensor = ggml_dup_tensor(ctx_clip.ctx_data.get(), cur);
2146                ggml_set_name(data_tensor, cur->name);
2147                loaded_tensor_names.insert(name);
2148                cur = data_tensor;
2149                // add to weight memory counter
2150                ctx_clip.mem_usage[ggml_backend_get_device(ctx_clip.backend)] += ggml_nbytes(cur);
2151            }
2152            return cur;
2153        };
2154
2155        // pocket-tts: the encoder and the decoder share the same layout, only the prefix differs
2156        auto load_seanet = [&](clip_seanet & seanet, bool is_decoder) {
2157            const char * conv_in  = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_IN    : TN_A_SEANET_CONV_IN;
2158            const char * conv_out = is_decoder ? TN_A_GEN_WAV_SEANET_CONV_OUT   : TN_A_SEANET_CONV_OUT;
2159            const char * res1     = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV1  : TN_A_SEANET_RES_CONV1;
2160            const char * res2     = is_decoder ? TN_A_GEN_WAV_SEANET_RES_CONV2  : TN_A_SEANET_RES_CONV2;
2161            const char * scale    = is_decoder ? TN_A_GEN_WAV_SEANET_SCALE_CONV : TN_A_SEANET_SCALE_CONV;
2162
2163            seanet.conv_in_w  = get_tensor(string_format(conv_in,  "weight"));
2164            seanet.conv_in_b  = get_tensor(string_format(conv_in,  "bias"));
2165            seanet.conv_out_w = get_tensor(string_format(conv_out, "weight"));
2166            seanet.conv_out_b = get_tensor(string_format(conv_out, "bias"));
2167
2168            seanet.stages.resize(hparams.seanet_n_stage);
2169            for (int i = 0; i < hparams.seanet_n_stage; i++) {
2170                auto & stage = seanet.stages[i];
2171                stage.res_conv1_w  = get_tensor(string_format(res1,  i, "weight"));
2172                stage.res_conv1_b  = get_tensor(string_format(res1,  i, "bias"));
2173                stage.res_conv2_w  = get_tensor(string_format(res2,  i, "weight"));
2174                stage.res_conv2_b  = get_tensor(string_format(res2,  i, "bias"));
2175                stage.scale_conv_w = get_tensor(string_format(scale, i, "weight"));
2176                stage.scale_conv_b = get_tensor(string_format(scale, i, "bias"));
2177            }
2178        };
2179
2180        auto get_vector = [&](const std::string & name) {
2181            std::vector<float> result;
2182            auto it = tensor_offset.find(name);
2183            if (it == tensor_offset.end()) {
2184                return result;
2185            }
2186
2187            const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str());
2188            if (idx < 0) {
2189                throw std::runtime_error(string_format("%s: failed to find tensor %s\n", __func__, name.c_str()));
2190            }
2191
2192            if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) {
2193                throw std::runtime_error(string_format("%s: %s must be %s, was %s\n", __func__,
2194                            name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type)));
2195            }
2196
2197            const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx);
2198            if (n_bytes == 0) {
2199                throw std::runtime_error(string_format("%s: tensor %s is empty\n", __func__, name.c_str()));
2200            }
2201
2202            const size_t n_elems = n_bytes / sizeof(float);
2203            result.resize(n_elems);
2204            fin.seekg(it->second, std::ios::beg);
2205            fin.read(reinterpret_cast<char*>(result.data()), n_bytes);
2206            return result;
2207        };
2208
2209        auto get_scalar = [&](const std::string & name, float default_val) {
2210            auto v = get_vector(name);
2211            if (v.empty()) {
2212                return default_val;
2213            }
2214            if (v.size() != 1) {
2215                throw std::runtime_error(string_format("%s: expected scalar tensor '%s' but got %d elements\n",
2216                            __func__, name.c_str(), (int) v.size()));
2217            }
2218
2219            return v[0];
2220        };
2221
2222        model.class_embedding = get_tensor(TN_CLASS_EMBD, false);
2223
2224        model.pre_ln_w = get_tensor(string_format(TN_LN_PRE, prefix, "weight"), false);
2225        model.pre_ln_b = get_tensor(string_format(TN_LN_PRE, prefix, "bias"),   false);
2226
2227        model.post_ln_w = get_tensor(string_format(TN_LN_POST, prefix, "weight"), false);
2228        model.post_ln_b = get_tensor(string_format(TN_LN_POST, prefix, "bias"),   false);
2229
2230        model.patch_bias = get_tensor(TN_PATCH_BIAS, false);
2231        model.patch_embeddings_0 = get_tensor(TN_PATCH_EMBD,   false);
2232        model.patch_embeddings_1 = get_tensor(TN_PATCH_EMBD_1, false);
2233
2234        model.norm_embd_w = get_tensor(string_format(TN_NORM_EMBD, "weight"), false);
2235        model.norm_embd_b = get_tensor(string_format(TN_NORM_EMBD, "bias"),   false);
2236
2237        model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
2238
2239        const bool has_standard_layers = (
2240            model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
2241            model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
2242            model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN);
2243
2244        // layers
2245        const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
2246        model.layers.resize(n_layers_to_load);
2247        for (int il = 0; il < n_layers_to_load; ++il) {
2248            auto & layer = model.layers[il];
2249            layer.k_w    = get_tensor(string_format(TN_ATTN_K,      prefix, il, "weight"), false);
2250            layer.q_w    = get_tensor(string_format(TN_ATTN_Q,      prefix, il, "weight"), false);
2251            layer.v_w    = get_tensor(string_format(TN_ATTN_V,      prefix, il, "weight"), false);
2252            layer.o_w    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"));
2253            layer.qkv_w  = get_tensor(string_format(TN_ATTN_QKV,    prefix, il, "weight"), false);
2254            layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "weight"), false);
2255            layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false);
2256            layer.ln_1_w = get_tensor(string_format(TN_LN_1,        prefix, il, "weight"), false);
2257            layer.ln_2_w = get_tensor(string_format(TN_LN_2,        prefix, il, "weight"), false);
2258            layer.ls_1_w        = get_tensor(string_format(TN_LS_1,         prefix, il, "weight"), false); // no bias
2259            layer.ls_2_w        = get_tensor(string_format(TN_LS_2,         prefix, il, "weight"), false); // no bias
2260            layer.ls_out_w      = get_tensor(string_format(TN_LS_OUT,        prefix, il, "weight"), false); // no bias
2261            layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias
2262            layer.ff_post_norm_w   = get_tensor(string_format(TN_FFN_POST_NORM,  prefix, il, "weight"), false); // no bias
2263
2264            layer.k_b    = get_tensor(string_format(TN_ATTN_K,      prefix, il, "bias"), false);
2265            layer.q_b    = get_tensor(string_format(TN_ATTN_Q,      prefix, il, "bias"), false);
2266            layer.v_b    = get_tensor(string_format(TN_ATTN_V,      prefix, il, "bias"), false);
2267            layer.o_b    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "bias"), false);
2268            layer.qkv_b  = get_tensor(string_format(TN_ATTN_QKV,    prefix, il, "bias"), false);
2269            layer.ln_1_b = get_tensor(string_format(TN_LN_1,        prefix, il, "bias"), false);
2270            layer.ln_2_b = get_tensor(string_format(TN_LN_2,        prefix, il, "bias"), false);
2271
2272            // MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present
2273            layer.ff_gate_inp_w  = get_tensor(string_format(TN_FFN_GATE_INP,  prefix, il, "weight"), false);
2274            layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false);
2275            layer.ff_up_exps_w   = get_tensor(string_format(TN_FFN_UP_EXPS,   prefix, il, "weight"), false);
2276            layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false);
2277            layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false);
2278            const bool is_moe = layer.ff_gate_exps_w != nullptr;
2279
2280            // ffn
2281            layer.ff_up_w   = get_tensor(string_format(TN_FFN_UP,   prefix, il, "weight"), !is_moe);
2282            layer.ff_up_b   = get_tensor(string_format(TN_FFN_UP,   prefix, il, "bias"),   false);
2283            layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false);
2284            layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"),   false);
2285            layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe);
2286            layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"),   false);
2287
2288            // mimovl per-head attention sink bias
2289            layer.attn_sinks = get_tensor(string_format(TN_ATTN_SINKS, prefix, il), false);
2290
2291            // qwen3vl deepstack layer
2292            layer.deepstack_norm_w = get_tensor(string_format(TN_DEEPSTACK_NORM, il, "weight"), false);
2293            layer.deepstack_norm_b = get_tensor(string_format(TN_DEEPSTACK_NORM, il, "bias"), false);
2294            layer.deepstack_fc1_w  = get_tensor(string_format(TN_DEEPSTACK_FC1,  il, "weight"), false);
2295            layer.deepstack_fc1_b  = get_tensor(string_format(TN_DEEPSTACK_FC1,  il, "bias"), false);
2296            layer.deepstack_fc2_w  = get_tensor(string_format(TN_DEEPSTACK_FC2,  il, "weight"), false);
2297            layer.deepstack_fc2_b  = get_tensor(string_format(TN_DEEPSTACK_FC2,  il, "bias"), false);
2298            if (layer.has_deepstack()) {
2299                model.n_deepstack_layers++;
2300            }
2301
2302            // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here
2303            // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check!
2304            bool is_ffn_swapped = (
2305                    // only old models need this fix
2306                    model.proj_type == PROJECTOR_TYPE_MLP
2307                    || model.proj_type == PROJECTOR_TYPE_MLP_NORM
2308                    || model.proj_type == PROJECTOR_TYPE_LDP
2309                    || model.proj_type == PROJECTOR_TYPE_LDPV2
2310                    || model.proj_type == PROJECTOR_TYPE_QWEN2VL
2311                    || model.proj_type == PROJECTOR_TYPE_QWEN25VL
2312                    || model.proj_type == PROJECTOR_TYPE_EXAONE4_5
2313                    || model.proj_type == PROJECTOR_TYPE_GLM_EDGE
2314                    || model.proj_type == PROJECTOR_TYPE_GEMMA3
2315                    || model.proj_type == PROJECTOR_TYPE_IDEFICS3
2316                    || model.proj_type == PROJECTOR_TYPE_MINICPMV
2317                    || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
2318                ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
2319            if (is_ffn_swapped) {
2320                // swap up and down weights
2321                ggml_tensor * tmp = layer.ff_up_w;
2322                layer.ff_up_w = layer.ff_down_w;
2323                layer.ff_down_w = tmp;
2324                // swap up and down biases
2325                tmp = layer.ff_up_b;
2326                layer.ff_up_b = layer.ff_down_b;
2327                layer.ff_down_b = tmp;
2328                if (il == 0) {
2329                    LOG_WRN("%s: ffn up/down are swapped\n", __func__);
2330                }
2331            }
2332        }
2333
2334
2335        switch (model.proj_type) {
2336            case PROJECTOR_TYPE_MLP:
2337            case PROJECTOR_TYPE_MLP_NORM:
2338                {
2339                    // LLaVA projection
2340                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"), false);
2341                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false);
2342                    // Yi-type llava
2343                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"), false);
2344                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false);
2345                    // missing in Yi-type llava
2346                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"), false);
2347                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
2348                    // Yi-type llava
2349                    model.mm_3_w = get_tensor(string_format(TN_LLAVA_PROJ, 3, "weight"), false);
2350                    model.mm_3_b = get_tensor(string_format(TN_LLAVA_PROJ, 3, "bias"), false);
2351                    model.mm_4_w = get_tensor(string_format(TN_LLAVA_PROJ, 4, "weight"), false);
2352                    model.mm_4_b = get_tensor(string_format(TN_LLAVA_PROJ, 4, "bias"), false);
2353                    if (model.mm_3_w) {
2354                        // TODO: this is a hack to support Yi-type llava
2355                        model.proj_type = PROJECTOR_TYPE_MLP_NORM;
2356                    }
2357                    model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
2358                } break;
2359            case PROJECTOR_TYPE_LDP:
2360                {
2361                    // MobileVLM projection
2362                    model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
2363                    model.mm_model_mlp_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias"));
2364                    model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
2365                    model.mm_model_mlp_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
2366                    model.mm_model_block_1_block_0_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));
2367                    model.mm_model_block_1_block_0_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));
2368                    model.mm_model_block_1_block_0_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));
2369                    model.mm_model_block_1_block_1_fc1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight"));
2370                    model.mm_model_block_1_block_1_fc1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias"));
2371                    model.mm_model_block_1_block_1_fc2_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight"));
2372                    model.mm_model_block_1_block_1_fc2_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias"));
2373                    model.mm_model_block_1_block_2_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));
2374                    model.mm_model_block_1_block_2_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));
2375                    model.mm_model_block_1_block_2_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));
2376                    model.mm_model_block_2_block_0_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));
2377                    model.mm_model_block_2_block_0_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));
2378                    model.mm_model_block_2_block_0_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));
2379                    model.mm_model_block_2_block_1_fc1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight"));
2380                    model.mm_model_block_2_block_1_fc1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias"));
2381                    model.mm_model_block_2_block_1_fc2_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight"));
2382                    model.mm_model_block_2_block_1_fc2_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias"));
2383                    model.mm_model_block_2_block_2_0_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));
2384                    model.mm_model_block_2_block_2_1_w = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));
2385                    model.mm_model_block_2_block_2_1_b = get_tensor(string_format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));
2386                } break;
2387            case PROJECTOR_TYPE_LDPV2:
2388                {
2389                    // MobilVLM_V2 projection
2390                    model.mm_model_mlp_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
2391                    model.mm_model_mlp_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias"));
2392                    model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight"));
2393                    model.mm_model_mlp_2_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "bias"));
2394                    model.mm_model_peg_0_w = get_tensor(string_format(TN_MVLM_PROJ_PEG, 0, "weight"));
2395                    model.mm_model_peg_0_b = get_tensor(string_format(TN_MVLM_PROJ_PEG, 0, "bias"));
2396                } break;
2397            case PROJECTOR_TYPE_MINICPMV:
2398                {
2399                    // model.mm_model_pos_embed = get_tensor(new_clip->ctx_data, TN_MINICPMV_POS_EMBD);
2400                    model.mm_model_pos_embed_k = get_tensor(TN_MINICPMV_POS_EMBD_K);
2401                    model.mm_model_query = get_tensor(TN_MINICPMV_QUERY);
2402                    model.mm_model_proj = get_tensor(TN_MINICPMV_PROJ);
2403                    model.mm_model_kv_proj = get_tensor(TN_MINICPMV_KV_PROJ);
2404                    model.mm_model_attn_q_w = get_tensor(string_format(TN_MINICPMV_ATTN, "q", "weight"));
2405                    model.mm_model_attn_k_w = get_tensor(string_format(TN_MINICPMV_ATTN, "k", "weight"));
2406                    model.mm_model_attn_v_w = get_tensor(string_format(TN_MINICPMV_ATTN, "v", "weight"));
2407                    model.mm_model_attn_q_b = get_tensor(string_format(TN_MINICPMV_ATTN, "q", "bias"));
2408                    model.mm_model_attn_k_b = get_tensor(string_format(TN_MINICPMV_ATTN, "k", "bias"));
2409                    model.mm_model_attn_v_b = get_tensor(string_format(TN_MINICPMV_ATTN, "v", "bias"));
2410                    model.mm_model_attn_o_w = get_tensor(string_format(TN_MINICPMV_ATTN, "out", "weight"));
2411                    model.mm_model_attn_o_b = get_tensor(string_format(TN_MINICPMV_ATTN, "out", "bias"));
2412                    model.mm_model_ln_q_w = get_tensor(string_format(TN_MINICPMV_LN, "q", "weight"));
2413                    model.mm_model_ln_q_b = get_tensor(string_format(TN_MINICPMV_LN, "q", "bias"));
2414                    model.mm_model_ln_kv_w = get_tensor(string_format(TN_MINICPMV_LN, "kv", "weight"));
2415                    model.mm_model_ln_kv_b = get_tensor(string_format(TN_MINICPMV_LN, "kv", "bias"));
2416                    model.mm_model_ln_post_w = get_tensor(string_format(TN_MINICPMV_LN, "post", "weight"));
2417                    model.mm_model_ln_post_b = get_tensor(string_format(TN_MINICPMV_LN, "post", "bias"));
2418                } break;
2419            case PROJECTOR_TYPE_MINICPMV4_6:
2420                {
2421                    const bool merger_required = hparams.n_merge == 4;
2422                    auto get_merger_tensor = [&](const std::string & name, bool required = true) {
2423                        return get_tensor(name, merger_required && required);
2424                    };
2425
2426                    // ViT merger: window self-attention
2427                    model.vit_merger_ln1_w     = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight"));
2428                    model.vit_merger_ln1_b     = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias"));
2429                    model.vit_merger_attn_q_w  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight"));
2430                    model.vit_merger_attn_q_b  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);
2431                    model.vit_merger_attn_k_w  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight"));
2432                    model.vit_merger_attn_k_b  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false);
2433                    model.vit_merger_attn_v_w  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight"));
2434                    model.vit_merger_attn_v_b  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false);
2435                    model.vit_merger_attn_o_w  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight"));
2436                    model.vit_merger_attn_o_b  = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false);
2437                    // ViT merger: MLP downsample
2438                    model.vit_merger_ds_ln_w   = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight"));
2439                    model.vit_merger_ds_ln_b   = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias"));
2440                    model.vit_merger_ds_up_w   = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight"));
2441                    model.vit_merger_ds_up_b   = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false);
2442                    model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight"));
2443                    model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);
2444                    // Final Merger (DownsampleMLP)
2445                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
2446                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);
2447                    model.mm_ffn_up_w     = get_tensor(string_format(TN_MM_UP,   "weight"));
2448                    model.mm_ffn_up_b     = get_tensor(string_format(TN_MM_UP,   "bias"), false);
2449                    model.mm_ffn_down_w   = get_tensor(string_format(TN_MM_DOWN, "weight"));
2450                    model.mm_ffn_down_b   = get_tensor(string_format(TN_MM_DOWN, "bias"), false);
2451                } break;
2452            case PROJECTOR_TYPE_GLM_EDGE:
2453                {
2454                    model.mm_model_adapter_conv_w = get_tensor(string_format(TN_GLM_ADAPER_CONV, "weight"));
2455                    model.mm_model_adapter_conv_b = get_tensor(string_format(TN_GLM_ADAPER_CONV, "bias"));
2456                    model.mm_model_mlp_0_w = get_tensor(string_format(TN_GLM_ADAPTER_LINEAR, "weight"));
2457                    model.mm_model_ln_q_w = get_tensor(string_format(TN_GLM_ADAPTER_NORM_1, "weight"));
2458                    model.mm_model_ln_q_b = get_tensor(string_format(TN_GLM_ADAPTER_NORM_1, "bias"));
2459                    model.mm_model_mlp_1_w = get_tensor(string_format(TN_GLM_ADAPTER_D_H_2_4H, "weight"));
2460                    model.mm_model_mlp_2_w = get_tensor(string_format(TN_GLM_ADAPTER_GATE, "weight"));
2461                    model.mm_model_mlp_3_w = get_tensor(string_format(TN_GLM_ADAPTER_D_4H_2_H, "weight"));
2462                    model.mm_boi = get_tensor(string_format(TN_TOK_GLM_BOI));
2463                    model.mm_eoi = get_tensor(string_format(TN_TOK_GLM_EOI));
2464                } break;
2465            case PROJECTOR_TYPE_QWEN2VL:
2466            case PROJECTOR_TYPE_QWEN25VL:
2467            case PROJECTOR_TYPE_EXAONE4_5:
2468                {
2469                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2470                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
2471                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2472                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2473                } break;
2474            case PROJECTOR_TYPE_QWEN3VL:
2475                {
2476                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2477                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
2478                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2479                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2480                } break;
2481            case PROJECTOR_TYPE_MIMOVL:
2482                {
2483                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2484                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false);
2485                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2486                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
2487                } break;
2488            case PROJECTOR_TYPE_MINIMAX_M3:
2489                {
2490                    // per-patch MLP: mm.1 -> gelu -> mm.2
2491                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2492                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
2493                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2494                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2495                    // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2
2496                    model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight"));
2497                    model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias"));
2498                    model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
2499                    model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
2500                } break;
2501            case PROJECTOR_TYPE_MUSE_GLIMMER:
2502                {
2503                    // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim)
2504                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2505                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2506                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2507                } break;
2508            case PROJECTOR_TYPE_STEP3VL:
2509                {
2510                    model.mm_0_w     = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2511                    model.mm_0_b     = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false);
2512                    model.mm_1_w     = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2513                    model.mm_1_b     = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false);
2514                    model.mm_model_proj = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
2515                } break;
2516            case PROJECTOR_TYPE_YOUTUVL:
2517                {
2518                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);        // merger.ln_q (RMS norm)
2519                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));  // merger.mlp.0
2520                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
2521                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));  // merger.mlp.2
2522                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2523                } break;
2524            case PROJECTOR_TYPE_YASA2:
2525                {
2526                    // reuse tensors already loaded by the common section
2527                    // (TN_PATCH_EMBD and TN_PATCH_BIAS have the same tensor names)
2528                    GGML_ASSERT(model.patch_embeddings_0 && "yasa2 requires v.patch_embd.weight");
2529                    model.yasa_patch_w = model.patch_embeddings_0;
2530                    model.yasa_patch_b = model.patch_bias;
2531                    model.yasa_patch_ln_w = get_tensor(TN_YASA_PATCH_LN_W, false);
2532                    model.yasa_patch_ln_b = get_tensor(TN_YASA_PATCH_LN_B, false);
2533                    model.yasa_backbone_ln_w = get_tensor(TN_YASA_BACKBONE_LN_W, false);
2534                    model.yasa_backbone_ln_b = get_tensor(TN_YASA_BACKBONE_LN_B, false);
2535                    model.yasa_vision_pos_embed = get_tensor(TN_YASA_POS_EMBD, false);
2536                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2537                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false);
2538                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2539                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
2540
2541                    model.yasa_stages.clear();
2542                    for (int s = 0; ; ++s) {
2543                        yasa2_stage stage;
2544                        stage.down_ln_w   = get_tensor(string_format(TN_YASA_STAGE_DOWN_LN, s, "weight"), false);
2545                        stage.down_ln_b   = get_tensor(string_format(TN_YASA_STAGE_DOWN_LN, s, "bias"), false);
2546                        stage.down_conv_w = get_tensor(string_format(TN_YASA_STAGE_DOWN_CONV, s, "weight"), false);
2547                        stage.down_conv_b = get_tensor(string_format(TN_YASA_STAGE_DOWN_CONV, s, "bias"), false);
2548
2549                        for (int bi = 0; ; ++bi) {
2550                            yasa2_block blk;
2551                            blk.dw_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "dw", "weight"), false);
2552                            if (!blk.dw_w) {
2553                                break;
2554                            }
2555                            blk.dw_b  = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "dw", "bias"), false);
2556                            blk.ln_w  = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "ln", "weight"), false);
2557                            blk.ln_b  = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "ln", "bias"), false);
2558                            blk.pw1_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "weight"), false);
2559                            blk.pw1_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw1", "bias"), false);
2560                            blk.grn_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "grn", "weight"), false);
2561                            blk.grn_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "grn", "bias"), false);
2562                            blk.pw2_w = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "weight"), false);
2563                            blk.pw2_b = get_tensor(string_format(TN_YASA_STAGE_BLK, s, bi, "pw2", "bias"), false);
2564                            stage.blocks.push_back(blk);
2565                        }
2566
2567                        if (!stage.down_conv_w && stage.blocks.empty()) {
2568                            break;
2569                        }
2570                        model.yasa_stages.push_back(std::move(stage));
2571                    }
2572                } break;
2573            case PROJECTOR_TYPE_GLM4V:
2574                {
2575                    model.mm_fc_w        = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
2576                    model.mm_ffn_up_w    = get_tensor(string_format(TN_MM_UP,        "weight"));
2577                    model.mm_ffn_up_b    = get_tensor(string_format(TN_MM_UP,        "bias"), false);
2578                    model.mm_ffn_gate_w  = get_tensor(string_format(TN_MM_GATE,      "weight"));
2579                    model.mm_ffn_gate_b  = get_tensor(string_format(TN_MM_GATE,      "bias"), false);
2580                    model.mm_ffn_down_w  = get_tensor(string_format(TN_MM_DOWN,      "weight"));
2581                    model.mm_ffn_down_b  = get_tensor(string_format(TN_MM_DOWN,      "bias"), false);
2582                    model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
2583                    model.mm_post_norm_b = get_tensor(string_format(TN_MM_POST_NORM, "bias"), false);
2584                    model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"));
2585                    model.mm_patch_merger_b = get_tensor(string_format(TN_MM_PATCH_MERGER, "bias"));
2586                } break;
2587            case PROJECTOR_TYPE_GEMMA3:
2588                {
2589                    model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
2590                    model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N);
2591                } break;
2592            case PROJECTOR_TYPE_GEMMA4V:
2593                {
2594                    model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
2595                    model.std_bias  = get_tensor(TN_STD_BIAS,  false);
2596                    model.std_scale = get_tensor(TN_STD_SCALE, false);
2597                    // load scalar for Gemma4ClippableLinear
2598                    for (auto * tensor : tensors_to_load) {
2599                        std::string name = tensor->name;
2600                        if (string_ends_with(name, ".weight")) {
2601                            std::string name_inp_max = name;
2602                            std::string name_inp_min = name;
2603                            std::string name_out_max = name;
2604                            std::string name_out_min = name;
2605                            string_replace_all(name_inp_max, ".weight", ".input_max");
2606                            string_replace_all(name_inp_min, ".weight", ".input_min");
2607                            string_replace_all(name_out_max, ".weight", ".output_max");
2608                            string_replace_all(name_out_min, ".weight", ".output_min");
2609                            model.clamp_info_map[name] = {
2610                                get_scalar(name_inp_max, FLT_MAX),
2611                                get_scalar(name_inp_min, -FLT_MAX),
2612                                get_scalar(name_out_max, FLT_MAX),
2613                                get_scalar(name_out_min, -FLT_MAX)
2614                            };
2615                        }
2616                    }
2617                } break;
2618            case PROJECTOR_TYPE_GEMMA4UV:
2619                {
2620                    model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
2621                    model.patch_norm_1_w = get_tensor(string_format(TN_PATCH_NORM, 1, "weight"));
2622                    model.patch_norm_1_b = get_tensor(string_format(TN_PATCH_NORM, 1, "bias"));
2623                    model.patch_norm_2_w = get_tensor(string_format(TN_PATCH_NORM, 2, "weight"));
2624                    model.patch_norm_2_b = get_tensor(string_format(TN_PATCH_NORM, 2, "bias"));
2625                    model.patch_norm_3_w = get_tensor(string_format(TN_PATCH_NORM, 3, "weight")); // pos_norm
2626                    model.patch_norm_3_b = get_tensor(string_format(TN_PATCH_NORM, 3, "bias"));   // pos_norm
2627                } break;
2628            case PROJECTOR_TYPE_GEMMA3NV:
2629                {
2630                    model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false);
2631                    model.mobilenet_stem_conv_b = get_tensor(TN_MNV5_STEM_BIAS, false);
2632                    model.mobilenet_stem_norm_w = get_tensor(TN_MNV5_STEM_BN, false);
2633
2634                    model.msfa_ffn_expand_w  = get_tensor(TN_MNV5_MSFA_FFN_EXP_W, false);
2635                    model.msfa_ffn_expand_bn = get_tensor(TN_MNV5_MSFA_FFN_EXP_BN, false); // Consume BN if present but likely folded
2636                    model.msfa_ffn_project_w = get_tensor(TN_MNV5_MSFA_FFN_PROJ_W, false);
2637                    model.msfa_ffn_project_bn = get_tensor(TN_MNV5_MSFA_FFN_PROJ_BN, false);
2638
2639                    model.msfa_concat_norm_w = get_tensor(TN_MNV5_MSFA_NORM, false);
2640
2641                    // Dynamically load blocks stage by stage
2642                    for (int stage = 0; stage < 4; ++stage) {
2643                        int blocks_found_in_stage = 0;
2644
2645                        for (int blk_idx = 0; ; ++blk_idx) {
2646                            bool found_block = false;
2647                            mobilenetv5_block block;
2648
2649                            // 1. Check for Edge Residual (S0)
2650                            block.s0_conv_exp_w = get_tensor(string_format(TN_MNV5_BLK_S0_EXP_W, stage, blk_idx), false);
2651                            if (block.s0_conv_exp_w) {
2652                                found_block = true;
2653                                block.s0_bn1_w      = get_tensor(string_format(TN_MNV5_BLK_S0_BN1_W, stage, blk_idx), false);
2654                                block.s0_conv_pwl_w = get_tensor(string_format(TN_MNV5_BLK_S0_PWL_W, stage, blk_idx), false);
2655                                block.s0_bn2_w      = get_tensor(string_format(TN_MNV5_BLK_S0_BN2_W, stage, blk_idx), false);
2656                            }
2657                            // 2. Check for UIR (Universal Inverted Residual)
2658                            else {
2659                                // Check for dw_start OR pw_exp (some UIR blocks skip dw_start)
2660                                block.dw_start_w = get_tensor(string_format(TN_MNV5_BLK_DW_START_W, stage, blk_idx), false);
2661                                block.pw_exp_w   = get_tensor(string_format(TN_MNV5_BLK_PW_EXP_W, stage, blk_idx), false);
2662
2663                                if (block.dw_start_w || block.pw_exp_w) {
2664                                    found_block = true;
2665                                    if (block.dw_start_w) {
2666                                        block.dw_start_bn_w = get_tensor(string_format(TN_MNV5_BLK_DW_START_BN, stage, blk_idx), false);
2667                                    }
2668                                    if (block.pw_exp_w) {
2669                                        block.pw_exp_bn_w   = get_tensor(string_format(TN_MNV5_BLK_PW_EXP_BN, stage, blk_idx), false);
2670                                    }
2671                                    block.dw_mid_w      = get_tensor(string_format(TN_MNV5_BLK_DW_MID_W, stage, blk_idx), false);
2672                                    if (block.dw_mid_w) {
2673                                        block.dw_mid_bn_w   = get_tensor(string_format(TN_MNV5_BLK_DW_MID_BN, stage, blk_idx), false);
2674                                    }
2675                                    block.pw_proj_w     = get_tensor(string_format(TN_MNV5_BLK_PW_PROJ_W, stage, blk_idx), false);
2676                                    if (block.pw_proj_w) {
2677                                        block.pw_proj_bn_w  = get_tensor(string_format(TN_MNV5_BLK_PW_PROJ_BN, stage, blk_idx), false);
2678                                    }
2679                                    block.layer_scale_w = get_tensor(string_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false);
2680                                }
2681                            }
2682
2683                            // 3. Check for Attention (MQA)
2684                            // Even if UIR/Edge check failed, this might be a pure attention block
2685                            ggml_tensor* attn_q_check = get_tensor(string_format(TN_MNV5_ATTN_Q_W, stage, blk_idx), false);
2686                            if (attn_q_check) {
2687                                found_block = true;
2688                                block.attn_q_w = attn_q_check;
2689                                block.attn_k_w = get_tensor(string_format(TN_MNV5_ATTN_K_W, stage, blk_idx), false);
2690                                block.attn_v_w = get_tensor(string_format(TN_MNV5_ATTN_V_W, stage, blk_idx), false);
2691                                block.attn_o_w = get_tensor(string_format(TN_MNV5_ATTN_O_W, stage, blk_idx), false);
2692                                block.attn_k_dw_w   = get_tensor(string_format(TN_MNV5_ATTN_K_DW, stage, blk_idx), false);
2693                                block.attn_k_norm_w = get_tensor(string_format(TN_MNV5_ATTN_K_NORM, stage, blk_idx), false);
2694                                block.attn_v_dw_w   = get_tensor(string_format(TN_MNV5_ATTN_V_DW, stage, blk_idx), false);
2695                                block.attn_v_norm_w = get_tensor(string_format(TN_MNV5_ATTN_V_NORM, stage, blk_idx), false);
2696                                block.attn_norm_w   = get_tensor(string_format(TN_MNV5_ATTN_NORM, stage, blk_idx), false);
2697                                // Note: Attention blocks also have layer_scale, load it if not already loaded by UIR check
2698                                if (!block.layer_scale_w) {
2699                                    block.layer_scale_w = get_tensor(string_format(TN_MNV5_BLK_LAYER_SCALE, stage, blk_idx), false);
2700                                }
2701                            }
2702
2703                            if (found_block) {
2704                                model.mobilenet_blocks.push_back(block);
2705                                blocks_found_in_stage++;
2706                            } else {
2707                                // End of blocks for this stage
2708                                break;
2709                            }
2710                        }
2711
2712                        // Track where this stage ends in the flat vector
2713                        if (blocks_found_in_stage > 0) {
2714                            model.mobilenet_stage_ends.push_back(model.mobilenet_blocks.size() - 1);
2715                            LOG_INF("%s: Stage %d ended at global block index %zu\n", __func__, stage, model.mobilenet_blocks.size() - 1);
2716                        }
2717                    }
2718                    model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
2719                    model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N);
2720                } break;
2721            case PROJECTOR_TYPE_IDEFICS3:
2722                {
2723                    model.mm_fc_w = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
2724                } break;
2725            case PROJECTOR_TYPE_LFM2:
2726                {
2727                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM, false);
2728                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);
2729                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2730                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
2731                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2732                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2733                } break;
2734            case PROJECTOR_TYPE_KIMIVL:
2735            case PROJECTOR_TYPE_PADDLEOCR:
2736            case PROJECTOR_TYPE_KIMIK25:
2737                {
2738                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
2739                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
2740                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2741                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
2742                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2743                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2744                } break;
2745            case PROJECTOR_TYPE_DEEPSEEK4V:
2746                {
2747                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2748                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
2749                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2750                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2751                    // sentinel token embeddings written into the output block
2752                    model.image_newline        = get_tensor(TN_IMAGE_NEWLINE);
2753                    model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
2754                    model.token_embd_img_end   = get_tensor(TN_TOK_IMG_END);
2755                    model.token_embd_img_pad   = get_tensor(TN_TOK_IMG_PAD);
2756                } break;
2757            case PROJECTOR_TYPE_PIXTRAL:
2758                {
2759                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2760                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false);
2761                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2762                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
2763                    // [IMG_BREAK] token embedding
2764                    model.token_embd_img_break = get_tensor(TN_TOK_IMG_BREAK);
2765                    // for mistral small 3.1
2766                    model.mm_input_norm_w   = get_tensor(TN_MM_INP_NORM, false);
2767                    model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false);
2768                } break;
2769            case PROJECTOR_TYPE_LIGHTONOCR:
2770                {
2771                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
2772                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"), false);
2773                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2774                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
2775                    model.mm_input_norm_w   = get_tensor(TN_MM_INP_NORM, false);
2776                    model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false);
2777                } break;
2778            case PROJECTOR_TYPE_DOTS_OCR:
2779            case PROJECTOR_TYPE_DOTS3NOTE_V:
2780                {
2781                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
2782                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
2783                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
2784                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
2785                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
2786                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
2787                    // post_trunk_norm: applied after all ViT blocks, before the merger
2788                    model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
2789                } break;
2790            case PROJECTOR_TYPE_DOTS3NOTE_A:
2791                {
2792                    model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight"));
2793                    model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias"));
2794                    model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight"));
2795                    model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias"));
2796                    model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight"));
2797                    model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias"));
2798                    model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias
2799                    // adapter: LayerNorm -> Linear -> GELU -> Linear
2800                    model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
2801                    model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
2802                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
2803                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
2804                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
2805                    model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
2806                } break;
2807            case PROJECTOR_TYPE_ULTRAVOX:
2808                {
2809                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
2810                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
2811                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
2812                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
2813                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
2814                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
2815                    model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
2816                    model.mm_norm_mid_w = get_tensor(string_format(TN_MM_NORM_MID, "weight"));
2817                } break;
2818            case PROJECTOR_TYPE_MERALION:
2819                {
2820                    // Whisper encoder conv layers
2821                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
2822                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
2823                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
2824                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
2825                    // MERaLiON adaptor: 4 linear layers + ln_pre
2826                    // linear_0 = frame compression (19200->6400) + SiLU
2827                    // linear_1 = gate_proj (6400->6400) for GLU
2828                    // linear_2 = pool_proj (6400->6400) for GLU
2829                    // linear_3 = out_proj  (6400->3584)
2830                    model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
2831                    model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
2832                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
2833                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
2834                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
2835                    model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
2836                    model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
2837                    model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
2838                    // ln_speech (LayerNorm before adaptor)
2839                    model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
2840                    model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
2841                } break;
2842            case PROJECTOR_TYPE_QWEN2A:
2843                {
2844                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
2845                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
2846                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
2847                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
2848                    model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
2849                    model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
2850                } break;
2851            case PROJECTOR_TYPE_QWEN3A:
2852                {
2853                    model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight"));
2854                    model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias"));
2855                    model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight"));
2856                    model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias"));
2857                    model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight"));
2858                    model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias"));
2859                    model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias
2860                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
2861                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
2862                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
2863                    model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
2864                } break;
2865            case PROJECTOR_TYPE_MIMO_AUDIO:
2866                {
2867                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
2868                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
2869                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
2870                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
2871                    model.downsample_conv_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight"));
2872                    model.downsample_norm_w = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "weight"));
2873                    model.downsample_norm_b = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "bias"));
2874                    model.rvq_codebook   = get_tensor(string_format(TN_A_RVQ_CODEBOOK, "weight"), false);
2875                    model.mm_a_code_embd = get_tensor(string_format(TN_MM_A_CODE_EMBD, "weight"), false);
2876                    if (!model.rvq_codebook || !model.mm_a_code_embd) {
2877                        throw std::runtime_error(string_format("%s: mimo_audio: missing %s or %s\n", __func__,
2878                            TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD));
2879                    }
2880                    // hparams.rvq_codebook_size comes from GGUF metadata and is independent of the
2881                    // tensors' actual shapes - bound it so codebook/code_embd views built from it
2882                    // (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab.
2883                    for (int32_t bins : hparams.rvq_codebook_size) {
2884                        if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) {
2885                            throw std::runtime_error(string_format(
2886                                "%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n",
2887                                __func__, KEY_A_RVQ_CODEBOOK_SIZE, bins));
2888                        }
2889                    }
2890
2891                    // LLM-side connector: input_local_transformer + projection
2892                    model.mm_a_local_layers.resize(hparams.audio_local_n_layer);
2893                    for (int il = 0; il < hparams.audio_local_n_layer; il++) {
2894                        auto & layer = model.mm_a_local_layers[il];
2895                        layer.q_w    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q,   il, "weight"));
2896                        layer.q_b    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q,   il, "bias"));
2897                        layer.k_w    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K,   il, "weight"));
2898                        layer.k_b    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K,   il, "bias"));
2899                        layer.v_w    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V,   il, "weight"));
2900                        layer.v_b    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V,   il, "bias"));
2901                        layer.o_w    = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight"));
2902                        layer.ff_gate_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight"));
2903                        layer.ff_up_w   = get_tensor(string_format(TN_MM_A_LOCAL_FFN_UP,   il, "weight"));
2904                        layer.ff_down_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight"));
2905                        layer.ln_1_w = get_tensor(string_format(TN_MM_A_LOCAL_LN1, il, "weight"));
2906                        layer.ln_2_w = get_tensor(string_format(TN_MM_A_LOCAL_LN2, il, "weight"));
2907                    }
2908                    model.mm_a_local_norm_w = get_tensor(string_format(TN_MM_A_LOCAL_NORM, "weight"));
2909
2910                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
2911                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
2912                } break;
2913            case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
2914                {
2915                    // stem TDNN (block 0)
2916                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight"));
2917                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias"));
2918
2919                    // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer)
2920                    model.layers.resize(hparams.n_layer);
2921                    for (int il = 0; il < hparams.n_layer; il++) {
2922                        auto & layer = model.layers[il];
2923                        int bid = il + 1;
2924                        layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight"));
2925                        layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias"));
2926                        layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight"));
2927                        layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias"));
2928                        layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight"));
2929                        layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias"));
2930                        layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight"));
2931                        layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias"));
2932                        layer.res2_conv_w.resize(7);
2933                        layer.res2_conv_b.resize(7);
2934                        for (int xid = 0; xid < 7; xid++) {
2935                            layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight"));
2936                            layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias"));
2937                        }
2938                    }
2939
2940                    // multi-layer feature aggregation
2941                    model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight"));
2942                    model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias"));
2943
2944                    // attentive statistics pooling
2945                    model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight"));
2946                    model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias"));
2947                    model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight"));
2948                    model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias"));
2949
2950                    // final speaker embedding projection
2951                    model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
2952                    model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
2953                } break;
2954            case PROJECTOR_TYPE_POCKETTTS_SPKENC:
2955                {
2956                    load_seanet(model.seanet, false);
2957                    model.downsample_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight"));
2958                    model.spk_proj_w   = get_tensor(string_format(TN_A_SPEAKER_PROJ, "weight"));
2959                } break;
2960            case PROJECTOR_TYPE_POCKETTTS_GEN:
2961                {
2962                    auto & flow = model.flow;
2963                    flow.input_proj_w = get_tensor(string_format(TN_A_GEN_FLOW_INPUT_PROJ, "weight"));
2964                    flow.input_proj_b = get_tensor(string_format(TN_A_GEN_FLOW_INPUT_PROJ, "bias"));
2965                    flow.cond_embd_w  = get_tensor(string_format(TN_A_GEN_FLOW_COND_EMBD, "weight"));
2966                    flow.cond_embd_b  = get_tensor(string_format(TN_A_GEN_FLOW_COND_EMBD, "bias"));
2967                    flow.final_ada_w  = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_ADA, "weight"));
2968                    flow.final_ada_b  = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_ADA, "bias"));
2969                    flow.final_proj_w = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_PROJ, "weight"));
2970                    flow.final_proj_b = get_tensor(string_format(TN_A_GEN_FLOW_FINAL_PROJ, "bias"));
2971
2972                    flow.time.resize(2);
2973                    for (size_t i = 0; i < flow.time.size(); i++) {
2974                        auto & t = flow.time[i];
2975                        t.freqs  = get_tensor(string_format(TN_A_GEN_FLOW_TIME_FREQS, (int) i));
2976                        t.up_w   = get_tensor(string_format(TN_A_GEN_FLOW_TIME_UP,   (int) i, "weight"));
2977                        t.up_b   = get_tensor(string_format(TN_A_GEN_FLOW_TIME_UP,   (int) i, "bias"));
2978                        t.down_w = get_tensor(string_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "weight"));
2979                        t.down_b = get_tensor(string_format(TN_A_GEN_FLOW_TIME_DOWN, (int) i, "bias"));
2980                        t.norm   = get_tensor(string_format(TN_A_GEN_FLOW_TIME_NORM, (int) i));
2981                    }
2982
2983                    // one AdaLN block per flow depth, the count is only known from the tensors
2984                    for (int il = 0; ; il++) {
2985                        ggml_tensor * probe = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "weight"), false);
2986                        if (probe == nullptr) {
2987                            break;
2988                        }
2989                        clip_flow_net::block blk;
2990                        blk.norm_w = probe;
2991                        blk.norm_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_NORM, il, "bias"));
2992                        blk.up_w   = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP,   il, "weight"));
2993                        blk.up_b   = get_tensor(string_format(TN_A_GEN_FLOW_BLK_UP,   il, "bias"));
2994                        blk.down_w = get_tensor(string_format(TN_A_GEN_FLOW_BLK_DOWN, il, "weight"));
2995                        blk.down_b = get_tensor(string_format(TN_A_GEN_FLOW_BLK_DOWN, il, "bias"));
2996                        blk.ada_w  = get_tensor(string_format(TN_A_GEN_FLOW_BLK_ADA,  il, "weight"));
2997                        blk.ada_b  = get_tensor(string_format(TN_A_GEN_FLOW_BLK_ADA,  il, "bias"));
2998                        flow.blocks.push_back(blk);
2999                    }
3000
3001                    model.gen_out_eos_w   = get_tensor(string_format(TN_A_GEN_OUT_EOS, "weight"));
3002                    model.gen_out_eos_b   = get_tensor(string_format(TN_A_GEN_OUT_EOS, "bias"));
3003                    model.gen_input_lin_w = get_tensor(string_format(TN_A_GEN_INPUT_LINEAR, "weight"));
3004                    model.gen_emb_mean    = get_tensor(TN_A_GEN_EMB_MEAN);
3005                    model.gen_emb_std     = get_tensor(TN_A_GEN_EMB_STD);
3006
3007                    // mimi decoder
3008                    model.gen_quant_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_OUT, "weight"));
3009                    model.gen_upsample_w  = get_tensor(string_format(TN_A_GEN_WAV_UPSAMPLE, "weight"));
3010                    load_seanet(model.seanet, true);
3011                    model.gen_tfm_layers.resize(hparams.n_layer);
3012                    for (int il = 0; il < hparams.n_layer; il++) {
3013                        auto & layer = model.gen_tfm_layers[il];
3014                        const char * p = "a.gen.wav.tfm";
3015                        layer.ln_1_w    = get_tensor(string_format(TN_LN_1,        p, il, "weight"));
3016                        layer.ln_1_b    = get_tensor(string_format(TN_LN_1,        p, il, "bias"));
3017                        layer.q_w       = get_tensor(string_format(TN_ATTN_Q,      p, il, "weight"));
3018                        layer.k_w       = get_tensor(string_format(TN_ATTN_K,      p, il, "weight"));
3019                        layer.v_w       = get_tensor(string_format(TN_ATTN_V,      p, il, "weight"));
3020                        layer.o_w       = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"));
3021                        layer.ls_1_w    = get_tensor(string_format(TN_LS_1,        p, il, "weight"));
3022                        layer.ln_2_w    = get_tensor(string_format(TN_LN_2,        p, il, "weight"));
3023                        layer.ln_2_b    = get_tensor(string_format(TN_LN_2,        p, il, "bias"));
3024                        layer.ff_up_w   = get_tensor(string_format(TN_FFN_UP,      p, il, "weight"));
3025                        layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN,    p, il, "weight"));
3026                        layer.ls_2_w    = get_tensor(string_format(TN_LS_2,        p, il, "weight"));
3027                    }
3028                } break;
3029            case PROJECTOR_TYPE_QWEN3TTS_GEN:
3030                {
3031                    // code_predictor, proj_in is absent when the talker and the predictor share the hidden size
3032                    model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false);
3033                    model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"),   false);
3034                    model.gen_code_embd_w     = get_tensor(string_format(TN_A_GEN_CODE_EMBD,     "weight"));
3035                    model.gen_code_head_w     = get_tensor(string_format(TN_A_GEN_CODE_HEAD,     "weight"));
3036                    model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
3037                    model.gen_code_norm_w     = get_tensor(string_format(TN_A_GEN_CODE_NORM,     "weight"));
3038
3039                    // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
3040                    {
3041                        auto & c2w = model.c2w;
3042
3043                        c2w.quant_first_in_w  = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN,  "weight"));
3044                        c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
3045                        c2w.quant_first_cb_w  = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB,  "weight"));
3046                        c2w.quant_rest_in_w   = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN,   "weight"));
3047                        c2w.quant_rest_out_w  = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT,  "weight"));
3048                        c2w.quant_rest_cb_w   = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB,   "weight"));
3049
3050                        c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight"));
3051                        c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias"));
3052
3053                        c2w.tfm_in_proj_w     = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ,  "weight"));
3054                        c2w.tfm_in_proj_b     = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ,  "bias"));
3055                        c2w.tfm_out_proj_w    = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"));
3056                        c2w.tfm_out_proj_b    = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
3057                        c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
3058
3059                        // loaded manually, the generic model.layers loop is taken by code_predictor
3060                        c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
3061                        for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
3062                            auto & layer = c2w.tfm_layers[il];
3063                            const char * p = "a.gen.wav.tfm";
3064                            layer.q_w      = get_tensor(string_format(TN_ATTN_Q,      p, il, "weight"));
3065                            layer.k_w      = get_tensor(string_format(TN_ATTN_K,      p, il, "weight"));
3066                            layer.v_w      = get_tensor(string_format(TN_ATTN_V,      p, il, "weight"));
3067                            layer.o_w      = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"));
3068                            layer.ln_1_w   = get_tensor(string_format(TN_LN_1,        p, il, "weight"));
3069                            layer.ln_2_w   = get_tensor(string_format(TN_LN_2,        p, il, "weight"));
3070                            layer.ls_1_w   = get_tensor(string_format(TN_LS_1,        p, il, "weight"));
3071                            layer.ls_2_w   = get_tensor(string_format(TN_LS_2,        p, il, "weight"));
3072                            layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE,   p, il, "weight"));
3073                            layer.ff_up_w   = get_tensor(string_format(TN_FFN_UP,     p, il, "weight"));
3074                            layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN,   p, il, "weight"));
3075                        }
3076
3077                        // upsample: 2x (causal ConvTranspose1d + ConvNeXt block)
3078                        c2w.upsample.resize(hparams.wav_upsample_n_block);
3079                        for (int il = 0; il < hparams.wav_upsample_n_block; il++) {
3080                            auto & up = c2w.upsample[il];
3081                            up.conv_w   = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV,   il, "weight"));
3082                            up.conv_b   = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV,   il, "bias"));
3083                            up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
3084                            up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
3085                            up.norm_w   = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM,   il, "weight"));
3086                            up.norm_b   = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM,   il, "bias"));
3087                            up.pw1_w    = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1,    il, "weight"));
3088                            up.pw1_b    = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1,    il, "bias"));
3089                            up.pw2_w    = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2,    il, "weight"));
3090                            up.pw2_b    = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2,    il, "bias"));
3091                            up.gamma    = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA,  il));
3092                        }
3093
3094                        // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post
3095                        c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight"));
3096                        c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias"));
3097
3098                        c2w.dac.resize(hparams.wav_dac_n_block);
3099                        for (int il = 0; il < hparams.wav_dac_n_block; il++) {
3100                            auto & blk = c2w.dac[il];
3101                            blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha"));
3102                            blk.snake_beta  = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta"));
3103                            blk.conv_w      = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV,  il, "weight"));
3104                            blk.conv_b      = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV,  il, "bias"));
3105
3106                            blk.res.resize(hparams.wav_dac_n_res);
3107                            for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) {
3108                                auto & res = blk.res[ir];
3109                                res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1,  il, ir, "alpha"));
3110                                res.act1_beta  = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1,  il, ir, "beta"));
3111                                res.conv1_w    = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight"));
3112                                res.conv1_b    = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias"));
3113                                res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2,  il, ir, "alpha"));
3114                                res.act2_beta  = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2,  il, ir, "beta"));
3115                                res.conv2_w    = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight"));
3116                                res.conv2_b    = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias"));
3117                            }
3118                        }
3119
3120                        c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha"));
3121                        c2w.dac_post_snake_beta  = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta"));
3122                        c2w.dac_post_conv_w      = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV,  "weight"));
3123                        c2w.dac_post_conv_b      = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV,  "bias"));
3124                    }
3125                } break;
3126            case PROJECTOR_TYPE_VOXTRAL:
3127                {
3128                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
3129                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
3130                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
3131                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
3132                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
3133                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
3134                } break;
3135            case PROJECTOR_TYPE_MUSIC_FLAMINGO:
3136                {
3137                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
3138                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
3139                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
3140                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
3141                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
3142                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
3143                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
3144                    model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
3145                } break;
3146            case PROJECTOR_TYPE_INTERNVL:
3147                {
3148                    model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
3149                    model.mm_0_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "bias"));
3150                    model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
3151                    model.mm_1_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "bias"));
3152                    model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
3153                    model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
3154                } break;
3155            case PROJECTOR_TYPE_NEMOTRON_V2_VL:
3156                {
3157                    model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
3158                    model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
3159                    model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
3160                } break;
3161            case PROJECTOR_TYPE_GLMA:
3162                {
3163                    model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
3164                    model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
3165                    model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
3166                    model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
3167                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
3168                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
3169                    model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
3170                    model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
3171                    model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
3172                    model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
3173                    model.mm_boi = get_tensor(string_format(TN_TOK_BOI));
3174                    model.mm_eoi = get_tensor(string_format(TN_TOK_EOI));
3175                } break;
3176            case PROJECTOR_TYPE_LLAMA4:
3177                {
3178                    model.mm_model_proj    = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
3179                    model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
3180                    model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 2, "weight"));
3181                } break;
3182            case PROJECTOR_TYPE_COGVLM:
3183                {
3184                    model.mm_model_proj     = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
3185                    model.mm_post_fc_norm_w = get_tensor(string_format(TN_MM_POST_FC_NORM, "weight"));
3186                    model.mm_post_fc_norm_b = get_tensor(string_format(TN_MM_POST_FC_NORM, "bias"));
3187                    model.mm_h_to_4h_w      = get_tensor(string_format(TN_MM_H_TO_4H,      "weight"));
3188                    model.mm_gate_w         = get_tensor(string_format(TN_MM_GATE,         "weight"));
3189                    model.mm_4h_to_h_w      = get_tensor(string_format(TN_MM_4H_TO_H,      "weight"));
3190                    model.mm_boi            = get_tensor(TN_TOK_BOI);
3191                    model.mm_eoi            = get_tensor(TN_TOK_EOI);
3192                } break;
3193            case PROJECTOR_TYPE_HUNYUANVL:
3194                {
3195                    // proj.0 -> mm.0 (conv1), proj.2 -> mm.2 (conv2), mlp -> mm.model.fc (linear)
3196                    model.mm_0_w            = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
3197                    model.mm_0_b            = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
3198                    model.mm_1_w            = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
3199                    model.mm_1_b            = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
3200                    model.mm_model_proj     = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
3201                    model.mm_model_proj_b   = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
3202                    model.mm_pre_norm_w     = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
3203                    model.mm_post_norm_w    = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
3204                    model.mm_img_begin      = get_tensor(TN_MM_IMG_BEGIN);
3205                    model.mm_img_end        = get_tensor(TN_MM_IMG_END);
3206                    model.image_newline     = get_tensor(TN_IMAGE_NEWLINE);
3207                    model.view_seperator    = get_tensor(TN_IMAGE_SEPERATOR, false);
3208                } break;
3209            case PROJECTOR_TYPE_JANUS_PRO:
3210                {
3211                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
3212                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
3213                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
3214                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
3215                } break;
3216            case PROJECTOR_TYPE_PHI4:
3217                {
3218                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
3219                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
3220                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
3221                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
3222                } break;
3223            case PROJECTOR_TYPE_DEEPSEEKOCR:
3224            case PROJECTOR_TYPE_DEEPSEEKOCR2:
3225                {
3226                    model.pos_embed          = get_tensor(string_format(TN_SAM_POS_EMBD,   "weight"));
3227                    model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
3228                    model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));
3229                    model.sam_layers.resize(model.n_sam_layers);
3230                    for (int il = 0; il < model.n_sam_layers; ++il) {
3231                        auto & layer    = model.sam_layers[il];
3232                        layer.qkv_w     = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "weight"));
3233                        layer.qkv_b     = get_tensor(string_format(TN_SAM_ATTN_QKV, il, "bias"));
3234                        layer.o_w       = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "weight"));
3235                        layer.o_b       = get_tensor(string_format(TN_SAM_ATTN_OUT, il, "bias"));
3236                        layer.ln_1_w    = get_tensor(string_format(TN_SAM_PRE_NORM, il, "weight"));
3237                        layer.ln_1_b    = get_tensor(string_format(TN_SAM_PRE_NORM, il, "bias"));
3238                        layer.ln_2_w    = get_tensor(string_format(TN_SAM_POST_NORM, il, "weight"));
3239                        layer.ln_2_b    = get_tensor(string_format(TN_SAM_POST_NORM, il, "bias"));
3240                        layer.rel_pos_h = get_tensor(string_format(TN_SAM_ATTN_POS_H, il, "weight"));
3241                        layer.rel_pos_w = get_tensor(string_format(TN_SAM_ATTN_POS_W, il, "weight"));
3242                        layer.ff_up_w   = get_tensor(string_format(TN_SAM_FFN_UP, il, "weight"));
3243                        layer.ff_up_b   = get_tensor(string_format(TN_SAM_FFN_UP, il, "bias"));
3244                        layer.ff_down_w = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "weight"));
3245                        layer.ff_down_b = get_tensor(string_format(TN_SAM_FFN_DOWN, il, "bias"));
3246                    }
3247                    model.neck_0_w       = get_tensor(string_format(TN_SAM_NECK, 0, "weight"));
3248                    model.neck_1_b       = get_tensor(string_format(TN_SAM_NECK, 1, "bias"));
3249                    model.neck_1_w       = get_tensor(string_format(TN_SAM_NECK, 1, "weight"));
3250                    model.neck_2_w       = get_tensor(string_format(TN_SAM_NECK, 2, "weight"));
3251                    model.neck_3_b       = get_tensor(string_format(TN_SAM_NECK, 3, "bias"));
3252                    model.neck_3_w       = get_tensor(string_format(TN_SAM_NECK, 3, "weight"));
3253                    model.net_2          = get_tensor(string_format(TN_SAM_NET, 2, "weight"));
3254                    model.net_3          = get_tensor(string_format(TN_SAM_NET, 3, "weight"));
3255                    model.image_newline  = get_tensor(TN_IMAGE_NEWLINE, false);
3256                    model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR);
3257                    model.mm_fc_w        = get_tensor(string_format(TN_MM_PROJECTOR, "weight"));
3258                    model.mm_fc_b        = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
3259                    model.resample_query_768  = get_tensor(string_format(TN_RESMPL_QUERY, 768, "weight"), false);
3260                    model.resample_query_1024 = get_tensor(string_format(TN_RESMPL_QUERY, 1024, "weight"), false);
3261                 } break;
3262            case PROJECTOR_TYPE_GEMMA4A:
3263                {
3264                    for (int i = 0; i < 2; i++) {
3265                        model.sscp_conv_w[i] = get_tensor(string_format(TN_A_CONV1D, i, "weight"));
3266                        model.sscp_conv_b[i] = get_tensor(string_format(TN_A_CONV1D, i, "bias"), false);
3267                        model.sscp_norm_w[i] = get_tensor(string_format(TN_A_CONV1D_NORM, i, "weight"), false);
3268                    }
3269                    model.sscp_inp_proj_w = get_tensor(string_format(TN_A_INP_PROJ, "weight"));
3270                    model.sscp_inp_proj_b = get_tensor(string_format(TN_A_INP_PROJ, "bias"), false);
3271                    model.audio_out_proj_w = get_tensor(string_format(TN_A_OUT_PROJ, "weight"), false);
3272                    model.audio_out_proj_b = get_tensor(string_format(TN_A_OUT_PROJ, "bias"), false);
3273                    // audio multimodal embedder (mm.a.* namespace, not mm.*)
3274                    model.mm_soft_emb_norm_w = get_tensor(string_format(TN_A_MM_SOFT_EMB_N, "weight"), false);
3275                    model.mm_input_proj_w    = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"), false);
3276
3277                    // Per-layer tensors NOT loaded by the generic loop above
3278                    for (int il = 0; il < hparams.n_layer; ++il) {
3279                        auto & layer = model.layers[il];
3280
3281                        // Gemma4 audio conformer-specific tensors
3282                        layer.ff_norm_w        = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
3283                        layer.attn_pre_norm_w  = get_tensor(string_format(TN_A_ATTN_PRE_NORM, prefix, il, "weight"), false);
3284                        layer.per_dim_scale_w  = get_tensor(string_format(TN_A_PER_DIM_SCALE, prefix, il, "weight"), false);
3285                        layer.per_dim_k_scale_w = get_tensor(string_format(TN_A_PER_DIM_K_SCALE, prefix, il, "weight"), false);
3286                        layer.attn_k_rel_w     = get_tensor(string_format(TN_A_ATTN_K_REL, prefix, il, "weight"), false);
3287
3288                        // Convolution module
3289                        // Note: conv_norm / norm_conv are swapped in GGUF due to
3290                        // upstream tensor_mapping.py, so we load them in reverse order
3291                        layer.norm_conv_w  = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"), false);
3292                        layer.norm_conv_b  = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"), false);
3293                        layer.conv_pw1_w   = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "weight"));
3294                        layer.conv_pw1_b   = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "bias"), false);
3295                        layer.conv_dw_w    = get_tensor(string_format(TN_CONV_DW,   prefix, il, "weight"));
3296                        layer.conv_dw_b    = get_tensor(string_format(TN_CONV_DW,   prefix, il, "bias"), false);
3297                        layer.conv_norm_w  = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"), false);
3298                        layer.conv_norm_b  = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"), false);
3299                        layer.conv_pw2_w   = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "weight"));
3300                        layer.conv_pw2_b   = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "bias"), false);
3301
3302                        // FFN2 (second half-step)
3303                        layer.ff_norm_1_w      = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
3304                        layer.ff_up_1_w        = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
3305                        layer.ff_up_1_b        = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
3306                        layer.ff_down_1_w      = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
3307                        layer.ff_down_1_b      = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
3308                        layer.ff_post_norm_1_w = get_tensor(string_format(TN_A_FFN_POST_NORM_1, prefix, il, "weight"), false);
3309                    }
3310
3311                    // Load clamp info for ClippableLinear AFTER all tensors are loaded
3312                    for (auto * tensor : tensors_to_load) {
3313                        std::string name = tensor->name;
3314                        if (string_ends_with(name, ".weight")) {
3315                            std::string name_inp_max = name;
3316                            std::string name_inp_min = name;
3317                            std::string name_out_max = name;
3318                            std::string name_out_min = name;
3319                            string_replace_all(name_inp_max, ".weight", ".input_max");
3320                            string_replace_all(name_inp_min, ".weight", ".input_min");
3321                            string_replace_all(name_out_max, ".weight", ".output_max");
3322                            string_replace_all(name_out_min, ".weight", ".output_min");
3323                            model.clamp_info_map[name] = {
3324                                get_scalar(name_inp_max, FLT_MAX),
3325                                get_scalar(name_inp_min, -FLT_MAX),
3326                                get_scalar(name_out_max, FLT_MAX),
3327                                get_scalar(name_out_min, -FLT_MAX)
3328                            };
3329                        }
3330                    }
3331                } break;
3332            case PROJECTOR_TYPE_GEMMA4UA:
3333                {
3334                    model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"));
3335                } break;
3336            case PROJECTOR_TYPE_LFM2A:
3337                {
3338                    for (int i : {0, 2, 3, 5, 6}) {
3339                        model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
3340                        model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
3341                    }
3342                    model.pre_encode_out_w    = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
3343                    model.pre_encode_out_b    = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
3344
3345                    model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
3346                    model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
3347                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
3348                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
3349                    model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
3350                    model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
3351
3352                    for (int il = 0; il < hparams.n_layer; ++il) {
3353                        auto & layer = model.layers[il];
3354
3355                        layer.ff_norm_w   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "weight"));
3356                        layer.ff_norm_b   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "bias"));
3357                        layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
3358                        layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
3359                        layer.ff_up_1_w   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "weight"));
3360                        layer.ff_up_1_b   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "bias"));
3361                        layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
3362                        layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
3363
3364                        layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
3365                        layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
3366
3367                        layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
3368                        layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
3369
3370                        layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
3371
3372                        layer.conv_norm_w  = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
3373                        layer.conv_norm_b  = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
3374                        layer.conv_dw_w    = get_tensor(string_format(TN_CONV_DW,   prefix, il, "weight"));
3375                        layer.conv_dw_b    = get_tensor(string_format(TN_CONV_DW,   prefix, il, "bias"));
3376                        layer.conv_pw1_w   = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "weight"));
3377                        layer.conv_pw1_b   = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "bias"));
3378                        layer.conv_pw2_w   = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "weight"));
3379                        layer.conv_pw2_b   = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "bias"));
3380                    }
3381                } break;
3382            case PROJECTOR_TYPE_PARAKEET:
3383                {
3384
3385                    hparams.mel_filters = get_vector(TN_MEL_FILTERS);
3386                    hparams.window      = get_vector(TN_WINDOW);
3387
3388                    // Subsampling layers (conv1d)
3389                    for (int i : {0, 2, 3, 5, 6}) {
3390                        model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
3391                        model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
3392                    }
3393                    model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
3394                    model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
3395
3396                    // Projection layers
3397                    model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false);
3398                    model.mm_0_w        = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
3399                    model.mm_1_w        = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
3400
3401                    // Encoder layers
3402                    for (int il = 0; il < hparams.n_layer; ++il) {
3403                        auto & layer = model.layers[il];
3404
3405                        // Attention (from shared above)
3406
3407                        // Relative position encoding
3408                        layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
3409                        layer.pos_bias_u   = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
3410                        layer.pos_bias_v   = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
3411
3412                        // Convolution module
3413                        layer.conv_pw1_w     = get_tensor(string_format(TN_CONV_PW1,       prefix, il, "weight"));
3414                        layer.conv_pw1_b     = get_tensor(string_format(TN_CONV_PW1,       prefix, il, "bias"), false);
3415                        layer.conv_dw_w      = get_tensor(string_format(TN_CONV_DW,        prefix, il, "weight"));
3416                        layer.conv_dw_b      = get_tensor(string_format(TN_CONV_DW,        prefix, il, "bias"), false);
3417                        layer.conv_norm_w    = get_tensor(string_format(TN_CONV_NORM,      prefix, il, "weight"));
3418                        layer.conv_norm_b    = get_tensor(string_format(TN_CONV_NORM,      prefix, il, "bias"));
3419                        layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il));
3420                        layer.conv_norm_var  = get_tensor(string_format(TN_CONV_NORM_VAR,  prefix, il));
3421                        layer.conv_pw2_w     = get_tensor(string_format(TN_CONV_PW2,       prefix, il, "weight"));
3422                        layer.conv_pw2_b     = get_tensor(string_format(TN_CONV_PW2,       prefix, il, "bias"), false);
3423
3424                        // Feed-forward networks
3425                        layer.ff_norm_w   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "weight"));
3426                        layer.ff_norm_b   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "bias"));
3427
3428                        layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
3429                        layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
3430                        layer.ff_up_1_w   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "weight"));
3431                        layer.ff_up_1_b   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "bias"), false);
3432                        layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
3433                        layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
3434
3435                        // Layer norms
3436                        layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
3437                        layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
3438                    }
3439
3440                    model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
3441                    model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
3442                    model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
3443                } break;
3444            case PROJECTOR_TYPE_GRANITE_SPEECH:
3445                {
3446                    model.inp_proj_w     = get_tensor(string_format(TN_INP_PROJ,    "weight"));
3447                    model.inp_proj_b     = get_tensor(string_format(TN_INP_PROJ,    "bias"));
3448                    model.ctc_out_w      = get_tensor(string_format(TN_CTC_OUT,     "weight"));
3449                    model.ctc_out_b      = get_tensor(string_format(TN_CTC_OUT,     "bias"));
3450                    model.ctc_out_mid_w  = get_tensor(string_format(TN_CTC_OUT_MID, "weight"));
3451                    model.ctc_out_mid_b  = get_tensor(string_format(TN_CTC_OUT_MID, "bias"));
3452
3453                    // per-layer tensors not loaded by the generic loop above
3454                    for (int il = 0; il < hparams.n_layer; ++il) {
3455                        auto & layer = model.layers[il];
3456
3457                        layer.attn_rel_pos_emb = get_tensor(string_format(TN_ATTN_REL_POS_EMB, prefix, il));
3458
3459                        layer.ff_norm_w   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "weight"));
3460                        layer.ff_norm_b   = get_tensor(string_format(TN_FFN_NORM,   prefix, il, "bias"));
3461
3462                        layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
3463                        layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
3464                        layer.ff_up_1_w   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "weight"));
3465                        layer.ff_up_1_b   = get_tensor(string_format(TN_FFN_UP_1,   prefix, il, "bias"));
3466                        layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
3467                        layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"));
3468
3469                        layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
3470                        layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
3471                        layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
3472                        layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
3473                        layer.conv_dw_w   = get_tensor(string_format(TN_CONV_DW,   prefix, il, "weight"));
3474                        layer.conv_pw1_w  = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "weight"));
3475                        layer.conv_pw1_b  = get_tensor(string_format(TN_CONV_PW1,  prefix, il, "bias"));
3476                        layer.conv_pw2_w  = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "weight"));
3477                        layer.conv_pw2_b  = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "bias"));
3478                    }
3479
3480                    model.qf_proj_blocks.resize(1);
3481                    auto & qf = model.qf_proj_blocks[0];
3482                    qf.qf_proj_query    = get_tensor(string_format(TN_QF_PROJ_QUERY, prefix));
3483                    qf.qf_proj_norm_w   = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "weight"));
3484                    qf.qf_proj_norm_b   = get_tensor(string_format(TN_QF_PROJ_NORM, prefix, "bias"));
3485                    qf.qf_proj_linear_w = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "weight"));
3486                    qf.qf_proj_linear_b = get_tensor(string_format(TN_QF_PROJ_LINEAR, prefix, "bias"));
3487
3488                    const int n_proj_layers = 2;
3489                    qf.qf_proj_layers.resize(n_proj_layers);
3490                    for (int il = 0; il < n_proj_layers; ++il) {
3491                        auto & pl = qf.qf_proj_layers[il];
3492
3493                        pl.q_w    = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "weight"));
3494                        pl.q_b    = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, il, "bias"));
3495                        pl.k_w    = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "weight"));
3496                        pl.k_b    = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, il, "bias"));
3497                        pl.v_w    = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "weight"));
3498                        pl.v_b    = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, il, "bias"));
3499                        pl.o_w    = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "weight"));
3500                        pl.o_b    = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, il, "bias"));
3501                        pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "weight"));
3502                        pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, il, "bias"));
3503
3504                        pl.cross_attn_q_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "weight"));
3505                        pl.cross_attn_q_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, il, "bias"));
3506                        pl.cross_attn_k_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "weight"));
3507                        pl.cross_attn_k_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, il, "bias"));
3508                        pl.cross_attn_v_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "weight"));
3509                        pl.cross_attn_v_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, il, "bias"));
3510                        pl.cross_attn_o_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "weight"));
3511                        pl.cross_attn_o_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, il, "bias"));
3512                        pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "weight"));
3513                        pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, il, "bias"));
3514
3515                        pl.ff_up_w   = get_tensor(string_format(TN_QF_FFN_UP,   prefix, il, "weight"));
3516                        pl.ff_up_b   = get_tensor(string_format(TN_QF_FFN_UP,   prefix, il, "bias"));
3517                        pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "weight"));
3518                        pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, il, "bias"));
3519                        pl.ln_2_w    = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "weight"));
3520                        pl.ln_2_b    = get_tensor(string_format(TN_QF_FFN_NORM, prefix, il, "bias"));
3521                    }
3522                } break;
3523            case PROJECTOR_TYPE_GRANITE4_VISION:
3524                {
3525                    // image_newline lives at the top-level.
3526                    model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
3527
3528                    // Load separate layerwise and spatial projector tensors
3529                    const auto projector_count = hparams.feature_layers.size();
3530                    model.qf_proj_blocks.resize(projector_count);
3531                    for (size_t bid = 0; bid < projector_count; ++bid) {
3532                        auto & b = model.qf_proj_blocks[bid];
3533
3534                        // non-layerwise tensors
3535                        b.qf_proj_img_pos     = get_tensor(string_format(TN_MULTI_PROJ_IMG_POS,           bid));
3536                        b.qf_proj_query       = get_tensor(string_format(TN_MULTI_PROJ_QUERY,     prefix, bid));
3537                        b.qf_proj_linear_w    = get_tensor(string_format(TN_MULTI_PROJ_LINEAR,    prefix, bid, "weight"));
3538                        b.qf_proj_linear_b    = get_tensor(string_format(TN_MULTI_PROJ_LINEAR,    prefix, bid, "bias"));
3539                        b.qf_proj_norm_w      = get_tensor(string_format(TN_MULTI_PROJ_NORM,      prefix, bid, "weight"));
3540                        b.qf_proj_norm_b      = get_tensor(string_format(TN_MULTI_PROJ_NORM,      prefix, bid, "bias"));
3541                        b.qf_proj_post_norm_w = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "weight"));
3542                        b.qf_proj_post_norm_b = get_tensor(string_format(TN_MULTI_PROJ_POST_NORM, prefix, bid, "bias"));
3543
3544                        // laywerwise tensors
3545                        // NOTE: If any model uses multi-layer qformers, this will need to change
3546                        b.qf_proj_layers.resize(1);
3547                        auto & pl = b.qf_proj_layers[0];
3548
3549                        pl.q_w    = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "weight"));
3550                        pl.q_b    = get_tensor(string_format(TN_QF_SELF_ATTN_Q, prefix, bid, "bias"));
3551                        pl.k_w    = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "weight"));
3552                        pl.k_b    = get_tensor(string_format(TN_QF_SELF_ATTN_K, prefix, bid, "bias"));
3553                        pl.v_w    = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "weight"));
3554                        pl.v_b    = get_tensor(string_format(TN_QF_SELF_ATTN_V, prefix, bid, "bias"));
3555                        pl.o_w    = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "weight"));
3556                        pl.o_b    = get_tensor(string_format(TN_QF_SELF_ATTN_O, prefix, bid, "bias"));
3557                        pl.ln_1_w = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "weight"));
3558                        pl.ln_1_b = get_tensor(string_format(TN_QF_SELF_ATTN_N, prefix, bid, "bias"));
3559
3560                        pl.cross_attn_q_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "weight"));
3561                        pl.cross_attn_q_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_Q, prefix, bid, "bias"));
3562                        pl.cross_attn_k_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "weight"));
3563                        pl.cross_attn_k_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_K, prefix, bid, "bias"));
3564                        pl.cross_attn_v_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "weight"));
3565                        pl.cross_attn_v_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_V, prefix, bid, "bias"));
3566                        pl.cross_attn_o_w    = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "weight"));
3567                        pl.cross_attn_o_b    = get_tensor(string_format(TN_QF_CROSS_ATTN_O, prefix, bid, "bias"));
3568                        pl.cross_attn_norm_w = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "weight"));
3569                        pl.cross_attn_norm_b = get_tensor(string_format(TN_QF_CROSS_ATTN_N, prefix, bid, "bias"));
3570
3571                        pl.ff_up_w   = get_tensor(string_format(TN_QF_FFN_UP,   prefix, bid, "weight"));
3572                        pl.ff_up_b   = get_tensor(string_format(TN_QF_FFN_UP,   prefix, bid, "bias"));
3573                        pl.ff_down_w = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "weight"));
3574                        pl.ff_down_b = get_tensor(string_format(TN_QF_FFN_DOWN, prefix, bid, "bias"));
3575                        pl.ln_2_w    = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "weight"));
3576                        pl.ln_2_b    = get_tensor(string_format(TN_QF_FFN_NORM, prefix, bid, "bias"));
3577                    }
3578
3579                } break;
3580            default:
3581                GGML_ASSERT(false && "unknown projector type");
3582        }
3583
3584        // load data
3585        {
3586            std::vector<uint8_t> read_buf;
3587
3588            // start loading event
3589            if (progress_callback){
3590                progress_callback(0.0, progress_callback_user_data);
3591            }
3592
3593            // compute total tensor data size for progress reporting
3594            size_t total_data_size = 0;
3595            for (auto & t : tensors_to_load) {
3596                total_data_size += ggml_nbytes(t);
3597            }
3598
3599            // alloc memory and offload data
3600            ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend);
3601            ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft));
3602            ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
3603            // read the weight from file
3604            if (!ctx_clip.no_alloc) {
3605                size_t data_loaded = 0;
3606                for (auto & t : tensors_to_load) {
3607                    ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name);
3608                    GGML_ASSERT(cur && "tensor not found in ctx_data");
3609                    auto it_off = tensor_offset.find(t->name);
3610                    GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
3611                    const size_t offset = it_off->second;
3612                    fin.seekg(offset, std::ios::beg);
3613                    if (!fin) {
3614                        throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
3615                    }
3616                    size_t num_bytes = ggml_nbytes(cur);
3617                    if (ggml_backend_buft_is_host(buft)) {
3618                        // for the CPU and Metal backend, we can read directly into the tensor
3619                        fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
3620                    } else {
3621                        // read into a temporary buffer first, then copy to device memory
3622                        read_buf.resize(num_bytes);
3623                        fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
3624                        ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
3625                    }
3626                    data_loaded += num_bytes;
3627                    if (progress_callback && total_data_size > 0) {
3628                        const float progress = (float)data_loaded / (float)total_data_size;
3629                        if (!progress_callback(progress, progress_callback_user_data)) {
3630                            throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__));
3631                        }
3632                    }
3633                }
3634                LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
3635            } else {
3636                LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
3637            }
3638            fin.close();
3639        }
3640
3641    }
3642
3643    struct support_info_op {
3644        ggml_tensor * op;
3645
3646        // true if the op runs on the accelerated ctx_clip.backend
3647        bool is_accel = true;
3648    };
3649
3650    struct support_info_graph {
3651        // whether the clip_ctx.backend supports flash attention
3652        bool fattn = true;
3653        ggml_tensor * fattn_op = nullptr; // for debugging
3654
3655        std::vector<support_info_op> ops;
3656    };
3657
3658    static clip_image_f32_batch get_dummy_batch(clip_ctx & ctx_clip) {
3659        // create a fake batch
3660        const auto & hparams = ctx_clip.model.hparams;
3661        clip_image_f32_batch batch;
3662        clip_image_f32 img;
3663        if (ctx_clip.model.modality == CLIP_MODALITY_VISION) {
3664            const int sz = hparams.warmup_image_size;
3665            img.set_size({sz, sz}, false, false);
3666            LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz);
3667        } else {
3668            // GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count,
3669            // so the [1, 256] bound only applies to FFT-based models.
3670            const bool fft_based = ctx_clip.model.proj_type != PROJECTOR_TYPE_GEMMA4UA;
3671            if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
3672                throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins));
3673            }
3674            img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false);
3675            LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size);
3676        }
3677        batch.entries.push_back(img);
3678        return batch;
3679    }
3680
3681    static void init_ctx(clip_ctx & ctx_clip) {
3682        ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
3683
3684        // check batching support
3685        auto batch = get_dummy_batch(ctx_clip);
3686        auto builder = clip_get_graph_builder(&ctx_clip, batch);
3687        ctx_clip.support_batch = builder->support_batch();
3688    }
3689
3690    static void warmup(clip_ctx & ctx_clip) {
3691        auto batch = get_dummy_batch(ctx_clip);
3692        warmup(ctx_clip, batch);
3693    }
3694
3695    static void warmup(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) {
3696        support_info_graph info;
3697
3698        if (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {
3699            // try to enable flash attention to see if it's supported
3700            ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_ENABLED;
3701            info = reserve_compute_meta(ctx_clip, batch);
3702            if (!info.fattn && info.fattn_op) {
3703                auto op = info.fattn_op;
3704                LOG_WRN("%s: *****************************************************************\n", __func__);
3705                LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(ctx_clip.backend));
3706                LOG_WRN("%s: op params: \n", __func__);
3707                static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) {
3708                    LOG_WRN("%s:   %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn,
3709                            name, ggml_type_name(t->type),
3710                            t->ne[0], t->ne[1], t->ne[2], t->ne[3],
3711                            t->nb[0], t->nb[1], t->nb[2], t->nb[3]);
3712                };
3713                print_shape(__func__, " dst", op);
3714                print_shape(__func__, "src0", op->src[0]);
3715                print_shape(__func__, "src1", op->src[1]);
3716                print_shape(__func__, "src2", op->src[2]);
3717                LOG_WRN("%s: please report this on github as an issue\n", __func__);
3718                LOG_WRN("%s: *****************************************************************\n", __func__);
3719                ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED;
3720                reserve_compute_meta(ctx_clip, batch);
3721            }
3722        } else {
3723            info = reserve_compute_meta(ctx_clip, batch);
3724            if (!info.fattn && ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
3725                LOG_WRN("%s: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded)\n", __func__);
3726            }
3727        }
3728
3729        ctx_clip.is_allocated = true; // mark buffers as allocated
3730
3731        LOG_INF("%s: flash attention is %s\n", __func__,
3732            (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
3733
3734        // print ops that are not supported by the GPU backend (if there is one)
3735        if (ctx_clip.backend && ctx_clip.backend != ctx_clip.backend_cpu) {
3736            std::vector<support_info_op> unsupported_ops;
3737            for (const auto & op : info.ops) {
3738                if (!op.is_accel) {
3739                    unsupported_ops.push_back(op);
3740                }
3741            }
3742            if (!unsupported_ops.empty()) {
3743                LOG_WRN("%s: *****************************************************************\n", __func__);
3744                LOG_WRN("%s: WARNING: the CLIP graph uses unsupported operators by the backend\n", __func__);
3745                LOG_WRN("%s:          the performance will be suboptimal                      \n", __func__);
3746                LOG_WRN("%s:          list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(ctx_clip.backend));
3747                for (const auto & op : unsupported_ops) {
3748                    LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__,
3749                            ggml_op_name(op.op->op),
3750                            ggml_type_name(op.op->type),
3751                            op.op->ne[0], op.op->ne[1], op.op->ne[2], op.op->ne[3]);
3752                }
3753                LOG_WRN("%s: flash attention is %s\n", __func__,
3754                    (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
3755                LOG_WRN("%s: please report this on github as an issue\n", __func__);
3756                LOG_WRN("%s: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118\n", __func__);
3757                LOG_WRN("%s: *****************************************************************\n", __func__);
3758            }
3759        }
3760    }
3761
3762    // only initialize backend buffers, but do not allocate them yet
3763    static support_info_graph reserve_compute_meta(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) {
3764        ggml_cgraph * gf = clip_get_graph_builder(&ctx_clip, batch)->build();
3765        ggml_backend_sched_reserve(ctx_clip.sched.get(), gf);
3766
3767        ctx_clip.mem_compute.clear();
3768        for (size_t i = 0; i < ctx_clip.backend_ptrs.size(); ++i) {
3769            ggml_backend_t backend = ctx_clip.backend_ptrs[i];
3770            ggml_backend_buffer_type_t buft = ctx_clip.backend_buft[i];
3771            size_t size = ggml_backend_sched_get_buffer_size(ctx_clip.sched.get(), backend);
3772            if (size > 1) {
3773                LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
3774                        ggml_backend_buft_name(buft),
3775                        size / 1024.0 / 1024.0);
3776            }
3777            ctx_clip.mem_compute[ggml_backend_get_device(backend)] += size;
3778        }
3779
3780        const int n_splits = ggml_backend_sched_get_n_splits(ctx_clip.sched.get());
3781        const int n_nodes  = ggml_graph_n_nodes(gf);
3782
3783        LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__,  n_splits, n_nodes);
3784
3785        support_info_graph res {
3786            /*.fattn    = */ true,
3787            /*.fattn_op = */ nullptr,
3788            /*.ops      = */ {},
3789        };
3790
3791        // check op support
3792        for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
3793            ggml_tensor * node = ggml_graph_node(gf, i);
3794            res.ops.push_back({node, true});
3795            if (!ggml_backend_supports_op(ctx_clip.backend, node)) {
3796                res.ops.back().is_accel = false;
3797                if (node->op == GGML_OP_FLASH_ATTN_EXT) {
3798                    res.fattn    = false;
3799                    res.fattn_op = node;
3800                }
3801            }
3802        }
3803
3804        return res;
3805    }
3806
3807    void get_bool(const std::string & key, bool & output, bool required = true) const {
3808        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3809        if (i < 0) {
3810            if (required) {
3811                throw std::runtime_error("Key not found: " + key);
3812            }
3813            return;
3814        }
3815        output = gguf_get_val_bool(ctx_gguf.get(), i);
3816    }
3817
3818    void get_i32(const std::string & key, int & output, bool required = true) const {
3819        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3820        if (i < 0) {
3821            if (required) {
3822                throw std::runtime_error("Key not found: " + key);
3823            }
3824            return;
3825        }
3826        output = gguf_get_val_i32(ctx_gguf.get(), i);
3827    }
3828
3829    void get_u32(const std::string & key, int & output, bool required = true) const {
3830        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3831        if (i < 0) {
3832            if (required) {
3833                throw std::runtime_error("Key not found: " + key);
3834            }
3835            return;
3836        }
3837        const uint32_t val = gguf_get_val_u32(ctx_gguf.get(), i);
3838        // sanity check
3839        if (val > (uint32_t) INT32_MAX) {
3840            throw std::runtime_error(string_format("%s: value %u for key '%s' exceeds INT32_MAX\n",
3841                __func__, val, key.c_str()));
3842        }
3843        output = (int) val;
3844    }
3845
3846    void get_f32(const std::string & key, float & output, bool required = true) const {
3847        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3848        if (i < 0) {
3849            if (required) {
3850                throw std::runtime_error("Key not found: " + key);
3851            }
3852            return;
3853        }
3854        output = gguf_get_val_f32(ctx_gguf.get(), i);
3855    }
3856
3857    void get_arr_f32(const std::string & key, std::vector<float> & output, bool required = true) const {
3858        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3859        if (i < 0) {
3860            if (required) {
3861                throw std::runtime_error("Key not found: " + key);
3862            }
3863            return;
3864        }
3865        if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
3866            throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
3867        }
3868        const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
3869        if (type != GGUF_TYPE_FLOAT32) {
3870            throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_FLOAT32)\n", __func__, key.c_str(), type, GGUF_TYPE_FLOAT32));
3871        }
3872        const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
3873        if (n > (size_t) std::numeric_limits<int>::max()) {
3874            throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
3875        }
3876        output.resize(n);
3877        const float * values = (const float *)gguf_get_arr_data(ctx_gguf.get(), i);
3878        for (size_t j = 0; j < n; ++j) {
3879            output[j] = values[j];
3880        }
3881    }
3882
3883    void get_string(const std::string & key, std::string & output, bool required = true) const {
3884        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3885        if (i < 0) {
3886            if (required) {
3887                throw std::runtime_error("Key not found: " + key);
3888            }
3889            return;
3890        }
3891        output = std::string(gguf_get_val_str(ctx_gguf.get(), i));
3892    }
3893
3894    void get_arr_int(const std::string & key, std::vector<int> & output, bool required = true) const {
3895        const int i = gguf_find_key(ctx_gguf.get(), key.c_str());
3896        if (i < 0) {
3897            if (required) {
3898                throw std::runtime_error("Key not found: " + key);
3899            }
3900            return;
3901        }
3902        if (gguf_get_kv_type(ctx_gguf.get(), i) != GGUF_TYPE_ARRAY) {
3903            throw std::runtime_error(string_format("%s: key '%s' is not an array\n", __func__, key.c_str()));
3904        }
3905        const auto type = gguf_get_arr_type(ctx_gguf.get(), i);
3906        if (type != GGUF_TYPE_INT32) {
3907            throw std::runtime_error(string_format("%s: array '%s' has type %d, expected %d (GGUF_TYPE_INT32)\n", __func__, key.c_str(), type, GGUF_TYPE_INT32));
3908        }
3909        const size_t n = gguf_get_arr_n(ctx_gguf.get(), i);
3910        if (n > (size_t) std::numeric_limits<int>::max()) {
3911            throw std::runtime_error(string_format("%s: array '%s' is too large (%zu elements)\n", __func__, key.c_str(), n));
3912        }
3913        output.resize(n);
3914        const int32_t * values = (const int32_t *)gguf_get_arr_data(ctx_gguf.get(), i);
3915        for (size_t j = 0; j < n; ++j) {
3916            output[j] = values[j];
3917        }
3918    }
3919
3920    static void set_llava_uhd_res_candidates(clip_model & model, const int max_patches_per_side) {
3921        auto & hparams = model.hparams;
3922        for (int x = 1; x <= max_patches_per_side; x++) {
3923            for (int y = 1; y <= max_patches_per_side; y++) {
3924                if (x == 1 && y == 1) {
3925                    continue; // skip the first point
3926                }
3927                hparams.image_res_candidates.push_back(clip_image_size{
3928                    x*hparams.image_size,
3929                    y*hparams.image_size,
3930                });
3931            }
3932        }
3933    }
3934
3935    static void set_internvl_dhr_res_candidates(clip_model & model) {
3936        auto & hparams = model.hparams;
3937        int min_num = hparams.preproc_min_tiles;
3938        int max_num = hparams.preproc_max_tiles;
3939        if (min_num < 1) {
3940           return; // avoid  divide by 0
3941        }
3942        for (int a = min_num; a <= max_num; ++a) {
3943            int b_lo = (min_num + a - 1) / a;
3944            int b_hi = max_num / a;
3945            b_lo = std::max(b_lo, min_num);
3946            b_hi = std::min(b_hi, max_num);
3947            for (int b = b_lo; b <= b_hi; ++b) {
3948                hparams.image_res_candidates.push_back(clip_image_size {
3949                    a*hparams.image_size,
3950                    b*hparams.image_size,
3951                });
3952            }
3953        }
3954    }
3955};
3956
3957struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) {
3958    clip_ctx * ctx_vision = nullptr;
3959    clip_ctx * ctx_audio = nullptr;
3960    clip_ctx * ctx_gen_audio = nullptr;
3961
3962    try {
3963        clip_model_loader loader(fname,
3964            /* skip_tensors */ false,
3965            ctx_params.progress_callback,
3966            ctx_params.progress_callback_user_data);
3967        bool skip_audio = false;
3968
3969        if (loader.has_vision) {
3970            ctx_vision = new clip_ctx(ctx_params);
3971            loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
3972            loader.load_tensors(*ctx_vision);
3973            loader.init_ctx(*ctx_vision);
3974            if (ctx_params.warmup) {
3975                loader.warmup(*ctx_vision);
3976            }
3977
3978            // TODO: we don't support audio for Gemma 3N, but GGUF contains audio tensors
3979            // we can remove this check when we implement audio support for Gemma 3N
3980            skip_audio = ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA3NV;
3981        }
3982
3983        if (loader.has_audio && !skip_audio) {
3984            ctx_audio = new clip_ctx(ctx_params);
3985            loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
3986            loader.load_tensors(*ctx_audio);
3987            loader.init_ctx(*ctx_audio);
3988            if (ctx_params.warmup) {
3989                loader.warmup(*ctx_audio);
3990            }
3991        }
3992
3993        if (loader.has_gen_audio) {
3994            ctx_gen_audio = new clip_ctx(ctx_params);
3995            loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
3996            loader.load_tensors(*ctx_gen_audio);
3997            // TODO: fix warmup
3998            ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
3999        }
4000
4001    } catch (const std::exception & e) {
4002        LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what());
4003
4004        delete ctx_vision;
4005        delete ctx_audio;
4006        delete ctx_gen_audio;
4007
4008        return {nullptr, nullptr, nullptr};
4009    }
4010
4011    return {ctx_vision, ctx_audio, ctx_gen_audio};
4012}
4013
4014struct clip_cap clip_get_cap(const char * fname) {
4015    clip_cap res;
4016    clip_model_loader loader(fname, /* skip_tensors= */ true);
4017    res.has_vision = loader.has_vision;
4018    res.has_audio  = loader.has_audio;
4019    return res;
4020}
4021
4022void clip_free(clip_ctx * ctx) {
4023    if (ctx == nullptr) {
4024        return;
4025    }
4026    delete ctx;
4027}
4028
4029const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
4030    return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat";
4031}
4032
4033int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
4034    const auto & params = ctx->model.hparams;
4035    const int n_total = clip_n_output_tokens(ctx, img);
4036    const auto & proj = ctx->proj_type();
4037    switch (proj) {
4038        case PROJECTOR_TYPE_QWEN2VL:
4039        case PROJECTOR_TYPE_QWEN25VL:
4040        case PROJECTOR_TYPE_QWEN3VL:
4041        case PROJECTOR_TYPE_EXAONE4_5:
4042        case PROJECTOR_TYPE_MIMOVL:
4043        case PROJECTOR_TYPE_GLM4V:
4044        case PROJECTOR_TYPE_PADDLEOCR:
4045        case PROJECTOR_TYPE_HUNYUANVL:
4046        case PROJECTOR_TYPE_YOUTUVL:
4047        case PROJECTOR_TYPE_MUSE_GLIMMER:
4048            return (img->nx() / params.patch_size) / 2;
4049        case PROJECTOR_TYPE_STEP3VL:
4050            return img->nx() / (params.patch_size * params.n_merge);
4051        case PROJECTOR_TYPE_DEEPSEEKOCR:
4052        case PROJECTOR_TYPE_DEEPSEEKOCR2:
4053            return (img->nx() / params.patch_size) / 4;
4054        default:
4055            break;
4056    }
4057    return n_total;
4058}
4059
4060int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
4061    const auto & params = ctx->model.hparams;
4062    const auto & proj = ctx->proj_type();
4063    switch (proj) {
4064        case PROJECTOR_TYPE_QWEN2VL:
4065        case PROJECTOR_TYPE_QWEN25VL:
4066        case PROJECTOR_TYPE_QWEN3VL:
4067        case PROJECTOR_TYPE_EXAONE4_5:
4068        case PROJECTOR_TYPE_MIMOVL:
4069        case PROJECTOR_TYPE_GLM4V:
4070        case PROJECTOR_TYPE_PADDLEOCR:
4071        case PROJECTOR_TYPE_HUNYUANVL:
4072        case PROJECTOR_TYPE_YOUTUVL:
4073        case PROJECTOR_TYPE_MUSE_GLIMMER:
4074            return (img->ny() / params.patch_size) / 2;
4075        case PROJECTOR_TYPE_STEP3VL:
4076            return img->ny() / (params.patch_size * params.n_merge);
4077        default:
4078            break;
4079    }
4080    return 1;
4081}
4082
4083int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
4084    const auto & params = ctx->model.hparams;
4085
4086    // for models with fixed size image, the input image is already pre-processed and resized to square
4087    int patch_size = params.patch_size;
4088    int n_patches = (img->nx() / patch_size) * (img->ny() / patch_size);
4089
4090    projector_type proj = ctx->proj_type();
4091
4092    switch (proj) {
4093        case PROJECTOR_TYPE_MLP:
4094        case PROJECTOR_TYPE_MLP_NORM:
4095        case PROJECTOR_TYPE_JANUS_PRO:
4096        case PROJECTOR_TYPE_PHI4:
4097            {
4098                // do nothing
4099            } break;
4100        case PROJECTOR_TYPE_YASA2:
4101            {
4102                n_patches = 64; // adaptive average pooling to 8x8 tokens
4103            } break;
4104        case PROJECTOR_TYPE_LDP:
4105        case PROJECTOR_TYPE_LDPV2:
4106        case PROJECTOR_TYPE_GLM_EDGE:
4107            {
4108                n_patches /= 4;
4109                if (ctx->model.mm_boi) {
4110                    n_patches += 2; // for BOI and EOI token embeddings
4111                }
4112            } break;
4113        case PROJECTOR_TYPE_MINICPMV:
4114            {
4115                // Use actual config value if available, otherwise fall back to hardcoded values
4116                if (params.minicpmv_query_num > 0) {
4117                    n_patches = params.minicpmv_query_num;
4118                } else {
4119                    // Fallback to hardcoded values for legacy models
4120                    if (params.minicpmv_version == 2) {
4121                        n_patches = 96;
4122                    } else if (params.minicpmv_version == 3) {
4123                        n_patches = 64;
4124                    } else if (params.minicpmv_version == 4) {
4125                        n_patches = 64;
4126                    } else if (params.minicpmv_version == 5) {
4127                        // MiniCPM-V 4.0
4128                        n_patches = 64;
4129                    } else if (params.minicpmv_version == 6) {
4130                        // MiniCPM-V 4.5
4131                        n_patches = 64;
4132                    } else if (params.minicpmv_version == 100045) {
4133                        // MiniCPM-o 4.5
4134                        n_patches = 64;
4135                    } else {
4136                        GGML_ABORT("Unknown minicpmv version");
4137                    }
4138                }
4139            } break;
4140        case PROJECTOR_TYPE_MINICPMV4_6:
4141            {
4142                n_patches /= params.n_merge * params.n_merge;
4143            } break;
4144        case PROJECTOR_TYPE_QWEN2VL:
4145        case PROJECTOR_TYPE_QWEN25VL:
4146        case PROJECTOR_TYPE_QWEN3VL:
4147        case PROJECTOR_TYPE_EXAONE4_5:
4148        case PROJECTOR_TYPE_MIMOVL:
4149        case PROJECTOR_TYPE_MINIMAX_M3:
4150        case PROJECTOR_TYPE_GLM4V:
4151        case PROJECTOR_TYPE_YOUTUVL:
4152        case PROJECTOR_TYPE_MUSE_GLIMMER:
4153            {
4154                // dynamic size (2 conv, so double patch size)
4155                int x_patch = img->nx() / (params.patch_size * 2);
4156                int y_patch = img->ny() / (params.patch_size * 2);
4157                n_patches = x_patch * y_patch;
4158            } break;
4159        case PROJECTOR_TYPE_STEP3VL:
4160            {
4161                int x_patch = img->nx() / (params.patch_size * params.n_merge);
4162                int y_patch = img->ny() / (params.patch_size * params.n_merge);
4163                n_patches = x_patch * y_patch;
4164            } break;
4165        case PROJECTOR_TYPE_GEMMA3:
4166        case PROJECTOR_TYPE_GEMMA4V:
4167        case PROJECTOR_TYPE_GEMMA4UV:
4168        case PROJECTOR_TYPE_IDEFICS3:
4169        case PROJECTOR_TYPE_INTERNVL:
4170        case PROJECTOR_TYPE_NEMOTRON_V2_VL:
4171        case PROJECTOR_TYPE_LLAMA4:
4172            {
4173                // both X and Y are downscaled by the scale factor
4174                int scale_factor = ctx->model.hparams.n_merge;
4175                n_patches /= (scale_factor * scale_factor);
4176            } break;
4177        case PROJECTOR_TYPE_GEMMA3NV:
4178            {
4179                // MobileNetV5 MSFA adapter always outputs fixed 16x16 resolution
4180                // regardless of input size (see architecture description)
4181                n_patches = ctx->model.hparams.image_size / ctx->model.hparams.patch_size;
4182            } break;
4183        case PROJECTOR_TYPE_LFM2:
4184        case PROJECTOR_TYPE_KIMIVL:
4185        case PROJECTOR_TYPE_KIMIK25:
4186            {
4187                // dynamic size
4188                int out_patch_size = params.patch_size * ctx->model.hparams.n_merge;
4189                int x_patch = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
4190                int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
4191                n_patches = x_patch * y_patch;
4192            } break;
4193        case PROJECTOR_TYPE_DEEPSEEK4V:
4194            {
4195                const int out_patch_size = params.patch_size * params.n_merge;
4196                const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
4197                const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
4198                n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
4199            } break;
4200        case PROJECTOR_TYPE_PADDLEOCR:
4201        case PROJECTOR_TYPE_DOTS_OCR:
4202        case PROJECTOR_TYPE_DOTS3NOTE_V:
4203            {
4204                // dynamic size
4205                int n_merge = ctx->model.hparams.n_merge;
4206                int stride = n_merge * n_merge;
4207                n_patches = CLIP_ALIGN(n_patches, stride) / stride;
4208            } break;
4209        case PROJECTOR_TYPE_DOTS3NOTE_A:
4210            {
4211                // 3x stride-2 conv2d over mel frames
4212                n_patches = (img->nx() + 7) / 8;
4213            } break;
4214        case PROJECTOR_TYPE_PIXTRAL:
4215        case PROJECTOR_TYPE_LIGHTONOCR:
4216            {
4217                // dynamic size
4218                int n_merge = ctx->model.hparams.n_merge;
4219                int n_patches_x = img->nx() / patch_size / n_merge;
4220                int n_patches_y = img->ny() / patch_size / n_merge;
4221                if (ctx->model.token_embd_img_break) {
4222                    n_patches = n_patches_y * n_patches_x + n_patches_y - 1; // + one [IMG_BREAK] per row, except the last row
4223                } else {
4224                    n_patches = n_patches_y * n_patches_x;
4225                }
4226            } break;
4227        case PROJECTOR_TYPE_VOXTRAL:
4228        case PROJECTOR_TYPE_ULTRAVOX:
4229        case PROJECTOR_TYPE_QWEN2A:
4230        case PROJECTOR_TYPE_MERALION:
4231        case PROJECTOR_TYPE_MUSIC_FLAMINGO:
4232            {
4233                n_patches = img->nx();
4234
4235                const int proj_stack_factor = ctx->model.hparams.proj_stack_factor;
4236                if (ctx->model.audio_has_stack_frames()) {
4237                    GGML_ASSERT(proj_stack_factor > 0);
4238                    const int n_len = CLIP_ALIGN(n_patches, proj_stack_factor);
4239                    n_patches = n_len / proj_stack_factor;
4240                }
4241
4242                // whisper downscales input token by half after conv1d
4243                n_patches /= 2;
4244
4245                if (ctx->model.audio_has_avgpool()) {
4246                    // divide by 2 because of nn.AvgPool1d(2, stride=2)
4247                    n_patches /= 2;
4248                }
4249            } break;
4250        case PROJECTOR_TYPE_QWEN3A:
4251            {
4252                // chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
4253                const int chunk_size       = 100;
4254                const int tokens_per_chunk = 13;
4255                n_patches = (img->nx() / chunk_size) * tokens_per_chunk;
4256            } break;
4257        case PROJECTOR_TYPE_GLMA:
4258            {
4259                n_patches = img->nx();
4260                // whisper downscales input token by half after conv1d
4261                n_patches /= 2;
4262                // reshape by merge_factor
4263                n_patches /= ctx->model.hparams.proj_stack_factor;
4264                // for BOI and EOI token embeddings
4265                n_patches += 2;
4266            } break;
4267        case PROJECTOR_TYPE_COGVLM:
4268            {
4269                n_patches += 2; // for BOI and EOI token embeddings
4270            } break;
4271        case PROJECTOR_TYPE_DEEPSEEKOCR:
4272        {
4273            // SAM encoder applies two stride-2 convolutions (net_2 and net_3)
4274            // that reduce spatial dimensions by 4x in each direction (16x total)
4275            // E.g., 64x64 -> 16x16 patches
4276            n_patches /= 16;
4277
4278            if (img->add_viewsep) {
4279                // global view: one image-newline per token-row + trailing view separator
4280                const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
4281                n_patches = h * (h + 1) + 1;
4282            } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
4283                // tile row: one image-newline per token-row
4284                const int grid_w = img->ny() / img->nx();
4285                const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
4286                const int h = tile_patches;
4287                n_patches = (tile_patches * grid_w + 1) * h;
4288            }
4289        } break;
4290        case PROJECTOR_TYPE_HUNYUANVL:
4291            {
4292                int merge = ctx->model.hparams.n_merge;
4293                int ow = (img->nx() / patch_size) / merge;
4294                int oh = (img->ny() / patch_size) / merge;
4295                n_patches = (ow + 1) * oh + 2;
4296            } break;
4297        case PROJECTOR_TYPE_DEEPSEEKOCR2:
4298        {
4299            // 1024 global view -> 256 query tokens + 1 view separator = 257;
4300            // 768 local tile   -> 144 query tokens, no separator.
4301            n_patches /= 16;
4302            if (img->add_viewsep) {
4303                n_patches += 1; // view separator, appended only after the global view
4304            }
4305        } break;
4306        case PROJECTOR_TYPE_LFM2A:
4307            {
4308                n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2;
4309            } break;
4310        case PROJECTOR_TYPE_GEMMA4A:
4311            {
4312                // Two Conv2D stride-2: O = floor((I + 2p - k) / s) + 1, p=1, k=3, s=2
4313                // O = floor((I - 1) / 2) + 1
4314                int n = img->nx();
4315                for (int i = 0; i < 2; i++) {
4316                    n = (n - 1) / 2 + 1;
4317                }
4318                n_patches = n;
4319            } break;
4320        case PROJECTOR_TYPE_PARAKEET:
4321            {
4322                n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
4323            } break;
4324        case PROJECTOR_TYPE_GEMMA4UA:
4325            {
4326                n_patches = img->nx();  // no downsampling: one token per raw waveform frame
4327            } break;
4328        case PROJECTOR_TYPE_MIMO_AUDIO:
4329            {
4330                // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
4331                int n = img->nx();
4332                n = (n - 1) / 2 + 1;         // conv1 + conv2
4333                n = (n - 2) / 2 + 1;         // downsample conv
4334                const int group_size = params.audio_local_group_size;
4335                n_patches = (n + group_size - 1) / group_size;
4336            } break;
4337        case PROJECTOR_TYPE_GRANITE_SPEECH:
4338            {
4339                const int ws = ctx->model.hparams.audio_proj_window_size;
4340                const int ds = ctx->model.hparams.audio_proj_downsample_rate;
4341                n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
4342            } break;
4343        case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
4344            {
4345                // pooling gives one speaker embedding, whatever the clip length is
4346                n_patches = 1;
4347            } break;
4348        case PROJECTOR_TYPE_QWEN3TTS_GEN:
4349            {
4350                // one hidden-state vector fed back to the talker per call
4351                n_patches = 1;
4352            } break;
4353        case PROJECTOR_TYPE_POCKETTTS_SPKENC:
4354            {
4355                // one conditioning row per 12.5Hz frame
4356                const int hop = ctx->model.hparams.mimi_downsample * 120;
4357                n_patches = img->nx() / hop;
4358            } break;
4359        case PROJECTOR_TYPE_POCKETTTS_GEN:
4360            {
4361                // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller
4362                n_patches = 1;
4363            } break;
4364        case PROJECTOR_TYPE_GRANITE4_VISION:
4365            {
4366                // Per-tile output token count: each projector block outputs
4367                // query_side^2 tokens per window x n^2 windows.
4368                // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.
4369                const int window_side = ctx->model.hparams.downsample_window_side;
4370                const int query_side  = ctx->model.hparams.downsample_query_side;
4371                const int side        = img->nx() / params.patch_size;
4372                const int n           = side / window_side;
4373                const int out_side    = query_side * n;
4374                n_patches             = out_side * out_side;
4375                if (img->anyres.is_tiled()) {
4376                    // overview tile, then the unpadded tile grid with one newline per row
4377                    int off_x, off_y, w, h;
4378                    clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,
4379                                      img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);
4380                    n_patches += h * (w + 1);
4381                }
4382            } break;
4383        default:
4384            GGML_ABORT("unsupported projector type");
4385    }
4386
4387    return n_patches;
4388}
4389
4390bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec) {
4391    clip_image_f32_batch imgs;
4392    clip_image_f32 img_copy = *img;
4393    imgs.entries.push_back(std::move(img_copy));
4394
4395    return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec);
4396}
4397
4398bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector<float> & out_batch_embd) {
4399    clip_encode_params params;
4400    params.imgs = imgs_c_ptr;
4401    params.n_threads = n_threads;
4402    params.out_embd = &out_batch_embd;
4403
4404    return clip_encode(ctx, &params);
4405}
4406
4407// persisted state slots of the gen-audio decoder, per pipeline
4408static std::vector<c2w_state_slot> list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
4409    switch (model.proj_type) {
4410        case PROJECTOR_TYPE_QWEN3TTS_GEN:  return list_c2w_state_slots(hparams, model);
4411        case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
4412        default:                           return {};
4413    }
4414}
4415
4416bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
4417    const clip_image_f32_batch & imgs = *params->imgs;
4418    int n_batch_cur = imgs.entries.size();
4419
4420    // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames
4421    if (!ctx->support_batch && n_batch_cur > clip_model_n_temporal_merge(ctx)) {
4422        LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, clip_model_n_temporal_merge(ctx));
4423        return false;
4424    }
4425
4426    // if buffers are not allocated, we need to do a warmup run to allocate them
4427    if (!ctx->is_allocated) {
4428        clip_model_loader::warmup(*ctx, *params->imgs);
4429    }
4430
4431    if (params->seed != ctx->rng_seed) {
4432        ctx->rng_seed = params->seed;
4433        ctx->rng.seed(params->seed == UINT32_MAX ? std::random_device{}() : params->seed);
4434    }
4435
4436    // build the inference graph
4437    ggml_backend_sched_reset(ctx->sched.get());
4438    ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build();
4439    ggml_backend_sched_alloc_graph(ctx->sched.get(), gf);
4440
4441    // set inputs
4442    const auto & model   = ctx->model;
4443    const auto & hparams = model.hparams;
4444
4445    const int image_size_width  = imgs.entries[0].nx();
4446    const int image_size_height = imgs.entries[0].ny();
4447
4448    const int patch_size    = hparams.patch_size;
4449    const int num_patches   = ((image_size_width / patch_size) * (image_size_height / patch_size));
4450    const int n_pos = num_patches + (model.class_embedding ? 1 : 0);
4451    const int pos_w = image_size_width  / patch_size;
4452    const int pos_h = image_size_height / patch_size;
4453
4454
4455    auto get_inp_tensor = [&gf](const char * name) {
4456        ggml_tensor * inp = ggml_graph_get_tensor(gf, name);
4457        if (inp == nullptr) {
4458            GGML_ABORT("Failed to get tensor %s", name);
4459        }
4460        if (!(inp->flags & GGML_TENSOR_FLAG_INPUT)) {
4461            GGML_ABORT("Tensor %s is not an input tensor", name);
4462        }
4463        return inp;
4464    };
4465
4466    auto set_input_f32 = [&get_inp_tensor](const char * name, const std::vector<float> & values) {
4467        ggml_tensor * cur = get_inp_tensor(name);
4468        GGML_ASSERT(cur->type == GGML_TYPE_F32);
4469        GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
4470        ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
4471    };
4472
4473    auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) {
4474        ggml_tensor * cur = get_inp_tensor(name);
4475        GGML_ASSERT(cur->type == GGML_TYPE_I32);
4476        GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());
4477        ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
4478    };
4479
4480    // upload the decoder state from the previous call, or zero-fill on a cold start
4481    auto set_gen_state_in = [&]() {
4482        size_t offset = 0;
4483        for (const auto & slot : list_gen_state_slots(hparams, model)) {
4484            ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
4485            const size_t nb = ggml_nbytes(t);
4486            if (params->state_in && params->state_in->size() >= offset + nb) {
4487                ggml_backend_tensor_set(t, params->state_in->data() + offset, 0, nb);
4488            } else {
4489                std::vector<uint8_t> zeros(nb, 0);
4490                ggml_backend_tensor_set(t, zeros.data(), 0, nb);
4491            }
4492            offset += nb;
4493        }
4494    };
4495
4496    // rope positions and attention mask of the mimi transformers (pocket-tts).
4497    // the mask is causal with a sliding window, see _build_attention_mask() in the reference
4498    auto set_pockettts_tfm_inputs = [&]() {
4499        const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
4500        GGML_ASSERT(n_pos > 0);
4501        std::vector<int32_t> positions((size_t) n_pos);
4502        for (int64_t i = 0; i < n_pos; i++) {
4503            positions[(size_t) i] = (int32_t) i;
4504        }
4505        set_input_i32("inp_pos", positions);
4506
4507        // the preprocessor truncates the waveform to keep this mask bounded
4508        const int64_t max_pos = (int64_t) clip_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
4509        GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
4510
4511        const int64_t context = hparams.mimi_tfm_context;
4512        std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
4513        for (int64_t q = 0; q < n_pos; q++) {
4514            for (int64_t k = 0; k < n_pos; k++) {
4515                const int64_t delta = q - k;
4516                if (delta >= 0 && delta < context) {
4517                    mask[(size_t) q * n_pos + k] = 0.0f;
4518                }
4519            }
4520        }
4521        set_input_f32("kq_mask", mask);
4522    };
4523
4524    // set input pixel values
4525    if (!imgs.is_audio) {
4526        size_t nelem = 0;
4527        for (const auto & img : imgs.entries) {
4528            nelem += img.nx() * img.ny() * 3;
4529        }
4530        std::vector<float> inp_raw(nelem);
4531
4532        // layout of data (note: the channel dim is unrolled to better visualize the layout):
4533        //
4534        // ┌──W──┐
4535        // │     H │  channel = R
4536        // ├─────┤ │
4537        // │     H │  channel = G
4538        // ├─────┤ │
4539        // │     H │  channel = B
4540        // └─────┘ │
4541        //   ──────┘ x B
4542
4543        // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models
4544        // All entries must have the same spatial size (enforced by can_batch_with() during merging)
4545        {
4546            const int nx = imgs.entries[0].nx();
4547            const int ny = imgs.entries[0].ny();
4548            const int n  = nx * ny;
4549
4550            for (int b = 0; b < n_batch_cur; b++) {
4551                LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny);
4552                const auto & buf = imgs.entries[b].get_ro_buf();
4553                float * batch_entry = inp_raw.data() + b * (3*n);
4554                for (int y = 0; y < ny; y++) {
4555                    for (int x = 0; x < nx; x++) {
4556                        size_t base_src = 3*(y * nx + x);
4557                        size_t base_dst =    y * nx + x;
4558                        batch_entry[      base_dst] = buf[base_src    ];
4559                        batch_entry[1*n + base_dst] = buf[base_src + 1];
4560                        batch_entry[2*n + base_dst] = buf[base_src + 2];
4561                    }
4562                }
4563            }
4564        }
4565        set_input_f32("inp_raw", inp_raw);
4566
4567    } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV) {
4568        // audio input. GEN_WAV is not here: it takes codes or feats, set in the switch below
4569        GGML_ASSERT(imgs.entries.size() == 1);
4570
4571        const auto & mel_inp = imgs.entries[0];
4572        const auto & buf = mel_inp.get_ro_buf();
4573        const int n_step = mel_inp.nx();
4574        const int n_mel  = mel_inp.ny();
4575        GGML_ASSERT((size_t)n_step * n_mel == buf.size());
4576
4577        set_input_f32("inp_raw", buf);
4578    }
4579
4580    // set input per projector
4581    switch (ctx->model.proj_type) {
4582        case PROJECTOR_TYPE_MUSE_GLIMMER:
4583            {
4584                const int grid_w = pos_w;            // image_size_width  / patch_size
4585                const int grid_h = pos_h;            // image_size_height / patch_size
4586                const int n_tok  = grid_w * grid_h;
4587                const int pgrid  = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32
4588                const int f      = hparams.n_merge;  // downsample 2
4589
4590                // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d);
4591                // pos-emb bilinear interp via resize_position_embeddings().
4592
4593                // --- sparse window grouping (pgrid x pgrid windows) ---
4594                const int win = pgrid;
4595                const int nwin_h = (grid_h + win - 1) / win;
4596                const int nwin_w = (grid_w + win - 1) / win;
4597                std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok);
4598                std::vector<int>     sp_slens;
4599                for (int wy = 0; wy < nwin_h; wy++) {
4600                    for (int wx = 0; wx < nwin_w; wx++) {
4601                        int cnt = 0;
4602                        for (int hh = 0; hh < win; hh++) {
4603                            for (int ww = 0; ww < win; ww++) {
4604                                const int gy = wy * win + hh;
4605                                const int gx = wx * win + ww;
4606                                if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; }
4607                            }
4608                        }
4609                        if (cnt > 0) sp_slens.push_back(cnt);
4610                    }
4611                }
4612                std::vector<int32_t> rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok);
4613                for (int i = 0; i < n_tok; i++) {
4614                    const int orig = sp_perm[i];
4615                    rpos_w[i] = (orig % grid_w) + 1; // 1-indexed
4616                    rpos_h[i] = (orig / grid_w) + 1;
4617                    inv_perm[orig] = i;
4618                }
4619                set_input_i32("muse_glimmer_sp_perm",  sp_perm);
4620                set_input_i32("muse_glimmer_inv_perm", inv_perm);
4621                set_input_i32("muse_glimmer_pos_w",    rpos_w);
4622                set_input_i32("muse_glimmer_pos_h",    rpos_h);
4623
4624                // block-diagonal window mask (permuted order)
4625                std::vector<float> sp_mask((size_t) n_tok * n_tok, -INFINITY);
4626                {
4627                    int off = 0;
4628                    for (int s : sp_slens) {
4629                        for (int a = 0; a < s; a++)
4630                            for (int b = 0; b < s; b++)
4631                                sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f;
4632                        off += s;
4633                    }
4634                }
4635                set_input_f32("muse_glimmer_sp_mask", sp_mask);
4636
4637                // pixel-shuffle gather (original order): f*f spatial neighbours grouped
4638                std::vector<int32_t> dsp; dsp.reserve(n_tok);
4639                for (int oy = 0; oy < grid_h / f; oy++)
4640                    for (int ox = 0; ox < grid_w / f; ox++)
4641                        for (int ry = 0; ry < f; ry++)
4642                            for (int rx = 0; rx < f; rx++)
4643                                dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx));
4644                set_input_i32("muse_glimmer_ds_perm", dsp);
4645            } break;
4646        case PROJECTOR_TYPE_MINICPMV:
4647            {
4648                // inspired from siglip:
4649                //    -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
4650                //    -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
4651                std::vector<int32_t> positions(pos_h * pos_w);
4652                int bucket_coords_h[1024];
4653                int bucket_coords_w[1024];
4654                for (int i = 0; i < pos_h; i++){
4655                    bucket_coords_h[i] = std::floor(70.0*i/pos_h);
4656                }
4657                for (int i = 0; i < pos_w; i++){
4658                    bucket_coords_w[i] = std::floor(70.0*i/pos_w);
4659                }
4660                for (int i = 0, id = 0; i < pos_h; i++){
4661                    for (int j = 0; j < pos_w; j++){
4662                        positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
4663                    }
4664                }
4665                set_input_i32("positions", positions);
4666
4667                // inputs for resampler projector
4668                // set the 2D positions (using float for sinusoidal embedding)
4669                int n_patches_per_col = image_size_width / patch_size;
4670                std::vector<float> pos_data(n_pos);
4671                // dimension H
4672                for (int i = 0; i < n_pos; i++) {
4673                    pos_data[i] = static_cast<float>(i / n_patches_per_col);
4674                }
4675                set_input_f32("pos_h", pos_data);
4676                // dimension W
4677                for (int i = 0; i < n_pos; i++) {
4678                    pos_data[i] = static_cast<float>(i % n_patches_per_col);
4679                }
4680                set_input_f32("pos_w", pos_data);
4681                // base frequency omega
4682                const float base_freq   = 10000.0f;
4683                const int   n_embd_proj = clip_n_mmproj_embd(ctx);
4684                std::vector<float> omega(n_embd_proj / 4);
4685                for (int i = 0; i < n_embd_proj / 4; ++i) {
4686                    omega[i] = 1.0f / std::pow(base_freq, static_cast<float>(i) / (n_embd_proj / 4));
4687                }
4688                set_input_f32("omega", omega);
4689            } break;
4690        case PROJECTOR_TYPE_MINICPMV4_6:
4691            {
4692                const bool is_4x = hparams.n_merge == 2;
4693
4694                // SigLIP position buckets (same as resampler path)
4695                std::vector<int32_t> positions(pos_h * pos_w);
4696                int bucket_coords_h[1024];
4697                int bucket_coords_w[1024];
4698                for (int i = 0; i < pos_h; i++){
4699                    bucket_coords_h[i] = std::floor(70.0*i/pos_h);
4700                }
4701                for (int i = 0; i < pos_w; i++){
4702                    bucket_coords_w[i] = std::floor(70.0*i/pos_w);
4703                }
4704                for (int i = 0, id = 0; i < pos_h; i++){
4705                    for (int j = 0; j < pos_w; j++){
4706                        positions[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
4707                    }
4708                }
4709                set_input_i32("positions", positions);
4710
4711                const int half_h = pos_h / 2;
4712                const int half_w = pos_w / 2;
4713
4714                auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) {
4715                    std::vector<int32_t> idx(ds_h * ds_w);
4716                    for (int i = 0; i < ds_h; i++) {
4717                        for (int j = 0; j < ds_w; j++) {
4718                            idx[i * ds_w + j] = (2*i + off_r) * stride_w + (2*j + off_c);
4719                        }
4720                    }
4721                    return idx;
4722                };
4723
4724                if (!is_4x) {
4725                    // window reorder indices for 2x2 windows
4726                    std::vector<int32_t> window_idx(n_pos);
4727                    std::vector<int32_t> inv_window_idx(n_pos);
4728                    {
4729                        int k = 0;
4730                        for (int wi = 0; wi < half_h; wi++) {
4731                            for (int wj = 0; wj < half_w; wj++) {
4732                                window_idx[k++] = (2*wi    ) * pos_w + (2*wj    );
4733                                window_idx[k++] = (2*wi    ) * pos_w + (2*wj + 1);
4734                                window_idx[k++] = (2*wi + 1) * pos_w + (2*wj    );
4735                                window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);
4736                            }
4737                        }
4738                        for (int i = 0; i < n_pos; i++) {
4739                            inv_window_idx[window_idx[i]] = i;
4740                        }
4741                    }
4742                    set_input_i32("vit_merger_window_idx",     window_idx);
4743                    set_input_i32("vit_merger_inv_window_idx", inv_window_idx);
4744
4745                    // block-diagonal attention mask: tokens in the same 4-token
4746                    // window attend to each other (mask = 0), all other positions
4747                    // are masked out (-inf). matches the window-major reorder above.
4748                    std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());
4749                    for (int wi = 0; wi < n_pos / 4; wi++) {
4750                        for (int i = 0; i < 4; i++) {
4751                            for (int j = 0; j < 4; j++) {
4752                                window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;
4753                            }
4754                        }
4755                    }
4756                    set_input_f32("vit_merger_window_mask", window_mask_data);
4757
4758                    // ViT merger 2x2 downsample indices
4759                    auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
4760                    auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);
4761                    auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);
4762                    auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);
4763                    set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);
4764                    set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);
4765                    set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);
4766                    set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);
4767                }
4768
4769                const int merger_h = is_4x ? pos_h : half_h;
4770                const int merger_w = is_4x ? pos_w : half_w;
4771                auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w);
4772                auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w);
4773                auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w);
4774                auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w);
4775                set_input_i32("merger_ds_idx_0", m_ds_0);
4776                set_input_i32("merger_ds_idx_1", m_ds_1);
4777                set_input_i32("merger_ds_idx_2", m_ds_2);
4778                set_input_i32("merger_ds_idx_3", m_ds_3);
4779            } break;
4780        case PROJECTOR_TYPE_QWEN2VL:
4781        case PROJECTOR_TYPE_QWEN3VL:
4782        case PROJECTOR_TYPE_GLM4V:
4783            {
4784                const int merge_ratio = hparams.n_merge;
4785                const int pw = image_size_width  / patch_size;
4786                const int ph = image_size_height / patch_size;
4787                std::vector<int> positions(n_pos * 4);
4788                int ptr = 0;
4789                for (int y = 0; y < ph; y += merge_ratio) {
4790                    for (int x = 0; x < pw; x += merge_ratio) {
4791                        for (int dy = 0; dy < 2; dy++) {
4792                            for (int dx = 0; dx < 2; dx++) {
4793                                positions[                  ptr] = y + dy;
4794                                positions[    num_patches + ptr] = x + dx;
4795                                positions[2 * num_patches + ptr] = y + dy;
4796                                positions[3 * num_patches + ptr] = x + dx;
4797                                ptr++;
4798                            }
4799                        }
4800                    }
4801                }
4802
4803                set_input_i32("positions", positions);
4804            } break;
4805        case PROJECTOR_TYPE_STEP3VL:
4806            {
4807                std::vector<int32_t> pos_data(n_pos);
4808                for (int i = 0; i < n_pos; i++) {
4809                    pos_data[i] = i / pos_w;
4810                }
4811                set_input_i32("pos_h", pos_data);
4812                for (int i = 0; i < n_pos; i++) {
4813                    pos_data[i] = i % pos_w;
4814                }
4815                set_input_i32("pos_w", pos_data);
4816            } break;
4817        case PROJECTOR_TYPE_PADDLEOCR:
4818            {
4819                const int merge_ratio = hparams.n_merge;
4820                const int pw = image_size_width  / patch_size;
4821                const int ph = image_size_height / patch_size;
4822                std::vector<int> positions(n_pos * 4);
4823                int ptr = 0;
4824                // NOTE: same as Qwen-VL, but x and y are swapped
4825                for (int y = 0; y < ph; y += merge_ratio) {
4826                    for (int dy = 0; dy < 2; dy++) {
4827                        for (int x = 0; x < pw; x += merge_ratio) {
4828                            for (int dx = 0; dx < 2; dx++) {
4829                                positions[                  ptr] = y + dy;
4830                                positions[    num_patches + ptr] = x + dx;
4831                                positions[2 * num_patches + ptr] = y + dy;
4832                                positions[3 * num_patches + ptr] = x + dx;
4833                                ptr++;
4834                            }
4835                        }
4836                    }
4837                }
4838
4839                set_input_i32("positions", positions);
4840            } break;
4841        case PROJECTOR_TYPE_MINIMAX_M3:
4842            {
4843                const int n_merge = hparams.n_merge;
4844                const int gh = image_size_height / patch_size;
4845                const int gw = image_size_width  / patch_size;
4846                std::vector<int32_t> pos_h, pos_w;
4847                pos_h.reserve(gh * gw);
4848                pos_w.reserve(gh * gw);
4849                for (int bh = 0; bh < gh / n_merge; bh++)
4850                for (int bw = 0; bw < gw / n_merge; bw++)
4851                for (int mh = 0; mh < n_merge; mh++)
4852                for (int mw = 0; mw < n_merge; mw++) {
4853                    pos_h.push_back(bh * n_merge + mh);
4854                    pos_w.push_back(bw * n_merge + mw);
4855                }
4856                set_input_i32("minimax_pos_h", pos_h);
4857                set_input_i32("minimax_pos_w", pos_w);
4858            } break;
4859        case PROJECTOR_TYPE_DOTS_OCR:
4860        case PROJECTOR_TYPE_DOTS3NOTE_V:
4861            {
4862                const int pw = image_size_width / patch_size;
4863                const int ph = image_size_height / patch_size;
4864                const int n_pos = ph * pw;
4865                std::vector<int> positions(n_pos * 4);
4866                int ptr = 0;
4867
4868                // flat layout: [h, w, h, w] for each patch
4869                // patches are in raster order (matching conv2d output)
4870                for (int y = 0; y < ph; y++) {
4871                    for (int x = 0; x < pw; x++) {
4872                        positions[          ptr] = y;
4873                        positions[  n_pos + ptr] = x;
4874                        positions[2*n_pos + ptr] = y;
4875                        positions[3*n_pos + ptr] = x;
4876                        ptr++;
4877                    }
4878                }
4879
4880                set_input_i32("positions", positions);
4881            } break;
4882        case PROJECTOR_TYPE_QWEN25VL:
4883        case PROJECTOR_TYPE_EXAONE4_5:
4884        case PROJECTOR_TYPE_YOUTUVL:
4885            {
4886                // pw * ph = number of tokens output by ViT after apply patch merger
4887                // ipw * ipw = number of vision token been processed inside ViT
4888                const bool use_window_attn =
4889                    (ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL || ctx->model.proj_type == PROJECTOR_TYPE_EXAONE4_5)
4890                        ? hparams.n_wa_pattern > 0
4891                        : !hparams.wa_layer_indexes.empty();
4892                const int merge_ratio = 2;
4893                const int pw  = image_size_width  / patch_size / merge_ratio;
4894                const int ph  = image_size_height / patch_size / merge_ratio;
4895                const int ipw = image_size_width  / patch_size;
4896                const int iph = image_size_height / patch_size;
4897
4898                std::vector<int> idx    (ph * pw);
4899                std::vector<int> inv_idx(ph * pw);
4900
4901                if (use_window_attn) {
4902                    const int attn_window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112;
4903                    const int grid_window = attn_window_size / patch_size / merge_ratio;
4904                    int dst = 0;
4905                    // [num_vision_tokens, num_vision_tokens] attention mask tensor
4906                    std::vector<float> mask(pow(ipw * iph, 2), std::numeric_limits<float>::lowest());
4907                    int mask_row = 0;
4908
4909                    for (int y = 0; y < ph; y += grid_window) {
4910                        for (int x = 0; x < pw; x += grid_window) {
4911                            const int win_h = std::min(grid_window, ph - y);
4912                            const int win_w = std::min(grid_window, pw - x);
4913                            const int dst_0 = dst;
4914                            // group all tokens belong to the same window togather (to a continue range)
4915                            for (int dy = 0; dy < win_h; dy++) {
4916                                for (int dx = 0; dx < win_w; dx++) {
4917                                    const int src = (y + dy) * pw + (x + dx);
4918                                    GGML_ASSERT(src < (int)idx.size());
4919                                    GGML_ASSERT(dst < (int)inv_idx.size());
4920                                    idx    [src] = dst;
4921                                    inv_idx[dst] = src;
4922                                    dst++;
4923                                }
4924                            }
4925
4926                            for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) {
4927                                int row_offset = mask_row * (ipw * iph);
4928                                std::fill(
4929                                    mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio),
4930                                    mask.begin() + row_offset + (dst   * merge_ratio * merge_ratio),
4931                                    0.0);
4932                                mask_row++;
4933                            }
4934                        }
4935                    }
4936
4937                    set_input_i32("window_idx",     idx);
4938                    set_input_i32("inv_window_idx", inv_idx);
4939                    set_input_f32("window_mask",    mask);
4940                } else {
4941                    for (int i = 0; i < ph * pw; i++) {
4942                        idx[i] = i;
4943                    }
4944                }
4945
4946                const int mpow = merge_ratio * merge_ratio;
4947                std::vector<int> positions(n_pos * 4);
4948
4949                int ptr = 0;
4950                for (int y = 0; y < iph; y += merge_ratio) {
4951                    for (int x = 0; x < ipw; x += merge_ratio) {
4952                        for (int dy = 0; dy < 2; dy++) {
4953                            for (int dx = 0; dx < 2; dx++) {
4954                                auto remap = idx[ptr / mpow];
4955                                remap = (remap * mpow) + (ptr % mpow);
4956
4957                                positions[                  remap] = y + dy;
4958                                positions[    num_patches + remap] = x + dx;
4959                                positions[2 * num_patches + remap] = y + dy;
4960                                positions[3 * num_patches + remap] = x + dx;
4961                                ptr++;
4962                            }
4963                        }
4964                    }
4965                }
4966
4967                set_input_i32("positions", positions);
4968            } break;
4969        case PROJECTOR_TYPE_MIMOVL:
4970            {
4971                const int merge      = hparams.n_merge;        // 2
4972                const int merge_unit = merge * merge;          // 4
4973                const int patch      = hparams.patch_size;     // 16
4974                const int H          = image_size_height / patch;
4975                const int W          = image_size_width  / patch;
4976                const int n_pos_full = H * W;
4977                const int llm_h      = H / merge;
4978                const int llm_w      = W / merge;
4979                const int n_units    = llm_h * llm_w;          // n_pos / merge_unit
4980
4981                // Row-major merge-tile-ordered (h, w) positions
4982                std::vector<int32_t> pos_h_row(n_pos_full);
4983                std::vector<int32_t> pos_w_row(n_pos_full);
4984                {
4985                    int idx = 0;
4986                    for (int ty = 0; ty < llm_h; ty++) {
4987                        for (int tx = 0; tx < llm_w; tx++) {
4988                            for (int dy = 0; dy < merge; dy++) {
4989                                for (int dx = 0; dx < merge; dx++) {
4990                                    pos_h_row[idx] = ty * merge + dy;
4991                                    pos_w_row[idx] = tx * merge + dx;
4992                                    idx++;
4993                                }
4994                            }
4995                        }
4996                    }
4997                }
4998
4999                // Col-major merge-unit permutation
5000                std::vector<float> idx_col(n_units);
5001                for (int r = 0; r < llm_h; r++) {
5002                    for (int c = 0; c < llm_w; c++) {
5003                        int u_row = r * llm_w + c;
5004                        int u_col = c * llm_h + r;
5005                        idx_col[u_col] = (float) u_row;
5006                    }
5007                }
5008
5009                // Col-mode positions: permute pos_*_row by idx_col
5010                std::vector<int32_t> pos_h_col(n_pos_full);
5011                std::vector<int32_t> pos_w_col(n_pos_full);
5012                for (int u = 0; u < n_units; u++) {
5013                    int src = (int) idx_col[u];
5014                    for (int k = 0; k < merge_unit; k++) {
5015                        pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k];
5016                        pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k];
5017                    }
5018                }
5019
5020                // Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels
5021                // only read slots 0 and 1, so pack h in slot 0, w in slot 1:
5022                //   positions[0..n_pos)         = h
5023                //   positions[n_pos..2*n_pos)   = w
5024                //   positions[2*n_pos..3*n_pos) = 0
5025                //   positions[3*n_pos..4*n_pos) = 0
5026                std::vector<int32_t> positions_row(static_cast<size_t>(n_pos_full) * 4, 0);
5027                std::vector<int32_t> positions_col(static_cast<size_t>(n_pos_full) * 4, 0);
5028                for (int i = 0; i < n_pos_full; i++) {
5029                    positions_row[0 * n_pos_full + i] = pos_h_row[i];
5030                    positions_row[1 * n_pos_full + i] = pos_w_row[i];
5031                    positions_col[0 * n_pos_full + i] = pos_h_col[i];
5032                    positions_col[1 * n_pos_full + i] = pos_w_col[i];
5033                }
5034
5035                // Banded 1D sliding-window mask
5036                const int window = hparams.attn_window_size;
5037                GGML_ASSERT(window > 0);
5038                std::vector<float> mask(static_cast<size_t>(n_pos_full) * n_pos_full, std::numeric_limits<float>::lowest());
5039                for (int q = 0; q < n_pos_full; q++) {
5040                    int lo = std::max(0, q - window);
5041                    int hi = std::min(n_pos_full - 1, q + window);
5042                    for (int k = lo; k <= hi; k++) {
5043                        mask[static_cast<size_t>(q) * n_pos_full + k] = 0.0f;
5044                    }
5045                }
5046
5047                set_input_i32("mimovl_positions_row", positions_row);
5048                set_input_i32("mimovl_positions_col", positions_col);
5049                set_input_f32("mimovl_idx_col",       idx_col);
5050                set_input_f32("mimovl_window_mask",   mask);
5051            } break;
5052        case PROJECTOR_TYPE_PIXTRAL:
5053        case PROJECTOR_TYPE_KIMIVL:
5054        case PROJECTOR_TYPE_KIMIK25:
5055        case PROJECTOR_TYPE_LIGHTONOCR:
5056            {
5057                // set the 2D positions
5058                int n_patches_per_col = image_size_width / patch_size;
5059                std::vector<int> pos_data(n_pos);
5060                // dimension H
5061                for (int i = 0; i < n_pos; i++) {
5062                    pos_data[i] = i / n_patches_per_col;
5063                }
5064                set_input_i32("pos_h", pos_data);
5065                // dimension W
5066                for (int i = 0; i < n_pos; i++) {
5067                    pos_data[i] = i % n_patches_per_col;
5068                }
5069                set_input_i32("pos_w", pos_data);
5070            } break;
5071        case PROJECTOR_TYPE_DEEPSEEK4V:
5072            {
5073                // set the 2D positions (mrope layout, only the first 2 channels are used)
5074                int n_patches_per_row = image_size_width / patch_size;
5075                std::vector<int32_t> positions(n_pos * 4, 0);
5076                for (int i = 0; i < n_pos; i++) {
5077                    positions[i]         = i / n_patches_per_row; // row
5078                    positions[n_pos + i] = i % n_patches_per_row; // col
5079                }
5080                set_input_i32("positions", positions);
5081
5082                // token block layout index (see clip_graph_deepseek4v::build)
5083                // rows [0, n_grid) are the aligner output, the sentinels follow
5084                const int n_merge = hparams.n_merge;
5085                const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
5086                const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
5087                const int n_grid  = n_llm_w * n_llm_h;
5088                const int idx_start   = n_grid;
5089                const int idx_end     = n_grid + 1;
5090                const int idx_newline = n_grid + 2;
5091                const int idx_pad     = n_grid + 3;
5092
5093                const int lead_pad = imgs.entries[0].lead_pad;
5094                const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
5095
5096                std::vector<int32_t> idx;
5097                idx.reserve(bl.n_out);
5098                for (int i = 0; i < lead_pad; i++) {
5099                    idx.push_back(idx_pad);
5100                }
5101                idx.push_back(idx_start);
5102                // pairs of adjacent rows are interleaved column-wise ("N-layout")
5103                // ref: build_image_block in inference/image_processor.py
5104                for (int t = 0; t < bl.rows * bl.row_len; t++) {
5105                    const int g   = t / (2 * bl.row_len);
5106                    const int rem = t % (2 * bl.row_len);
5107                    const int c   = rem / 2; // column
5108                    const int r   = 2 * g + rem % 2; // row
5109                    if (r >= n_llm_h) {
5110                        idx.push_back(idx_pad);
5111                    } else if (c == n_llm_w) {
5112                        idx.push_back(idx_newline);
5113                    } else {
5114                        idx.push_back(r * n_llm_w + c);
5115                    }
5116                }
5117                for (int i = 0; i < bl.pad_last; i++) {
5118                    idx.push_back(idx_pad);
5119                }
5120                idx.push_back(idx_end);
5121                set_input_i32("layout_idx", idx);
5122            } break;
5123        case PROJECTOR_TYPE_GLM_EDGE:
5124        {
5125            // llava and other models
5126            std::vector<int32_t> positions(n_pos);
5127            for (int i = 0; i < n_pos; i++) {
5128                positions[i] = i;
5129            }
5130            set_input_i32("positions", positions);
5131        } break;
5132        case PROJECTOR_TYPE_MLP:
5133        case PROJECTOR_TYPE_MLP_NORM:
5134        case PROJECTOR_TYPE_LDP:
5135        case PROJECTOR_TYPE_LDPV2:
5136            {
5137                // llava and other models
5138                std::vector<int32_t> positions(n_pos);
5139                for (int i = 0; i < n_pos; i++) {
5140                    positions[i] = i;
5141                }
5142                set_input_i32("positions", positions);
5143
5144                // The patches vector is used to get rows to index into the embeds with;
5145                // we should skip dim 0 only if we have CLS to avoid going out of bounds
5146                // when retrieving the rows.
5147                int patch_offset = model.class_embedding ? 1 : 0;
5148                std::vector<int32_t> patches(num_patches);
5149                for (int i = 0; i < num_patches; i++) {
5150                    patches[i] = i + patch_offset;
5151                }
5152                set_input_i32("patches", patches);
5153            } break;
5154        case PROJECTOR_TYPE_POCKETTTS_SPKENC:
5155            {
5156                set_pockettts_tfm_inputs();
5157            } break;
5158        case PROJECTOR_TYPE_POCKETTTS_GEN:
5159            {
5160                if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
5161                    GGML_ASSERT(params->feats != nullptr);
5162                    set_input_f32("inp_feats", *params->feats);
5163                    // positions and mask are derived in-graph from the persisted counter
5164                    set_gen_state_in();
5165                } else {
5166                    // flow matching starts from gaussian noise, std = sqrt(temp)
5167                    ggml_tensor * t = get_inp_tensor("inp_noise");
5168                    // Config.default_temperature, for a caller that does not set one
5169                    const float temp = params->temp > 0.0f ? params->temp : 0.7f;
5170                    std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
5171                    std::vector<float> noise(ggml_nelements(t));
5172                    for (auto & v : noise) {
5173                        v = dist(ctx->rng);
5174                    }
5175                    set_input_f32("inp_noise", noise);
5176                }
5177            } break;
5178        case PROJECTOR_TYPE_GEMMA4V:
5179        case PROJECTOR_TYPE_GEMMA4UV:
5180            {
5181                // set (col, row) patch positions for learned positional embedding
5182                const int n_cols = image_size_width  / patch_size;
5183                std::vector<int> pos_x(num_patches), pos_y(num_patches);
5184                for (int i = 0; i < num_patches; i++) {
5185                    pos_x[i] = i % n_cols;
5186                    pos_y[i] = i / n_cols;
5187                }
5188                set_input_i32("pos_x", pos_x);
5189                set_input_i32("pos_y", pos_y);
5190            } break;
5191        case PROJECTOR_TYPE_DEEPSEEKOCR:
5192        case PROJECTOR_TYPE_DEEPSEEKOCR2:
5193            {
5194                GGML_ASSERT(
5195                    (pos_w == pos_h) // overview image
5196                    || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
5197                );
5198
5199                const int window = hparams.attn_window_size;
5200                const int pos = pos_w;
5201                std::vector<int32_t> rel_pos_indices_local(window * window);
5202                std::vector<int32_t> rel_pos_indices_global(pos * pos);
5203
5204                for (int q = 0; q < window; q++) {
5205                    for (int k = 0; k < window; k++) {
5206                        rel_pos_indices_local[q * window + k] = q - k + window - 1;
5207                    }
5208                }
5209
5210                for (int q = 0; q < pos; q++) {
5211                    for (int k = 0; k < pos; k++) {
5212                        rel_pos_indices_global[q * pos + k] = q - k + pos - 1;
5213                    }
5214                }
5215
5216                set_input_i32("rel_pos_indices_local", rel_pos_indices_local);
5217                set_input_i32("rel_pos_indices_global", rel_pos_indices_global);
5218
5219                if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEKOCR2) {
5220
5221                    // qwen2 encoder attention mask
5222
5223                    // num_image_tokens = num_patches / 16
5224                    //   256 for 1024 global view
5225                    //   144 for 768 tile views
5226                    const int   num_image_tokens = num_patches / 16;
5227                    const int   seq_len          = num_image_tokens * 2;
5228                    std::vector qwen2_mask(static_cast<size_t>(seq_len) * seq_len, 0.0f);
5229
5230                    // attention mask layout
5231                    //  +--------------+---------------+
5232                    //  |    all 0     |   all -inf    |
5233                    //  +--------------+---------------+
5234                    //  |    all 0     |  lower tri 0  |
5235                    //  +--------------+---------------+
5236                    for (int i = 0; i < seq_len; i++) {
5237                        for (int j = 0; j < seq_len; j++) {
5238                            const bool zero = i < num_image_tokens ?
5239                                                     j < num_image_tokens :
5240                                                     j < num_image_tokens || j <= i;
5241                            qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;
5242                        }
5243                    }
5244                    set_input_f32("qwen2_attn_mask", qwen2_mask);
5245                }
5246            } break;
5247        case PROJECTOR_TYPE_GEMMA3:
5248        case PROJECTOR_TYPE_GEMMA3NV:
5249        case PROJECTOR_TYPE_IDEFICS3:
5250        case PROJECTOR_TYPE_INTERNVL:
5251        case PROJECTOR_TYPE_NEMOTRON_V2_VL:
5252        case PROJECTOR_TYPE_QWEN2A:
5253        case PROJECTOR_TYPE_QWEN3A:
5254        case PROJECTOR_TYPE_GLMA:
5255        case PROJECTOR_TYPE_ULTRAVOX:
5256        case PROJECTOR_TYPE_LFM2:
5257        case PROJECTOR_TYPE_VOXTRAL:
5258        case PROJECTOR_TYPE_MERALION:
5259        case PROJECTOR_TYPE_MUSIC_FLAMINGO:
5260        case PROJECTOR_TYPE_JANUS_PRO:
5261        case PROJECTOR_TYPE_PHI4:
5262        case PROJECTOR_TYPE_COGVLM:
5263        case PROJECTOR_TYPE_YASA2:
5264        case PROJECTOR_TYPE_GEMMA4UA:
5265        case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
5266            {
5267                // do nothing
5268            } break;
5269        case PROJECTOR_TYPE_QWEN3TTS_GEN:
5270            {
5271                if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
5272                    GGML_ASSERT(params->codes != nullptr);
5273
5274                    // frame-major input to group-major, rear-padded with code 0 up to one window
5275                    const int64_t n_codes  = model.gen_code_head_w->ne[2] + 1;
5276                    const int64_t n_frames_w = hparams.wav_tfm_swa;
5277                    const int64_t n_frames   = (int64_t) params->codes->size() / n_codes;
5278                    GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w);
5279
5280                    // codes are used as ggml_get_rows indices, so check them against the codebook vocab
5281                    const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1];
5282                    const int64_t vocab_rest  = model.c2w.quant_rest_cb_w->ne[1];
5283                    for (int64_t f = 0; f < n_frames; f++) {
5284                        for (int64_t g = 0; g < n_codes; g++) {
5285                            const int32_t c = (*params->codes)[f * n_codes + g];
5286                            const int64_t vocab = (g == 0) ? vocab_first : vocab_rest;
5287                            if (c < 0 || (int64_t) c >= vocab) {
5288                                LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n",
5289                                        __func__, (long long) f, (long long) g, c, (long long) vocab);
5290                                return false;
5291                            }
5292                        }
5293                    }
5294
5295                    std::vector<int32_t> codes(n_frames_w * n_codes, 0);
5296                    for (int64_t f = 0; f < n_frames; f++) {
5297                        for (int64_t g = 0; g < n_codes; g++) {
5298                            codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g];
5299                        }
5300                    }
5301                    set_input_i32("inp_codes", codes);
5302                    set_gen_state_in();
5303                } else {
5304                    // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it
5305                    const int64_t vocab0 = model.gen_code_out_embd_w->ne[1];
5306                    if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) {
5307                        LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0);
5308                        return false;
5309                    }
5310                    std::vector<int32_t> code0 = { params->code0 };
5311                    set_input_i32("inp_code0", code0);
5312
5313                    // one uniform(0,1) draw per codebook, used by do_sampling()
5314                    std::uniform_real_distribution<float> dist(0.0f, 1.0f);
5315                    const int64_t n_acoustic = model.gen_code_head_w->ne[2];
5316                    for (int64_t g = 0; g < n_acoustic; g++) {
5317                        std::vector<float> r = { dist(ctx->rng) };
5318                        set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r);
5319                    }
5320                }
5321            } break;
5322        case PROJECTOR_TYPE_HUNYUANVL:
5323            {
5324                // Compute the HunyuanVL 2D position embedding on CPU (with the
5325                // custom sf=(target+0.1)/n_grid bilinear sampling that the
5326                // reference implementation uses) and upload it to the graph
5327                // input declared in clip_graph_hunyuanvl::build().
5328                GGML_ASSERT(model.position_embeddings != nullptr);
5329                ggml_tensor * src_t   = model.position_embeddings;
5330                const int64_t n_embd  = src_t->ne[0];
5331                const int64_t n_pos   = src_t->ne[1];            // = n_grid * n_grid
5332                const int     n_grid  = (int)std::lround(std::sqrt((double)n_pos));
5333                GGML_ASSERT((int64_t)n_grid * n_grid == n_pos);
5334                const int     out_w   = pos_w;                    // pw
5335                const int     out_h   = pos_h;                    // ph
5336
5337                // Pull weight to host.
5338                std::vector<float> src(n_embd * n_pos);
5339                ggml_backend_tensor_get(src_t, src.data(), 0, ggml_nbytes(src_t));
5340
5341                // Output layout matches ggml_new_tensor_2d(F32, n_embd, out_h*out_w):
5342                //   ne[0] = n_embd (fastest), ne[1] = out_h*out_w
5343                //   dst[(y*out_w + x) * n_embd + c]
5344                std::vector<float> dst((size_t)n_embd * out_h * out_w);
5345
5346                const float sx = (float)(out_w + 0.1f) / (float)n_grid;
5347                const float sy = (float)(out_h + 0.1f) / (float)n_grid;
5348
5349                for (int y = 0; y < out_h; ++y) {
5350                    // Match ggml_compute_forward_upscale_f32 pixel-center
5351                    // convention (align_corners=False): src_y = (y+0.5)/sy - 0.5.
5352                    const float fy = ((float)y + 0.5f) / sy - 0.5f;
5353                    int y0 = (int)std::floor(fy);
5354                    int y1 = y0 + 1;
5355                    y0 = std::clamp(y0, 0, n_grid - 1);
5356                    y1 = std::clamp(y1, 0, n_grid - 1);
5357                    float wy1 = std::clamp(fy - (float)y0, 0.0f, 1.0f);
5358                    const float wy0 = 1.0f - wy1;
5359                    for (int x = 0; x < out_w; ++x) {
5360                        const float fx = ((float)x + 0.5f) / sx - 0.5f;
5361                        int x0 = (int)std::floor(fx);
5362                        int x1 = x0 + 1;
5363                        x0 = std::clamp(x0, 0, n_grid - 1);
5364                        x1 = std::clamp(x1, 0, n_grid - 1);
5365                        float wx1 = std::clamp(fx - (float)x0, 0.0f, 1.0f);
5366                        const float wx0 = 1.0f - wx1;
5367
5368                        const float w00 = wy0 * wx0;
5369                        const float w01 = wy0 * wx1;
5370                        const float w10 = wy1 * wx0;
5371                        const float w11 = wy1 * wx1;
5372
5373                        const float * s00 = &src[((size_t)y0 * n_grid + x0) * n_embd];
5374                        const float * s01 = &src[((size_t)y0 * n_grid + x1) * n_embd];
5375                        const float * s10 = &src[((size_t)y1 * n_grid + x0) * n_embd];
5376                        const float * s11 = &src[((size_t)y1 * n_grid + x1) * n_embd];
5377                        float * d         = &dst[((size_t)y * out_w + x) * n_embd];
5378                        for (int c = 0; c < n_embd; ++c) {
5379                            d[c] = w00 * s00[c] + w01 * s01[c] + w10 * s10[c] + w11 * s11[c];
5380                        }
5381                    }
5382                }
5383
5384                set_input_f32("hunyuanvl_pos_embd", dst);
5385            } break;
5386        case PROJECTOR_TYPE_LLAMA4:
5387            {
5388                // set the 2D positions
5389                int n_patches_per_col = image_size_width / patch_size;
5390                std::vector<int> pos_data(num_patches + 1, 0); // +1 for the [CLS] token
5391                // last pos is always kept 0, it's for CLS
5392                // dimension H
5393                for (int i = 0; i < num_patches; i++) {
5394                    pos_data[i] = (i / n_patches_per_col) + 1;
5395                }
5396                set_input_i32("pos_h", pos_data);
5397                // dimension W
5398                for (int i = 0; i < num_patches; i++) {
5399                    pos_data[i] = (i % n_patches_per_col) + 1;
5400                }
5401                set_input_i32("pos_w", pos_data);
5402            } break;
5403        case PROJECTOR_TYPE_DOTS3NOTE_A:
5404            {
5405                GGML_ASSERT(imgs.entries.size() == 1);
5406                const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
5407                std::vector<int32_t> positions(n_pos);
5408                for (int i = 0; i < n_pos; i++) {
5409                    positions[i] = i;
5410                }
5411                set_input_i32("positions", positions);
5412            } break;
5413        case PROJECTOR_TYPE_GEMMA4A:
5414            {
5415                GGML_ASSERT(imgs.entries.size() == 1);
5416                const auto & img0 = imgs.entries.front();
5417                // Compute n_pos matching SSCP output: two stride-2 convs
5418                int n_pos = img0.nx();
5419                for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; }
5420
5421                // Chunked local attention: blocked causal mask and RPE
5422                const int chunk_size   = 12;
5423                const int max_past     = 12;
5424                const int context_size = chunk_size + max_past;
5425                const int num_blocks   = (n_pos + chunk_size - 1) / chunk_size;
5426
5427                // Blocked causal attention mask: [context_size, chunk_size, num_blocks]
5428                {
5429                    std::vector<float> mask(context_size * chunk_size * num_blocks, -1e9f);
5430                    for (int b = 0; b < num_blocks; b++) {
5431                        for (int q = 0; q < chunk_size; q++) {
5432                            int gq = b * chunk_size + q;
5433                            for (int k = 0; k < context_size; k++) {
5434                                int gk = b * chunk_size - max_past + k;
5435                                if (gq < n_pos && gk >= 0 && gk < n_pos && gk <= gq && (gq - gk) < max_past) {
5436                                    mask[k + q * context_size + b * context_size * chunk_size] = 0.0f;
5437                                }
5438                            }
5439                        }
5440                    }
5441                    set_input_f32("kq_mask", mask);
5442                }
5443
5444                // Sinusoidal RPE: 13 positions [12, 11, ..., 0]
5445                {
5446                    const int n_embd = ctx->model.hparams.n_embd;
5447                    const int num_timescales = n_embd / 2;
5448                    const float log_timescale_increment = logf(10000.0f) / std::max(num_timescales - 1, 1);
5449                    const int rpe_len = max_past + 1;
5450                    std::vector<float> pos_emb(n_embd * rpe_len, 0.0f);
5451                    for (int p = 0; p < rpe_len; p++) {
5452                        float position = (float)(max_past - p);
5453                        for (int i = 0; i < num_timescales; i++) {
5454                            float inv_ts = expf(-(float)i * log_timescale_increment);
5455                            float scaled = position * inv_ts;
5456                            pos_emb[p * n_embd + i]                 = sinf(scaled);
5457                            pos_emb[p * n_embd + i + num_timescales] = cosf(scaled);
5458                        }
5459                    }
5460                    set_input_f32("pos_emb", pos_emb);
5461                }
5462            } break;
5463        case PROJECTOR_TYPE_MIMO_AUDIO:
5464            {
5465                GGML_ASSERT(imgs.entries.size() == 1);
5466                const int n_frames = imgs.entries.front().nx();
5467                const int n_pos    = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length
5468
5469                std::vector<int32_t> positions(n_pos);
5470                for (int i = 0; i < n_pos; i++) {
5471                    positions[i] = i;
5472                }
5473                set_input_i32("mimo_audio_positions", positions);
5474
5475                const int window = hparams.attn_window_size;
5476                GGML_ASSERT(window > 0);
5477
5478                const float neg_inf = std::numeric_limits<float>::lowest();
5479                std::vector<float> full_mask((size_t) n_pos * n_pos);
5480                std::vector<float> window_mask((size_t) n_pos * n_pos);
5481                for (int q = 0; q < n_pos; q++) {
5482                    for (int k = 0; k < n_pos; k++) {
5483                        const bool causal_ok = k <= q;
5484                        full_mask[(size_t) q * n_pos + k]   = causal_ok ? 0.0f : neg_inf;
5485                        window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
5486                    }
5487                }
5488                set_input_f32("mimo_audio_full_mask", full_mask);
5489                set_input_f32("mimo_audio_window_mask", window_mask);
5490
5491                // input_local_transformer: block-diagonal mask + in-group positions
5492                {
5493                    const int n_pos_ds   = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0)
5494                    const int group_size = hparams.audio_local_group_size;
5495                    GGML_ASSERT(group_size > 0);
5496                    const int n_groups = (n_pos_ds + group_size - 1) / group_size;
5497                    const int n_padded = n_groups * group_size;
5498
5499                    std::vector<int32_t> local_positions(n_padded);
5500                    for (int i = 0; i < n_padded; i++) {
5501                        local_positions[i] = i % group_size;
5502                    }
5503                    set_input_i32("mimo_audio_local_positions", local_positions);
5504
5505                    std::vector<float> local_mask((size_t) n_padded * n_padded);
5506                    for (int q = 0; q < n_padded; q++) {
5507                        for (int k = 0; k < n_padded; k++) {
5508                            const bool same_group = (q / group_size) == (k / group_size);
5509                            local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
5510                        }
5511                    }
5512                    set_input_f32("mimo_audio_local_mask", local_mask);
5513                }
5514            } break;
5515        case PROJECTOR_TYPE_LFM2A:
5516            {
5517                GGML_ASSERT(imgs.entries.size() == 1);
5518                const auto n_frames = clip_n_output_tokens(ctx, &imgs.entries.front());
5519
5520                auto d_model = 512;
5521                auto seq_len = n_frames * 2 - 1;
5522                std::vector<float> pos_emb(d_model*seq_len);
5523                std::vector<double> inv_freq(d_model / 2);
5524                for (size_t i = 0; i < inv_freq.size(); ++i) {
5525                    inv_freq[i] = std::exp(-(std::log(10000.0) / (float)d_model) * (2.0f * (float)(i)));
5526                }
5527                for (int64_t pos = 0; pos < seq_len; ++pos) {
5528                    for (size_t i = 0; i < inv_freq.size(); ++i) {
5529                        const float ang = (n_frames - pos - 1) * inv_freq[i];
5530                        pos_emb[pos*d_model + 2*i + 0] = sinf(ang);  // even
5531                        pos_emb[pos*d_model + 2*i + 1] = cosf(ang);  // odd
5532                    }
5533                }
5534                set_input_f32("pos_emb", pos_emb);
5535            } break;
5536        case PROJECTOR_TYPE_PARAKEET:
5537            {
5538                GGML_ASSERT(imgs.entries.size() == 1);
5539                struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask");
5540                const int n_q = attn_mask->ne[1];
5541                const int n_k = attn_mask->ne[0];
5542                const int n_frames = imgs.entries.front().nx();
5543                const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor;
5544                const float mask_value  = -1e30f;
5545
5546                std::vector<float> mask_data(n_q * n_k);
5547                if (n_k == n_q) {
5548                    // full attention: mask keys that are padding
5549                    for (int q = 0; q < n_q; ++q) {
5550                        for (int k = 0; k < n_k; ++k) {
5551                            mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f;
5552                        }
5553                    }
5554                } else {
5555                    // local attention: mask keys outside the valid window
5556                    const int att_left = n_k / 2;
5557                    for (int q = 0; q < n_q; ++q) {
5558                        for (int k = 0; k < n_k; ++k) {
5559                            const int key = q - att_left + k;
5560                            mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value;
5561                        }
5562                    }
5563                }
5564                set_input_f32(attn_mask->name, mask_data);
5565
5566                // local attention skew mask: zeroes out the probs that were
5567                // computed for keys outside the valid sliding window.
5568                if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) {
5569                    const int lm_k = local_mask->ne[0];
5570                    const int lm_q = local_mask->ne[1];
5571                    const int window_size = lm_k - lm_q + 1;
5572                    std::vector<float> lm_data(lm_q * lm_k);
5573                    for (int q = 0; q < lm_q; ++q) {
5574                        for (int k = 0; k < lm_k; ++k) {
5575                            const int rel = k - q;
5576                            lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f;
5577                        }
5578                    }
5579                    set_input_f32(local_mask->name, lm_data);
5580                }
5581
5582                // Generate rotation frequencies for relative positional encoding.
5583                {
5584                    const int n_state     = hparams.n_embd;
5585                    const int d_half      = n_state / 2;
5586                    const float log_10000 = logf(10000.0f);
5587                    std::vector<float> freqs(d_half);
5588                    for (int k = 0; k < d_half; ++k) {
5589                        freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state)));
5590                    }
5591                    set_input_f32("pos_freqs", freqs);
5592                }
5593
5594                // Generate relative positional distance values which scaled by
5595                // the frequency to produce the angles for sin/cos.
5596                {
5597                    // window_size is only known after graph construction since it depends on
5598                    // n_time from the conv output, so we read it back from the graph tensor.
5599                    struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions");
5600                    const int window_size = rel_pos->ne[1];
5601                    std::vector<float> pos(window_size);
5602                    // local attention: window is fixed at [att_left, att_right]
5603                    // full attention: window covers the full sequence, centered
5604                    if (ggml_graph_get_tensor(gf, "local_mask")) {
5605                        const int att_left = window_size / 2;
5606                        for (int t = 0; t < window_size; ++t) {
5607                            pos[t] = float(att_left - t);
5608                        }
5609                    } else {
5610                        const int n_time = (window_size + 1) / 2;
5611                        for (int t = 0; t < window_size; ++t) {
5612                            pos[t] = float(n_time - 1 - t);
5613                        }
5614                    }
5615                    set_input_f32(rel_pos->name, pos);
5616                }
5617            } break;
5618        case PROJECTOR_TYPE_GRANITE_SPEECH:
5619            {
5620                const int context_size = ctx->model.hparams.audio_chunk_size;
5621                const int max_pos_emb  = ctx->model.hparams.audio_max_pos_emb;
5622
5623                std::vector<int32_t> dists((size_t) context_size * (size_t) context_size);
5624                for (int i = 0; i < context_size; i++) {
5625                    for (int j = 0; j < context_size; j++) {
5626                        int d = i - j;
5627                        if (d < -context_size) d = -context_size;
5628                        if (d >  context_size) d =  context_size;
5629                        dists[(size_t) i * (size_t) context_size + (size_t) j] = d + max_pos_emb;
5630                    }
5631                }
5632                set_input_i32("attn_dists", dists);
5633
5634                const int n_frames   = image_size_width;
5635                const int remainder  = n_frames % context_size;
5636                if (remainder > 0) {
5637                    const int num_blocks = (n_frames + context_size - 1) / context_size;
5638                    std::vector<float> mask((size_t) context_size * (size_t) context_size * (size_t) num_blocks, 0.0f);
5639                    const float neg_inf = -INFINITY;
5640                    const size_t last_block_offset = (size_t) (num_blocks - 1) * (size_t) context_size * (size_t) context_size;
5641                    for (int q = 0; q < context_size; q++) {
5642                        for (int k = 0; k < context_size; k++) {
5643                            if (q >= remainder || k >= remainder) {
5644                                mask[last_block_offset + (size_t) q * (size_t) context_size + (size_t) k] = neg_inf;
5645                            }
5646                        }
5647                    }
5648                    set_input_f32("attn_mask", mask);
5649                }
5650            } break;
5651        case PROJECTOR_TYPE_GRANITE4_VISION:
5652            {
5653                // Granite Vision 4.1 uses precomputed permutation index
5654                // tensors to express the _win / _unwin / spatial sampling
5655                // reshapes as ggml_get_rows gathers. The names are set
5656                // by g4v_gather() in models/granite4-vision.cpp.
5657                const int patch_size  = model.hparams.patch_size;
5658                const int image_side  = imgs.entries.front().nx() / patch_size;
5659                const int window_side = hparams.downsample_window_side;
5660                const int query_side  = hparams.downsample_query_side;
5661                const int n           = image_side / window_side;
5662                const int new_side    = n * query_side;
5663
5664                // Builds the raster→window permutation indices for a
5665                // (side, side) grid split into (n × n) windows of (win × win)
5666                // tokens each.  dst[w * win*win + p] = source raster index.
5667                auto make_win_idx = [](int side, int win) {
5668                    const int nn = side / win;
5669                    std::vector<int32_t> idx(static_cast<size_t>(side) * side);
5670                    for (int wy = 0; wy < nn; ++wy) {
5671                        for (int wx = 0; wx < nn; ++wx) {
5672                            for (int iy = 0; iy < win; ++iy) {
5673                                for (int ix = 0; ix < win; ++ix) {
5674                                    const int w  = wy * nn + wx;
5675                                    const int p  = iy * win + ix;
5676                                    const int y  = wy * win + iy;
5677                                    const int x  = wx * win + ix;
5678                                    idx[static_cast<size_t>(w) * (win*win) + p] = y * side + x;
5679                                }
5680                            }
5681                        }
5682                    }
5683                    return idx;
5684                };
5685
5686                auto make_unwin_idx = [&](int side, int win) {
5687                    const std::vector<int32_t> fwd = make_win_idx(side, win);
5688                    std::vector<int32_t> inv(fwd.size());
5689                    for (size_t i = 0; i < fwd.size(); ++i) {
5690                        inv[fwd[i]] = static_cast<int32_t>(i);
5691                    }
5692                    return inv;
5693                };
5694
5695                auto make_spatial_idx = [](int side, int offset) {
5696                    const int off_y = (offset >> 1) & 1;
5697                    const int off_x = offset & 1;
5698                    const int new_s = side / 2;
5699                    std::vector<int32_t> idx(static_cast<size_t>(new_s) * new_s);
5700                    for (int y = 0; y < new_s; ++y) {
5701                        for (int x = 0; x < new_s; ++x) {
5702                            idx[y * new_s + x] = (y * 2 + off_y) * side + (x * 2 + off_x);
5703                        }
5704                    }
5705                    return idx;
5706                };
5707
5708                // the same permutation is applied to every tile of the stacked image
5709                auto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {
5710                    ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());
5711                    GGML_ASSERT(t);
5712                    GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);
5713                    const int n_rep = ggml_nelements(t) / idx.size();
5714                    std::vector<int32_t> buf;
5715                    buf.reserve(idx.size() * n_rep);
5716                    for (int i = 0; i < n_rep; ++i) {
5717                        buf.insert(buf.end(), idx.begin(), idx.end());
5718                    }
5719                    ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));
5720                };
5721
5722                // Stage 1b only uses block 0's permutations; future stages
5723                // will upload all blocks.
5724                for (size_t bid = 0; bid < hparams.feature_layers.size(); ++bid) {
5725                    const std::string prefix = "g4v_blk" + std::to_string(bid) + "_";
5726                    upload(prefix + "win_idx",     make_win_idx(image_side, window_side));
5727                    upload(prefix + "qwin_idx",    make_win_idx(new_side, query_side));
5728                    upload(prefix + "unwin_idx",   make_unwin_idx(new_side, query_side));
5729                    const auto spatial_offset = hparams.proj_spatial_offsets[bid];
5730                    if (spatial_offset >= 0) {
5731                        upload(prefix + "spatial_idx", make_spatial_idx(image_side,spatial_offset));
5732                    }
5733                }
5734            } break;
5735        default:
5736            GGML_ABORT("Unknown projector type");
5737    }
5738
5739    // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads);
5740    ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu);
5741    ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
5742    if (reg) {
5743        auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
5744        if (ggml_backend_set_n_threads_fn) {
5745            ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads);
5746        }
5747    }
5748
5749    auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf);
5750    if (status != GGML_STATUS_SUCCESS) {
5751        LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
5752        return false;
5753    }
5754
5755    // the last node is the embedding tensor, code2wav has no out_embd
5756    ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
5757
5758    if (embeddings != nullptr) {
5759        // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
5760        const int n_tokens_out = embeddings->ne[1];
5761        const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]);
5762        if (n_tokens_out != expected_n_tokens_out) {
5763            LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out);
5764            GGML_ABORT("Invalid number of output tokens");
5765        }
5766
5767        LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
5768            (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
5769
5770        // copy output to user buffer if provided
5771        // if output is empty, skip the copy
5772        auto & out_batch_embd = *params->out_embd;
5773        if (!out_batch_embd.empty()) {
5774            if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
5775                LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
5776                GGML_ABORT("Output buffer size mismatch");
5777            }
5778            ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
5779        } else {
5780            LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
5781        }
5782    }
5783
5784    //
5785    // for audio gen models
5786    //
5787
5788    // optional outputs: a pipeline yields codes or feats, and not all have an eos head
5789    if (params->out_codes != nullptr) {
5790        ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes");
5791        if (codes != nullptr) {
5792            auto & out_codes = *params->out_codes;
5793            out_codes.resize(ggml_nelements(codes));
5794            ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes));
5795        }
5796    }
5797    if (params->out_feats != nullptr) {
5798        ggml_tensor * feats = ggml_graph_get_tensor(gf, "out_feats");
5799        if (feats != nullptr) {
5800            auto & out_feats = *params->out_feats;
5801            out_feats.resize(ggml_nelements(feats));
5802            ggml_backend_tensor_get(feats, out_feats.data(), 0, ggml_nbytes(feats));
5803        }
5804    }
5805    if (params->out_is_eos != nullptr) {
5806        ggml_tensor * eos = ggml_graph_get_tensor(gf, "out_eos_score");
5807        if (eos != nullptr) {
5808            GGML_ASSERT(ggml_nelements(eos) == 1);
5809            float score = 0.0f;
5810            ggml_backend_tensor_get(eos, &score, 0, sizeof(float));
5811            *params->out_is_eos = score > hparams.gen_eos_threshold;
5812        }
5813    }
5814    if (params->out_audio != nullptr) {
5815        ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio");
5816        if (audio == nullptr) {
5817            GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor");
5818        }
5819        auto & out_audio = *params->out_audio;
5820        out_audio.resize(ggml_nelements(audio));
5821        ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
5822
5823        // drop the tail audio that comes from the code-0 rear padding
5824        const int64_t n_codes    = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0;
5825        const int64_t n_frames_w = hparams.wav_tfm_swa;
5826        const int64_t n_frames   = params->codes ? (int64_t) params->codes->size() / n_codes : n_frames_w;
5827        if (n_frames < n_frames_w) {
5828            const size_t hop = out_audio.size() / n_frames_w;
5829            out_audio.resize((size_t) n_frames * hop);
5830        }
5831    }
5832    if (params->state_out != nullptr) {
5833        auto & state_out = *params->state_out;
5834        size_t total = 0;
5835        for (const auto & slot : list_gen_state_slots(hparams, model)) {
5836            total += (size_t) (slot.ne0 * slot.ne1) * sizeof(float);
5837        }
5838        state_out.resize(total);
5839        size_t offset = 0;
5840        for (const auto & slot : list_gen_state_slots(hparams, model)) {
5841            ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
5842            if (t == nullptr) {
5843                GGML_ABORT("state_out requested but graph has no \"state_out_%s\" tensor", slot.name.c_str());
5844            }
5845            const size_t nb = ggml_nbytes(t);
5846            ggml_backend_tensor_get(t, state_out.data() + offset, 0, nb);
5847            offset += nb;
5848        }
5849    }
5850
5851    //
5852    // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
5853    //
5854
5855    if (ctx->debug_output_embeddings && embeddings != nullptr) {
5856        const int64_t n_embd = embeddings->ne[0];
5857        const int64_t n_tokens = embeddings->ne[1];
5858        std::vector<float> emb_data(ggml_nelements(embeddings));
5859        ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings));
5860
5861        LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
5862        LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
5863
5864        // TEMP debugging (parity validation), will be removed before merge
5865        // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
5866        const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
5867        if (dump_path && strcmp(dump_path, "1") != 0) {
5868            FILE * f = fopen(dump_path, "wb");
5869            if (f) {
5870                const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
5871                fwrite(hdr, sizeof(hdr), 1, f);
5872                fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
5873                fclose(f);
5874            }
5875        }
5876
5877        // Print first few values of first token
5878        LOG_INF("Token 0 (first 16 values): ");
5879        for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
5880            LOG_INF("%.6f ", emb_data[i]);
5881        }
5882        LOG_INF("\n");
5883
5884        // Print last few values of first token
5885        if (n_embd > 16) {
5886            LOG_INF("Token 0 (last 16 values):  ");
5887            for (int64_t i = n_embd - 16; i < n_embd; i++) {
5888                LOG_INF("%.6f ", emb_data[i]);
5889            }
5890            LOG_INF("\n");
5891        }
5892
5893        // Compute and print statistics
5894        float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0];
5895        for (size_t i = 0; i < emb_data.size(); i++) {
5896            sum += emb_data[i];
5897            sum_sq += emb_data[i] * emb_data[i];
5898            min_val = std::min(min_val, emb_data[i]);
5899            max_val = std::max(max_val, emb_data[i]);
5900        }
5901        float mean = sum / emb_data.size();
5902        float variance = (sum_sq / emb_data.size()) - (mean * mean);
5903        LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n",
5904                mean, sqrtf(variance), min_val, max_val, sum);
5905        LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n");
5906    }
5907
5908    return true;
5909}
5910
5911int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
5912    switch (ctx->model.proj_type) {
5913        case PROJECTOR_TYPE_LDP:
5914            return ctx->model.mm_model_block_1_block_2_1_b->ne[0];
5915        case PROJECTOR_TYPE_LDPV2:
5916            return ctx->model.mm_model_peg_0_b->ne[0];
5917        case PROJECTOR_TYPE_MLP:
5918        case PROJECTOR_TYPE_PHI4:
5919        case PROJECTOR_TYPE_PIXTRAL:
5920        case PROJECTOR_TYPE_LIGHTONOCR:
5921        case PROJECTOR_TYPE_DOTS_OCR:
5922        case PROJECTOR_TYPE_DOTS3NOTE_V:
5923        case PROJECTOR_TYPE_DOTS3NOTE_A:
5924            return ctx->model.mm_2_w->ne[1];
5925        case PROJECTOR_TYPE_MLP_NORM:
5926            return ctx->model.mm_3_b->ne[0];
5927        case PROJECTOR_TYPE_MINICPMV:
5928            return ctx->model.mm_model_proj->ne[0];
5929        case PROJECTOR_TYPE_MINICPMV4_6:
5930            return ctx->model.mm_ffn_down_w->ne[1];
5931        case PROJECTOR_TYPE_GLM_EDGE:
5932            return ctx->model.mm_model_mlp_3_w->ne[1];
5933        case PROJECTOR_TYPE_MINIMAX_M3:
5934            return ctx->model.mm_merger_fc2_b->ne[0];
5935        case PROJECTOR_TYPE_MUSE_GLIMMER:
5936            return ctx->model.mm_2_w->ne[1];
5937        case PROJECTOR_TYPE_QWEN2VL:
5938        case PROJECTOR_TYPE_QWEN25VL:
5939        case PROJECTOR_TYPE_EXAONE4_5:
5940        case PROJECTOR_TYPE_JANUS_PRO:
5941        case PROJECTOR_TYPE_YOUTUVL:
5942            return ctx->model.mm_1_b->ne[0];
5943        case PROJECTOR_TYPE_QWEN3VL:
5944            // main path + deepstack paths
5945            return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
5946        case PROJECTOR_TYPE_MIMOVL:
5947            return ctx->model.mm_1_w->ne[1];
5948        case PROJECTOR_TYPE_STEP3VL:
5949            return ctx->model.mm_model_proj->ne[1];
5950        case PROJECTOR_TYPE_GEMMA3:
5951        case PROJECTOR_TYPE_GEMMA3NV:
5952            return ctx->model.mm_input_proj_w->ne[0];
5953        case PROJECTOR_TYPE_GEMMA4V:
5954        case PROJECTOR_TYPE_GEMMA4UV:
5955        case PROJECTOR_TYPE_GEMMA4A:
5956        case PROJECTOR_TYPE_GEMMA4UA:
5957            return ctx->model.mm_input_proj_w->ne[1];
5958        case PROJECTOR_TYPE_IDEFICS3:
5959            return ctx->model.mm_fc_w->ne[1];
5960        case PROJECTOR_TYPE_ULTRAVOX:
5961        case PROJECTOR_TYPE_VOXTRAL:
5962        case PROJECTOR_TYPE_MUSIC_FLAMINGO:
5963            return ctx->model.mm_2_w->ne[1];
5964        case PROJECTOR_TYPE_MERALION:
5965            return ctx->model.mm_3_w->ne[1]; // out_proj output dim
5966        case PROJECTOR_TYPE_INTERNVL:
5967        case PROJECTOR_TYPE_NEMOTRON_V2_VL:
5968            return ctx->model.mm_3_w->ne[1];
5969        case PROJECTOR_TYPE_LLAMA4:
5970            return ctx->model.mm_model_proj->ne[1];
5971        case PROJECTOR_TYPE_QWEN2A:
5972            return ctx->model.mm_fc_w->ne[1];
5973        case PROJECTOR_TYPE_QWEN3A:
5974            return ctx->model.mm_2_w->ne[1];
5975        case PROJECTOR_TYPE_GLMA:
5976        case PROJECTOR_TYPE_LFM2:
5977        case PROJECTOR_TYPE_KIMIVL:
5978        case PROJECTOR_TYPE_PADDLEOCR:
5979        case PROJECTOR_TYPE_KIMIK25:
5980        case PROJECTOR_TYPE_YASA2:
5981        case PROJECTOR_TYPE_DEEPSEEK4V:
5982            return ctx->model.mm_2_w->ne[1];
5983        case PROJECTOR_TYPE_HUNYUANVL:
5984            return ctx->model.mm_model_proj->ne[1];
5985        case PROJECTOR_TYPE_COGVLM:
5986            return ctx->model.mm_4h_to_h_w->ne[1];
5987        case PROJECTOR_TYPE_DEEPSEEKOCR:
5988        case PROJECTOR_TYPE_DEEPSEEKOCR2:
5989            return ctx->model.mm_fc_w->ne[1];
5990        case PROJECTOR_TYPE_LFM2A:
5991            return ctx->model.position_embeddings->ne[0];
5992        case PROJECTOR_TYPE_GRANITE_SPEECH:
5993            return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1];
5994        case PROJECTOR_TYPE_GRANITE4_VISION:
5995            return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
5996        case PROJECTOR_TYPE_GLM4V:
5997            return ctx->model.mm_ffn_down_w->ne[1];
5998        case PROJECTOR_TYPE_MIMO_AUDIO:
5999            return ctx->model.mm_2_w->ne[1];
6000        case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
6001            return ctx->model.mm_fc_w->ne[2];
6002        case PROJECTOR_TYPE_QWEN3TTS_GEN:
6003            return ctx->model.gen_code_out_embd_w->ne[0];
6004        case PROJECTOR_TYPE_POCKETTTS_SPKENC:
6005            return ctx->model.spk_proj_w->ne[1];
6006        case PROJECTOR_TYPE_POCKETTTS_GEN:
6007            return ctx->model.gen_input_lin_w->ne[1];
6008        case PROJECTOR_TYPE_PARAKEET:
6009            return ctx->model.mm_1_w->ne[1];
6010        default:
6011            GGML_ABORT("Unknown projector type");
6012    }
6013}
6014
6015bool clip_is_llava(const struct clip_ctx * ctx) {
6016    return ctx->model.hparams.has_llava_projector;
6017}
6018
6019bool clip_has_vision_encoder(const struct clip_ctx * ctx) {
6020    return ctx->model.modality == CLIP_MODALITY_VISION;
6021}
6022
6023bool clip_has_audio_encoder(const struct clip_ctx * ctx) {
6024    return ctx->model.modality == CLIP_MODALITY_AUDIO;
6025}
6026
6027bool clip_support_batch(const struct clip_ctx * ctx) {
6028    return ctx->support_batch;
6029}
6030
6031// TODO @ngxson : this is no longer correct with mtmd_batch API
6032// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support)
6033// this logic should be refactored in near future to distinctly handle "merge frames" and "batching"
6034int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
6035    switch (ctx->proj_type()) {
6036        case PROJECTOR_TYPE_QWEN2VL:
6037        case PROJECTOR_TYPE_QWEN25VL:
6038        case PROJECTOR_TYPE_QWEN3VL:
6039            return 2;
6040        default:
6041            return 1;
6042    }
6043}
6044
6045//
6046// API used internally with mtmd
6047//
6048
6049projector_type clip_get_projector_type(const struct clip_ctx * ctx) {
6050    return ctx->proj_type();
6051}
6052
6053const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) {
6054    return &ctx->model.hparams;
6055}
6056
6057std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx) {
6058    std::map<ggml_backend_dev_t, size_t> result = ctx->mem_usage;
6059    for (auto & [dev, size] : ctx->mem_compute) {
6060        result[dev] += size;
6061    }
6062    return result;
6063}
6064
6065//
6066// API for debugging
6067//
6068
6069void clip_set_debug_output_embeddings(clip_ctx * ctx, bool enable) {
6070    ctx->debug_output_embeddings = enable;
6071}