char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

tc-mbmtmd: add minicpmv46 downsample (#25993)de699957b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
14.4 KiB360 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_minicpmv::build() {
4    GGML_ASSERT(model.class_embedding == nullptr);
5    const int n_pos       = n_patches;
6    const int n_embd_proj = n_mmproj_embd;
7
8    // position embeddings for the projector (not for ViT)
9    // see: https://huggingface.co/openbmb/MiniCPM-o-2_6/blob/main/resampler.py#L70
10    // base frequency omega
11    ggml_tensor * omega = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_embd_proj / 4);
12    ggml_set_name(omega, "omega");
13    ggml_set_input(omega);
14
15    // 2D input positions (using float for sinusoidal embeddings)
16    ggml_tensor * pos_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_pos);
17    ggml_set_name(pos_h, "pos_h");
18    ggml_set_input(pos_h);
19    ggml_tensor * pos_w = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_pos);
20    ggml_set_name(pos_w, "pos_w");
21    ggml_set_input(pos_w);
22
23    // for selecting learned pos embd, used by ViT
24    struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
25    ggml_set_name(positions, "positions");
26    ggml_set_input(positions);
27
28    ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
29
30    ggml_tensor * inp = build_inp();
31    ggml_tensor * embeddings = build_vit(
32                            inp, n_pos,
33                            NORM_TYPE_NORMAL,
34                            hparams.ffn_op,
35                            learned_pos_embd,
36                            nullptr);
37
38    // resampler projector (it is just another transformer)
39
40    ggml_tensor * q = model.mm_model_query;
41    ggml_tensor * v = build_mm(model.mm_model_kv_proj, embeddings);
42
43    // norm
44    q = build_norm(q, model.mm_model_ln_q_w,  model.mm_model_ln_q_b,  NORM_TYPE_NORMAL, eps, -1);
45    v = build_norm(v, model.mm_model_ln_kv_w, model.mm_model_ln_kv_b, NORM_TYPE_NORMAL, eps, -1);
46
47    // calculate sinusoidal pos embd
48    ggml_tensor * pos_embed = nullptr;
49    {
50        // outer product
51        ggml_tensor * omega_b = ggml_repeat_4d(ctx0, omega, omega->ne[0], n_pos, 1, 1); // n_pos rows
52        ggml_tensor * theta_x = ggml_mul(ctx0, omega_b, pos_w);
53        ggml_tensor * theta_y = ggml_mul(ctx0, omega_b, pos_h);
54        // sin and cos
55        ggml_tensor * pos_embd_x = ggml_concat(
56            ctx0,
57            ggml_sin(ctx0, theta_x),
58            ggml_cos(ctx0, theta_x),
59            0 // concat on first dim
60        );
61        ggml_tensor * pos_embd_y = ggml_concat(
62            ctx0,
63            ggml_sin(ctx0, theta_y),
64            ggml_cos(ctx0, theta_y),
65            0 // concat on first dim
66        );
67        pos_embed = ggml_concat(ctx0, pos_embd_x, pos_embd_y, 0);
68    }
69
70    // k = v + pos_embed
71    ggml_tensor * k = ggml_add(ctx0, v, pos_embed);
72
73    // attention
74    {
75        const int d_head = 128;
76        int n_head = n_embd_proj/d_head;
77        // Use actual config value if available, otherwise fall back to hardcoded values
78        int num_query = hparams.minicpmv_query_num;
79        ggml_tensor * Q = ggml_add(ctx0,
80            build_mm(model.mm_model_attn_q_w, q),
81            model.mm_model_attn_q_b);
82        ggml_tensor * K = ggml_add(ctx0,
83            build_mm(model.mm_model_attn_k_w, k),
84            model.mm_model_attn_k_b);
85        ggml_tensor * V = ggml_add(ctx0,
86            build_mm(model.mm_model_attn_v_w, v),
87            model.mm_model_attn_v_b);
88
89        Q = ggml_reshape_3d(ctx0, Q, d_head, n_head, num_query);
90        K = ggml_reshape_3d(ctx0, K, d_head, n_head, n_pos);
91        V = ggml_reshape_3d(ctx0, V, d_head, n_head, n_pos);
92
93        cb(Q, "resampler_Q", -1);
94        cb(K, "resampler_K", -1);
95        cb(V, "resampler_V", -1);
96
97        float resampler_kq_scale = 1.0f/ sqrtf(float(d_head));
98        embeddings = build_attn(
99            model.mm_model_attn_o_w,
100            model.mm_model_attn_o_b,
101            Q, K, V, nullptr, resampler_kq_scale, -1);
102        cb(embeddings, "resampler_attn_out", -1);
103    }
104    // layernorm
105    embeddings = build_norm(embeddings, model.mm_model_ln_post_w, model.mm_model_ln_post_b, NORM_TYPE_NORMAL, eps, -1);
106
107    // projection
108    embeddings = build_mm(model.mm_model_proj, embeddings);
109
110    // build the graph
111    ggml_build_forward_expand(gf, embeddings);
112
113    return gf;
114}
115
116ggml_cgraph * clip_graph_minicpmv4_6::build() {
117    const bool is_4x = hparams.n_merge == 2;
118    const int n_pos  = n_patches;
119    const int half_h = n_patches_y / 2;
120    const int half_w = n_patches_x / 2;
121    const int n_ds   = half_h * half_w;
122    const int n_out  = is_4x ? n_ds : (half_h / 2) * (half_w / 2);
123
124    auto add_i32_input = [&](const char * name, int n) {
125        ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
126        ggml_set_name(t, name);
127        ggml_set_input(t);
128        return t;
129    };
130
131    // position indices for ViT learned positional embeddings
132    ggml_tensor * positions = add_i32_input("positions", n_pos);
133    ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
134
135    ggml_tensor * vit_merger_window_idx     = nullptr;
136    ggml_tensor * vit_merger_inv_window_idx = nullptr;
137    ggml_tensor * vit_merger_window_mask    = nullptr;
138    ggml_tensor * vit_merger_ds_idx_0       = nullptr;
139    ggml_tensor * vit_merger_ds_idx_1       = nullptr;
140    ggml_tensor * vit_merger_ds_idx_2       = nullptr;
141    ggml_tensor * vit_merger_ds_idx_3       = nullptr;
142
143    if (!is_4x) {
144        // ViT merger window reorder indices + block-diagonal mask
145        // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
146        // so each window-major group of 4 tokens only attends to itself)
147        vit_merger_window_idx     = add_i32_input("vit_merger_window_idx", n_pos);
148        vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
149        vit_merger_window_mask    = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
150        ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
151        ggml_set_input(vit_merger_window_mask);
152        if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
153            vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
154        }
155
156        // ViT merger 2x2 downsample gather indices
157        vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
158        vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
159        vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
160        vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
161    }
162
163    // final merger 2x2 downsample gather indices
164    ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out);
165    ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out);
166    ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out);
167    ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out);
168
169    // patch embedding + positional embedding
170    ggml_tensor * inp = build_inp();
171    inp = ggml_add(ctx0, inp, learned_pos_embd);
172    cb(inp, "pos_embed", -1);
173
174    ggml_tensor * inpL = inp;
175    if (model.pre_ln_w) {
176        inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, NORM_TYPE_NORMAL, eps, -1);
177        cb(inpL, "pre_ln", -1);
178    }
179
180    auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) {
181        for (int il = il_begin; il < il_end; il++) {
182            auto & layer = model.layers[il];
183            ggml_tensor * cur = input;
184
185            cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
186            cb(cur, "layer_inp_normed", il);
187
188            {
189                ggml_tensor * Qcur = build_mm(layer.q_w, cur);
190                if (layer.q_b) {
191                    Qcur = ggml_add(ctx0, Qcur, layer.q_b);
192                }
193                ggml_tensor * Kcur = build_mm(layer.k_w, cur);
194                if (layer.k_b) {
195                    Kcur = ggml_add(ctx0, Kcur, layer.k_b);
196                }
197                ggml_tensor * Vcur = build_mm(layer.v_w, cur);
198                if (layer.v_b) {
199                    Vcur = ggml_add(ctx0, Vcur, layer.v_b);
200                }
201
202                Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer);
203                Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer);
204                Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer);
205                cb(Qcur, "Qcur", il);
206                cb(Kcur, "Kcur", il);
207                cb(Vcur, "Vcur", il);
208
209                cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il);
210                cb(cur, "attn_out", il);
211            }
212
213            if (layer.ls_1_w) {
214                cur = ggml_mul(ctx0, cur, layer.ls_1_w);
215                cb(cur, "attn_out_scaled", il);
216            }
217            cur = ggml_add(ctx0, cur, input);
218            input = cur;
219            cb(cur, "ffn_inp", il);
220
221            cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
222            cb(cur, "ffn_inp_normed", il);
223
224            cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b,
225                            layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);
226            cb(cur, "ffn_out", il);
227
228            if (layer.ls_2_w) {
229                cur = ggml_mul(ctx0, cur, layer.ls_2_w);
230                cb(cur, "ffn_out_scaled", il);
231            }
232            input = ggml_add(ctx0, input, cur);
233            cb(input, "layer_out", il);
234        }
235        return input;
236    };
237
238    if (!is_4x) {
239        const int insert_lid = hparams.insert_layer_id;
240
241        inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos);
242
243        // ViT merger: window self-attention
244        // Tokens are reordered to window-major (4 tokens per window are contiguous),
245        // and a block-diagonal mask restricts attention to within each window. This
246        // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
247        // flash-attention path when available.
248        {
249            ggml_tensor * residual = inpL;
250            ggml_tensor * cur = build_norm(inpL,
251                model.vit_merger_ln1_w, model.vit_merger_ln1_b,
252                NORM_TYPE_NORMAL, eps, -1);
253            cb(cur, "vit_merger_attn_inp_normed", -1);
254
255            cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
256            cb(cur, "vit_merger_window_reorder", -1);
257
258            ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
259            if (model.vit_merger_attn_q_b) {
260                Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
261            }
262            ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
263            if (model.vit_merger_attn_k_b) {
264                Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
265            }
266            ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
267            if (model.vit_merger_attn_v_b) {
268                Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
269            }
270
271            Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
272            Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
273            Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
274            cb(Qcur, "vit_merger_Qcur", -1);
275            cb(Kcur, "vit_merger_Kcur", -1);
276            cb(Vcur, "vit_merger_Vcur", -1);
277
278            cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
279                             Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
280            cb(cur, "vit_merger_attn_out", -1);
281
282            cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
283            inpL = ggml_add(ctx0, cur, residual);
284            cb(inpL, "vit_merger_attn_residual", -1);
285        }
286
287        // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
288        {
289            ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
290            ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
291            ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
292            ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
293
294            ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
295            mean_res = ggml_add(ctx0, mean_res, p2);
296            mean_res = ggml_add(ctx0, mean_res, p3);
297            mean_res = ggml_scale(ctx0, mean_res, 0.25f);
298            cb(mean_res, "vit_merger_ds_mean_res", -1);
299
300            ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
301            cat = ggml_concat(ctx0, cat, p2, 0);
302            cat = ggml_concat(ctx0, cat, p3, 0);
303
304            ggml_tensor * cur = build_norm(cat,
305                model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
306                NORM_TYPE_NORMAL, eps, -1);
307            cb(cur, "vit_merger_ds_normed", -1);
308
309            // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
310            cur = build_ffn(cur,
311                model.vit_merger_ds_up_w,   model.vit_merger_ds_up_b,
312                nullptr, nullptr,
313                model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
314                FFN_GELU, -1);
315            cb(cur, "vit_merger_ds_mlp_out", -1);
316
317            inpL = ggml_add(ctx0, cur, mean_res);
318            cb(inpL, "vit_merger_ds_out", -1);
319        }
320
321        inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds);
322    } else {
323        inpL = build_vit_layers(inpL, 0, n_layer, n_pos);
324    }
325
326    if (model.post_ln_w) {
327        inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1);
328        cb(inpL, "post_ln", -1);
329    }
330
331    // Final Merger (DownsampleMLP): another 2x2 spatial merge -> projector embedding
332    {
333        ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, merger_ds_idx_0);
334        ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, merger_ds_idx_1);
335        ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, merger_ds_idx_2);
336        ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, merger_ds_idx_3);
337
338        ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
339        cat = ggml_concat(ctx0, cat, p2, 0);
340        cat = ggml_concat(ctx0, cat, p3, 0);
341
342        ggml_tensor * cur = build_norm(cat,
343            model.mm_input_norm_w, model.mm_input_norm_b,
344            NORM_TYPE_NORMAL, eps, -1);
345        cb(cur, "merger_normed", -1);
346
347        // MiniCPMV4_6DownsampleMLP uses nn.GELU() (erf-based, FFN_GELU_ERF)
348        cur = build_ffn(cur,
349            model.mm_ffn_up_w,   model.mm_ffn_up_b,
350            nullptr, nullptr,
351            model.mm_ffn_down_w, model.mm_ffn_down_b,
352            FFN_GELU_ERF, -1);
353        cb(cur, "merger_out", -1);
354
355        inpL = cur;
356    }
357
358    ggml_build_forward_expand(gf, inpL);
359    return gf;
360}