char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: use ggml_rope_set_offset (#27521)b21e4de74

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.8 KiB133 linesraw
1#include "models.h"
2#include <cmath>
3
4ggml_cgraph * clip_graph_gemma4v::build() {
5    ggml_tensor * inp_raw = build_inp_raw();
6
7    // patches = 2 * (patches - 0.5)
8    // equivalent to: patches * 2 - 1
9    inp_raw = ggml_scale_bias(ctx0, inp_raw, 2.0f, -1.0f);
10    ggml_set_name(inp_raw, "inp_raw_scaled");
11
12    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
13    inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
14    inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
15    ggml_set_name(inp, "inp");
16    // note: no patch bias
17
18    ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
19    ggml_set_name(pos_x, "pos_x");
20    ggml_set_input(pos_x);
21
22    ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
23    ggml_set_name(pos_y, "pos_y");
24    ggml_set_input(pos_y);
25
26    {
27        const int64_t pos_size = model.position_embeddings->ne[1];
28        const size_t  nb1      = ggml_row_size(model.position_embeddings->type, n_embd);
29
30        // positional embeddings are stored as lookup tables (one for x, one for y)
31        ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings,
32                                             n_embd, pos_size, nb1, 0);
33        ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings,
34                                             n_embd, pos_size, nb1, pos_size * nb1);
35
36        // ggml_get_rows: [n_embd, n_patches]
37        ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x);
38        ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y);
39
40        inp = ggml_add(ctx0, inp, emb_x);
41        inp = ggml_add(ctx0, inp, emb_y);
42        cb(inp, "pos_embd", -1);
43    }
44
45    // similar to build_rope_2d, but use neox ordering
46    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
47        const int64_t n_dim = cur->ne[0];
48
49        // first half, dims [0, n_dim/2)
50        cur = ggml_rope_ext(
51            ctx0,
52            cur,
53            pos_x,      // positions
54            nullptr,    // freq factors
55            n_dim/2,    // n_dims
56            GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
57            1.0f, 0.0f, 1.0f, 0.0f, 0.0f
58        );
59
60        // second half, dims [n_dim/2, n_dim)
61        cur = ggml_rope_ext(
62            ctx0,
63            cur,
64            pos_y,      // positions
65            nullptr,    // freq factors
66            n_dim/2,    // n_dims
67            GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
68            1.0f, 0.0f, 1.0f, 0.0f, 0.0f
69        );
70        cur = ggml_rope_set_offset(cur, n_dim/2);
71
72        return cur;
73    };
74
75    kq_scale = 1.0f;
76    ggml_tensor * cur = build_vit(
77                        inp, n_patches,
78                        NORM_TYPE_RMS,
79                        hparams.ffn_op,
80                        nullptr, // pos embd is already handled above
81                        add_pos);
82
83    // Gemma4VisionPooler
84    {
85        const int kernel_size = hparams.n_merge;
86        GGML_ASSERT(kernel_size > 0);
87
88        // [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, n_batch]
89        cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, n_batch);
90        cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG,
91                           kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
92        const int out_x = n_patches_x / kernel_size;
93        const int out_y = n_patches_y / kernel_size;
94        // [out_x, out_y, n_embd, n_batch] -> [n_embd, out_x * out_y, n_batch]
95        cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, n_batch);
96        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
97        cur = ggml_scale(ctx0, cur, sqrtf((float)n_embd));
98        cb(cur, "pooled", -1);
99    }
100
101    // hidden_states = (hidden_states - self.std_bias) * self.std_scale
102    if (model.std_bias && model.std_scale) {
103        cur = ggml_sub(ctx0, cur, model.std_bias);
104        cur = ggml_mul(ctx0, cur, model.std_scale);
105        cb(cur, "std_scaled", -1);
106    }
107
108    // Gemma4MultimodalEmbedder
109    {
110        // embedding_pre_projection_norm
111        cur = ggml_rms_norm(ctx0, cur, hparams.eps);
112        cur = build_mm(model.mm_input_proj_w, cur);
113        cb(cur, "projected", -1);
114    }
115
116    ggml_build_forward_expand(gf, cur);
117    return gf;
118}
119
120ggml_tensor * clip_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const {
121    // Gemma4ClippableLinear
122
123    auto it = model.clamp_info_map.find(w->name);
124    if (it == model.clamp_info_map.end()) {
125        return ggml_mul_mat(ctx0, w, x);
126    } else {
127        const auto & clamp_info = it->second;
128        ggml_tensor * clamped = ggml_clamp(ctx0, x, clamp_info.inp_min, clamp_info.inp_max);
129        ggml_tensor * out = ggml_mul_mat(ctx0, w, clamped);
130        out = ggml_clamp(ctx0, out, clamp_info.out_min, clamp_info.out_max);
131        return out;
132    }
133}