char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: use ggml_rope_set_offset (#27521)b21e4de74

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.0 KiB76 linesraw
1#include "models.h"
2
3ggml_tensor * clip_graph_minimax_m3::apply_rope(
4        ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
5    const int dh  = (int) x->ne[0];
6    const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
7
8    GGML_ASSERT(3 * axd <= dh);
9
10    const float th  = hparams.rope_theta;
11
12    // layout of x is [t, h, w, pad]
13    // t is unrotated, h and w are rotated, pad is unrotated
14    x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
15    x = ggml_rope_set_offset(x, axd);
16
17    x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
18    x = ggml_rope_set_offset(x, 2 * axd);
19
20    return x;
21}
22
23ggml_cgraph * clip_graph_minimax_m3::build() {
24    GGML_ASSERT(model.patch_bias     == nullptr);
25    GGML_ASSERT(model.class_embedding == nullptr);
26    GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);
27    GGML_ASSERT(model.mm_1_w && model.mm_2_w);
28    GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w);
29
30    const int batch_size = 1;
31    const int n_pos      = n_patches;
32    const int merge      = hparams.n_merge;
33
34    // patch embedding
35    ggml_tensor * inp_raw = build_inp_raw();
36    ggml_tensor * inp = ggml_add(ctx0,
37        ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
38        ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
39
40    // spatial merge
41    {
42        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
43        inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size);
44        inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge));
45        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
46        inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);
47    }
48
49    // t (time axis) is always 0 for now, so we leave it unrotated
50    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
51    ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h);
52    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
53    ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w);
54
55    ggml_tensor * inpL = build_vit(
56        inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
57        [&](ggml_tensor * c, const clip_layer &) {
58            return apply_rope(c, pos_h, pos_w);
59        });
60
61    // projector
62    ggml_tensor * emb = inpL;
63    emb = build_ffn(emb, model.mm_1_w, model.mm_1_b,
64                    nullptr, nullptr,
65                    model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);
66
67    const int64_t proj = emb->ne[0];
68    emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge));
69
70    emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b,
71                    nullptr, nullptr,
72                    model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1);
73
74    ggml_build_forward_expand(gf, emb);
75    return gf;
76}