char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd, server: add "placeholder bitmap" for counting tokens , add */input_tokens API (#23913)f5c6ae182

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.3 KiB170 linesraw
1// similar to qwen2vl, except for GQA attention
2#include "models.h"
3
4ggml_cgraph * clip_graph_exaone4_5::build() {
5    GGML_ASSERT(model.patch_bias == nullptr);
6    GGML_ASSERT(model.class_embedding == nullptr);
7
8    const int batch_size = 1;
9    const bool use_window_attn = hparams.n_wa_pattern > 0;
10    const int n_wa_pattern     = hparams.n_wa_pattern;
11    const int n_pos            = n_patches;
12    const int num_position_ids = n_pos * 4;
13
14    const norm_type norm_t = NORM_TYPE_RMS;
15
16    const int64_t n_kv_head = hparams.n_head_kv > 0 ? hparams.n_head_kv : n_head;
17    GGML_ASSERT(n_head % n_kv_head == 0);
18
19    int rope_sections[4] = { d_head / 4, d_head / 4, d_head / 4, d_head / 4 };
20    const float rope_freq_base = hparams.rope_theta > 0.0f ? hparams.rope_theta : 10000.0f;
21
22    ggml_tensor * inp_raw = build_inp_raw();
23    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
24
25    GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
26    GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
27
28    {
29        ggml_tensor * inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
30        inp = ggml_add(ctx0, inp, inp_1);
31        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
32        inp = ggml_cont_4d(
33            ctx0, inp,
34            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
35        inp = ggml_reshape_4d(
36            ctx0, inp,
37            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
38        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
39        inp = ggml_cont_3d(
40            ctx0, inp,
41            n_embd, n_patches_x * n_patches_y, batch_size);
42    }
43
44    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
45    ggml_set_name(positions, "positions");
46    ggml_set_input(positions);
47
48    ggml_tensor * window_mask    = nullptr;
49    ggml_tensor * window_idx     = nullptr;
50    ggml_tensor * inv_window_idx = nullptr;
51
52    if (use_window_attn) {
53        window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
54        ggml_set_name(window_idx, "window_idx");
55        ggml_set_input(window_idx);
56
57        inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
58        ggml_set_name(inv_window_idx, "inv_window_idx");
59        ggml_set_input(inv_window_idx);
60
61        window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
62        ggml_set_name(window_mask, "window_mask");
63        ggml_set_input(window_mask);
64
65        if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
66            window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);
67        }
68    }
69
70    ggml_tensor * inpL = inp;
71
72    if (use_window_attn) {
73        GGML_ASSERT(batch_size == 1);
74        inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);
75        inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);
76        inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);
77    }
78
79    for (int il = 0; il < n_layer; il++) {
80        const auto & layer = model.layers[il];
81        const bool full_attn = use_window_attn ? (il + 1) % n_wa_pattern == 0 : true;
82        ggml_tensor * cur = inpL;
83
84        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
85        cb(cur, "ln1", il);
86
87        {
88            GGML_ASSERT(layer.qkv_w != nullptr);
89            cur = build_mm(layer.qkv_w, cur);
90            if (layer.qkv_b) {
91                cur = ggml_add(ctx0, cur, layer.qkv_b);
92            }
93
94            const int64_t n_embd_kv = d_head * n_kv_head;
95            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_patches,
96                ggml_row_size(cur->type, d_head),
97                cur->nb[1],
98                0);
99            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,
100                ggml_row_size(cur->type, d_head),
101                cur->nb[1],
102                ggml_row_size(cur->type, n_embd));
103            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,
104                ggml_row_size(cur->type, d_head),
105                cur->nb[1],
106                ggml_row_size(cur->type, n_embd + n_embd_kv));
107
108            cb(Qcur, "Qcur", il);
109            cb(Kcur, "Kcur", il);
110            cb(Vcur, "Vcur", il);
111
112            Qcur = ggml_rope_multi(
113                ctx0, Qcur, positions, nullptr,
114                d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);
115            Kcur = ggml_rope_multi(
116                ctx0, Kcur, positions, nullptr,
117                d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);
118
119            cb(Qcur, "Qcur_rope", il);
120            cb(Kcur, "Kcur_rope", il);
121            cb(Vcur, "Vcur", il);
122
123            ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;
124            cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
125            cb(cur, "attn_out", il);
126        }
127
128        cur = ggml_add(ctx0, cur, inpL);
129        inpL = cur;
130
131        cb(cur, "ffn_inp", il);
132
133        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
134        cb(cur, "ffn_inp_normed", il);
135
136        cur = build_ffn(cur,
137            layer.ff_up_w, layer.ff_up_b,
138            layer.ff_gate_w, layer.ff_gate_b,
139            layer.ff_down_w, layer.ff_down_b,
140            hparams.ffn_op, il);
141
142        cb(cur, "ffn_out", il);
143
144        cur = ggml_add(ctx0, inpL, cur);
145        cb(cur, "layer_out", il);
146
147        inpL = cur;
148    }
149
150    ggml_tensor * embeddings = inpL;
151    embeddings = build_norm(embeddings, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);
152    embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
153    embeddings = build_ffn(embeddings,
154        model.mm_0_w, model.mm_0_b,
155        nullptr, nullptr,
156        model.mm_1_w, model.mm_1_b,
157        FFN_GELU,
158        -1);
159
160    if (use_window_attn) {
161        GGML_ASSERT(batch_size == 1);
162        embeddings = ggml_reshape_2d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4);
163        embeddings = ggml_get_rows(ctx0, embeddings, window_idx);
164        embeddings = ggml_reshape_3d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4, batch_size);
165    }
166
167    ggml_build_forward_expand(gf, embeddings);
168
169    return gf;
170}