char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd, server: add "placeholder bitmap" for counting tokens , add */input_tokens API (#23913)f5c6ae182

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.5 KiB122 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_glm4v::build() {
4    GGML_ASSERT(model.patch_bias != nullptr);
5    GGML_ASSERT(model.class_embedding == nullptr);
6
7    const int batch_size = 1;
8
9    norm_type norm_t = NORM_TYPE_RMS;
10
11    ggml_tensor * inp_raw = build_inp_raw();
12    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
13
14    int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
15    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
16    ggml_set_name(positions, "positions");
17    ggml_set_input(positions);
18
19    GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
20    GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
21
22    // second conv dimension
23    {
24        auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
25        inp = ggml_add(ctx0, inp, inp_1);
26
27        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w, h, c, b] -> [c, w, h, b]
28        inp = ggml_cont_4d(
29            ctx0, inp,
30            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
31        inp = ggml_reshape_4d(
32            ctx0, inp,
33            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
34        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
35        inp = ggml_cont_3d(
36            ctx0, inp,
37            n_embd, n_patches_x * n_patches_y, batch_size);
38    }
39
40    // add patch bias
41    inp = ggml_add(ctx0, inp, model.patch_bias);
42    cb(inp, "patch_bias", -1);
43
44    // pos-conv norm
45    inp = build_norm(inp, model.norm_embd_w, model.norm_embd_b, norm_t, eps, -1);
46
47    ggml_tensor * learned_pos_embd = nullptr;
48    // Note: GLM-OCR does not have learned position embeddings
49    if (model.position_embeddings != nullptr) {
50        learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
51        learned_pos_embd = ggml_cont_4d(
52            ctx0, learned_pos_embd,
53            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
54        learned_pos_embd = ggml_reshape_4d(
55            ctx0, learned_pos_embd,
56            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
57        learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
58        learned_pos_embd = ggml_cont_3d(
59            ctx0, learned_pos_embd,
60            n_embd, n_patches_x * n_patches_y, batch_size);
61        cb(learned_pos_embd, "learned_pos_embd", -1);
62    }
63
64    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
65        return ggml_rope_multi(
66                    ctx0, cur, positions, nullptr,
67                    d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
68                    32768, hparams.rope_theta, 1, 0, 1, 32, 1);
69    };
70
71    ggml_tensor * cur = build_vit(
72                            inp, n_patches,
73                            norm_t,
74                            hparams.ffn_op,
75                            learned_pos_embd,
76                            add_pos);
77
78    cb(cur, "vit_out", -1);
79    // cb(ggml_sum(ctx0, cur), "vit_out_sum", -1);
80
81    // GLM4V projector
82    // ref: https://github.com/huggingface/transformers/blob/40dc11cd3eb4126652aa41ef8272525affd4a636/src/transformers/models/glm4v/modeling_glm4v.py#L116-L130
83
84    // patch merger (downsample)
85    {
86        int n_merge = hparams.n_merge;
87        GGML_ASSERT(n_merge > 0);
88
89        int n_token_out = n_patches / n_merge / n_merge;
90        cur = ggml_reshape_4d(ctx0, cur, n_embd, n_merge, n_merge, n_token_out);
91        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3)); // [n_merge, n_merge, n_embd, n_token_out]
92        cur = ggml_conv_2d(ctx0, model.mm_patch_merger_w, cur, n_merge, n_merge, 0, 0, 1, 1);
93        cur = ggml_reshape_2d(ctx0, cur, cur->ne[2], n_token_out); // [n_embd_out, n_token_out]
94
95        cur = ggml_add(ctx0, cur, model.mm_patch_merger_b);
96    }
97
98    // FC projector
99    {
100        cur = build_mm(model.mm_fc_w, cur);
101        // default LayerNorm (post_projection_norm)
102        cur = build_norm(cur, model.mm_post_norm_w, model.mm_post_norm_b, NORM_TYPE_NORMAL, 1e-5, -1);
103        cur = ggml_gelu_erf(ctx0, cur);
104        cb(cur, "after_fc_proj", -1);
105    }
106
107    // FFN projector
108    {
109        cur = build_ffn(cur,
110            model.mm_ffn_up_w, model.mm_ffn_up_b,
111            model.mm_ffn_gate_w, model.mm_ffn_gate_b,
112            model.mm_ffn_down_w, model.mm_ffn_down_b,
113            hparams.ffn_op, -1);
114        cb(cur, "after_ffn_proj", -1);
115        // cb(ggml_sum(ctx0, cur), "merged_sum", -1);
116    }
117
118    // build the graph
119    ggml_build_forward_expand(gf, cur);
120
121    return gf;
122}