char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: support multi-row batching for deepseek-ocr (#26154)717dad5c8

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.1 KiB85 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_deepseekocr2::build() {
4    GGML_ASSERT(hparams.n_head_kv > 0);
5    GGML_ASSERT(n_head % hparams.n_head_kv == 0);
6
7    // patch embedding
8    ggml_tensor * inp_raw = build_inp_raw();
9
10    ggml_tensor * sam_out = build_sam(inp_raw);
11
12    ggml_tensor * qwen2_out;
13    // Building Qwen2 encoder
14    {
15        ggml_tensor * inp;
16
17        // H*W, C, B
18        inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);
19        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B
20
21        auto num_image_tokens = inp->ne[1]; // H*W
22        GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);
23
24        // query based on numbers of image tokens (in SAM output)
25        // 16x16 -> query_1024 (1024x1024 images)
26        // 12x12 -> query_768 (768x768 images)
27
28        ggml_tensor * query_embed = model.resample_query_1024;
29        int           num_queries = 256;
30
31        if (num_image_tokens == 144) {
32            query_embed = model.resample_query_768;
33            num_queries = 144;
34        }
35
36        // repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)
37        query_embed = ggml_cast(ctx0, query_embed, inp->type);
38        query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);
39        inp = ggml_concat(ctx0, inp, query_embed, 1);
40
41        auto seq_len = inp->ne[1];
42
43        // qwen2 encoder attention mask
44        ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, seq_len, seq_len);
45        ggml_set_name(attn_mask, "qwen2_attn_mask");
46        ggml_set_input(attn_mask);
47
48        ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);
49
50        auto add_rope = [&](ggml_tensor * x, const clip_layer &) {
51            return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
52                                 GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0);
53        };
54
55        build_vit_opts vit_opts;
56        vit_opts.attn_mask = attn_mask;
57
58        // build_vit applies model.post_ln_w internally; do not re-apply
59        ggml_tensor * cur = build_vit(inp, seq_len, NORM_TYPE_RMS, FFN_SILU,
60                                      /* learned_pos_embd */ nullptr, add_rope, vit_opts);
61
62        cur = ggml_cont(ctx0,
63                        ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],
64                                     cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output
65
66        ggml_build_forward_expand(gf, cur);
67        qwen2_out = cur;
68    }
69
70    ggml_tensor * cur;
71
72    cur = ggml_mul_mat(ctx0, model.mm_fc_w, qwen2_out);
73    cur = ggml_add(ctx0, cur, model.mm_fc_b);
74
75    // view_seperator only after the global view
76    if (img.add_viewsep) {
77        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);
78        cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)
79    }
80
81    cb(cur, "dsocr2_output", -1);
82
83    ggml_build_forward_expand(gf, cur);
84    return gf;
85}