char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: add batching support for internvl (#24775)db52540f7

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.6 KiB73 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_internvl::build() {
4    GGML_ASSERT(model.class_embedding != nullptr);
5    GGML_ASSERT(model.position_embeddings != nullptr);
6
7    const int n_pos = n_patches + 1;
8    ggml_tensor * inp = build_inp();
9
10    // add CLS token
11    ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,
12            model.class_embedding->ne[0], 1, n_batch, 1);
13    inp = ggml_concat(ctx0, inp, cls_repeated, 1);
14
15    // The larger models use a different ViT, which uses RMS norm instead of layer norm
16    // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188
17    norm_type norm_t = (hparams.n_embd == 3200 && hparams.n_layer == 45)
18        ? NORM_TYPE_RMS // 6B ViT (Used by InternVL 2.5/3 - 26B, 38B, 78B)
19        : NORM_TYPE_NORMAL; // 300M ViT (Used by all smaller InternVL models)
20
21    ggml_tensor * cur = build_vit(
22                            inp, n_pos,
23                            norm_t,
24                            hparams.ffn_op,
25                            model.position_embeddings,
26                            nullptr);
27
28    // remove CLS token
29    cur = ggml_view_3d(ctx0, cur,
30        n_embd, n_patches, n_batch,
31        cur->nb[1], cur->nb[2], 0);
32    cur = ggml_cont(ctx0, cur);
33
34    // pixel shuffle
35    {
36        const int scale_factor = model.hparams.n_merge;
37        const int bsz    = n_batch;
38        const int height = n_patches_y;
39        const int width  = n_patches_x;
40        GGML_ASSERT(scale_factor > 0);
41        cur = ggml_reshape_4d(ctx0, cur, n_embd * scale_factor, height / scale_factor, width, bsz);
42        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
43        cur = ggml_cont_4d(ctx0, cur,
44            n_embd * scale_factor * scale_factor,
45            height / scale_factor,
46            width / scale_factor,
47            bsz);
48        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
49        // flatten to 2D
50        cur = ggml_cont_3d(ctx0, cur,
51            n_embd * scale_factor * scale_factor,
52            cur->ne[1] * cur->ne[2],
53            cur->ne[3]);
54    }
55
56    // projector (always using GELU activation)
57    {
58        // projector LayerNorm uses pytorch's default eps = 1e-5
59        // ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L79
60        cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
61        cur = build_ffn(cur,
62            model.mm_1_w, model.mm_1_b,
63            nullptr, nullptr,
64            model.mm_3_w, model.mm_3_b,
65            FFN_GELU,
66            -1);
67    }
68
69    // build the graph
70    ggml_build_forward_expand(gf, cur);
71
72    return gf;
73}