char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

forforever73model : support step3-vl-10b (#21287)09343c019

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.6 KiB81 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_step3vl::build() {
4    GGML_ASSERT(model.class_embedding == nullptr);
5    GGML_ASSERT(model.patch_embeddings_0 != nullptr);
6    GGML_ASSERT(model.position_embeddings != nullptr);
7
8    norm_type norm_t = NORM_TYPE_NORMAL;
9
10    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
11    ggml_set_name(pos_h, "pos_h");
12    ggml_set_input(pos_h);
13
14    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
15    ggml_set_name(pos_w, "pos_w");
16    ggml_set_input(pos_w);
17
18    ggml_tensor * inp = build_inp();
19    ggml_tensor * learned_pos_embd = resize_position_embeddings();
20
21    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
22        return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
23    };
24
25    auto add_spatial_bias = [&](ggml_tensor * cur, ggml_tensor * bias) {
26        if (bias == nullptr) {
27            return cur;
28        }
29
30        const int64_t width    = cur->ne[0];
31        const int64_t height   = cur->ne[1];
32        const int64_t channels = cur->ne[2];
33
34        cur = ggml_reshape_2d(ctx0, cur, width * height, channels);
35        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
36        cur = ggml_add(ctx0, cur, bias);
37        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
38        cur = ggml_reshape_3d(ctx0, cur, width, height, channels);
39
40        return cur;
41    };
42
43    ggml_tensor * cur = build_vit(
44        inp,
45        n_patches,
46        norm_t,
47        hparams.ffn_op,
48        learned_pos_embd,
49        add_pos);
50    cb(cur, "vit_out", -1);
51
52    // [n_embd, n_patches] -> [w, h, n_embd] for spatial downsampling convolutions.
53    cur = ggml_permute(ctx0, cur, 1, 0, 2, 3);
54    cur = ggml_cont_3d(ctx0, cur, n_patches_x, n_patches_y, n_embd);
55
56    // First downsampler: Conv2d(1536 -> 3072, k=3, s=2, p=1)
57    cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, 2, 2, 1, 1, 1, 1);
58    cur = add_spatial_bias(cur, model.mm_0_b);
59    cb(cur, "downsample_0", -1);
60
61    // Second downsampler: Conv2d(3072 -> 6144, k=3, s=2, p=1)
62    cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 2, 2, 1, 1, 1, 1);
63    cur = add_spatial_bias(cur, model.mm_1_b);
64    cb(cur, "downsample_1", -1);
65
66    // [w, h, c] -> [c, w*h]
67    {
68        const int64_t w = cur->ne[0];
69        const int64_t h = cur->ne[1];
70        cur = ggml_reshape_3d(ctx0, cur, w * h, cur->ne[2], cur->ne[3]);
71        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 0, 2, 3));
72    }
73    cb(cur, "downsample_flatten", -1);
74
75    // Final projector: Linear(6144 -> projection_dim)
76    cur = ggml_mul_mat(ctx0, model.mm_model_proj, cur);
77    cb(cur, "projector_out", -1);
78
79    ggml_build_forward_expand(gf, cur);
80    return gf;
81}