char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: support dots3-note vision+audio (#27524)54ee5ee64

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB61 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_dots3note_a::build() {
4    // inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded
5    // the reference impl zero-masks conv inputs beyond the valid length at each stage;
6    // running on exactly the valid frames with the convs' zero padding is equivalent
7    ggml_tensor * inp = build_inp_raw(1);
8    GGML_ASSERT(inp->type == GGML_TYPE_F32);
9
10    // 3x conv2d (k=3, s=2, p=1) + gelu
11    {
12        auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
13            x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
14            x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
15            return ggml_gelu_erf(ctx0, x);
16        };
17
18        inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
19        inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
20        inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
21        // inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1]
22        cb(inp, "after_conv_stem", -1);
23    }
24
25    // [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape)
26    inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
27    inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]);
28
29    // project to d_model (no bias)
30    inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
31    cb(inp, "after_conv_out", -1);
32
33    const int64_t n_pos = inp->ne[1];
34
35    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
36    ggml_set_name(positions, "positions");
37    ggml_set_input(positions);
38
39    // partial rotary: first half of each head, NEOX style
40    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
41        return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2,
42                             GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
43    };
44
45    ggml_tensor * cur = build_vit(inp, n_pos,
46        NORM_TYPE_RMS, hparams.ffn_op,
47        nullptr, add_pos);
48    cb(cur, "after_transformer", -1);
49
50    // adapter: LayerNorm -> Linear -> GELU -> Linear
51    cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1);
52    cur = build_ffn(cur,
53        model.mm_1_w, model.mm_1_b,
54        nullptr, nullptr,
55        model.mm_2_w, model.mm_2_b,
56        FFN_GELU_ERF, -1);
57    cb(cur, "projected", -1);
58
59    ggml_build_forward_expand(gf, cur);
60    return gf;
61}