char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: add chunks and fix preproc for qwen3a (#23073)72e60f500

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.5 KiB88 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_qwen3a::build() {
4    // Ref implementation: https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py
5
6    // inp_raw: [n_frames, n_mel, 1]  (nx=n_frames, ny=n_mel)
7    ggml_tensor * inp = build_inp_raw(1);
8
9    const int64_t n_frames   = inp->ne[0]; // total frames, padded to multiple of chunk_size
10    const int64_t n_mel      = inp->ne[1]; // 128
11    const int64_t chunk_size = 100;        // n_window * 2 (n_window=50 from model config)
12    const int64_t n_chunks   = n_frames / chunk_size;
13
14    GGML_ASSERT(n_frames % chunk_size == 0); // preprocessor should already pad the input
15    GGML_ASSERT(inp->type == GGML_TYPE_F32);
16
17    // View mel spectrogram as batched 100-frame chunks: [chunk_size, n_mel, 1, n_chunks]
18    inp = ggml_view_4d(ctx0, inp,
19        chunk_size, n_mel, 1, n_chunks,
20        n_frames   * (int64_t)sizeof(float), // nb[1]: stride over mel bins
21        chunk_size * (int64_t)sizeof(float), // nb[2]: stride for C=1 (unused)
22        chunk_size * (int64_t)sizeof(float), // nb[3]: stride over chunks
23        0);
24    inp = ggml_cont(ctx0, inp);
25    cb(inp, "inp_chunks", -1);
26
27    // 3 x conv2d + gelu
28    {
29        // conv output [OW, OH, C_out, n_chunks]
30        auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
31            x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
32            if (b) {
33                x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
34            }
35            return ggml_gelu_erf(ctx0, x);
36        };
37
38        inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
39        inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
40        inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
41        // inp: [OW=13, OH=16, OC=480, n_chunks]
42        cb(inp, "after_conv_blocks", -1);
43    }
44
45    // permute [OW=25, OH=16, OC=480, n_chunks] -> [OH=16, OC=480, OW=25, n_chunks]
46    // reshape to [OH*OC=7680, OW*n_chunks]
47    // feature index h+16*c = c*16+f (matches python code)
48    inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
49    inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2] * inp->ne[3]);
50
51    // Project to d_model: [d_model, 25*n_chunks]
52    inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
53    if (model.conv_out_b) {
54        inp = ggml_add(ctx0, inp, model.conv_out_b);
55    }
56    cb(inp, "after_conv_out", -1);
57
58    const int64_t n_pos = inp->ne[1]; // 25 * n_chunks
59
60    // Per-chunk positional embeddings: repeat pos[0:13] for each chunk
61    // (position indices reset 0..12 per chunk, not sequential across chunks)
62    {
63        const int64_t tokens_per_chunk = n_pos / n_chunks; // 13
64        ggml_tensor * pos_tmp = ggml_view_2d(ctx0, model.position_embeddings,
65            model.position_embeddings->ne[0], tokens_per_chunk,
66            model.position_embeddings->nb[1], 0);
67        ggml_tensor * tgt = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,
68            model.position_embeddings->ne[0], n_pos);
69        inp = ggml_add(ctx0, inp, ggml_repeat(ctx0, pos_tmp, tgt));
70    }
71
72    ggml_tensor * cur = build_vit(inp, n_pos,
73        NORM_TYPE_NORMAL, hparams.ffn_op,
74        nullptr,  // pos embd already added above
75        nullptr);
76    cb(cur, "after_transformer", -1);
77
78    // MLP projector
79    cur = build_ffn(cur,
80        model.mm_1_w, model.mm_1_b,
81        nullptr, nullptr,
82        model.mm_2_w, model.mm_2_b,
83        FFN_GELU_ERF, -1);
84    cb(cur, "projected", -1);
85
86    ggml_build_forward_expand(gf, cur);
87    return gf;
88}