char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: support pocket-tts (#26871)6e62ba538

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.2 KiB77 linesraw
1#include "models.h"
2
3// voice-prompt encoder: raw 24kHz waveform -> one conditioning row per 12.5Hz frame
4// mimi encoder (SEANet + transformer + downsample), then flow_lm.speaker_proj_weight
5
6// pre-norm block with layer scale on both residual paths, see mimi_transformer.py
7ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
8    ggml_tensor * inp = cur;
9
10    cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
11
12    ggml_tensor * Qcur = build_mm(layer.q_w, cur);
13    ggml_tensor * Kcur = build_mm(layer.k_w, cur);
14    ggml_tensor * Vcur = build_mm(layer.v_w, cur);
15
16    const int64_t n_pos = cur->ne[1];
17    Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
18    Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
19    Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
20
21    Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
22                         hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
23    Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
24                         hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
25
26    cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
27    cur = ggml_mul(ctx0, cur, layer.ls_1_w);
28    cur = ggml_add(ctx0, cur, inp);
29
30    inp = cur;
31    cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
32    cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
33    cur = ggml_mul(ctx0, cur, layer.ls_2_w);
34    cur = ggml_add(ctx0, cur, inp);
35
36    return cur;
37}
38
39ggml_cgraph * clip_graph_pockettts_spkenc::build() {
40    // the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
41    ggml_tensor * inp_raw = build_inp_raw(1);
42    ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
43
44    clip_graph_pockettts_seanet seanet(*this);
45    cur = seanet.encode(cur);
46    cb(cur, "mimi_enc", -1);
47
48    // [T, 512] -> transformer works on [512, T]
49    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
50
51    ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, cur->ne[1]);
52    ggml_set_name(inp_pos, "inp_pos");
53    ggml_set_input(inp_pos);
54
55    // the mimi transformer is causal with a sliding window, see _build_attention_mask()
56    ggml_tensor * kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, cur->ne[1], cur->ne[1]);
57    ggml_set_name(kq_mask, "kq_mask");
58    ggml_set_input(kq_mask);
59
60    for (int il = 0; il < n_layer; il++) {
61        cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
62    }
63    cb(cur, "mimi_enc_tfm", -1);
64
65    // downsample to the model frame rate, [512, T] -> [T, 512] -> [T / 16, 32]
66    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
67    cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, true);
68    cb(cur, "mimi_downsample", -1);
69
70    // voice latent -> backbone embd
71    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
72    cur = build_mm(model.spk_proj_w, cur);
73    cb(cur, "spk_proj", -1);
74
75    ggml_build_forward_expand(gf, cur);
76    return gf;
77}