char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

ynankanimodel : NvFP4 quantized LM head support (#23046)42928bc14

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
10.3 KiB264 linesraw
1#include "models.h"
2
3void llama_model_wavtokenizer_dec::load_arch_hparams(llama_model_loader & ml) {
4    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
5    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS,    hparams.f_norm_group_eps);
6    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
7}
8
9void llama_model_wavtokenizer_dec::load_arch_tensors(llama_model_loader &) {
10    LLAMA_LOAD_LOCALS;
11
12    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hparams.n_embd, n_vocab}, 0);
13
14    conv1d   = create_tensor(tn(LLM_TENSOR_CONV1D, "weight", 0), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);
15    conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias",   0), {1, hparams.posnet.n_embd}, 0);
16
17    // posnet
18    {
19        const int64_t n_embd = hparams.posnet.n_embd;
20
21        for (uint32_t i = 0; i < hparams.posnet.n_layer; ++i) {
22            auto & layer = layers[i].posnet;
23
24            // posnet:
25            //
26            //  - resnet
27            //  - resnet
28            //  - attn
29            //  - resnet
30            //  - resnet
31            //  - norm
32            //
33            switch (i) {
34                case 0:
35                case 1:
36                case 3:
37                case 4:
38                    {
39                        layer.norm1   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "weight", i), {1, n_embd}, 0);
40                        layer.norm1_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "bias",   i), {1, n_embd}, 0);
41
42                        layer.conv1   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "weight", i), {3, n_embd, n_embd}, 0);
43                        layer.conv1_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "bias",   i), {1, n_embd}, 0);
44
45                        layer.norm2   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "weight", i), {1, n_embd}, 0);
46                        layer.norm2_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "bias",   i), {1, n_embd}, 0);
47
48                        layer.conv2   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "weight", i), {3, n_embd, n_embd}, 0);
49                        layer.conv2_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "bias",   i), {1, n_embd}, 0);
50                    } break;
51                case 2:
52                    {
53                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
54                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);
55
56                        layer.attn_q      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "weight", i), {1, n_embd, n_embd}, 0);
57                        layer.attn_q_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "bias",   i), {1, n_embd}, 0);
58
59                        layer.attn_k      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "weight", i), {1, n_embd, n_embd}, 0);
60                        layer.attn_k_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "bias",   i), {1, n_embd}, 0);
61
62                        layer.attn_v      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "weight", i), {1, n_embd, n_embd}, 0);
63                        layer.attn_v_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "bias",   i), {1, n_embd}, 0);
64
65                        layer.attn_o      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "weight", i), {1, n_embd, n_embd}, 0);
66                        layer.attn_o_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "bias",   i), {1, n_embd}, 0);
67                    } break;
68                case 5:
69                    {
70                        layer.norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
71                        layer.norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);
72                    } break;
73                default: GGML_ABORT("unknown posnet layer");
74            };
75        }
76    }
77
78    GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);
79
80    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {hparams.posnet.n_embd}, 0);
81    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias",   0), {hparams.posnet.n_embd}, 0);
82
83    // convnext
84    {
85        const int64_t n_embd = hparams.convnext.n_embd;
86
87        for (uint32_t i = 0; i < hparams.convnext.n_layer; ++i) {
88            auto & layer = layers[i].convnext;
89
90            layer.dw     = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "weight", i), {7, 1, n_embd}, 0);
91            layer.dw_b   = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "bias",   i), {1, n_embd}, 0);
92
93            layer.norm   = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "weight", i), {n_embd}, 0);
94            layer.norm_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "bias",   i), {n_embd}, 0);
95
96            layer.pw1    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "weight", i), {n_embd, n_ff}, 0);
97            layer.pw1_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "bias",   i), {n_ff}, 0);
98
99            layer.pw2    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "weight", i), {n_ff, n_embd}, 0);
100            layer.pw2_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "bias",   i), {n_embd}, 0);
101
102            layer.gamma  = create_tensor(tn(LLM_TENSOR_CONVNEXT_GAMMA, "weight", i), {n_embd}, 0);
103        }
104
105        // output
106        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
107        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"),   {n_embd}, 0);
108    }
109
110    output   = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, hparams.n_embd_out()}, 0);
111    output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"),   {hparams.n_embd_out()}, 0);
112}
113
114std::unique_ptr<llm_graph_context> llama_model_wavtokenizer_dec::build_arch_graph(const llm_graph_params & params) const {
115    return std::make_unique<graph>(*this, params);
116}
117
118llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
119    ggml_tensor * cur;
120    ggml_tensor * inpL;
121
122    inpL = build_inp_embd(model.tok_embd);
123
124    cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
125
126    cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1);
127    cur = ggml_add(ctx0, cur, model.conv1d_b);
128
129    // posnet
130    for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) {
131        const auto & layer = model.layers[il].posnet;
132
133        inpL = cur;
134
135        switch (il) {
136            case 0:
137            case 1:
138            case 3:
139            case 4:
140                {
141                    cur = build_norm(cur,
142                            layer.norm1,
143                            layer.norm1_b,
144                            LLM_NORM_GROUP, 0);
145
146                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);
147
148                    cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1);
149                    cur = ggml_add(ctx0, cur, layer.conv1_b);
150
151                    cur = build_norm(cur,
152                            layer.norm2,
153                            layer.norm2_b,
154                            LLM_NORM_GROUP, 0);
155
156                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);
157
158                    cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1);
159                    cur = ggml_add(ctx0, cur, layer.conv2_b);
160
161                    cur = ggml_add(ctx0, cur, inpL);
162                } break;
163            case 2:
164                {
165                    cur = build_norm(cur,
166                            layer.attn_norm,
167                            layer.attn_norm_b,
168                            LLM_NORM_GROUP, 0);
169
170                    ggml_tensor * q;
171                    ggml_tensor * k;
172                    ggml_tensor * v;
173
174                    q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1);
175                    k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1);
176                    v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1);
177
178                    q = ggml_add(ctx0, q, layer.attn_q_b);
179                    k = ggml_add(ctx0, k, layer.attn_k_b);
180                    v = ggml_add(ctx0, v, layer.attn_v_b);
181
182                    q = ggml_cont(ctx0, ggml_transpose(ctx0, q));
183                    k = ggml_cont(ctx0, ggml_transpose(ctx0, k));
184
185                    ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
186
187                    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f);
188
189                    cur = ggml_mul_mat(ctx0, kq, v);
190
191                    cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1);
192                    cur = ggml_add(ctx0, cur, layer.attn_o_b);
193
194                    cur = ggml_add(ctx0, cur, inpL);
195                } break;
196            case 5:
197                {
198                    cur = build_norm(cur,
199                            layer.norm,
200                            layer.norm_b,
201                            LLM_NORM_GROUP, 0);
202                } break;
203            default: GGML_ABORT("unknown posnet layer");
204        };
205    }
206    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
207
208    cur = build_norm(cur,
209            model.tok_norm,
210            model.tok_norm_b,
211            LLM_NORM, 0);
212
213    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
214
215    inpL = cur;
216
217    // convnext
218    for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) {
219        const auto & layer = model.layers[il].convnext;
220
221        cur = inpL;
222
223        cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1);
224        cur = ggml_add(ctx0, cur, layer.dw_b);
225
226        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
227
228        cur = build_norm(cur,
229                layer.norm,
230                layer.norm_b,
231                LLM_NORM, -1);
232
233        cur = build_ffn(cur,
234                layer.pw1, layer.pw1_b, NULL,
235                NULL,      NULL,        NULL,
236                layer.pw2, layer.pw2_b, NULL,
237                NULL,
238                LLM_FFN_GELU, LLM_FFN_SEQ, il);
239
240        cur = ggml_mul(ctx0, cur, layer.gamma);
241
242        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
243
244        inpL = ggml_add(ctx0, cur, inpL);
245    }
246    cur = inpL;
247
248    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
249
250    cur = build_norm(cur,
251            model.output_norm,
252            model.output_norm_b,
253            LLM_NORM, -1);
254
255    // lm_head
256    cur = build_lora_mm(model.output, cur, model.output_s);
257
258    cur = ggml_add(ctx0, cur, model.output_b);
259
260    cb(cur, "result_embd", -1);
261    res->t_embd = cur;
262
263    ggml_build_forward_expand(gf, cur);
264}