char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovhparams : refactor `hparams.n_layer` (#24060)7acb4e8cd

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
10.2 KiB211 linesraw
1#include "models.h"
2
3void llama_model_rwkv7::load_arch_hparams(llama_model_loader & ml) {
4    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,                hparams.f_norm_eps, false);
5    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,            hparams.f_norm_rms_eps, false);
6    ml.get_key(LLM_KV_WKV_HEAD_SIZE,                          hparams.wkv_head_size);
7    ml.get_key(LLM_KV_ATTENTION_DECAY_LORA_RANK,              hparams.n_lora_decay);
8    ml.get_key(LLM_KV_ATTENTION_ICLR_LORA_RANK,               hparams.n_lora_iclr);
9    ml.get_key(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);
10    ml.get_key(LLM_KV_ATTENTION_GATE_LORA_RANK,               hparams.n_lora_gate, false);
11    ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT,                      hparams.token_shift_count, false);
12
13    switch (hparams.n_layer()) {
14        case 12:
15            switch (hparams.n_embd) {
16                case 768: type = LLM_TYPE_190M; break;
17                default: type = LLM_TYPE_UNKNOWN;
18            } break;
19        case 24:
20            switch (hparams.n_embd) {
21                case 1024: type = LLM_TYPE_450M; break;
22                case 2048: type = LLM_TYPE_1_5B; break;
23                default: type = LLM_TYPE_UNKNOWN;
24            } break;
25        case 28:
26            switch (hparams.n_embd) {
27                case 1536: type = LLM_TYPE_1_5B; break;
28                case 3584: type = LLM_TYPE_7B; break;
29                default: type = LLM_TYPE_UNKNOWN;
30            } break;
31        case 32:
32            switch (hparams.n_embd) {
33                case 2560: type = LLM_TYPE_2_9B; break;
34                case 4096: type = LLM_TYPE_7B; break;
35                default: type = LLM_TYPE_UNKNOWN;
36            } break;
37        case 61:
38            switch (hparams.n_embd) {
39                case 4096: type = LLM_TYPE_14B; break;
40                default: type = LLM_TYPE_UNKNOWN;
41            } break;
42        default: type = LLM_TYPE_UNKNOWN;
43    }
44}
45
46void llama_model_rwkv7::load_arch_tensors(llama_model_loader &) {
47    LLAMA_LOAD_LOCALS;
48
49    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
50
51    // Block 0, LN0
52    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
53    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias",   0), {n_embd}, 0);
54
55    // output
56    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
57    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0);
58    output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
59
60    const int n_lora_decay = hparams.n_lora_decay;
61    const int n_lora_iclr = hparams.n_lora_iclr;
62    const int n_lora_value_res_mix = hparams.n_lora_value_res_mix;
63    const int n_lora_gate = hparams.n_lora_gate;
64    const int attn_hidden_size = n_embd;
65    const int ffn_size = hparams.n_ff_arr[0];
66
67    for (int i = 0; i < n_layer; ++i) {
68        auto & layer = layers[i];
69
70        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
71        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i),   {n_embd}, 0);
72
73        layer.attn_norm_2   = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, 0);
74        layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, "bias", i),   {n_embd}, 0);
75
76        layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, "weight", i), {n_embd}, 0);
77        layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, "weight", i), {n_embd, n_lora_decay}, 0);
78        layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, "weight", i), {n_lora_decay, n_embd}, 0);
79
80        layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, "weight", i), {n_embd}, 0);
81        layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, "weight", i), {n_embd, n_lora_iclr}, 0);
82        layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, "weight", i), {n_lora_iclr, n_embd}, 0);
83
84        if (i == 0) {
85            // actually not used
86            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0);
87            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_iclr}, 0);
88            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_iclr, n_embd}, 0);
89        } else {
90            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, "weight", i), {n_embd}, 0);
91            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, "weight", i), {n_embd, n_lora_value_res_mix}, 0);
92            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, "weight", i), {n_lora_value_res_mix, n_embd}, 0);
93        }
94
95        layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, "weight", i), {n_embd, n_lora_gate}, 0);
96        layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, "weight", i), {n_lora_gate, n_embd}, 0);
97
98        layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, "weight", i), {n_embd, 1, 1, 6}, 0);
99
100        layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, "weight", i), {attn_hidden_size}, 0);
101        layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, "weight", i), {attn_hidden_size}, 0);
102        layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, "weight", i), {attn_hidden_size}, 0);
103
104        layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, "weight", i), {attn_hidden_size, n_embd}, 0);
105        layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, "weight", i), {attn_hidden_size, n_embd}, 0);
106        layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, "weight", i), {attn_hidden_size, n_embd}, 0);
107
108        layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "weight", i), {n_embd}, 0);
109        layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, "bias", i), {n_embd}, 0);
110        layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, "weight", i), {n_embd, attn_hidden_size}, 0);
111
112        layer.channel_mix_lerp_k = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, "weight", i), {n_embd, 1, 1}, 0);
113
114        layer.channel_mix_key = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_KEY, "weight", i), {n_embd, ffn_size}, 0);
115        layer.channel_mix_value = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_VALUE, "weight", i), {ffn_size, n_embd}, 0);
116    }
117
118}
119
120std::unique_ptr<llm_graph_context> llama_model_rwkv7::build_arch_graph(const llm_graph_params & params) const {
121    return std::make_unique<graph>(*this, params);
122}
123
124llama_model_rwkv7::graph::graph(const llama_model & model, const llm_graph_params & params) :
125    llm_build_rwkv7_base(model, params) {
126    GGML_ASSERT(hparams.token_shift_count == 2);
127
128    ggml_tensor * cur;
129    ggml_tensor * inpL;
130    ggml_tensor * v_first = nullptr;
131
132    inpL = build_inp_embd(model.tok_embd);
133    inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
134
135    auto * rs_inp = build_rs_inp();
136
137    const auto n_embd       = hparams.n_embd;
138    const auto n_seq_tokens = ubatch.n_seq_tokens;
139    const auto n_seqs       = ubatch.n_seqs;
140
141    ggml_tensor * inp_out_ids = build_inp_out_ids();
142
143    for (int il = 0; il < n_layer; ++il) {
144        const llama_layer * layer = &model.layers[il];
145        inpL                      = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);
146
147        ggml_tensor * token_shift = build_rwkv_token_shift_load(rs_inp, ubatch, il);
148
149        ggml_tensor * att_shift =
150            ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0);
151        ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1],
152                                               token_shift->nb[2], n_embd * ggml_element_size(token_shift));
153
154        ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il);
155        cb(att_norm, "attn_norm", il);
156
157        ggml_tensor * x_prev = ggml_concat(
158            ctx0, att_shift,
159            ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), 1);
160
161        cur = build_rwkv7_time_mix(rs_inp, att_norm, x_prev, v_first, ubatch, il);
162
163        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
164        cb(ffn_inp, "ffn_inp", il);
165
166        ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il);
167        cb(ffn_norm, "ffn_norm", il);
168
169        x_prev = ggml_concat(
170            ctx0, ffn_shift,
171            ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), 1);
172
173        token_shift = ggml_concat(ctx0,
174                                  ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2],
175                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(att_norm)),
176                                  ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2],
177                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(ffn_norm)),
178                                  1);
179        ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il));
180
181        ffn_inp  = ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens);
182        ffn_norm = ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens);
183        x_prev   = ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens);
184
185        if (il == n_layer - 1 && inp_out_ids) {
186            ffn_inp  = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);
187            ffn_norm = ggml_get_rows(ctx0, ffn_norm, inp_out_ids);
188            x_prev   = ggml_get_rows(ctx0, x_prev, inp_out_ids);
189        }
190        cur = build_rwkv7_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV7);
191        cur = ggml_add(ctx0, cur, ffn_inp);
192
193        cur = build_cvec(cur, il);
194        cb(cur, "l_out", il);
195
196        // input for next layer
197        inpL = cur;
198    }
199    cur = inpL;
200    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1);
201
202    cb(cur, "result_norm", -1);
203    res->t_embd = cur;
204
205    cur = build_lora_mm(model.output, cur, model.output_s);
206
207    cb(cur, "result_output", -1);
208    res->t_logits = cur;
209
210    ggml_build_forward_expand(gf, cur);
211}