char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovhparams : refactor `hparams.n_layer` (#24060)7acb4e8cd

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.8 KiB209 linesraw
1#include "models.h"
2
3void llama_model_falcon_h1::load_arch_hparams(llama_model_loader & ml) {
4    // Common parameters
5    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
6
7    // SSM parameters
8    ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);
9    ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);
10    ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);
11    ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
12    ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);
13
14    std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), true);
15
16    switch (hparams.n_layer()) {
17        case 36:
18            type = LLM_TYPE_0_5B; break;
19        case 24:
20            type = LLM_TYPE_1_5B; break;
21        case 66:
22            type = LLM_TYPE_1B; break;
23        case 32:
24            type = LLM_TYPE_3B; break;
25        case 44:
26            type = LLM_TYPE_7B; break;
27        case 72:
28            type = LLM_TYPE_34B; break;
29        default:
30            type = LLM_TYPE_UNKNOWN;
31    }
32}
33
34void llama_model_falcon_h1::load_arch_tensors(llama_model_loader &) {
35    LLAMA_LOAD_LOCALS;
36
37    // Common
38    const int64_t hidden_size = hparams.n_embd; // hidden_size
39
40    // mamba2 Mixer SSM params
41    const int64_t ssm_conv_kernel_size  = hparams.ssm_d_conv; // ssm_conv_kernel_size
42    const int64_t ssm_n_groups          = hparams.ssm_n_group; // ssm_n_groups
43    const int64_t ssm_state_size        = hparams.ssm_d_state; // ssm_state_size
44    const int64_t ssm_intermediate_size = hparams.ssm_d_inner; // TODO expand
45    const int64_t ssm_num_heads         = hparams.ssm_dt_rank; // ssm_num_heads
46    const int64_t ssm_conv_dim          = ssm_intermediate_size + 2 * ssm_n_groups * ssm_state_size;
47    const int64_t ssm_projection_size   = ssm_intermediate_size + ssm_conv_dim + ssm_num_heads;
48
49    // attn params
50    const int64_t attn_num_attention_head = hparams.n_head(0); // rename to: attn_num_attention_head
51    const int64_t attn_num_key_value_head = hparams.n_head_kv(0);
52
53    // ffn params
54    const int64_t ffn_intermediate_size = hparams.n_ff(0);
55
56    // embeddings
57    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hidden_size, n_vocab}, 0);
58
59    // output
60    output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hidden_size, n_vocab}, TENSOR_NOT_REQUIRED);
61    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {hidden_size}, 0);
62
63    // if output is NULL, init from the input tok embed
64    if (output == NULL) {
65        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hidden_size, n_vocab}, TENSOR_DUPLICATED);
66    }
67
68    for (int i = 0; i < n_layer; ++i) {
69        auto & layer = layers[i];
70
71        /*SSM LAYERS*/
72        // ssm in
73        layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", i), {hidden_size, ssm_projection_size}, 0);
74        // ssm 1d conv
75        layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {ssm_conv_kernel_size, ssm_conv_dim}, 0);
76        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {ssm_conv_dim}, TENSOR_NOT_REQUIRED);
77        // ssm_dt
78        layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {ssm_num_heads}, 0);
79        // no "weight" suffix for these
80        layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, ssm_num_heads}, 0);
81        layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, ssm_num_heads}, 0);
82        // ssm_norm
83        layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {ssm_intermediate_size / ssm_n_groups, ssm_n_groups}, TENSOR_NOT_REQUIRED);
84        // out_proj
85        layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {ssm_intermediate_size, hidden_size}, 0);
86
87        /*ATTENTION LAYERS*/
88        // attention layers (with optional bias)
89        create_tensor_qkv(layer, i, hidden_size, n_embd_head_k * attn_num_attention_head, attn_num_key_value_head * n_embd_head_k, attn_num_key_value_head * n_embd_head_v, 0);
90        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * attn_num_attention_head, hidden_size}, 0);
91        layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);
92        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {hidden_size}, 0);
93
94
95        // feed forward (w/ optional biases)
96        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, i), {hidden_size}, 0);
97        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
98        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {hidden_size,   ffn_intermediate_size}, 0);
99        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  ffn_intermediate_size, hidden_size}, 0);
100        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {hidden_size,   ffn_intermediate_size}, 0);
101
102        layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);
103        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);
104        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "bias", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);
105    }
106}
107
108std::unique_ptr<llm_graph_context> llama_model_falcon_h1::build_arch_graph(const llm_graph_params & params) const {
109    return std::make_unique<graph>(*this, params);
110}
111
112llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_params & params) :
113    llm_build_mamba_base(params) {
114    const int64_t n_embd_head = hparams.n_embd_head_v();
115
116    ggml_tensor * cur;
117    ggml_tensor * inpL;
118
119    inpL = build_inp_embd(model.tok_embd);
120
121    // inp_pos - contains the positions
122    ggml_tensor * inp_pos = build_inp_pos();
123
124    // Build the inputs in the recurrent & kv cache
125    auto * inp = build_inp_mem_hybrid();
126
127    const float kq_scale =
128        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
129
130    ggml_tensor * inp_out_ids = build_inp_out_ids();
131
132    for (int il = 0; il < n_layer; ++il) {
133        ggml_tensor * inpSA = inpL;
134
135        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
136        cb(cur, "attn_norm", il);
137
138        // self-attention
139        auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
140                n_embd_head, n_head, n_head_kv, il);
141
142        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,
143                             ext_factor, attn_factor, beta_fast, beta_slow);
144
145        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,
146                             ext_factor, attn_factor, beta_fast, beta_slow);
147
148        cb(Qcur, "Qcur-post-rope", il);
149        cb(Kcur, "Kcur-post-rope", il);
150        cb(Vcur, "Vcur-post-rope", il);
151
152        ggml_tensor * attn_out = build_attn(inp->get_attn(),
153                                    model.layers[il].wo, NULL, model.layers[il].wo_s,
154                                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
155        cb(attn_out, "attn_out", il);
156
157        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
158        // Mamba2 layer
159        cb(cur, "ssm_in", il);
160
161        ggml_tensor * ssm_out = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);
162        cb(ssm_out, "ssm_out", il);
163
164        // // Aggregation
165        cur   = ggml_add(ctx0, attn_out, ssm_out);
166        inpSA = ggml_add(ctx0, cur, inpSA);
167        cb(cur, "layer_out", il);
168
169        if (il == n_layer - 1 && inp_out_ids) {
170            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
171            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
172        }
173        ggml_tensor * ffn_inp = inpSA;
174        cb(ffn_inp, "ffn_inp", il);
175
176        // feed-forward network
177        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
178        cb(cur, "ffn_norm", il);
179
180        cur = build_ffn(cur,
181                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
182                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
183                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
184                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
185        cb(cur, "ffn_out", il);
186
187        cur = ggml_add(ctx0, cur, inpSA);
188
189        cur = build_cvec(cur, il);
190        cb(cur, "l_out", il);
191
192        // input for next layer
193        inpL = cur;
194    }
195    cur = inpL;
196
197    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
198
199    cb(cur, "result_norm", -1);
200    res->t_embd = cur;
201
202    // lm_head
203    cur = build_lora_mm(model.output, cur, model.output_s);
204
205    cb(cur, "result_output", -1);
206    res->t_logits = cur;
207
208    ggml_build_forward_expand(gf, cur);
209}