char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

lnigamggml : recurrent state rollback for ggml_ssm_scan (#26623)1692f9e50

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
17.6 KiB393 linesraw
1#pragma once
2
3#include "ggml-et-common.h"
4#include "ggml.h"
5
6#include <inttypes.h>
7
8// Performance logging macros for ET ops
9// Logs in machine-parseable pipe-delimited format: ET_PERF|field=value|...
10#ifdef ET_PERF_RECORD
11#    define ET_PERF_START() int64_t _et_perf_start = ggml_time_us()
12
13#    define ET_PERF_END(op_name, kernel_name, node)                                                                \
14        do {                                                                                                       \
15            int64_t _et_perf_end      = ggml_time_us();                                                            \
16            int64_t _et_perf_duration = _et_perf_end - _et_perf_start;                                             \
17            GGML_LOG_DEBUG("ET_PERF|op=%s|kernel=%s|duration_us=%" PRId64 "|tensor=%s|shape=[%" PRId64 ",%" PRId64 \
18                           ",%" PRId64 ",%" PRId64 "]|start_us=%" PRId64 "|end_us=%" PRId64 "\n",                  \
19                           op_name, kernel_name, _et_perf_duration, (node)->name, (node)->ne[0], (node)->ne[1],    \
20                           (node)->ne[2], (node)->ne[3], _et_perf_start, _et_perf_end);                            \
21        } while (0)
22
23#    define ET_PERF_END_EXT(op_name, kernel_name, node, fmt, ...)                                                  \
24        do {                                                                                                       \
25            int64_t _et_perf_end      = ggml_time_us();                                                            \
26            int64_t _et_perf_duration = _et_perf_end - _et_perf_start;                                             \
27            GGML_LOG_DEBUG("ET_PERF|op=%s|kernel=%s|duration_us=%" PRId64 "|tensor=%s|shape=[%" PRId64 ",%" PRId64 \
28                           ",%" PRId64 ",%" PRId64 "]|start_us=%" PRId64 "|end_us=%" PRId64 "|" fmt "\n",          \
29                           op_name, kernel_name, _et_perf_duration, (node)->name, (node)->ne[0], (node)->ne[1],    \
30                           (node)->ne[2], (node)->ne[3], _et_perf_start, _et_perf_end, ##__VA_ARGS__);             \
31        } while (0)
32#else
33
34#    define ET_PERF_START() \
35        do {                \
36        } while (0)
37#    define ET_PERF_END_EXT(op_name, kernel_name, node, fmt, ...) \
38        do {                                                      \
39            (void) (node);                                        \
40        } while (0)
41#    define ET_PERF_END(op_name, kernel_name, node) \
42        do {                                        \
43            (void) (node);                          \
44        } while (0)
45
46#endif  // ET_PERF_RECORD
47
48struct ggml_et_binary_params {
49    ggml_tensor src0;
50    ggml_tensor src1;
51    ggml_tensor dst;
52};
53
54// Q8_0 mul_mat with optional residual bias.
55// bias.data == NULL means "no bias" - kernel skips the add.
56// When non-NULL, bias must have the same shape and strides as dst.
57struct ggml_et_mm_q8_params {
58    ggml_tensor src0;
59    ggml_tensor src1;
60    ggml_tensor dst;
61    ggml_tensor bias;
62};
63
64struct ggml_et_im2col_params {
65    ggml_tensor src0;
66    ggml_tensor src1;
67    ggml_tensor dst;
68};
69
70// Element map parameters for embarrassingly parallel binary operations (MUL, ADD, etc.)
71// Operation type is determined by dst->op (GGML_OP_MUL, GGML_OP_ADD, etc.)
72struct ggml_et_elmap_params {
73    ggml_tensor src0;
74    ggml_tensor src1;
75    ggml_tensor dst;
76};
77
78struct ggml_et_rope_settings {
79    int32_t n_past;
80    int32_t n_dims;  // Number of dimensions to apply ROPE to (must be even)
81    int32_t mode;    // ROPE mode, GGML_ROPE_TYPE_*
82    int32_t n_ctx;
83    int32_t n_ctx_orig;
84    float   freq_base;    // Base frequency (usually 10000.0f)
85    float   freq_scale;   // Frequency scaling factor
86    float   ext_factor;   // Extension factor for YaRN
87    float   attn_factor;  // Attention factor for YaRN
88    float   beta_fast;    // Fast beta for YaRN
89    float   beta_slow;    // Slow beta for YaRN
90    int32_t sections[4];  // Sections for multi-modal ROPE
91};
92
93struct ggml_et_rope_params {
94    ggml_tensor           src0;
95    ggml_tensor           src1;
96    ggml_tensor           src2;
97    ggml_tensor           dst;
98    ggml_et_rope_settings rope_params;
99};
100
101struct ggml_et_rms_norm_params {
102    ggml_tensor src0;  // F32 input tensor
103    ggml_tensor dst;   // F32 output tensor
104    float       eps;   // Epsilon parameter for numerical stability
105};
106
107struct ggml_et_norm_params {
108    ggml_tensor src0;  // F32 input tensor
109    ggml_tensor dst;   // F32 output tensor
110    float       eps;   // Epsilon parameter for numerical stability
111};
112
113struct ggml_et_l2_norm_params {
114    ggml_tensor src0;  // F32 input tensor
115    ggml_tensor dst;   // F32 output tensor
116    float       eps;   // Epsilon parameter for numerical stability
117};
118
119struct ggml_et_group_norm_params {
120    ggml_tensor src0;      // F32 input tensor
121    ggml_tensor dst;       // F32 output tensor
122    int32_t     n_groups;  // Number of channel groups
123    float       eps;       // Epsilon parameter for numerical stability
124};
125
126struct ggml_et_glu_params {
127    ggml_tensor src0;         // F32 input tensor A (or combined tensor if src1 is null)
128    ggml_tensor src1;         // F32 input tensor B (null for single tensor mode)
129    ggml_tensor dst;          // F32 output tensor (n/2 columns)
130    int32_t     glu_op_type;  // GLU operation type (REGLU=0, GEGLU=1, SWIGLU=2, etc.)
131    int32_t     swapped;      // Whether gate and value are swapped
132    float       alpha;        // SWIGLU_OAI: sigmoid scaling factor (unused for other variants)
133    float       limit;        // SWIGLU_OAI: clamp limit (unused for other variants)
134};
135
136struct ggml_et_softmax_params {
137    ggml_tensor src0;      // F32 input tensor
138    ggml_tensor src1;      // F32 mask tensor (optional, may be zeroed if not used)
139    ggml_tensor src2;      // F32 sinks tensor (optional, may be zeroed if not used)
140    ggml_tensor dst;       // F32 output tensor
141    float       scale;     // Scale factor
142    float       max_bias;  // Max bias for ALiBi (0.0f if not used)
143};
144
145struct ggml_et_flash_attn_ext_params {
146    ggml_tensor src0;      // Q tensor (F32)
147    ggml_tensor src1;      // K tensor (F32)
148    ggml_tensor src2;      // V tensor (F32)
149    ggml_tensor mask;      // mask tensor (F16 or F32), zeroed when absent
150    ggml_tensor dst;       // Output tensor (F32)
151    float       scale;     // Scale factor applied to QK
152    int32_t     has_mask;  // nonzero if mask is present
153};
154
155struct ggml_et_get_rows_params {
156    ggml_tensor src0;  // Data tensor (F32 or Q8_0)
157    ggml_tensor src1;  // Row indices tensor (I32)
158    ggml_tensor dst;   // Output tensor (F32)
159};
160
161struct ggml_et_cont_params {
162    ggml_tensor src0;  // F32 input tensor (non-contiguous)
163    ggml_tensor dst;   // F32 output tensor (contiguous)
164};
165
166struct ggml_et_concat_params {
167    ggml_tensor src0;  // F32 input tensor 0
168    ggml_tensor src1;  // F32 input tensor 1
169    ggml_tensor dst;   // F32 output tensor
170    int32_t     dim;   // Concatenation dimension
171};
172
173struct ggml_et_repeat_params {
174    ggml_tensor src0;  // F32 input tensor (tile)
175    ggml_tensor dst;   // F32 output tensor (tiled result)
176};
177
178struct ggml_et_fill_params {
179    ggml_tensor dst;  // F32 output tensor (contiguous)
180    float       c;    // Constant value to fill
181};
182
183struct ggml_et_tri_params {
184    ggml_tensor src0;      // F32 input tensor
185    ggml_tensor dst;       // F32 output tensor
186    int32_t     tri_type;  // ggml_tri_type enum value
187};
188
189struct ggml_et_solve_tri_params {
190    ggml_tensor src0;  // A: lower-triangular [n, n, B1, B2]
191    ggml_tensor src1;  // B: RHS [k, n, B1, B2]
192    ggml_tensor dst;   // X: solution [k, n, B1, B2]
193};
194
195struct ggml_et_pad_params {
196    ggml_tensor src0;   // F32 input (may be non-contiguous, nb[0] must == 4)
197    ggml_tensor dst;    // F32 output (contiguous, ne[0] % 16 == 0)
198    int32_t     lp[4];  // left padding per dimension
199    int32_t     rp[4];  // right padding per dimension
200};
201
202struct ggml_et_diag_params {
203    ggml_tensor src0;  // F32 input vector
204    ggml_tensor dst;   // F32 output diagonal matrix
205};
206
207struct ggml_et_ssm_conv_params {
208    ggml_tensor src0;  // conv_x: [d_conv - 1 + n_t, d_inner, n_seqs]
209    ggml_tensor src1;  // conv1d.weight: [d_conv, d_inner]
210    ggml_tensor dst;   // output: [d_inner, n_t, n_seqs]
211};
212
213struct ggml_et_ssm_scan_params {
214    ggml_tensor src0;  // s:   [d_state, head_dim, n_head, n_seqs]
215    ggml_tensor src1;  // x:   [head_dim, n_head, n_seq_tokens, n_seqs]
216    ggml_tensor src2;  // dt:  [n_head, n_seq_tokens, n_seqs]
217    ggml_tensor src3;  // A:   [d_state, n_head] or [1, n_head]
218    ggml_tensor src4;  // B:   [d_state, n_group, n_seq_tokens, n_seqs]
219    ggml_tensor src5;  // C:   [d_state, n_group, n_seq_tokens, n_seqs]
220    ggml_tensor src6;  // ids: [n_seqs] i32
221    ggml_tensor dst;   // [y, states] packed output from ggml_ssm_scan()
222    int32_t     K;
223};
224
225struct ggml_et_rwkv_wkv6_params {
226    float * k;         // src[0]: [S, H, T]  key
227    float * v;         // src[1]: [S, H, T]  value
228    float * r;         // src[2]: [S, H, T]  receptance
229    float * tf;        // src[3]: [S, H]     time_faaaa (per-head)
230    float * td;        // src[4]: [S, H, T]  time_decay
231    float * state_in;  // src[5]: [S*S*H, n_seqs]  initial state
232    float * dst;       // [C, T + S*n_seqs]  output + state_out
233    int32_t C;         // total channels (S * H)
234    int32_t H;         // number of heads
235    int32_t S;         // head size
236    int32_t T;         // number of tokens
237    int32_t n_seqs;    // number of sequences
238};
239
240struct ggml_et_rwkv_wkv7_params {
241    float * r;         // [S, H, T]  receptance
242    float * w;         // [S, H, T]  decay
243    float * k;         // [S, H, T]  key
244    float * v;         // [S, H, T]  value
245    float * a;         // [S, H, T]  bonus gate
246    float * b;         // [S, H, T]  bonus key
247    float * state_in;  // [S*S*H, n_seqs]  initial state
248    float * dst;       // [C, T + S*n_seqs]  output + state_out
249    int32_t C;         // total channels (S * H)
250    int32_t H;         // number of heads
251    int32_t S;         // head size
252    int32_t T;         // number of tokens
253    int32_t n_seqs;    // number of sequences
254};
255
256struct ggml_et_gated_delta_net_params {
257    ggml_tensor q;         // [S_v, H_q, n_tokens, n_seqs_q]
258    ggml_tensor k;         // [S_v, H_k, n_tokens, n_seqs_k]
259    ggml_tensor v;         // [S_v, H, n_tokens, n_seqs]
260    ggml_tensor g;         // [1 or S_v, H, n_tokens, n_seqs]
261    ggml_tensor beta;      // [1, H, n_tokens, n_seqs]
262    ggml_tensor state_in;  // [S_v*S_v*H, K, n_seqs]
263    ggml_tensor dst;       // [S_v*H, n_tokens*n_seqs + S_v*n_seqs*K]
264    int32_t     S_v;       // head dimension (value size)
265    int32_t     H;         // number of value heads
266    int32_t     H_q;       // number of Q heads
267    int32_t     H_k;       // number of K heads
268    int32_t     n_tokens;  // total tokens
269    int32_t     n_seqs;    // number of sequences (from V)
270    int32_t     n_seqs_q;  // Q sequence count
271    int32_t     n_seqs_k;  // K sequence count
272    int32_t     kda;       // 1 if per-element gate (g_ne0 == S_v), 0 if scalar
273    int32_t     K;         // snapshot slot count
274    float       scale;     // 1/sqrt(S_v)
275};
276
277struct ggml_et_set_rows_params {
278    ggml_tensor src0;  // F32 source data tensor
279    ggml_tensor src1;  // I64 row indices tensor
280    ggml_tensor dst;   // F32/F16 destination tensor
281};
282
283struct ggml_et_set_params {
284    ggml_tensor src1;    // F32 source view to write into dst
285    ggml_tensor dst;     // F32 destination/base tensor
286    int32_t     nb1;     // destination view stride for dim 1
287    int32_t     nb2;     // destination view stride for dim 2
288    int32_t     nb3;     // destination view stride for dim 3
289    int32_t     offset;  // byte offset into destination
290};
291
292struct ggml_et_rms_norm_mul_params {
293    ggml_tensor src0;  // F32 input tensor (to be normalized)
294    ggml_tensor src1;  // F32 weights tensor (element-wise multiply)
295    ggml_tensor dst;   // F32 output tensor
296    float       eps;   // Epsilon for numerical stability
297};
298
299struct ggml_et_mul_mat_id_params {
300    ggml_tensor src0;  // Expert weight matrices (Q8_0/F16/F32) [K, M, n_expert]
301    ggml_tensor src1;  // Activations (F32) [K, n_expert_used, batch]
302    ggml_tensor src2;  // Expert indices (I32) [n_expert_used, batch]
303    ggml_tensor dst;   // Output (F32) [M, n_expert_used, batch, 1]
304};
305
306struct ggml_et_sqr_params {
307    ggml_tensor src0;  // F32 input tensor
308    ggml_tensor dst;   // F32 output tensor
309};
310
311struct ggml_et_unary_params {
312    ggml_tensor src0;      // F32 input tensor
313    ggml_tensor dst;       // F32 output tensor
314    int32_t     unary_op;  // ggml_unary_op enum value
315};
316
317struct ggml_et_sum_rows_params {
318    ggml_tensor src0;  // F32 input tensor [ne00, ne01, ne02, ne03]
319    ggml_tensor dst;   // F32 output tensor [1, ne01, ne02, ne03]
320};
321
322struct ggml_et_mean_params {
323    ggml_tensor src0;  // F32 input tensor [ne00, ne01, ne02, ne03]
324    ggml_tensor dst;   // F32 output tensor [1, ne01, ne02, ne03]
325};
326
327struct ggml_et_clamp_params {
328    ggml_tensor src0;  // F32 input tensor (contiguous)
329    ggml_tensor dst;   // F32 output tensor (contiguous; may alias src0)
330    float       min_val;
331    float       max_val;
332};
333
334struct ggml_et_cumsum_params {
335    ggml_tensor src0;  // F32 input tensor [ne00, ne01, ne02, ne03]
336    ggml_tensor dst;   // F32 output tensor [ne00, ne01, ne02, ne03]
337};
338
339struct ggml_et_scale_params {
340    ggml_tensor src0;   // F32 input tensor
341    ggml_tensor dst;    // F32 output tensor
342    float       scale;  // Scale factor
343    float       bias;   // Bias (additive offset)
344};
345
346bool ggml_et_op_cumsum(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
347bool ggml_et_op_sqr(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
348bool ggml_et_op_unary(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
349bool ggml_et_op_sum_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
350bool ggml_et_op_mean(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
351bool ggml_et_op_clamp(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
352bool ggml_et_op_scale(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
353bool ggml_et_op_mul(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
354bool ggml_et_op_add(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
355bool ggml_et_op_sub(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
356// add_node is optional: when non-NULL and the pair (node, add_node) was
357// validated by ggml_et_can_fuse({MUL_MAT, ADD}), the Q8_0 path writes
358// dst = mm(...) + add_node's "other" operand (the bias) in one launch.
359bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx,
360                        const ggml_tensor *              node,
361                        const ggml_tensor *              add_node = nullptr);
362bool ggml_et_op_mul_mat_id(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
363bool ggml_et_op_rope(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
364bool ggml_et_op_rms_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
365bool ggml_et_op_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
366bool ggml_et_op_l2_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
367bool ggml_et_op_group_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
368bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
369bool ggml_et_op_softmax(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
370bool ggml_et_op_im2col(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
371bool ggml_et_op_conv_2d(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
372bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
373bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
374bool ggml_et_op_set_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
375bool ggml_et_op_cont(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
376bool ggml_et_op_concat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
377bool ggml_et_op_repeat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
378bool ggml_et_op_rwkv_wkv6(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
379bool ggml_et_op_rwkv_wkv7(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
380bool ggml_et_op_cpy(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
381bool ggml_et_op_gated_delta_net(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
382bool ggml_et_op_elmap(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
383bool ggml_et_op_fill(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
384bool ggml_et_op_diag(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
385bool ggml_et_op_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
386bool ggml_et_op_solve_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
387bool ggml_et_op_pad(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
388bool ggml_et_op_set(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
389bool ggml_et_op_ssm_conv(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
390bool ggml_et_op_ssm_scan(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node);
391bool ggml_et_op_rms_norm_mul(ggml_backend_et_device_context * dev_ctx,
392                             const ggml_tensor *              rms_norm_node,
393                             const ggml_tensor *              mul_node);