char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

alex-spacemitggml-cpu: Add IME2 Instruction Support for the SpacemiT Backend (#22863)81b0d882a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.8 KiB189 linesraw
1#pragma once
2
3#include <cassert>
4#include <cstddef>
5#include <functional>
6
7namespace spacemit_kernels {
8
9#define BLOCK_QNK_LEN 256
10
11template <int N> struct nrow_block_q2_k {
12    // [4bit scale + 4bit zp] * N * 16
13    uint8_t  scales[N * BLOCK_QNK_LEN / 16];
14    // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63]
15    // [b64, b80, b96, b112] ...[b79, b95, b111, b127]
16    // [b128, b144, b160, b176] ...[b143, b159, b175, b191]
17    // [b192, b208, b224, b240] ...[b207, b223, b239, b255]
18    uint8_t  qs[N * BLOCK_QNK_LEN / 4];
19    uint16_t scales16[N];
20    uint16_t zeros16[N];
21};
22
23template <int N> struct nrow_block_q3_k {
24    // [8bit scale] * N * 16
25    int8_t   scales[N * 16];
26    // [b0, b1, b2, b3, b4, b5, b6, b7] ... [b248, b249, b250, b251, b252, b253, b254, b255]
27    uint8_t  hmask[N * BLOCK_QNK_LEN / 8];
28    // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63]
29    // [b64, b80, b96, b112] ...[b79, b95, b111, b127]
30    // [b128, b144, b160, b176] ...[b143, b159, b175, b191]
31    // [b192, b208, b224, b240] ...[b207, b223, b239, b255]
32    uint8_t  qs[N * BLOCK_QNK_LEN / 4];
33    uint16_t scales16[N];
34};
35
36template <int N> struct nrow_block_mxfp4 {
37    uint8_t e[N];
38    uint8_t qh[4 * N];
39    uint8_t qs[16 * N];
40};
41
42template <int N> struct __attribute__((packed)) nrow_block_q5_1 {
43    uint16_t scales16[N];
44    uint8_t  zp[N];
45    // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] ....
46    uint8_t  qh[4 * N];
47    // n0 [b0, b1], [b2, b3] ....  [b30, b31]
48    // n1 [b0, b1], [b2, b3] ....  [b30, b31]
49    uint8_t  qs[16 * N];
50};
51
52static_assert(sizeof(nrow_block_q5_1<1>) == sizeof(uint8_t) + 22, "wrong nrow_block_q5_1 block size/padding");
53
54template <int N> struct __attribute__((packed)) nrow_block_q5_0 {
55    uint16_t scales16[N];
56    // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] ....
57    uint8_t  qh[4 * N];
58    // n0 [b0, b1], [b2, b3] ....  [b30, b31]
59    // n1 [b0, b1], [b2, b3] ....  [b30, b31]
60    uint8_t  qs[16 * N];
61};
62
63static_assert(sizeof(nrow_block_q5_0<1>) == 22, "wrong nrow_block_q5_0 block size/padding");
64
65using gemm_kernel_quantize_def = std::function<
66    size_t(size_t, const uint8_t *, const uint8_t *, const uint8_t *, float *, size_t, size_t, size_t, size_t)>;
67
68using moe_gemm_kernel_quantize_def = std::function<
69    size_t(size_t, const uint8_t **, const uint8_t *, const uint8_t *, float **, size_t, size_t, size_t, size_t)>;
70
71namespace ime1 {
72size_t gemm_kernel_i8i4(size_t          blk_len,
73                        const uint8_t * quant_a_ptr,
74                        const uint8_t * quant_b_data,
75                        const uint8_t * quant_b_zp,
76                        float *         c_ptr,
77                        size_t          count_m,
78                        size_t          count_n,
79                        size_t          k_blks,
80                        size_t          ldc);
81
82void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
83
84void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
85
86}  // namespace ime1
87
88namespace ime2 {
89size_t gemm_kernel_i8i2k(size_t          blk_len,
90                         const uint8_t * quant_a_ptr,
91                         const uint8_t * quant_b_data,
92                         const uint8_t * quant_b_zp,
93                         float *         c_ptr,
94                         size_t          count_m,
95                         size_t          count_n,
96                         size_t          k_blks,
97                         size_t          ldc);
98
99size_t gemm_kernel_i8i3k(size_t          blk_len,
100                         const uint8_t * quant_a_ptr,
101                         const uint8_t * quant_b_data,
102                         const uint8_t * quant_b_zp,
103                         float *         c_ptr,
104                         size_t          count_m,
105                         size_t          count_n,
106                         size_t          k_blks,
107                         size_t          ldc);
108
109size_t gemm_kernel_i8i4(size_t          blk_len,
110                        const uint8_t * quant_a_ptr,
111                        const uint8_t * quant_b_data,
112                        const uint8_t * quant_b_zp,
113                        float *         c_ptr,
114                        size_t          count_m,
115                        size_t          count_n,
116                        size_t          k_blks,
117                        size_t          ldc);
118
119size_t gemm_kernel_i8i4_hp(size_t          blk_len,
120                           const uint8_t * quant_a_ptr,
121                           const uint8_t * quant_b_data,
122                           const uint8_t * quant_b_zp,
123                           float *         c_ptr,
124                           size_t          count_m,
125                           size_t          count_n,
126                           size_t          k_blks,
127                           size_t          ldc);
128
129size_t moe_m2_gemm_kernel_i8i4(size_t           blk_len,
130                               const uint8_t ** quant_a_ptr,
131                               const uint8_t *  quant_b_data,
132                               const uint8_t *  quant_b_zp,
133                               float **         c_ptr,
134                               size_t           count_m,
135                               size_t           count_n,
136                               size_t           k_blks,
137                               size_t           ldc);
138
139size_t gemm_kernel_i8i8(size_t          blk_len,
140                        const uint8_t * quant_a_ptr,
141                        const uint8_t * quant_b_data,
142                        const uint8_t * quant_b_zp,
143                        float *         c_ptr,
144                        size_t          count_m,
145                        size_t          count_n,
146                        size_t          k_blks,
147                        size_t          ldc);
148
149size_t gemm_kernel_i8mxfp4(size_t          blk_len,
150                           const uint8_t * quant_a_ptr,
151                           const uint8_t * quant_b_data,
152                           const uint8_t * quant_b_zp,
153                           float *         c_ptr,
154                           size_t          count_m,
155                           size_t          count_n,
156                           size_t          k_blks,
157                           size_t          ldc);
158
159size_t moe_m2_gemm_kernel_i8mxfp4(size_t           blk_len,
160                                  const uint8_t ** quant_a_ptr,
161                                  const uint8_t *  quant_b_data,
162                                  const uint8_t *  quant_b_zp,
163                                  float **         c_ptr,
164                                  size_t           count_m,
165                                  size_t           count_n,
166                                  size_t           k_blks,
167                                  size_t           ldc);
168
169size_t gemm_kernel_i8i5(size_t          blk_len,
170                        const uint8_t * quant_a_ptr,
171                        const uint8_t * quant_b_data,
172                        const uint8_t * quant_b_zp,
173                        float *         c_ptr,
174                        size_t          count_m,
175                        size_t          count_n,
176                        size_t          k_blks,
177                        size_t          ldc);
178
179size_t moe_m2_gemm_kernel_i8i5(size_t           blk_len,
180                               const uint8_t ** quant_a_ptr,
181                               const uint8_t *  quant_b_data,
182                               const uint8_t *  quant_b_zp,
183                               float **         c_ptr,
184                               size_t           count_m,
185                               size_t           count_n,
186                               size_t           k_blks,
187                               size_t           ldc);
188}  // namespace ime2
189}  // namespace spacemit_kernels