char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Bernard Ladenthinggml-cpu(s390x): guard VXE-only repack helpers (#28775)697805298

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.7 KiB225 linesraw
1#define GGML_COMMON_IMPL_CPP
2#define GGML_COMMON_DECL_CPP
3#include "ggml-common.h"
4#include "ggml-backend-impl.h"
5
6#include "ggml-impl.h"
7#include "ggml-cpu.h"
8#include "ggml-cpu-impl.h"
9#include "simd-mappings.h"
10#include "traits.h"
11
12#include <cmath>
13#include <cstring>
14#include <cassert>
15
16#define GGML_CPU_CLANG_WORKAROUND
17#include "../../repack.h"
18
19#define UNUSED GGML_UNUSED
20
21void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
22    assert(QK8_0 == 32);
23    assert(k % QK8_0 == 0);
24    const int nb = k / QK8_0;
25
26    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;
27
28#if defined(__VXE__) || defined(__VXE2__)
29    float32x4_t v_src[4][8];
30    float       id[4];
31
32    for (int i = 0; i < nb; i++) {
33        float32x4_t v_asrc[8];
34        float32x4_t v_amax[8];
35
36        for (int row_iter = 0; row_iter < 4; row_iter++) {
37            for (int j = 0; j < 8; j++) v_src[row_iter][j] = vec_xl(0, x + row_iter * k + i * 32 + 4 * j);
38            for (int j = 0; j < 8; j++) v_asrc[j] = vec_abs(v_src[row_iter][j]);
39
40            for (int j = 0; j < 4; j++) v_amax[2 * j] = vec_max(v_asrc[2 * j], v_asrc[2 * j + 1]);
41            for (int j = 0; j < 2; j++) v_amax[4 * j] = vec_max(v_amax[4 * j], v_amax[4 * j + 2]);
42            for (int j = 0; j < 1; j++) v_amax[8 * j] = vec_max(v_amax[8 * j], v_amax[8 * j + 4]);
43
44            const float amax = MAX(MAX(vec_extract(v_amax[0], 0), vec_extract(v_amax[0], 1)),
45                                   MAX(vec_extract(v_amax[0], 2), vec_extract(v_amax[0], 3)));
46
47            const float d = amax / ((1 << 7) - 1);
48            id[row_iter] = d ? 1.0f / d : 0.0f;
49
50            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);
51        }
52
53        for (int j = 0; j < 8; j++) {
54            /* Uses non-default rounding for vec_signed or vec_round */
55            const int32x4_t v_qs0 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[0][j], id[0]), 4, 1));
56            const int32x4_t v_qs1 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[1][j], id[1]), 4, 1));
57            const int32x4_t v_qs2 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[2][j], id[2]), 4, 1));
58            const int32x4_t v_qs3 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[3][j], id[3]), 4, 1));
59
60            const int16x8_t v_qs01 = vec_packs(v_qs0, v_qs1);
61            const int16x8_t v_qs23 = vec_packs(v_qs2, v_qs3);
62
63            vec_xst(vec_packs(v_qs01, v_qs23), 0, y[i].qs + 16 * j);
64        }
65    }
66#else
67    UNUSED(nb);
68    UNUSED(y);
69    ggml_quantize_mat_q8_0_4x4_generic(x, vy, k);
70#endif
71}
72
73#if defined(__VXE__) || defined(__VXE2__)
74static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) {
75    return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc));
76}
77
78static inline int8x16_t vxe_splat_granule(const int8_t * qs) {
79    uint32_t g;
80    memcpy(&g, qs, sizeof(g));
81    return (int8x16_t)vec_splats(g);
82}
83
84static inline int32x4_t vxe_fold(const int16x8_t v_sumi) {
85    const int16x8_t v_ones = vec_splats((int16_t)1);
86    return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones));
87}
88#endif
89
90void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
91    const int qk = QK8_0;
92    const int nb = n / qk;
93    const int ncols_interleaved = 4;
94
95    assert(nr == 1);
96    assert(n % qk == 0);
97    assert(nc % ncols_interleaved == 0);
98
99    UNUSED(bs);
100    UNUSED(nr);
101
102#if defined(__VXE__) || defined(__VXE2__)
103    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
104    float * res_ptr = s;
105
106    for (int x = 0; x < nc / ncols_interleaved; x++) {
107        const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
108
109        float32x4_t v_sumf = vec_splats(0.0f);
110
111        for (int l = 0; l < nb; l++) {
112            const int8_t * x_qs = b_ptr[l].qs;
113
114            const int8x16_t v_x0 = vec_xl( 0, x_qs);
115            const int8x16_t v_x1 = vec_xl(16, x_qs);
116            const int8x16_t v_x2 = vec_xl(32, x_qs);
117            const int8x16_t v_x3 = vec_xl(48, x_qs);
118
119            const int8x16_t v_x0l = vec_sra(vec_sl(v_x0, 4), 4);
120            const int8x16_t v_x1l = vec_sra(vec_sl(v_x1, 4), 4);
121            const int8x16_t v_x2l = vec_sra(vec_sl(v_x2, 4), 4);
122            const int8x16_t v_x3l = vec_sra(vec_sl(v_x3, 4), 4);
123
124            const int8x16_t v_x0h = vec_sra(v_x0, 4);
125            const int8x16_t v_x1h = vec_sra(v_x1, 4);
126            const int8x16_t v_x2h = vec_sra(v_x2, 4);
127            const int8x16_t v_x3h = vec_sra(v_x3, 4);
128
129            const int8_t * y_lo = a_ptr[l].qs;
130            const int8_t * y_hi = y_lo + qk / 2;
131
132            int16x8_t v_sumi = vec_splats((int16_t)0);
133
134            v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo +  0), v_sumi);
135            v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo +  4), v_sumi);
136            v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo +  8), v_sumi);
137            v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi);
138
139            v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi +  0), v_sumi);
140            v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi +  4), v_sumi);
141            v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi +  8), v_sumi);
142            v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi);
143
144            const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d));
145            const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);
146            const float32x4_t v_d  = vec_mul(v_yd, v_xd);
147
148            v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf);
149        }
150
151        vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved);
152    }
153#else
154    UNUSED(nb);
155    UNUSED(ncols_interleaved);
156    ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
157#endif
158}
159
160void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
161    const int qk = QK8_0;
162    const int nb = n / qk;
163    const int ncols_interleaved = 4;
164
165    assert(nr % 4 == 0);
166    assert(n % qk == 0);
167    assert(nc % ncols_interleaved == 0);
168
169#if defined(__VXE__) || defined(__VXE2__)
170    for (int y = 0; y < nr / 4; y++) {
171        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
172
173        for (int x = 0; x < nc / ncols_interleaved; x++) {
174            const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
175
176            float32x4_t v_sumf[4];
177            for (int m = 0; m < 4; m++) {
178                v_sumf[m] = vec_splats(0.0f);
179            }
180
181            for (int l = 0; l < nb; l++) {
182                int16x8_t v_sumi0 = vec_splats((int16_t)0);
183                int16x8_t v_sumi1 = vec_splats((int16_t)0);
184                int16x8_t v_sumi2 = vec_splats((int16_t)0);
185                int16x8_t v_sumi3 = vec_splats((int16_t)0);
186
187                for (int k = 0; k < 4; k++) {
188                    const int8x16_t v_x  = vec_xl(0, b_ptr[l].qs + 16 * k);
189                    const int8x16_t v_xl = vec_sra(vec_sl(v_x, 4), 4);
190                    const int8x16_t v_xh = vec_sra(v_x, 4);
191
192                    const int8_t * y_lo = a_ptr[l].qs + 16 * k;
193                    const int8_t * y_hi = y_lo + qk / 2 * 4;
194
195                    v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  0), v_sumi0);
196                    v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  4), v_sumi1);
197                    v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  8), v_sumi2);
198                    v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3);
199
200                    v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  0), v_sumi0);
201                    v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  4), v_sumi1);
202                    v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  8), v_sumi2);
203                    v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3);
204                }
205
206                const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d);
207                const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);
208
209                v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]);
210                v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]);
211                v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]);
212                v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]);
213            }
214
215            for (int m = 0; m < 4; m++) {
216                vec_xst(v_sumf[m], 0, s + (y * 4 + m) * bs + x * ncols_interleaved);
217            }
218        }
219    }
220#else
221    UNUSED(nb);
222    UNUSED(ncols_interleaved);
223    ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
224#endif
225}