char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aaron Teotests(s390x): add non-vxe build to tests (#28776)be2c6d7d1

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
55.7 KiB1534 linesraw
1#define GGML_COMMON_IMPL_C
2#include "ggml-common.h"
3#include "ggml-quants.h"
4#include "ggml-impl.h"
5#include "ggml-cpu.h"
6#include "simd-mappings.h"
7
8#include "../../quants.h"
9#include "../../ggml-cpu-impl.h"
10
11#include <math.h>
12#include <string.h>
13#include <assert.h>
14#include <float.h>
15#include <stdlib.h> // for qsort
16#include <stdio.h>  // for GGML_ASSERT
17
18#define GROUP_MAX_EPS 1e-15f
19#define GROUP_MAX_EPS_IQ3_XXS 1e-8f
20#define GROUP_MAX_EPS_IQ2_S 1e-8f
21#define GROUP_MAX_EPS_IQ1_M 1e-7f
22#define GROUP_MAX_EPS_IQ1_S 1e-12f
23
24#define UNUSED GGML_UNUSED
25
26#if defined(__VXE__) || defined(__VXE2__)
27#define B1(c,s,n)  0x ## n ## c ,  0x ## n ## s
28#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)
29#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)
30#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)
31#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)
32#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)
33#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)
34#define B8(c,s  ) B7(c,s,     c), B7(c,s,     s)
35
36// precomputed tables for expanding 8bits to 8 bytes:
37static const __attribute__((aligned(16))) uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b ) << 4
38static const __attribute__((aligned(16))) uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4
39
40// permute mask for byteswapping
41static const uint8x16_t v_kperm = (const uint8x16_t){
42     7,  6,  5,  4,  3,  2, 1, 0,
43    15, 14, 13, 12, 11, 10, 9, 8
44};
45#endif
46
47void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
48    assert(QK8_0 == 32);
49    assert(k % QK8_0 == 0);
50    const int nb = k / QK8_0;
51
52    block_q8_0 * GGML_RESTRICT y = vy;
53
54#if defined(__VXE__) || defined(__VXE2__)
55    for (int i = 0; i < nb; i++) {
56        float32x4_t srcv [8];
57        float32x4_t asrcv[8];
58        float32x4_t amaxv[8];
59
60        for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);
61        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);
62        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);
63        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);
64        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);
65
66        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),
67                                   vec_extract(amaxv[0], 1)),
68                               MAX(vec_extract(amaxv[0], 2),
69                                   vec_extract(amaxv[0], 3)));
70
71        const float d = amax / ((1 << 7) - 1);
72        const float id = d ? 1.0f / d : 0.0f;
73
74        y[i].d = GGML_CPU_FP32_TO_FP16(d);
75
76        for (int j = 0; j < 8; j++) {
77            const float32x4_t v = vec_mul(srcv[j], vec_splats(id));
78            /* Uses non-default rounding for vec_signed or vec_round */
79            const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));
80
81            y[i].qs[4*j + 0] = vec_extract(vi, 0);
82            y[i].qs[4*j + 1] = vec_extract(vi, 1);
83            y[i].qs[4*j + 2] = vec_extract(vi, 2);
84            y[i].qs[4*j + 3] = vec_extract(vi, 3);
85        }
86    }
87#else
88    GGML_UNUSED(nb);
89    // scalar
90    quantize_row_q8_0_ref(x, y, k);
91#endif
92}
93
94void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
95    assert(k % QK8_1 == 0);
96    const int nb = k / QK8_1;
97
98    block_q8_1 * GGML_RESTRICT y = vy;
99
100#if defined(__VXE__) || defined(__VXE2__)
101    for (int i = 0; i < nb; i++) {
102        float32x4_t srcv [8];
103        float32x4_t asrcv[8];
104        float32x4_t amaxv[8];
105
106        for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);
107        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);
108        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);
109        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);
110        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);
111
112        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),
113                                   vec_extract(amaxv[0], 1)),
114                               MAX(vec_extract(amaxv[0], 2),
115                                   vec_extract(amaxv[0], 3)));
116
117        const float d = amax / ((1 << 7) - 1);
118        const float id = d ? 1.0f / d : 0.0f;
119
120        y[i].d = GGML_CPU_FP32_TO_FP16(d);
121
122        int32x4_t acc = vec_splats(0);
123
124        for (int j = 0; j < 8; j++) {
125            const float32x4_t v = vec_mul(srcv[j], vec_splats(id));
126            /* Uses non-default rounding for vec_signed or vec_round */
127            const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));
128
129            y[i].qs[4*j + 0] = vec_extract(vi, 0);
130            y[i].qs[4*j + 1] = vec_extract(vi, 1);
131            y[i].qs[4*j + 2] = vec_extract(vi, 2);
132            y[i].qs[4*j + 3] = vec_extract(vi, 3);
133
134            acc = vec_add(acc, vi);
135        }
136
137        y[i].s = GGML_CPU_FP32_TO_FP16(d * (acc[0] + acc[1] + acc[2] + acc[3]));
138    }
139#else
140    GGML_UNUSED(nb);
141    // scalar
142    quantize_row_q8_1_ref(x, y, k);
143#endif
144}
145
146
147//===================================== Dot products =================================
148
149void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
150    const int qk = QK1_0;  // 128
151    const int nb = n / qk;
152
153    assert(n % qk == 0);
154    assert(nrc == 1);
155    UNUSED(nrc);
156    UNUSED(bx);
157    UNUSED(by);
158    UNUSED(bs);
159
160    const block_q1_0 * GGML_RESTRICT x = vx;
161    const block_q8_0 * GGML_RESTRICT y = vy;
162
163#if defined(__VXE__) || defined(__VXE2__)
164    float32x4_t v_sumf = vec_splats(0.0f);
165
166    const uint8x16_t v_zero = vec_splats((uint8_t)0x00);  // zero
167    const uint8x16_t v_bias = vec_splats((uint8_t)0x80);  // bias from signed to unsigned
168                                                          // v ^ 0x80 == v + 128
169
170    const uint8x16_t v_idx = (const uint8x16_t){ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
171    const uint8x16_t v_bit = (const uint8x16_t){ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 };
172
173    for (int i = 0; i < nb; ++i) {
174        const uint8x16_t  v_x  = vec_xl(0, (const uint8_t *)x[i].qs);
175        const float32x4_t v_xd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));
176
177        for (int k = 0; k < 4; ++k) {
178            // sub-block k holds elements 32k .. 32k+31
179            const block_q8_0 * GGML_RESTRICT yb = &y[i*4 + k];
180            const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(yb->d));
181
182            const uint8x16_t v_xrl = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 0))));
183            const uint8x16_t v_xrh = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 2))));
184
185            // isolate each lane's bit, then set all ones where that bit is clear, the -d case
186            const int8x16_t v_ml = (int8x16_t)vec_cmpeq(vec_and(v_xrl, v_bit), v_zero);
187            const int8x16_t v_mh = (int8x16_t)vec_cmpeq(vec_and(v_xrh, v_bit), v_zero);
188
189            const int8x16_t v_yl = vec_xl(0,       (const int8_t *)yb->qs);
190            const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)yb->qs);
191
192            // weights are only +1 or -1, so negate y
193            const int8x16_t v_ysl = vec_sub(vec_xor(v_yl, v_ml), v_ml);
194            const int8x16_t v_ysh = vec_sub(vec_xor(v_yh, v_mh), v_mh);
195
196            // bias to unsigned, then vec_sum4 adds each group of 4 bytes into one word
197            const uint32x4_t v_p = vec_add(vec_sum4(vec_xor((uint8x16_t)v_ysl, v_bias), v_zero),
198                                           vec_sum4(vec_xor((uint8x16_t)v_ysh, v_bias), v_zero));
199
200            // each word summed 8 biased bytes, so take back 8 * 128
201            const int32x4_t v_xy = vec_sub((int32x4_t)v_p, vec_splats((int32_t)1024));
202
203            // apply both block scales and add into the running total
204            v_sumf = vec_madd(vec_float(v_xy), vec_mul(v_xd, v_yd), v_sumf);
205        }
206    }
207
208    *s = vec_hsum_f32x4(v_sumf);
209#else
210    UNUSED(nb);
211    UNUSED(x);
212    UNUSED(y);
213    ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
214#endif
215}
216
217void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
218    const int qk = QK8_0;
219    const int nb = n / qk;
220
221    assert(n % qk == 0);
222    assert(nrc == 1);
223    UNUSED(nrc);
224    UNUSED(bx);
225    UNUSED(by);
226    UNUSED(bs);
227
228    const block_q4_0 * GGML_RESTRICT x = vx;
229    const block_q8_0 * GGML_RESTRICT y = vy;
230
231    int ib = 0;
232    float sumf = 0;
233
234#if defined(__VXE__) || defined(__VXE2__)
235    float32x4_t acc = vec_splats(0.0f);
236
237    const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);
238    const int8x16_t  v_s = vec_splats( (const int8_t)0x08);
239
240    for (; ib < nb; ++ib) {
241        const uint8x16_t v_x = vec_xl(0, x[ib].qs);
242        const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);
243        const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);
244
245        const int8x16_t v_xls = vec_sub(v_xl, v_s);
246        const int8x16_t v_xhs = vec_sub(v_xh, v_s);
247
248        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);
249        const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);
250
251        const int16x8_t v_xylso = vec_mulo(v_xls, v_yl);
252        const int16x8_t v_xyl = vec_meadd(v_xls, v_yl, v_xylso);
253        const int16x8_t v_xyhso = vec_mulo(v_xhs, v_yh);
254        const int16x8_t v_xyh = vec_meadd(v_xhs, v_yh, v_xyhso);
255
256        int16x8_t v_xy_ = v_xyl + v_xyh; v_xy_ += vec_reve(v_xy_);
257
258        const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_));
259        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
260
261        acc = vec_madd(v_xy, v_d, acc);
262    }
263
264    sumf = vec_hsum_f32x4(acc);
265    *s = sumf;
266#else
267    UNUSED(nb);
268    UNUSED(x);
269    UNUSED(y);
270    UNUSED(ib);
271    UNUSED(sumf);
272    ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
273#endif
274}
275
276void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
277    const int qk = QK8_1;
278    const int nb = n / qk;
279
280    assert(n % qk == 0);
281    assert(nrc == 1);
282    UNUSED(nrc);
283    UNUSED(bx);
284    UNUSED(by);
285    UNUSED(bs);
286
287    const block_q4_1 * GGML_RESTRICT x = vx;
288    const block_q8_1 * GGML_RESTRICT y = vy;
289
290    int ib = 0;
291    float sumf = 0;
292
293#if defined(__VXE__) || defined(__VXE2__)
294    float summs = 0;
295    float32x4_t acc = vec_splats(0.0f);
296
297    const uint8x16_t v_m = vec_splat_u8(0x0F);
298
299#pragma GCC unroll 4
300    for (; ib < nb; ++ib) {
301        __builtin_prefetch(x[ib].qs, 0, 1);
302        __builtin_prefetch(y[ib].qs, 0, 1);
303
304        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);
305
306        const uint8x16_t v_x = vec_xl(0, x[ib].qs);
307        const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);
308        const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);
309
310        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);
311        const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs);
312
313        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
314        const float32x4_t v_xy = vec_float(v_xy_);
315
316        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
317
318        acc = vec_madd(v_xy, v_d, acc);
319    }
320
321    sumf = vec_hsum_f32x4(acc) + summs;
322    *s = sumf;
323#else
324    UNUSED(nb);
325    UNUSED(x);
326    UNUSED(y);
327    UNUSED(ib);
328    UNUSED(sumf);
329    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
330#endif
331}
332
333void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
334    assert(nrc == 1);
335    UNUSED(nrc);
336    UNUSED(bx);
337    UNUSED(by);
338    UNUSED(bs);
339    assert(n % QK_MXFP4 == 0);
340    static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same");
341
342    const int qk = QK_MXFP4;
343    const int nb = n / qk;
344
345    const block_mxfp4 * GGML_RESTRICT x = vx;
346    const block_q8_0  * GGML_RESTRICT y = vy;
347
348    int ib = 0;
349    float sumf = 0.0f;
350
351#if defined(__VXE__) || defined(__VXE2__)
352    const int8x16_t  v_k = vec_xl(0, kvalues_mxfp4);
353    const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);
354
355    float32x4_t v_acc = vec_splats(0.0f);
356
357    #pragma GCC unroll 8
358    for (; ib + 1 < nb; ib += 2) {
359        const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];
360        const block_mxfp4 * GGML_RESTRICT x1 = &x[ib + 1];
361        const block_q8_0  * GGML_RESTRICT y0 = &y[ib + 0];
362        const block_q8_0  * GGML_RESTRICT y1 = &y[ib + 1];
363
364        const uint8x16_t v_x0 = vec_xl(0, x0->qs);
365        const uint8x16_t v_x1 = vec_xl(0, x1->qs);
366
367        int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);
368        int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);
369        int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);
370        int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);
371
372        v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);
373        v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);
374        v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);
375        v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);
376
377        const int8x16_t v_y0l = vec_xl(0,       y0->qs);
378        const int8x16_t v_y0h = vec_xl(QK8_0/2, y0->qs);
379        const int8x16_t v_y1l = vec_xl(0,       y1->qs);
380        const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs);
381
382        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);
383        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);
384
385        const float32x4_t v_xy0f = vec_float(v_xy0);
386        const float32x4_t v_xy1f = vec_float(v_xy1);
387
388        const float32x4_t v_d0 = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));
389        const float32x4_t v_d1 = vec_splats(GGML_E8M0_TO_FP32_HALF(x1->e) * GGML_CPU_FP16_TO_FP32(y1->d));
390
391        v_acc = vec_madd(v_xy0f, v_d0, v_acc);
392        v_acc = vec_madd(v_xy1f, v_d1, v_acc);
393    }
394
395    for (; ib < nb; ++ib) {
396        const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];
397        const block_q8_0  * GGML_RESTRICT y0 = &y[ib + 0];
398
399        const uint8x16_t v_x = vec_xl(0, x0->qs);
400
401        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);
402        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);
403
404        v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);
405        v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);
406
407        const int8x16_t v_yl = vec_xl(0,       y0->qs);
408        const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);
409
410        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
411        const float32x4_t v_xyf = vec_float(v_xy);
412
413        const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));
414        v_acc = vec_madd(v_xyf, v_d, v_acc);
415    }
416
417    sumf = vec_hsum_f32x4(v_acc);
418    *s = sumf;
419#else
420    UNUSED(nb);
421    UNUSED(x);
422    UNUSED(y);
423    UNUSED(ib);
424    UNUSED(sumf);
425    ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
426#endif
427}
428
429void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
430    const int qk = QK8_0;
431    const int nb = n / qk;
432
433    assert(n % qk == 0);
434    assert(qk == QK5_0);
435    assert(nrc == 1);
436    UNUSED(nrc);
437    UNUSED(bx);
438    UNUSED(by);
439    UNUSED(bs);
440
441    const block_q5_0 * GGML_RESTRICT x = vx;
442    const block_q8_0 * GGML_RESTRICT y = vy;
443
444    int ib = 0;
445    float sumf = 0.0f;
446
447#if defined(__VXE__) || defined(__VXE2__)
448    float32x4_t v_sum0 = vec_splats(0.0f);
449    float32x4_t v_sum1 = vec_splats(0.0f);
450
451    uint32_t qh0, qh1;
452    uint64_t tmp0[4], tmp1[4];
453
454    const uint8x16_t v_m = vec_splats((uint8_t)0x0F);
455
456    #pragma GCC unroll 4
457    for (; ib + 1 < nb; ib += 2) {
458        const block_q5_0 * GGML_RESTRICT x0 = &x[ib + 0];
459        const block_q5_0 * GGML_RESTRICT x1 = &x[ib + 1];
460        const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];
461        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];
462
463        memcpy(&qh0, x0->qh, sizeof(qh0));
464        memcpy(&qh1, x1->qh, sizeof(qh1));
465
466        tmp0[0] = table_b2b_1[(qh0 >>  0) & 0xFF];
467        tmp0[1] = table_b2b_1[(qh0 >>  8) & 0xFF];
468        tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];
469        tmp0[3] = table_b2b_1[(qh0 >> 24)       ];
470
471        tmp1[0] = table_b2b_1[(qh1 >>  0) & 0xFF];
472        tmp1[1] = table_b2b_1[(qh1 >>  8) & 0xFF];
473        tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];
474        tmp1[3] = table_b2b_1[(qh1 >> 24)       ];
475
476        int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));
477        int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));
478        int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));
479        int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));
480
481        // required for fixing the byteorder
482        v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);
483        v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);
484        v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);
485        v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);
486
487        const uint8x16_t v_x0 = vec_xl(0, (const uint8_t *)x0->qs);
488        const uint8x16_t v_x1 = vec_xl(0, (const uint8_t *)x1->qs);
489
490        int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);
491        int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);
492        int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);
493        int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);
494
495        const int8x16_t v_x0lf = vec_sub(v_x0l, v_qh0l);
496        const int8x16_t v_x0hf = vec_sub(v_x0h, v_qh0h);
497        const int8x16_t v_x1lf = vec_sub(v_x1l, v_qh1l);
498        const int8x16_t v_x1hf = vec_sub(v_x1h, v_qh1h);
499
500        const int8x16_t v_y0l = vec_xl(0,       (const int8_t *)y0->qs);
501        const int8x16_t v_y0h = vec_xl(QK8_0/2, (const int8_t *)y0->qs);
502        const int8x16_t v_y1l = vec_xl(0,       (const int8_t *)y1->qs);
503        const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs);
504
505        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
506        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
507
508        const float32x4_t v_xy0f = vec_float(v_xy0);
509        const float32x4_t v_xy1f = vec_float(v_xy1);
510
511        const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
512        const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));
513
514        v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);
515        v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);
516    }
517
518    sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1);
519
520    #pragma GCC unroll 4
521    for (; ib < nb; ++ib) {
522        const block_q5_0 * GGML_RESTRICT x0 = &x[ib];
523        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];
524
525        uint32_t qh;
526        memcpy(&qh, x0->qh, sizeof(qh));
527
528        uint64_t tmp[4];
529        tmp[0] = table_b2b_1[(qh >>  0) & 0xFF];
530        tmp[1] = table_b2b_1[(qh >>  8) & 0xFF];
531        tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];
532        tmp[3] = table_b2b_1[(qh >> 24)       ];
533
534        int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));
535        int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));
536
537        // required for fixing the byteorder
538        v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);
539        v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);
540
541        const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x0->qs);
542        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);
543        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);
544
545        const int8x16_t v_xlf = vec_sub(v_xl, v_qhl);
546        const int8x16_t v_xhf = vec_sub(v_xh, v_qhh);
547
548        const int8x16_t v_yl = vec_xl(0,       (const int8_t *)y0->qs);
549        const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs);
550
551        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
552        const float32x4_t v_xyf = vec_float(v_xy);
553
554        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
555        const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f));
556
557        sumf += vec_hsum_f32x4(v_acc);
558    }
559
560    *s = sumf;
561#else
562    UNUSED(nb);
563    UNUSED(x);
564    UNUSED(y);
565    UNUSED(ib);
566    UNUSED(sumf);
567    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
568#endif
569}
570
571void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
572    const int qk = QK8_1;
573    const int nb = n / qk;
574
575    assert(n % qk == 0);
576    assert(qk == QK5_1);
577    assert(nrc == 1);
578    UNUSED(nrc);
579    UNUSED(bx);
580    UNUSED(by);
581    UNUSED(bs);
582
583    const block_q5_1 * GGML_RESTRICT x = vx;
584    const block_q8_1 * GGML_RESTRICT y = vy;
585
586    int ib = 0;
587    float sumf = 0.0f;
588
589#if defined(__VXE__) || defined(__VXE2__)
590    float32x4_t v_sum0 = vec_splats(0.0f);
591    float32x4_t v_sum1 = vec_splats(0.0f);
592
593    float summs0 = 0.0f;
594    float summs1 = 0.0f;
595
596    uint32_t qh0;
597    uint32_t qh1;
598
599    uint64_t tmp0[4];
600    uint64_t tmp1[4];
601
602    const uint8x16_t v_m = vec_splats((uint8_t)0x0F);
603
604    #pragma GCC unroll 4
605    for (; ib + 1 < nb; ib += 2) {
606        const block_q5_1 * GGML_RESTRICT x0 = &x[ib + 0];
607        const block_q5_1 * GGML_RESTRICT x1 = &x[ib + 1];
608        const block_q8_1 * GGML_RESTRICT y0 = &y[ib + 0];
609        const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];
610
611        summs0 += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);
612        summs1 += GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);
613
614        memcpy(&qh0, x0->qh, sizeof(qh0));
615        memcpy(&qh1, x1->qh, sizeof(qh1));
616
617        tmp0[0] = table_b2b_0[(qh0 >>  0) & 0xFF];
618        tmp0[1] = table_b2b_0[(qh0 >>  8) & 0xFF];
619        tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];
620        tmp0[3] = table_b2b_0[(qh0 >> 24)       ];
621
622        tmp1[0] = table_b2b_0[(qh1 >>  0) & 0xFF];
623        tmp1[1] = table_b2b_0[(qh1 >>  8) & 0xFF];
624        tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];
625        tmp1[3] = table_b2b_0[(qh1 >> 24)       ];
626
627        int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));
628        int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));
629        int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));
630        int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));
631
632        // required for fixing the byteorder
633        v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);
634        v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);
635        v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);
636        v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);
637
638        const uint8x16_t v_x0 = vec_xl(0, x0->qs);
639        const uint8x16_t v_x1 = vec_xl(0, x1->qs);
640
641        const int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);
642        const int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);
643        const int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);
644        const int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);
645
646        const int8x16_t v_x0lf = vec_or(v_x0l, v_qh0l);
647        const int8x16_t v_x0hf = vec_or(v_x0h, v_qh0h);
648        const int8x16_t v_x1lf = vec_or(v_x1l, v_qh1l);
649        const int8x16_t v_x1hf = vec_or(v_x1h, v_qh1h);
650
651        const int8x16_t v_y0l = vec_xl(0      , y0->qs);
652        const int8x16_t v_y0h = vec_xl(QK8_1/2, y0->qs);
653        const int8x16_t v_y1l = vec_xl(0      , y1->qs);
654        const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs);
655
656        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
657        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
658
659        const float32x4_t v_xy0f = vec_float(v_xy0);
660        const float32x4_t v_xy1f = vec_float(v_xy1);
661
662        const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
663        const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));
664
665        v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);
666        v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);
667    }
668
669    sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1) + summs0 + summs1;
670
671    #pragma GCC unroll 4
672    for (; ib < nb; ++ib) {
673        const block_q5_1 * GGML_RESTRICT x0 = &x[ib];
674        const block_q8_1 * GGML_RESTRICT y0 = &y[ib];
675
676        float summs = GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);
677
678        uint32_t qh;
679        memcpy(&qh, x0->qh, sizeof(qh));
680
681        uint64_t tmp[4];
682        tmp[0] = table_b2b_0[(qh >>  0) & 0xFF];
683        tmp[1] = table_b2b_0[(qh >>  8) & 0xFF];
684        tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];
685        tmp[3] = table_b2b_0[(qh >> 24)       ];
686
687        int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));
688        int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));
689
690        // required for fixing the byteorder
691        v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);
692        v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);
693
694        const uint8x16_t v_x = vec_xl(0, x0->qs);
695        const int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);
696        const int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);
697
698        const int8x16_t v_xlf = vec_or(v_xl, v_qhl);
699        const int8x16_t v_xhf = vec_or(v_xh, v_qhh);
700
701        const int8x16_t v_yl = vec_xl(0      , y0->qs);
702        const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs);
703
704        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
705        const float32x4_t v_xyf = vec_float(v_xy);
706
707        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
708        const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f));
709
710        sumf += vec_hsum_f32x4(v_acc) + summs;
711    }
712
713    *s = sumf;
714#else
715    UNUSED(nb);
716    UNUSED(x);
717    UNUSED(y);
718    UNUSED(ib);
719    UNUSED(sumf);
720    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
721#endif
722}
723
724void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
725    const int qk = QK8_0;
726    const int nb = n / qk;
727
728    assert(n % qk == 0);
729    assert(nrc == 1);
730    UNUSED(nrc);
731    UNUSED(bx);
732    UNUSED(by);
733    UNUSED(bs);
734
735    const block_q8_0 * GGML_RESTRICT x = vx;
736    const block_q8_0 * GGML_RESTRICT y = vy;
737
738    int ib = 0;
739    float sumf = 0;
740
741#if defined(__VXE__) || defined(__VXE2__)
742    float32x4_t acc = vec_splats(0.0f);
743
744#pragma GCC unroll 8
745    for (; ib < nb; ++ib) {
746        __builtin_prefetch(x[ib].qs, 0, 1);
747        __builtin_prefetch(y[ib].qs, 0, 1);
748
749        const int8x16_t v_xl = vec_xl(0      , x[ib].qs);
750        const int8x16_t v_xh = vec_xl(QK8_0/2, x[ib].qs);
751        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);
752        const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);
753
754        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
755        const float32x4_t v_xy = vec_float(v_xy_);
756        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
757
758        acc = vec_madd(v_xy, v_d, acc);
759    }
760
761    sumf = vec_hsum_f32x4(acc);
762
763    *s = sumf;
764#else
765    UNUSED(nb);
766    UNUSED(x);
767    UNUSED(y);
768    UNUSED(ib);
769    UNUSED(sumf);
770    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
771#endif
772}
773
774void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
775    assert(n % QK_K == 0);
776    assert(nrc == 1);
777    UNUSED(nrc);
778    UNUSED(bx);
779    UNUSED(by);
780    UNUSED(bs);
781
782    const uint32_t kmask1 = 0x03030303;
783    const uint32_t kmask2 = 0x0f0f0f0f;
784
785    const block_q3_K * GGML_RESTRICT x = vx;
786    const block_q8_K * GGML_RESTRICT y = vy;
787
788    const int nb = n / QK_K;
789
790#if defined(__VXE__) || defined(__VXE2__)
791    uint32_t aux[3];
792    uint32_t utmp[4];
793
794    const int32x4_t v_z = vec_splat_s32(0);
795    const uint8x16_t v_3m = vec_splat_u8(0x03);
796
797    const uint8x16_t v_0c = vec_splat_u8(1);
798    const uint8x16_t v_1c = vec_sl(v_0c, 1);
799    const uint8x16_t v_2c = vec_sl(v_0c, 2);
800    const uint8x16_t v_3c = vec_sl(v_0c, 3);
801
802    uint8x16_t q3h[4];
803    uint8x16_t q3b[2];
804    int8x16_t q3bytes[4];
805    int8x16_t q8bytes[8];
806    uint8x16_t qhbits[2];
807
808    float sum = 0;
809
810    for (int i = 0; i < nb; ++i) {
811        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
812
813        const uint8_t * restrict x0l = x[i].qs;
814        const uint8_t * restrict x0h = x[i].hmask;
815        const int8_t  * restrict y0  = y[i].qs;
816
817        qhbits[0] = vec_xl(0 , x0h);
818        qhbits[1] = vec_xl(16, x0h);
819
820        int32_t isum = 0;
821
822        memcpy(aux, x[i].scales, 12);
823        utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
824        utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
825        utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
826        utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
827
828        int8_t * scale = (int8_t *)utmp;
829        for (int j = 0; j < 16; ++j) scale[j] -= 32;
830
831        for (int j = 0; j < QK_K/128; ++j) {
832            int32x4_t isum0, isum1, isum2, isum3;
833
834            q3b[0] = vec_xl(0 , x0l);
835            q3b[1] = vec_xl(16, x0l);
836            x0l += 32;
837
838            q8bytes[0] = vec_xl(0  , y0);
839            q8bytes[1] = vec_xl(16 , y0);
840            q8bytes[2] = vec_xl(32 , y0);
841            q8bytes[3] = vec_xl(48 , y0);
842            q8bytes[4] = vec_xl(64 , y0);
843            q8bytes[5] = vec_xl(80 , y0);
844            q8bytes[6] = vec_xl(96 , y0);
845            q8bytes[7] = vec_xl(112, y0);
846            y0 += 128;
847
848            q3h[0] = vec_sl(vec_andc(v_0c, qhbits[0]), 2);
849            q3h[1] = vec_sl(vec_andc(v_0c, qhbits[1]), 2);
850            q3h[2] = vec_sl(vec_andc(v_1c, qhbits[0]), 1);
851            q3h[3] = vec_sl(vec_andc(v_1c, qhbits[1]), 1);
852
853            q3bytes[0] = vec_sub((int8x16_t)vec_and(q3b[0], v_3m), (int8x16_t)q3h[0]);
854            q3bytes[1] = vec_sub((int8x16_t)vec_and(q3b[1], v_3m), (int8x16_t)q3h[1]);
855            q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]);
856            q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]);
857
858            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]);
859            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]);
860            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]);
861            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]);
862
863            isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0];
864            isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1];
865            isum += (isum2[0] + isum2[1] + isum2[2] + isum2[3]) * scale[2];
866            isum += (isum3[0] + isum3[1] + isum3[2] + isum3[3]) * scale[3];
867
868            scale += 4;
869
870            q3h[0] = vec_andc(v_2c, qhbits[0]);
871            q3h[1] = vec_andc(v_2c, qhbits[1]);
872            q3h[2] = vec_sr(vec_andc(v_3c, qhbits[0]), 1);
873            q3h[3] = vec_sr(vec_andc(v_3c, qhbits[1]), 1);
874
875            q3bytes[0] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 4), v_3m), (int8x16_t)q3h[0]);
876            q3bytes[1] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 4), v_3m), (int8x16_t)q3h[1]);
877            q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]);
878            q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]);
879
880            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]);
881            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]);
882            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]);
883            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]);
884
885            isum += vec_hsum_i32x4(isum0) * scale[0];
886            isum += vec_hsum_i32x4(isum1) * scale[1];
887            isum += vec_hsum_i32x4(isum2) * scale[2];
888            isum += vec_hsum_i32x4(isum3) * scale[3];
889
890            scale += 4;
891
892            if (j == 0) {
893                qhbits[0] = vec_sr(qhbits[0], 4);
894                qhbits[1] = vec_sr(qhbits[1], 4);
895            }
896        }
897
898        sum += d * isum;
899    }
900
901    *s = sum;
902
903#else
904    UNUSED(kmask1);
905    UNUSED(kmask2);
906    UNUSED(x);
907    UNUSED(y);
908    UNUSED(nb);
909    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
910#endif
911}
912
913void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
914    assert(n % QK_K == 0);
915    assert(nrc == 1);
916    UNUSED(nrc);
917    UNUSED(bx);
918    UNUSED(by);
919    UNUSED(bs);
920
921    const block_q4_K * GGML_RESTRICT x = vx;
922    const block_q8_K * GGML_RESTRICT y = vy;
923
924    const int nb = n / QK_K;
925
926    static const uint32_t kmask1 = 0x3f3f3f3f;
927    static const uint32_t kmask2 = 0x0f0f0f0f;
928    static const uint32_t kmask3 = 0x03030303;
929
930    uint32_t utmp[4];
931
932#if defined(__VXE__) || defined(__VXE2__)
933    const uint8x16_t v_lm = vec_splat_u8(0x0F);
934    const int32x4_t v_z = vec_splat_s32(0);
935
936    uint8x16_t v_x[2];
937    int8x16_t  v_xl[2];
938    int8x16_t  v_y[2];
939
940    float sumf = 0;
941
942    for (int i = 0; i < nb; ++i) {
943        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
944        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
945
946        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);
947        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);
948        const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);
949
950        memcpy(utmp, x[i].scales, 12);
951
952        uint32x4_t v_mins8 = { 0 };
953        v_mins8 = vec_insert(utmp[1] & kmask1, v_mins8, 0);
954        v_mins8 = vec_insert(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1);
955
956        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
957        utmp[0] &= kmask1;
958
959        const int16x8_t v_minsh = (int16x8_t)vec_unpackh((uint8x16_t)v_mins8);
960
961        const int32x4_t v_minso = vec_mulo(v_ysums, v_minsh);
962        const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minso);
963        sumf -= dmin * (v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]);
964
965        const uint8_t * scales = (const uint8_t *)utmp;
966        const uint8_t * GGML_RESTRICT x0 = x[i].qs;
967        const int8_t  * GGML_RESTRICT y0 = y[i].qs;
968
969        int32_t sumi1 = 0;
970        int32_t sumi2 = 0;
971
972        for (int j = 0; j < QK_K/64; ++j) {
973            v_x[0] = vec_xl(0 , x0);
974            v_x[1] = vec_xl(16, x0);
975            x0 += 32;
976
977            v_y[0] = vec_xl(0 , y0);
978            v_y[1] = vec_xl(16, y0);
979            y0 += 32;
980
981            v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm);
982            v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm);
983
984            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
985            sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0];
986
987            v_y[0] = vec_xl(0 , y0);
988            v_y[1] = vec_xl(16, y0);
989            y0 += 32;
990
991            v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4);
992            v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4);
993
994            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
995            sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1];
996        }
997
998        sumf += d * (sumi1 + sumi2);
999    }
1000
1001    *s = sumf;
1002
1003#else
1004    UNUSED(x);
1005    UNUSED(y);
1006    UNUSED(nb);
1007    UNUSED(kmask1);
1008    UNUSED(kmask2);
1009    UNUSED(kmask3);
1010    UNUSED(utmp);
1011    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1012#endif
1013}
1014
1015void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {
1016    assert(n % QK_K == 0);
1017    assert(nrc == 1);
1018    UNUSED(nrc);
1019    UNUSED(bx);
1020    UNUSED(by);
1021    UNUSED(bs);
1022
1023    const block_q5_K * GGML_RESTRICT x = vx;
1024    const block_q8_K * GGML_RESTRICT y = vy;
1025
1026    const int nb = n / QK_K;
1027
1028    static const uint32_t kmask1 = 0x3f3f3f3f;
1029    static const uint32_t kmask2 = 0x0f0f0f0f;
1030    static const uint32_t kmask3 = 0x03030303;
1031
1032    uint32_t utmp[4];
1033
1034#if defined(__VXE__) || defined(__VXE2__)
1035    const uint8x16_t v_lm = vec_splat_u8(0x0F);
1036    const uint8x16_t v_1m = vec_splat_u8(0x01);
1037    const uint8x16_t v_2m = vec_splat_u8(0x02);
1038
1039    const int32x4_t v_z = vec_splat_s32(0);
1040
1041    const uchar8x16_t v_minsm = {
1042        0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
1043        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
1044    };
1045
1046    int8x16_t  q5b[4];
1047    uint8x16_t q5h[4];
1048
1049    uint8x16_t v_xl[2];
1050    uint8x16_t v_xh[2];
1051    int8x16_t  v_y[4];
1052
1053    float sumf = 0;
1054
1055    for (int i = 0; i < nb; ++i) {
1056        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1057        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
1058
1059        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);
1060        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);
1061        const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);
1062
1063        memcpy(utmp, x[i].scales, 12);
1064        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
1065        const uint32_t uaux = utmp[1] & kmask1;
1066        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
1067        utmp[2] = uaux;
1068        utmp[0] &= kmask1;
1069
1070        const uint8x16_t v_mins16 = vec_xl(0, (const uint8_t *)utmp);
1071        const uint8x16_t v_mins8 = vec_perm(v_mins16, v_mins16, v_minsm);
1072        const int16x8_t v_minsh = (int16x8_t)vec_unpackh(v_mins8);
1073
1074        const int32x4_t v_minsho = vec_mulo(v_ysums, v_minsh);
1075        const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minsho);
1076        const int32_t mins = vec_hsum_i32x4(v_mins);
1077
1078        const uint8_t * scales = (const uint8_t *)utmp;
1079        const uint8_t * GGML_RESTRICT x0l = x[i].qs;
1080        const uint8_t * GGML_RESTRICT x0h = x[i].qh;
1081        const int8_t  * GGML_RESTRICT y0 = y[i].qs;
1082
1083        v_xh[0] = vec_xl(0 , x0h);
1084        v_xh[1] = vec_xl(16, x0h);
1085
1086        int32_t sumi = 0;
1087        for (int j = 0; j < QK_K/64; ++j) {
1088            v_xl[0] = vec_xl(0 , x0l);
1089            v_xl[1] = vec_xl(16, x0l);
1090            x0l += 32;
1091
1092            v_y[0] = vec_xl(0 , y0);
1093            v_y[1] = vec_xl(16, y0);
1094            v_y[2] = vec_xl(32, y0);
1095            v_y[3] = vec_xl(48, y0);
1096            y0 += 64;
1097
1098            q5h[0] = vec_sl(vec_and(v_1m, v_xh[0]), 4);
1099            q5h[1] = vec_sl(vec_and(v_1m, v_xh[1]), 4);
1100            q5h[2] = vec_sl(vec_and(v_2m, v_xh[0]), 3);
1101            q5h[3] = vec_sl(vec_and(v_2m, v_xh[1]), 3);
1102            v_xh[0] = vec_sr(v_xh[0], 2);
1103            v_xh[1] = vec_sr(v_xh[1], 2);
1104
1105            q5b[0] = (int8x16_t)vec_or(vec_and(v_xl[0], v_lm), q5h[0]);
1106            q5b[1] = (int8x16_t)vec_or(vec_and(v_xl[1], v_lm), q5h[1]);
1107            q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]);
1108            q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]);
1109
1110            int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);
1111            int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);
1112
1113            sumi += vec_hsum_i32x4(sumi0) * *scales++;
1114            sumi += vec_hsum_i32x4(sumi1) * *scales++;
1115        }
1116
1117        sumf += d * sumi - dmin * mins;
1118    }
1119
1120    *s = sumf;
1121
1122#else
1123    UNUSED(x);
1124    UNUSED(y);
1125    UNUSED(nb);
1126    UNUSED(kmask1);
1127    UNUSED(kmask2);
1128    UNUSED(kmask3);
1129    UNUSED(utmp);
1130    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1131#endif
1132}
1133
1134void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1135    assert(n % QK_K == 0);
1136    assert(nrc == 1);
1137    UNUSED(nrc);
1138    UNUSED(bx);
1139    UNUSED(by);
1140    UNUSED(bs);
1141
1142    const block_q6_K * GGML_RESTRICT x = vx;
1143    const block_q8_K * GGML_RESTRICT y = vy;
1144
1145    const int nb = n / QK_K;
1146
1147#if defined(__VXE__) || defined(__VXE2__)
1148    float sum = 0;
1149
1150    // Lower 4-bit and upper 2-bit masks
1151    const uint8x16_t v_lm = vec_splat_u8(0x0F);
1152    const uint8x16_t v_um = vec_splat_u8(0x03);
1153
1154    const int32x4_t v_z = vec_splat_s32(0);
1155
1156    int8x16_t  q6b[4];
1157    uint8x16_t q6h[4];
1158
1159    uint8x16_t v_xl[4];
1160    uint8x16_t v_xh[2];
1161    int8x16_t  v_y[4];
1162
1163    for (int i = 0; i < nb; ++i) {
1164        const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);
1165
1166        const uint8_t * GGML_RESTRICT x0l = x[i].ql;
1167        const uint8_t * GGML_RESTRICT x0h = x[i].qh;
1168        const int8_t  * GGML_RESTRICT y0 = y[i].qs;
1169
1170        const int8_t  * GGML_RESTRICT scale = x[i].scales;
1171
1172        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);
1173        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);
1174
1175        const int8x16_t v_scale  = vec_xl(0, scale);
1176        const int16x8_t v_scalel = vec_unpackh(v_scale);
1177        const int16x8_t v_scaleh = vec_unpackl(v_scale);
1178
1179        const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel);
1180        const int32x4_t v_minsl = vec_meadd(v_ysumsl, v_scalel, v_minslo);
1181        const int32x4_t v_minsho = vec_mulo(v_ysumsh, v_scaleh);
1182        const int32x4_t v_minsh = vec_meadd(v_ysumsh, v_scaleh, v_minsho);
1183        const int32x4_t v_mins = vec_add(v_minsl, v_minsh);
1184
1185        const int32_t mins = vec_hsum_i32x4(v_mins);
1186
1187        int32_t isum = 0;
1188        for (int j = 0; j < QK_K/128; ++j) {
1189            // Load model upper 2 bits
1190            v_xh[0] = vec_xl(0 , x0h);
1191            v_xh[1] = vec_xl(16, x0h);
1192            x0h += 32;
1193
1194            // Load model lower 4 bits
1195            v_xl[0] = vec_xl(0 , x0l);
1196            v_xl[1] = vec_xl(16, x0l);
1197            v_xl[2] = vec_xl(32, x0l);
1198            v_xl[3] = vec_xl(48, x0l);
1199            x0l += 64;
1200
1201            // Load activation quants
1202            v_y[0] = vec_xl(0 , y0);
1203            v_y[1] = vec_xl(16, y0);
1204            v_y[2] = vec_xl(32, y0);
1205            v_y[3] = vec_xl(48, y0);
1206            y0 += 64;
1207
1208            q6h[0] = vec_sl(vec_and(v_um, v_xh[0]), 4);
1209            q6h[1] = vec_sl(vec_and(v_um, v_xh[1]), 4);
1210            uint8x16_t shifted = vec_sr(v_xh[0], 2);
1211            q6h[2] = vec_sl(vec_and(v_um, shifted), 4);
1212            shifted = vec_sr(v_xh[1], 2);
1213            q6h[3] = vec_sl(vec_and(v_um, shifted), 4);
1214
1215            q6b[0] = (int8x16_t)(vec_or(vec_and(v_xl[0], v_lm), q6h[0]));
1216            q6b[1] = (int8x16_t)(vec_or(vec_and(v_xl[1], v_lm), q6h[1]));
1217            q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2]));
1218            q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3]));
1219
1220            int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);
1221            int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);
1222            int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);
1223            int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);
1224
1225            isum += vec_hsum_i32x4(summs0) * scale[0] +
1226                    vec_hsum_i32x4(summs1) * scale[1] +
1227                    vec_hsum_i32x4(summs2) * scale[2] +
1228                    vec_hsum_i32x4(summs3) * scale[3];
1229
1230            scale += 4;
1231
1232
1233            // Load activation quants
1234            v_y[0] = vec_xl(0 , y0);
1235            v_y[1] = vec_xl(16, y0);
1236            v_y[2] = vec_xl(32, y0);
1237            v_y[3] = vec_xl(48, y0);
1238            y0 += 64;
1239
1240            shifted = vec_sr(v_xh[0], 4);
1241            q6h[0] = vec_sl(vec_and(v_um, shifted), 4);
1242            shifted = vec_sr(v_xh[1], 4);
1243            q6h[1] = vec_sl(vec_and(v_um, shifted), 4);
1244            shifted = vec_sr(v_xh[0], 6);
1245            q6h[2] = vec_sl(vec_and(v_um, shifted), 4);
1246            shifted = vec_sr(v_xh[1], 6);
1247            q6h[3] = vec_sl(vec_and(v_um, shifted), 4);
1248
1249            q6b[0] = (int8x16_t)(vec_or(vec_sr(v_xl[0], 4), q6h[0]));
1250            q6b[1] = (int8x16_t)(vec_or(vec_sr(v_xl[1], 4), q6h[1]));
1251            q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2]));
1252            q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3]));
1253
1254            summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);
1255            summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);
1256            summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);
1257            summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);
1258
1259            isum += vec_hsum_i32x4(summs0) * scale[0] +
1260                    vec_hsum_i32x4(summs1) * scale[1] +
1261                    vec_hsum_i32x4(summs2) * scale[2] +
1262                    vec_hsum_i32x4(summs3) * scale[3];
1263
1264            scale += 4;
1265        }
1266
1267        sum += d_all * y[i].d * (isum - 32 * mins);
1268    }
1269
1270    *s = sum;
1271
1272#else
1273    UNUSED(x);
1274    UNUSED(y);
1275    UNUSED(nb);
1276    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1277#endif
1278}
1279
1280// #if defined(__VXE__) || defined(__VXE2__)
1281// static const int8_t keven_signs_q2xs[1024] = {
1282//      1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,
1283//      1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,
1284//      1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,
1285//      1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,
1286//      1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,
1287//      1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,
1288//      1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,
1289//      1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,
1290//      1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,
1291//      1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,
1292//      1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,
1293//      1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,
1294//      1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,
1295//      1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,
1296//      1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,
1297//      1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,
1298//      1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,
1299//      1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,
1300//      1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,
1301//      1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,
1302//      1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,
1303//      1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,
1304//      1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,
1305//      1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,
1306//      1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,
1307//      1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,
1308//      1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,
1309//      1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,
1310//      1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,
1311//      1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,
1312//      1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,
1313//      1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,
1314// };
1315// #endif
1316
1317// void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1318//     assert(n % QK_K == 0);
1319//     assert(nrc == 1);
1320//     UNUSED(nrc);
1321//     UNUSED(bx);
1322//     UNUSED(by);
1323//     UNUSED(bs);
1324
1325//     const block_iq2_xxs * GGML_RESTRICT x = vx;
1326//     const block_q8_K    * GGML_RESTRICT y = vy;
1327
1328//     const int nb = n / QK_K;
1329
1330// #if defined(__VXE__) || defined(__VXE2__)
1331//    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
1332
1333//    uint32_t aux32[4];
1334//    const uint8_t * aux8 = (const uint8_t *)aux32;
1335
1336//    float sumf = 0;
1337
1338//    for (int i = 0; i < nb; ++i) {
1339//        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1340//        const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1341//        const int8_t   * GGML_RESTRICT q8 = y[i].qs;
1342
1343//        float sumf1 = 0, sumf2 = 0;
1344
1345//        for (int ib32 = 0; ib32 < QK_K/32; ib += 2) {
1346//            int8x16_t q8b0 = vec_xl( 0, q8);
1347//            int8x16_t qb81 = vec_xl(16, q8);
1348//            int8x16_t q8b2 = vec_xl(32, q8);
1349//            int8x16_t q8b3 = vec_xl(48, q8);
1350//            q8 += 64;
1351
1352//            memcpy(aux32, q2, 4 * sizeof(uint32_t));
1353//            q2 += 8;
1354
1355//            int8x16_t q2u0 = { *(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1]) };
1356//            int8x16_t q2u1 = { *(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3]) };
1357//            int8x16_t q2u2 = { *(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9]) };
1358//            int8x16_t q2u3 = { *(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11]) };
1359
1360//            int8x16_t q2s0 = { *(const int64_t *)(signs64 + ((aux32[1] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >>  7) & 127)) };
1361//            int8x16_t q2s1 = { *(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127)) };
1362//            int8x16_t q2s2 = { *(const int64_t *)(signs64 + ((aux32[3] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >>  7) & 127)) };
1363//            int8x16_t q2s3 = { *(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127)) };
1364
1365//            q2u0 = vec_mul(q2u0, q2s0);
1366//            q2u1 = vec_mul(q2u1, q2s1);
1367//            q2u2 = vec_mul(q2u2, q2s2);
1368//            q2u3 = vec_mul(q2u3, q2s3);
1369
1370//            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);
1371//            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);
1372
1373//            sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28));
1374//            sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28));
1375//        }
1376
1377//        sumf += d * (sumf1 + sumf2);
1378//    }
1379
1380//    *s = 0.25f * sumf;
1381
1382// #else
1383
1384//     uint32_t aux32[2];
1385//     const uint8_t * aux8 = (const uint8_t *)aux32;
1386
1387//     float sumf = 0.f;
1388//     for (int i = 0; i < nb; ++i) {
1389//         const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1390//         const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1391//         const int8_t   * GGML_RESTRICT q8 = y[i].qs;
1392//         int32_t bsum = 0;
1393//         for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
1394//             memcpy(aux32, q2, 2*sizeof(uint32_t));
1395//             q2 += 4;
1396//             const uint32_t ls = 2*(aux32[1] >> 28) + 1;
1397//             int32_t sumi = 0;
1398//             for (int l = 0; l < 4; ++l) {
1399//                 const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1400//                 const uint8_t  signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];
1401//                 for (int j = 0; j < 8; ++j) {
1402//                     sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1403//                 }
1404//                 q8 += 8;
1405//             }
1406//             bsum += sumi * ls;
1407//         }
1408//         sumf += d * bsum;
1409//     }
1410//     *s = 0.125f * sumf;
1411// #endif
1412// }
1413
1414void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1415    assert(nrc == 1);
1416    UNUSED(nrc);
1417    UNUSED(bx);
1418    UNUSED(by);
1419    UNUSED(bs);
1420    assert(n % QK4_NL == 0);
1421    static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");
1422
1423    const block_iq4_nl * GGML_RESTRICT x = vx;
1424    const block_q8_0   * GGML_RESTRICT y = vy;
1425
1426    const int nb = n / QK4_NL;
1427
1428    int ib = 0;
1429    float sumf = 0;
1430
1431#if defined(__VXE__) || defined(__VXE2__)
1432    const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);
1433    const uint8x16_t v_m = vec_splat_u8(0x0F);
1434
1435    for (; ib < nb; ++ib) {
1436        const block_iq4_nl * GGML_RESTRICT x0 = &x[ib];
1437        const block_q8_0   * GGML_RESTRICT y0 = &y[ib];
1438
1439        const uint8x16_t v_x = vec_xl(0, x0->qs);
1440        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);
1441        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);
1442
1443        v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);
1444        v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);
1445
1446        const int8x16_t v_yl = vec_xl(0      , y0->qs);
1447        const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);
1448        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
1449
1450        sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy);
1451    }
1452
1453    *s = sumf;
1454#else
1455    UNUSED(x);
1456    UNUSED(y);
1457    UNUSED(nb);
1458    UNUSED(ib);
1459    UNUSED(sumf);
1460    ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
1461#endif
1462}
1463
1464void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1465    assert(nrc == 1);
1466    UNUSED(nrc);
1467    UNUSED(bx);
1468    UNUSED(by);
1469    UNUSED(bs);
1470    assert(n % QK_K == 0);
1471
1472    const block_iq4_xs * GGML_RESTRICT x = vx;
1473    const block_q8_K   * GGML_RESTRICT y = vy;
1474
1475    const int nb = n / QK_K;
1476
1477#if defined(__VXE__) || defined(__VXE2__)
1478    const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);
1479    const uint8x16_t v_m = vec_splat_u8(0x0F);
1480
1481    float sumf = 0;
1482
1483    for (int ibl = 0; ibl < nb; ++ibl) {
1484        const uint8_t * GGML_RESTRICT q4 = x[ibl].qs;
1485        const int8_t  * GGML_RESTRICT q8 = y[ibl].qs;
1486
1487        uint16_t h = x[ibl].scales_h;
1488
1489        int sumi1 = 0, sumi2 = 0;
1490        for (int ib = 0; ib < QK_K/64; ++ib) {
1491            const uint8x16_t v_x0 = vec_xl(0       , q4);
1492            const uint8x16_t v_x1 = vec_xl(QK4_NL/2, q4);
1493            q4 += 32;
1494
1495            int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);
1496            int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);
1497            int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);
1498            int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);
1499
1500            v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);
1501            v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);
1502            v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);
1503            v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);
1504
1505            const int8x16_t v_y0 = vec_xl( 0, q8);
1506            const int8x16_t v_y1 = vec_xl(16, q8);
1507            const int8x16_t v_y2 = vec_xl(32, q8);
1508            const int8x16_t v_y3 = vec_xl(48, q8);
1509            q8 += 64;
1510
1511            int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);
1512            int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);
1513
1514            int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32;
1515            int ls2 = ((x[ibl].scales_l[ib] >>  4) | ((h << 2) & 0x30)) - 32;
1516
1517            h >>= 4;
1518
1519            sumi1 += vec_hsum_i32x4(vsumi0) * ls1;
1520            sumi2 += vec_hsum_i32x4(vsumi1) * ls2;
1521        }
1522
1523        sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);
1524    }
1525
1526    *s = sumf;
1527
1528#else
1529    UNUSED(x);
1530    UNUSED(y);
1531    UNUSED(nb);
1532    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1533#endif
1534}