hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Jinyang Heggml : add some lsx support (#23798)d48a56eff2026-05-30

1#define GGML_COMMON_IMPL_C
2#include "ggml-common.h"
3#include "ggml-quants.h"
4#include "ggml-impl.h"
5#include "ggml-cpu.h"
6#include "simd-mappings.h"
7
8#include "../../quants.h"
9#include "../../ggml-cpu-impl.h"
10
11#include <math.h>
12#include <string.h>
13#include <assert.h>
14#include <float.h>
15#include <stdlib.h> // for qsort
16#include <stdio.h>  // for GGML_ASSERT
17
18#define GROUP_MAX_EPS 1e-15f
19#define GROUP_MAX_EPS_IQ3_XXS 1e-8f
20#define GROUP_MAX_EPS_IQ2_S 1e-8f
21#define GROUP_MAX_EPS_IQ1_M 1e-7f
22#define GROUP_MAX_EPS_IQ1_S 1e-12f
23
24#define UNUSED GGML_UNUSED
25
26#if defined(__loongarch_sx)
27
28static __m128i lsx_packs_w(__m128i a, __m128i b) {
29    __m128i tmp, tmp1;
30    tmp = __lsx_vsat_w(a, 15);
31    tmp1 = __lsx_vsat_w(b, 15);
32    return __lsx_vpickev_h(tmp1, tmp);
33}
34
35static __m128i lsx_packs_h(__m128i a, __m128i b) {
36    __m128i tmp, tmp1;
37    tmp = __lsx_vsat_h(a, 7);
38    tmp1 = __lsx_vsat_h(b, 7);
39    return __lsx_vpickev_b(tmp1, tmp);
40}
41
42static __m128i lsx_packus_h(__m128i a, __m128i b) {
43    __m128i tmp, tmp1;
44    tmp = __lsx_vsat_hu(a, 7);
45    tmp1 = __lsx_vsat_hu(b, 7);
46    return __lsx_vpickev_b(tmp1, tmp);
47}
48
49static __m128i lsx_maddubs_h(__m128i a, __m128i b) {
50    __m128i tmp1, tmp2;
51    tmp1 = __lsx_vmulwev_h_b(a, b);
52    tmp2 = __lsx_vmulwod_h_b(a, b);
53    return __lsx_vsadd_h(tmp1, tmp2);
54}
55
56static __m128i lsx_madd_h(__m128i a, __m128i b) {
57    __m128i tmp1, tmp2;
58    tmp1 = __lsx_vmulwev_w_h(a, b);
59    tmp2 = __lsx_vmulwod_w_h(a, b);
60    return __lsx_vadd_w(tmp1, tmp2);
61}
62
63static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) {
64    v4i32 __ret = {d, c, b, a};
65    return (__m128i)__ret;
66}
67
68static __m128i lsx_shuffle_b(__m128i a, __m128i b) {
69    __m128i mask_f, zero, tmp0, tmp2, mask;
70    int f = 0x8f;
71    mask_f = __lsx_vreplgr2vr_b(f);
72    zero = __lsx_vldi(0);
73    tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits
74    tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or  with 0x10 prepare for positive
75    mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask
76    tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones
77    return __lsx_vshuf_b(a, zero, tmp2);
78}
79
80static __m128i lsx_hadd_h(__m128i a, __m128i b) {
81    __m128i tmp1 = __lsx_vpickev_h(b, a);
82    __m128i tmp2 = __lsx_vpickod_h(b, a);
83    return __lsx_vadd_h(tmp1, tmp2);
84}
85
86static __m128i lsx_hadd_w(__m128i a, __m128i b) {
87    __m128i tmp1 = __lsx_vpickev_w(b, a);
88    __m128i tmp2 = __lsx_vpickod_w(b, a);
89    return __lsx_vadd_w(tmp1, tmp2);
90}
91
92static __m128 lsx_hadd_s(__m128 a, __m128 b) {
93    __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a);
94    __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a);
95
96    return __lsx_vfadd_s(tmp1, tmp2);
97}
98
99static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {
100    __m128 res_0 =lsx_hadd_s(a, b);
101    __m128 res_1 =lsx_hadd_s(c, d);
102    __m128 res =lsx_hadd_s(res_0, res_1);
103    res =lsx_hadd_s(res, res);
104    res =lsx_hadd_s(res, res);
105
106    return ((v4f32)res)[0];
107}
108
109// multiply int8_t, add results pairwise twice
110static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {
111    // Get absolute values of x vectors
112    const __m128i ax = __lsx_vsigncov_b(x, x);
113    // Sign the values of the y vectors
114    const __m128i sy = __lsx_vsigncov_b(x, y);
115    // Perform multiplication and create 16-bit values
116    const __m128i dot = lsx_maddubs_h(ax, sy);
117    const __m128i ones = __lsx_vreplgr2vr_h(1);
118    return lsx_madd_h(ones, dot);
119}
120#endif
121
122#if defined(__loongarch_asx)
123
124#ifdef __clang__
125#define VREGS_PREFIX "$vr"
126#define XREGS_PREFIX "$xr"
127#else // GCC
128#define VREGS_PREFIX "$f"
129#define XREGS_PREFIX "$f"
130#endif
131#define __ALL_REGS "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
132// Convert __m128i to __m256i
133static inline __m256i ____m256i(__m128i in) {
134    __m256i out = __lasx_xvldi(0);
135    __asm__ volatile (
136        ".irp i," __ALL_REGS                "\n\t"
137        " .ifc %[out], " XREGS_PREFIX"\\i    \n\t"
138        "  .irp j," __ALL_REGS              "\n\t"
139        "   .ifc %[in], " VREGS_PREFIX "\\j  \n\t"
140        "    xvpermi.q $xr\\i, $xr\\j, 0x20  \n\t"
141        "   .endif                           \n\t"
142        "  .endr                             \n\t"
143        " .endif                             \n\t"
144        ".endr                               \n\t"
145        : [out] "+f" (out) : [in] "f" (in)
146    );
147    return out;
148}
149// Convert two __m128i to __m256i
150static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) {
151    __m256i out;
152    __asm__ volatile (
153        ".irp i," __ALL_REGS                "\n\t"
154        " .ifc %[hi], " VREGS_PREFIX "\\i    \n\t"
155        "  .irp j," __ALL_REGS              "\n\t"
156        "   .ifc %[lo], " VREGS_PREFIX "\\j  \n\t"
157        "    xvpermi.q $xr\\i, $xr\\j, 0x20  \n\t"
158        "   .endif                           \n\t"
159        "  .endr                             \n\t"
160        " .endif                             \n\t"
161        ".endr                               \n\t"
162        ".ifnc %[out], %[hi]                 \n\t"
163        ".irp i," __ALL_REGS                "\n\t"
164        " .ifc %[out], " XREGS_PREFIX "\\i   \n\t"
165        "  .irp j," __ALL_REGS              "\n\t"
166        "   .ifc %[hi], " VREGS_PREFIX "\\j  \n\t"
167        "    xvori.b $xr\\i, $xr\\j, 0       \n\t"
168        "   .endif                           \n\t"
169        "  .endr                             \n\t"
170        " .endif                             \n\t"
171        ".endr                               \n\t"
172        ".endif                              \n\t"
173        : [out] "=f" (out), [hi] "+f" (inhi)
174        : [lo] "f" (inlo)
175    );
176    return out;
177}
178// Convert __m256i low part to __m128i
179static inline __m128i lasx_extracti128_lo(__m256i in) {
180    __m128i out;
181    __asm__ volatile (
182        ".ifnc %[out], %[in]                 \n\t"
183        ".irp i," __ALL_REGS                "\n\t"
184        " .ifc %[out], " VREGS_PREFIX "\\i   \n\t"
185        "  .irp j," __ALL_REGS              "\n\t"
186        "   .ifc %[in], " XREGS_PREFIX "\\j  \n\t"
187        "    vori.b $vr\\i, $vr\\j, 0        \n\t"
188        "   .endif                           \n\t"
189        "  .endr                             \n\t"
190        " .endif                             \n\t"
191        ".endr                               \n\t"
192        ".endif                              \n\t"
193        : [out] "=f" (out) : [in] "f" (in)
194    );
195    return out;
196}
197// Convert __m256i high part to __m128i
198static inline __m128i lasx_extracti128_hi(__m256i in) {
199    __m128i out;
200    __asm__ volatile (
201        ".irp i," __ALL_REGS                "\n\t"
202        " .ifc %[out], " VREGS_PREFIX "\\i   \n\t"
203        "  .irp j," __ALL_REGS              "\n\t"
204        "   .ifc %[in], " XREGS_PREFIX "\\j  \n\t"
205        "    xvpermi.q $xr\\i, $xr\\j, 0x11  \n\t"
206        "   .endif                           \n\t"
207        "  .endr                             \n\t"
208        " .endif                             \n\t"
209        ".endr                               \n\t"
210        : [out] "=f" (out) : [in] "f" (in)
211    );
212    return out;
213}
214
215static __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) {
216    v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7};
217    return (__m256i)__ret;
218}
219
220static __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) {
221    v4i64 __ret = {d, c, b, a};
222    return (__m256i)__ret;
223}
224
225static __m256i lasx_insertf128( __m128i x, __m128i y) {
226    return lasx_set_q(x, y);
227}
228
229static __m256i lasx_shuffle_b(__m256i a, __m256i b) {
230    __m256i mask_f, zero, tmp0, tmp2, mask;
231    int f = 0x8f;
232    mask_f = __lasx_xvreplgr2vr_b(f);
233    zero = __lasx_xvldi(0);
234    tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits
235    tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or  with 0x10 prepare for positive
236    mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask
237    tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones
238    return __lasx_xvshuf_b(a, zero, tmp2);
239}
240
241static __m256i lasx_extu8_16(__m128i a) {
242    return __lasx_vext2xv_hu_bu(____m256i(a));
243}
244
245static __m256i lasx_ext8_16(__m128i a) {
246    return __lasx_vext2xv_h_b(____m256i(a));
247}
248
249static __m256i lasx_ext16_32(__m128i a) {
250    return __lasx_vext2xv_w_h(____m256i(a));
251}
252
253static __m128i lasx_extracti128( __m256i a, int pos) {
254    __m128i ret;
255    if( pos == 0)
256    {
257       ret = lasx_extracti128_lo(a);
258    } else {
259       ret = lasx_extracti128_hi(a);
260    }
261    return ret;
262}
263
264static __m128 lasx_extractf128( __m256 a, int pos) {
265    __m128 ret;
266    if( pos == 0)
267    {
268       ret = (__m128)lasx_extracti128_lo((__m256i)a);
269    } else {
270       ret = (__m128)lasx_extracti128_hi((__m256i)a);
271    }
272    return ret;
273}
274
275static __m256i lasx_maddubs_h(__m256i a, __m256i b) {
276    __m256i tmp1, tmp2;
277    tmp1 = __lasx_xvmulwev_h_b(a, b);
278    tmp2 = __lasx_xvmulwod_h_b(a, b);
279    return __lasx_xvsadd_h(tmp1, tmp2);
280}
281
282static __m256i lasx_madd_h(__m256i a, __m256i b) {
283    __m256i tmp1, tmp2;
284    tmp1 = __lasx_xvmulwev_w_h(a, b);
285    tmp2 = __lasx_xvmulwod_w_h(a, b);
286    return __lasx_xvadd_w(tmp1, tmp2);
287}
288
289static __m256i lasx_packs_w(__m256i a, __m256i b) {
290    __m256i tmp, tmp1;
291    tmp = __lasx_xvsat_w(a, 15);
292    tmp1 = __lasx_xvsat_w(b, 15);
293    return __lasx_xvpickev_h(tmp1, tmp);
294}
295
296static __m256i lasx_packs_h(__m256i a, __m256i b) {
297    __m256i tmp, tmp1;
298    tmp = __lasx_xvsat_h(a, 7);
299    tmp1 = __lasx_xvsat_h(b, 7);
300    return __lasx_xvpickev_b(tmp1, tmp);
301}
302
303static inline __m256i lasx_madd_h_b(__m256i a, __m256i b) {
304    __m256i tmp1, tmp2;
305    tmp1 = __lasx_xvmulwev_h_b(a, b);
306    tmp2 = __lasx_xvmulwod_h_b(a, b);
307    return __lasx_xvadd_h(tmp1, tmp2);
308}
309
310static inline __m256i lasx_xvrepl128vei_h(__m256i a, const unsigned int b) {
311    switch (b) {
312        case 0: return __lasx_xvrepl128vei_h(a, 0);
313        case 1: return __lasx_xvrepl128vei_h(a, 1);
314        case 2: return __lasx_xvrepl128vei_h(a, 2);
315        case 3: return __lasx_xvrepl128vei_h(a, 3);
316        case 4: return __lasx_xvrepl128vei_h(a, 4);
317        case 5: return __lasx_xvrepl128vei_h(a, 5);
318        case 6: return __lasx_xvrepl128vei_h(a, 6);
319        case 7: return __lasx_xvrepl128vei_h(a, 7);
320        default: __builtin_unreachable();
321    }
322}
323
324static inline __m256i lasx_xvandi_b_bit(__m256i a, const unsigned int b) {
325    switch (b) {
326        case 0: return __lasx_xvandi_b(a, 1 << 0);
327        case 1: return __lasx_xvandi_b(a, 1 << 1);
328        case 2: return __lasx_xvandi_b(a, 1 << 2);
329        case 3: return __lasx_xvandi_b(a, 1 << 3);
330        case 4: return __lasx_xvandi_b(a, 1 << 4);
331        case 5: return __lasx_xvandi_b(a, 1 << 5);
332        case 6: return __lasx_xvandi_b(a, 1 << 6);
333        case 7: return __lasx_xvandi_b(a, 1 << 7);
334        default: __builtin_unreachable();
335    }
336}
337
338// horizontally add 8 floats
339static inline float hsum_float_8(const __m256 x) {
340    __m128 res = lasx_extractf128(x, 1);
341    res = __lsx_vfadd_s(res, lasx_extractf128(x, 0));
342    res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res));
343    res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0));
344    return ((v4f32)res)[0];
345}
346
347// horizontally add 8 int32_t
348static inline int hsum_i32_8(const __m256i a) {
349
350    __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11);
351    __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00);
352
353    __m128i  tmp1_128 = lasx_extracti128_lo(tmp1);
354    __m128i  tmp2_128 = lasx_extracti128_lo(tmp2);
355
356    __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128);
357
358    __m128i ev = __lsx_vpickev_w(sum128, sum128);
359    __m128i od = __lsx_vpickod_w(sum128, sum128);
360    __m128i sum64 = __lsx_vadd_w(ev, od);
361
362    int sum64_1, sum64_2;
363    sum64_1 = __lsx_vpickve2gr_w(sum64, 0);
364    sum64_2 = __lsx_vpickve2gr_w(sum64, 1);
365
366    return  sum64_1 + sum64_2;
367}
368
369// horizontally add 4 int32_t
370static inline int hsum_i32_4(const __m128i a) {
371    __m128i ev = __lsx_vpickev_w(a, a);
372    __m128i od = __lsx_vpickod_w(a, a);
373    __m128i sum64 = __lsx_vadd_w(ev, od);
374
375    int sum64_1, sum64_2;
376    sum64_1 = __lsx_vpickve2gr_w(sum64, 0);
377    sum64_2 = __lsx_vpickve2gr_w(sum64, 1);
378
379    return  sum64_1 + sum64_2;
380}
381
382// spread 32 bits to 32 bytes { 0x00, 0xFF }
383static inline __m256i bytes_from_bits_32(const uint8_t * x) {
384
385    uint32_t x32;
386    memcpy(&x32, x, sizeof(uint32_t));
387    const __m256i shuf_mask = lasx_set_d(
388            0x0303030303030303, 0x0202020202020202,
389            0x0101010101010101, 0x0000000000000000);
390
391    __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask);
392    const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe);
393    bytes = __lasx_xvor_v(bytes, bit_mask);
394    return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1));
395}
396
397// Unpack 32 4-bit fields into 32 bytes
398// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
399static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {
400    const __m128i lo = __lsx_vld((const __m128i *)rsi, 0);
401    __m128i hi = __lsx_vsrli_h(lo, 4);
402    return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf);
403}
404
405// add int16_t pairwise and return as float vector
406static inline __m256 sum_i16_pairs_float(const __m256i x) {
407    __m256i v = __lasx_xvpackod_h(x, x);
408    __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v);
409    return __lasx_xvffint_s_w(summed_pairs);
410}
411
412static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
413    // Perform multiplication and create 16-bit values
414    const __m256i dot = lasx_maddubs_h(ax, sy);
415    return sum_i16_pairs_float(dot);
416}
417
418// multiply int8_t, add results pairwise twice and return as float vector
419static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
420    const __m256i dot = lasx_madd_h_b(x, y);
421    return sum_i16_pairs_float(dot);
422}
423
424static inline __m128i packNibbles( __m256i bytes ) {
425    // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
426    const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF);
427     __m256i high = __lasx_xvandn_v(lowByte, bytes);
428    __m256i low = __lasx_xvand_v(lowByte, bytes);
429    high = __lasx_xvsrli_h(high, 4);
430    bytes = __lasx_xvor_v(low, high);
431    // Compress uint16_t lanes into bytes
432    __m128i *r0 = (__m128i *)&bytes;
433    __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11);
434    __m128i *r1 = (__m128i *)&tmp_h128;
435
436    __m128i zero = __lsx_vldi(0);
437    __m128i tmp, tmp2, tmp3;
438
439    tmp = __lsx_vmax_h(zero, *r0);
440    tmp2 = __lsx_vsat_hu(tmp, 7);
441
442    tmp = __lsx_vmax_h(zero, *r1);
443    tmp3 = __lsx_vsat_hu(tmp, 7);
444    return  __lsx_vpickev_b(tmp3, tmp2);
445}
446#endif  //__loongarch_asx
447
448void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
449    assert(QK8_0 == 32);
450    assert(k % QK8_0 == 0);
451    const int nb = k / QK8_0;
452
453    block_q8_0 * GGML_RESTRICT y = vy;
454
455#if defined(__loongarch_asx)
456    for (int i = 0; i < nb; i++) {
457        __m256 v0 = (__m256)__lasx_xvld( x , 0);
458        __m256 v1 = (__m256)__lasx_xvld( x , 32);
459        __m256 v2 = (__m256)__lasx_xvld( x , 64);
460        __m256 v3 = (__m256)__lasx_xvld( x , 96);
461        x += 32;
462
463        // Compute max(abs(e)) for the block
464        const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );
465        __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );
466        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );
467        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );
468        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );
469
470        __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) );
471        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );
472        __m128 tmp = max4;
473        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 ));
474        const float max_scalar = ((v4f32)max4)[0];
475
476        // Quantize these floats
477        const float d = max_scalar / 127.f;
478        y[i].d = GGML_CPU_FP32_TO_FP16(d);
479        const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;
480        const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id );
481
482        // Apply the multiplier
483        v0 = __lasx_xvfmul_s( v0, mul );
484        v1 = __lasx_xvfmul_s( v1, mul );
485        v2 = __lasx_xvfmul_s( v2, mul );
486        v3 = __lasx_xvfmul_s( v3, mul );
487
488        // Round to nearest integer
489        __m256i i0 = __lasx_xvftintrne_w_s( v0 );
490        __m256i i1 = __lasx_xvftintrne_w_s( v1 );
491        __m256i i2 = __lasx_xvftintrne_w_s( v2 );
492        __m256i i3 = __lasx_xvftintrne_w_s( v3 );
493
494        __m128i ni0 = lasx_extracti128( i0, 0 );
495        __m128i ni1 = lasx_extracti128( i0, 1);
496        __m128i ni2 = lasx_extracti128( i1, 0);
497        __m128i ni3 = lasx_extracti128( i1, 1);
498        __m128i ni4 = lasx_extracti128( i2, 0);
499        __m128i ni5 = lasx_extracti128( i2, 1);
500        __m128i ni6 = lasx_extracti128( i3, 0);
501        __m128i ni7 = lasx_extracti128( i3, 1);
502
503        // Convert int32 to int16
504        ni0 = lsx_packs_w( ni0, ni1 );
505        ni2 = lsx_packs_w( ni2, ni3 );
506        ni4 = lsx_packs_w( ni4, ni5 );
507        ni6 = lsx_packs_w( ni6, ni7 );
508        // Convert int16 to int8
509        ni0 = lsx_packs_h( ni0, ni2 );
510        ni4 = lsx_packs_h( ni4, ni6 );
511
512        __lsx_vst(ni0, (__m128i *)(y[i].qs +  0), 0);
513        __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);
514
515    }
516#else
517    GGML_UNUSED(nb);
518    // scalar
519    quantize_row_q8_0_ref(x, y, k);
520#endif
521}
522
523void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
524    assert(k % QK8_1 == 0);
525    const int nb = k / QK8_1;
526
527    block_q8_1 * GGML_RESTRICT y = vy;
528
529#if defined(__loongarch_asx)
530    for (int i = 0; i < nb; i++) {
531        __m256 v0 = (__m256)__lasx_xvld( x , 0 );
532        __m256 v1 = (__m256)__lasx_xvld( x , 32 );
533        __m256 v2 = (__m256)__lasx_xvld( x , 64 );
534        __m256 v3 = (__m256)__lasx_xvld( x , 96 );
535        x += 32;
536
537        // Compute max(abs(e)) for the block
538        const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );
539        __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );
540        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );
541        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );
542        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );
543
544        __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) );
545        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );
546        __m128 tmp = max4;
547        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x1 ));
548        const float max_scalar = ((v4f32)max4)[0];
549
550        // Quantize these floats
551        const float d = max_scalar / 127.f;
552        y[i].d = GGML_CPU_FP32_TO_FP16(d);
553        const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;
554        const __m256 mul = __lasx_xvreplfr2vr_s( id );
555
556        // Apply the multiplier
557        v0 = __lasx_xvfmul_s( v0, mul );
558        v1 = __lasx_xvfmul_s( v1, mul );
559        v2 = __lasx_xvfmul_s( v2, mul );
560        v3 = __lasx_xvfmul_s( v3, mul );
561
562        // Round to nearest integer
563        __m256i i0 = __lasx_xvftintrne_w_s( v0 );
564        __m256i i1 = __lasx_xvftintrne_w_s( v1 );
565        __m256i i2 = __lasx_xvftintrne_w_s( v2 );
566        __m256i i3 = __lasx_xvftintrne_w_s( v3 );
567
568        __m128i ni0 = lasx_extracti128(i0, 0);
569        __m128i ni1 = lasx_extracti128( i0, 1);
570        __m128i ni2 = lasx_extracti128( i1, 0);
571        __m128i ni3 = lasx_extracti128( i1, 1);
572        __m128i ni4 = lasx_extracti128( i2, 0 );
573        __m128i ni5 = lasx_extracti128( i2, 1);
574        __m128i ni6 = lasx_extracti128( i3, 0);
575        __m128i ni7 = lasx_extracti128( i3, 1);
576
577        // Compute the sum of the quants and set y[i].s
578        const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3));
579        const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7));
580        y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1)));
581
582        // Convert int32 to int16
583        ni0 = lsx_packs_w( ni0, ni1 );
584        ni2 = lsx_packs_w( ni2, ni3 );
585        ni4 = lsx_packs_w( ni4, ni5 );
586        ni6 = lsx_packs_w( ni6, ni7 );
587        // Convert int16 to int8
588        ni0 = lsx_packs_h( ni0, ni2 );
589        ni4 = lsx_packs_h( ni4, ni6 );
590
591        __lsx_vst(ni0, (__m128i *)(y[i].qs +  0), 0);
592        __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);
593    }
594#else
595    GGML_UNUSED(nb);
596    // scalar
597    quantize_row_q8_1_ref(x, y, k);
598#endif
599}
600
601
602//===================================== Dot products =================================
603
604//
605// Helper functions
606//
607
608#if defined(__loongarch_asx)
609// shuffles to pick the required scales in dot products
610static inline __m256i get_scale_shuffle_q3k(int i) {
611    static const uint8_t k_shuffle[128] = {
612         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,     2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
613         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,     6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
614         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,    10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
615        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,    14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,
616    };
617    return __lasx_xvld((const __m256i*)k_shuffle + i, 0);
618}
619static inline __m256i get_scale_shuffle_k4(int i) {
620    static const uint8_t k_shuffle[256] = {
621         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
622         2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
623         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,
624         6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
625         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,
626        10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,
627        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,
628        14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15
629    };
630    return __lasx_xvld((const __m256i*)k_shuffle + i, 0);
631}
632static inline __m128i get_scale_shuffle(int i) {
633    static const uint8_t k_shuffle[128] = {
634         0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
635         2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
636         4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
637         6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
638         8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
639        10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,
640        12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,
641        14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15
642    };
643    return __lsx_vld((const __m128i*)k_shuffle + i, 0);
644}
645#endif
646
647void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
648    const int qk = QK8_0;
649    const int nb = n / qk;
650
651    assert(n % qk == 0);
652    assert(nrc == 1);
653    UNUSED(nrc);
654    UNUSED(bx);
655    UNUSED(by);
656    UNUSED(bs);
657
658    const block_q4_0 * GGML_RESTRICT x = vx;
659    const block_q8_0 * GGML_RESTRICT y = vy;
660
661    int ib = 0;
662    float sumf = 0;
663
664#if defined(__loongarch_asx)
665    // Initialize accumulator with zeros
666    __m256 acc = (__m256)__lasx_xvldi(0);
667
668    // Main loop
669    for (; ib < nb; ++ib) {
670        /* Compute combined scale for the block */
671        const __m256 d = __lasx_xvreplfr2vr_s( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );
672
673        __m256i qx = bytes_from_nibbles_32(x[ib].qs);
674
675        // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
676        const __m256i off = __lasx_xvreplgr2vr_b( 8 );
677        qx = __lasx_xvsub_b( qx, off );
678
679        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
680
681        const __m256 q = mul_sum_i8_pairs_float(qx, qy);
682
683        /* Multiply q with scale and accumulate */
684        acc = __lasx_xvfmadd_s( d, q, acc );
685    }
686
687    sumf = hsum_float_8(acc);
688
689#elif defined(__loongarch_sx)
690    // set constants
691    const __m128i low_mask = __lsx_vreplgr2vr_b(0xF);
692    const __m128i off = __lsx_vreplgr2vr_b(8);
693
694    // Initialize accumulator with zeros
695    __m128 acc_0 = (__m128)__lsx_vldi(0);
696    __m128 acc_1 = (__m128)__lsx_vldi(0);
697    __m128 acc_2 = (__m128)__lsx_vldi(0);
698    __m128 acc_3 = (__m128)__lsx_vldi(0);
699
700    for (; ib + 1 < nb; ib += 2) {
701
702        // Compute combined scale for the block 0 and 1
703        const float ft0 = GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d);
704        const __m128 d_0_1 = (__m128)(v4f32){ft0, ft0, ft0, ft0};
705
706        const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[ib].qs, 0);
707
708        __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);
709        __m128i by_0 = __lsx_vld((const __m128i *)y[ib].qs, 0);
710        bx_0 = __lsx_vsub_b(bx_0, off);
711        const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
712
713        __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));
714        __m128i by_1 = __lsx_vld((const __m128i *)(y[ib].qs + 16), 0);
715        bx_1 = __lsx_vsub_b(bx_1, off);
716        const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
717
718        // Compute combined scale for the block 2 and 3
719        const float ft1 = GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d);
720        const __m128 d_2_3 = (__m128)(v4f32){ft1, ft1, ft1, ft1};
721
722        const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[ib + 1].qs, 0);
723
724        __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);
725        __m128i by_2 = __lsx_vld((const __m128i *)y[ib + 1].qs, 0);
726        bx_2 = __lsx_vsub_b(bx_2, off);
727        const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
728
729        __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));
730        __m128i by_3 = __lsx_vld((const __m128i *)(y[ib + 1].qs + 16), 0);
731        bx_3 = __lsx_vsub_b(bx_3, off);
732        const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
733
734        // Convert int32_t to float
735        __m128 p0 = __lsx_vffint_s_w(i32_0);
736        __m128 p1 = __lsx_vffint_s_w(i32_1);
737        __m128 p2 = __lsx_vffint_s_w(i32_2);
738        __m128 p3 = __lsx_vffint_s_w(i32_3);
739
740        // Apply the scale
741        __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 );
742        __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 );
743        __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 );
744        __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 );
745
746        // Acummulate
747        acc_0 = __lsx_vfadd_s(p0_d, acc_0);
748        acc_1 = __lsx_vfadd_s(p1_d, acc_1);
749        acc_2 = __lsx_vfadd_s(p2_d, acc_2);
750        acc_3 = __lsx_vfadd_s(p3_d, acc_3);
751    }
752
753    sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);
754
755#endif
756    for (; ib < nb; ++ib) {
757        int sumi0 = 0;
758        int sumi1 = 0;
759
760        for (int j = 0; j < qk/2; ++j) {
761            const int v0 = (x[ib].qs[j] & 0x0F) - 8;
762            const int v1 = (x[ib].qs[j] >>   4) - 8;
763
764            sumi0 += (v0 * y[ib].qs[j]);
765            sumi1 += (v1 * y[ib].qs[j + qk/2]);
766        }
767
768        int sumi = sumi0 + sumi1;
769        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);
770    }
771
772    *s = sumf;
773}
774
775void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
776    const int qk = QK8_1;
777    const int nb = n / qk;
778
779    assert(n % qk == 0);
780    assert(nrc == 1);
781    UNUSED(nrc);
782    UNUSED(bx);
783    UNUSED(by);
784    UNUSED(bs);
785
786    const block_q4_1 * GGML_RESTRICT x = vx;
787    const block_q8_1 * GGML_RESTRICT y = vy;
788
789    int ib = 0;
790    float sumf = 0;
791
792#if defined(__loongarch_asx)
793    // Initialize accumulator with zeros
794    __m256 acc = (__m256)__lasx_xvldi(0);
795
796    float summs = 0;
797
798    // Main loop
799    for (; ib < nb; ++ib) {
800        const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);
801        const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);
802
803        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);
804
805        const __m256 d0v = __lasx_xvreplfr2vr_s( d0 );
806        const __m256 d1v = __lasx_xvreplfr2vr_s( d1 );
807
808        // Compute combined scales
809        const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v );
810
811        // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
812        const __m256i qx = bytes_from_nibbles_32(x[ib].qs);
813        const __m256i qy = __lasx_xvld( (const __m256i *)y[ib].qs, 0);
814
815        const __m256 xy = mul_sum_us8_pairs_float(qx, qy);
816
817        // Accumulate d0*d1*x*y
818        acc = __lasx_xvfmadd_s( d0d1, xy, acc );
819    }
820
821    sumf = hsum_float_8(acc) + summs;
822
823    *s = sumf;
824#else
825    UNUSED(nb);
826    UNUSED(x);
827    UNUSED(y);
828    UNUSED(ib);
829    UNUSED(sumf);
830    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
831#endif
832}
833
834void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
835    const int qk = QK8_0;
836    const int nb = n / qk;
837
838    int ib = 0;
839    float sumf = 0;
840
841    assert(n % qk == 0);
842    assert(qk == QK5_0);
843    assert(nrc == 1);
844    UNUSED(nrc);
845    UNUSED(bx);
846    UNUSED(by);
847    UNUSED(bs);
848
849    const block_q5_0 * GGML_RESTRICT x = vx;
850    const block_q8_0 * GGML_RESTRICT y = vy;
851
852#if defined(__loongarch_asx)
853    // Initialize accumulator with zeros
854    __m256 acc = (__m256)__lasx_xvldi(0);
855
856    // Main loop
857    for (; ib < nb; ++ib) {
858        /* Compute combined scale for the block */
859        const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); //FIXME
860
861        __m256i qx = bytes_from_nibbles_32(x[ib].qs);
862        __m256i bxhi = bytes_from_bits_32(x[ib].qh);
863        bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));
864        qx = __lasx_xvor_v(qx, bxhi);
865
866        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
867
868        const __m256 q = mul_sum_i8_pairs_float(qx, qy);
869
870        /* Multiply q with scale and accumulate */
871        acc = __lasx_xvfmadd_s(d, q, acc);
872    }
873
874    sumf = hsum_float_8(acc);
875
876    *s = sumf;
877#else
878    UNUSED(nb);
879    UNUSED(ib);
880    UNUSED(sumf);
881    UNUSED(x);
882    UNUSED(y);
883    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
884#endif
885}
886
887void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
888    const int qk = QK8_1;
889    const int nb = n / qk;
890
891    int ib = 0;
892    float sumf = 0;
893
894    assert(n % qk == 0);
895    assert(qk == QK5_1);
896    assert(nrc == 1);
897    UNUSED(nrc);
898    UNUSED(bx);
899    UNUSED(by);
900    UNUSED(bs);
901
902    const block_q5_1 * GGML_RESTRICT x = vx;
903    const block_q8_1 * GGML_RESTRICT y = vy;
904
905#if defined(__loongarch_asx)
906    // Initialize accumulator with zeros
907    __m256 acc = (__m256)__lasx_xvldi(0);
908
909    float summs = 0.0f;
910
911    // Main loop
912    for (; ib < nb; ++ib) {
913        const __m256 dx = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d));
914
915        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);
916
917        __m256i qx = bytes_from_nibbles_32(x[ib].qs);
918        __m256i bxhi = bytes_from_bits_32(x[ib].qh);
919        bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10));
920        qx = __lasx_xvor_v(qx, bxhi);
921
922        const __m256 dy = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib].d));
923        const __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
924
925        const __m256 q = mul_sum_us8_pairs_float(qx, qy);
926
927        acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc);
928    }
929
930    sumf = hsum_float_8(acc) + summs;
931
932    *s = sumf;
933#else
934    UNUSED(nb);
935    UNUSED(ib);
936    UNUSED(sumf);
937    UNUSED(x);
938    UNUSED(y);
939    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
940#endif
941}
942
943void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
944    const int qk = QK8_0;
945    const int nb = n / qk;
946
947    assert(n % qk == 0);
948    assert(nrc == 1);
949    UNUSED(nrc);
950    UNUSED(bx);
951    UNUSED(by);
952    UNUSED(bs);
953
954    const block_q8_0 * GGML_RESTRICT x = vx;
955    const block_q8_0 * GGML_RESTRICT y = vy;
956
957    int ib = 0;
958    float sumf = 0;
959
960#if defined(__loongarch_asx)
961    // Initialize accumulator with zeros
962    __m256 acc = (__m256)__lasx_xvldi(0);
963
964    // Main loop
965    for (; ib < nb; ++ib) {
966        // Compute combined scale for the block
967        const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
968        __m256i qx = __lasx_xvld((const __m256i *)x[ib].qs, 0);
969        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
970
971        const __m256 q = mul_sum_i8_pairs_float(qx, qy);
972
973        // Multiply q with scale and accumulate
974        acc = __lasx_xvfmadd_s( d, q, acc );
975    }
976
977    sumf = hsum_float_8(acc);
978
979    *s = sumf;
980
981#elif defined(__loongarch_sx)
982
983    __m128 acc = (__m128)__lsx_vldi(0);
984
985    for (; ib < nb; ++ib) {
986        const float d = GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d);
987        const __m128i qx_0 = __lsx_vld((const __m128i *)x[ib].qs, 0);
988        const __m128i qx_1 = __lsx_vld((const __m128i *)x[ib].qs + 1, 0);
989        const __m128i qy_0 = __lsx_vld((const __m128i *)y[ib].qs, 0);
990        const __m128i qy_1 = __lsx_vld((const __m128i *)y[ib].qs + 1, 0);
991
992        const __m128i p16_0 = lsx_maddubs_h(qx_0, qy_0);
993        const __m128i p16_1 = lsx_maddubs_h(qx_1, qy_1);
994
995        // Sum int16 pairs → int32
996        const __m128i s_0 = __lsx_vaddwev_w_h(p16_0, p16_1);
997        const __m128i s_1 = __lsx_vaddwod_w_h(p16_0, p16_1);
998
999        const __m128 q = __lsx_vffint_s_w(__lsx_vadd_w(s_0, s_1));
1000        acc = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(d), q, acc);
1001    }
1002
1003    __m128 res = lsx_hadd_s(acc, acc);
1004    res = lsx_hadd_s(res, res);
1005    sumf = ((v4f32)res)[0];
1006
1007    *s = sumf;
1008
1009#else
1010    UNUSED(nb);
1011    UNUSED(ib);
1012    UNUSED(sumf);
1013    UNUSED(x);
1014    UNUSED(y);
1015    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
1016#endif
1017}
1018
1019void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1020    assert(nrc == 1);
1021    UNUSED(nrc);
1022    UNUSED(bx);
1023    UNUSED(by);
1024    UNUSED(bs);
1025
1026    const block_q2_K * GGML_RESTRICT x = vx;
1027    const block_q8_K * GGML_RESTRICT y = vy;
1028
1029    const int nb = n / QK_K;
1030
1031#if defined __loongarch_asx
1032
1033    __m256 acc = (__m256)__lasx_xvldi(0);
1034
1035    for (int i = 0; i < nb; ++i) {
1036
1037        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1038        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
1039
1040        const uint8_t * GGML_RESTRICT q2 = x[i].qs;
1041        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1042
1043        const __m128i mins_and_scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);
1044        const __m128i scales128 = __lsx_vandi_b(mins_and_scales128, 0xf);
1045        const __m256i mins = lasx_ext8_16(__lsx_vsrli_b(mins_and_scales128, 4));
1046        const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0));
1047
1048        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc);
1049
1050        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};
1051        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));
1052
1053        __m256i sumi = __lasx_xvldi(0);
1054
1055        for (int j = 0; j < QK_K/128; ++j) {
1056
1057            const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32;
1058
1059            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1060            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1061            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1062            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1063
1064            const __m256i q2_0 = __lasx_xvandi_b(q2bits, 3);
1065            const __m256i q2_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 2), 3);
1066            const __m256i q2_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 4), 3);
1067            const __m256i q2_3 = __lasx_xvsrli_b(q2bits, 6);
1068
1069            __m256i p0 = lasx_madd_h_b(q2_0, q8_0);
1070            __m256i p1 = lasx_madd_h_b(q2_1, q8_1);
1071            __m256i p2 = lasx_madd_h_b(q2_2, q8_2);
1072            __m256i p3 = lasx_madd_h_b(q2_3, q8_3);
1073
1074            p0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p0);
1075            p1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p1);
1076            p2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p2);
1077            p3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p3);
1078
1079            p0 = __lasx_xvadd_w(p0, p1);
1080            p2 = __lasx_xvadd_w(p2, p3);
1081
1082            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2));
1083        }
1084
1085        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);
1086
1087    }
1088
1089    *s = hsum_float_8(acc);
1090
1091#else
1092    UNUSED(x);
1093    UNUSED(y);
1094    UNUSED(nb);
1095    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1096#endif
1097}
1098
1099void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1100    assert(n % QK_K == 0);
1101    assert(nrc == 1);
1102    UNUSED(nrc);
1103    UNUSED(bx);
1104    UNUSED(by);
1105    UNUSED(bs);
1106
1107    const uint32_t kmask1 = 0x03030303;
1108    const uint32_t kmask2 = 0x0f0f0f0f;
1109
1110    const block_q3_K * GGML_RESTRICT x = vx;
1111    const block_q8_K * GGML_RESTRICT y = vy;
1112
1113    const int nb = n / QK_K;
1114
1115#if defined __loongarch_asx
1116
1117    const __m128i m32 = __lsx_vreplgr2vr_b(32);
1118
1119    __m256 acc = (__m256)__lasx_xvldi(0);
1120
1121    uint32_t aux[3];
1122
1123    for (int i = 0; i < nb; ++i) {
1124
1125        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1126        const uint8_t * GGML_RESTRICT q3 = x[i].qs;
1127        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1128        // Set up scales
1129        memcpy(aux, x[i].scales, 12);
1130        __m128i scales128 = lsx_set_w(
1131                ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
1132                ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
1133                (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
1134                (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
1135        scales128 = __lsx_vsub_b(scales128, m32);
1136
1137        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};
1138        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));
1139
1140        // high bit
1141        const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0);
1142
1143        // integer accumulator
1144        __m256i sumi = __lasx_xvldi(0);
1145
1146        for (int j = 0; j < QK_K/128; ++j) {
1147            // load low 2 bits
1148            const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32;
1149
1150            // prepare low and high bits
1151            const __m256i q3l_0 = __lasx_xvandi_b(q3bits, 3);
1152            const __m256i q3l_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 2), 3);
1153            const __m256i q3l_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 4), 3);
1154            const __m256i q3l_3 = __lasx_xvsrli_b(q3bits, 6);
1155            const __m256i q3h_0 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 0), 0), 2);
1156            const __m256i q3h_1 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 1), 0), 2);
1157            const __m256i q3h_2 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 2), 0), 2);
1158            const __m256i q3h_3 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 3), 0), 2);
1159            const __m256i q3_0 = __lasx_xvor_v(q3h_0, q3l_0);
1160            const __m256i q3_1 = __lasx_xvor_v(q3h_1, q3l_1);
1161            const __m256i q3_2 = __lasx_xvor_v(q3h_2, q3l_2);
1162            const __m256i q3_3 = __lasx_xvor_v(q3h_3, q3l_3);
1163
1164            // load Q8 quants
1165            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1166            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1167            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1168            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1169
1170            __m256i p16_0 = lasx_madd_h_b(q8_0, q3_0);
1171            __m256i p16_1 = lasx_madd_h_b(q8_1, q3_1);
1172            __m256i p16_2 = lasx_madd_h_b(q8_2, q3_2);
1173            __m256i p16_3 = lasx_madd_h_b(q8_3, q3_3);
1174
1175            // multiply with scales
1176            p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);
1177            p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);
1178            p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);
1179            p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);
1180
1181            // accumulate
1182            p16_0 = __lasx_xvadd_w(p16_0, p16_1);
1183            p16_2 = __lasx_xvadd_w(p16_2, p16_3);
1184            sumi  = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2));
1185        }
1186        // multiply with block scale and accumulate
1187        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);
1188    }
1189
1190    *s = hsum_float_8(acc);
1191
1192#else
1193    UNUSED(kmask1);
1194    UNUSED(kmask2);
1195    UNUSED(x);
1196    UNUSED(y);
1197    UNUSED(nb);
1198    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1199#endif
1200}
1201
1202void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1203    assert(n % QK_K == 0);
1204    assert(nrc == 1);
1205    UNUSED(nrc);
1206    UNUSED(bx);
1207    UNUSED(by);
1208    UNUSED(bs);
1209
1210    const block_q4_K * GGML_RESTRICT x = vx;
1211    const block_q8_K * GGML_RESTRICT y = vy;
1212
1213    const int nb = n / QK_K;
1214
1215    static const uint32_t kmask1 = 0x3f3f3f3f;
1216    static const uint32_t kmask2 = 0x0f0f0f0f;
1217    static const uint32_t kmask3 = 0x03030303;
1218
1219    uint32_t utmp[4];
1220
1221#if defined __loongarch_asx
1222
1223    __m256 acc = (__m256)__lasx_xvldi(0);
1224    __m128 acc_m = (__m128)__lsx_vldi(0);
1225
1226   for (int i = 0; i < nb; ++i) {
1227
1228        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1229        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
1230
1231        memcpy(utmp, x[i].scales, 12);
1232        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
1233        const uint32_t uaux = utmp[1] & kmask1;
1234        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
1235        utmp[2] = uaux;
1236        utmp[0] &= kmask1;
1237
1238        const uint8_t * GGML_RESTRICT q4 = x[i].qs;
1239        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1240
1241        const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);
1242        const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);
1243        const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);
1244
1245        const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);
1246        const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));
1247        const __m128i prod = lsx_madd_h(mins128, q8s);
1248        acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);
1249
1250        const __m256i scales = lasx_insertf128(scales128, scales128);
1251
1252        __m256i sumi = __lasx_xvldi(0);
1253
1254        for (int j = 0; j < QK_K/64; ++j) {
1255
1256            const __m256i scale_l = lasx_xvrepl128vei_h(scales, 2 * j + 0);
1257            const __m256i scale_h = lasx_xvrepl128vei_h(scales, 2 * j + 1);
1258
1259            const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
1260            const __m256i q4l = __lasx_xvandi_b(q4bits, 0xf);
1261            const __m256i q4h = __lasx_xvsrli_b(q4bits, 4);
1262
1263            const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1264            __m256i p16l = lasx_madd_h_b(q4l, q8l);
1265            p16l = lasx_madd_h(scale_l, p16l);
1266
1267            const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1268            __m256i p16h = lasx_madd_h_b(q4h, q8h);
1269            p16h = lasx_madd_h(scale_h, p16h);
1270            const __m256i sumj = __lasx_xvadd_w(p16l, p16h);
1271
1272            sumi = __lasx_xvadd_w(sumi, sumj);
1273        }
1274
1275        __m256 vd = __lasx_xvreplfr2vr_s(d);
1276        acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);
1277
1278    }
1279
1280    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee));
1281    __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0);
1282    acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1);
1283
1284
1285    *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];
1286
1287#else
1288    UNUSED(x);
1289    UNUSED(y);
1290    UNUSED(nb);
1291    UNUSED(kmask1);
1292    UNUSED(kmask2);
1293    UNUSED(kmask3);
1294    UNUSED(utmp);
1295    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1296#endif
1297}
1298
1299void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {
1300    assert(n % QK_K == 0);
1301    assert(nrc == 1);
1302    UNUSED(nrc);
1303    UNUSED(bx);
1304    UNUSED(by);
1305    UNUSED(bs);
1306
1307    const block_q5_K * GGML_RESTRICT x = vx;
1308    const block_q8_K * GGML_RESTRICT y = vy;
1309
1310    const int nb = n / QK_K;
1311
1312    static const uint32_t kmask1 = 0x3f3f3f3f;
1313    static const uint32_t kmask2 = 0x0f0f0f0f;
1314    static const uint32_t kmask3 = 0x03030303;
1315
1316    uint32_t utmp[4];
1317
1318#if defined __loongarch_asx
1319
1320    __m256 acc = (__m256)__lasx_xvldi(0);
1321    __m128 acc_m = (__m128)__lsx_vldi(0);
1322
1323    for (int i = 0; i < nb; ++i) {
1324
1325        const uint8_t * GGML_RESTRICT q5 = x[i].qs;
1326        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1327
1328        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1329        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
1330
1331        memcpy(utmp, x[i].scales, 12);
1332        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
1333        const uint32_t uaux = utmp[1] & kmask1;
1334        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
1335        utmp[2] = uaux;
1336        utmp[0] &= kmask1;
1337
1338        const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);
1339        const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);
1340        const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);
1341
1342        const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);
1343        const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));
1344        const __m128i prod = lsx_madd_h(mins128, q8s);
1345        acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);
1346
1347        const __m256i scales = lasx_insertf128(scales128, scales128);
1348
1349        const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0);
1350
1351        __m256i sumi = __lasx_xvldi(0);
1352
1353        for (int j = 0; j < QK_K/64; ++j) {
1354
1355            const __m256i scale_0 = lasx_xvrepl128vei_h(scales, 2 * j + 0);
1356            const __m256i scale_1 = lasx_xvrepl128vei_h(scales, 2 * j + 1);
1357
1358            const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32;
1359
1360            const __m256i q5l_0 = __lasx_xvandi_b(q5bits, 0xf);
1361            const __m256i q5l_1 = __lasx_xvsrli_b(q5bits, 4);
1362            const __m256i q5h_0 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 0), 0), 0xef);
1363            const __m256i q5h_1 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 1), 0), 0xef);
1364            const __m256i q5_0  = __lasx_xvor_v(q5l_0, q5h_0);
1365            const __m256i q5_1  = __lasx_xvor_v(q5l_1, q5h_1);
1366
1367            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1368            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1369
1370            __m256i p16_0 = lasx_madd_h_b(q5_0, q8_0);
1371            __m256i p16_1 = lasx_madd_h_b(q5_1, q8_1);
1372
1373            p16_0 = lasx_madd_h(scale_0, p16_0);
1374            p16_1 = lasx_madd_h(scale_1, p16_1);
1375
1376            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));
1377
1378        }
1379
1380        __m256 vd = __lasx_xvreplfr2vr_s(d);
1381        acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);
1382
1383    }
1384
1385    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 8));
1386    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 4));
1387
1388    *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];
1389
1390#else
1391    UNUSED(x);
1392    UNUSED(y);
1393    UNUSED(nb);
1394    UNUSED(kmask1);
1395    UNUSED(kmask2);
1396    UNUSED(kmask3);
1397    UNUSED(utmp);
1398    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1399#endif
1400}
1401
1402void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1403    assert(n % QK_K == 0);
1404    assert(nrc == 1);
1405    UNUSED(nrc);
1406    UNUSED(bx);
1407    UNUSED(by);
1408    UNUSED(bs);
1409
1410    const block_q6_K * GGML_RESTRICT x = vx;
1411    const block_q8_K * GGML_RESTRICT y = vy;
1412
1413    const int nb = n / QK_K;
1414
1415#if defined __loongarch_asx
1416
1417    const __m256i m32s = __lasx_xvreplgr2vr_b(32);
1418
1419    __m256 acc = (__m256)__lasx_xvldi(0);
1420
1421    for (int i = 0; i < nb; ++i) {
1422
1423        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1424
1425        const uint8_t * GGML_RESTRICT q4 = x[i].ql;
1426        const uint8_t * GGML_RESTRICT qh = x[i].qh;
1427        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1428
1429        const __m128i scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);
1430        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};
1431        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));
1432
1433        __m256i sumi = __lasx_xvldi(0);
1434
1435        for (int j = 0; j < QK_K/128; ++j) {
1436
1437            const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
1438            const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;
1439            const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32;
1440
1441            const __m256i q4h_0 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3), 4);
1442            const __m256i q4h_1 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3 << 2), 2);
1443            const __m256i q4h_2 = __lasx_xvandi_b(q4bitsH, 3 << 4);
1444            const __m256i q4h_3 = __lasx_xvsrli_b(__lasx_xvandi_b(q4bitsH, 3 << 6), 2);
1445
1446            const __m256i q4_0 = __lasx_xvor_v(__lasx_xvandi_b(q4bits1, 0xf), q4h_0);
1447            const __m256i q4_1 = __lasx_xvor_v(__lasx_xvandi_b(q4bits2, 0xf), q4h_1);
1448            const __m256i q4_2 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits1, 4), q4h_2);
1449            const __m256i q4_3 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits2, 4), q4h_3);
1450
1451            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1452            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1453            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1454            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
1455
1456            __m256i p16_0 = lasx_madd_h_b(__lasx_xvsub_b(q4_0, m32s), q8_0);
1457            __m256i p16_1 = lasx_madd_h_b(__lasx_xvsub_b(q4_1, m32s), q8_1);
1458            __m256i p16_2 = lasx_madd_h_b(__lasx_xvsub_b(q4_2, m32s), q8_2);
1459            __m256i p16_3 = lasx_madd_h_b(__lasx_xvsub_b(q4_3, m32s), q8_3);
1460
1461            p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);
1462            p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);
1463            p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);
1464            p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);
1465
1466            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));
1467            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3));
1468        }
1469
1470        acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);
1471    }
1472
1473    *s = hsum_float_8(acc);
1474
1475#elif defined(__loongarch_sx)
1476
1477    const __m128i m32s = __lsx_vreplgr2vr_b(32);
1478
1479    __m128 acc_0 = (__m128)__lsx_vldi(0);
1480    __m128 acc_1 = (__m128)__lsx_vldi(0);
1481
1482    for (int i = 0; i < nb; ++i) {
1483
1484        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1485
1486        const uint8_t * GGML_RESTRICT q4 = x[i].ql;
1487        const uint8_t * GGML_RESTRICT qh = x[i].qh;
1488        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1489
1490        const __m128i scale_i8 = __lsx_vld(x[i].scales, 0);
1491        const __m128i scales_lo = __lsx_vsllwil_h_b(scale_i8, 0);
1492        const __m128i scales_hi = __lsx_vsllwil_h_b(__lsx_vbsrl_v(scale_i8, 8), 0);
1493
1494        __m128i sumi_0 = __lsx_vldi(0);
1495        __m128i sumi_1 = __lsx_vldi(0);
1496
1497        for (int j = 0; j < QK_K/128; ++j) {
1498
1499            const __m128i q4bitsH_0 = __lsx_vld((const __m128i*)qh, 0); qh += 16;
1500            const __m128i q4bitsH_1 = __lsx_vld((const __m128i*)qh, 0); qh += 16;
1501
1502            const __m128i q4h_0 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_0, 3), 4);
1503            const __m128i q4h_1 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_1, 3), 4);
1504            const __m128i q4h_2 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_0, 3 << 2), 2);
1505            const __m128i q4h_3 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_1, 3 << 2), 2);
1506            const __m128i q4h_4 = __lsx_vandi_b(q4bitsH_0, 3 << 4);
1507            const __m128i q4h_5 = __lsx_vandi_b(q4bitsH_1, 3 << 4);
1508            const __m128i q4h_6 = __lsx_vsrli_b(__lsx_vandi_b(q4bitsH_0, 3 << 6), 2);
1509            const __m128i q4h_7 = __lsx_vsrli_b(__lsx_vandi_b(q4bitsH_1, 3 << 6), 2);
1510
1511            const __m128i q4bits1_0 = __lsx_vld((const __m128i*)q4, 0); q4 += 16;
1512            const __m128i q4bits1_1 = __lsx_vld((const __m128i*)q4, 0); q4 += 16;
1513            const __m128i q4bits2_0 = __lsx_vld((const __m128i*)q4, 0); q4 += 16;
1514            const __m128i q4bits2_1 = __lsx_vld((const __m128i*)q4, 0); q4 += 16;
1515
1516            const __m128i q4_0 = __lsx_vor_v(__lsx_vandi_b(q4bits1_0, 0xf), q4h_0);
1517            const __m128i q4_1 = __lsx_vor_v(__lsx_vandi_b(q4bits1_1, 0xf), q4h_1);
1518            const __m128i q4_2 = __lsx_vor_v(__lsx_vandi_b(q4bits2_0, 0xf), q4h_2);
1519            const __m128i q4_3 = __lsx_vor_v(__lsx_vandi_b(q4bits2_1, 0xf), q4h_3);
1520            const __m128i q4_4 = __lsx_vor_v(__lsx_vsrli_b(q4bits1_0, 4), q4h_4);
1521            const __m128i q4_5 = __lsx_vor_v(__lsx_vsrli_b(q4bits1_1, 4), q4h_5);
1522            const __m128i q4_6 = __lsx_vor_v(__lsx_vsrli_b(q4bits2_0, 4), q4h_6);
1523            const __m128i q4_7 = __lsx_vor_v(__lsx_vsrli_b(q4bits2_1, 4), q4h_7);
1524
1525            const __m128i q8_0 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1526            const __m128i q8_1 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1527            const __m128i q8_2 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1528            const __m128i q8_3 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1529            const __m128i q8_4 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1530            const __m128i q8_5 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1531            const __m128i q8_6 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1532            const __m128i q8_7 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
1533
1534            __m128i p16_0 = lsx_maddubs_h(__lsx_vsub_b(q4_0, m32s), q8_0);
1535            __m128i p16_1 = lsx_maddubs_h(__lsx_vsub_b(q4_1, m32s), q8_1);
1536            __m128i p16_2 = lsx_maddubs_h(__lsx_vsub_b(q4_2, m32s), q8_2);
1537            __m128i p16_3 = lsx_maddubs_h(__lsx_vsub_b(q4_3, m32s), q8_3);
1538            __m128i p16_4 = lsx_maddubs_h(__lsx_vsub_b(q4_4, m32s), q8_4);
1539            __m128i p16_5 = lsx_maddubs_h(__lsx_vsub_b(q4_5, m32s), q8_5);
1540            __m128i p16_6 = lsx_maddubs_h(__lsx_vsub_b(q4_6, m32s), q8_6);
1541            __m128i p16_7 = lsx_maddubs_h(__lsx_vsub_b(q4_7, m32s), q8_7);
1542
1543            const __m128i sc_vec = j == 0 ? scales_lo : scales_hi;
1544
1545            p16_0 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 0), p16_0);
1546            p16_1 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 1), p16_1);
1547            p16_2 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 2), p16_2);
1548            p16_3 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 3), p16_3);
1549            p16_4 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 4), p16_4);
1550            p16_5 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 5), p16_5);
1551            p16_6 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 6), p16_6);
1552            p16_7 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 7), p16_7);
1553
1554            sumi_0 = __lsx_vadd_w(sumi_0, __lsx_vadd_w(p16_0, p16_2));
1555            sumi_1 = __lsx_vadd_w(sumi_1, __lsx_vadd_w(p16_1, p16_3));
1556            sumi_0 = __lsx_vadd_w(sumi_0, __lsx_vadd_w(p16_4, p16_6));
1557            sumi_1 = __lsx_vadd_w(sumi_1, __lsx_vadd_w(p16_5, p16_7));
1558        }
1559
1560        __m128 p_0 = __lsx_vfmul_s(__lsx_vreplfr2vr_s(d), __lsx_vffint_s_w(sumi_0));
1561        __m128 p_1 = __lsx_vfmul_s(__lsx_vreplfr2vr_s(d), __lsx_vffint_s_w(sumi_1));
1562        acc_0 = __lsx_vfadd_s(p_0, acc_0);
1563        acc_1 = __lsx_vfadd_s(p_1, acc_1);
1564    }
1565
1566    *s = hsum_float_4x4(acc_0, acc_1, (__m128)__lsx_vldi(0), (__m128)__lsx_vldi(0));
1567
1568#else
1569    UNUSED(x);
1570    UNUSED(y);
1571    UNUSED(nb);
1572    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1573#endif
1574}
1575
1576#if defined(__loongarch_asx)
1577static const int8_t keven_signs_q2xs[1024] = {
1578     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,
1579     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,
1580     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,
1581     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,
1582     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,
1583     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,
1584     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,
1585     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,
1586     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,
1587     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,
1588     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,
1589     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,
1590     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,
1591     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,
1592     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,
1593     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,
1594     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,
1595     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,
1596     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,
1597     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,
1598     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,
1599     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,
1600     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,
1601     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,
1602     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,
1603     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,
1604     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,
1605     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,
1606     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,
1607     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,
1608     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,
1609     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,
1610};
1611#endif
1612
1613void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1614    assert(n % QK_K == 0);
1615    assert(nrc == 1);
1616    UNUSED(nrc);
1617    UNUSED(bx);
1618    UNUSED(by);
1619    UNUSED(bs);
1620
1621    const block_iq2_xxs * GGML_RESTRICT x = vx;
1622    const block_q8_K    * GGML_RESTRICT y = vy;
1623
1624    const int nb = n / QK_K;
1625
1626#if defined(__loongarch_asx)
1627
1628    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
1629
1630    uint32_t aux32[4];
1631    const uint8_t * aux8 = (const uint8_t *)aux32;
1632
1633    __m256 accumf = (__m256)__lasx_xvldi(0);
1634    for (int i = 0; i < nb; ++i) {
1635        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1636        const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1637        const int8_t   * GGML_RESTRICT q8 = y[i].qs;
1638        __m256i sumi1 = __lasx_xvldi(0);
1639        __m256i sumi2 = __lasx_xvldi(0);
1640        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
1641            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1642            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1643            memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;
1644
1645            const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);
1646            const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);
1647            const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
1648                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);
1649            const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],
1650                                                   signs64[(aux32[3] >>  7) & 127], signs64[(aux32[3] >>  0) & 127]);
1651            const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);
1652            const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);
1653            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);
1654            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);
1655            const uint16_t ls1 = aux32[1] >> 28;
1656            const uint16_t ls2 = aux32[3] >> 28;
1657            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
1658            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
1659            sumi1 = __lasx_xvadd_w(sumi1, p1);
1660            sumi2 = __lasx_xvadd_w(sumi2, p2);
1661        }
1662
1663        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
1664    }
1665
1666    *s = 0.125f * hsum_float_8(accumf);
1667
1668#else
1669    UNUSED(x);
1670    UNUSED(y);
1671    UNUSED(nb);
1672    ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1673#endif
1674}
1675
1676void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1677    assert(n % QK_K == 0);
1678    assert(nrc == 1);
1679    UNUSED(nrc);
1680    UNUSED(bx);
1681    UNUSED(by);
1682    UNUSED(bs);
1683
1684    const block_iq2_xs * GGML_RESTRICT x = vx;
1685    const block_q8_K   * GGML_RESTRICT y = vy;
1686
1687    const int nb = n / QK_K;
1688
1689#if defined(__loongarch_asx)
1690
1691    const __m256i mone = __lasx_xvreplgr2vr_b(1);
1692    static const char block_sign_shuffle_mask_1[32] = {
1693        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
1694        0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
1695    };
1696    static const char block_sign_shuffle_mask_2[32] = {
1697        0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,
1698        0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,
1699    };
1700    static const uint8_t bit_selector_mask_bytes[32] = {
1701        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1702        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1703    };
1704
1705    const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0);
1706    const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0);
1707    const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0);
1708
1709    static const uint8_t k_bit_helper[32] = {
1710        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
1711        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,
1712    };
1713    const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0);
1714    const __m256i m511 = __lasx_xvreplgr2vr_h(511);
1715    const __m128i m4 = __lsx_vreplgr2vr_b(0xf);
1716    const __m128i m1 = __lsx_vreplgr2vr_b(1);
1717
1718    uint64_t aux64;
1719
1720    // somewhat hacky, but gives a significant boost in performance
1721    __m256i aux_gindex;
1722    const uint16_t * gindex = (const uint16_t *)&aux_gindex;
1723
1724    __m256 accumf = (__m256)__lasx_xvldi(0);
1725    for (int i = 0; i < nb; ++i) {
1726        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1727        const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1728        const int8_t   * GGML_RESTRICT q8 = y[i].qs;
1729
1730        memcpy(&aux64, x[i].scales, 8);
1731        __m128i stmp = __lsx_vreplgr2vr_d(aux64);
1732        stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4));
1733        const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1);
1734
1735        __m256i sumi1 = __lasx_xvldi(0);
1736        __m256i sumi2 = __lasx_xvldi(0);
1737        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {
1738
1739            const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0);  q2 += 16;
1740            aux_gindex = __lasx_xvand_v(q2_data, m511);
1741
1742            const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9);
1743            const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13);
1744            const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper);
1745
1746            const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting);
1747            const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits);
1748
1749            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1750            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1751            const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1752            const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1753
1754            const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],
1755                                                   iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);
1756            const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],
1757                                                   iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);
1758            const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],
1759                                                   iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);
1760            const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],
1761                                                   iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);
1762
1763            const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0);
1764            const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1);
1765            const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l);
1766            const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h);
1767
1768            __m256i signs;
1769            signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1);
1770            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
1771            const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);
1772
1773            signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2);
1774            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
1775            const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);
1776
1777            signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1);
1778            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
1779            const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3);
1780
1781            signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2);
1782            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);
1783            const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4);
1784
1785            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);
1786            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);
1787            const __m256i dot3  = lasx_maddubs_h(q2_3, q8s_3);
1788            const __m256i dot4  = lasx_maddubs_h(q2_4, q8s_4);
1789
1790            const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0)));
1791            const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1)));
1792            const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2)));
1793            const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3)));
1794
1795            sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1));
1796            sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2));
1797            sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3));
1798            sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4));
1799        }
1800
1801        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
1802
1803    }
1804
1805    *s = 0.125f * hsum_float_8(accumf);
1806
1807#else
1808    UNUSED(x);
1809    UNUSED(y);
1810    UNUSED(nb);
1811    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1812#endif
1813}
1814
1815void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1816    assert(n % QK_K == 0);
1817    assert(nrc == 1);
1818    UNUSED(nrc);
1819    UNUSED(bx);
1820    UNUSED(by);
1821    UNUSED(bs);
1822
1823    const block_iq2_s * GGML_RESTRICT x = vx;
1824    const block_q8_K  * GGML_RESTRICT y = vy;
1825
1826    const int nb = n / QK_K;
1827
1828#if defined(__loongarch_asx)
1829
1830   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
1831                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
1832   };
1833
1834    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1835                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1836    };
1837
1838
1839    const __m128i m4 = __lsx_vreplgr2vr_b(0xf);
1840    const __m128i m1 = __lsx_vreplgr2vr_b(1);
1841
1842    const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);
1843    const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);
1844    uint64_t aux64;
1845
1846    __m256 accumf = (__m256)__lasx_xvldi(0);
1847    for (int i = 0; i < nb; ++i) {
1848        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1849        const uint8_t * GGML_RESTRICT qs = x[i].qs;
1850        const uint8_t * GGML_RESTRICT qh = x[i].qh;
1851        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);
1852        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1853
1854        __m128i tmp1;
1855        memcpy(&aux64, x[i].scales, 8);
1856        tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0);
1857        tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1);
1858        const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1);
1859        const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15
1860
1861        __m256i sumi1 = __lasx_xvldi(0);
1862        __m256i sumi2 = __lasx_xvldi(0);
1863        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
1864            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1865            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1866            const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],
1867                                                   iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],
1868                                                   iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],
1869                                                   iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);
1870            const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],
1871                                                   iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],
1872                                                   iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],
1873                                                   iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);
1874            qs += 8;
1875
1876            __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16));
1877            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
1878            const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);
1879            const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);
1880
1881            aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16));
1882            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
1883            const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);
1884            const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);
1885
1886            signs += 4;
1887
1888            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1
1889            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3
1890
1891            const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0)));
1892            const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1)));
1893            sumi1 = __lasx_xvadd_w(sumi1, p1);
1894            sumi2 = __lasx_xvadd_w(sumi2, p2);
1895        }
1896
1897        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
1898    }
1899
1900    *s = 0.125f * hsum_float_8(accumf);
1901
1902#else
1903    UNUSED(x);
1904    UNUSED(y);
1905    UNUSED(nb);
1906    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1907#endif
1908}
1909
1910void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1911    assert(n % QK_K == 0);
1912    assert(nrc == 1);
1913    UNUSED(nrc);
1914    UNUSED(bx);
1915    UNUSED(by);
1916    UNUSED(bs);
1917
1918    const block_iq3_xxs * GGML_RESTRICT x = vx;
1919    const block_q8_K    * GGML_RESTRICT y = vy;
1920
1921    const int nb = n / QK_K;
1922
1923#if defined(__loongarch_asx)
1924
1925    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;
1926
1927    uint32_t aux32[2];
1928
1929    __m256 accumf = (__m256)__lasx_xvldi(0);
1930    for (int i = 0; i < nb; ++i) {
1931        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1932        const uint8_t * GGML_RESTRICT q3 = x[i].qs;
1933        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;
1934        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
1935        __m256i sumi1 = __lasx_xvldi(0);
1936        __m256i sumi2 = __lasx_xvldi(0);
1937        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
1938            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1939            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
1940            const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
1941                                                iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
1942            q3 += 8;
1943            const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],
1944                                                iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);
1945            q3 += 8;
1946            memcpy(aux32, gas, 8); gas += 8;
1947
1948            const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],
1949                                                   signs64[(aux32[0] >>  7) & 127], signs64[(aux32[0] >>  0) & 127]);
1950            const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],
1951                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);
1952            const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);
1953            const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);
1954            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);
1955            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);
1956            const uint16_t ls1 = aux32[0] >> 28;
1957            const uint16_t ls2 = aux32[1] >> 28;
1958
1959            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
1960            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
1961            sumi1 = __lasx_xvadd_w(sumi1, p1);
1962            sumi2 = __lasx_xvadd_w(sumi2, p2);
1963        }
1964
1965        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
1966    }
1967
1968    *s = 0.25f * hsum_float_8(accumf);
1969
1970#else
1971    UNUSED(x);
1972    UNUSED(y);
1973    UNUSED(nb);
1974    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1975#endif
1976}
1977
1978void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
1979    assert(n % QK_K == 0);
1980    assert(nrc == 1);
1981    UNUSED(nrc);
1982    UNUSED(bx);
1983    UNUSED(by);
1984    UNUSED(bs);
1985
1986    const block_iq3_s * GGML_RESTRICT x = vx;
1987    const block_q8_K  * GGML_RESTRICT y = vy;
1988
1989    const int nb = n / QK_K;
1990
1991#if defined(__loongarch_asx)
1992
1993   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
1994                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03
1995   };
1996
1997    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1998                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
1999    };
2000
2001    const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);
2002    const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);
2003
2004    __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8);
2005    const __m256i idx_mask  = __lasx_xvreplgr2vr_w(256);
2006
2007    typedef union {
2008        __m256i  vec[2];
2009        uint32_t index[16];
2010    } index_t;
2011
2012    index_t idx;
2013
2014    __m256 accumf = (__m256)__lasx_xvldi(0);
2015    for (int i = 0; i < nb; ++i) {
2016        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
2017        const uint8_t * GGML_RESTRICT qs = x[i].qs;
2018        const uint8_t * GGML_RESTRICT qh = x[i].qh;
2019        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;
2020        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
2021        __m256i sumi1 = __lasx_xvldi(0);
2022        __m256i sumi2 = __lasx_xvldi(0);
2023        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
2024            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
2025            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
2026            const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16;
2027            idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]);
2028            idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]);
2029            idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask);
2030            idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask);
2031            idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0)));
2032            idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1)));
2033
2034            // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.
2035            //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);
2036            //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);
2037            const __m256i q2_1 = lasx_set_w(
2038                    iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],
2039                    iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]
2040            );
2041            const __m256i q2_2 = lasx_set_w(
2042                    iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],
2043                    iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]
2044            );
2045
2046            __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16));
2047            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
2048            const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);
2049            const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);
2050
2051            aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16));
2052            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);
2053            const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);
2054            const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);
2055
2056            signs += 4;
2057
2058            const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);
2059            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);
2060            const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;
2061            const uint16_t ls2 = x[i].scales[ib32/2] >>  4;
2062            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));
2063            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));
2064            sumi1 = __lasx_xvadd_w(sumi1, p1);
2065            sumi2 = __lasx_xvadd_w(sumi2, p2);
2066        }
2067
2068        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);
2069    }
2070
2071    *s = hsum_float_8(accumf);
2072
2073#else
2074    UNUSED(x);
2075    UNUSED(y);
2076    UNUSED(nb);
2077    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2078#endif
2079}
2080
2081#if defined(__loongarch_asx)
2082static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {
2083    const __m256i a = __lasx_xvmulwev_h_b(x, y);
2084    const __m256i b = __lasx_xvmulwod_h_b(x, y);
2085    return __lasx_xvadd_h(a, b);
2086}
2087#endif
2088
2089void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
2090    assert(n % QK_K == 0);
2091    assert(nrc == 1);
2092    UNUSED(nrc);
2093    UNUSED(bx);
2094    UNUSED(by);
2095    UNUSED(bs);
2096
2097    const block_iq1_s * GGML_RESTRICT x = vx;
2098    const block_q8_K  * GGML_RESTRICT y = vy;
2099
2100    const int nb = n / QK_K;
2101
2102#if defined(__loongarch_asx)
2103
2104    __m256 accum = (__m256)__lasx_xvldi(0);
2105    float accum1 = 0;
2106    for (int i = 0; i < nb; ++i) {
2107
2108        const int8_t   * q8 = y[i].qs;
2109        const uint8_t  * qs = x[i].qs;
2110        const uint16_t * qh = x[i].qh;
2111
2112        __m256i sumi = __lasx_xvldi(0);
2113        int sumi1 = 0;
2114        for (int ib = 0; ib < QK_K/32; ib += 2) {
2115            __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0);
2116            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1);
2117            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2);
2118            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3);
2119
2120            __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0);
2121            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1);
2122            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2);
2123            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3);
2124
2125            qs += 8;
2126            const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
2127            const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;
2128
2129            const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);
2130            const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);
2131            const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;
2132            const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;
2133
2134            __m256i tmp1, tmp5, tmp6;
2135            tmp1 = __lasx_xvreplgr2vr_h(ls1);
2136            tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1);
2137            tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1);
2138            const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6);
2139
2140            tmp1 = __lasx_xvreplgr2vr_h(ls2);
2141            tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1);
2142            tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1);
2143            const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6);
2144
2145            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2));
2146            sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1
2147                   + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;
2148        }
2149
2150        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
2151        accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum);
2152        accum1 += d * sumi1;
2153    }
2154
2155    *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;
2156
2157#else
2158    UNUSED(x);
2159    UNUSED(y);
2160    UNUSED(nb);
2161    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2162#endif
2163}
2164
2165void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
2166    assert(nrc == 1);
2167    UNUSED(nrc);
2168    UNUSED(bx);
2169    UNUSED(by);
2170    UNUSED(bs);
2171    assert(n % QK4_NL == 0);
2172    static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");
2173
2174    const block_iq4_nl * GGML_RESTRICT x = vx;
2175    const block_q8_0   * GGML_RESTRICT y = vy;
2176
2177    const int nb = n / QK4_NL;
2178
2179    int ib = 0;
2180    float sumf = 0;
2181
2182#if defined (__loongarch_asx)
2183
2184    const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);
2185    const __m128i m4b  = __lsx_vreplgr2vr_b(0x0f);
2186    const __m256i mone = __lasx_xvreplgr2vr_h(1);
2187
2188    __m256 accum1 = (__m256)__lasx_xvldi(0);
2189    __m256 accum2 = (__m256)__lasx_xvldi(0);
2190    for (; ib + 1 < nb; ib += 2) {
2191        const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[ib + 0].qs, 0);
2192        const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[ib + 1].qs, 0);
2193        const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[ib + 0].qs, 0);
2194        const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[ib + 1].qs, 0);
2195        const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)),
2196                                              lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b)));
2197        const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)),
2198                                              lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b)));
2199        const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
2200        const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
2201        const __m256i p_1 = lasx_madd_h(p16_1, mone);
2202        const __m256i p_2 = lasx_madd_h(p16_2, mone);
2203        accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),
2204                __lasx_xvffint_s_w(p_1), accum1);
2205        accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),
2206                __lasx_xvffint_s_w(p_2), accum2);
2207    }
2208
2209    sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2));
2210
2211#endif
2212    for (; ib < nb; ++ib) {
2213        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);
2214        int sumi1 = 0, sumi2 = 0;
2215        for (int j = 0; j < QK4_NL/2; ++j) {
2216            sumi1 += y[ib].qs[j+       0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];
2217            sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >>  4];
2218        }
2219        sumf += d * (sumi1 + sumi2);
2220    }
2221    *s = sumf;
2222}
2223
2224void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
2225    assert(nrc == 1);
2226    UNUSED(nrc);
2227    UNUSED(bx);
2228    UNUSED(by);
2229    UNUSED(bs);
2230    assert(n % QK_K == 0);
2231
2232    const block_iq4_xs * GGML_RESTRICT x = vx;
2233    const block_q8_K   * GGML_RESTRICT y = vy;
2234
2235    const int nb = n / QK_K;
2236
2237#if defined(__loongarch_asx)
2238
2239    const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);
2240
2241    __m256 accum = (__m256)__lasx_xvldi(0);
2242
2243    for (int ibl = 0; ibl < nb; ++ibl) {
2244        const uint8_t * qs = x[ibl].qs;
2245        const int8_t  * q8 = y[ibl].qs;
2246        uint16_t sh = x[ibl].scales_h;
2247        __m256i sumi1 = __lasx_xvldi(0);
2248        __m256i sumi2 = __lasx_xvldi(0);
2249        for (int ib = 0; ib < QK_K/32; ib += 2) {
2250            const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16;
2251            const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16;
2252            const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
2253            const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;
2254            const __m256i q4b_1 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_1, 4)),
2255                                                  __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_1, 0xf)));
2256            const __m256i q4b_2 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_2, 4)),
2257                                                  __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_2, 0xf)));
2258            const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);
2259            const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);
2260            const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;
2261            const int16_t ls2 = ((x[ibl].scales_l[ib/2] >>  4) | ((sh << 2) & 0x30)) - 32;
2262            sh >>= 4;
2263            const __m256i p_1 = lasx_madd_h(p16_1, __lasx_xvreplgr2vr_h(ls1));
2264            const __m256i p_2 = lasx_madd_h(p16_2, __lasx_xvreplgr2vr_h(ls2));
2265            sumi1 = __lasx_xvadd_w(p_1, sumi1);
2266            sumi2 = __lasx_xvadd_w(p_2, sumi2);
2267        }
2268        accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),
2269                __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum);
2270    }
2271
2272    *s = hsum_float_8(accum);
2273
2274#elif defined(__loongarch_sx)
2275
2276    const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);
2277
2278    __m128 accum = (__m128)__lsx_vldi(0);
2279    for (int ibl = 0; ibl < nb; ++ibl) {
2280        const uint8_t * qs = x[ibl].qs;
2281        const int8_t  * q8 = y[ibl].qs;
2282        uint16_t sh = x[ibl].scales_h;
2283        __m128i sumi = __lsx_vldi(0);
2284        for (int ib = 0; ib < QK_K/32; ++ib) {
2285            const __m128i q4bits = __lsx_vld((const __m128i*)qs, 0); qs += 16;
2286            const __m128i q8b_0 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
2287            const __m128i q8b_1 = __lsx_vld((const __m128i*)q8, 0); q8 += 16;
2288            const __m128i q4b_0 = __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits, 0xf));
2289            const __m128i q4b_1 = __lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits, 4));
2290            const __m128i p16_0 = lsx_maddubs_h(q4b_0, q8b_0);
2291            const __m128i p16_1 = lsx_maddubs_h(q4b_1, q8b_1);
2292            const int16_t ls = (((x[ibl].scales_l[ib/2] >> ((ib & 1) * 4)) & 0xf) | ((sh & 0x3) << 4)) - 32;
2293            sh >>= 2;
2294            sumi = __lsx_vadd_w(lsx_madd_h(p16_0, __lsx_vreplgr2vr_h(ls)), sumi);
2295            sumi = __lsx_vadd_w(lsx_madd_h(p16_1, __lsx_vreplgr2vr_h(ls)), sumi);
2296        }
2297        const float ds = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;
2298        accum = __lsx_vfadd_s(__lsx_vfmul_s(__lsx_vreplfr2vr_s(ds), __lsx_vffint_s_w(sumi)), accum);
2299    }
2300
2301    *s = ((v4f32)lsx_hadd_s(lsx_hadd_s(accum, accum), lsx_hadd_s(accum, accum)))[0];
2302
2303#else
2304    UNUSED(x);
2305    UNUSED(y);
2306    UNUSED(nb);
2307    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2308#endif
2309}