char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
81b0d882a
char/aliceai-t5-moe
1#include "ggml-impl.h" 2#include "ggml.h" 3#include "ime_kernels.h" 4#include "rvv_kernels.h" 5 6#include <algorithm> 7#include <cmath> 8#include <stdexcept> 9 10#if !defined(__riscv_v )|| !defined(__riscv_v_intrinsic ) 11# error "riscv v extension or v_intrinsic not enabled" 12#else 13# include <riscv_vector.h> 14#endif 15 16#if !defined(__riscv_zfh ) 17# error "riscv zfh extension not enabled" 18#endif 19 20#if defined(RISCV64_SPACEMIT_IME1 ) 21#else 22# error "RISCV64_SPACEMIT_IME1 not defined" 23#endif 24 25// clang-format off 26#if defined(__GNUC__ ) 27#pragma GCC diagnostic ignored "-Woverlength-strings" 28#pragma GCC diagnostic ignored "-Wcast-qual" 29#pragma GCC diagnostic ignored "-Wunused-parameter" 30#endif 31// clang-format on 32namespace spacemit_kernels { 33 34#define QUANTIZEM4ROW_KERNEL \ 35 "vmv.s.x v16, zero \n\t" \ 36 "vfabs.v v8, v0 \n\t" \ 37 "vfredmax.vs v16, v8, v16 \n\t" \ 38 "vfmv.f.s f10, v16 \n\t" \ 39 "fmul.s f10, f10, %[RMAXREC] \n\t" \ 40 "fsw f10, (a1) \n\t" \ 41 "fdiv.s f11, %[FONE], f10 \n\t" \ 42 "vfmul.vf v16, v0, f11 \n\t" \ 43 "vfcvt.x.f.v v16, v16 \n\t" \ 44 "vsetvli t0, zero, e16, mf2 \n\t" \ 45 "vnclip.wx v16, v16, zero \n\t" \ 46 "vnclip.wx v17, v17, zero \n\t" \ 47 "vnclip.wx v18, v18, zero \n\t" \ 48 "vnclip.wx v19, v19, zero \n\t" \ 49 "vnclip.wx v20, v20, zero \n\t" \ 50 "vnclip.wx v21, v21, zero \n\t" \ 51 "vnclip.wx v22, v22, zero \n\t" \ 52 "vnclip.wx v23, v23, zero \n\t" \ 53 "vsetvli t0, zero, e8, mf4 \n\t" \ 54 "vnclip.wx v24, v16, zero \n\t" \ 55 "vnclip.wx v25, v17, zero \n\t" \ 56 "vnclip.wx v26, v18, zero \n\t" \ 57 "vnclip.wx v27, v19, zero \n\t" \ 58 "vnclip.wx v28, v20, zero \n\t" \ 59 "vnclip.wx v29, v21, zero \n\t" \ 60 "vnclip.wx v30, v22, zero \n\t" \ 61 "vnclip.wx v31, v23, zero \n\t" 62 63#define QUANTIZEM4ROW_STORE \ 64 "addi t1, %[BlkLen], 0 \n\t" \ 65 "vsetvli t0, t1, e8, mf4 \n\t" \ 66 "vse8.v v24, (s1) \n\t" \ 67 "addi s1, s1, 32 \n\t" \ 68 "sub t1, t1, t0 \n\t" \ 69 "vsetvli t0, t1, e8, mf4 \n\t" \ 70 "vse8.v v25, (s1) \n\t" \ 71 "addi s1, s1, 32 \n\t" \ 72 "sub t1, t1, t0 \n\t" \ 73 "vsetvli t0, t1, e8, mf4 \n\t" \ 74 "vse8.v v26, (s1) \n\t" \ 75 "addi s1, s1, 32 \n\t" \ 76 "sub t1, t1, t0 \n\t" \ 77 "vsetvli t0, t1, e8, mf4 \n\t" \ 78 "vse8.v v27, (s1) \n\t" \ 79 "addi s1, s1, 32 \n\t" \ 80 "sub t1, t1, t0 \n\t" \ 81 "vsetvli t0, t1, e8, mf4 \n\t" \ 82 "vse8.v v28, (s1) \n\t" \ 83 "addi s1, s1, 32 \n\t" \ 84 "sub t1, t1, t0 \n\t" \ 85 "vsetvli t0, t1, e8, mf4 \n\t" \ 86 "vse8.v v29, (s1) \n\t" \ 87 "addi s1, s1, 32 \n\t" \ 88 "sub t1, t1, t0 \n\t" \ 89 "vsetvli t0, t1, e8, mf4 \n\t" \ 90 "vse8.v v30, (s1) \n\t" \ 91 "addi s1, s1, 32 \n\t" \ 92 "sub t1, t1, t0 \n\t" \ 93 "vsetvli t0, t1, e8, mf4 \n\t" \ 94 "vse8.v v31, (s1) \n\t" 95 96namespace ime1 { 97void quantize_a_4row_i8 (size_t BlkLen ,const float * A ,size_t CountK ,uint8_t * QuantA ) { 98constexpr float range_max_reciprocal = 1.0f / ((1 <<7 )- 1 ); 99const float fone = 1.0f ; 100 101for (size_t row_index = 0 ;row_index < 4 ;++ row_index ) { 102const float * SRC = A + row_index * CountK ; 103uint8_t * DST = QuantA + row_index * sizeof (float ); 104 105const size_t offset = (4 - row_index )* 4 + row_index * 8 ; 106const size_t stride = 4 * (sizeof (float )+ BlkLen ); 107 __asm__volatile ( 108"vsetvli t0, zero, e32, m8 \n\t" 109"addi t2, %[CountK], 0 \n\t" 110"addi a1, %[DST], 0 \n\t" 111"blt t2, %[BlkLen], TAIL%= \n\t" 112 113"LOOP%=: \n\t" 114"vsetvli t0, %[BlkLen], e32, m8 \n\t" 115"vle32.v v0, (%[SRC]) \n\t" 116"sub t2, t2, t0 \n\t" 117"slli t1, t0, 2 \n\t" 118"add %[SRC], %[SRC], t1 \n\t" 119"add s1, a1, %[OFFSET] \n\t" 120 121QUANTIZEM4ROW_KERNEL QUANTIZEM4ROW_STORE 122 123"add a1, a1, %[STRIDE] \n\t" 124"bge t2, %[BlkLen], LOOP%= \n\t" 125 126"TAIL%=: \n\t" 127"blez t2, QUIT%= \n\t" 128"vsetvli t0, zero, e32, m8 \n\t" 129"vxor.vv v16, v16, v16 \n\t" 130"vxor.vv v24, v24, v24 \n\t" 131"vsetvli t0, t2, e32, m8 \n\t" 132"vle32.v v0, (%[SRC]) \n\t" 133"add s1, a1, %[OFFSET] \n\t" 134 135QUANTIZEM4ROW_KERNEL 136 137"addi t3, %[BlkLen], 0 \n\t" 138"addi s2, s1, 0 \n\t" 139"vsetvli t0, zero, e8, mf4 \n\t" 140"vxor.vv v8, v8, v8 \n\t" 141"SET_ZERO%=: \n\t" 142"vse8.v v8, (s2) \n\t" 143"addi s2, s2, 32 \n\t" 144"addi t3, t3, -8 \n\t" 145"bnez t3, SET_ZERO%= \n\t" 146 147QUANTIZEM4ROW_STORE 148 149"QUIT%=: \n\t" 150 : [SRC ]"+r" (SRC ) 151 : [DST ]"r" (DST ), [BlkLen ]"r" (BlkLen ), [OFFSET ]"r" (offset ), [STRIDE ]"r" (stride ), [CountK ]"r" (CountK ), 152 [FONE ]"f" (fone ), [RMAXREC ]"f" (range_max_reciprocal ) 153 :"cc" ,"t0" ,"t1" ,"t2" ,"t3" ,"a1" ,"s1" ,"s2" ,"f10" ,"f11" ); 154 } 155} 156 157void quantize_a_row_i8 (size_t BlkLen ,const float * A ,size_t CountK ,uint8_t * QuantA ) { 158const float * SRC = A ; 159uint8_t * DST = QuantA ; 160constexpr float range_max_reciprocal = 1.0f / ((1 <<7 )- 1 ); 161const float fone = 1.0f ; 162uint8_t * QuantA_offset = QuantA + CountK + 4 * ((CountK + BlkLen - 1 ) /BlkLen ); 163size_t offset = (CountK + BlkLen - 1 ) /BlkLen * BlkLen - CountK ; 164 165 __asm__volatile ( 166"addi t3, zero, 32*4 \n\t" 167"addi t2, zero, 32 \n\t" 168 169"addi a1, %[SRC], 0 \n\t" 170"addi a2, %[SRC], 128 \n\t" 171"addi a3, %[SRC], 256 \n\t" 172"addi a4, %[SRC], 384 \n\t" 173 174"addi s1, %[DST], 0 \n\t" 175"addi s2, %[DST], 36 \n\t" 176"addi s3, %[DST], 72 \n\t" 177"addi s4, %[DST], 108 \n\t" 178"blt %[K], t3, LOOP_K%= \n\t" 179"blt %[K], t2, TAIL%= \n\t" 180 181"LOOP_MAIN%=: \n\t" 182"vsetvli t1, zero, e32, m4 \n\t" 183"addi %[K], %[K], -128 \n\t" 184"vle32.v v0, (a1) \n\t" 185"addi a1, a1, 512 \n\t" 186"vle32.v v4, (a2) \n\t" 187"addi a2, a2, 512 \n\t" 188"vle32.v v8, (a3) \n\t" 189"addi a3, a3, 512 \n\t" 190"vle32.v v12, (a4) \n\t" 191"addi a4, a4, 512 \n\t" 192"vfabs.v v16, v0 \n\t" 193"vfabs.v v20, v4 \n\t" 194"vfabs.v v24, v8 \n\t" 195"vfabs.v v28, v12 \n\t" 196"vsetvli t0, zero, e32, m2 \n\t" 197"vfmax.vv v16, v16, v18 \n\t" 198"vfmax.vv v20, v20, v22 \n\t" 199"vfmax.vv v24, v24, v26 \n\t" 200"vfmax.vv v28, v28, v30 \n\t" 201"vsetvli t0, zero, e32, m1 \n\t" 202"vfmax.vv v16, v16, v17 \n\t" 203"vfmax.vv v20, v20, v21 \n\t" 204"vfmax.vv v24, v24, v25 \n\t" 205"vfmax.vv v28, v28, v29 \n\t" 206 207"vfredmax.vs v17, v16, v17 \n\t" 208"vfredmax.vs v21, v20, v21 \n\t" 209"vfredmax.vs v25, v24, v25 \n\t" 210"vfredmax.vs v29, v28, v29 \n\t" 211"vfmv.f.s f10, v17 \n\t" 212"vfmv.f.s f11, v21 \n\t" 213"vfmv.f.s f12, v25 \n\t" 214"vfmv.f.s f13, v29 \n\t" 215 216"fmul.s f10, f10, %[RMAXREC] \n\t" 217"fmul.s f11, f11, %[RMAXREC] \n\t" 218"fmul.s f12, f12, %[RMAXREC] \n\t" 219"fmul.s f13, f13, %[RMAXREC] \n\t" 220"fsw f10, (s1) \n\t" 221"addi s1, s1, 4 \n\t" 222 223"fsw f11, (s2) \n\t" 224"addi s2, s2, 4 \n\t" 225"fsw f12, (s3) \n\t" 226"addi s3, s3, 4 \n\t" 227"fsw f13, (s4) \n\t" 228"addi s4, s4, 4 \n\t" 229"fdiv.s f10, %[FONE], f10 \n\t" 230"fdiv.s f11, %[FONE], f11 \n\t" 231"fdiv.s f12, %[FONE], f12 \n\t" 232"fdiv.s f13, %[FONE], f13 \n\t" 233"vsetvli t0, zero, e32, m4 \n\t" 234"vfmul.vf v16, v0, f10 \n\t" 235"vfmul.vf v20, v4, f11 \n\t" 236"vfmul.vf v24, v8, f12 \n\t" 237"vfmul.vf v28, v12, f13 \n\t" 238"vfcvt.x.f.v v16, v16 \n\t" 239"vfcvt.x.f.v v20, v20 \n\t" 240"vfcvt.x.f.v v24, v24 \n\t" 241"vfcvt.x.f.v v28, v28 \n\t" 242"vsetvli t0, zero, e16, m2 \n\t" 243"vnclip.wx v16, v16, zero \n\t" 244"vnclip.wx v20, v20, zero \n\t" 245"vnclip.wx v24, v24, zero \n\t" 246"vnclip.wx v28, v28, zero \n\t" 247"vsetvli t0, t1, e8, m1 \n\t" 248"vnclip.wx v16, v16, zero \n\t" 249"vnclip.wx v20, v20, zero \n\t" 250"vnclip.wx v24, v24, zero \n\t" 251"vnclip.wx v28, v28, zero \n\t" 252"vse8.v v16, (s1) \n\t" 253"addi s1, s1, 140 \n\t" 254"vse8.v v20, (s2) \n\t" 255"addi s2, s2, 140 \n\t" 256"vse8.v v24, (s3) \n\t" 257"addi s3, s3, 140 \n\t" 258"vse8.v v28, (s4) \n\t" 259"addi s4, s4, 140 \n\t" 260"bge %[K], t3, LOOP_MAIN%= \n\t" 261"blt %[K], t2, TAIL%= \n\t" 262"LOOP_K%=: \n\t" 263"vsetvli t1, %[K], e32, m4 \n\t" 264"vle32.v v0, (a1) \n\t" 265"addi a1, a1, 128 \n\t" 266"sub %[K], %[K], t1 \n\t" 267"vfabs.v v16, v0 \n\t" 268"vsetvli t0, zero, e32, m2 \n\t" 269"vfmax.vv v16, v16, v18 \n\t" 270"vsetvli t0, zero, e32, m1 \n\t" 271"vfmax.vv v16, v16, v17 \n\t" 272"vfredmax.vs v17, v16, v17 \n\t" 273"vfmv.f.s f10, v17 \n\t" 274 275"fmul.s f10, f10, %[RMAXREC] \n\t" 276"fsw f10, (s1) \n\t" 277"addi s1, s1, 4 \n\t" 278"fdiv.s f11, %[FONE], f10 \n\t" 279"vsetvli t0, zero, e32, m4 \n\t" 280"vfmul.vf v16, v0, f11 \n\t" 281"vfcvt.x.f.v v16, v16 \n\t" 282"vsetvli t0, zero, e16, m2 \n\t" 283"vnclip.wx v16, v16, zero \n\t" 284"vsetvli t0, zero, e8, m1 \n\t" 285"vnclip.wx v16, v16, zero \n\t" 286"vse8.v v16, (s1) \n\t" 287"addi s1, s1, 32 \n\t" 288"bge %[K], t2, LOOP_K%= \n\t" 289"TAIL%=: \n\t" 290"blez %[K], END%= \n\t" 291"vsetvli t0, t3, e32, m4 \n\t" 292"vxor.vv v0, v0, v0 \n\t" 293"vxor.vv v16, v16, v16 \n\t" 294"jal x0, LOOP_K%= \n\t" 295"END%=: \n\t" 296 : [K ]"+r" (CountK ) 297 : [FONE ]"f" (fone ), [RMAXREC ]"f" (range_max_reciprocal ), [SRC ]"r" (SRC ), [DST ]"r" (DST ) 298 :"cc" ,"t3" ,"t2" ,"t1" ,"t0" ,"a1" ,"a2" ,"a3" ,"a4" ,"s1" ,"s2" ,"s3" ,"s4" ,"f10" ,"f11" ,"f12" ,"f13" ); 299} 300 301}// namespace ime1 302 303namespace { 304#define SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 \ 305 "vmadot v16, v14, v0 \n\t" \ 306 "vmadot v18, v14, v1 \n\t" \ 307 "vmadot v20, v14, v2 \n\t" \ 308 "vmadot v22, v14, v3 \n\t" \ 309 "vmadot v16, v15, v4 \n\t" \ 310 "vmadot v18, v15, v5 \n\t" \ 311 "vmadot v20, v15, v6 \n\t" \ 312 "vmadot v22, v15, v7 \n\t" 313 314#define SQ4BIT_KERNEL_ACC_1X4X4 \ 315 "vfcvt.f.x.v v16, v16 \n\t" \ 316 "vfcvt.f.x.v v18, v18 \n\t" \ 317 "vfcvt.f.x.v v20, v20 \n\t" \ 318 "vfcvt.f.x.v v22, v22 \n\t" \ 319 "addi s2, s1, 16 \n\t" \ 320 "addi s3, s1, 32 \n\t" \ 321 "addi s4, s1, 48 \n\t" \ 322 "addi s6, s5, 12 \n\t" \ 323 "vfmacc.vv v28, v16, v24 \n\t" \ 324 "vfmacc.vv v29, v18, v25 \n\t" \ 325 "vfmacc.vv v30, v20, v26 \n\t" \ 326 "vfmacc.vv v31, v22, v27 \n\t" 327 328#define SQ4BIT_KERNEL_ACC_F16_1X4X4 \ 329 "vfcvt.f.x.v v16, v16 \n\t" \ 330 "vfcvt.f.x.v v18, v18 \n\t" \ 331 "vfcvt.f.x.v v20, v20 \n\t" \ 332 "vfcvt.f.x.v v22, v22 \n\t" \ 333 "addi s2, s1, 8 \n\t" \ 334 "addi s3, s1, 16 \n\t" \ 335 "addi s4, s1, 24 \n\t" \ 336 "addi s6, s5, 12 \n\t" \ 337 "vfmacc.vv v28, v16, v24 \n\t" \ 338 "vfmacc.vv v29, v18, v25 \n\t" \ 339 "vfmacc.vv v30, v20, v26 \n\t" \ 340 "vfmacc.vv v31, v22, v27 \n\t" 341 342#define SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 \ 343 "vle8.v v4, (s1) \n\t" \ 344 "addi s1, s1, 128 \n\t" \ 345 "vle8.v v5, (s2) \n\t" \ 346 "addi s2, s2, 128 \n\t" \ 347 "vle8.v v6, (s3) \n\t" \ 348 "addi s3, s3, 128 \n\t" \ 349 "vle8.v v7, (s4) \n\t" \ 350 "addi s4, s4, 128 \n\t" \ 351 "vsetvli t0, zero, e8, mf4 \n\t" \ 352 "vle8.v v14, (s5) \n\t" \ 353 "addi s5, s5, 16 \n\t" \ 354 "vle8.v v15, (s6) \n\t" \ 355 "addi s6, s6, 16 \n\t" \ 356 "addi t5, t5, -1 \n\t" \ 357 "vsetvli t0, zero, e8, m1 \n\t" \ 358 "vand.vi v0, v4, 15 \n\t" \ 359 "vand.vi v1, v5, 15 \n\t" \ 360 "vand.vi v2, v6, 15 \n\t" \ 361 "vand.vi v3, v7, 15 \n\t" \ 362 "vsrl.vi v4, v4, 4 \n\t" \ 363 "vsrl.vi v5, v5, 4 \n\t" \ 364 "vsrl.vi v6, v6, 4 \n\t" \ 365 "vsrl.vi v7, v7, 4 \n\t" 366 367#define SQ4BIT_KERNEL_LOAD_ZP_16X1 \ 368 "vsetvli t0, zero, e8, mf2 \n\t" \ 369 "vle8.v v1, (s7) \n\t" \ 370 "vsetvli t0, zero, e8, m1 \n\t" \ 371 "vrgather.vv v8, v1, v13 \n\t" \ 372 "vadd.vi v13, v13, 4 \n\t" \ 373 "vrgather.vv v9, v1, v13 \n\t" \ 374 "vadd.vi v13, v13, 4 \n\t" \ 375 "vrgather.vv v10, v1, v13 \n\t" \ 376 "vadd.vi v13, v13, 4 \n\t" \ 377 "vrgather.vv v11, v1, v13 \n\t" \ 378 "vadd.vi v13, v13, -12 \n\t" 379 380// using for M4Kernel 381#define LOAD_B_16x8x2 \ 382 "vsetvli t0, zero, e8, m1 \n\t" \ 383 "vle8.v v6, (s1) \n\t" \ 384 "addi s1, s1, 32*4 \n\t" \ 385 "vle8.v v7, (s2) \n\t" \ 386 "addi s2, s2, 32*4 \n\t" \ 387 "vle8.v v8, (s3) \n\t" \ 388 "addi s3, s3, 32*4 \n\t" \ 389 "vle8.v v9, (s4) \n\t" \ 390 "addi s4, s4, 32*4 \n\t" \ 391 \ 392 "vand.vi v2, v6, 15 \n\t" \ 393 "vand.vi v3, v7, 15 \n\t" \ 394 "vand.vi v4, v8, 15 \n\t" \ 395 "vand.vi v5, v9, 15 \n\t" \ 396 \ 397 "vsrl.vi v6, v6, 4 \n\t" \ 398 "vsrl.vi v7, v7, 4 \n\t" \ 399 "vsrl.vi v8, v8, 4 \n\t" \ 400 "vsrl.vi v9, v9, 4 \n\t" 401 402// [s2|s5, s3, s4, s6] 403#define LOAD_SCALE_4x16_FP16 \ 404 "addi s2, s5, -8 \n\t" \ 405 "addi s3, s5, 8 \n\t" \ 406 "addi s4, s5, 16 \n\t" \ 407 "addi s6, s5, 24 \n\t" \ 408 "li t1, 0xf0 \n\t" \ 409 "vmv.s.x v0, t1 \n\t" \ 410 "vsetvli t0, zero, e16, mf4 \n\t" \ 411 "vle16.v v9, (s5) \n\t" \ 412 "vle16.v v11, (s3) \n\t" \ 413 "vle16.v v13, (s4) \n\t" \ 414 "vle16.v v15, (s6) \n\t" \ 415 "vsetvli t0, zero, e16, mf2 \n\t" \ 416 "vle16.v v9, (s2), v0.t \n\t" \ 417 "vle16.v v11, (s5), v0.t \n\t" \ 418 "vle16.v v13, (s3), v0.t \n\t" \ 419 "vle16.v v15, (s4), v0.t \n\t" \ 420 "vfwcvt.f.f.v v8, v9 \n\t" \ 421 "vfwcvt.f.f.v v10, v11 \n\t" \ 422 "vfwcvt.f.f.v v12, v13 \n\t" \ 423 "vfwcvt.f.f.v v14, v15 \n\t" \ 424 "vsetvli t0, zero, e32, m1 \n\t" \ 425 "vmv.v.v v9, v8 \n\t" \ 426 "vmv.v.v v11, v10 \n\t" \ 427 "vmv.v.v v13, v12 \n\t" \ 428 "vmv.v.v v15, v14 \n\t" \ 429 "li t1, 0xf0 \n\t" \ 430 "vmv.s.x v0, t1 \n\t" \ 431 "vsetvli t0, zero, e32, mf2 \n\t" \ 432 "vfmul.vf v8, v8, f1 \n\t" \ 433 "vfmul.vf v10, v10, f1 \n\t" \ 434 "vfmul.vf v12, v12, f1 \n\t" \ 435 "vfmul.vf v14, v14, f1 \n\t" \ 436 "vfmul.vf v9, v9, f3 \n\t" \ 437 "vfmul.vf v11, v11, f3 \n\t" \ 438 "vfmul.vf v13, v13, f3 \n\t" \ 439 "vfmul.vf v15, v15, f3 \n\t" \ 440 "vsetvli t0, zero, e32, m1 \n\t" \ 441 "vfmul.vf v8, v8, f2, v0.t \n\t" \ 442 "vfmul.vf v10, v10, f2, v0.t \n\t" \ 443 "vfmul.vf v12, v12, f2, v0.t \n\t" \ 444 "vfmul.vf v14, v14, f2, v0.t \n\t" \ 445 "vfmul.vf v9, v9, f4, v0.t \n\t" \ 446 "vfmul.vf v11, v11, f4, v0.t \n\t" \ 447 "vfmul.vf v13, v13, f4, v0.t \n\t" \ 448 "vfmul.vf v15, v15, f4, v0.t \n\t" 449 450// [s2|s5, s3, s4, s6] 451#define LOAD_SCALE_4x16 \ 452 "addi s2, s5, -16 \n\t" \ 453 "addi s3, s5, 16 \n\t" \ 454 "addi s4, s5, 32 \n\t" \ 455 "addi s6, s5, 48 \n\t" \ 456 "li t1, 0xf0 \n\t" \ 457 "vmv.s.x v0, t1 \n\t" \ 458 "vsetvli t0, zero, e32, mf2 \n\t" \ 459 "vle32.v v8, (s5) \n\t" \ 460 "vle32.v v10, (s3) \n\t" \ 461 "vle32.v v12, (s4) \n\t" \ 462 "vle32.v v14, (s6) \n\t" \ 463 "vsetvli t0, zero, e32, m1 \n\t" \ 464 "vle32.v v8, (s2), v0.t \n\t" \ 465 "vle32.v v10, (s5), v0.t \n\t" \ 466 "vle32.v v12, (s3), v0.t \n\t" \ 467 "vle32.v v14, (s4), v0.t \n\t" \ 468 "vmv.v.v v9, v8 \n\t" \ 469 "vmv.v.v v11, v10 \n\t" \ 470 "vmv.v.v v13, v12 \n\t" \ 471 "vmv.v.v v15, v14 \n\t" \ 472 "vsetvli t0, zero, e32, mf2 \n\t" \ 473 "vfmul.vf v8, v8, f1 \n\t" \ 474 "vfmul.vf v10, v10, f1 \n\t" \ 475 "vfmul.vf v12, v12, f1 \n\t" \ 476 "vfmul.vf v14, v14, f1 \n\t" \ 477 "vfmul.vf v9, v9, f3 \n\t" \ 478 "vfmul.vf v11, v11, f3 \n\t" \ 479 "vfmul.vf v13, v13, f3 \n\t" \ 480 "vfmul.vf v15, v15, f3 \n\t" \ 481 "vsetvli t0, zero, e32, m1 \n\t" \ 482 "vfmul.vf v8, v8, f2, v0.t \n\t" \ 483 "vfmul.vf v10, v10, f2, v0.t \n\t" \ 484 "vfmul.vf v12, v12, f2, v0.t \n\t" \ 485 "vfmul.vf v14, v14, f2, v0.t \n\t" \ 486 "vfmul.vf v9, v9, f4, v0.t \n\t" \ 487 "vfmul.vf v11, v11, f4, v0.t \n\t" \ 488 "vfmul.vf v13, v13, f4, v0.t \n\t" \ 489 "vfmul.vf v15, v15, f4, v0.t \n\t" 490 491//[s1| BIAS, s2, s3, s4] 492#define LOAD_BIAS \ 493 "vsetvli t0, zero, e32, mf2 \n\t" \ 494 "li t1, 0xf0 \n\t" \ 495 "vmv.s.x v0, t1 \n\t" \ 496 "addi s1, %[BIAS], -16 \n\t" \ 497 "addi s2, %[BIAS], 16 \n\t" \ 498 "addi s3, %[BIAS], 32 \n\t" \ 499 "addi s4, %[BIAS], 48 \n\t" \ 500 \ 501 "vle32.v v24, (%[BIAS]) \n\t" \ 502 "vle32.v v26, (s2) \n\t" \ 503 "vle32.v v28, (s3) \n\t" \ 504 "vle32.v v30, (s4) \n\t" \ 505 "vsetvli t0, zero, e32, m1 \n\t" \ 506 "vle32.v v24, (s1), v0.t \n\t" \ 507 "vle32.v v26, (%[BIAS]), v0.t \n\t" \ 508 "vle32.v v28, (s2), v0.t \n\t" \ 509 "vle32.v v30, (s3), v0.t \n\t" \ 510 "vmv.v.v v25, v24 \n\t" \ 511 "vmv.v.v v27, v26 \n\t" \ 512 "vmv.v.v v29, v28 \n\t" \ 513 "vmv.v.v v31, v30 \n\t" 514 515#define SQ4BIT_KERNEL_COMP_4x16x16 \ 516 "vmadot v16, v10, v2 \n\t" \ 517 "vmadot v18, v10, v3 \n\t" \ 518 "vmadot v20, v10, v4 \n\t" \ 519 "vmadot v22, v10, v5 \n\t" \ 520 "vmadot v16, v11, v6 \n\t" \ 521 "vmadot v18, v11, v7 \n\t" \ 522 "vmadot v20, v11, v8 \n\t" \ 523 "vmadot v22, v11, v9 \n\t" 524 525#define SAVE_RESULT_4x16 \ 526 "addi a1, %[C], 0 \n\t" \ 527 "add a2, %[C], %[LDC] \n\t" \ 528 "add a3, a2, %[LDC] \n\t" \ 529 "add a4, a3, %[LDC] \n\t" \ 530 "addi a2, a2, -16 \n\t" \ 531 "addi a4, a4, -16 \n\t" \ 532 "li t1, 0xf0 \n\t" \ 533 "vmv.s.x v0, t1 \n\t" \ 534 "vsetvli t0, zero, e32, mf2 \n\t" \ 535 \ 536 "vse32.v v24, (a1) \n\t" \ 537 "addi a1, a1, 16 \n\t" \ 538 "vse32.v v25, (a3) \n\t" \ 539 "addi a3, a3, 16 \n\t" \ 540 \ 541 "vse32.v v26, (a1) \n\t" \ 542 "addi a1, a1, 16 \n\t" \ 543 "vse32.v v27, (a3) \n\t" \ 544 "addi a3, a3, 16 \n\t" \ 545 \ 546 "vse32.v v28, (a1) \n\t" \ 547 "addi a1, a1, 16 \n\t" \ 548 "vse32.v v29, (a3) \n\t" \ 549 "addi a3, a3, 16 \n\t" \ 550 \ 551 "vse32.v v30, (a1) \n\t" \ 552 "vse32.v v31, (a3) \n\t" \ 553 "vsetvli t0, zero, e32, m1 \n\t" \ 554 \ 555 "vse32.v v24, (a2), v0.t \n\t" \ 556 "addi a2, a2, 16 \n\t" \ 557 "vse32.v v25, (a4), v0.t \n\t" \ 558 "addi a4, a4, 16 \n\t" \ 559 \ 560 "vse32.v v26, (a2), v0.t \n\t" \ 561 "addi a2, a2, 16 \n\t" \ 562 "vse32.v v27, (a4), v0.t \n\t" \ 563 "addi a4, a4, 16 \n\t" \ 564 \ 565 "vse32.v v28, (a2), v0.t \n\t" \ 566 "addi a2, a2, 16 \n\t" \ 567 "vse32.v v29, (a4), v0.t \n\t" \ 568 "addi a4, a4, 16 \n\t" \ 569 \ 570 "vse32.v v30, (a2), v0.t \n\t" \ 571 "vse32.v v31, (a4), v0.t \n\t" 572 573#define SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 \ 574 "vsetvli t0, zero, e8, mf2 \n\t" \ 575 "vle8.v v11, (s6) \n\t" \ 576 "vsetvli t0, zero, e8, m1 \n\t" \ 577 "vrgather.vv v12, v11, v1 \n\t" \ 578 "vadd.vi v1, v1, 4 \n\t" \ 579 "vrgather.vv v13, v11, v1 \n\t" \ 580 "vadd.vi v1, v1, 4 \n\t" \ 581 "vrgather.vv v14, v11, v1 \n\t" \ 582 "vadd.vi v1, v1, 4 \n\t" \ 583 "vrgather.vv v15, v11, v1 \n\t" \ 584 "vadd.vi v1, v1, -12 \n\t" 585 586template < bool HasZeroPoint > 587void SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl (size_t BlkLen , 588const uint8_t * QuantA , 589const uint8_t * QuantBData , 590float * C , 591size_t CountN , 592size_t BlockCountK , 593const size_t ldc ) { 594size_t LDC = ldc * sizeof (float ); 595const size_t INNER = BlkLen /16 ; 596float tmp [4 * 16 ]; 597 598if constexpr (HasZeroPoint ) { 599for (size_t n = 0 ;n < CountN ;n += 16 ) { 600size_t NBLKS = (CountN - n )> 16 ?16 :CountN - n ; 601uint8_t * QuantBDataPtr = (uint8_t * )QuantBData + // 602n * BlockCountK * BlkLen /2 + // b data 603n * BlockCountK * sizeof (uint8_t )+ // zp 604n * BlockCountK * sizeof (_Float16 );// scale 605float * CPtr = C + n ; 606if (NBLKS < 16 ) { 607CPtr = tmp ; 608LDC = 16 * sizeof (float ); 609 } 610 611 __asm__volatile ( 612"vsetvli t0, zero, e32, m8 \n\t" 613"vxor.vv v24, v24, v24 \n\t" 614"addi t3, %[BlockCountK], 0 \n\t" 615"vsetvli t0, zero, e8, m1 \n\t" 616"li s1, 24 \n\t" 617"vmv.v.i v1, 3 \n\t" 618"vsetvli t0, s1, e8, m1 \n\t" 619"vmv.v.i v1, 2 \n\t" 620"vsetvli t0, zero, e8, mf2 \n\t" 621"vmv.v.i v1, 1 \n\t" 622"vsetvli t0, zero, e8, mf4 \n\t" 623"vmv.v.i v1, 0 \n\t" 624"addi a1, %[A], 0 \n\t" 625"addi s1, %[B], 0 \n\t" 626"BLOCK_COUNTK_LOOP%=: \n\t" 627// scale offset 628"addi s5, s1, 0 \n\t" 629// zp offset 630"addi s6, s1, 32 \n\t" 631"addi s1, s6, 16 \n\t" 632"addi s2, s1, 32 \n\t" 633"addi s3, s1, 32*2 \n\t" 634"addi s4, s1, 32*3 \n\t" 635 636"vsetvli t0, zero, e32, m8 \n\t" 637"vxor.vv v16, v16, v16 \n\t" 638// load a scale 639"flw f1, (a1) \n\t" 640"flw f2, 4(a1) \n\t" 641"flw f3, 8(a1) \n\t" 642"flw f4, 12(a1) \n\t" 643"addi a1, a1, 16 \n\t" 644"addi t2, %[INNER], 0 \n\t" 645 646SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 647 648"BLOCK_INNER_LOOP%=: \n\t" 649 650LOAD_B_16x8x2 651 652"vle8.v v10, (a1) \n\t" 653"addi a1, a1, 32 \n\t" 654"vle8.v v11, (a1) \n\t" 655"addi a1, a1, 32 \n\t" 656"vsub.vv v2, v2, v12 \n\t" 657"vsub.vv v6, v6, v12 \n\t" 658"vsub.vv v3, v3, v13 \n\t" 659"vsub.vv v7, v7, v13 \n\t" 660"vsub.vv v4, v4, v14 \n\t" 661"vsub.vv v8, v8, v14 \n\t" 662"vsub.vv v5, v5, v15 \n\t" 663"vsub.vv v9, v9, v15 \n\t" 664 665SQ4BIT_KERNEL_COMP_4x16x16 666 667"addi t2, t2, -1 \n\t" 668"bnez t2, BLOCK_INNER_LOOP%= \n\t" 669 670LOAD_SCALE_4x16_FP16 671 672"vsetvli t0, zero, e32, m8 \n\t" 673"vfcvt.f.x.v v16, v16 \n\t" 674"vfmacc.vv v24, v16, v8 \n\t" 675"addi t3, t3, -1 \n\t" 676"bnez t3, BLOCK_COUNTK_LOOP%= \n\t" 677 678"RESULT_SAVE%=: \n\t" 679 680SAVE_RESULT_4x16 681 682 : 683 : [INNER ]"r" (INNER ), [A ]"r" (QuantA ), [B ]"r" (QuantBDataPtr ), [LDC ]"r" (LDC ), 684 [BlockCountK ]"r" (BlockCountK ), [C ]"r" (CPtr ) 685 :"cc" ,"t0" ,"t1" ,"t2" ,"t3" ,"a1" ,"a2" ,"a3" ,"a4" ,"f1" ,"f2" ,"f3" ,"f4" ,"s1" ,"s2" ,"s3" ,"s4" , 686"s5" ,"s6" ); 687 } 688 }else { 689for (size_t n = 0 ;n < CountN ;n += 16 ) { 690size_t NBLKS = (CountN - n )> 16 ?16 :CountN - n ; 691uint8_t * QuantBDataPtr = (uint8_t * )QuantBData + // 692n * BlockCountK * BlkLen /2 + // b data 693n * BlockCountK * sizeof (_Float16 );// scale 694float * CPtr = C + n ; 695if (NBLKS < 16 ) { 696CPtr = tmp ; 697LDC = 16 * sizeof (float ); 698 } 699 700 __asm__volatile ( 701"vsetvli t0, zero, e32, m8 \n\t" 702"vxor.vv v24, v24, v24 \n\t" 703"addi t3, %[BlockCountK], 0 \n\t" 704"addi a1, %[A], 0 \n\t" 705"addi s1, %[B], 0 \n\t" 706"BLOCK_COUNTK_LOOP%=: \n\t" 707"addi s5, s1, 0 \n\t" 708"addi s1, s5, 32 \n\t" 709"addi s2, s1, 32 \n\t" 710"addi s3, s1, 32*2 \n\t" 711"addi s4, s1, 32*3 \n\t" 712"vsetvli t0, zero, e32, m8 \n\t" 713"vxor.vv v16, v16, v16 \n\t" 714// load a scale 715"flw f1, (a1) \n\t" 716"flw f2, 4(a1) \n\t" 717"flw f3, 8(a1) \n\t" 718"flw f4, 12(a1) \n\t" 719"addi a1, a1, 16 \n\t" 720"addi t2, %[INNER], 0 \n\t" 721"BLOCK_INNER_LOOP%=: \n\t" 722 723LOAD_B_16x8x2 724 725"vsetvli t0, zero, e8, m1 \n\t" 726"vle8.v v10, (a1) \n\t" 727"addi a1, a1, 32 \n\t" 728"vle8.v v11, (a1) \n\t" 729"addi a1, a1, 32 \n\t" 730"vadd.vi v2, v2, -8 \n\t" 731"vadd.vi v3, v3, -8 \n\t" 732"vadd.vi v4, v4, -8 \n\t" 733"vadd.vi v5, v5, -8 \n\t" 734"vadd.vi v6, v6, -8 \n\t" 735"vadd.vi v7, v7, -8 \n\t" 736"vadd.vi v8, v8, -8 \n\t" 737"vadd.vi v9, v9, -8 \n\t" 738 739SQ4BIT_KERNEL_COMP_4x16x16 740 741"addi t2, t2, -1 \n\t" 742"bnez t2, BLOCK_INNER_LOOP%= \n\t" 743 744LOAD_SCALE_4x16_FP16 745 746"vsetvli t0, zero, e32, m8 \n\t" 747"vfcvt.f.x.v v16, v16 \n\t" 748"vfmacc.vv v24, v16, v8 \n\t" 749"addi t3, t3, -1 \n\t" 750"bnez t3, BLOCK_COUNTK_LOOP%= \n\t" 751"RESULT_SAVE%=: \n\t" 752 753SAVE_RESULT_4x16 754 755 : 756 : [INNER ]"r" (INNER ), [A ]"r" (QuantA ), [B ]"r" (QuantBDataPtr ), [LDC ]"r" (LDC ), 757 [BlockCountK ]"r" (BlockCountK ), [C ]"r" (CPtr ) 758 :"cc" ,"t0" ,"t1" ,"t2" ,"t3" ,"a1" ,"a2" ,"a3" ,"a4" ,"f1" ,"f2" ,"f3" ,"f4" ,"s1" ,"s2" ,"s3" ,"s4" , 759"s5" ,"s6" ); 760 } 761 } 762} 763 764template < bool HasZeroPoint > 765void SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl (size_t BlkLen , 766const uint8_t * QuantA , 767const uint8_t * QuantBData , 768float * C , 769size_t CountN , 770size_t BlockCountK , 771const size_t ldc ) { 772GGML_UNUSED (ldc ); 773size_t INNER = BlkLen /16 ; 774 775if constexpr (HasZeroPoint ) { 776for (size_t n = 0 ;n < CountN ;n += 16 ) { 777size_t nblks = (CountN - n )> 16 ?16 :CountN - n ; 778uint8_t * QuantBDataPtr = (uint8_t * )QuantBData + // 779n * BlockCountK * BlkLen /2 + // b data 780n * BlockCountK * sizeof (uint8_t )+ // zp 781n * BlockCountK * sizeof (_Float16 );// scale 782float * CPtr = C + n ; 783size_t cnt = BlockCountK ; 784 785 __asm__volatile ( 786"vsetvli t0, zero, e32, m4 \n\t" 787"vxor.vv v28, v28, v28 \n\t" 788 789"vsetvli t0, zero, e8, m1 \n\t" 790"vmv.v.i v13, 3 \n\t" 791"li s1, 24 \n\t" 792"vsetvli t0, s1, e8, m1 \n\t" 793"vmv.v.i v13, 2 \n\t" 794"vsetvli t0, zero, e8, mf2 \n\t" 795"vmv.v.i v13, 1 \n\t" 796"vsetvli t0, zero, e8, mf4 \n\t" 797"vmv.v.i v13, 0 \n\t" 798 799"addi s1, %[B], 0 \n\t" 800"addi s2, %[B], 8 \n\t" 801"addi s3, %[B], 16 \n\t" 802"addi s4, %[B], 24 \n\t" 803 804"addi s7, %[B], 32 \n\t" 805 806"addi s5, %[A], 0 \n\t" 807"addi s6, %[A], 12 \n\t" 808"LOOP_K%=: \n\t" 809"vsetvli t0, zero, e16, mf4 \n\t" 810"vle16.v v4, (s1) \n\t" 811"addi s1, s1, 48 \n\t" 812"vle16.v v5, (s2) \n\t" 813"addi s2, s2, 72 \n\t" 814"vle16.v v6, (s3) \n\t" 815"addi s3, s3, 96 \n\t" 816"vle16.v v7, (s4) \n\t" 817"addi s4, s4, 120 \n\t" 818"flw f1, (s5) \n\t" 819"addi s5, s5, 4 \n\t" 820 821"vfwcvt.f.f.v v8, v4 \n\t" 822"vfwcvt.f.f.v v9, v5 \n\t" 823"vfwcvt.f.f.v v10, v6 \n\t" 824"vfwcvt.f.f.v v11, v7 \n\t" 825"vsetvli t0, zero, e32, mf2 \n\t" 826 827"addi t5, %[INNER], 0 \n\t" 828"vxor.vv v16, v16, v16 \n\t" 829"vxor.vv v18, v18, v18 \n\t" 830"vxor.vv v20, v20, v20 \n\t" 831"vxor.vv v22, v22, v22 \n\t" 832"vfmul.vf v24, v8, f1 \n\t" 833"vfmul.vf v25, v9, f1 \n\t" 834"vfmul.vf v26, v10, f1 \n\t" 835"vfmul.vf v27, v11, f1 \n\t" 836"addi %[CNT], %[CNT], -1 \n\t" 837 838SQ4BIT_KERNEL_LOAD_ZP_16X1 839 840"LOOP_INNER%=: \n\t" 841 842SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 843 844"vsub.vv v0, v0, v8 \n\t" 845"vsub.vv v4, v4, v8 \n\t" 846"vsub.vv v1, v1, v9 \n\t" 847"vsub.vv v5, v5, v9 \n\t" 848"vsub.vv v2, v2, v10 \n\t" 849"vsub.vv v6, v6, v10 \n\t" 850"vsub.vv v3, v3, v11 \n\t" 851"vsub.vv v7, v7, v11 \n\t" 852 853SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 854 855"bnez t5, LOOP_INNER%= \n\t" 856"vsetvli t0, zero, e32, mf2 \n\t" 857 858SQ4BIT_KERNEL_ACC_F16_1X4X4 859"addi s7, s1, 32 \n\t" 860 861"bnez %[CNT], LOOP_K%= \n\t" 862"addi t3, zero, 16 \n\t" 863"addi s1, %[C], 16 \n\t" 864"addi s2, %[C], 32 \n\t" 865"addi s3, %[C], 48 \n\t" 866"blt %[NBLKS], t3, ST_TAIL%= \n\t" 867"vse32.v v28, (%[C]) \n\t" 868"vse32.v v29, (s1) \n\t" 869"vse32.v v30, (s2) \n\t" 870"vse32.v v31, (s3) \n\t" 871"jal x0, END%= \n\t" 872 873"ST_TAIL%=: \n\t" 874"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 875"sub %[NBLKS], %[NBLKS], t0 \n\t" 876"vse32.v v28, (%[C]) \n\t" 877"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 878"sub %[NBLKS], %[NBLKS], t0 \n\t" 879"vse32.v v29, (s1) \n\t" 880"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 881"sub %[NBLKS], %[NBLKS], t0 \n\t" 882"vse32.v v30, (s2) \n\t" 883"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 884"sub %[NBLKS], %[NBLKS], t0 \n\t" 885"vse32.v v31, (s3) \n\t" 886"END%=: \n\t" 887 888 : [CNT ]"+r" (cnt ), [NBLKS ]"+r" (nblks ) 889 : [INNER ]"r" (INNER ), [A ]"r" (QuantA ), [B ]"r" (QuantBDataPtr ), [C ]"r" (CPtr ) 890 :"cc" ,"t0" ,"t5" ,"t3" ,"f1" ,"s1" ,"s2" ,"s3" ,"s4" ,"s5" ,"s6" ,"s7" ); 891 } 892 }else { 893for (size_t n = 0 ;n < CountN ;n += 16 ) { 894size_t nblks = (CountN - n )> 16 ?16 :CountN - n ; 895uint8_t * QuantBDataPtr = (uint8_t * )QuantBData + // 896n * BlockCountK * BlkLen /2 + // b data 897n * BlockCountK * sizeof (_Float16 );// scale 898float * CPtr = C + n ; 899size_t cnt = BlockCountK ; 900 901 __asm__volatile ( 902"vsetvli t0, zero, e32, m4 \n\t" 903"vxor.vv v28, v28, v28 \n\t" 904"addi s1, %[B], 0 \n\t" 905"addi s2, %[B], 8 \n\t" 906"addi s3, %[B], 16 \n\t" 907"addi s4, %[B], 24 \n\t" 908 909"addi s5, %[A], 0 \n\t" 910"addi s6, %[A], 12 \n\t" 911"LOOP_K%=: \n\t" 912"vsetvli t0, zero, e16, mf4 \n\t" 913"vle16.v v4, (s1) \n\t" 914"addi s1, s1, 32 \n\t" 915"vle16.v v5, (s2) \n\t" 916"addi s2, s2, 56 \n\t" 917"vle16.v v6, (s3) \n\t" 918"addi s3, s3, 80 \n\t" 919"vle16.v v7, (s4) \n\t" 920"addi s4, s4, 104 \n\t" 921"flw f1, (s5) \n\t" 922"addi s5, s5, 4 \n\t" 923 924"vfwcvt.f.f.v v8, v4 \n\t" 925"vfwcvt.f.f.v v9, v5 \n\t" 926"vfwcvt.f.f.v v10, v6 \n\t" 927"vfwcvt.f.f.v v11, v7 \n\t" 928"vsetvli t0, zero, e32, mf2 \n\t" 929 930"addi t5, %[INNER], 0 \n\t" 931"vxor.vv v16, v16, v16 \n\t" 932"vxor.vv v18, v18, v18 \n\t" 933"vxor.vv v20, v20, v20 \n\t" 934"vxor.vv v22, v22, v22 \n\t" 935"vfmul.vf v24, v8, f1 \n\t" 936"vfmul.vf v25, v9, f1 \n\t" 937"vfmul.vf v26, v10, f1 \n\t" 938"vfmul.vf v27, v11, f1 \n\t" 939"addi %[CNT], %[CNT], -1 \n\t" 940"vsetvli t0, zero, e8, m1 \n\t" 941"LOOP_INNER%=: \n\t" 942 943SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 944 945"vadd.vi v0, v0, -8 \n\t" 946"vadd.vi v1, v1, -8 \n\t" 947"vadd.vi v2, v2, -8 \n\t" 948"vadd.vi v3, v3, -8 \n\t" 949"vadd.vi v4, v4, -8 \n\t" 950"vadd.vi v5, v5, -8 \n\t" 951"vadd.vi v6, v6, -8 \n\t" 952"vadd.vi v7, v7, -8 \n\t" 953 954SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 955 956"bnez t5, LOOP_INNER%= \n\t" 957"vsetvli t0, zero, e32, mf2 \n\t" 958 959SQ4BIT_KERNEL_ACC_F16_1X4X4 960 961"bnez %[CNT], LOOP_K%= \n\t" 962"addi t3, zero, 16 \n\t" 963"addi s1, %[C], 16 \n\t" 964"addi s2, %[C], 32 \n\t" 965"addi s3, %[C], 48 \n\t" 966"blt %[NBLKS], t3, ST_TAIL%= \n\t" 967"vse32.v v28, (%[C]) \n\t" 968"vse32.v v29, (s1) \n\t" 969"vse32.v v30, (s2) \n\t" 970"vse32.v v31, (s3) \n\t" 971"jal x0, END%= \n\t" 972 973"ST_TAIL%=: \n\t" 974"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 975"sub %[NBLKS], %[NBLKS], t0 \n\t" 976"vse32.v v28, (%[C]) \n\t" 977"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 978"sub %[NBLKS], %[NBLKS], t0 \n\t" 979"vse32.v v29, (s1) \n\t" 980"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 981"sub %[NBLKS], %[NBLKS], t0 \n\t" 982"vse32.v v30, (s2) \n\t" 983"vsetvli t0, %[NBLKS], e32, mf2 \n\t" 984"sub %[NBLKS], %[NBLKS], t0 \n\t" 985"vse32.v v31, (s3) \n\t" 986"END%=: \n\t" 987 988 : [CNT ]"+r" (cnt ), [NBLKS ]"+r" (nblks ) 989 : [INNER ]"r" (INNER ), [A ]"r" (QuantA ), [B ]"r" (QuantBDataPtr ), [C ]"r" (CPtr ) 990 :"cc" ,"t0" ,"t5" ,"t3" ,"f1" ,"s1" ,"s2" ,"s3" ,"s4" ,"s5" ,"s6" ); 991 } 992 } 993} 994}// namespace 995 996namespace ime1 { 997size_t gemm_kernel_i8i4 (size_t blk_len , 998const uint8_t * quant_a_ptr , 999const uint8_t * quant_b_data , 1000const uint8_t * quant_b_zp , 1001float * c_ptr , 1002size_t count_m , 1003size_t count_n , 1004size_t k_blks , 1005size_t ldc ) { 1006if (count_m >=4 ) { 1007if (quant_b_zp != nullptr ) { 1008SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl < true> (blk_len ,quant_a_ptr ,quant_b_data ,c_ptr ,count_n ,k_blks , 1009ldc ); 1010 }else { 1011SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl < false> (blk_len ,quant_a_ptr ,quant_b_data ,c_ptr ,count_n , 1012k_blks ,ldc ); 1013 } 1014return 4 ; 1015 }else { 1016if (quant_b_zp != nullptr ) { 1017SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl < true> (blk_len ,quant_a_ptr ,quant_b_data ,c_ptr ,count_n ,k_blks , 1018ldc ); 1019 }else { 1020SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl < false> (blk_len ,quant_a_ptr ,quant_b_data ,c_ptr ,count_n , 1021k_blks ,ldc ); 1022 } 1023return 1 ; 1024 } 1025} 1026}// namespace ime1 1027}// namespace spacemit_kernels