char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
cd644c395
char/aliceai-t5-moe
1#pragma once 2 3#include "ggml-cpu-impl.h" 4 5#ifdef __ARM_FEATURE_SVE 6#include <arm_sve.h> 7#endif // __ARM_FEATURE_SVE 8 9#if defined(__ARM_NEON )&& !defined(__CUDACC__ )&& !defined(__MUSACC__ ) 10// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example: 11// 12// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/ 13// 14#include <arm_neon.h> 15#endif 16 17#if defined(__riscv_v_intrinsic ) 18#include <riscv_vector.h> 19#endif 20 21#ifdef __cplusplus 22extern "C" { 23#endif 24 25// 26// simd mappings 27// 28 29// FP16 to FP32 conversion 30 31// 16-bit float 32// on Arm, we use __fp16, which requires the IEEE fp16 format: implied on 33// AArch64, selected by -mfp16-format=ieee on 32 bit Arm, where the compiler 34// may otherwise reject the type 35// on x86, we use uint16_t 36// 37// for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616 38// for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843 39// 40#if defined(__ARM_NEON )&& defined(__ARM_FP16_FORMAT_IEEE )&& !(defined(__CUDACC__ )&& __CUDACC_VER_MAJOR__ <=11 )&& !defined(__MUSACC__ ) 41#define GGML_CPU_COMPUTE_FP16_TO_FP32 (x ) neon_compute_fp16_to_fp32(x) 42#define GGML_CPU_COMPUTE_FP32_TO_FP16 (x ) neon_compute_fp32_to_fp16(x) 43 44#define GGML_CPU_FP16_TO_FP32 (x ) GGML_CPU_COMPUTE_FP16_TO_FP32(x) 45 46static inline float neon_compute_fp16_to_fp32 (ggml_fp16_t h ) { 47__fp16 tmp ; 48memcpy (& tmp ,& h ,sizeof (ggml_fp16_t )); 49return (float )tmp ; 50 } 51 52static inline ggml_fp16_t neon_compute_fp32_to_fp16 (float f ) { 53ggml_fp16_t res ; 54__fp16 tmp = f ; 55memcpy (& res ,& tmp ,sizeof (ggml_fp16_t )); 56return res ; 57 } 58#elif defined(__F16C__ ) 59#ifdef _MSC_VER 60#define GGML_CPU_COMPUTE_FP16_TO_FP32 (x ) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x))) 61#define GGML_CPU_COMPUTE_FP32_TO_FP16 (x ) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0) 62#else 63#define GGML_CPU_COMPUTE_FP16_TO_FP32 (x ) _cvtsh_ss(x) 64#define GGML_CPU_COMPUTE_FP32_TO_FP16 (x ) _cvtss_sh(x, 0) 65#endif 66#elif defined(__POWER9_VECTOR__ ) 67#define GGML_CPU_COMPUTE_FP16_TO_FP32 (x ) power_compute_fp16_to_fp32(x) 68#define GGML_CPU_COMPUTE_FP32_TO_FP16 (x ) power_compute_fp32_to_fp16(x) 69/* the inline asm below is about 12% faster than the lookup method */ 70#define GGML_CPU_FP16_TO_FP32 (x ) GGML_CPU_COMPUTE_FP16_TO_FP32(x) 71#define GGML_CPU_FP32_TO_FP16 (x ) GGML_CPU_COMPUTE_FP32_TO_FP16(x) 72 73static inline float power_compute_fp16_to_fp32 (ggml_fp16_t h ) { 74float f ; 75double d ; 76 __asm__( 77"mtfprd %0,%2\n" 78"xscvhpdp %0,%0\n" 79"frsp %1,%0\n" : 80/* temp */ "=d" (d ), 81/* out */ "=f" (f ): 82/* in */ "r" (h )); 83return f ; 84 } 85 86static inline ggml_fp16_t power_compute_fp32_to_fp16 (float f ) { 87double d ; 88ggml_fp16_t r ; 89 __asm__(/* xscvdphp can work on double or single precision */ 90"xscvdphp %0,%2\n" 91"mffprd %1,%0\n" : 92/* temp */ "=d" (d ), 93/* out */ "=r" (r ): 94/* in */ "f" (f )); 95return r ; 96 } 97#elif defined(__riscv )&& defined(__riscv_zfhmin ) 98static inline float riscv_compute_fp16_to_fp32 (ggml_fp16_t h ) { 99_Float16 hf ; 100memcpy (& hf ,& h ,sizeof (ggml_fp16_t )); 101return hf ; 102 } 103 104static inline ggml_fp16_t riscv_compute_fp32_to_fp16 (float f ) { 105ggml_fp16_t res ; 106_Float16 hf = (_Float16 )f ; 107memcpy (& res ,& hf ,sizeof (ggml_fp16_t )); 108return res ; 109 } 110 111#define GGML_CPU_COMPUTE_FP16_TO_FP32 (x ) riscv_compute_fp16_to_fp32(x) 112#define GGML_CPU_COMPUTE_FP32_TO_FP16 (x ) riscv_compute_fp32_to_fp16(x) 113#define GGML_CPU_FP16_TO_FP32 (x ) GGML_CPU_COMPUTE_FP16_TO_FP32(x) 114#define GGML_CPU_FP32_TO_FP16 (x ) GGML_CPU_COMPUTE_FP32_TO_FP16(x) 115#endif 116 117// precomputed f32 table for f16 (256 KB) 118// defined in ggml-cpu.c, initialized in ggml_cpu_init() 119extern float ggml_table_f32_f16 [1 <<16 ]; 120 121// precomputed f32 table for e8m0 half (1 KB) 122// defined in ggml-cpu.c, initialized in ggml_cpu_init() 123extern float ggml_table_f32_e8m0_half [1 <<8 ]; 124 125// precomputed f32 table for ue4m3 (1 KB) 126// defined in ggml-cpu.c, initialized in ggml_cpu_init() 127extern float ggml_table_f32_ue4m3 [1 <<8 ]; 128 129// Use lookup table for E8M0 on x86 (faster than bit manipulation) 130#if defined(__AVX__ )|| defined(__AVX2__ )|| defined(__AVX512F__ ) 131#define GGML_CPU_E8M0_TO_FP32_HALF (x ) ggml_table_f32_e8m0_half[(uint8_t)(x)] 132#else 133#define GGML_CPU_E8M0_TO_FP32_HALF (x ) GGML_E8M0_TO_FP32_HALF(x) 134#endif 135 136// Use lookup table for UE4M3 on x86 and ARM (faster than bit manipulation) 137#if defined(__AVX__ )|| defined(__AVX2__ )|| defined(__AVX512F__ )|| defined(__ARM_NEON ) 138#define GGML_CPU_UE4M3_TO_FP32 (x ) ggml_table_f32_ue4m3[(uint8_t)(x)] 139#else 140#define GGML_CPU_UE4M3_TO_FP32 (x ) ggml_ue4m3_to_fp32(x) 141#endif 142 143// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, 144// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. 145// This is also true for POWER9. 146#if !defined(GGML_CPU_FP16_TO_FP32 ) 147inline static float ggml_lookup_fp16_to_fp32 (ggml_fp16_t f ) { 148uint16_t s ; 149memcpy (& s ,& f ,sizeof (uint16_t )); 150return ggml_table_f32_f16 [s ]; 151} 152 153#define GGML_CPU_FP16_TO_FP32 (x ) ggml_lookup_fp16_to_fp32(x) 154#endif 155 156#if !defined(GGML_CPU_FP32_TO_FP16 ) 157#define GGML_CPU_FP32_TO_FP16 (x ) GGML_COMPUTE_FP32_TO_FP16(x) 158#endif 159 160 161// we define a common set of C macros which map to specific intrinsics based on the current architecture 162// we then implement the fundamental computation operations below using only these macros 163// adding support for new architectures requires to define the corresponding SIMD macros 164// 165// GGML_F32_STEP / GGML_F16_STEP 166// number of elements to process in a single step 167// 168// GGML_F32_EPR / GGML_F16_EPR 169// number of elements to fit in a single register 170// 171 172#if defined(__ARM_FEATURE_SVE )&& defined(__ARM_FEATURE_FMA ) 173 174#define GGML_SIMD 175 176// F32 SVE 177#define GGML_F32_EPR 8 178#define DEFAULT_PG svptrue_b32() 179 180#define GGML_F32xt svfloat32_t 181#define GGML_F32xt_ZERO svdup_n_f32(0.0f) 182#define GGML_F32xt_SET1 (x ) svdup_n_f32(x) 183#define GGML_F32xt_LOAD_IMPL (pg ,a ) svld1_f32(pg, a) 184#define GGML_F32xt_LOAD (a ) GGML_F32xt_LOAD_IMPL(DEFAULT_PG, a) 185#define GGML_F32xt_STORE_IMPL (pg ,a ,b ) svst1_f32(pg, a, b) 186#define GGML_F32xt_STORE (a ,b ) GGML_F32xt_STORE_IMPL(DEFAULT_PG, a, b) 187#define GGML_F32xt_FMA_IMPL (pg ,a ,b ,c ) svmad_f32_m(pg, b, c, a) 188#define GGML_F32xt_FMA (a ,b ,c ) GGML_F32xt_FMA_IMPL(DEFAULT_PG, a, b, c) 189#define GGML_F32xt_ADD_IMPL (pg ,a ,b ) svadd_f32_m(pg, a, b) 190#define GGML_F32xt_ADD (a ,b ) GGML_F32xt_ADD_IMPL(DEFAULT_PG, a, b) 191#define GGML_F32xt_MUL_IMPL (pg ,a ,b ) svmul_f32_m(pg, a, b) 192#define GGML_F32xt_MUL (a ,b ) GGML_F32xt_MUL_IMPL(DEFAULT_PG, a, b) 193#define GGML_F32xt_REDUCE_ONE_IMPL (pg ,a ) svaddv(pg, a) 194#define GGML_F32xt_REDUCE_ONE (a ) GGML_F32xt_REDUCE_ONE_IMPL(DEFAULT_PG, a) 195#define GGML_F32xt_REDUCE_IMPL (pg ,res ,sum1 ,sum2 ,sum3 ,sum4 ,sum5 ,sum6 ,sum7 ,sum8 ) \ 196{ \ 197 sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum2); \ 198 sum3 = svadd_f32_m(DEFAULT_PG, sum3, sum4); \ 199 sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum6); \ 200 sum7 = svadd_f32_m(DEFAULT_PG, sum7, sum8); \ 201 sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum3); \ 202 sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum7); \ 203 sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum5); \ 204 (res) = (ggml_float) GGML_F32xt_REDUCE_ONE(sum1); \ 205} 206#define GGML_F32xt_REDUCE (res ,sum1 ,sum2 ,sum3 ,sum4 ,sum5 ,sum6 ,sum7 ,sum8 ) \ 207 GGML_F32xt_REDUCE_IMPL(DEFAULT_PG, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) 208 209#define GGML_F32_VEC GGML_F32xt 210#define GGML_F32_VEC_ZERO GGML_F32xt_ZERO 211#define GGML_F32_VEC_SET1 GGML_F32xt_SET1 212#define GGML_F32_VEC_LOAD GGML_F32xt_LOAD 213#define GGML_F32_VEC_STORE GGML_F32xt_STORE 214#define GGML_F32_VEC_FMA GGML_F32xt_FMA 215#define GGML_F32_VEC_ADD GGML_F32xt_ADD 216#define GGML_F32_VEC_MUL GGML_F32xt_MUL 217#define GGML_F32_VEC_REDUCE GGML_F32xt_REDUCE 218 219// F16 SVE 220#define DEFAULT_PG32 svptrue_b32() 221#define DEFAULT_PG16 svptrue_b16() 222 223#define GGML_F32Cxt svfloat16_t 224#define GGML_F32Cxt_ZERO svdup_n_f16(0.0f) 225#define GGML_F32Cxt_SET1 (x ) svdup_n_f16(x) 226#define GGML_F32Cxt_LOAD (p ) svld1_f16(DEFAULT_PG16, (const __fp16 *)(p)) 227#define GGML_F32Cxt_STORE (dst_ptr ,src_vec ) svst1_f16(DEFAULT_PG16, (__fp16 *)(dst_ptr), (src_vec)) 228 229#define GGML_F32Cxt_FMA_IMPL (pg ,a ,b ,c ) svmad_f16_x(pg, b, c, a) 230#define GGML_F32Cxt_FMA (a ,b ,c ) GGML_F32Cxt_FMA_IMPL(DEFAULT_PG16, a, b, c) 231#define GGML_F32Cxt_ADD_IMPL (pg ,a ,b ) svadd_f16_x(pg, a, b) 232#define GGML_F32Cxt_ADD (a ,b ) GGML_F32Cxt_ADD_IMPL(DEFAULT_PG16, a, b) 233#define GGML_F32Cxt_MUL_IMPL (pg ,a ,b ) svmul_f16_x(pg, a, b) 234#define GGML_F32Cxt_MUL (a ,b ) GGML_F32Cxt_MUL_IMPL(DEFAULT_PG16, a, b) 235#define GGML_F32Cxt_REDUCE GGML_F16xt_REDUCE_MIXED 236 237#define GGML_F16x_VEC GGML_F32Cxt 238#define GGML_F16x_VEC_ZERO GGML_F32Cxt_ZERO 239#define GGML_F16x_VEC_SET1 GGML_F32Cxt_SET1 240#define GGML_F16x_VEC_LOAD (p ,i ) GGML_F32Cxt_LOAD(p) 241#define GGML_F16x_VEC_STORE (p ,r ,i ) GGML_F32Cxt_STORE((__fp16 *)(p), r) 242#define GGML_F16x_VEC_FMA GGML_F32Cxt_FMA 243#define GGML_F16x_VEC_ADD GGML_F32Cxt_ADD 244#define GGML_F16x_VEC_MUL GGML_F32Cxt_MUL 245#define GGML_F16x_VEC_REDUCE GGML_F32Cxt_REDUCE 246 247#define GGML_F16xt_REDUCE_ONE_IMPL (pg ,a ) svaddv_f16(pg, a) 248#define GGML_F16xt_REDUCE_ONE (a ) GGML_F16xt_REDUCE_ONE_IMPL(DEFAULT_PG16, a) 249 250#define GGML_F16xt_REDUCE_MIXED_IMPL (pg16 ,res ,sum1 ,sum2 ,sum3 ,sum4 ) \ 251{ \ 252 sum1 = svadd_f16_x(pg16, sum1, sum2); \ 253 sum3 = svadd_f16_x(pg16, sum3, sum4); \ 254 sum1 = svadd_f16_x(pg16, sum1, sum3); \ 255 __fp16 sum_f16 = svaddv_f16(pg16, sum1); \ 256 (res) = (ggml_float) sum_f16; \ 257} 258#define GGML_F16xt_REDUCE_MIXED (res ,sum1 ,sum2 ,sum3 ,sum4 ) \ 259 GGML_F16xt_REDUCE_MIXED_IMPL(DEFAULT_PG16, res, sum1, sum2, sum3, sum4) 260 261// F16 NEON 262 263#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC ) 264#define GGML_F16_STEP 32 265#define GGML_F16_EPR 8 266 267#define GGML_F16x8 float16x8_t 268#define GGML_F16x8_ZERO vdupq_n_f16(0.0f) 269#define GGML_F16x8_SET1 (x ) vdupq_n_f16(x) 270#define GGML_F16x8_LOAD (x ) vld1q_f16((const __fp16 *)(x)) 271#define GGML_F16x8_STORE vst1q_f16 272#define GGML_F16x8_FMA (a ,b ,c ) vfmaq_f16(a, b, c) 273#define GGML_F16x8_ADD vaddq_f16 274#define GGML_F16x8_MUL vmulq_f16 275#define GGML_F16x8_REDUCE (res ,x ) \ 276 do { \ 277 int offset = GGML_F16_ARR >> 1; \ 278 for (int i = 0; i < offset; ++i) { \ 279 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 280 } \ 281 offset >>= 1; \ 282 for (int i = 0; i < offset; ++i) { \ 283 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 284 } \ 285 offset >>= 1; \ 286 for (int i = 0; i < offset; ++i) { \ 287 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 288 } \ 289 const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \ 290 const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \ 291 (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \ 292 } while (0) 293 294#define GGML_F16_VEC GGML_F16x8 295#define GGML_F16_VEC_ZERO GGML_F16x8_ZERO 296#define GGML_F16_VEC_SET1 GGML_F16x8_SET1 297#define GGML_F16_VEC_LOAD (p ,i ) GGML_F16x8_LOAD(p) 298#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F16x8_STORE((__fp16 *)(p), (r)[i]) 299#define GGML_F16_VEC_FMA GGML_F16x8_FMA 300#define GGML_F16_VEC_ADD GGML_F16x8_ADD 301#define GGML_F16_VEC_MUL GGML_F16x8_MUL 302#define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE 303#else 304// if FP16 vector arithmetic is not supported, we use FP32 instead 305// and take advantage of the vcvt_ functions to convert to/from FP16 306 307#define GGML_F16_STEP 16 308#define GGML_F16_EPR 4 309 310#define GGML_F32Cx4 float32x4_t 311#define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) 312#define GGML_F32Cx4_SET1 (x ) vdupq_n_f32(x) 313#define GGML_F32Cx4_LOAD (x ) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) 314#define GGML_F32Cx4_STORE (x ,y ) vst1_f16(x, vcvt_f16_f32(y)) 315#define GGML_F32Cx4_FMA (a ,b ,c ) vfmaq_f32(a, b, c) 316#define GGML_F32Cx4_ADD vaddq_f32 317#define GGML_F32Cx4_MUL vmulq_f32 318#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE 319 320#define GGML_F16_VEC GGML_F32Cx4 321#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO 322#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 323#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx4_LOAD(p) 324#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx4_STORE((__fp16 *)(p), r[i]) 325#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA 326#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD 327#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL 328#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE 329#endif 330 331#elif defined(__ARM_NEON )&& defined(__ARM_FEATURE_FMA )&& defined(__ARM_FP16_FORMAT_IEEE ) 332 333#define GGML_SIMD 334 335// F32 NEON 336 337#define GGML_F32_STEP 16 338#define GGML_F32_EPR 4 339 340#define GGML_F32x4 float32x4_t 341#define GGML_F32x4_ZERO vdupq_n_f32(0.0f) 342#define GGML_F32x4_SET1 (x ) vdupq_n_f32(x) 343#define GGML_F32x4_LOAD vld1q_f32 344#define GGML_F32x4_STORE vst1q_f32 345#define GGML_F32x4_FMA (a ,b ,c ) vfmaq_f32(a, b, c) 346#define GGML_F32x4_ADD vaddq_f32 347#define GGML_F32x4_MUL vmulq_f32 348#define GGML_F32x4_REDUCE_ONE (x ) vaddvq_f32(x) 349#define GGML_F32x4_REDUCE (res ,x ) \ 350{ \ 351 int offset = GGML_F32_ARR >> 1; \ 352 for (int i = 0; i < offset; ++i) { \ 353 (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ 354 } \ 355 offset >>= 1; \ 356 for (int i = 0; i < offset; ++i) { \ 357 (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ 358 } \ 359 offset >>= 1; \ 360 for (int i = 0; i < offset; ++i) { \ 361 (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ 362 } \ 363 (res) = (ggml_float) GGML_F32x4_REDUCE_ONE((x)[0]); \ 364} 365 366#define GGML_F32_VEC GGML_F32x4 367#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 368#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 369#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 370#define GGML_F32_VEC_STORE GGML_F32x4_STORE 371#define GGML_F32_VEC_FMA GGML_F32x4_FMA 372#define GGML_F32_VEC_ADD GGML_F32x4_ADD 373#define GGML_F32_VEC_MUL GGML_F32x4_MUL 374#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 375 376// F16 NEON 377 378#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC ) 379#define GGML_F16_STEP 32 380#define GGML_F16_EPR 8 381 382#define GGML_F16x8 float16x8_t 383#define GGML_F16x8_ZERO vdupq_n_f16(0.0f) 384#define GGML_F16x8_SET1 (x ) vdupq_n_f16(x) 385#define GGML_F16x8_LOAD (x ) vld1q_f16((const __fp16 *)(x)) 386#define GGML_F16x8_STORE vst1q_f16 387#define GGML_F16x8_FMA (a ,b ,c ) vfmaq_f16(a, b, c) 388#define GGML_F16x8_ADD vaddq_f16 389#define GGML_F16x8_MUL vmulq_f16 390#define GGML_F16x8_REDUCE (res ,x ) \ 391 do { \ 392 int offset = GGML_F16_ARR >> 1; \ 393 for (int i = 0; i < offset; ++i) { \ 394 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 395 } \ 396 offset >>= 1; \ 397 for (int i = 0; i < offset; ++i) { \ 398 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 399 } \ 400 offset >>= 1; \ 401 for (int i = 0; i < offset; ++i) { \ 402 (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ 403 } \ 404 const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \ 405 const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \ 406 (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \ 407 } while (0) 408 409#define GGML_F16_VEC GGML_F16x8 410#define GGML_F16_VEC_ZERO GGML_F16x8_ZERO 411#define GGML_F16_VEC_SET1 GGML_F16x8_SET1 412#define GGML_F16_VEC_LOAD (p ,i ) GGML_F16x8_LOAD(p) 413#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F16x8_STORE((__fp16 *)(p), (r)[i]) 414#define GGML_F16_VEC_FMA GGML_F16x8_FMA 415#define GGML_F16_VEC_ADD GGML_F16x8_ADD 416#define GGML_F16_VEC_MUL GGML_F16x8_MUL 417#define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE 418#else 419// if FP16 vector arithmetic is not supported, we use FP32 instead 420// and take advantage of the vcvt_ functions to convert to/from FP16 421 422#define GGML_F16_STEP 16 423#define GGML_F16_EPR 4 424 425#define GGML_F32Cx4 float32x4_t 426#define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) 427#define GGML_F32Cx4_SET1 (x ) vdupq_n_f32(x) 428#define GGML_F32Cx4_LOAD (x ) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) 429#define GGML_F32Cx4_STORE (x ,y ) vst1_f16(x, vcvt_f16_f32(y)) 430#define GGML_F32Cx4_FMA (a ,b ,c ) vfmaq_f32(a, b, c) 431#define GGML_F32Cx4_ADD vaddq_f32 432#define GGML_F32Cx4_MUL vmulq_f32 433#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE 434 435#define GGML_F16_VEC GGML_F32Cx4 436#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO 437#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 438#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx4_LOAD(p) 439#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx4_STORE((__fp16 *)(p), r[i]) 440#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA 441#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD 442#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL 443#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE 444#endif 445 446#elif defined(__AVX512F__ ) 447 448#define GGML_SIMD 449 450// F32 AVX512 451 452#define GGML_F32_STEP 64 453#define GGML_F32_EPR 16 454 455#define GGML_F32x16 __m512 456#define GGML_F32x16_ZERO _mm512_setzero_ps() 457#define GGML_F32x16_SET1 (x ) _mm512_set1_ps(x) 458#define GGML_F32x16_LOAD _mm512_loadu_ps 459#define GGML_F32x16_STORE _mm512_storeu_ps 460// _mm512_fmadd_ps is defined in AVX512F so no guard is required 461#define GGML_F32x16_FMA (a ,b ,c ) _mm512_fmadd_ps(b, c, a) 462#define GGML_F32x16_ADD _mm512_add_ps 463#define GGML_F32x16_MUL _mm512_mul_ps 464#define GGML_F32x16_REDUCE (res ,x ) \ 465do { \ 466 int offset = GGML_F32_ARR >> 1; \ 467 for (int i = 0; i < offset; ++i) { \ 468 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 469 } \ 470 offset >>= 1; \ 471 for (int i = 0; i < offset; ++i) { \ 472 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 473 } \ 474 offset >>= 1; \ 475 for (int i = 0; i < offset; ++i) { \ 476 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 477 } \ 478 res = (ggml_float) _mm512_reduce_add_ps(x[0]); \ 479} while (0) 480 481// TODO: is this optimal ? 482 483#define GGML_F32_VEC GGML_F32x16 484#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO 485#define GGML_F32_VEC_SET1 GGML_F32x16_SET1 486#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD 487#define GGML_F32_VEC_STORE GGML_F32x16_STORE 488#define GGML_F32_VEC_FMA GGML_F32x16_FMA 489#define GGML_F32_VEC_ADD GGML_F32x16_ADD 490#define GGML_F32_VEC_MUL GGML_F32x16_MUL 491#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE 492 493// F16 AVX512 494 495#if defined(__AVX512FP16__ ) 496 497#define GGML_F16_STEP 128 498#define GGML_F16_EPR 32 499 500#define GGML_F16x32 __m512h 501#define GGML_F16x32_ZERO _mm512_setzero_ph() 502#define GGML_F16x32_SET1 (x ) _mm512_set1_ph(__extension__(_Float16)(x)) 503#define GGML_F16x32_LOAD (x ) _mm512_loadu_ph(x) 504#define GGML_F16x32_STORE (x ,y ) _mm512_storeu_ph(x, y) 505#define GGML_F16x32_FMA (a ,b ,c ) _mm512_fmadd_ph(b, c, a) 506#define GGML_F16x32_ADD _mm512_add_ph 507#define GGML_F16x32_MUL _mm512_mul_ph 508#define GGML_F16x32_REDUCE (res ,x ) \ 509do { \ 510 int offset = GGML_F16_ARR >> 1; \ 511 for (int i = 0; i < offset; ++i) { \ 512 x[i] = _mm512_add_ph(x[i], x[offset+i]); \ 513 } \ 514 offset >>= 1; \ 515 for (int i = 0; i < offset; ++i) { \ 516 x[i] = _mm512_add_ph(x[i], x[offset+i]); \ 517 } \ 518 offset >>= 1; \ 519 for (int i = 0; i < offset; ++i) { \ 520 x[i] = _mm512_add_ph(x[i], x[offset+i]); \ 521 } \ 522 res = (ggml_float) _mm512_reduce_add_ph(x[0]); \ 523} while (0) 524 525#define GGML_F16_VEC GGML_F16x32 526#define GGML_F16_VEC_ZERO GGML_F16x32_ZERO 527#define GGML_F16_VEC_SET1 GGML_F16x32_SET1 528#define GGML_F16_VEC_LOAD (p ,i ) GGML_F16x32_LOAD(p) 529#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F16x32_STORE(p, r[i]) 530#define GGML_F16_VEC_FMA GGML_F16x32_FMA 531#define GGML_F16_VEC_ADD GGML_F16x32_ADD 532#define GGML_F16_VEC_MUL GGML_F16x32_MUL 533#define GGML_F16_VEC_REDUCE GGML_F16x32_REDUCE 534 535#else // Fallback FP16 <-> FP32 536 537#define GGML_F16_STEP 64 538#define GGML_F16_EPR 16 539 540#define GGML_F32Cx16 __m512 541#define GGML_F32Cx16_ZERO _mm512_setzero_ps() 542#define GGML_F32Cx16_SET1 (x ) _mm512_set1_ps(x) 543 544// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F 545// so F16C guard isn't required 546#define GGML_F32Cx16_LOAD (x ) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x))) 547#define GGML_F32Cx16_STORE (x ,y ) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0)) 548 549#define GGML_F32Cx16_FMA (a ,b ,c ) _mm512_fmadd_ps(b, c, a) 550#define GGML_F32Cx16_ADD _mm512_add_ps 551#define GGML_F32Cx16_MUL _mm512_mul_ps 552#define GGML_F32Cx16_REDUCE (res ,x ) \ 553do { \ 554 int offset = GGML_F32_ARR >> 1; \ 555 for (int i = 0; i < offset; ++i) { \ 556 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 557 } \ 558 offset >>= 1; \ 559 for (int i = 0; i < offset; ++i) { \ 560 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 561 } \ 562 offset >>= 1; \ 563 for (int i = 0; i < offset; ++i) { \ 564 x[i] = _mm512_add_ps(x[i], x[offset+i]); \ 565 } \ 566 res = (ggml_float) _mm512_reduce_add_ps(x[0]); \ 567} while (0) 568 569#define GGML_F16_VEC GGML_F32Cx16 570#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO 571#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1 572#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx16_LOAD(p) 573#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx16_STORE(p, r[i]) 574#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA 575#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD 576#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL 577 578#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE 579 580#endif // __AVX512FP16__ 581#elif defined(__AVX__ ) 582 583#define GGML_SIMD 584 585// F32 AVX 586 587#define GGML_F32_STEP 32 588#define GGML_F32_EPR 8 589 590#define GGML_F32x8 __m256 591#define GGML_F32x8_ZERO _mm256_setzero_ps() 592#define GGML_F32x8_SET1 (x ) _mm256_set1_ps(x) 593#define GGML_F32x8_LOAD _mm256_loadu_ps 594#define GGML_F32x8_STORE _mm256_storeu_ps 595#if defined(__FMA__ ) 596#define GGML_F32x8_FMA (a ,b ,c ) _mm256_fmadd_ps(b, c, a) 597#else 598#define GGML_F32x8_FMA (a ,b ,c ) _mm256_add_ps(_mm256_mul_ps(b, c), a) 599#endif 600#define GGML_F32x8_ADD _mm256_add_ps 601#define GGML_F32x8_MUL _mm256_mul_ps 602#define GGML_F32x8_REDUCE (res ,x ) \ 603do { \ 604 int offset = GGML_F32_ARR >> 1; \ 605 for (int i = 0; i < offset; ++i) { \ 606 x[i] = _mm256_add_ps(x[i], x[offset+i]); \ 607 } \ 608 offset >>= 1; \ 609 for (int i = 0; i < offset; ++i) { \ 610 x[i] = _mm256_add_ps(x[i], x[offset+i]); \ 611 } \ 612 offset >>= 1; \ 613 for (int i = 0; i < offset; ++i) { \ 614 x[i] = _mm256_add_ps(x[i], x[offset+i]); \ 615 } \ 616 const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \ 617 _mm256_extractf128_ps(x[0], 1)); \ 618 const __m128 t1 = _mm_hadd_ps(t0, t0); \ 619 res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \ 620} while (0) 621// TODO: is this optimal ? 622 623#define GGML_F32_VEC GGML_F32x8 624#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO 625#define GGML_F32_VEC_SET1 GGML_F32x8_SET1 626#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD 627#define GGML_F32_VEC_STORE GGML_F32x8_STORE 628#define GGML_F32_VEC_FMA GGML_F32x8_FMA 629#define GGML_F32_VEC_ADD GGML_F32x8_ADD 630#define GGML_F32_VEC_MUL GGML_F32x8_MUL 631#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE 632 633// F16 AVX 634 635#define GGML_F16_STEP 32 636#define GGML_F16_EPR 8 637 638// F16 arithmetic is not supported by AVX, so we use F32 instead 639 640#define GGML_F32Cx8 __m256 641#define GGML_F32Cx8_ZERO _mm256_setzero_ps() 642#define GGML_F32Cx8_SET1 (x ) _mm256_set1_ps(x) 643 644#if defined(__F16C__ ) 645// the _mm256_cvt intrinsics require F16C 646#define GGML_F32Cx8_LOAD (x ) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x))) 647#define GGML_F32Cx8_STORE (x ,y ) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0)) 648#else 649static inline __m256 __avx_f32cx8_load (const ggml_fp16_t * x ) { 650float tmp [8 ]; 651 652for (int i = 0 ;i < 8 ;i ++ ) { 653tmp [i ]= GGML_CPU_FP16_TO_FP32 (x [i ]); 654 } 655 656return _mm256_loadu_ps (tmp ); 657} 658static inline void __avx_f32cx8_store (ggml_fp16_t * x ,__m256 y ) { 659float arr [8 ]; 660 661_mm256_storeu_ps (arr ,y ); 662 663for (int i = 0 ;i < 8 ;i ++ ) 664x [i ]= GGML_CPU_FP32_TO_FP16 (arr [i ]); 665} 666#define GGML_F32Cx8_LOAD (x ) __avx_f32cx8_load(x) 667#define GGML_F32Cx8_STORE (x ,y ) __avx_f32cx8_store(x, y) 668#endif 669 670#define GGML_F32Cx8_FMA GGML_F32x8_FMA 671#define GGML_F32Cx8_ADD _mm256_add_ps 672#define GGML_F32Cx8_MUL _mm256_mul_ps 673#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE 674 675#define GGML_F16_VEC GGML_F32Cx8 676#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO 677#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1 678#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx8_LOAD(p) 679#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx8_STORE(p, r[i]) 680#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA 681#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD 682#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL 683#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE 684 685#elif defined(__POWER9_VECTOR__ ) 686 687#define GGML_SIMD 688 689// F32 POWER9 690 691#define GGML_F32_STEP 32 692#define GGML_F32_EPR 4 693 694#define GGML_F32x4 vector float 695#define GGML_F32x4_ZERO {0.0f} 696#define GGML_F32x4_SET1 vec_splats 697#define GGML_F32x4_LOAD (p ) vec_xl(0, p) 698#define GGML_F32x4_STORE (p ,r ) vec_xst(r, 0, p) 699#define GGML_F32x4_FMA (a ,b ,c ) vec_madd(b, c, a) 700#define GGML_F32x4_ADD vec_add 701#define GGML_F32x4_MUL vec_mul 702#define GGML_F32x4_REDUCE (res ,x ) \ 703{ \ 704 int offset = GGML_F32_ARR >> 1; \ 705 for (int i = 0; i < offset; ++i) { \ 706 x[i] = vec_add(x[i], x[offset+i]); \ 707 } \ 708 offset >>= 1; \ 709 for (int i = 0; i < offset; ++i) { \ 710 x[i] = vec_add(x[i], x[offset+i]); \ 711 } \ 712 offset >>= 1; \ 713 for (int i = 0; i < offset; ++i) { \ 714 x[i] = vec_add(x[i], x[offset+i]); \ 715 } \ 716 res = vec_extract(x[0], 0) + \ 717 vec_extract(x[0], 1) + \ 718 vec_extract(x[0], 2) + \ 719 vec_extract(x[0], 3); \ 720} 721#define GGML_F32x4_REDUCE_4 (res ,s0 ,s1 ,s2 ,s3 ) \ 722{ \ 723 vector float v = vec_add(vec_add(s0, s1), \ 724 vec_add(s2, s3)); \ 725 v = vec_add(v, vec_sld(v, v, 8)); \ 726 v = vec_add(v, vec_sld(v, v, 4)); \ 727 res += (ggml_float) vec_extract(v, 0); \ 728} 729 730#define GGML_F32_VEC GGML_F32x4 731#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 732#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 733#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 734#define GGML_F32_VEC_STORE GGML_F32x4_STORE 735#define GGML_F32_VEC_FMA GGML_F32x4_FMA 736#define GGML_F32_VEC_ADD GGML_F32x4_ADD 737#define GGML_F32_VEC_MUL GGML_F32x4_MUL 738#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 739 740// F16 POWER9 741#define GGML_F16_STEP GGML_F32_STEP 742#define GGML_F16_EPR GGML_F32_EPR 743#define GGML_F16_VEC GGML_F32x4 744#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO 745#define GGML_F16_VEC_SET1 GGML_F32x4_SET1 746#define GGML_F16_VEC_FMA GGML_F32x4_FMA 747#define GGML_F16_VEC_ADD GGML_F32x4_ADD 748#define GGML_F16_VEC_MUL GGML_F32x4_MUL 749#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE 750// Use vec_xl, not vec_ld, in case the load address is not aligned. 751#define GGML_F16_VEC_LOAD (p ,i ) (i & 0x1) ? \ 752 vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \ 753 vec_extract_fp32_from_shortl(vec_xl(0, p)) 754static inline unsigned char ggml_endian_byte (int i ) { 755uint16_t tmp_val = 1 ; 756return ((unsigned char * )& tmp_val )[i ]; 757} 758#define GGML_ENDIAN_BYTE (i ) ggml_endian_byte(i) 759#define GGML_F16_VEC_STORE (p ,r ,i ) \ 760 if (i & 0x1) \ 761 vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \ 762 r[i - GGML_ENDIAN_BYTE(0)]), \ 763 0, p - GGML_F16_EPR) 764 765//BF16 POWER9 766#define GGML_BF16_STEP 16 767#define GGML_BF16_EPR 8 768 769#define GGML_BF16x8 vector unsigned short 770#define GGML_BF16x8_ZERO vec_splats((unsigned short)0) 771#define GGML_BF16x8_LOAD (p ) vec_xl(0, (const unsigned short *)(p)) 772 773#define GGML_BF16_VEC GGML_BF16x8 774#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO 775#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD 776#if defined(__LITTLE_ENDIAN__ ) 777#define GGML_BF16_TO_F32_LO (v ) ((vector float) vec_mergel(GGML_BF16_VEC_ZERO, (v))) 778#define GGML_BF16_TO_F32_HI (v ) ((vector float) vec_mergeh(GGML_BF16_VEC_ZERO, (v))) 779#else 780#define GGML_BF16_TO_F32_LO (v ) ((vector float) vec_mergel((v), GGML_BF16_VEC_ZERO)) 781#define GGML_BF16_TO_F32_HI (v ) ((vector float) vec_mergeh((v), GGML_BF16_VEC_ZERO)) 782#endif 783#define GGML_BF16_FMA_LO (acc ,x ,y ) \ 784 (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y)) 785#define GGML_BF16_FMA_HI (acc ,x ,y ) \ 786 (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) 787 788#elif defined(__wasm_simd128__ ) 789 790#define GGML_SIMD 791 792// F32 WASM 793 794#define GGML_F32_STEP 16 795#define GGML_F32_EPR 4 796 797#define GGML_F32x4 v128_t 798#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f) 799#define GGML_F32x4_SET1 (x ) wasm_f32x4_splat(x) 800#define GGML_F32x4_LOAD wasm_v128_load 801#define GGML_F32x4_STORE wasm_v128_store 802#define GGML_F32x4_FMA (a ,b ,c ) wasm_f32x4_add(wasm_f32x4_mul(b, c), a) 803#define GGML_F32x4_ADD wasm_f32x4_add 804#define GGML_F32x4_MUL wasm_f32x4_mul 805#define GGML_F32x4_REDUCE (res ,x ) \ 806{ \ 807 int offset = GGML_F32_ARR >> 1; \ 808 for (int i = 0; i < offset; ++i) { \ 809 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 810 } \ 811 offset >>= 1; \ 812 for (int i = 0; i < offset; ++i) { \ 813 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 814 } \ 815 offset >>= 1; \ 816 for (int i = 0; i < offset; ++i) { \ 817 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 818 } \ 819 res = wasm_f32x4_extract_lane(x[0], 0) + \ 820 wasm_f32x4_extract_lane(x[0], 1) + \ 821 wasm_f32x4_extract_lane(x[0], 2) + \ 822 wasm_f32x4_extract_lane(x[0], 3); \ 823} 824 825#define GGML_F32_VEC GGML_F32x4 826#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 827#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 828#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 829#define GGML_F32_VEC_STORE GGML_F32x4_STORE 830#define GGML_F32_VEC_FMA GGML_F32x4_FMA 831#define GGML_F32_VEC_ADD GGML_F32x4_ADD 832#define GGML_F32_VEC_MUL GGML_F32x4_MUL 833#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 834 835// F16 WASM 836 837#define GGML_F16_STEP 16 838#define GGML_F16_EPR 4 839 840inline static v128_t __wasm_f16x4_load (const ggml_fp16_t * p ) { 841float tmp [4 ]; 842 843tmp [0 ]= GGML_CPU_FP16_TO_FP32 (p [0 ]); 844tmp [1 ]= GGML_CPU_FP16_TO_FP32 (p [1 ]); 845tmp [2 ]= GGML_CPU_FP16_TO_FP32 (p [2 ]); 846tmp [3 ]= GGML_CPU_FP16_TO_FP32 (p [3 ]); 847 848return wasm_v128_load (tmp ); 849} 850 851inline static void __wasm_f16x4_store (ggml_fp16_t * p ,v128_t x ) { 852float tmp [4 ]; 853 854wasm_v128_store (tmp ,x ); 855 856p [0 ]= GGML_CPU_FP32_TO_FP16 (tmp [0 ]); 857p [1 ]= GGML_CPU_FP32_TO_FP16 (tmp [1 ]); 858p [2 ]= GGML_CPU_FP32_TO_FP16 (tmp [2 ]); 859p [3 ]= GGML_CPU_FP32_TO_FP16 (tmp [3 ]); 860} 861 862#define GGML_F16x4 v128_t 863#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f) 864#define GGML_F16x4_SET1 (x ) wasm_f32x4_splat(x) 865#define GGML_F16x4_LOAD (x ) __wasm_f16x4_load(x) 866#define GGML_F16x4_STORE (x ,y ) __wasm_f16x4_store(x, y) 867#define GGML_F16x4_FMA GGML_F32x4_FMA 868#define GGML_F16x4_ADD wasm_f32x4_add 869#define GGML_F16x4_MUL wasm_f32x4_mul 870#define GGML_F16x4_REDUCE (res ,x ) \ 871{ \ 872 int offset = GGML_F16_ARR >> 1; \ 873 for (int i = 0; i < offset; ++i) { \ 874 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 875 } \ 876 offset >>= 1; \ 877 for (int i = 0; i < offset; ++i) { \ 878 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 879 } \ 880 offset >>= 1; \ 881 for (int i = 0; i < offset; ++i) { \ 882 x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ 883 } \ 884 res = (ggml_float) (wasm_f32x4_extract_lane(x[0], 0) + \ 885 wasm_f32x4_extract_lane(x[0], 1) + \ 886 wasm_f32x4_extract_lane(x[0], 2) + \ 887 wasm_f32x4_extract_lane(x[0], 3)); \ 888} 889 890#define GGML_F16_VEC GGML_F16x4 891#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO 892#define GGML_F16_VEC_SET1 GGML_F16x4_SET1 893#define GGML_F16_VEC_LOAD (p ,i ) GGML_F16x4_LOAD(p) 894#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F16x4_STORE(p, r[i]) 895#define GGML_F16_VEC_FMA GGML_F16x4_FMA 896#define GGML_F16_VEC_ADD GGML_F16x4_ADD 897#define GGML_F16_VEC_MUL GGML_F16x4_MUL 898#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE 899 900#elif defined(__SSE3__ ) 901 902#define GGML_SIMD 903 904// F32 SSE 905 906#define GGML_F32_STEP 32 907#define GGML_F32_EPR 4 908 909#define GGML_F32x4 __m128 910#define GGML_F32x4_ZERO _mm_setzero_ps() 911#define GGML_F32x4_SET1 (x ) _mm_set1_ps(x) 912#define GGML_F32x4_LOAD _mm_loadu_ps 913#define GGML_F32x4_STORE _mm_storeu_ps 914#if defined(__FMA__ ) 915// TODO: Does this work? 916#define GGML_F32x4_FMA (a ,b ,c ) _mm_fmadd_ps(b, c, a) 917#else 918#define GGML_F32x4_FMA (a ,b ,c ) _mm_add_ps(_mm_mul_ps(b, c), a) 919#endif 920#define GGML_F32x4_ADD _mm_add_ps 921#define GGML_F32x4_MUL _mm_mul_ps 922#define GGML_F32x4_REDUCE (res ,x ) \ 923{ \ 924 int offset = GGML_F32_ARR >> 1; \ 925 for (int i = 0; i < offset; ++i) { \ 926 x[i] = _mm_add_ps(x[i], x[offset+i]); \ 927 } \ 928 offset >>= 1; \ 929 for (int i = 0; i < offset; ++i) { \ 930 x[i] = _mm_add_ps(x[i], x[offset+i]); \ 931 } \ 932 offset >>= 1; \ 933 for (int i = 0; i < offset; ++i) { \ 934 x[i] = _mm_add_ps(x[i], x[offset+i]); \ 935 } \ 936 const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \ 937 res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \ 938} 939// TODO: is this optimal ? 940 941#define GGML_F32_VEC GGML_F32x4 942#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 943#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 944#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 945#define GGML_F32_VEC_STORE GGML_F32x4_STORE 946#define GGML_F32_VEC_FMA GGML_F32x4_FMA 947#define GGML_F32_VEC_ADD GGML_F32x4_ADD 948#define GGML_F32_VEC_MUL GGML_F32x4_MUL 949#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 950 951// F16 SSE 952 953#define GGML_F16_STEP 32 954#define GGML_F16_EPR 4 955 956static inline __m128 __sse_f16x4_load (const ggml_fp16_t * x ) { 957float tmp [4 ]; 958 959tmp [0 ]= GGML_CPU_FP16_TO_FP32 (x [0 ]); 960tmp [1 ]= GGML_CPU_FP16_TO_FP32 (x [1 ]); 961tmp [2 ]= GGML_CPU_FP16_TO_FP32 (x [2 ]); 962tmp [3 ]= GGML_CPU_FP16_TO_FP32 (x [3 ]); 963 964return _mm_loadu_ps (tmp ); 965} 966 967static inline void __sse_f16x4_store (ggml_fp16_t * x ,__m128 y ) { 968float arr [4 ]; 969 970_mm_storeu_ps (arr ,y ); 971 972x [0 ]= GGML_CPU_FP32_TO_FP16 (arr [0 ]); 973x [1 ]= GGML_CPU_FP32_TO_FP16 (arr [1 ]); 974x [2 ]= GGML_CPU_FP32_TO_FP16 (arr [2 ]); 975x [3 ]= GGML_CPU_FP32_TO_FP16 (arr [3 ]); 976} 977 978#define GGML_F32Cx4 __m128 979#define GGML_F32Cx4_ZERO _mm_setzero_ps() 980#define GGML_F32Cx4_SET1 (x ) _mm_set1_ps(x) 981#define GGML_F32Cx4_LOAD (x ) __sse_f16x4_load(x) 982#define GGML_F32Cx4_STORE (x ,y ) __sse_f16x4_store(x, y) 983#define GGML_F32Cx4_FMA GGML_F32x4_FMA 984#define GGML_F32Cx4_ADD _mm_add_ps 985#define GGML_F32Cx4_MUL _mm_mul_ps 986#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE 987 988#define GGML_F16_VEC GGML_F32Cx4 989#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO 990#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 991#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx4_LOAD(p) 992#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx4_STORE(p, r[i]) 993#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA 994#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD 995#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL 996#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE 997 998#elif defined(__loongarch_asx ) 999 1000#define GGML_SIMD 1001 1002// F32 LASX 1003#define GGML_F32_STEP 32 1004#define GGML_F32_EPR 8 1005 1006#define GGML_F32x8 __m256 1007#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0) 1008#define GGML_F32x8_SET1 (x ) (__m256)__lasx_xvreplfr2vr_s((x)) 1009#define GGML_F32x8_LOAD (x ) (__m256)__lasx_xvld((x), 0) 1010#define GGML_F32x8_STORE (x ,y ) __lasx_xvst((y), (x), 0) 1011#define GGML_F32x8_FMA (a ,b ,c ) __lasx_xvfmadd_s(b, c, a) 1012#define GGML_F32x8_ADD __lasx_xvfadd_s 1013#define GGML_F32x8_MUL __lasx_xvfmul_s 1014#define GGML_F32x8_REDUCE (res ,x ) \ 1015do { \ 1016 int offset = GGML_F32_ARR >> 1; \ 1017 for (int i = 0; i < offset; ++i) { \ 1018 x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ 1019 } \ 1020 offset >>= 1; \ 1021 for (int i = 0; i < offset; ++i) { \ 1022 x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ 1023 } \ 1024 offset >>= 1; \ 1025 for (int i = 0; i < offset; ++i) { \ 1026 x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ 1027 } \ 1028 float *tmp_p = (float *)&x[0]; \ 1029 res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \ 1030} while (0) 1031// TODO: is this optimal ? 1032 1033#define GGML_F32_VEC GGML_F32x8 1034#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO 1035#define GGML_F32_VEC_SET1 GGML_F32x8_SET1 1036#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD 1037#define GGML_F32_VEC_STORE GGML_F32x8_STORE 1038#define GGML_F32_VEC_FMA GGML_F32x8_FMA 1039#define GGML_F32_VEC_ADD GGML_F32x8_ADD 1040#define GGML_F32_VEC_MUL GGML_F32x8_MUL 1041#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE 1042 1043// F16 LASX 1044 1045#define GGML_F16_STEP 32 1046#define GGML_F16_EPR 8 1047 1048// F16 arithmetic is not supported by LASX, so we use F32 instead 1049 1050#define GGML_F32Cx8 __m256 1051#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0) 1052#define GGML_F32Cx8_SET1 (x ) (__m256)__lasx_xvreplfr2vr_s((x)) 1053 1054static inline __m256 __lasx_f32cx8_load (const ggml_fp16_t * x ) { 1055__m256i a ; 1056memcpy (& a ,x ,sizeof (ggml_fp16_t )* 8 ); 1057a = __lasx_xvpermi_d (a ,0 | (1 <<4 )); 1058return __lasx_xvfcvtl_s_h (a ); 1059} 1060 1061static inline void __lasx_f32cx8_store (ggml_fp16_t * x ,__m256 y ) { 1062__m256i a = __lasx_xvfcvt_h_s (y ,y ); 1063a = __lasx_xvpermi_d (a ,0 | (2 <<2 )); 1064memcpy (x ,& a ,sizeof (ggml_fp16_t )* 8 ); 1065} 1066#define GGML_F32Cx8_LOAD (x ) __lasx_f32cx8_load(x) 1067#define GGML_F32Cx8_STORE (x ,y ) __lasx_f32cx8_store(x, y) 1068 1069#define GGML_F32Cx8_FMA GGML_F32x8_FMA 1070#define GGML_F32Cx8_ADD __lasx_xvfadd_s 1071#define GGML_F32Cx8_MUL __lasx_xvfmul_s 1072#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE 1073 1074#define GGML_F16_VEC GGML_F32Cx8 1075#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO 1076#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1 1077#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx8_LOAD(p) 1078#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx8_STORE(p, r[i]) 1079#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA 1080#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD 1081#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL 1082#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE 1083 1084#elif defined(__loongarch_sx ) 1085 1086#define GGML_SIMD 1087 1088// F32 LSX 1089 1090#define GGML_F32_STEP 32 1091#define GGML_F32_EPR 4 1092 1093#define GGML_F32x4 __m128 1094#define GGML_F32x4_ZERO (__m128)__lsx_vldi(0) 1095#define GGML_F32x4_SET1 (x ) (__m128)__lsx_vreplfr2vr_s((x)) 1096#define GGML_F32x4_LOAD (x ) (__m128)__lsx_vld((x), 0) 1097#define GGML_F32x4_STORE (x ,y ) __lsx_vst(y, x, 0) 1098#define GGML_F32x4_FMA (a ,b ,c ) __lsx_vfmadd_s(b, c, a) 1099#define GGML_F32x4_ADD __lsx_vfadd_s 1100#define GGML_F32x4_MUL __lsx_vfmul_s 1101 1102#define GGML_F32x4_REDUCE (res ,x ) \ 1103{ \ 1104 int offset = GGML_F32_ARR >> 1; \ 1105 for (int i = 0; i < offset; ++i) { \ 1106 x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ 1107 } \ 1108 offset >>= 1; \ 1109 for (int i = 0; i < offset; ++i) { \ 1110 x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ 1111 } \ 1112 offset >>= 1; \ 1113 for (int i = 0; i < offset; ++i) { \ 1114 x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ 1115 } \ 1116 __m128i t0 = __lsx_vpickev_w((__m128i)x[0], (__m128i)x[0]); \ 1117 __m128i t1 = __lsx_vpickod_w((__m128i)x[0], (__m128i)x[0]); \ 1118 __m128 t2 = __lsx_vfadd_s((__m128)t0, (__m128)t1); \ 1119 __m128i t3 = __lsx_vpickev_w((__m128i)t2, (__m128i)t2); \ 1120 __m128i t4 = __lsx_vpickod_w((__m128i)t2, (__m128i)t2); \ 1121 __m128 t5 = __lsx_vfadd_s((__m128)t3, (__m128)t4); \ 1122 res = (ggml_float) ((v4f32)t5)[0]; \ 1123} 1124 1125#define GGML_F32_VEC GGML_F32x4 1126#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 1127#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 1128#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 1129#define GGML_F32_VEC_STORE GGML_F32x4_STORE 1130#define GGML_F32_VEC_FMA GGML_F32x4_FMA 1131#define GGML_F32_VEC_ADD GGML_F32x4_ADD 1132#define GGML_F32_VEC_MUL GGML_F32x4_MUL 1133#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 1134 1135// F16 LSX 1136 1137#define GGML_F16_STEP 32 1138#define GGML_F16_EPR 4 1139 1140static inline __m128 __lsx_f16x4_load (const ggml_fp16_t * x ) { 1141return __lsx_vfcvtl_s_h (__lsx_vld ((const void * )x ,0 )); 1142} 1143 1144static inline void __lsx_f16x4_store (ggml_fp16_t * x ,__m128 y ) { 1145__m128i a = __lsx_vfcvt_h_s (y ,y ); 1146memcpy (x ,& a ,sizeof (ggml_fp16_t )* 4 ); 1147} 1148 1149#define GGML_F32Cx4 __m128 1150#define GGML_F32Cx4_ZERO (__m128)__lsx_vldi(0) 1151#define GGML_F32Cx4_SET1 (x ) (__m128)__lsx_vreplfr2vr_s((x)) 1152#define GGML_F32Cx4_LOAD (x ) (__m128)__lsx_f16x4_load(x) 1153#define GGML_F32Cx4_STORE (x ,y ) __lsx_f16x4_store(x, y) 1154#define GGML_F32Cx4_FMA GGML_F32x4_FMA 1155#define GGML_F32Cx4_ADD __lsx_vfadd_s 1156#define GGML_F32Cx4_MUL __lsx_vfmul_s 1157#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE 1158 1159#define GGML_F16_VEC GGML_F32Cx4 1160#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO 1161#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 1162#define GGML_F16_VEC_LOAD (p ,i ) GGML_F32Cx4_LOAD(p) 1163#define GGML_F16_VEC_STORE (p ,r ,i ) GGML_F32Cx4_STORE(p, r[i]) 1164#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA 1165#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD 1166#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL 1167#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE 1168 1169#elif defined(__VXE__ )|| defined(__VXE2__ ) 1170 1171#define GGML_SIMD 1172 1173// F32 s390x 1174 1175#define GGML_F32_STEP 32 1176#define GGML_F32_EPR 4 1177 1178#define GGML_F32x4 float32x4_t 1179#define GGML_F32x4_ZERO vec_splats(0.0f) 1180#define GGML_F32x4_SET1 vec_splats 1181#define GGML_F32x4_LOAD (p ) vec_xl(0, p) 1182#define GGML_F32x4_STORE (p ,r ) vec_xst(r, 0, p) 1183#define GGML_F32x4_FMA (a ,b ,c ) vec_madd(b, c, a) 1184#define GGML_F32x4_ADD vec_add 1185#define GGML_F32x4_MUL vec_mul 1186#define GGML_F32x4_REDUCE (res ,x ) \ 1187{ \ 1188 int offset = GGML_F32_ARR >> 1; \ 1189 for (int i = 0; i < offset; ++i) { \ 1190 x[i] = vec_add(x[i], x[offset + i]); \ 1191 } \ 1192 offset >>= 1; \ 1193 for (int i = 0; i < offset; ++i) { \ 1194 x[i] = vec_add(x[i], x[offset + i]); \ 1195 } \ 1196 offset >>= 1; \ 1197 for (int i = 0; i < offset; ++i) { \ 1198 x[i] = vec_add(x[i], x[offset + i]); \ 1199 } \ 1200 float32x4_t tmp = x[0] + vec_reve(x[0]); \ 1201 res = tmp[0] + tmp[1]; \ 1202} 1203#define GGML_F32x4_REDUCE_4 (res ,s0 ,s1 ,s2 ,s3 ) \ 1204{ \ 1205 float32x4_t v = vec_add(vec_add(s0, s1), \ 1206 vec_add(s2, s3)); \ 1207 v = vec_add(v, vec_sld(v, v, 8)); \ 1208 v = vec_add(v, vec_sld(v, v, 4)); \ 1209 res += (ggml_float)vec_extract(v, 0); \ 1210} 1211 1212#define GGML_F32_VEC GGML_F32x4 1213#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 1214#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 1215#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 1216#define GGML_F32_VEC_STORE GGML_F32x4_STORE 1217#define GGML_F32_VEC_FMA GGML_F32x4_FMA 1218#define GGML_F32_VEC_ADD GGML_F32x4_ADD 1219#define GGML_F32_VEC_MUL GGML_F32x4_MUL 1220#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 1221 1222// F16 s390x 1223#define GGML_F16_STEP GGML_F32_STEP 1224#define GGML_F16_EPR GGML_F32_EPR 1225 1226static inline float32x4_t __lzs_f16cx4_load (const ggml_fp16_t * x ) { 1227float tmp [4 ]; 1228 1229for (int i = 0 ;i < 4 ;i ++ ) { 1230tmp [i ]= GGML_CPU_FP16_TO_FP32 (x [i ]); 1231 } 1232 1233// note: keep type-cast here to prevent compiler bugs 1234// see: https://github.com/ggml-org/llama.cpp/issues/12846 1235return vec_xl (0 , (const float * )(tmp )); 1236} 1237 1238static inline void __lzs_f16cx4_store (ggml_fp16_t * x ,float32x4_t v_y ) { 1239float arr [4 ]; 1240 1241// note: keep type-cast here to prevent compiler bugs 1242// see: https://github.com/ggml-org/llama.cpp/issues/12846 1243vec_xst (v_y ,0 , (float * )(arr )); 1244 1245for (int i = 0 ;i < 4 ;i ++ ) { 1246x [i ]= GGML_CPU_FP32_TO_FP16 (arr [i ]); 1247 } 1248} 1249 1250#define GGML_F16_VEC GGML_F32x4 1251#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO 1252#define GGML_F16_VEC_SET1 GGML_F32x4_SET1 1253#define GGML_F16_VEC_LOAD (p ,i ) __lzs_f16cx4_load(p) 1254#define GGML_F16_VEC_STORE (p ,r ,i ) __lzs_f16cx4_store(p, r[i]) 1255#define GGML_F16_VEC_FMA GGML_F32x4_FMA 1256#define GGML_F16_VEC_ADD GGML_F32x4_ADD 1257#define GGML_F16_VEC_MUL GGML_F32x4_MUL 1258#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE 1259 1260// BF16 s390x 1261#define GGML_BF16_STEP 16 1262#define GGML_BF16_EPR 8 1263 1264#define GGML_BF16x8 __vector unsigned short 1265#define GGML_BF16x8_ZERO vec_splats((unsigned short)0) 1266#define GGML_BF16x8_LOAD (p ) vec_xl(0, (const unsigned short *)(p)) 1267 1268#define GGML_BF16_VEC GGML_BF16x8 1269#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO 1270#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD 1271#define GGML_BF16_TO_F32_LO (v ) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO)) 1272#define GGML_BF16_TO_F32_HI (v ) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO)) 1273#define GGML_BF16_FMA_LO (acc ,x ,y ) \ 1274 (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y)) 1275#define GGML_BF16_FMA_HI (acc ,x ,y ) \ 1276 (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) 1277 1278#elif defined(__riscv_v_intrinsic ) 1279 1280// compatible with vlen >= 128 1281 1282#define GGML_SIMD 1283 1284// F32 1285 1286#define GGML_F32_STEP 16 1287#define GGML_F32_EPR 4 1288 1289#define GGML_F32x4 vfloat32m1_t 1290#define GGML_F32x4_ZERO __riscv_vfmv_v_f_f32m1(0.0f, GGML_F32_EPR) 1291#define GGML_F32x4_SET1 (x ) __riscv_vfmv_v_f_f32m1(x, GGML_F32_EPR) 1292#define GGML_F32x4_LOAD (x ) __riscv_vle32_v_f32m1(x, GGML_F32_EPR) 1293#define GGML_F32x4_STORE (b ,v ) __riscv_vse32_v_f32m1(b, v, GGML_F32_EPR) 1294#define GGML_F32x4_FMA (a ,b ,c ) __riscv_vfmacc_vv_f32m1(a, b, c, GGML_F32_EPR) 1295#define GGML_F32x4_ADD (a ,b ) __riscv_vfadd_vv_f32m1(a, b, GGML_F32_EPR) 1296#define GGML_F32x4_MUL (a ,b ) __riscv_vfmul_vv_f32m1(a, b, GGML_F32_EPR) 1297 1298#define GGML_F32_VEC GGML_F32x4 1299#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO 1300#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 1301#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD 1302#define GGML_F32_VEC_STORE GGML_F32x4_STORE 1303#define GGML_F32_VEC_FMA GGML_F32x4_FMA 1304#define GGML_F32_VEC_ADD GGML_F32x4_ADD 1305#define GGML_F32_VEC_MUL GGML_F32x4_MUL 1306#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE 1307 1308#endif 1309 1310// GGML_F32_ARR / GGML_F16_ARR 1311// number of registers to use per step 1312#ifdef GGML_SIMD 1313#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) 1314#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) 1315#endif 1316 1317#ifdef __cplusplus 1318} 1319#endif