char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
eafe15a5e
char/aliceai-t5-moe
1#ifndef HVX_DIV_H 2#define HVX_DIV_H 3 4#include <HAP_farf.h> 5 6#include <math.h> 7#include <string.h> 8#include <assert.h> 9#include <stddef.h> 10#include <stdint.h> 11 12#include "hvx-base.h" 13#include "hex-utils.h" 14#include "hvx-inverse.h" 15#include "hvx-arith.h" 16 17#if __HVX_ARCH__ < 79 18#define HVX_OP_MUL_F32 (a ,b ) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) 19#define HVX_OP_MUL_F16 (a ,b ) Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b)) 20#else 21#define HVX_OP_MUL_F32 (a ,b ) Q6_Vsf_vmpy_VsfVsf(a, b) 22#define HVX_OP_MUL_F16 (a ,b ) Q6_Vhf_vmpy_VhfVhf(a, b) 23#endif 24 25// Compute div by scaler in f32. Requires first by expanding fp32 to fp16 and converting the result back to fp32. 26static inline HVX_Vector hvx_div_mul_f16_const_using_f32 (HVX_Vector vec1_hf ,HVX_Vector vec2_sf_const ,HVX_Vector vec_hf_one_1_0 ) { 27#if __HVX_ARCH__ < 79 28HVX_VectorPair src_to_f32 = Q6_Wqf32_vmpy_VhfVhf (vec1_hf ,vec_hf_one_1_0 ); 29HVX_Vector src_to_f32_0 = Q6_Vsf_equals_Vqf32 (Q6_V_lo_W (src_to_f32 )); 30HVX_Vector src_to_f32_1 = Q6_Vsf_equals_Vqf32 (Q6_V_hi_W (src_to_f32 )); 31#else 32HVX_VectorPair src_to_f32 = Q6_Wsf_vmpy_VhfVhf (vec1_hf ,vec_hf_one_1_0 ); 33HVX_Vector src_to_f32_0 = Q6_V_lo_W (src_to_f32 ); 34HVX_Vector src_to_f32_1 = Q6_V_hi_W (src_to_f32 ); 35#endif 36 37HVX_Vector div_f32_0 = HVX_OP_MUL_F32 (src_to_f32_0 ,vec2_sf_const ); 38HVX_Vector div_f32_1 = HVX_OP_MUL_F32 (src_to_f32_1 ,vec2_sf_const ); 39 40#if __HVX_ARCH__ < 79 41HVX_Vector res = hvx_vec_f32_to_f16 (div_f32_0 ,div_f32_1 ); 42#else 43HVX_Vector res = Q6_Vhf_vcvt_VsfVsf (div_f32_0 ,div_f32_1 ); 44#endif 45return res ; 46} 47 48// Variant for <v79: Use pre-computed f16 reciprocal constant 49static inline HVX_Vector hvx_div_mul_f16_const_using_f16 (HVX_Vector vec1_hf ,HVX_Vector const_inv_hf ) { 50// Multiply by pre-computed f16 reciprocal constant 51return HVX_OP_MUL_F16 (vec1_hf ,const_inv_hf ); 52} 53 54#define hvx_div_scaler_f16_loop_body (dst_type ,src_type ,vec_store ) \ 55 do { \ 56 dst_type * restrict vdst = (dst_type *) dst; \ 57 src_type * restrict vsrc = (src_type *) src; \ 58 \ 59 HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \ 60 \ 61 const uint32_t nvec = n / VLEN_FP16; \ 62 const uint32_t nloe = n % VLEN_FP16; \ 63 \ 64 uint32_t i = 0; \ 65 \ 66 _Pragma("unroll(4)") \ 67 for (; i < nvec; i++) { \ 68 HVX_Vector res; \ 69 if (__HVX_ARCH__ < 79) { \ 70 res = hvx_div_mul_f16_const_using_f16(vsrc[i], val_vec_f16); \ 71 } else { \ 72 res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \ 73 } \ 74 vdst[i] = res; \ 75 } \ 76 if (nloe) { \ 77 HVX_Vector res; \ 78 if (__HVX_ARCH__ < 79) { \ 79 res = hvx_div_mul_f16_const_using_f16(vsrc[i], val_vec_f16); \ 80 } else { \ 81 res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \ 82 } \ 83 vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \ 84 } \ 85 } while(0) 86 87static inline void hvx_div_scalar_f16_aa (uint8_t * restrictdst ,const uint8_t * restrictsrc ,const _Float16 val ,uint32_t n ) { 88const HVX_Vector val_vec_f32 = hvx_vec_splat_f32 (1.0f /((float )val )); 89const HVX_Vector val_vec_f16 = hvx_vec_splat_f16 (1.0f /val ); 90assert ((uintptr_t )dst %128 == 0 ); 91assert ((uintptr_t )src %128 == 0 ); 92hvx_div_scaler_f16_loop_body (HVX_Vector ,HVX_Vector ,hvx_vec_store_a ); 93} 94static inline void hvx_div_scalar_f16_au (uint8_t * restrictdst ,const uint8_t * restrictsrc ,const _Float16 val ,uint32_t n ) { 95const HVX_Vector val_vec_f32 = hvx_vec_splat_f32 (1.0f /((float )val )); 96const HVX_Vector val_vec_f16 = hvx_vec_splat_f16 (1.0f /val ); 97assert ((uintptr_t )dst %128 == 0 ); 98hvx_div_scaler_f16_loop_body (HVX_Vector ,HVX_UVector ,hvx_vec_store_a ); 99} 100static inline void hvx_div_scalar_f16_ua (uint8_t * restrictdst ,const uint8_t * restrictsrc ,const _Float16 val ,uint32_t n ) { 101const HVX_Vector val_vec_f32 = hvx_vec_splat_f32 (1.0f /((float )val )); 102const HVX_Vector val_vec_f16 = hvx_vec_splat_f16 (1.0f /val ); 103assert ((uintptr_t )src %128 == 0 ); 104hvx_div_scaler_f16_loop_body (HVX_UVector ,HVX_Vector ,hvx_vec_store_u ); 105} 106static inline void hvx_div_scalar_f16_uu (uint8_t * restrictdst ,const uint8_t * restrictsrc ,const _Float16 val ,uint32_t n ) { 107const HVX_Vector val_vec_f32 = hvx_vec_splat_f32 (1.0f /((float )val )); 108const HVX_Vector val_vec_f16 = hvx_vec_splat_f16 (1.0f /val ); 109hvx_div_scaler_f16_loop_body (HVX_UVector ,HVX_UVector ,hvx_vec_store_u ); 110} 111 112// Compute div by using hvx_vec_inverse_f32_guard. Requires first by exapnding fp32 to fp16 and convert the result back to fp32. 113static inline HVX_Vector hvx_vec_div_f16_using_f32 (HVX_Vector vec1 ,HVX_Vector vec2 ,HVX_Vector f32_nan_inf_mask ,HVX_Vector vec_hf_one_1_0 ) { 114#if __HVX_ARCH__ < 79 115// Convert first input to fp32 116HVX_VectorPair vec1_to_f32 = Q6_Wqf32_vmpy_VhfVhf (vec1 ,vec_hf_one_1_0 );// *1.0 117HVX_Vector vec1_to_f32_0 = Q6_Vsf_equals_Vqf32 (Q6_V_lo_W (vec1_to_f32 )); 118HVX_Vector vec1_to_f32_1 = Q6_Vsf_equals_Vqf32 (Q6_V_hi_W (vec1_to_f32 )); 119 120// Convert second input to fp32 121HVX_VectorPair vec2_to_f32 = Q6_Wqf32_vmpy_VhfVhf (vec2 ,vec_hf_one_1_0 );// *1.0 122HVX_Vector vec2_to_f32_0 = Q6_Vsf_equals_Vqf32 (Q6_V_lo_W (vec2_to_f32 )); 123HVX_Vector vec2_to_f32_1 = Q6_Vsf_equals_Vqf32 (Q6_V_hi_W (vec2_to_f32 )); 124#else 125// Convert first input to fp32 126HVX_VectorPair vec1_to_f32 = Q6_Wsf_vmpy_VhfVhf (vec1 ,vec_hf_one_1_0 );// *1.0 127HVX_Vector vec1_to_f32_0 = Q6_V_lo_W (vec1_to_f32 ); 128HVX_Vector vec1_to_f32_1 = Q6_V_hi_W (vec1_to_f32 ); 129 130// Convert second input to fp32 131HVX_VectorPair vec2_to_f32 = Q6_Wsf_vmpy_VhfVhf (vec2 ,vec_hf_one_1_0 );// *1.0 132HVX_Vector vec2_to_f32_0 = Q6_V_lo_W (vec2_to_f32 ); 133HVX_Vector vec2_to_f32_1 = Q6_V_hi_W (vec2_to_f32 ); 134#endif 135 136// Inverse second input in fp32 137HVX_Vector vec2_inv_f32_0 = hvx_vec_inverse_f32_guard (vec2_to_f32_0 ,f32_nan_inf_mask ); 138HVX_Vector vec2_inv_f32_1 = hvx_vec_inverse_f32_guard (vec2_to_f32_1 ,f32_nan_inf_mask ); 139 140// Multiply first input by inverse of second, in fp32 141HVX_Vector div_f32_0 = HVX_OP_MUL_F32 (vec1_to_f32_0 ,vec2_inv_f32_0 ); 142HVX_Vector div_f32_1 = HVX_OP_MUL_F32 (vec1_to_f32_1 ,vec2_inv_f32_1 ); 143 144// Convert back to fp16 145#if __HVX_ARCH__ < 79 146HVX_Vector recip = hvx_vec_f32_to_f16 (div_f32_0 ,div_f32_1 ); 147#else 148HVX_Vector recip = Q6_Vhf_vcvt_VsfVsf (div_f32_0 ,div_f32_1 ); 149#endif 150 151return recip ; 152} 153 154// Hybrid approach: f16 reciprocal for <v79, f32 precision for >=v79 155static inline HVX_Vector hvx_vec_hybrid_div_f16 (HVX_Vector vec1 ,HVX_Vector vec2 ,HVX_Vector f32_nan_inf_mask ,HVX_Vector f16_nan_inf_mask ,HVX_Vector vec_hf_one_1_0 ) { 156#if __HVX_ARCH__ < 79 157// For older architectures, use f16 reciprocal to avoid NaN/-inf issues 158HVX_Vector vec2_inv = hvx_vec_inverse_f16_guard (vec2 ,f16_nan_inf_mask ); 159return HVX_OP_MUL_F16 (vec1 ,vec2_inv ); 160#else 161return hvx_vec_div_f16_using_f32 (vec1 ,vec2 ,f32_nan_inf_mask ,vec_hf_one_1_0 ); 162#endif 163} 164 165#define hvx_div_f16_loop_body (dst_type ,src0_type ,src1_type ,vec_store ) \ 166 do { \ 167 dst_type * restrict vdst = (dst_type *) dst; \ 168 src0_type * restrict vsrc0 = (src0_type *) src0; \ 169 src1_type * restrict vsrc1 = (src1_type *) src1; \ 170 \ 171 const HVX_Vector f32_nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \ 172 const HVX_Vector f16_nan_inf_mask = Q6_Vh_vsplat_R(0x7c00); \ 173 const HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \ 174 \ 175 const uint32_t nvec = n / VLEN_FP16; \ 176 const uint32_t nloe = n % VLEN_FP16; \ 177 \ 178 uint32_t i = 0; \ 179 \ 180 _Pragma("unroll(4)") \ 181 for (; i < nvec; i++) { \ 182 HVX_Vector res = hvx_vec_hybrid_div_f16(vsrc0[i], vsrc1[i], \ 183 f32_nan_inf_mask, f16_nan_inf_mask, \ 184 hf_one); \ 185 vdst[i] = res; \ 186 } \ 187 if (nloe) { \ 188 HVX_Vector res = hvx_vec_hybrid_div_f16(vsrc0[i], vsrc1[i], \ 189 f32_nan_inf_mask, f16_nan_inf_mask, \ 190 hf_one); \ 191 vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \ 192 } \ 193 } while(0) 194 195#define hvx_div_f32_loop_body (dst_type ,src0_type ,src1_type ,vec_store ) \ 196 do { \ 197 dst_type * restrict vdst = (dst_type *) dst; \ 198 src0_type * restrict vsrc0 = (src0_type *) src0; \ 199 src1_type * restrict vsrc1 = (src1_type *) src1; \ 200 \ 201 const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \ 202 \ 203 const uint32_t nvec = n / VLEN_FP32; \ 204 const uint32_t nloe = n % VLEN_FP32; \ 205 \ 206 uint32_t i = 0; \ 207 \ 208 _Pragma("unroll(4)") \ 209 for (; i < nvec; i++) { \ 210 HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \ 211 HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \ 212 vdst[i] = res; \ 213 } \ 214 if (nloe) { \ 215 HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \ 216 HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \ 217 vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, res); \ 218 } \ 219 } while(0) 220 221// Generic macro to define alignment permutations for an op 222#define DEFINE_HVX_DIV_OP_VARIANTS (OP_NAME ,OP_LOOP_BODY ) \ 223static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 224 assert((uintptr_t) dst % 128 == 0); \ 225 assert((uintptr_t) src0 % 128 == 0); \ 226 assert((uintptr_t) src1 % 128 == 0); \ 227 OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ 228} \ 229static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 230 assert((uintptr_t) dst % 128 == 0); \ 231 assert((uintptr_t) src0 % 128 == 0); \ 232 OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ 233} \ 234static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 235 assert((uintptr_t) dst % 128 == 0); \ 236 assert((uintptr_t) src1 % 128 == 0); \ 237 OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ 238} \ 239static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 240 assert((uintptr_t) dst % 128 == 0); \ 241 OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ 242} \ 243static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 244 assert((uintptr_t) src0 % 128 == 0); \ 245 assert((uintptr_t) src1 % 128 == 0); \ 246 OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ 247} \ 248static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 249 assert((uintptr_t) src0 % 128 == 0); \ 250 OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ 251} \ 252static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 253 assert((uintptr_t) src1 % 128 == 0); \ 254 OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ 255} \ 256static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ 257 OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ 258} \ 259 260// Dispatcher logic 261#define HVX_DIV_DISPATCHER (OP_NAME ) \ 262static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ 263 if (hex_is_aligned((void *) dst, 128)) { \ 264 if (hex_is_aligned((void *) src0, 128)) { \ 265 if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ 266 else OP_NAME##_aau(dst, src0, src1, num_elems); \ 267 } else { \ 268 if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ 269 else OP_NAME##_auu(dst, src0, src1, num_elems); \ 270 } \ 271 } else { \ 272 if (hex_is_aligned((void *) src0, 128)) { \ 273 if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ 274 else OP_NAME##_uau(dst, src0, src1, num_elems); \ 275 } else { \ 276 if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ 277 else OP_NAME##_uuu(dst, src0, src1, num_elems); \ 278 } \ 279 } \ 280} 281 282DEFINE_HVX_DIV_OP_VARIANTS (hvx_div_f32 ,hvx_div_f32_loop_body ) 283DEFINE_HVX_DIV_OP_VARIANTS (hvx_div_f16 ,hvx_div_f16_loop_body ) 284 285HVX_DIV_DISPATCHER (hvx_div_f32 ) 286HVX_DIV_DISPATCHER (hvx_div_f16 ) 287 288#undef HVX_OP_MUL_F32 289#undef HVX_OP_MUL_F16 290 291#endif // HVX_DIV_H