char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

cqderekggml-hexagon: add F16 support for unary ops (#28228)f027c4f1b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.9 KiB118 linesraw
1#ifndef HVX_LOG_H
2#define HVX_LOG_H
3
4#include "hvx-base.h"
5
6// Approximates ln(x) element-wise for float vectors.
7// x must contain positive float elements.
8// Uses Abramowitz & Stegun polynomial approximation 4.1.44 for ln(1+y) over [0, 1].
9static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {
10    // x = m * 2^e, where m in [1, 2)
11    HVX_Vector biased_e = Q6_Vuw_vlsr_VuwR(x, 23);
12    HVX_Vector e_int = Q6_Vw_vsub_VwVw(biased_e, Q6_V_vsplat_R(127));
13    HVX_Vector e_float = Q6_Vsf_equals_Vw(e_int);
14
15    // Extract mantissa and set exponent to 127 (which represents float value in [1.0, 2.0))
16    HVX_Vector mant_mask = Q6_V_vsplat_R(0x007FFFFF);
17    HVX_Vector exp_127 = Q6_V_vsplat_R(0x3F800000);
18    HVX_Vector m = Q6_V_vor_VV(Q6_V_vand_VV(x, mant_mask), exp_127);
19
20    // y = m - 1.0f, y in [0, 1)
21    HVX_Vector y = hvx_vec_sub_f32_f32(m, hvx_vec_splat_f32(1.0f));
22
23    // Abramowitz & Stegun 4.1.44 polynomial approximation of ln(1+y)
24    HVX_Vector c;
25    HVX_Vector res;
26
27    c   = hvx_vec_splat_f32(-0.0064535442f);
28    res = hvx_vec_mul_f32_f32(y, c);
29
30    c   = hvx_vec_splat_f32(0.0360884937f);
31    res = hvx_vec_add_f32_f32(res, c);
32    res = hvx_vec_mul_f32_f32(y, res);
33
34    c   = hvx_vec_splat_f32(-0.0953293897f);
35    res = hvx_vec_add_f32_f32(res, c);
36    res = hvx_vec_mul_f32_f32(y, res);
37
38    c   = hvx_vec_splat_f32(0.1676540711f);
39    res = hvx_vec_add_f32_f32(res, c);
40    res = hvx_vec_mul_f32_f32(y, res);
41
42    c   = hvx_vec_splat_f32(-0.2407338084f);
43    res = hvx_vec_add_f32_f32(res, c);
44    res = hvx_vec_mul_f32_f32(y, res);
45
46    c   = hvx_vec_splat_f32(0.3317990258f);
47    res = hvx_vec_add_f32_f32(res, c);
48    res = hvx_vec_mul_f32_f32(y, res);
49
50    c   = hvx_vec_splat_f32(-0.4998741238f);
51    res = hvx_vec_add_f32_f32(res, c);
52    res = hvx_vec_mul_f32_f32(y, res);
53
54    c   = hvx_vec_splat_f32(0.9999964239f);
55    res = hvx_vec_add_f32_f32(res, c);
56    res = hvx_vec_mul_f32_f32(y, res);
57
58    // ln(x) = e * ln(2) + ln(1+y)
59    HVX_Vector ln2 = hvx_vec_splat_f32(0.69314718056f);
60    HVX_Vector term_e = hvx_vec_mul_f32_f32(e_float, ln2);
61
62    return hvx_vec_add_f32_f32(term_e, res);
63}
64
65static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
66    assert((unsigned long) dst % 128 == 0);
67    assert((unsigned long) src % 128 == 0);
68
69    HVX_Vector * restrict vdst = (HVX_Vector *) dst;
70    HVX_Vector * restrict vsrc = (HVX_Vector *) src;
71
72    const uint32_t elem_size = sizeof(float);
73    const uint32_t epv       = 128 / elem_size;
74    const uint32_t nvec      = n / epv;
75    const uint32_t nloe      = n % epv;
76
77    uint32_t i = 0;
78
79    _Pragma("unroll(4)")
80    for (; i < nvec; i++) {
81        vdst[i] = hvx_vec_log_f32(vsrc[i]);
82    }
83    if (nloe) {
84        HVX_Vector v = hvx_vec_log_f32(vsrc[i]);
85        hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
86    }
87}
88
89// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back.
90static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
91    assert((unsigned long) dst % 128 == 0);
92    assert((unsigned long) src % 128 == 0);
93
94    HVX_Vector * restrict vdst = (HVX_Vector *) dst;
95    HVX_Vector * restrict vsrc = (HVX_Vector *) src;
96
97    const uint32_t nvec = n / VLEN_FP16;
98    const uint32_t nloe = n % VLEN_FP16;
99
100    uint32_t i = 0;
101
102    _Pragma("unroll(4)")
103    for (; i < nvec; i++) {
104        HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
105        HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
106        HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
107        vdst[i] = hvx_vec_f32_to_f16(r0, r1);
108    }
109    if (nloe) {
110        HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
111        HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
112        HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
113        HVX_Vector v = hvx_vec_f32_to_f16(r0, r1);
114        hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v);
115    }
116}
117
118#endif /* HVX_LOG_H */