hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Tarek Dakhranggml-cpu: fix SVE leftover path in ggml_vec_dot_f32 (#24699)c16c35b812026-06-26

1#include "vec.h"
2
3#include <cassert>
4
5// precomputed gelu table for f16 (128 KB)
6ggml_fp16_t ggml_table_gelu_f16[1 << 16];
7
8// precomputed quick gelu table for f16 (128 KB)
9ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];
10
11void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {
12   assert(nrc == 1);
13   GGML_UNUSED(nrc);
14   GGML_UNUSED(bx);
15   GGML_UNUSED(by);
16   GGML_UNUSED(bs);
17
18#if defined(GGML_SIMD)
19    float sumf = 0.0f;
20
21    #if defined(__ARM_FEATURE_SVE)
22        const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;
23        const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16
24        const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers
25
26        const int np = (n & ~(ggml_f32_step - 1));
27        svfloat32_t sum1 = svdup_n_f32(0.0f);
28        svfloat32_t sum2 = svdup_n_f32(0.0f);
29        svfloat32_t sum3 = svdup_n_f32(0.0f);
30        svfloat32_t sum4 = svdup_n_f32(0.0f);
31        svfloat32_t sum5 = svdup_n_f32(0.0f);
32        svfloat32_t sum6 = svdup_n_f32(0.0f);
33        svfloat32_t sum7 = svdup_n_f32(0.0f);
34        svfloat32_t sum8 = svdup_n_f32(0.0f);
35        svfloat32_t ax1,ax2,ax3,ax4,ax5,ax6,ax7,ax8;
36        svfloat32_t ay1,ay2,ay3,ay4,ay5,ay6,ay7,ay8;
37        for (int i = 0; i < np; i += ggml_f32_step) {
38            ax1 = GGML_F32_VEC_LOAD(x + i);
39            ay1 = GGML_F32_VEC_LOAD(y + i);
40            sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);
41
42            ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);
43            ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);
44            sum2 = GGML_F32_VEC_FMA(sum2, ax2, ay2);
45
46            ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);
47            ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);
48            sum3 = GGML_F32_VEC_FMA(sum3, ax3, ay3);
49
50            ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);
51            ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);
52            sum4 = GGML_F32_VEC_FMA(sum4, ax4, ay4);
53
54            ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);
55            ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);
56            sum5 = GGML_F32_VEC_FMA(sum5, ax5, ay5);
57
58            ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);
59            ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);
60            sum6 = GGML_F32_VEC_FMA(sum6, ax6, ay6);
61
62            ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);
63            ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);
64            sum7 = GGML_F32_VEC_FMA(sum7, ax7, ay7);
65
66            ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);
67            ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);
68            sum8 = GGML_F32_VEC_FMA(sum8, ax8, ay8);
69        }
70        // leftovers
71        // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop
72        const int np2 = (n & ~(ggml_f32_epr - 1));
73        for (int i = np; i < np2; i += ggml_f32_epr) {
74            ax1 = GGML_F32_VEC_LOAD(x + i);
75            ay1 = GGML_F32_VEC_LOAD(y + i);
76            sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);
77        }
78        // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmla on available elements only
79        if (np2 < n) {
80            svbool_t pg = svwhilelt_b32(np2, n);
81            ax1 = svld1_f32(pg, x + np2);
82            ay1 = svld1_f32(pg, y + np2);
83            sum1 = svmla_f32_m(pg, sum1, ax1, ay1);
84        }
85        // reduce sum1,sum2 to sum1
86        GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);
87    #elif defined(__riscv_v_intrinsic)
88        int vl = __riscv_vsetvlmax_e32m8();
89        vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);
90        vfloat32m8_t vsum;
91        vfloat32m8_t ax;
92        vfloat32m8_t ay;
93        vsum = __riscv_vfmv_v_f_f32m8_tu(vsum, 0.0f, vl);
94        for (int i = 0; i < n; i += vl) {
95            vl = __riscv_vsetvl_e32m8(n - i);
96            ax = __riscv_vle32_v_f32m8_tu(ax, &x[i], vl);
97            ay = __riscv_vle32_v_f32m8_tu(ay, &y[i], vl);
98            vsum = __riscv_vfmacc_vv_f32m8_tu(vsum, ax, ay, vl);
99        }
100        vl = __riscv_vsetvlmax_e32m8();
101        vs = __riscv_vfredusum_vs_f32m8_f32m1(vsum, vs, vl);
102        sumf += __riscv_vfmv_f_s_f32m1_f32(vs);
103    #else
104        const int np = (n & ~(GGML_F32_STEP - 1));
105
106        GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
107
108        GGML_F32_VEC ax[GGML_F32_ARR];
109        GGML_F32_VEC ay[GGML_F32_ARR];
110
111        for (int i = 0; i < np; i += GGML_F32_STEP) {
112            for (int j = 0; j < GGML_F32_ARR; j++) {
113                ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
114                ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
115
116                sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);
117            }
118        }
119
120        // reduce sum0..sum3 to sum0
121        GGML_F32_VEC_REDUCE(sumf, sum);
122
123        // leftovers
124        for (int i = np; i < n; ++i) {
125            sumf += x[i]*y[i];
126        }
127    #endif
128#else
129    // scalar
130    ggml_float sumf = 0.0;
131    for (int i = 0; i < n; ++i) {
132        sumf += (ggml_float)(x[i]*y[i]);
133    }
134#endif
135
136    *s = sumf;
137}
138
139void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc) {
140    assert(nrc == 1);
141    GGML_UNUSED(nrc);
142    GGML_UNUSED(bx);
143    GGML_UNUSED(by);
144    GGML_UNUSED(bs);
145    int i = 0;
146    ggml_float sumf = 0;
147
148#if defined(__AVX512BF16__)
149    __m512 c1 = _mm512_setzero_ps();
150    __m512 c2 = _mm512_setzero_ps();
151    for (; i + 64 <= n; i += 64) {
152        c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),
153                             m512bh(_mm512_loadu_si512((y + i))));
154        c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),
155                             m512bh(_mm512_loadu_si512((y + i + 32))));
156    }
157    sumf += (ggml_float)_mm512_reduce_add_ps(c1);
158    sumf += (ggml_float)_mm512_reduce_add_ps(c2);
159
160#elif defined(__AVX512F__)
161#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))
162    __m512 c1 = _mm512_setzero_ps();
163    __m512 c2 = _mm512_setzero_ps();
164    for (; i + 32 <= n; i += 32) {
165        c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
166        c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);
167    }
168    sumf += (ggml_float)_mm512_reduce_add_ps(c1);
169    sumf += (ggml_float)_mm512_reduce_add_ps(c2);
170
171#undef LOAD
172#elif defined(__AVX2__) || defined(__AVX__)
173#if defined(__AVX2__)
174#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))
175#else
176#define LOAD(p) _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16)), (_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_bsrli_si128(_mm_loadu_si128((const __m128i *)(p)), 8)), 16)), 1))
177#endif
178    __m256 c1 = _mm256_setzero_ps();
179    __m256 c2 = _mm256_setzero_ps();
180    __m256 c3 = _mm256_setzero_ps();
181    __m256 c4 = _mm256_setzero_ps();
182    for (; i + 32 <= n; i += 32) {
183        c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
184        c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);
185        c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);
186        c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);
187    }
188    __m128 g;
189    c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),
190                       _mm256_add_ps(c2, c4));
191    g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),
192                   _mm256_castps256_ps128(c1));
193    g = _mm_add_ps(g, _mm_movehl_ps(g, g));
194    g = _mm_add_ss(g, _mm_movehdup_ps(g));
195    sumf += (ggml_float)_mm_cvtss_f32(g);
196
197#undef LOAD
198#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma)
199    size_t vl = __riscv_vsetvlmax_e32m4();
200
201    // initialize accumulators to all zeroes
202    vfloat32m4_t vsum0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);
203    vfloat32m4_t vsum1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);
204
205    // calculate step size
206    const size_t epr = __riscv_vsetvlmax_e16m2();
207    const size_t step = epr * 2;
208    const int np = (n & ~(step - 1));
209
210    // unroll by 2
211    for (; i < np; i += step) {
212        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], epr);
213        vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], epr);
214        vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, epr);
215        __asm__ __volatile__ ("" ::: "memory");
216
217        vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i + epr], epr);
218        vbfloat16m2_t ay1 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i + epr], epr);
219        vsum1 = __riscv_vfwmaccbf16_vv_f32m4(vsum1, ax1, ay1, epr);
220        __asm__ __volatile__ ("" ::: "memory");
221    }
222
223    // accumulate in 1 register
224    vsum0 = __riscv_vfadd_vv_f32m4(vsum0, vsum1, vl);
225
226    // leftovers
227    for (i = np; i < n; i += vl) {
228        vl = __riscv_vsetvl_e16m2(n - i);
229        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], vl);
230        vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], vl);
231        vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, vl);
232    }
233
234    // reduce
235    vl = __riscv_vsetvlmax_e32m4();
236    vfloat32m1_t redsum = __riscv_vfredusum_vs_f32m4_f32m1(vsum0, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);
237    sumf += __riscv_vfmv_f_s_f32m1_f32(redsum);
238
239#elif defined(__POWER9_VECTOR__) || defined(__VXE__) || defined(__VXE2__)
240    const int np = (n & ~(GGML_BF16_STEP - 1));
241    if (np > 0) {
242        GGML_F32_VEC sum[4] = {GGML_F32_VEC_ZERO};
243        for (; i < np; i += GGML_BF16_STEP) {
244            GGML_BF16_VEC vx0 = GGML_BF16_VEC_LOAD(x + i);
245            GGML_BF16_VEC vx1 = GGML_BF16_VEC_LOAD(x + i + 8);
246            GGML_BF16_VEC vy0 = GGML_BF16_VEC_LOAD(y + i);
247            GGML_BF16_VEC vy1 = GGML_BF16_VEC_LOAD(y + i + 8);
248            GGML_BF16_FMA_LO(sum[0], vx0, vy0);
249            GGML_BF16_FMA_HI(sum[1], vx0, vy0);
250            GGML_BF16_FMA_LO(sum[2], vx1, vy1);
251            GGML_BF16_FMA_HI(sum[3], vx1, vy1);
252        }
253        GGML_F32x4_REDUCE_4(sumf, sum[0], sum[1], sum[2], sum[3]);
254    }
255#endif
256
257    for (; i < n; ++i) {
258        sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *
259                             GGML_BF16_TO_FP32(y[i]));
260    }
261    *s = sumf;
262}
263
264void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc) {
265    assert(nrc == 1);
266    GGML_UNUSED(nrc);
267    GGML_UNUSED(bx);
268    GGML_UNUSED(by);
269    GGML_UNUSED(bs);
270
271    ggml_float sumf = 0.0;
272
273
274#if defined(GGML_SIMD)
275    #if defined(__ARM_FEATURE_SVE)
276        const int ggml_f16_epr = svcnth();
277        const int ggml_f16_step = 8 * ggml_f16_epr;
278        const int np = n - (n % ggml_f16_step);
279        const int np2 = n - (n % ggml_f16_epr);
280
281        svfloat32_t sum1_lo = svdup_n_f32(0.0f);
282        svfloat32_t sum1_hi = svdup_n_f32(0.0f);
283        svfloat32_t sum2_lo = svdup_n_f32(0.0f);
284        svfloat32_t sum2_hi = svdup_n_f32(0.0f);
285        svfloat32_t sum3_lo = svdup_n_f32(0.0f);
286        svfloat32_t sum3_hi = svdup_n_f32(0.0f);
287        svfloat32_t sum4_lo = svdup_n_f32(0.0f);
288        svfloat32_t sum4_hi = svdup_n_f32(0.0f);
289
290        for (int i = 0; i < np; i += ggml_f16_step) {
291            ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0), GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0));
292            ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1), GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1));
293            ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2), GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2));
294            ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3), GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3));
295            ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4), GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4));
296            ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5), GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5));
297            ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6), GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6));
298            ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7), GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7));
299        }
300
301        for (int i = np; i < np2; i += ggml_f16_epr) {
302            ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i, 0), GGML_F16x_VEC_LOAD(y + i, 0));
303        }
304
305        if (np2 < n) {
306            const svbool_t pg = svwhilelt_b16(np2, n);
307            const svfloat16_t rx = svld1_f16(pg, (const __fp16 *)(x + np2));
308            const svfloat16_t ry = svld1_f16(pg, (const __fp16 *)(y + np2));
309
310            ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, rx, ry);
311        }
312
313        sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum2_lo);
314        sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum2_hi);
315        sum3_lo = svadd_f32_m(DEFAULT_PG32, sum3_lo, sum4_lo);
316        sum3_hi = svadd_f32_m(DEFAULT_PG32, sum3_hi, sum4_hi);
317        sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum3_lo);
318        sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi);
319
320        sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi);
321    #elif defined(__riscv_v_intrinsic)
322        #if defined(__riscv_zvfh)
323            int vl = __riscv_vsetvlmax_e32m2();
324            vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);
325            vfloat32m2_t vsum;
326            vfloat16m1_t ax;
327            vfloat16m1_t ay;
328            vsum = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vmv_v_x_u32m2(0, vl));
329            for (int i = 0; i < n; i += vl) {
330                vl = __riscv_vsetvl_e16m1(n - i);
331                ax = __riscv_vle16_v_f16m1_tu(ax, (const _Float16 *)&x[i], vl);
332                ay = __riscv_vle16_v_f16m1_tu(ay, (const _Float16 *)&y[i], vl);
333                vsum = __riscv_vfwmacc_vv_f32m2_tu(vsum, ax, ay, vl);
334            }
335            vl = __riscv_vsetvlmax_e32m1();
336            vfloat32m1_t ac0 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(vsum, 0), __riscv_vget_v_f32m2_f32m1(vsum, 1), vl);
337            vs = __riscv_vfredusum_vs_f32m1_f32m1(ac0, vs, vl);
338            sumf += __riscv_vfmv_f_s_f32m1_f32(vs);
339        #else
340            for (int i = 0; i < n; ++i) {
341                sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));
342            }
343        #endif // __riscv_zvfh
344    #else
345        const int np = (n & ~(GGML_F16_STEP - 1));
346
347        GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };
348
349        GGML_F16_VEC ax[GGML_F16_ARR];
350        GGML_F16_VEC ay[GGML_F16_ARR];
351
352        for (int i = 0; i < np; i += GGML_F16_STEP) {
353            for (int j = 0; j < GGML_F16_ARR; j++) {
354                ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
355                ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
356
357                sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
358            }
359        }
360
361        // reduce sum0..sum3 to sum0
362        GGML_F16_VEC_REDUCE(sumf, sum);
363
364        // leftovers
365        for (int i = np; i < n; ++i) {
366            sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));
367        }
368        // if you hit this, you are likely running outside the FP range
369        assert(!isnan(sumf) && !isinf(sumf));
370    #endif
371#else
372    for (int i = 0; i < n; ++i) {
373        sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));
374    }
375#endif // GGML_SIMD
376
377    *s = sumf;
378}
379
380void ggml_vec_silu_f32(const int n, float * y, const float * x) {
381    int i = 0;
382#if defined(__AVX512F__) && defined(__AVX512DQ__)
383    for (; i + 15 < n; i += 16) {
384        _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));
385    }
386#elif defined(__AVX2__) && defined(__FMA__)
387    for (; i + 7 < n; i += 8) {
388        _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));
389    }
390#elif defined(__SSE2__)
391    for (; i + 3 < n; i += 4) {
392        _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));
393    }
394#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)
395    const int vlen = svcntw();
396    for (; i < n; i += vlen) {
397        const svbool_t pg = svwhilelt_b32_s32(i, n);
398        svst1_f32(pg, y + i, ggml_v_silu(pg, svld1_f32(pg, x + i)));
399    }
400#elif defined(__ARM_NEON) && defined(__aarch64__)
401    for (; i + 3 < n; i += 4) {
402        vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));
403    }
404#elif defined(__riscv_v_intrinsic)
405    for (int vl; i < n; i += vl) {
406        vl = __riscv_vsetvl_e32m2(n - i);
407        vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
408        vfloat32m2_t vy = ggml_v_silu_m2(vx, vl);
409        __riscv_vse32_v_f32m2(&y[i], vy, vl);
410    }
411#endif
412    for (; i < n; ++i) {
413        y[i] = ggml_silu_f32(x[i]);
414    }
415}
416
417void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g) {
418    int i = 0;
419#if defined(__AVX512F__) && defined(__AVX512DQ__)
420    for (; i + 15 < n; i += 16) {
421        _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_silu(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i)));
422    }
423#elif defined(__AVX2__) && defined(__FMA__)
424    for (; i + 7 < n; i += 8) {
425        _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_silu(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i)));
426    }
427#elif defined(__SSE2__)
428    for (; i + 3 < n; i += 4) {
429        _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_silu(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i)));
430    }
431#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)
432    const int vlen = svcntw();
433    for (; i < n; i += vlen) {
434        const svbool_t pg = svwhilelt_b32_s32(i, n);
435        svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_silu(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i)));
436    }
437#elif defined(__ARM_NEON) && defined(__aarch64__)
438    for (; i + 3 < n; i += 4) {
439        vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));
440    }
441#elif defined(__riscv_v_intrinsic)
442    for (int vl; i < n; i += vl) {
443        vl = __riscv_vsetvl_e32m2(n - i);
444        vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
445        vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl);
446        vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_silu_m2(vx, vl), vg, vl);
447        __riscv_vse32_v_f32m2(&y[i], vy, vl);
448    }
449#endif
450    for (; i < n; ++i) {
451        y[i] = ggml_silu_f32(x[i]) * g[i];
452    }
453}
454
455ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) {
456    int i = 0;
457    ggml_float sum = 0;
458// TODO: optimize to process the remaining elements in groups using the smaller vector sizes from AVX2 and SSE
459// ref: https://github.com/ggml-org/llama.cpp/pull/15953#pullrequestreview-3310928344
460#if defined(__AVX512F__) && defined(__AVX512DQ__)
461    for (; i + 15 < n; i += 16) {
462        __m512 val = _mm512_sub_ps(_mm512_loadu_ps(x + i),
463                                   _mm512_set1_ps(mean));
464        _mm512_storeu_ps(y + i, val);
465        sum += (ggml_float)_mm512_reduce_add_ps(_mm512_mul_ps(val, val));
466    }
467#elif defined(__AVX2__) && defined(__FMA__)
468    for (; i + 7 < n; i += 8) {
469        __m256 val = _mm256_sub_ps(_mm256_loadu_ps(x + i),
470                                   _mm256_set1_ps(mean));
471        _mm256_storeu_ps(y + i, val);
472        val = _mm256_mul_ps(val,val);
473        __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
474                                 _mm256_castps256_ps128(val));
475        val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));
476        val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));
477        sum += (ggml_float)_mm_cvtss_f32(val2);
478    }
479#elif defined(__SSE2__)
480    for (; i + 3 < n; i += 4) {
481        __m128 val = _mm_sub_ps(_mm_loadu_ps(x + i),
482                                _mm_set1_ps(mean));
483        _mm_storeu_ps(y + i, val);
484        val = _mm_mul_ps(val, val);
485#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
486        val = _mm_add_ps(val, _mm_movehl_ps(val, val));
487        val = _mm_add_ss(val, _mm_movehdup_ps(val));
488#else
489        __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));
490        val = _mm_add_ps(val, tmp);
491        tmp = _mm_movehl_ps(tmp, val);
492        val = _mm_add_ss(val, tmp);
493#endif  // __AVX__ || __AVX2__ || __AVX512F__
494        sum += (ggml_float)_mm_cvtss_f32(val);
495    }
496#elif defined(__ARM_NEON) && defined(__aarch64__)
497    for (; i + 3 < n; i += 4) {
498        float32x4_t val = vsubq_f32(vld1q_f32(x + i),
499                                    vdupq_n_f32(mean));
500        vst1q_f32(y + i, val);
501        val = vmulq_f32(val, val);
502        sum += (ggml_float)vaddvq_f32(val);
503    }
504#elif defined(__VXE__) || defined(__VXE2__)
505    for (; i + 3 < n; i += 4) {
506        float32x4_t val = vec_sub(vec_xl(0, x + i), vec_splats(mean));
507        vec_xst(val, 0, y + i);
508        val = vec_mul(val, val);
509        sum += (ggml_float)vec_hsum_f32x4(val);
510    }
511#elif defined(__riscv_v_intrinsic)
512    vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
513    for (int vl; i < n; i += vl) {
514        vl = __riscv_vsetvl_e32m2(n - i);
515        vfloat32m2_t val = __riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], vl), mean, vl);
516        __riscv_vse32_v_f32m2(&y[i], val, vl);
517        val = __riscv_vfmul_vv_f32m2(val, val, vl);
518        vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, vl);
519    }
520    sum = (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);
521#endif
522    for (; i < n; ++i) {
523        float val = x[i] - mean;
524        y[i] = val;
525        val *= val;
526        sum += (ggml_float)val;
527    }
528    return sum/n;
529}
530
531ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
532    int i = 0;
533    ggml_float sum = 0;
534#if defined(__AVX512F__) && defined(__AVX512DQ__)
535    for (; i + 15 < n; i += 16) {
536        __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
537                                               _mm512_set1_ps(max)));
538        _mm512_storeu_ps(y + i, val);
539        sum += (ggml_float)_mm512_reduce_add_ps(val);
540    }
541#elif defined(__AVX2__) && defined(__FMA__)
542    for (; i + 7 < n; i += 8) {
543        __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
544                                               _mm256_set1_ps(max)));
545        _mm256_storeu_ps(y + i, val);
546        __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
547                                 _mm256_castps256_ps128(val));
548        val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));
549        val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));
550        sum += (ggml_float)_mm_cvtss_f32(val2);
551    }
552#elif defined(__SSE2__)
553    for (; i + 3 < n; i += 4) {
554        __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),
555                                            _mm_set1_ps(max)));
556        _mm_storeu_ps(y + i, val);
557#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
558        val = _mm_add_ps(val, _mm_movehl_ps(val, val));
559        val = _mm_add_ss(val, _mm_movehdup_ps(val));
560#else
561        __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));
562        val = _mm_add_ps(val, tmp);
563        tmp = _mm_movehl_ps(tmp, val);
564        val = _mm_add_ss(val, tmp);
565#endif
566        sum += (ggml_float)_mm_cvtss_f32(val);
567    }
568#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)
569    const int vlen = svcntw();
570    for (; i < n; i += vlen) {
571        const svbool_t pg = svwhilelt_b32_s32(i, n);
572        svfloat32_t val = ggml_v_expf(pg, svsub_f32_x(pg, svld1_f32(pg, x + i),
573                                                svdup_n_f32_x(pg, max)));
574        svst1_f32(pg, y + i, val);
575        sum += (ggml_float)svaddv_f32(pg, val);
576    }
577#elif defined(__ARM_NEON) && defined(__aarch64__)
578    for (; i + 3 < n; i += 4) {
579        float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),
580                                                vdupq_n_f32(max)));
581        vst1q_f32(y + i, val);
582        sum += (ggml_float)vaddvq_f32(val);
583    }
584#elif defined(__riscv_v_intrinsic)
585    vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
586    for (int avl; i < n; i += avl) {
587        avl = __riscv_vsetvl_e32m2(n - i);
588        vfloat32m2_t val = ggml_v_expf_m2(__riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], avl), max, avl), avl);
589        __riscv_vse32_v_f32m2(&y[i], val, avl);
590        vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, avl);
591    }
592    return (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);
593#endif
594    for (; i < n; ++i) {
595        float val = expf(x[i] - max);
596        sum += (ggml_float)val;
597        y[i] = val;
598    }
599    return sum;
600}
601
602ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {
603    // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)
604
605    int i = 0;
606    ggml_float sum = 0;
607    for (; i < n; ++i) {
608        float val = x[i] - max;
609        y[i] = val;
610        sum += (ggml_float)expf(val);
611    }
612    return sum = (ggml_float)logf(sum);
613}