char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
79.9 KiB1331 linesraw
1#pragma clang diagnostic ignored "-Wunused-variable"
2#pragma clang diagnostic ignored "-Wunused-function"
3#pragma clang diagnostic ignored "-Wunused-but-set-variable"
4
5#include <HAP_farf.h>
6#include <HAP_perf.h>
7
8#include <math.h>
9#include <string.h>
10
11#include "hex-dma.h"
12#include "hex-fastdiv.h"
13#include "hvx-exp.h"
14#include "hvx-sigmoid.h"
15#include "hvx-utils.h"
16#include "unary-ops.h"
17
18#define GGML_COMMON_DECL_C
19#include "ggml-common.h"
20#include "htp-ctx.h"
21#include "htp-ops.h"
22#include "htp-tensor.h"
23#include "htp-vtcm.h"
24#include "hex-profile.h"
25
26struct htp_unary_context {
27    struct htp_ops_context * octx;
28    const struct htp_unary_kernel_params * kparams;
29
30    const uint8_t *           data_src0;
31    const uint8_t *           data_src1;            // weight/scale tensor for RMS_NORM_MUL
32    uint8_t *                 data_dst;
33
34    size_t                    src0_data_row_size;   // actual data bytes per row
35    size_t                    src1_data_row_size;
36    size_t                    dst_data_row_size;    // actual data bytes per row
37
38    size_t                    src0_row_size_aligned;
39    size_t                    src1_row_size_aligned;
40    size_t                    dst_row_size_aligned;
41
42    size_t                    src0_vtcm_half_size;
43    size_t                    src1_vtcm_half_size;
44    size_t                    dst_vtcm_half_size;
45
46    uint32_t                  block;
47    uint32_t                  src0_nrows;
48    uint32_t                  src0_nrows_per_thread;
49    uint32_t                  row_start;
50    uint32_t                  nc;
51    uint32_t                  col_tile;             // tiled mode
52    bool                      broadcast_weight;
53
54    uint8_t *                 vtcm_src0;
55    uint8_t *                 vtcm_src1;
56    uint8_t *                 vtcm_dst;
57
58    size_t                    vtcm_src0_size_per_thread;
59    size_t                    vtcm_src1_size_per_thread;
60    size_t                    vtcm_dst_size_per_thread;
61};
62
63// Convert flat row index to DDR byte offset using the tensor's actual strides.
64// ir = i1 + ne1*(i2 + ne2*i3)  =>  offset = i1*nb1 + i2*nb2 + i3*nb3
65static inline size_t unary_row_offset(uint32_t ir,
66                                      uint32_t ne1, uint32_t ne2,
67                                      const struct fastdiv_values * div_ne1,
68                                      const struct fastdiv_values * div_ne2,
69                                      const struct fastdiv_values * div_ne12,
70                                      size_t nb1, size_t nb2, size_t nb3) {
71    const uint32_t i1 = fastmodulo(ir, ne1, div_ne1);
72    const uint32_t ir_div_ne1 = fastdiv(ir, div_ne1);
73    const uint32_t i2 = fastmodulo(ir_div_ne1, ne2, div_ne2);
74    const uint32_t i3 = fastdiv(ir, div_ne12);
75    return i1 * nb1 + i2 * nb2 + i3 * nb3;
76}
77
78// Safe DMA block size from row `ir`: clamp to the tighter dim-1 slice
79// boundary of src and dst so the nb1 stride stays valid for all rows.
80static inline uint32_t unary_block_size(uint32_t ir,
81                                        uint32_t end_row,
82                                        uint32_t block,
83                                        bool src_contig,
84                                        bool dst_contig,
85                                        uint32_t ne1,
86                                        const struct fastdiv_values * div_ne1) {
87    uint32_t limit = MIN(block, end_row - ir);
88
89    if (!src_contig || !dst_contig) {
90        const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1;
91        limit = MIN(limit, slice_end - ir);
92    }
93
94    return limit;
95}
96
97#define htp_unary_preamble            \
98    const uint32_t ne00 = src->ne[0]; \
99    const uint32_t ne01 = src->ne[1]; \
100    const uint32_t ne02 = src->ne[2]; \
101    const uint32_t ne03 = src->ne[3]; \
102                                      \
103    const uint32_t ne0 = dst->ne[0];  \
104    const uint32_t ne1 = dst->ne[1];  \
105    const uint32_t ne2 = dst->ne[2];  \
106    const uint32_t ne3 = dst->ne[3];  \
107                                      \
108    const uint32_t nb00 = src->nb[0]; \
109    const uint32_t nb01 = src->nb[1]; \
110    const uint32_t nb02 = src->nb[2]; \
111    const uint32_t nb03 = src->nb[3]; \
112                                      \
113    const uint32_t nb0 = dst->nb[0];  \
114    const uint32_t nb1 = dst->nb[1];  \
115    const uint32_t nb2 = dst->nb[2];  \
116    const uint32_t nb3 = dst->nb[3];
117
118#define htp_unary_op_preamble                                         \
119    int32_t * op_params = uctx->octx->op_params;                      \
120    const uint32_t ne0 = uctx->nc;                                    \
121    const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \
122    const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;
123
124static void scale_f32(const float * restrict src,
125                      float * restrict dst,
126                      const uint32_t num_rows,
127                      const struct htp_unary_context * uctx) {
128    htp_unary_op_preamble;
129    float scale = 0.f;
130    float bias  = 0.f;
131    memcpy(&scale, &op_params[0], sizeof(float));
132    memcpy(&bias,  &op_params[1], sizeof(float));
133
134    for (uint32_t ir = 0; ir < num_rows; ir++) {
135        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
136        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
137
138        hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias);
139    }
140}
141
142static void clamp_f32(const float * restrict src,
143                      float * restrict dst,
144                      const uint32_t num_rows,
145                      const struct htp_unary_context * uctx) {
146    htp_unary_op_preamble;
147    float min = 0.f;
148    float max = 0.f;
149    memcpy(&min, &op_params[0], sizeof(float));
150    memcpy(&max, &op_params[1], sizeof(float));
151
152    for (uint32_t ir = 0; ir < num_rows; ir++) {
153        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
154        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
155
156        hvx_clamp_scalar_f32(dst_local, src_local, min, max, ne0);
157    }
158}
159
160static void leaky_relu_f32(const float * restrict src,
161                           float * restrict dst,
162                           const uint32_t num_rows,
163                           const struct htp_unary_context * uctx) {
164    htp_unary_op_preamble;
165    float negative_slope = 0.f;
166    memcpy(&negative_slope, &op_params[0], sizeof(float));
167
168    for (uint32_t ir = 0; ir < num_rows; ir++) {
169        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
170        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
171
172        hvx_leaky_relu_scalar_f32(dst_local, src_local, negative_slope, ne0);
173    }
174}
175
176static void rms_norm_f32(const float * restrict src,
177                         float * restrict dst,
178                         const uint32_t num_rows,
179                         const struct htp_unary_context * uctx) {
180    htp_unary_op_preamble;
181    float epsilon = 0.f;
182    memcpy(&epsilon, op_params, sizeof(float));
183
184    for (uint32_t ir = 0; ir < num_rows; ir++) {
185        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
186        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
187
188        hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
189    }
190}
191
192static void rms_norm_mul_f32(const float * restrict src,
193                             const float * restrict weight,
194                             float * restrict dst,
195                             const uint32_t num_rows,
196                             const struct htp_unary_context * uctx) {
197    htp_unary_op_preamble;
198    float epsilon = 0.f;
199    memcpy(&epsilon, op_params, sizeof(float));
200
201    for (uint32_t ir = 0; ir < num_rows; ir++) {
202        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
203        const uint8_t * restrict w_local   = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned);
204        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
205
206        hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon);
207    }
208}
209
210static void norm_f32(const float * restrict src,
211                     float * restrict dst,
212                     const uint32_t num_rows,
213                     const struct htp_unary_context * uctx) {
214    htp_unary_op_preamble;
215    float epsilon = 0.f;
216    memcpy(&epsilon, op_params, sizeof(float));
217
218    for (uint32_t ir = 0; ir < num_rows; ir++) {
219        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
220        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
221
222        hvx_fast_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
223    }
224}
225
226static void sqr_f32(const float * restrict src,
227                    float * restrict dst,
228                    const uint32_t num_rows,
229                    const struct htp_unary_context * uctx) {
230    htp_unary_op_preamble;
231
232    for (uint32_t ir = 0; ir < num_rows; ir++) {
233        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
234        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
235
236        hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
237    }
238}
239
240static void sqrt_f32(const float * restrict src,
241                     float * restrict dst,
242                     const uint32_t num_rows,
243                     const struct htp_unary_context * uctx) {
244    htp_unary_op_preamble;
245
246    for (uint32_t ir = 0; ir < num_rows; ir++) {
247        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
248        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
249
250        hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
251    }
252}
253
254static void scale_f16(const _Float16 * restrict src,
255                      _Float16 * restrict dst,
256                      const uint32_t num_rows,
257                      const struct htp_unary_context * uctx) {
258    htp_unary_op_preamble;
259    float scale = 0.f;
260    float bias  = 0.f;
261    memcpy(&scale, &op_params[0], sizeof(float));
262    memcpy(&bias,  &op_params[1], sizeof(float));
263
264    for (uint32_t ir = 0; ir < num_rows; ir++) {
265        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
266        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
267
268        hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias);
269    }
270}
271
272static void clamp_f16(const _Float16 * restrict src,
273                      _Float16 * restrict dst,
274                      const uint32_t num_rows,
275                      const struct htp_unary_context * uctx) {
276    htp_unary_op_preamble;
277    float min = 0.f;
278    float max = 0.f;
279    memcpy(&min, &op_params[0], sizeof(float));
280    memcpy(&max, &op_params[1], sizeof(float));
281
282    for (uint32_t ir = 0; ir < num_rows; ir++) {
283        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
284        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
285
286        hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0);
287    }
288}
289
290static void rms_norm_f16(const _Float16 * restrict src,
291                         _Float16 * restrict dst,
292                         const uint32_t num_rows,
293                         const struct htp_unary_context * uctx) {
294    htp_unary_op_preamble;
295    float epsilon = 0.f;
296    memcpy(&epsilon, op_params, sizeof(float));
297
298    for (uint32_t ir = 0; ir < num_rows; ir++) {
299        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
300        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
301
302        hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
303    }
304}
305
306static void norm_f16(const _Float16 * restrict src,
307                     _Float16 * restrict dst,
308                     const uint32_t num_rows,
309                     const struct htp_unary_context * uctx) {
310    htp_unary_op_preamble;
311    float epsilon = 0.f;
312    memcpy(&epsilon, op_params, sizeof(float));
313
314    for (uint32_t ir = 0; ir < num_rows; ir++) {
315        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
316        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
317
318        hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
319    }
320}
321
322static void sqr_f16(const _Float16 * restrict src,
323                    _Float16 * restrict dst,
324                    const uint32_t num_rows,
325                    const struct htp_unary_context * uctx) {
326    htp_unary_op_preamble;
327
328    for (uint32_t ir = 0; ir < num_rows; ir++) {
329        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
330        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
331
332        hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
333    }
334}
335
336static void sqrt_f16(const _Float16 * restrict src,
337                     _Float16 * restrict dst,
338                     const uint32_t num_rows,
339                     const struct htp_unary_context * uctx) {
340    htp_unary_op_preamble;
341
342    for (uint32_t ir = 0; ir < num_rows; ir++) {
343        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
344        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
345
346        hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
347    }
348}
349
350static void abs_f16(const _Float16 * restrict src,
351                    _Float16 * restrict dst,
352                    const uint32_t num_rows,
353                    const struct htp_unary_context * uctx) {
354    htp_unary_op_preamble;
355
356    for (uint32_t ir = 0; ir < num_rows; ir++) {
357        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
358        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
359
360        hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
361    }
362}
363
364static void log_f16(const _Float16 * restrict src,
365                    _Float16 * restrict dst,
366                    const uint32_t num_rows,
367                    const struct htp_unary_context * uctx) {
368    htp_unary_op_preamble;
369
370    for (uint32_t ir = 0; ir < num_rows; ir++) {
371        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
372        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
373
374        hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
375    }
376}
377
378static void l2_norm_f16(const _Float16 * restrict src,
379                        _Float16 * restrict dst,
380                        const uint32_t num_rows,
381                        const struct htp_unary_context * uctx) {
382    htp_unary_op_preamble;
383    float epsilon = 0.f;
384    memcpy(&epsilon, op_params, sizeof(float));
385
386    for (uint32_t ir = 0; ir < num_rows; ir++) {
387        const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned);
388        uint8_t * restrict dst_f       = (uint8_t *)dst + (ir * dst_row_size_aligned);
389
390        hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon);
391    }
392}
393
394static void neg_f32(const float * restrict src,
395                    float * restrict dst,
396                    const uint32_t num_rows,
397                    const struct htp_unary_context * uctx) {
398    htp_unary_op_preamble;
399
400    for (uint32_t ir = 0; ir < num_rows; ir++) {
401        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
402        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
403
404        hvx_scale_f32_aa(dst_local, src_local, ne0, -1.0f);
405    }
406}
407
408static void exp_f32(const float * restrict src,
409                    float * restrict dst,
410                    const uint32_t num_rows,
411                    const struct htp_unary_context * uctx) {
412    htp_unary_op_preamble;
413
414    for (uint32_t ir = 0; ir < num_rows; ir++) {
415        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
416        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
417
418        hvx_exp_f32(dst_local, src_local, ne0, false);
419    }
420}
421
422static void sigmoid_f32(const float * restrict src,
423                        float * restrict dst,
424                        const uint32_t num_rows,
425                        const struct htp_unary_context * uctx) {
426    htp_unary_op_preamble;
427
428    for (uint32_t ir = 0; ir < num_rows; ir++) {
429        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
430        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
431
432        hvx_sigmoid_f32_aa(dst_local, src_local, ne0);
433    }
434}
435
436// silu(x) = x * sigmoid(x)
437static void silu_f32(const float * restrict src,
438                     float * restrict dst,
439                     const uint32_t num_rows,
440                     const struct htp_unary_context * uctx) {
441    htp_unary_op_preamble;
442
443    for (uint32_t ir = 0; ir < num_rows; ir++) {
444        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
445        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
446
447        hvx_sigmoid_f32_aa(dst_local, src_local, ne0);
448        hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);
449    }
450}
451
452// gelu(x) = x * sigmoid(1.702 * x)  (quick/sigmoid approximation, matches CPU GELU_QUICK reference)
453static void gelu_f32(const float * restrict src,
454                     float * restrict dst,
455                     const uint32_t num_rows,
456                     const struct htp_unary_context * uctx) {
457    htp_unary_op_preamble;
458
459    for (uint32_t ir = 0; ir < num_rows; ir++) {
460        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
461        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
462
463        hvx_mul_scalar_f32(dst_local, src_local, 1.702f, ne0);
464        hvx_sigmoid_f32_aa(dst_local, dst_local, ne0);
465        hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);
466    }
467}
468
469static void tri_f32(const float * restrict src,
470                    float * restrict dst,
471                    const uint32_t num_rows,
472                    const uint32_t ir,
473                    const struct htp_unary_context * uctx) {
474    htp_unary_op_preamble;
475    const int32_t ttype = op_params[0];
476    const HVX_Vector zero = hvx_vec_splat_f32(0.0f);
477    const uint32_t nvec  = ne0 / VLEN_FP32;
478    const uint32_t nloe  = ne0 % VLEN_FP32;
479
480    const uint32_t ne01 = uctx->octx->src[0]->ne[1];
481
482    for (uint32_t b = 0; b < num_rows; b++) {
483        const uint32_t abs_row = ir + b;
484        const uint32_t i01     = abs_row % ne01;
485
486        const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned);
487        HVX_Vector * restrict v_dst       = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned);
488
489        uint32_t boundary;
490        int      keep_left;
491        switch (ttype) {
492            case 0: boundary = i01;     keep_left = 0; break;  // keep col >= row
493            case 1: boundary = i01 + 1; keep_left = 0; break;  // keep col > row
494            case 2: boundary = i01 + 1; keep_left = 1; break;  // keep col <= row
495            case 3: boundary = i01;     keep_left = 1; break;  // keep col < row
496            default: boundary = 0; keep_left = 0; break;
497        }
498        if (boundary > ne0) boundary = ne0;
499
500        // Full HVX vectors - each starts at a 128-byte aligned offset
501        for (uint32_t i = 0; i < nvec; i++) {
502            const uint32_t vec_start = i * VLEN_FP32;
503            const uint32_t vec_end   = vec_start + VLEN_FP32;
504            if (keep_left) {
505                if (vec_end <= boundary) {
506                    v_dst[i] = v_src[i];
507                } else if (vec_start >= boundary) {
508                    v_dst[i] = zero;
509                } else {
510                    HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float));
511                    v_dst[i]            = Q6_V_vmux_QVV(mask, v_src[i], zero);
512                }
513            } else {
514                if (vec_end <= boundary) {
515                    v_dst[i] = zero;
516                } else if (vec_start >= boundary) {
517                    v_dst[i] = v_src[i];
518                } else {
519                    HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float));
520                    v_dst[i]            = Q6_V_vmux_QVV(mask, zero, v_src[i]);
521                }
522            }
523        }
524
525        // Tail elements (row_elems not a multiple of VLEN_FP32)
526        if (nloe > 0) {
527            const uint32_t abs_start = nvec * VLEN_FP32;
528            const uint32_t abs_end   = abs_start + nloe;
529            HVX_Vector     tail_val;
530            if (keep_left) {
531                if (abs_end <= boundary) {
532                    tail_val = v_src[nvec];
533                } else if (abs_start >= boundary) {
534                    tail_val = zero;
535                } else {
536                    HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
537                    tail_val            = Q6_V_vmux_QVV(mask, v_src[nvec], zero);
538                }
539            } else {
540                if (abs_end <= boundary) {
541                    tail_val = zero;
542                } else if (abs_start >= boundary) {
543                    tail_val = v_src[nvec];
544                } else {
545                    HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
546                    tail_val            = Q6_V_vmux_QVV(mask, zero, v_src[nvec]);
547                }
548            }
549            hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), tail_val);
550        }
551    }
552}
553
554static void softplus_f32(const float * restrict src,
555                         float * restrict dst,
556                         const uint32_t num_rows,
557                         const struct htp_unary_context * uctx) {
558    htp_unary_op_preamble;
559    // softplus(x) = log(1 + exp(x))
560    // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h
561    for (uint32_t ir = 0; ir < num_rows; ir++) {
562        const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));
563        float * restrict dst_f       = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));
564
565        for (uint32_t i = 0; i < ne0; i++) {
566            float x = src_f[i];
567            // For x > 20: softplus(x) ~ x (avoids exp overflow)
568            dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
569        }
570    }
571}
572
573static void l2_norm_f32(const float * restrict src,
574                        float * restrict dst,
575                        const uint32_t num_rows,
576                        const struct htp_unary_context * uctx) {
577    htp_unary_op_preamble;
578    float epsilon = 0.f;
579    memcpy(&epsilon, op_params, sizeof(float));
580
581    for (uint32_t ir = 0; ir < num_rows; ir++) {
582        const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));
583        float * restrict dst_f       = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));
584
585        hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon);
586    }
587}
588
589static void tanh_f32(const float * restrict src,
590                     float * restrict dst,
591                     const uint32_t num_rows,
592                     const struct htp_unary_context * uctx) {
593    htp_unary_op_preamble;
594
595    for (uint32_t ir = 0; ir < num_rows; ir++) {
596        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
597        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
598
599        hvx_tanh_f32_aa(dst_local, src_local, ne0);
600    }
601}
602
603static void abs_f32(const float * restrict src,
604                    float * restrict dst,
605                    const uint32_t num_rows,
606                    const struct htp_unary_context * uctx) {
607    htp_unary_op_preamble;
608
609    for (uint32_t ir = 0; ir < num_rows; ir++) {
610        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
611        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
612
613        hvx_abs_f32_aa(dst_local, src_local, ne0);
614    }
615}
616
617static void relu_f32(const float * restrict src,
618                     float * restrict dst,
619                     const uint32_t num_rows,
620                     const struct htp_unary_context * uctx) {
621    htp_unary_op_preamble;
622
623    for (uint32_t ir = 0; ir < num_rows; ir++) {
624        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
625        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
626
627        hvx_max_scalar_f32(dst_local, src_local, 0.0f, ne0);
628    }
629}
630
631static void log_f32(const float * restrict src,
632                    float * restrict dst,
633                    const uint32_t num_rows,
634                    const struct htp_unary_context * uctx) {
635    htp_unary_op_preamble;
636
637    for (uint32_t ir = 0; ir < num_rows; ir++) {
638        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
639        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
640
641        hvx_log_f32_aa(dst_local, src_local, ne0);
642    }
643}
644
645#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR)                              \
646static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) {                         \
647    const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;                                \
648    struct htp_ops_context * octx = uctx->octx;                                                                     \
649    const struct htp_tensor * src = octx->src[0];                                                                   \
650    const struct htp_tensor * dst = octx->dst;                                                                      \
651    struct htp_thread_trace * tr = &octx->ctx->trace[ith];                                                          \
652                                                                                                                    \
653    htp_unary_preamble;                                                                                             \
654                                                                                                                    \
655    int32_t *    op_params = octx->op_params;                                                                       \
656    uint32_t     src0_nrows_per_thread = uctx->src0_nrows_per_thread;                                               \
657                                                                                                                    \
658    const size_t src0_data_row_size = uctx->src0_data_row_size;                                                     \
659    const size_t dst_data_row_size  = uctx->dst_data_row_size;                                                      \
660                                                                                                                    \
661    const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;                                               \
662    const size_t dst_row_size_aligned  = uctx->dst_row_size_aligned;                                                \
663                                                                                                                    \
664    const uint32_t src0_nrows = uctx->src0_nrows;                                                                   \
665    const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;                                  \
666    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);      \
667                                                                                                                    \
668    if (src0_start_row >= src0_end_row) {                                                                           \
669        return;                                                                                                     \
670    }                                                                                                               \
671                                                                                                                    \
672    const uint8_t * restrict data_src = uctx->data_src0;                                                            \
673    const uint8_t * restrict data_src1 = uctx->data_src1;                                                           \
674    uint8_t * restrict       data_dst = uctx->data_dst;                                                             \
675                                                                                                                    \
676    const struct htp_tensor * src1 = (IS_RMS_NORM_MUL) ? octx->src[1] : NULL;                                       \
677    const uint32_t nb11 = src1 ? src1->nb[1] : 0;                                                                   \
678    const uint32_t nb12 = src1 ? src1->nb[2] : 0;                                                                   \
679    const uint32_t nb13 = src1 ? src1->nb[3] : 0;                                                                   \
680    const uint32_t nb11_bc = (src1 && src1->ne[1] > 1) ? nb11 : 0;                                                  \
681    const uint32_t nb12_bc = (src1 && src1->ne[2] > 1) ? nb12 : 0;                                                  \
682    const uint32_t nb13_bc = (src1 && src1->ne[3] > 1) ? nb13 : 0;                                                  \
683    const bool src1_contig = src1 ? ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)) : false;       \
684                                                                                                                    \
685    uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);                           \
686    uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;\
687    uint8_t * dst_vtcm_data  = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);                             \
688                                                                                                                    \
689    size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;                                                         \
690    size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size;                                                         \
691    size_t dst_vtcm_half_size  = uctx->dst_vtcm_half_size;                                                          \
692                                                                                                                    \
693    const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&                                                       \
694                             (nb03 == (size_t)ne02 * nb02);                                                         \
695    const bool dst_contig  = (nb2  == (size_t)ne1  * nb1)  &&                                                       \
696                             (nb3  == (size_t)ne2  * nb2);                                                          \
697                                                                                                                    \
698    const struct fastdiv_values * div_ne01  = &uctx->kparams->div_ne01;                                             \
699    const struct fastdiv_values * div_ne02  = &uctx->kparams->div_ne02;                                             \
700    const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;                                            \
701                                                                                                                    \
702    const bool src1_needs_row_clip = (IS_RMS_NORM_MUL) && !uctx->broadcast_weight && !src1_contig;                  \
703    const bool block_src0_contig = src0_contig && !src1_needs_row_clip;                                             \
704    const bool block_dst_contig  = dst_contig  && !src1_needs_row_clip;                                             \
705                                                                                                                    \
706    const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);             \
707    const uint32_t dst_max_block  = block_dst_contig  ? uctx->block : MIN((uint32_t)uctx->block, ne1);              \
708    const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);                                                      \
709    if (BLOCK == 0) {                                                                                               \
710        FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n",         \
711             uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);                                               \
712        return;                                                                                                     \
713    }                                                                                                               \
714                                                                                                                    \
715    dma_queue * dma_queue = octx->ctx->dma[ith];                                                                    \
716                                                                                                                    \
717    if ((IS_RMS_NORM_MUL) && uctx->broadcast_weight) {                                                              \
718        dma_queue_push(dma_queue, dma_make_ptr(src1_vtcm_data, data_src1),                                          \
719                       uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1);                                \
720        dma_queue_flush(dma_queue);                                                                                 \
721    }                                                                                                               \
722                                                                                                                    \
723    for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {               \
724        const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,  \
725                                                     ne01, div_ne01);                                               \
726                                                                                                                    \
727        dma_queue_push(dma_queue,                                                                                   \
728            dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),                                \
729            nb1, dst_row_size_aligned, dst_data_row_size, 0);                                                       \
730                                                                                                                    \
731        const size_t src0_off = src0_contig ? (ir * nb01) :                                                         \
732            unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);                      \
733        dma_queue_push(dma_queue,                                                                                   \
734            dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),                   \
735            src0_row_size_aligned, nb01, src0_data_row_size, block_size);                                           \
736                                                                                                                    \
737        if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) {                                                         \
738            const size_t src1_off = src1_contig ? (ir * nb11) :                                                     \
739                unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc);         \
740            dma_queue_push(dma_queue,                                                                               \
741                dma_make_ptr(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off),              \
742                uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size);                           \
743        }                                                                                                           \
744                                                                                                                    \
745        ir += block_size;                                                                                           \
746    }                                                                                                               \
747                                                                                                                    \
748    for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {                                                       \
749        const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,  \
750                                                     ne01, div_ne01);                                               \
751                                                                                                                    \
752        TYPE * dst_vtcm  = (TYPE *) dma_queue_pop(dma_queue).src;                                                   \
753        TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst;                                                   \
754        TYPE * src1_vtcm = NULL;                                                                                    \
755        if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) {                                                         \
756            src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst;                                                      \
757        }                                                                                                           \
758                                                                                                                    \
759        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir);                                                      \
760        CORE_EXPR;                                                                                                  \
761        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);                                                       \
762                                                                                                                    \
763        const size_t dst_off = dst_contig ? (ir * nb1) :                                                            \
764            unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);                           \
765        dma_queue_push(dma_queue,                                                                                   \
766            dma_make_ptr(data_dst + dst_off, dst_vtcm),                                                             \
767            nb1, dst_row_size_aligned, dst_data_row_size, block_size);                                              \
768                                                                                                                    \
769        const uint32_t next_ir = ir + block_size;                                                                   \
770        if (next_ir < src0_end_row) {                                                                               \
771            const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig,      \
772                                                              block_dst_contig, ne01, div_ne01);                    \
773            const uint32_t pref_ir = next_ir + next_block_size;                                                     \
774            if (pref_ir < src0_end_row) {                                                                           \
775                const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig,  \
776                                                                  block_dst_contig, ne01, div_ne01);                \
777                const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :                                       \
778                    unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);         \
779                dma_queue_push(dma_queue,                                                                           \
780                    dma_make_ptr(src0_vtcm, data_src + src0_pref_off),                                              \
781                    src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);                              \
782                                                                                                                    \
783                if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) {                                                 \
784                    const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) :                                   \
785                        unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc,      \
786                                          nb13_bc);                                                                 \
787                    dma_queue_push(dma_queue,                                                                       \
788                        dma_make_ptr(src1_vtcm, data_src1 + src1_pref_off),                                         \
789                        uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size);              \
790                }                                                                                                   \
791            }                                                                                                       \
792        }                                                                                                           \
793        ir += block_size;                                                                                           \
794    }                                                                                                               \
795                                                                                                                    \
796    dma_queue_flush(dma_queue);                                                                                     \
797}
798
799// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers.
800#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
801    DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR)
802
803DEFINE_UNARY_TASK(norm,           false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
804DEFINE_UNARY_TASK(rms_norm,       false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
805DEFINE_UNARY_TASK(rms_norm_mul,   true,  false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx))
806DEFINE_UNARY_TASK(scale,          false, false, scale_f32(src0_vtcm, dst_vtcm, block_size, uctx))
807DEFINE_UNARY_TASK(clamp,          false, false, clamp_f32(src0_vtcm, dst_vtcm, block_size, uctx))
808DEFINE_UNARY_TASK(leaky_relu,     false, false, leaky_relu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
809DEFINE_UNARY_TASK(sqr,            false, false, sqr_f32(src0_vtcm, dst_vtcm, block_size, uctx))
810DEFINE_UNARY_TASK(sqrt,           false, false, sqrt_f32(src0_vtcm, dst_vtcm, block_size, uctx))
811DEFINE_UNARY_TASK(unary_neg,      false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx))
812DEFINE_UNARY_TASK(unary_exp,      false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx))
813DEFINE_UNARY_TASK(unary_sigmoid,  false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx))
814DEFINE_UNARY_TASK(unary_silu,     false, false, silu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
815DEFINE_UNARY_TASK(unary_gelu,     false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
816DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))
817DEFINE_UNARY_TASK(unary_tanh,     false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))
818DEFINE_UNARY_TASK(unary_abs,      false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx))
819DEFINE_UNARY_TASK(unary_log,      false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx))
820DEFINE_UNARY_TASK(unary_relu,     false, false, relu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
821DEFINE_UNARY_TASK(l2_norm,        false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
822DEFINE_UNARY_TASK(tri,            false, true,  tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
823
824// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are
825// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support.
826DEFINE_UNARY_TASK_IMPL(norm,      _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
827DEFINE_UNARY_TASK_IMPL(rms_norm,  _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
828DEFINE_UNARY_TASK_IMPL(scale,     _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx))
829DEFINE_UNARY_TASK_IMPL(clamp,     _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx))
830DEFINE_UNARY_TASK_IMPL(sqr,       _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx))
831DEFINE_UNARY_TASK_IMPL(sqrt,      _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx))
832DEFINE_UNARY_TASK_IMPL(l2_norm,   _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
833DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx))
834DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx))
835
836// Apply a pointwise unary op to one column tile that is already in VTCM.
837#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR)                                                         \
838static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) {                            \
839    const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;                                  \
840    struct htp_ops_context * octx = uctx->octx;                                                                       \
841    const struct htp_tensor * src = octx->src[0];                                                                     \
842    const struct htp_tensor * dst = octx->dst;                                                                        \
843    struct htp_thread_trace * tr = &octx->ctx->trace[ith];                                                            \
844                                                                                                                      \
845    htp_unary_preamble;                                                                                               \
846                                                                                                                      \
847    uint32_t     src0_nrows_per_thread = uctx->src0_nrows_per_thread;                                                 \
848                                                                                                                      \
849    int32_t *      op_params = octx->op_params;                                                                       \
850    const uint32_t col_tile  = uctx->col_tile;                                                                        \
851                                                                                                                      \
852    const uint32_t src0_nrows = uctx->src0_nrows;                                                                     \
853    const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith;                                    \
854    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows);        \
855                                                                                                                      \
856    if (src0_start_row >= src0_end_row) {                                                                             \
857        return;                                                                                                       \
858    }                                                                                                                 \
859                                                                                                                      \
860    const uint8_t * restrict data_src = uctx->data_src0;                                                              \
861    uint8_t * restrict       data_dst = uctx->data_dst;                                                               \
862                                                                                                                      \
863    uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);                             \
864    uint8_t * dst_vtcm_data  = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread);                               \
865                                                                                                                      \
866    const size_t src0_half = uctx->src0_vtcm_half_size;                                                               \
867    const size_t dst_half  = uctx->dst_vtcm_half_size;                                                                \
868                                                                                                                      \
869    dma_queue * dmaq = octx->ctx->dma[ith];                                                                           \
870                                                                                                                      \
871    const struct fastdiv_values * div_ne01  = &uctx->kparams->div_ne01;                                               \
872    const struct fastdiv_values * div_ne02  = &uctx->kparams->div_ne02;                                               \
873    const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;                                              \
874    const struct fastdiv_values * div_tpr   = &uctx->kparams->div_tpr;                                                \
875                                                                                                                      \
876    const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile;                                                   \
877    const int32_t  tri_ttype     = (IS_TRI) ? op_params[0] : 0;                                                       \
878                                                                                                                      \
879    const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&                                                         \
880                             (nb03 == (size_t)ne02 * nb02);                                                           \
881    const bool dst_contig  = (nb2  == (size_t)ne1  * nb1)  &&                                                         \
882                             (nb3  == (size_t)ne2  * nb2);                                                            \
883                                                                                                                      \
884    const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row;                                     \
885                                                                                                                      \
886    for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) {                            \
887        const uint32_t row  = src0_start_row + t / tiles_per_row;                                                     \
888        const uint32_t col  = (t % tiles_per_row) * col_tile;                                                         \
889        const uint32_t tw   = MIN(col_tile, ne0 - col);                                                               \
890        const size_t   tb   = (size_t) tw * sizeof(float);                                                            \
891        const size_t   soff = (src0_contig ? (row * nb01) :                                                           \
892                               unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +  \
893                               (size_t) col * sizeof(float);                                                          \
894                                                                                                                      \
895        dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0);              \
896        dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);  \
897    }                                                                                                                 \
898                                                                                                                      \
899    uint32_t row = src0_start_row;                                                                                    \
900    uint32_t col = 0;                                                                                                 \
901    uint32_t tile_in_row = 0;                                                                                         \
902    uint32_t i01 = fastmodulo(row, ne01, div_ne01);                                                                   \
903                                                                                                                      \
904    uint32_t prow = src0_start_row + fastdiv(2, div_tpr);                                                             \
905    uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile;                                                 \
906    uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr);                                                    \
907                                                                                                                      \
908    for (uint32_t t = 0; t < total_tiles; t++) {                                                                      \
909        uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src;                                                     \
910        uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst;                                                     \
911                                                                                                                      \
912        const uint32_t tw  = MIN(col_tile, ne0 - col);                                                                \
913                                                                                                                      \
914        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t);                                                         \
915        CORE_TILE_EXPR;                                                                                               \
916        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t);                                                          \
917                                                                                                                      \
918        const size_t doff = (dst_contig ? (row * nb1) :                                                               \
919                             unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) +         \
920                             (size_t) col * sizeof(float);                                                            \
921        const size_t tb   = (size_t) tw * sizeof(float);                                                              \
922        dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1);                                 \
923                                                                                                                      \
924        const uint32_t pt = t + 2;                                                                                    \
925        if (pt < total_tiles) {                                                                                       \
926            const uint32_t ptw  = MIN(col_tile, ne0 - pcol);                                                          \
927            const size_t   ptb  = (size_t) ptw * sizeof(float);                                                       \
928            const size_t   psoff = (src0_contig ? (prow * nb01) :                                                     \
929                                    unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02,     \
930                                                     nb03)) +                                                         \
931                                   (size_t) pcol * sizeof(float);                                                     \
932            dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1);                         \
933        }                                                                                                             \
934                                                                                                                      \
935        tile_in_row++;                                                                                                \
936        col += col_tile;                                                                                              \
937        if (tile_in_row == tiles_per_row) {                                                                           \
938            tile_in_row = 0;                                                                                          \
939            col = 0;                                                                                                  \
940            row++;                                                                                                    \
941            i01++;                                                                                                    \
942            if (i01 == ne01) {                                                                                        \
943                i01 = 0;                                                                                              \
944            }                                                                                                         \
945        }                                                                                                             \
946                                                                                                                      \
947        ptile_in_row++;                                                                                               \
948        pcol += col_tile;                                                                                             \
949        if (ptile_in_row == tiles_per_row) {                                                                          \
950            ptile_in_row = 0;                                                                                         \
951            pcol = 0;                                                                                                 \
952            prow++;                                                                                                   \
953        }                                                                                                             \
954    }                                                                                                                 \
955                                                                                                                      \
956    dma_queue_flush(dmaq);                                                                                            \
957}
958
959static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) {
960    float scale = 0.f;
961    float bias = 0.f;
962    memcpy(&scale, &op_params[0], sizeof(float));
963    memcpy(&bias,  &op_params[1], sizeof(float));
964    hvx_scale_offset_f32_aa(dst_vtcm, src_vtcm, tw, scale, bias);
965}
966
967static inline void tile_clamp_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) {
968    float min = 0.f;
969    float max = 0.f;
970    memcpy(&min, &op_params[0], sizeof(float));
971    memcpy(&max, &op_params[1], sizeof(float));
972    hvx_clamp_scalar_f32(dst_vtcm, src_vtcm, min, max, tw);
973}
974
975static inline void tile_leaky_relu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) {
976    float negative_slope = 0.f;
977    memcpy(&negative_slope, &op_params[0], sizeof(float));
978    hvx_leaky_relu_scalar_f32(dst_vtcm, src_vtcm, negative_slope, tw);
979}
980
981static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {
982    const float * restrict sf = (const float *) src_vtcm;
983    float * restrict df       = (float *) dst_vtcm;
984    for (uint32_t i = 0; i < tw; i++) {
985        float x = sf[i];
986        df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
987    }
988}
989
990// silu(x) = x * sigmoid(x)
991static inline void tile_silu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {
992    hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw);
993    hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);
994}
995
996// gelu(x) = x * sigmoid(1.702 * x)  (quick/sigmoid approximation, matches CPU GELU_QUICK reference)
997static inline void tile_gelu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {
998    hvx_mul_scalar_f32(dst_vtcm, src_vtcm, 1.702f, tw);
999    hvx_sigmoid_f32_aa(dst_vtcm, dst_vtcm, tw);
1000    hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);
1001}
1002
1003// Triangular mask applied to one column tile. Boundary is an absolute column index, so
1004// each vector compares against its absolute column position (col_start + i*VLEN_FP32).
1005static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst,
1006                                      uint32_t tile_elems, uint32_t col_start, uint32_t i01,
1007                                      uint32_t ne0, int32_t ttype) {
1008    const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
1009    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
1010    const HVX_Vector zero = hvx_vec_splat_f32(0.0f);
1011
1012    uint32_t boundary;
1013    int      keep_left;
1014    switch (ttype) {
1015        case 0: boundary = i01;     keep_left = 0; break;
1016        case 1: boundary = i01 + 1; keep_left = 0; break;
1017        case 2: boundary = i01 + 1; keep_left = 1; break;
1018        case 3: boundary = i01;     keep_left = 1; break;
1019        default: boundary = 0; keep_left = 0; break;
1020    }
1021    if (boundary > ne0) boundary = ne0;
1022
1023    const uint32_t nvec = tile_elems / VLEN_FP32;
1024    const uint32_t nloe = tile_elems % VLEN_FP32;
1025
1026    for (uint32_t i = 0; i < nvec; i++) {
1027        const uint32_t abs_start = col_start + i * VLEN_FP32;
1028        const uint32_t abs_end   = abs_start + VLEN_FP32;
1029        if (keep_left) {
1030            if (abs_end <= boundary) {
1031                v_dst[i] = v_src[i];
1032            } else if (abs_start >= boundary) {
1033                v_dst[i] = zero;
1034            } else {
1035                HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
1036                v_dst[i]            = Q6_V_vmux_QVV(mask, v_src[i], zero);
1037            }
1038        } else {
1039            if (abs_end <= boundary) {
1040                v_dst[i] = zero;
1041            } else if (abs_start >= boundary) {
1042                v_dst[i] = v_src[i];
1043            } else {
1044                HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
1045                v_dst[i]            = Q6_V_vmux_QVV(mask, zero, v_src[i]);
1046            }
1047        }
1048    }
1049
1050    if (nloe > 0) {
1051        const uint32_t abs_start = col_start + nvec * VLEN_FP32;
1052        const uint32_t abs_end   = abs_start + nloe;
1053        HVX_Vector     tail_val;
1054        if (keep_left) {
1055            if (abs_end <= boundary) {
1056                tail_val = v_src[nvec];
1057            } else if (abs_start >= boundary) {
1058                tail_val = zero;
1059            } else {
1060                HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
1061                tail_val            = Q6_V_vmux_QVV(mask, v_src[nvec], zero);
1062            }
1063        } else {
1064            if (abs_end <= boundary) {
1065                tail_val = zero;
1066            } else if (abs_start >= boundary) {
1067                tail_val = v_src[nvec];
1068            } else {
1069                HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));
1070                tail_val            = Q6_V_vmux_QVV(mask, zero, v_src[nvec]);
1071            }
1072        }
1073        hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), tail_val);
1074    }
1075}
1076
1077DEFINE_UNARY_TILED_TASK(scale,          false, tile_scale_f32(dst_vtcm, src_vtcm, tw, op_params))
1078DEFINE_UNARY_TILED_TASK(clamp,          false, tile_clamp_f32(dst_vtcm, src_vtcm, tw, op_params))
1079DEFINE_UNARY_TILED_TASK(leaky_relu,     false, tile_leaky_relu_f32(dst_vtcm, src_vtcm, tw, op_params))
1080DEFINE_UNARY_TILED_TASK(sqr,            false, hvx_sqr_f32_aa(dst_vtcm, src_vtcm, tw))
1081DEFINE_UNARY_TILED_TASK(sqrt,           false, hvx_sqrt_f32_aa(dst_vtcm, src_vtcm, tw))
1082DEFINE_UNARY_TILED_TASK(unary_neg,      false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f))
1083DEFINE_UNARY_TILED_TASK(unary_exp,      false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false))
1084DEFINE_UNARY_TILED_TASK(unary_sigmoid,  false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw))
1085DEFINE_UNARY_TILED_TASK(unary_silu,     false, tile_silu_f32(dst_vtcm, src_vtcm, tw))
1086DEFINE_UNARY_TILED_TASK(unary_gelu,     false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))
1087DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))
1088DEFINE_UNARY_TILED_TASK(unary_tanh,     false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))
1089DEFINE_UNARY_TILED_TASK(unary_abs,      false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw))
1090DEFINE_UNARY_TILED_TASK(unary_log,      false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
1091DEFINE_UNARY_TILED_TASK(unary_relu,     false, hvx_max_scalar_f32(dst_vtcm, src_vtcm, 0.0f, tw))
1092DEFINE_UNARY_TILED_TASK(tri,            true,  tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
1093
1094static int execute_op_unary(struct htp_ops_context * octx) {
1095    int err = HTP_STATUS_OK;
1096
1097    const struct htp_tensor * src0 = octx->src[0];
1098    const struct htp_tensor * dst  = octx->dst;
1099
1100    const bool is_f16 = (src0->type == HTP_TYPE_F16);
1101
1102    const char * op_type = NULL;
1103
1104    switch (octx->op) {
1105        case HTP_OP_NORM:            op_type = is_f16 ? "norm-f16"     : "norm-f32";         break;
1106        case HTP_OP_RMS_NORM:        op_type = is_f16 ? "rmsnorm-f16"  : "rmsnorm-f32";      break;
1107        case HTP_OP_RMS_NORM_MUL:    op_type = "rmsnorm-mul-f32";                            break;
1108        case HTP_OP_SCALE:           op_type = is_f16 ? "scale-f16"    : "scale-f32";        break;
1109        case HTP_OP_CLAMP:           op_type = is_f16 ? "clamp-f16"    : "clamp-f32";        break;
1110        case HTP_OP_LEAKY_RELU:      op_type = "leaky-relu-f32";                             break;
1111        case HTP_OP_SQR:             op_type = is_f16 ? "sqr-f16"      : "sqr-f32";          break;
1112        case HTP_OP_SQRT:            op_type = is_f16 ? "sqrt-f16"     : "sqrt-f32";         break;
1113        case HTP_OP_UNARY_NEG:       op_type = "neg-f32";                                    break;
1114        case HTP_OP_UNARY_EXP:       op_type = "exp-f32";                                    break;
1115        case HTP_OP_UNARY_SIGMOID:   op_type = "sigmoid-f32";                                break;
1116        case HTP_OP_UNARY_SILU:      op_type = "silu-f32";                                   break;
1117        case HTP_OP_UNARY_GELU:      op_type = "gelu-f32";                                   break;
1118        case HTP_OP_UNARY_SOFTPLUS:  op_type = "softplus-f32";                               break;
1119        case HTP_OP_UNARY_TANH:      op_type = "tanh-f32";                                   break;
1120        case HTP_OP_UNARY_ABS:       op_type = is_f16 ? "abs-f16"      : "abs-f32";          break;
1121        case HTP_OP_UNARY_LOG:       op_type = is_f16 ? "log-f16"      : "log-f32";          break;
1122        case HTP_OP_UNARY_RELU:      op_type = "relu-f32";                                   break;
1123        case HTP_OP_L2_NORM:         op_type = is_f16 ? "l2norm-f16"   : "l2norm-f32";       break;
1124        case HTP_OP_TRI:             op_type = "tri-f32";                                    break;
1125        default:
1126            FARF(ERROR, "Unsupported unary Op %u\n", octx->op);
1127            return HTP_STATUS_NO_SUPPORT;
1128    }
1129
1130    // F16 only has row-block kernels for this subset of ops (see the dispatch switch
1131    // below) - reject everything else up front, before touching kparams/VTCM.
1132    if (is_f16) {
1133        switch (octx->op) {
1134            case HTP_OP_NORM:
1135            case HTP_OP_RMS_NORM:
1136            case HTP_OP_SCALE:
1137            case HTP_OP_CLAMP:
1138            case HTP_OP_SQR:
1139            case HTP_OP_SQRT:
1140            case HTP_OP_L2_NORM:
1141            case HTP_OP_UNARY_ABS:
1142            case HTP_OP_UNARY_LOG:
1143                break;
1144            default:
1145                FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
1146                return HTP_STATUS_NO_SUPPORT;
1147        }
1148    }
1149
1150    const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
1151
1152    if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
1153        return HTP_STATUS_INVAL_PARAMS;
1154    }
1155
1156    const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
1157    const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
1158    const size_t src0_data_row_size = src0->ne[0] * elem_size;
1159    const size_t dst_data_row_size  = dst->ne[0]  * elem_size;
1160
1161    uint32_t row_start = 0;
1162    uint32_t nrows     = src0_nrows;
1163
1164    if (octx->ctx->mdev.count > 1) {
1165        uint32_t rows_per_chunk = 0;
1166        htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
1167        const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
1168        row_start = range.start;
1169        nrows     = range.count;
1170    }
1171
1172    if (nrows == 0) {
1173        return HTP_STATUS_OK;
1174    }
1175
1176    const uint32_t n_threads = octx->n_threads;
1177
1178    const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
1179    const size_t dst_row_size_aligned  = kparams->dst_row_size_aligned;
1180
1181    // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path,
1182    // since only F32 has unary_task_f32_tiled_* kernels.
1183    const uint32_t col_tile = kparams->col_tile;
1184
1185    size_t src1_data_row_size = 0;
1186    size_t src1_row_size_aligned = kparams->src1_row_size_aligned;
1187    bool broadcast_weight = kparams->broadcast_weight;
1188    const struct htp_tensor * src1 = NULL;
1189
1190    // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see
1191    // try_fuse_node()'s type guard), so this never triggers when is_f16 is true.
1192    if (octx->op == HTP_OP_RMS_NORM_MUL) {
1193        src1 = octx->src[1];
1194        src1_data_row_size = src1->ne[0] * sizeof(float);
1195    }
1196
1197    if (octx->ctx->vtcm_size < (size_t)kparams->vtcm_size) {
1198        FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t)kparams->vtcm_size);
1199        return HTP_STATUS_VTCM_TOO_SMALL;
1200    }
1201
1202    octx->src0_spad.src = NULL;
1203    octx->src1_spad.src = NULL;
1204    octx->dst_spad.src  = NULL;
1205
1206    FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %u src1-vtcm-size %u dst-vtcm-size %u\n", op_type,
1207         src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
1208         kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size);
1209
1210    if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
1211        uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
1212        struct htp_unary_context uctx = {
1213            .octx                  = octx,
1214            .kparams               = kparams,
1215            .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
1216            .src0_nrows            = nrows,
1217            .row_start             = row_start,
1218
1219            .data_src0             = (const uint8_t *)src0->data,
1220            .data_src1             = (octx->op == HTP_OP_RMS_NORM_MUL) ? (const uint8_t *)src1->data : NULL,
1221            .data_dst              = (uint8_t *)dst->data,
1222
1223            .src0_data_row_size    = src0_data_row_size,
1224            .src1_data_row_size    = src1_data_row_size,
1225            .dst_data_row_size     = dst_data_row_size,
1226
1227            .src0_row_size_aligned = src0_row_size_aligned,
1228            .src1_row_size_aligned = src1_row_size_aligned,
1229            .dst_row_size_aligned  = dst_row_size_aligned,
1230
1231            .src0_vtcm_half_size   = kparams->vtcm_src0_size_per_thread / 2,
1232            .src1_vtcm_half_size   = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0,
1233            .dst_vtcm_half_size    = kparams->vtcm_dst_size_per_thread / 2,
1234
1235            .block                 = kparams->block,
1236            .nc                    = src0->ne[0],
1237            .col_tile              = col_tile,
1238            .broadcast_weight      = broadcast_weight,
1239
1240            .vtcm_src0             = VTCM_LAYOUT_PTR(uint8_t, base, 0),
1241            .vtcm_src1             = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0),
1242            .vtcm_dst              = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size),
1243
1244            .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,
1245            .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,
1246            .vtcm_dst_size_per_thread  = kparams->vtcm_dst_size_per_thread,
1247        };
1248
1249        FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile);
1250
1251        worker_callback_t task_func = NULL;
1252        if (col_tile) {
1253            switch (octx->op) {
1254                case HTP_OP_SCALE:           task_func = unary_task_f32_tiled_scale;          break;
1255                case HTP_OP_CLAMP:           task_func = unary_task_f32_tiled_clamp;          break;
1256                case HTP_OP_LEAKY_RELU:      task_func = unary_task_f32_tiled_leaky_relu;     break;
1257                case HTP_OP_SQR:             task_func = unary_task_f32_tiled_sqr;            break;
1258                case HTP_OP_SQRT:            task_func = unary_task_f32_tiled_sqrt;           break;
1259                case HTP_OP_UNARY_NEG:       task_func = unary_task_f32_tiled_unary_neg;      break;
1260                case HTP_OP_UNARY_EXP:       task_func = unary_task_f32_tiled_unary_exp;      break;
1261                case HTP_OP_UNARY_SIGMOID:   task_func = unary_task_f32_tiled_unary_sigmoid;  break;
1262                case HTP_OP_UNARY_SILU:      task_func = unary_task_f32_tiled_unary_silu;     break;
1263                case HTP_OP_UNARY_GELU:      task_func = unary_task_f32_tiled_unary_gelu;     break;
1264                case HTP_OP_UNARY_SOFTPLUS:  task_func = unary_task_f32_tiled_unary_softplus; break;
1265                case HTP_OP_UNARY_TANH:      task_func = unary_task_f32_tiled_unary_tanh;     break;
1266                case HTP_OP_UNARY_ABS:       task_func = unary_task_f32_tiled_unary_abs;      break;
1267                case HTP_OP_UNARY_LOG:       task_func = unary_task_f32_tiled_unary_log;      break;
1268                case HTP_OP_UNARY_RELU:      task_func = unary_task_f32_tiled_unary_relu;     break;
1269                case HTP_OP_TRI:             task_func = unary_task_f32_tiled_tri;            break;
1270                default:                     break;
1271            }
1272        } else if (is_f16) {
1273            switch (octx->op) {
1274                case HTP_OP_NORM:            task_func = unary_task_f16_norm;                 break;
1275                case HTP_OP_RMS_NORM:        task_func = unary_task_f16_rms_norm;             break;
1276                case HTP_OP_SCALE:           task_func = unary_task_f16_scale;                break;
1277                case HTP_OP_CLAMP:           task_func = unary_task_f16_clamp;                break;
1278                case HTP_OP_SQR:             task_func = unary_task_f16_sqr;                  break;
1279                case HTP_OP_SQRT:            task_func = unary_task_f16_sqrt;                 break;
1280                case HTP_OP_L2_NORM:         task_func = unary_task_f16_l2_norm;              break;
1281                case HTP_OP_UNARY_ABS:       task_func = unary_task_f16_unary_abs;            break;
1282                case HTP_OP_UNARY_LOG:       task_func = unary_task_f16_unary_log;            break;
1283                default:                     break;
1284            }
1285        } else {
1286            switch (octx->op) {
1287                case HTP_OP_NORM:            task_func = unary_task_f32_norm;                 break;
1288                case HTP_OP_RMS_NORM:        task_func = unary_task_f32_rms_norm;             break;
1289                case HTP_OP_RMS_NORM_MUL:    task_func = unary_task_f32_rms_norm_mul;         break;
1290                case HTP_OP_SCALE:           task_func = unary_task_f32_scale;                break;
1291                case HTP_OP_CLAMP:           task_func = unary_task_f32_clamp;                break;
1292                case HTP_OP_LEAKY_RELU:      task_func = unary_task_f32_leaky_relu;           break;
1293                case HTP_OP_SQR:             task_func = unary_task_f32_sqr;                  break;
1294                case HTP_OP_SQRT:            task_func = unary_task_f32_sqrt;                 break;
1295                case HTP_OP_UNARY_NEG:       task_func = unary_task_f32_unary_neg;            break;
1296                case HTP_OP_UNARY_EXP:       task_func = unary_task_f32_unary_exp;            break;
1297                case HTP_OP_UNARY_SIGMOID:   task_func = unary_task_f32_unary_sigmoid;        break;
1298                case HTP_OP_UNARY_SILU:      task_func = unary_task_f32_unary_silu;           break;
1299                case HTP_OP_UNARY_GELU:      task_func = unary_task_f32_unary_gelu;           break;
1300                case HTP_OP_UNARY_SOFTPLUS:  task_func = unary_task_f32_unary_softplus;       break;
1301                case HTP_OP_UNARY_TANH:      task_func = unary_task_f32_unary_tanh;           break;
1302                case HTP_OP_UNARY_ABS:       task_func = unary_task_f32_unary_abs;            break;
1303                case HTP_OP_UNARY_LOG:       task_func = unary_task_f32_unary_log;            break;
1304                case HTP_OP_UNARY_RELU:      task_func = unary_task_f32_unary_relu;           break;
1305                case HTP_OP_L2_NORM:         task_func = unary_task_f32_l2_norm;              break;
1306                case HTP_OP_TRI:             task_func = unary_task_f32_tri;                  break;
1307                default:                     break;
1308            }
1309        }
1310
1311        if (task_func) {
1312            work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads);
1313        } else {
1314            FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op);
1315            err = HTP_STATUS_NO_SUPPORT;
1316        }
1317    }
1318
1319    return err;
1320}
1321
1322int op_unary(struct htp_ops_context * octx) {
1323    switch (octx->src[0]->type) {
1324        case HTP_TYPE_F32:
1325        case HTP_TYPE_F16:
1326            return execute_op_unary(octx);
1327
1328        default:
1329            return HTP_STATUS_NO_SUPPORT;
1330    }
1331}