char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: further improved pipeline of the core bits (L2, DMA, MM, FA) (#26049)0a50d9909

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
63.5 KiB1378 linesraw
1#include "hmx-utils.h"
2#include "hmx-queue.h"
3
4// MXFP4 dequantization LUT: maps 4-bit index to fp16 mantissa value
5// kvalues: 0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6
6static const __fp16 mxfp4_to_fp16_lut[64] __attribute__((aligned(VLEN))) = {
7    0, 0, 0.5, 0, 1, 0, 1.5, 0, 2, 0, 3, 0, 4, 0, 6, 0, 0, 0, -0.5, 0, -1, 0, -1.5, 0, -2, 0, -3, 0, -4, 0, -6, 0,
8};
9
10static const __fp16 iq4_nl_to_fp16_lut[64] __attribute__((aligned(VLEN))) = {
11    -127, 0, -104, 0, -83, 0, -65, 0, -49, 0, -35, 0, -22, 0, -10, 0,
12    1,    0, 13,   0, 25,  0, 38,  0, 53,  0, 69,  0, 89,  0, 113, 0,
13};
14
15// --- tiled format dequantizers ---
16
17typedef struct {
18    struct htp_context      * ctx;
19    struct htp_thread_trace * traces;
20    __fp16                  * dst;
21    const uint8_t           * src;
22
23    struct fastdiv_values     n_k_tiles_div;
24    uint32_t                  n_k_tiles;
25    uint32_t                  n_tot_tiles;
26    uint32_t                  n_tiles_per_task;
27    uint32_t                  tile_size;
28    uint32_t                  aligned_tile_size;
29    uint32_t                  n_tasks;
30    uint32_t                  n_cols;
31    uint32_t                  k_block;
32    size_t                    row_stride;
33    uint32_t                  weight_type;
34} tiled_dequantize_state_t;
35
36// Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16.
37static void dequantize_tiled_weight_to_fp16_task_q4_0(
38        const tiled_dequantize_state_t *state,
39        uint32_t start_tile, uint32_t end_tile) {
40
41    const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);
42    const HVX_Vector i8 = Q6_Vb_vsplat_R(8);
43
44    for (uint32_t t = start_tile; t < end_tile; t++) {
45        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
46        __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
47
48        HVX_Vector v_sc = hvx_vmem(tile_src + 512);
49        HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
50
51        // Load all 4 groups in parallel
52        HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128);
53        HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128);
54        HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128);
55        HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128);
56
57        // Nibble extraction
58        HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4);
59        HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4);
60        HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4);
61        HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4);
62        HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4);
63        HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4);
64        HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4);
65        HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4);
66
67        // Offsetting (-8)
68        v_lo0 = Q6_Vb_vsub_VbVb(v_lo0, i8);
69        v_hi0 = Q6_Vb_vsub_VbVb(v_hi0, i8);
70        v_lo1 = Q6_Vb_vsub_VbVb(v_lo1, i8);
71        v_hi1 = Q6_Vb_vsub_VbVb(v_hi1, i8);
72        v_lo2 = Q6_Vb_vsub_VbVb(v_lo2, i8);
73        v_hi2 = Q6_Vb_vsub_VbVb(v_hi2, i8);
74        v_lo3 = Q6_Vb_vsub_VbVb(v_lo3, i8);
75        v_hi3 = Q6_Vb_vsub_VbVb(v_hi3, i8);
76
77        // Shuffling
78        HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1);
79        HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1);
80        HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1);
81        HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1);
82
83        // Unpack to 16-bit
84        HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0));
85        HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0));
86        HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1));
87        HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1));
88        HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2));
89        HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2));
90        HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3));
91        HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3));
92
93        // Convert and scale multiplication
94        HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated));
95        HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated));
96        HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated));
97        HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated));
98
99        HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated));
100        HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated));
101        HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated));
102        HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated));
103
104        HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated));
105        HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated));
106        HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated));
107        HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated));
108
109        HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated));
110        HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated));
111        HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated));
112        HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated));
113
114        hvx_vmem(dst_ptr +  0 * 64) = v_grp0_0;
115        hvx_vmem(dst_ptr +  1 * 64) = v_grp0_1;
116        hvx_vmem(dst_ptr +  2 * 64) = v_grp0_2;
117        hvx_vmem(dst_ptr +  3 * 64) = v_grp0_3;
118
119        hvx_vmem(dst_ptr +  4 * 64) = v_grp1_0;
120        hvx_vmem(dst_ptr +  5 * 64) = v_grp1_1;
121        hvx_vmem(dst_ptr +  6 * 64) = v_grp1_2;
122        hvx_vmem(dst_ptr +  7 * 64) = v_grp1_3;
123
124        hvx_vmem(dst_ptr +  8 * 64) = v_grp2_0;
125        hvx_vmem(dst_ptr +  9 * 64) = v_grp2_1;
126        hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2;
127        hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3;
128
129        hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0;
130        hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1;
131        hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2;
132        hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3;
133    }
134}
135
136static void dequantize_tiled_weight_to_fp16_task_q4_1(
137        const tiled_dequantize_state_t *state,
138        uint32_t start_tile, uint32_t end_tile) {
139
140    const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);
141
142    for (uint32_t t = start_tile; t < end_tile; t++) {
143        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
144        __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
145
146        HVX_Vector vscale_offset = hvx_vmem(tile_src + 512);
147        HVX_VectorPair dm_deal = Q6_W_vdeal_VVR(vscale_offset, vscale_offset, -2);
148        HVX_Vector vd = Q6_V_lo_W(dm_deal);
149        HVX_Vector vm = Q6_V_hi_W(dm_deal);
150
151        HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vd, vd, -2));
152        HVX_Vector v_offset_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vm, vm, -2));
153
154        // Load all 4 groups in parallel
155        HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128);
156        HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128);
157        HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128);
158        HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128);
159
160        // Nibble extraction
161        HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4);
162        HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4);
163        HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4);
164        HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4);
165        HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4);
166        HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4);
167        HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4);
168        HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4);
169
170        // Shuffling
171        HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1);
172        HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1);
173        HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1);
174        HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1);
175
176        // Unpack to 16-bit
177        HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0));
178        HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0));
179        HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1));
180        HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1));
181        HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2));
182        HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2));
183        HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3));
184        HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3));
185
186        // Convert, multiply, add offset
187        HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated));
188        HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated));
189        HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated));
190        HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated));
191
192        HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated));
193        HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated));
194        HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated));
195        HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated));
196
197        HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated));
198        HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated));
199        HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated));
200        HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated));
201
202        HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated));
203        HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated));
204        HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated));
205        HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated));
206
207        // Parallel Stores
208        hvx_vmem(dst_ptr +  0 * 64) = v_grp0_0;
209        hvx_vmem(dst_ptr +  1 * 64) = v_grp0_1;
210        hvx_vmem(dst_ptr +  2 * 64) = v_grp0_2;
211        hvx_vmem(dst_ptr +  3 * 64) = v_grp0_3;
212
213        hvx_vmem(dst_ptr +  4 * 64) = v_grp1_0;
214        hvx_vmem(dst_ptr +  5 * 64) = v_grp1_1;
215        hvx_vmem(dst_ptr +  6 * 64) = v_grp1_2;
216        hvx_vmem(dst_ptr +  7 * 64) = v_grp1_3;
217
218        hvx_vmem(dst_ptr +  8 * 64) = v_grp2_0;
219        hvx_vmem(dst_ptr +  9 * 64) = v_grp2_1;
220        hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2;
221        hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3;
222
223        hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0;
224        hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1;
225        hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2;
226        hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3;
227    }
228}
229
230static void dequantize_tiled_weight_to_fp16_task_iq4_nl(
231        const tiled_dequantize_state_t *state,
232        uint32_t start_tile, uint32_t end_tile) {
233
234    const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);
235    const HVX_Vector vlut_cvt = hvx_vmem(iq4_nl_to_fp16_lut);
236
237    for (uint32_t t = start_tile; t < end_tile; t++) {
238        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
239        __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
240
241        HVX_Vector v_sc = hvx_vmem(tile_src + 512);
242        HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
243
244        // Load all 4 groups in parallel
245        HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128);
246        HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128);
247        HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128);
248        HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128);
249
250        // Nibble extraction
251        HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4);
252        HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4);
253        HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4);
254        HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4);
255        HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4);
256        HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4);
257        HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4);
258        HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4);
259
260        // Shuffling
261        HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1);
262        HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1);
263        HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1);
264        HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1);
265
266        // Shuffle for LUT lookup
267        HVX_Vector v_q_lo0 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf0));
268        HVX_Vector v_q_hi0 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf0));
269        HVX_Vector v_q_lo1 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf1));
270        HVX_Vector v_q_hi1 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf1));
271        HVX_Vector v_q_lo2 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf2));
272        HVX_Vector v_q_hi2 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf2));
273        HVX_Vector v_q_lo3 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf3));
274        HVX_Vector v_q_hi3 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf3));
275
276        // LUT lookup
277        HVX_VectorPair vp_lo0 = Q6_Wh_vlut16_VbVhR(v_q_lo0, vlut_cvt, 0);
278        HVX_VectorPair vp_hi0 = Q6_Wh_vlut16_VbVhR(v_q_hi0, vlut_cvt, 0);
279        HVX_VectorPair vp_lo1 = Q6_Wh_vlut16_VbVhR(v_q_lo1, vlut_cvt, 0);
280        HVX_VectorPair vp_hi1 = Q6_Wh_vlut16_VbVhR(v_q_hi1, vlut_cvt, 0);
281        HVX_VectorPair vp_lo2 = Q6_Wh_vlut16_VbVhR(v_q_lo2, vlut_cvt, 0);
282        HVX_VectorPair vp_hi2 = Q6_Wh_vlut16_VbVhR(v_q_hi2, vlut_cvt, 0);
283        HVX_VectorPair vp_lo3 = Q6_Wh_vlut16_VbVhR(v_q_lo3, vlut_cvt, 0);
284        HVX_VectorPair vp_hi3 = Q6_Wh_vlut16_VbVhR(v_q_hi3, vlut_cvt, 0);
285
286        // Convert and scale multiplication
287        HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo0), v_scale_duplicated));
288        HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo0), v_scale_duplicated));
289        HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi0), v_scale_duplicated));
290        HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi0), v_scale_duplicated));
291
292        HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo1), v_scale_duplicated));
293        HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo1), v_scale_duplicated));
294        HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi1), v_scale_duplicated));
295        HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi1), v_scale_duplicated));
296
297        HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo2), v_scale_duplicated));
298        HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo2), v_scale_duplicated));
299        HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi2), v_scale_duplicated));
300        HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi2), v_scale_duplicated));
301
302        HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo3), v_scale_duplicated));
303        HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo3), v_scale_duplicated));
304        HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi3), v_scale_duplicated));
305        HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi3), v_scale_duplicated));
306
307        hvx_vmem(dst_ptr +  0 * 64) = v_grp0_0;
308        hvx_vmem(dst_ptr +  1 * 64) = v_grp0_1;
309        hvx_vmem(dst_ptr +  2 * 64) = v_grp0_2;
310        hvx_vmem(dst_ptr +  3 * 64) = v_grp0_3;
311
312        hvx_vmem(dst_ptr +  4 * 64) = v_grp1_0;
313        hvx_vmem(dst_ptr +  5 * 64) = v_grp1_1;
314        hvx_vmem(dst_ptr +  6 * 64) = v_grp1_2;
315        hvx_vmem(dst_ptr +  7 * 64) = v_grp1_3;
316
317        hvx_vmem(dst_ptr +  8 * 64) = v_grp2_0;
318        hvx_vmem(dst_ptr +  9 * 64) = v_grp2_1;
319        hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2;
320        hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3;
321
322        hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0;
323        hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1;
324        hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2;
325        hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3;
326    }
327}
328
329static void dequantize_tiled_weight_to_fp16_task_mxfp4(
330        const tiled_dequantize_state_t *state,
331        uint32_t start_tile, uint32_t end_tile) {
332
333    const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);
334    const HVX_Vector vlut_cvt = hvx_vmem(mxfp4_to_fp16_lut);
335
336    for (uint32_t t = start_tile; t < end_tile; t++) {
337        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
338        __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
339
340        HVX_Vector v = hvx_vmem(tile_src + 512);
341        HVX_Vector vh = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(v));
342        vh = Q6_Vh_vsub_VhVh(vh, Q6_Vh_vsplat_R(112));
343        vh = Q6_Vh_vmax_VhVh(vh, Q6_V_vzero());
344        vh = Q6_Vh_vmin_VhVh(vh, Q6_Vh_vsplat_R(30));
345        vh = Q6_Vh_vasl_VhR(vh, 10);
346
347        HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vh, vh, -2));
348
349        // Load all 4 groups in parallel
350        HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128);
351        HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128);
352        HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128);
353        HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128);
354
355        // Nibble extraction
356        HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4);
357        HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4);
358        HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4);
359        HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4);
360        HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4);
361        HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4);
362        HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4);
363        HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4);
364
365        // Shuffling
366        HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1);
367        HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1);
368        HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1);
369        HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1);
370
371        // Shuffle for LUT lookup
372        HVX_Vector v_q_lo0 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf0));
373        HVX_Vector v_q_hi0 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf0));
374        HVX_Vector v_q_lo1 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf1));
375        HVX_Vector v_q_hi1 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf1));
376        HVX_Vector v_q_lo2 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf2));
377        HVX_Vector v_q_hi2 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf2));
378        HVX_Vector v_q_lo3 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf3));
379        HVX_Vector v_q_hi3 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf3));
380
381        // LUT lookup
382        HVX_VectorPair vp_lo0 = Q6_Wh_vlut16_VbVhR(v_q_lo0, vlut_cvt, 0);
383        HVX_VectorPair vp_hi0 = Q6_Wh_vlut16_VbVhR(v_q_hi0, vlut_cvt, 0);
384        HVX_VectorPair vp_lo1 = Q6_Wh_vlut16_VbVhR(v_q_lo1, vlut_cvt, 0);
385        HVX_VectorPair vp_hi1 = Q6_Wh_vlut16_VbVhR(v_q_hi1, vlut_cvt, 0);
386        HVX_VectorPair vp_lo2 = Q6_Wh_vlut16_VbVhR(v_q_lo2, vlut_cvt, 0);
387        HVX_VectorPair vp_hi2 = Q6_Wh_vlut16_VbVhR(v_q_hi2, vlut_cvt, 0);
388        HVX_VectorPair vp_lo3 = Q6_Wh_vlut16_VbVhR(v_q_lo3, vlut_cvt, 0);
389        HVX_VectorPair vp_hi3 = Q6_Wh_vlut16_VbVhR(v_q_hi3, vlut_cvt, 0);
390
391        // Convert and scale multiplication
392        HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo0), v_scale_duplicated));
393        HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo0), v_scale_duplicated));
394        HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi0), v_scale_duplicated));
395        HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi0), v_scale_duplicated));
396
397        HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo1), v_scale_duplicated));
398        HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo1), v_scale_duplicated));
399        HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi1), v_scale_duplicated));
400        HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi1), v_scale_duplicated));
401
402        HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo2), v_scale_duplicated));
403        HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo2), v_scale_duplicated));
404        HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi2), v_scale_duplicated));
405        HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi2), v_scale_duplicated));
406
407        HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo3), v_scale_duplicated));
408        HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo3), v_scale_duplicated));
409        HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi3), v_scale_duplicated));
410        HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi3), v_scale_duplicated));
411
412        hvx_vmem(dst_ptr +  0 * 64) = v_grp0_0;
413        hvx_vmem(dst_ptr +  1 * 64) = v_grp0_1;
414        hvx_vmem(dst_ptr +  2 * 64) = v_grp0_2;
415        hvx_vmem(dst_ptr +  3 * 64) = v_grp0_3;
416
417        hvx_vmem(dst_ptr +  4 * 64) = v_grp1_0;
418        hvx_vmem(dst_ptr +  5 * 64) = v_grp1_1;
419        hvx_vmem(dst_ptr +  6 * 64) = v_grp1_2;
420        hvx_vmem(dst_ptr +  7 * 64) = v_grp1_3;
421
422        hvx_vmem(dst_ptr +  8 * 64) = v_grp2_0;
423        hvx_vmem(dst_ptr +  9 * 64) = v_grp2_1;
424        hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2;
425        hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3;
426
427        hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0;
428        hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1;
429        hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2;
430        hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3;
431    }
432}
433
434static void dequantize_tiled_weight_to_fp16_task_q8_0(
435        const tiled_dequantize_state_t *state,
436        uint32_t start_tile, uint32_t end_tile) {
437
438    for (uint32_t t = start_tile; t < end_tile; t++) {
439        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
440        __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
441
442        HVX_Vector v_sc = hvx_vmem(tile_src + 1024);
443        HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
444
445        // Load groups 0-3 in parallel
446        HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128);
447        HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128);
448        HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128);
449        HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128);
450
451        HVX_VectorPair vp_int16_0 = Q6_Wh_vunpack_Vb(vq0);
452        HVX_VectorPair vp_int16_1 = Q6_Wh_vunpack_Vb(vq1);
453        HVX_VectorPair vp_int16_2 = Q6_Wh_vunpack_Vb(vq2);
454        HVX_VectorPair vp_int16_3 = Q6_Wh_vunpack_Vb(vq3);
455
456        // Load groups 4-7 in parallel
457        HVX_Vector vq4 = hvx_vmem(tile_src + 4 * 128);
458        HVX_Vector vq5 = hvx_vmem(tile_src + 5 * 128);
459        HVX_Vector vq6 = hvx_vmem(tile_src + 6 * 128);
460        HVX_Vector vq7 = hvx_vmem(tile_src + 7 * 128);
461
462        HVX_VectorPair vp_int16_4 = Q6_Wh_vunpack_Vb(vq4);
463        HVX_VectorPair vp_int16_5 = Q6_Wh_vunpack_Vb(vq5);
464        HVX_VectorPair vp_int16_6 = Q6_Wh_vunpack_Vb(vq6);
465        HVX_VectorPair vp_int16_7 = Q6_Wh_vunpack_Vb(vq7);
466
467        // Convert and scale multiply for groups 0-3
468        HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_0)), v_scale_duplicated));
469        HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_0)), v_scale_duplicated));
470        HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_1)), v_scale_duplicated));
471        HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_1)), v_scale_duplicated));
472        HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_2)), v_scale_duplicated));
473        HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_2)), v_scale_duplicated));
474        HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_3)), v_scale_duplicated));
475        HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_3)), v_scale_duplicated));
476
477        // Store groups 0-3
478        hvx_vmem(dst_ptr +  0 * 64) = v_grp0_0;
479        hvx_vmem(dst_ptr +  1 * 64) = v_grp0_1;
480        hvx_vmem(dst_ptr +  2 * 64) = v_grp1_0;
481        hvx_vmem(dst_ptr +  3 * 64) = v_grp1_1;
482        hvx_vmem(dst_ptr +  4 * 64) = v_grp2_0;
483        hvx_vmem(dst_ptr +  5 * 64) = v_grp2_1;
484        hvx_vmem(dst_ptr +  6 * 64) = v_grp3_0;
485        hvx_vmem(dst_ptr +  7 * 64) = v_grp3_1;
486
487        // Convert and scale multiply for groups 4-7
488        HVX_Vector v_grp4_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_4)), v_scale_duplicated));
489        HVX_Vector v_grp4_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_4)), v_scale_duplicated));
490        HVX_Vector v_grp5_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_5)), v_scale_duplicated));
491        HVX_Vector v_grp5_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_5)), v_scale_duplicated));
492        HVX_Vector v_grp6_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_6)), v_scale_duplicated));
493        HVX_Vector v_grp6_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_6)), v_scale_duplicated));
494        HVX_Vector v_grp7_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_7)), v_scale_duplicated));
495        HVX_Vector v_grp7_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_7)), v_scale_duplicated));
496
497        // Store groups 4-7
498        hvx_vmem(dst_ptr +  8 * 64) = v_grp4_0;
499        hvx_vmem(dst_ptr +  9 * 64) = v_grp4_1;
500        hvx_vmem(dst_ptr + 10 * 64) = v_grp5_0;
501        hvx_vmem(dst_ptr + 11 * 64) = v_grp5_1;
502        hvx_vmem(dst_ptr + 12 * 64) = v_grp6_0;
503        hvx_vmem(dst_ptr + 13 * 64) = v_grp6_1;
504        hvx_vmem(dst_ptr + 14 * 64) = v_grp7_0;
505        hvx_vmem(dst_ptr + 15 * 64) = v_grp7_1;
506    }
507}
508
509static __attribute__((noinline))
510void convert_f16_weight_to_fp16_tiles_task(
511        const tiled_dequantize_state_t *state,
512        uint32_t start_tile, uint32_t end_tile) {
513
514    const uint32_t n_k_tiles = state->n_k_tiles;
515    const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div;
516
517    const HVX_Vector v_scat_base  = hvx_vmem(hmx_transpose_scatter_offsets);
518    const HVX_Vector v_scat_step  = Q6_V_vsplat_R(4);
519    const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64);
520
521    unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div);
522    unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div);
523
524    for (unsigned t = start_tile; t < (unsigned)end_tile; ) {
525        if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; }
526
527        __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
528        {
529            uint32_t byte_off = kt * 32 * sizeof(__fp16);
530
531            HVX_Vector v_off = v_scat_base;
532            for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) {
533                uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r;
534                uint32_t row1 = row0 + 1;
535
536                const uint8_t *r0 = state->src + row0 * state->row_stride;
537                const uint8_t *r1 = state->src + row1 * state->row_stride;
538
539                HVX_Vector v0 = hvx_vmemu((const __fp16 *)(r0 + byte_off));
540                HVX_Vector v1 = (row1 < state->n_cols) ? hvx_vmemu((const __fp16 *)(r1 + byte_off)) : Q6_V_vzero();
541
542                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v0);
543                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
544                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v1);
545                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
546            }
547        }
548        ++t; ++kt;
549    }
550}
551
552static __attribute__((noinline))
553void quantize_f32_weight_to_fp16_tiles_task(
554        const tiled_dequantize_state_t *state,
555        uint32_t start_tile, uint32_t end_tile) {
556
557    const uint32_t n_k_tiles = state->n_k_tiles;
558    const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div;
559
560    const HVX_Vector v_scat_base  = hvx_vmem(hmx_transpose_scatter_offsets);
561    const HVX_Vector v_scat_step  = Q6_V_vsplat_R(4);
562    const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64);
563
564    unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div);
565    unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div);
566
567    for (unsigned t = start_tile; t < (unsigned)end_tile; ) {
568        if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; }
569
570        __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
571        {
572            uint32_t byte_off = kt * 32 * sizeof(float);
573
574            HVX_Vector v_off = v_scat_base;
575            for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) {
576                uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r;
577                uint32_t row1 = row0 + 1;
578
579                const uint8_t *r0 = state->src + row0 * state->row_stride;
580                const uint8_t *r1 = state->src + row1 * state->row_stride;
581
582                HVX_Vector v0_f32 = hvx_vmem((const float *)(r0 + byte_off));
583                HVX_Vector v1_f32 = (row1 < state->n_cols) ? hvx_vmem((const float *)(r1 + byte_off)) : Q6_V_vzero();
584
585                HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32);
586
587                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out);
588                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
589
590                HVX_Vector v_out_hi = Q6_V_vror_VR(v_out, 64);
591                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi);
592                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
593            }
594        }
595        ++t; ++kt;
596    }
597}
598
599// --- End tiled dequantizers ---
600
601// dot-chunk functions require external HMX lock
602
603static void core_dot_chunk_fp16_short(__fp16 *restrict output, const __fp16 *restrict activation,
604                                const __fp16 *restrict weight, const __fp16 *restrict scales,
605                                uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) {
606    __builtin_assume(n_row_tiles > 0);
607    __builtin_assume(n_col_tiles > 0);
608    __builtin_assume(n_dot_tiles > 0);
609    __builtin_assume(n_dot_tiles <= 32);
610
611    asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales));
612
613    const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS;
614    const uint32_t range = 2048u * n_dot_tiles - 1;
615
616    for (uint32_t r = 0; r < n_row_tiles; ++r) {
617        const __fp16 *row_base = activation + r * dot_stride;
618        const __fp16 *col_base = weight;
619        __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS;
620
621        for (size_t c = 0; c < n_col_tiles; ++c) {
622            asm volatile(HMX_CLRACC_F16());
623            asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base));
624            asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory");
625            col_base += dot_stride;
626            out_tile += HTP_MM_HMX_TILE_N_ELMS;
627        }
628    }
629}
630
631static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation,
632                          const __fp16 *restrict weight, const __fp16 *restrict scales,
633                          uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) {
634    if (n_dot_tiles <= 32) {
635        core_dot_chunk_fp16_short(output, activation, weight, scales, n_row_tiles, n_col_tiles, n_dot_tiles);
636        return;
637    }
638    __builtin_assume(n_row_tiles > 0);
639    __builtin_assume(n_col_tiles > 0);
640    __builtin_assume(n_dot_tiles > 32);
641
642    asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales));
643
644    const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS;
645
646    for (uint32_t r = 0; r < n_row_tiles; ++r) {
647        const __fp16 *row_base = activation + r * dot_stride;
648        const __fp16 *col_base = weight;
649        __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS;
650
651        for (size_t c = 0; c < n_col_tiles; ++c) {
652            const __fp16 *row_tiles = row_base;
653            const __fp16 *col_tiles = col_base;
654
655            asm volatile(HMX_CLRACC_F16());
656
657            const uint32_t n_loops = n_dot_tiles / 32;
658            const uint32_t rem = n_dot_tiles % 32;
659
660            for (uint32_t l = 0; l < n_loops; ++l) {
661                asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles));
662                row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS;
663                col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS;
664            }
665
666            if (rem > 0) {
667                const uint32_t range = 2048u * rem - 1;
668                asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles));
669            }
670
671            asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory");
672
673            col_base += dot_stride;
674            out_tile += HTP_MM_HMX_TILE_N_ELMS;
675        }
676    }
677}
678
679static void core_mma_chunk_fp16_short(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b,
680                                const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile,
681                                uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) {
682    __builtin_assume(n_row_tiles > 0);
683    __builtin_assume(n_col_tiles > 0);
684    __builtin_assume(n_dot_tiles > 0);
685    __builtin_assume(n_dot_tiles <= 32);
686
687    asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales));
688
689    const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS;
690    const uint32_t range = 2048u * n_dot_tiles - 1;
691
692    for (size_t i = 0; i < n_row_tiles; ++i) {
693        const __fp16 *row_base = a + i * dot_tile_stride;
694        __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS;
695        const __fp16 *col_base = b;
696        __fp16 *accum_tile = res_base;
697
698        for (size_t j = 0; j < n_col_tiles; ++j) {
699            asm volatile(HMX_CLRACC_F16());
700
701            if (!zero_init) {
702                asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile));
703            }
704
705            asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base));
706
707            asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory");
708
709            col_base   += dot_tile_stride;
710            accum_tile += HTP_MM_HMX_TILE_N_ELMS;
711        }
712    }
713}
714
715static void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b,
716                          const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile,
717                          uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) {
718    if (n_dot_tiles <= 32) {
719        core_mma_chunk_fp16_short(c, a, b, col_scales, eye_tile, n_row_tiles, n_col_tiles, n_dot_tiles, zero_init);
720        return;
721    }
722    __builtin_assume(n_row_tiles > 0);
723    __builtin_assume(n_col_tiles > 0);
724    __builtin_assume(n_dot_tiles > 32);
725
726    asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales));
727
728    const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS;
729
730    for (size_t i = 0; i < n_row_tiles; ++i) {
731        const __fp16 *row_base = a + i * dot_tile_stride;
732        __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS;
733        const __fp16 *col_base = b;
734        __fp16 *accum_tile = res_base;
735
736        for (size_t j = 0; j < n_col_tiles; ++j) {
737            const __fp16 *col_tiles = col_base;
738            const __fp16 *row_tiles = row_base;
739
740            asm volatile(HMX_CLRACC_F16());
741
742            if (!zero_init) {
743                asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile));
744            }
745
746            const uint32_t n_loops = n_dot_tiles / 32;
747            const uint32_t rem = n_dot_tiles % 32;
748
749            for (uint32_t l = 0; l < n_loops; ++l) {
750                asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles));
751                row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS;
752                col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS;
753            }
754
755            if (rem > 0) {
756                const uint32_t range = 2048u * rem - 1;
757                asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles));
758            }
759
760            asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory");
761
762            col_base += dot_tile_stride;
763            accum_tile += HTP_MM_HMX_TILE_N_ELMS;
764        }
765    }
766}
767
768// output : fp16 -> f32p
769
770static void transfer_output_chunk_fp16_to_fp32_col_chunk(
771    float *restrict dst,
772    const float *restrict src2,
773    const __fp16 *restrict vtcm_src,
774    uint32_t start_row,
775    uint32_t n_rows,
776    uint32_t c_len,
777    uint32_t total_n_cols,
778    uint32_t dst_stride,
779    uint32_t src2_stride,
780    uint32_t dst_cols
781) {
782    assert(c_len % HTP_MM_HMX_TILE_N_COLS == 0);
783    assert(total_n_cols % HTP_MM_HMX_TILE_N_COLS == 0);
784    const size_t tile_row_stride = (total_n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS;
785
786    const HVX_Vector one = hvx_vec_splat_f16(1.0);
787
788    const size_t limit_c         = hex_smin(c_len, dst_cols);
789    const size_t limit_c_aligned = (limit_c & ~31);
790
791    for (size_t r = 0; r < n_rows; r += 2) {
792        const size_t r_idx0 = start_row + r + 0;
793        const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS;
794        const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2;  // index of the row pair within the tile
795        const __fp16 *row_base = vtcm_src + r0 * tile_row_stride;
796        float *output_row_base = dst + r * dst_stride;  // global memory row base for row r (and r+1)
797        const float *src2_row_base = src2 ? (src2 + r * src2_stride) : NULL;
798
799        #pragma unroll(4)
800        for (size_t c = 0; c < limit_c_aligned; c += HTP_MM_HMX_TILE_N_COLS) {
801            const size_t c0    = c / HTP_MM_HMX_TILE_N_COLS;
802            const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS;
803            HVX_Vector v = ((const HVX_Vector *) tile)[r1];
804            HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);
805
806            HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0);
807            HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride);
808
809            HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
810            if (src2_row_base) {
811                HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0);
812                v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0);
813            }
814            *pv_out0 = v_out0;
815
816            if (r + 1 < n_rows) {
817                HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
818                if (src2_row_base) {
819                    HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride);
820                    v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1);
821                }
822                *pv_out1 = v_out1;
823            }
824        }
825
826        if (limit_c_aligned < limit_c) {
827            size_t c = limit_c_aligned;
828            size_t valid_c = limit_c - c;
829            const size_t c0 = c / HTP_MM_HMX_TILE_N_COLS;
830            const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS;
831            HVX_Vector v = ((const HVX_Vector *) tile)[r1];
832            HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);
833
834            HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
835            if (src2_row_base) {
836                HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0);
837                v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0);
838            }
839            hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), v_out0);
840
841            if (r + 1 < n_rows) {
842                HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
843                if (src2_row_base) {
844                    HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride);
845                    v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1);
846                }
847                hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), v_out1);
848            }
849        }
850    }
851}
852
853static inline void transfer_output_chunk_fp16_to_fp32(
854    float *restrict dst,
855    const float *restrict src2,
856    const __fp16 *restrict vtcm_src,
857    uint32_t start_row,
858    uint32_t n_rows,
859    uint32_t n_cols,
860    uint32_t dst_stride,
861    uint32_t src2_stride,
862    uint32_t dst_cols
863) {
864    transfer_output_chunk_fp16_to_fp32_col_chunk(
865        dst, src2, vtcm_src, start_row, n_rows, n_cols, n_cols, dst_stride, src2_stride, dst_cols
866    );
867}
868
869typedef struct {
870    const __fp16  *vtcm_src;
871    float         *dst;
872    const float   *src2;
873    uint32_t       n_tasks;
874    uint32_t       n_tot_chunks;
875    uint32_t       n_chunks_per_task;
876    uint32_t       n_cols;
877    uint32_t       dst_stride;  // DDR row stride
878    uint32_t       src2_stride; // DDR row stride for residual
879    uint32_t       dst_cols;    // Actual output columns
880    struct htp_thread_trace * traces;
881} output_transfer_task_state_t;
882
883// activations : fp32 -> fp16
884
885static void transfer_activation_chunk_fp32_to_fp16(__fp16 *restrict vtcm_dst, const float *restrict src, uint32_t n_rows, uint32_t k_block, uint32_t k_stride, uint32_t k_valid) {
886    const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);
887    const uint32_t n_rows_tiled  = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS;
888
889    uint32_t r = 0;
890
891    #pragma unroll(2)
892    for (r = 0; r < n_rows_tiled; r += 2) {
893        uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
894        uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
895
896        const float *ptr_in0 = src + (r + 0) * k_stride;
897        const float *ptr_in1 = src + (r + 1) * k_stride;
898
899        uint32_t c = 0;
900        for (; c + 32 <= k_valid; c += 32) {
901            HVX_Vector v0 = *(const HVX_Vector *)(ptr_in0 + c);
902            HVX_Vector v1 = *(const HVX_Vector *)(ptr_in1 + c);
903            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
904
905            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
906            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
907
908            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
909            tile[r1 / 2]     = v_out;
910        }
911        if (c < k_block) {
912            HVX_Vector v0 = *(const HVX_Vector *)(ptr_in0 + c);
913            HVX_Vector v1 = *(const HVX_Vector *)(ptr_in1 + c);
914
915            uint32_t rem = k_valid - c;
916            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
917            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
918            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
919
920            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
921
922            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
923            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
924
925            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
926            tile[r1 / 2]     = v_out;
927        }
928    }
929
930    for (; r < n_rows_padded; r += 2) {
931        uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
932        uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
933
934        const bool row0_valid = r       < n_rows;
935        const bool row1_valid = (r + 1) < n_rows;
936
937        const float *ptr_in0 = row0_valid ? (src + (r + 0) * k_stride) : NULL;
938        const float *ptr_in1 = row1_valid ? (src + (r + 1) * k_stride) : NULL;
939
940        uint32_t c = 0;
941        for (; c + 32 <= k_valid; c += 32) {
942            HVX_Vector v0 = Q6_V_vzero();
943            HVX_Vector v1 = Q6_V_vzero();
944            if (row0_valid) v0 = *(const HVX_Vector *)(ptr_in0 + c);
945            if (row1_valid) v1 = *(const HVX_Vector *)(ptr_in1 + c);
946
947            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
948
949            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
950            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
951
952            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
953            tile[r1 / 2]     = v_out;
954        }
955        if (c < k_block) {
956            HVX_Vector v0 = Q6_V_vzero();
957            HVX_Vector v1 = Q6_V_vzero();
958            if (row0_valid) v0 = *(const HVX_Vector *)(ptr_in0 + c);
959            if (row1_valid) v1 = *(const HVX_Vector *)(ptr_in1 + c);
960
961            uint32_t rem = k_valid - c;
962            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
963            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
964            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
965
966            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
967
968            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
969            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
970
971            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
972            tile[r1 / 2]     = v_out;
973        }
974    }
975}
976
977static void transfer_activation_row_pair_fp32_to_fp16(
978        __fp16 *restrict vtcm_dst,
979        const float *restrict row0,
980        const float *restrict row1,
981        uint32_t r,
982        uint32_t k_block,
983        uint32_t k_valid,
984        bool row0_valid,
985        bool row1_valid) {
986
987    uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
988    uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
989
990    uint32_t c = 0;
991    for (; c + 32 <= k_valid; c += 32) {
992        HVX_Vector v0 = Q6_V_vzero();
993        HVX_Vector v1 = Q6_V_vzero();
994        if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
995        if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
996
997        HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
998
999        uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
1000        uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1001
1002        HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1003        tile[r1 / 2]     = v_out;
1004    }
1005    if (c < k_block) {
1006        HVX_Vector v0 = Q6_V_vzero();
1007        HVX_Vector v1 = Q6_V_vzero();
1008        if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
1009        if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
1010
1011        uint32_t rem = k_valid - c;
1012        HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1013        v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1014        v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1015
1016        HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1017
1018        uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;  // tile column index
1019        uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1020
1021        HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1022        tile[r1 / 2]     = v_out;
1023    }
1024}
1025
1026static void transfer_activation_row_pair_fp32_to_fp16_col_chunk(
1027        __fp16 *restrict vtcm_dst,
1028        const float *restrict row0, // offset by c_first
1029        const float *restrict row1, // offset by c_first
1030        uint32_t r,
1031        uint32_t k_block,
1032        uint32_t c_first,
1033        uint32_t c_len,
1034        uint32_t k_chunk_valid,
1035        bool row0_valid,
1036        bool row1_valid) {
1037
1038    uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
1039    uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
1040
1041    uint32_t c = 0;
1042    for (; c + 32 <= k_chunk_valid; c += 32) {
1043        HVX_Vector v0 = Q6_V_vzero();
1044        HVX_Vector v1 = Q6_V_vzero();
1045        if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
1046        if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
1047
1048        HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1049
1050        uint32_t c0       = (c_first + c) / HTP_MM_HMX_TILE_N_COLS;  // tile column index
1051        uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1052
1053        HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1054        tile[r1 / 2]     = v_out;
1055    }
1056    if (c < c_len) {
1057        HVX_Vector v0 = Q6_V_vzero();
1058        HVX_Vector v1 = Q6_V_vzero();
1059        if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
1060        if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
1061
1062        uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0;
1063        HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1064        v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1065        v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1066
1067        HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1068
1069        uint32_t c0       = (c_first + c) / HTP_MM_HMX_TILE_N_COLS;  // tile column index
1070        uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1071
1072        HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1073        tile[r1 / 2]     = v_out;
1074    }
1075}
1076
1077static void transfer_activation_chunk_fp32_to_fp16_gathered(
1078            __fp16 *restrict vtcm_dst,
1079            const float *restrict src,
1080            uint32_t start_row,
1081            uint32_t vtcm_start_row,
1082            uint32_t n_rows,
1083            uint32_t k_block,
1084            const struct mmid_row_mapping *matrix_rows,
1085            uint32_t cur_a,
1086            uint32_t mapping_stride,
1087            uint32_t ne11,
1088            const struct fastdiv_values * ne11_div,
1089            size_t nb11,
1090            size_t nb12,
1091            uint32_t cne1,
1092            uint32_t k_valid) {
1093    const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);
1094    const uint32_t n_rows_tiled  = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS;
1095
1096    uint32_t r = 0;
1097
1098    #pragma unroll(2)
1099    for (r = 0; r < n_rows_tiled; r += 2) {
1100        uint32_t r_idx0 = start_row + r + 0;
1101        uint32_t r_idx1 = start_row + r + 1;
1102        uint32_t lr = vtcm_start_row + r;           // vtcm-local row
1103        uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
1104        uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
1105
1106        struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];
1107        struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];
1108
1109        uint32_t i11_0 = fastmodulo(mapping0.i1, ne11, ne11_div);
1110        uint32_t i11_1 = fastmodulo(mapping1.i1, ne11, ne11_div);
1111
1112        const float *row0_ptr = (const float *) ((const uint8_t *) src + i11_0 * nb11 + mapping0.i2 * nb12);
1113        const float *row1_ptr = (const float *) ((const uint8_t *) src + i11_1 * nb11 + mapping1.i2 * nb12);
1114
1115        uint32_t c = 0;
1116        for (; c + 32 <= k_valid; c += 32) {
1117            HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c);
1118            HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c);
1119            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1120
1121            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1122            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1123
1124            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1125            tile[r1 / 2]     = v_out;
1126        }
1127        if (c < k_block) {
1128            HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c);
1129            HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c);
1130
1131            uint32_t rem = k_valid - c;
1132            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1133            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1134            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1135
1136            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1137
1138            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1139            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1140
1141            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1142            tile[r1 / 2]     = v_out;
1143        }
1144    }
1145
1146    for (; r < n_rows_padded; r += 2) {
1147        uint32_t lr = vtcm_start_row + r;           // vtcm-local row
1148        uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
1149        uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
1150
1151        const bool row0_valid = (start_row + r + 0) < cne1;
1152        const bool row1_valid = (start_row + r + 1) < cne1;
1153
1154        const float *row0_ptr = NULL;
1155        const float *row1_ptr = NULL;
1156
1157        if (row0_valid) {
1158            struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + (start_row + r + 0)];
1159            uint32_t i11_0 = fastmodulo(mapping0.i1, ne11, ne11_div);
1160            row0_ptr = (const float *) ((const uint8_t *) src + i11_0 * nb11 + mapping0.i2 * nb12);
1161        }
1162        if (row1_valid) {
1163            struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + (start_row + r + 1)];
1164            uint32_t i11_1 = fastmodulo(mapping1.i1, ne11, ne11_div);
1165            row1_ptr = (const float *) ((const uint8_t *) src + i11_1 * nb11 + mapping1.i2 * nb12);
1166        }
1167
1168        uint32_t c = 0;
1169        for (; c + 32 <= k_valid; c += 32) {
1170            HVX_Vector v0 = Q6_V_vzero();
1171            HVX_Vector v1 = Q6_V_vzero();
1172            if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c);
1173            if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c);
1174
1175            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1176
1177            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1178            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1179
1180            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1181            tile[r1 / 2]     = v_out;
1182        }
1183        if (c < k_block) {
1184            HVX_Vector v0 = Q6_V_vzero();
1185            HVX_Vector v1 = Q6_V_vzero();
1186            if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c);
1187            if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c);
1188
1189            uint32_t rem = k_valid - c;
1190            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1191            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1192            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1193
1194            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1195
1196            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1197            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1198
1199            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1200            tile[r1 / 2]     = v_out;
1201        }
1202    }
1203}
1204
1205static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(
1206            __fp16 *restrict vtcm_dst,
1207            const float *restrict src,
1208            uint32_t start_row,
1209            uint32_t vtcm_start_row,
1210            uint32_t n_rows,
1211            uint32_t k_block,
1212            const struct mmid_row_mapping *matrix_rows,
1213            uint32_t cur_a,
1214            uint32_t mapping_stride,
1215            size_t nb12,
1216            uint32_t cne1,
1217            uint32_t k_valid) {
1218    const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);
1219    const uint32_t n_rows_tiled  = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS;
1220
1221    uint32_t r = 0;
1222
1223    #pragma unroll(2)
1224    for (r = 0; r < n_rows_tiled; r += 2) {
1225        uint32_t r_idx0 = start_row + r + 0;
1226        uint32_t r_idx1 = start_row + r + 1;
1227        uint32_t lr = vtcm_start_row + r;           // vtcm-local row
1228        uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
1229        uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
1230
1231        struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];
1232        struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];
1233
1234        const float *row0_ptr = (const float *) ((const uint8_t *) src + mapping0.i2 * nb12);
1235        const float *row1_ptr = (const float *) ((const uint8_t *) src + mapping1.i2 * nb12);
1236
1237        uint32_t c = 0;
1238        for (; c + 32 <= k_valid; c += 32) {
1239            HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c);
1240            HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c);
1241            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1242
1243            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1244            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1245
1246            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1247            tile[r1 / 2]     = v_out;
1248        }
1249        if (c < k_block) {
1250            HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c);
1251            HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c);
1252
1253            uint32_t rem = k_valid - c;
1254            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1255            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1256            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1257
1258            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1259
1260            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1261            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1262
1263            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1264            tile[r1 / 2]     = v_out;
1265        }
1266    }
1267
1268    for (; r < n_rows_padded; r += 2) {
1269        uint32_t lr = vtcm_start_row + r;           // vtcm-local row
1270        uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS;  // tile row index
1271        uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS;  // intra-tile row idx
1272
1273        const bool row0_valid = (start_row + r + 0) < cne1;
1274        const bool row1_valid = (start_row + r + 1) < cne1;
1275
1276        const float *row0_ptr = NULL;
1277        const float *row1_ptr = NULL;
1278
1279        if (row0_valid) {
1280            struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + (start_row + r + 0)];
1281            row0_ptr = (const float *) ((const uint8_t *) src + mapping0.i2 * nb12);
1282        }
1283        if (row1_valid) {
1284            struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + (start_row + r + 1)];
1285            row1_ptr = (const float *) ((const uint8_t *) src + mapping1.i2 * nb12);
1286        }
1287
1288        uint32_t c = 0;
1289        for (; c + 32 <= k_valid; c += 32) {
1290            HVX_Vector v0 = Q6_V_vzero();
1291            HVX_Vector v1 = Q6_V_vzero();
1292            if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c);
1293            if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c);
1294
1295            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1296
1297            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1298            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1299
1300            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1301            tile[r1 / 2]     = v_out;
1302        }
1303        if (c < k_block) {
1304            HVX_Vector v0 = Q6_V_vzero();
1305            HVX_Vector v1 = Q6_V_vzero();
1306            if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c);
1307            if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c);
1308
1309            uint32_t rem = k_valid - c;
1310            HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
1311            v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
1312            v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
1313
1314            HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
1315
1316            uint32_t c0       = c / HTP_MM_HMX_TILE_N_COLS;
1317            uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
1318
1319            HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
1320            tile[r1 / 2]     = v_out;
1321        }
1322    }
1323}
1324
1325static void transfer_output_chunk_fp16_to_fp32_scattered(
1326            float *restrict dst,
1327            const __fp16 *restrict vtcm_src,
1328            uint32_t start_row,
1329            uint32_t vtcm_start_row,
1330            uint32_t n_rows,
1331            uint32_t n_cols,
1332            const struct mmid_row_mapping *matrix_rows,
1333            uint32_t cur_a,
1334            uint32_t mapping_stride,
1335            size_t dst_nb1,
1336            size_t dst_nb2,
1337            uint32_t cne1) {
1338    assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0);
1339    const size_t tile_row_stride = (n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS;
1340
1341    const HVX_Vector one = hvx_vec_splat_f16(1.0);
1342
1343    for (size_t r = 0; r < n_rows; r += 2) {
1344        uint32_t r_idx0 = start_row + r + 0;
1345        uint32_t r_idx1 = start_row + r + 1;
1346        uint32_t     lr = vtcm_start_row + r;                 // vtcm-local row
1347        const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS);
1348        const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2;  // index of the row pair within the tile
1349        const __fp16 *row_base = vtcm_src + r0 * tile_row_stride;
1350
1351        if (r_idx0 >= cne1) break;
1352
1353        struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];
1354        float *output_row0 = (float *) ((uint8_t *) dst + mapping0.i1 * dst_nb1 + mapping0.i2 * dst_nb2);
1355
1356        float *output_row1 = NULL;
1357        if (r_idx1 < cne1) {
1358            struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];
1359            output_row1 = (float *) ((uint8_t *) dst + mapping1.i1 * dst_nb1 + mapping1.i2 * dst_nb2);
1360        }
1361
1362        #pragma unroll(4)
1363        for (size_t c = 0; c < (size_t)n_cols; c += HTP_MM_HMX_TILE_N_COLS) {
1364            const size_t c0 = c / HTP_MM_HMX_TILE_N_COLS;
1365            const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS;
1366            HVX_Vector v = ((const HVX_Vector *) tile)[r1];
1367            HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);
1368
1369            HVX_Vector *pv_out0 = (HVX_Vector *) (output_row0 + c);
1370            HVX_Vector *pv_out1 = output_row1 ? (HVX_Vector *) (output_row1 + c) : NULL;
1371
1372            *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
1373            if (pv_out1) {
1374                *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
1375            }
1376        }
1377    }
1378}