hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)192067b722026-08-26

1#ifndef HVX_QUANT_H
2#define HVX_QUANT_H
3
4#include <math.h>
5#include <stdint.h>
6#include <string.h>
7
8#include "hvx-arith.h"
9#include "hvx-base.h"
10#include "hvx-reduce.h"
11#include "hvx-repl.h"
12#include "hvx-utils.h"
13
14#ifndef GGML_COMMON_DECL_C
15#define GGML_COMMON_DECL_C
16#endif
17#include "ggml-common.h"
18#include "ggml-impl.h"
19
20static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) {
21    const int nb = n / QK8_0;
22    block_q8_0 * dst = (block_q8_0 *) dst_ptr;
23    HVX_Vector zero = Q6_V_vzero();
24
25    int i = 0;
26    for (; i + 3 < nb; i += 4) {
27        HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0);
28
29        HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
30        HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
31        HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
32        HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
33
34        HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
35        HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
36        HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
37        HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
38
39        HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
40        HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
41        HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
42        HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
43
44        HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
45        HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
46
47        HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
48        HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
49
50        HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
51        HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
52        HVX_Vector vd01_hf   = Q6_Vhf_equals_Vqf16(vd01_qf16);
53        HVX_Vector vd23_hf   = Q6_Vhf_equals_Vqf16(vd23_qf16);
54
55        HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
56        HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
57        vx01_hf              = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
58        vx23_hf              = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
59
60        HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
61        HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
62        HVX_Vector vx_i8    = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);
63
64        hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf);
65        hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8);
66
67        hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64));
68        hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32));
69
70        hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf);
71        hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64));
72
73        hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64));
74        hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96));
75    }
76
77    for (; i < nb; i++) {
78        const float * block_src = src_ptr + i * QK8_0;
79        HVX_Vector vx = *(const HVX_UVector *) block_src;
80        HVX_Vector v_abs = hvx_vec_abs_f32(vx);
81        HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs);
82        float amax = hvx_vec_get_f32(v_max);
83
84        const float d = amax / 127.0f;
85        const float id = d ? (1.0f / d) : 0.0f;
86        dst[i].d = GGML_FP32_TO_FP16(d);
87
88        HVX_Vector vid = hvx_vec_splat_f32(id);
89        HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid);
90        HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero);
91        HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf)));
92        HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf);
93        HVX_Vector v_i8  = Q6_Vb_vpack_VhVh_sat(zero, v_i16);
94
95        hvx_vec_store_u(dst[i].qs, 32, v_i8);
96    }
97}
98
99static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
100    const int nb = n / QK8_0;
101    const block_q8_0 * src = (const block_q8_0 *) src_ptr;
102
103    for (int i = 0; i < nb; i++) {
104        HVX_Vector vd_f16     = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
105        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
106        HVX_Vector vd         = Q6_V_lo_W(vp_f32);
107
108        HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
109
110        HVX_VectorPair p16   = Q6_Wh_vunpack_Vb(vq_i8);
111        HVX_Vector     v_i16 = Q6_V_lo_W(p16);
112        HVX_VectorPair p32   = Q6_Ww_vunpack_Vh(v_i16);
113        HVX_Vector     v_i32 = Q6_V_lo_W(p32);
114
115        HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
116        HVX_Vector res   = hvx_vec_mul_f32_f32(v_f32, vd);
117
118        float * block_dst = dst_ptr + i * QK8_0;
119        hvx_vmem(block_dst) = res;
120    }
121}
122
123static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) {
124    const int nb = n / QK8_0;
125    const block_q8_0 * src = (const block_q8_0 *) src_ptr;
126
127    for (int i = nb - 1; i >= 0; i--) {
128        HVX_Vector vd_f16     = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
129        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
130        HVX_Vector vd         = Q6_V_lo_W(vp_f32);
131
132        HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
133
134        HVX_VectorPair p16   = Q6_Wh_vunpack_Vb(vq_i8);
135        HVX_Vector     v_i16 = Q6_V_lo_W(p16);
136        HVX_VectorPair p32   = Q6_Ww_vunpack_Vh(v_i16);
137        HVX_Vector     v_i32 = Q6_V_lo_W(p32);
138
139        HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
140        HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd);
141
142        HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero());
143
144        __fp16 * block_dst = dst_ptr + i * QK8_0;
145        hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16);
146    }
147}
148
149static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
150    const int nb = n / 32;
151    const _Float16 * src = (const _Float16 *) src_ptr;
152
153    for (int i = 0; i < nb; i++) {
154        HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32);
155        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
156        HVX_Vector res = Q6_V_lo_W(vp_f32);
157
158        float * block_dst = dst_ptr + i * 32;
159        hvx_vmem(block_dst) = res;
160    }
161}
162
163
164
165#endif // HVX_QUANT_H