char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
eafe15a5e
char/aliceai-t5-moe
1#pragma clang diagnostic ignored "-Wunused-variable" 2#pragma clang diagnostic ignored "-Wunused-function" 3#pragma clang diagnostic ignored "-Wunused-but-set-variable" 4 5#include <HAP_farf.h> 6#include <HAP_perf.h> 7 8#include <math.h> 9#include <string.h> 10 11#define GGML_COMMON_DECL_C 12#include "ggml-common.h" 13#include "hex-common.h" 14#include "htp-ctx.h" 15#include "htp-ops.h" 16#include "htp-tensor.h" 17#include "hvx-utils.h" 18#include "hvx-quant.h" 19#include "get-rows-ops.h" 20#include "work-queue.h" 21 22struct get_rows_context { 23struct htp_ops_context * octx ; 24const struct htp_get_rows_kernel_params * kparams ; 25struct htp_get_rows_vtcm_layout vtcm_layout ; 26uint8_t * vtcm_base ; 27uint32_t task_start ; 28uint32_t tasks ; 29uint32_t tasks_per_thread ; 30}; 31 32#define get_rows_preamble \ 33 const uint32_t ne00 = octx->src[0]->ne[0]; \ 34 const uint32_t ne01 = octx->src[0]->ne[1]; \ 35 const uint32_t ne02 = octx->src[0]->ne[2]; \ 36 const uint32_t ne03 = octx->src[0]->ne[3]; \ 37 \ 38 const uint32_t ne10 = octx->src[1]->ne[0]; \ 39 const uint32_t ne11 = octx->src[1]->ne[1]; \ 40 const uint32_t ne12 = octx->src[1]->ne[2]; \ 41 const uint32_t ne13 = octx->src[1]->ne[3]; \ 42 \ 43 const uint32_t ne0 = octx->dst->ne[0]; \ 44 const uint32_t ne1 = octx->dst->ne[1]; \ 45 const uint32_t ne2 = octx->dst->ne[2]; \ 46 const uint32_t ne3 = octx->dst->ne[3]; \ 47 \ 48 const uint32_t nb01 = octx->src[0]->nb[1]; \ 49 const uint32_t nb02 = octx->src[0]->nb[2]; \ 50 const uint32_t nb03 = octx->src[0]->nb[3]; \ 51 \ 52 const uint32_t nb10 = octx->src[1]->nb[0]; \ 53 const uint32_t nb11 = octx->src[1]->nb[1]; \ 54 const uint32_t nb12 = octx->src[1]->nb[2]; \ 55 \ 56 const uint32_t nb1 = octx->dst->nb[1]; \ 57 const uint32_t nb2 = octx->dst->nb[2]; \ 58 const uint32_t nb3 = octx->dst->nb[3]; \ 59 \ 60 const uint32_t nr = ne10 * ne11 * ne12; 61 62#define GET_ROWS_THREAD_ST_FN (IDX_TYPE ) \ 63static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ 64 struct get_rows_context * grctx = (struct get_rows_context *)data; \ 65 struct htp_ops_context * octx = grctx->octx; \ 66 const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ 67 get_rows_preamble; \ 68 const uint32_t dr = grctx->tasks_per_thread; \ 69 const uint32_t ir0 = grctx->task_start + dr * ith; \ 70 if (ir0 >= grctx->task_start + grctx->tasks) { \ 71 return; \ 72 } \ 73 const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ 74 const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ 75 dma_queue * dma_queue = octx->ctx->dma[ith]; \ 76 for (uint32_t i = ir0; i < ir1; ++i) { \ 77 const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \ 78 const uint32_t rem = i - i12 * ne11 * ne10; \ 79 const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ 80 const uint32_t i10 = rem - i11 * ne10; \ 81 const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ 82 const uint32_t i01 = (uint32_t)*src1_ptr; \ 83 assert(i01 < ne01); \ 84 const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ 85 const uint32_t i02 = i11 - q02 * ne02; \ 86 const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ 87 const uint32_t i03 = i12 - q03 * ne03; \ 88 const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \ 89 const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \ 90 while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \ 91 row_size_bytes, 1)) { \ 92 dma_queue_pop(dma_queue); \ 93 } \ 94 } \ 95 dma_queue_flush(dma_queue); \ 96} 97 98GET_ROWS_THREAD_ST_FN (int32_t ) 99GET_ROWS_THREAD_ST_FN (int64_t ) 100 101#define GET_ROWS_THREAD_DT_FN (TYPE_NAME ,SRC0_SIZE_EXPR ,IDX_TYPE ,COMPUTE_EXPR ) \ 102static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ 103 struct get_rows_context * grctx = (struct get_rows_context *)data; \ 104 struct htp_ops_context * octx = grctx->octx; \ 105 const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ 106 get_rows_preamble; \ 107 struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ 108 const uint32_t dr = grctx->tasks_per_thread; \ 109 const uint32_t ir0 = grctx->task_start + dr * ith; \ 110 if (ir0 >= grctx->task_start + grctx->tasks) { \ 111 return; \ 112 } \ 113 const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ 114 const uint32_t chunks_per_row = kparams->chunks_per_row; \ 115 const uint32_t chunk_size = kparams->chunk_size; \ 116 dma_queue * dma_queue = octx->ctx->dma[ith]; \ 117 const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \ 118 uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ 119 uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ 120 for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \ 121 const uint32_t i = ir0 + step; \ 122 const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ 123 const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ 124 const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ 125 const uint32_t rem = row_idx - i12 * ne11 * ne10; \ 126 const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ 127 const uint32_t i10 = rem - i11 * ne10; \ 128 const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \ 129 const uint32_t i01 = (uint32_t)*src1_ptr; \ 130 assert(i01 < ne01); \ 131 const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \ 132 const uint32_t i02 = i11 - q02 * ne02; \ 133 const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \ 134 const uint32_t i03 = i12 - q03 * ne03; \ 135 const uint32_t offset = chunk_idx * chunk_size; \ 136 const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ 137 const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \ 138 const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ 139 const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \ 140 dma_queue_push(dma_queue, \ 141 dma_make_ptr((void *)(uintptr_t)octx->dst->data, \ 142 vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ 143 cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \ 144 dma_queue_push(dma_queue, \ 145 dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ 146 (const void *)src0_ptr), \ 147 vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \ 148 } \ 149 for (uint32_t step = 0; step < ir1 - ir0; ++step) { \ 150 const uint32_t i = ir0 + step; \ 151 void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ 152 void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ 153 const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \ 154 const uint32_t chunk_idx = i - row_idx * chunks_per_row; \ 155 const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \ 156 const uint32_t rem = row_idx - i12 * ne11 * ne10; \ 157 const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \ 158 const uint32_t i10 = rem - i11 * ne10; \ 159 const uint32_t offset = chunk_idx * chunk_size; \ 160 const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \ 161 const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \ 162 htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \ 163 COMPUTE_EXPR; \ 164 htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \ 165 const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \ 166 dma_queue_push(dma_queue, \ 167 dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ 168 cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \ 169 const uint32_t next_step = step + 2; \ 170 if (next_step < ir1 - ir0) { \ 171 const uint32_t pi = ir0 + next_step; \ 172 const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \ 173 const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \ 174 const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \ 175 const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \ 176 const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \ 177 const uint32_t pi10 = prem - pi11 * ne10; \ 178 const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \ 179 const uint32_t pi01 = (uint32_t)*psrc1_ptr; \ 180 assert(pi01 < ne01); \ 181 const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \ 182 const uint32_t pi02 = pi11 - pq02 * ne02; \ 183 const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \ 184 const uint32_t pi03 = pi12 - pq03 * ne03; \ 185 const uint32_t poffset = pchunk_idx * chunk_size; \ 186 const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \ 187 const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \ 188 const uintptr_t psrc0_ptr = \ 189 octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \ 190 dma_queue_push(dma_queue, \ 191 dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ 192 vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \ 193 } \ 194 } \ 195 dma_queue_flush(dma_queue); \ 196} 197 198#define F32_BYTES (n ) ((n) * sizeof(float)) 199#define F16_BYTES (n ) ((n) * sizeof(__fp16)) 200#define Q8_0_BYTES (n ) (((n) / 32) * sizeof(block_q8_0)) 201 202GET_ROWS_THREAD_DT_FN (f32 ,F32_BYTES ,int32_t , {if (cur_elems > 0 )hvx_copy_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,cur_elems ); }) 203GET_ROWS_THREAD_DT_FN (f32 ,F32_BYTES ,int64_t , {if (cur_elems > 0 )hvx_copy_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,cur_elems ); }) 204 205GET_ROWS_THREAD_DT_FN (f16 ,F16_BYTES ,int32_t , {hvx_dequantize_row_f16_f32 ((float * )dst_spad ,src_spad ,ne00 ); }) 206GET_ROWS_THREAD_DT_FN (f16 ,F16_BYTES ,int64_t , {hvx_dequantize_row_f16_f32 ((float * )dst_spad ,src_spad ,ne00 ); }) 207 208GET_ROWS_THREAD_DT_FN (q8_0 ,Q8_0_BYTES ,int32_t , {hvx_dequantize_row_q8_0_f32 ((float * )dst_spad ,src_spad ,ne00 ); }) 209GET_ROWS_THREAD_DT_FN (q8_0 ,Q8_0_BYTES ,int64_t , {hvx_dequantize_row_q8_0_f32 ((float * )dst_spad ,src_spad ,ne00 ); }) 210 211int op_get_rows (struct htp_ops_context * octx ) { 212const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params * )octx -> kernel_params ; 213 214if (octx -> src [0 ]-> type != HTP_TYPE_F32 && 215octx -> src [0 ]-> type != HTP_TYPE_F16 && 216octx -> src [0 ]-> type != HTP_TYPE_Q8_0 ) { 217return HTP_STATUS_NO_SUPPORT ; 218 } 219 220if (octx -> dst -> type != HTP_TYPE_F32 ) { 221return HTP_STATUS_NO_SUPPORT ; 222 } 223 224if (octx -> src [1 ]-> type != HTP_TYPE_I32 && octx -> src [1 ]-> type != HTP_TYPE_I64 ) { 225return HTP_STATUS_NO_SUPPORT ; 226 } 227 228if (octx -> flags & HTP_OPFLAGS_SKIP_COMPUTE ) { 229return HTP_STATUS_OK ; 230 } 231 232const struct htp_tensor * dst = octx -> dst ; 233const uint32_t total_tasks = kparams -> total_tasks ; 234const size_t dst_row_size = htp_tensor_get_row_size (dst -> type ,dst -> ne [0 ]); 235 236uint32_t task_start = 0 ; 237uint32_t tasks = total_tasks ; 238 239if (octx -> ctx -> mdev .count > 1 ) { 240uint32_t tasks_per_chunk = 1 ; 241htp_tensor_mdev_rows_per_chunk (dst ,dst_row_size /dst -> ne [0 ], (uint32_t )dst_row_size ,& tasks_per_chunk ); 242const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition (total_tasks ,tasks_per_chunk ,octx -> ctx -> mdev .idx ,octx -> ctx -> mdev .count ,& octx -> ctx -> mdev .count_div ); 243task_start = range .start ; 244tasks = range .count ; 245 } 246 247if (tasks == 0 ) { 248return HTP_STATUS_OK ; 249 } 250 251if (!htp_ops_context_set_n_threads (octx , (uint32_t )kparams -> n_threads )) { 252return HTP_STATUS_INVAL_PARAMS ; 253 } 254 255const uint32_t n_threads = octx -> n_threads ; 256 257struct get_rows_context grctx ; 258grctx .octx = octx ; 259grctx .kparams = kparams ; 260grctx .vtcm_base = (uint8_t * )octx -> ctx -> vtcm_base ; 261grctx .task_start = task_start ; 262grctx .tasks = tasks ; 263grctx .tasks_per_thread = fastdiv (tasks + n_threads - 1 ,& octx -> n_threads_div ); 264 265const uint32_t ne00 = octx -> src [0 ]-> ne [0 ]; 266htp_get_rows_vtcm_layout_build (& grctx .vtcm_layout ,octx -> src [0 ]-> type ,ne00 ,n_threads ); 267 268const bool is_i32 = (octx -> src [1 ]-> type == HTP_TYPE_I32 ); 269 270work_queue_func_t q_func = NULL ; 271if (kparams -> use_dma ) { 272q_func = (work_queue_func_t )(is_i32 ?get_rows_thread_st_int32_t :get_rows_thread_st_int64_t ); 273 }else { 274switch (octx -> src [0 ]-> type ) { 275case HTP_TYPE_F32 :q_func = (work_queue_func_t )(is_i32 ?get_rows_thread_f32_int32_t :get_rows_thread_f32_int64_t );break ; 276case HTP_TYPE_F16 :q_func = (work_queue_func_t )(is_i32 ?get_rows_thread_f16_int32_t :get_rows_thread_f16_int64_t );break ; 277case HTP_TYPE_Q8_0 :q_func = (work_queue_func_t )(is_i32 ?get_rows_thread_q8_0_int32_t :get_rows_thread_q8_0_int64_t );break ; 278default :return HTP_STATUS_NO_SUPPORT ; 279 } 280 } 281 282FARF (HIGH ,"get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n" , 283octx -> src [0 ]-> ne [0 ],octx -> src [0 ]-> ne [1 ],octx -> src [0 ]-> ne [2 ],octx -> src [0 ]-> ne [3 ], 284octx -> src [1 ]-> ne [0 ],octx -> src [1 ]-> ne [1 ],octx -> src [1 ]-> ne [2 ],octx -> src [1 ]-> ne [3 ], 285octx -> dst -> ne [0 ],octx -> dst -> ne [1 ],octx -> dst -> ne [2 ],octx -> dst -> ne [3 ], 286grctx .vtcm_layout .src0_bytes_per_thread * n_threads , 287grctx .vtcm_layout .dst_bytes_per_thread * n_threads , 288kparams -> use_dma ,n_threads ); 289 290work_queue_run (octx -> ctx -> work_queue ,q_func ,& grctx ,n_threads ); 291return HTP_STATUS_OK ; 292}