char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
eafe15a5e
char/aliceai-t5-moe
1#pragma clang diagnostic ignored "-Wunused-variable" 2#pragma clang diagnostic ignored "-Wunused-function" 3#pragma clang diagnostic ignored "-Wunused-but-set-variable" 4 5#include <HAP_farf.h> 6#include <HAP_perf.h> 7 8#include <math.h> 9#include <string.h> 10 11#include "dma-queue.h" 12#include "work-queue.h" 13#include "hvx-utils.h" 14#include "hex-utils.h" 15#include "hvx-copy.h" 16#include "hvx-quant.h" 17 18#define GGML_COMMON_DECL_C 19#include "ggml-common.h" 20 21#include "hex-common.h" 22#include "htp-ctx.h" 23#include "htp-ops.h" 24#include "htp-tensor.h" 25#include "htp/set-rows-ops.h" 26 27#define set_rows_preamble \ 28 const uint32_t ne00 = octx->src[0]->ne[0]; \ 29 const uint32_t ne01 = octx->src[0]->ne[1]; \ 30 const uint32_t ne02 = octx->src[0]->ne[2]; \ 31 const uint32_t ne03 = octx->src[0]->ne[3]; \ 32 \ 33 const uint32_t ne10 = octx->src[1]->ne[0]; \ 34 const uint32_t ne11 = octx->src[1]->ne[1]; \ 35 const uint32_t ne12 = octx->src[1]->ne[2]; \ 36 const uint32_t ne13 = octx->src[1]->ne[3]; \ 37 \ 38 const uint32_t nb01 = octx->src[0]->nb[1]; \ 39 const uint32_t nb02 = octx->src[0]->nb[2]; \ 40 const uint32_t nb03 = octx->src[0]->nb[3]; \ 41 \ 42 const uint32_t nb10 = octx->src[1]->nb[0]; \ 43 const uint32_t nb11 = octx->src[1]->nb[1]; \ 44 const uint32_t nb12 = octx->src[1]->nb[2]; \ 45 \ 46 const uint32_t nb1 = octx->dst->nb[1]; \ 47 const uint32_t nb2 = octx->dst->nb[2]; \ 48 const uint32_t nb3 = octx->dst->nb[3]; \ 49 \ 50 const uint32_t ne0 = octx->dst->ne[0]; \ 51 const uint32_t ne1 = octx->dst->ne[1]; \ 52 const uint32_t ne2 = octx->dst->ne[2]; \ 53 const uint32_t ne3 = octx->dst->ne[3]; \ 54 \ 55 const uint32_t nr = ne01; 56 57struct set_rows_context { 58struct htp_ops_context * octx ; 59const struct htp_set_rows_kernel_params * kparams ; 60struct htp_set_rows_vtcm_layout vtcm_layout ; 61uint8_t * vtcm_base ; 62uint32_t task_start ; 63uint32_t tasks ; 64uint32_t tasks_per_thread ; 65}; 66 67#define SET_ROWS_THREAD_DMA_FN (TYPE_NAME ,IDX_TYPE ,COMPUTE_EXPR ) \ 68static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ 69 struct set_rows_context * srctx = (struct set_rows_context *)data; \ 70 struct htp_ops_context * octx = srctx->octx; \ 71 const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ 72 set_rows_preamble; \ 73 struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ 74 const uint32_t dr = srctx->tasks_per_thread; \ 75 const uint32_t ir0 = srctx->task_start + dr * ith; \ 76 if (ir0 >= srctx->task_start + srctx->tasks) { \ 77 return; \ 78 } \ 79 const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ 80 dma_queue * dma_queue = octx->ctx->dma[ith]; \ 81 const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ 82 uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ 83 uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \ 84 const uint32_t src0_row_size = ne00 * sizeof(float); \ 85 const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ 86 const uint32_t nrows_per_thread = ir1 - ir0; \ 87 const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ 88 uint32_t pi_step = 0; \ 89 uint32_t pi02 = 0; \ 90 uint32_t pi03 = 0; \ 91 for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \ 92 uint32_t i = ir0 + pi_step; \ 93 const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \ 94 dma_queue_push(dma_queue, \ 95 dma_make_ptr((void *)octx->dst->data, \ 96 vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ 97 dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ 98 dma_queue_push(dma_queue, \ 99 dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \ 100 (const void *)src0_ptr), \ 101 vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ 102 pi_step++; \ 103 if (pi_step == nrows_per_thread) { \ 104 pi_step = 0; \ 105 pi02++; \ 106 if (pi02 == ne02) { \ 107 pi02 = 0; \ 108 pi03++; \ 109 } \ 110 } \ 111 } \ 112 uint32_t ci_step = 0; \ 113 uint32_t ci02 = 0; \ 114 uint32_t ci03 = 0; \ 115 uint32_t ci11_base = 0; \ 116 uint32_t ci12_base = 0; \ 117 for (uint32_t step = 0; step < total_steps; ++step) { \ 118 void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \ 119 void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \ 120 uint32_t i = ir0 + ci_step; \ 121 const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \ 122 const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \ 123 const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \ 124 const uint32_t target_i1 = (uint32_t)i1; \ 125 htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \ 126 if (valid_i1) { \ 127 COMPUTE_EXPR; \ 128 } \ 129 htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \ 130 if (valid_i1) { \ 131 const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \ 132 dma_queue_push(dma_queue, \ 133 dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \ 134 dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \ 135 } else { \ 136 dma_queue_push(dma_queue, \ 137 dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \ 138 dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ 139 } \ 140 const uint32_t next_step = step + 2; \ 141 if (next_step < total_steps) { \ 142 uint32_t ni = ir0 + pi_step; \ 143 const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \ 144 dma_queue_push(dma_queue, \ 145 dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \ 146 vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ 147 pi_step++; \ 148 if (pi_step == nrows_per_thread) { \ 149 pi_step = 0; \ 150 pi02++; \ 151 if (pi02 == ne02) { \ 152 pi02 = 0; \ 153 pi03++; \ 154 } \ 155 } \ 156 } \ 157 ci_step++; \ 158 if (ci_step == nrows_per_thread) { \ 159 ci_step = 0; \ 160 ci02++; \ 161 ci11_base++; \ 162 if (ci11_base == ne11) { \ 163 ci11_base = 0; \ 164 } \ 165 if (ci02 == ne02) { \ 166 ci02 = 0; \ 167 ci03++; \ 168 ci12_base++; \ 169 if (ci12_base == ne12) { \ 170 ci12_base = 0; \ 171 } \ 172 } \ 173 } \ 174 } \ 175 dma_queue_flush(dma_queue); \ 176} 177 178SET_ROWS_THREAD_DMA_FN (f32 ,int32_t , {hvx_copy_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,ne00 ); }) 179SET_ROWS_THREAD_DMA_FN (f32 ,int64_t , {hvx_copy_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,ne00 ); }) 180 181SET_ROWS_THREAD_DMA_FN (f16 ,int32_t , {hvx_copy_f16_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,ne00 ); }) 182SET_ROWS_THREAD_DMA_FN (f16 ,int64_t , {hvx_copy_f16_f32_uu ((uint8_t * )dst_spad , (const uint8_t * )src_spad ,ne00 ); }) 183 184SET_ROWS_THREAD_DMA_FN (q8_0 ,int32_t , {hvx_quantize_row_q8_0_f32 (dst_spad , (const float * )src_spad ,ne00 ); }) 185SET_ROWS_THREAD_DMA_FN (q8_0 ,int64_t , {hvx_quantize_row_q8_0_f32 (dst_spad , (const float * )src_spad ,ne00 ); }) 186 187int op_set_rows (struct htp_ops_context * octx ) { 188const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params * )octx -> kernel_params ; 189set_rows_preamble ; 190 191if (octx -> src [0 ]-> type != HTP_TYPE_F32 ) { 192return HTP_STATUS_NO_SUPPORT ; 193 } 194 195if (octx -> dst -> type != HTP_TYPE_F32 && octx -> dst -> type != HTP_TYPE_F16 && octx -> dst -> type != HTP_TYPE_Q8_0 ) { 196return HTP_STATUS_NO_SUPPORT ; 197 } 198 199if (octx -> flags & HTP_OPFLAGS_SKIP_COMPUTE ) { 200return HTP_STATUS_OK ; 201 } 202 203const struct htp_tensor * dst = octx -> dst ; 204const uint32_t total_tasks = kparams -> total_tasks ; 205 206uint32_t task_start = 0 ; 207uint32_t tasks = total_tasks ; 208 209if (octx -> ctx -> mdev .count > 1 ) { 210const bool can_split = htp_tensor_mdev_data_aligned (dst )&& (dst -> nb [1 ]& (HTP_TENSOR_MDEV_LINE_SIZE - 1 ))== 0 && !htp_tensor_is_permuted (dst ); 211const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition (total_tasks ,can_split ?1 :0 ,octx -> ctx -> mdev .idx ,octx -> ctx -> mdev .count ,& octx -> ctx -> mdev .count_div ); 212task_start = range .start ; 213tasks = range .count ; 214 } 215 216if (tasks == 0 ) { 217return HTP_STATUS_OK ; 218 } 219 220if (!htp_ops_context_set_n_threads (octx , (uint32_t )kparams -> n_threads )) { 221return HTP_STATUS_INVAL_PARAMS ; 222 } 223 224const uint32_t n_threads = octx -> n_threads ; 225 226// l2fetch the src1 (indices) tensor in the main thread 227hex_l2fetch_block ((const void * )octx -> src [1 ]-> data ,octx -> src [1 ]-> ne [3 ]* octx -> src [1 ]-> nb [3 ]); 228 229struct set_rows_context srctx ; 230srctx .octx = octx ; 231srctx .kparams = kparams ; 232srctx .task_start = task_start ; 233srctx .tasks = tasks ; 234srctx .tasks_per_thread = fastdiv (tasks + n_threads - 1 ,& octx -> n_threads_div ); 235 236htp_set_rows_vtcm_layout_build (& srctx .vtcm_layout ,octx -> dst -> type ,ne00 ,n_threads ); 237srctx .vtcm_base = (uint8_t * )octx -> ctx -> vtcm_base ; 238 239work_queue_func_t q_func = NULL ; 240const bool is_i32 = (octx -> src [1 ]-> type == HTP_TYPE_I32 ); 241 242switch (octx -> dst -> type ) { 243case HTP_TYPE_F32 :q_func = is_i32 ?set_rows_thread_dma_f32_int32_t :set_rows_thread_dma_f32_int64_t ;break ; 244case HTP_TYPE_F16 :q_func = is_i32 ?set_rows_thread_dma_f16_int32_t :set_rows_thread_dma_f16_int64_t ;break ; 245case HTP_TYPE_Q8_0 :q_func = is_i32 ?set_rows_thread_dma_q8_0_int32_t :set_rows_thread_dma_q8_0_int64_t ;break ; 246default :return HTP_STATUS_NO_SUPPORT ; 247 } 248 249FARF (HIGH ,"set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n" , 250octx -> src [0 ]-> ne [0 ],octx -> src [0 ]-> ne [1 ],octx -> src [0 ]-> ne [2 ],octx -> src [0 ]-> ne [3 ], 251octx -> src [1 ]-> ne [0 ],octx -> src [1 ]-> ne [1 ],octx -> src [1 ]-> ne [2 ],octx -> src [1 ]-> ne [3 ], 252octx -> dst -> ne [0 ],octx -> dst -> ne [1 ],octx -> dst -> ne [2 ],octx -> dst -> ne [3 ], 253srctx .vtcm_layout .src0_bytes_per_thread * n_threads , 254srctx .vtcm_layout .dst_bytes_per_thread * n_threads , 255n_threads ); 256 257work_queue_run (octx -> ctx -> work_queue ,q_func ,& srctx ,n_threads ); 258 259return HTP_STATUS_OK ; 260}