char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)192067b72

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.2 KiB77 linesraw
1#ifndef HTP_GET_ROWS_OPS_H
2#define HTP_GET_ROWS_OPS_H
3
4#include "hex-fastdiv.h"
5
6struct htp_get_rows_kernel_params {
7    int32_t  n_threads;
8    int32_t  use_dma;
9    int32_t  chunks_per_row;
10    int32_t  chunk_size;
11    int32_t  total_tasks;
12    int32_t  tasks_per_thread;
13    int32_t  vtcm_size;
14
15    // Fastdiv helpers
16    struct fastdiv_values div_ne10;
17    struct fastdiv_values div_ne10_ne11;
18    struct fastdiv_values div_chunks_per_row;
19    struct fastdiv_values div_ne02;
20    struct fastdiv_values div_ne03;
21};
22
23struct htp_get_rows_vtcm_layout {
24    size_t total_bytes;
25    size_t off_src0;
26    size_t off_dst;
27
28    size_t src0_bytes_per_thread;
29    size_t dst_bytes_per_thread;
30
31    size_t src0_spad_half_size;
32    size_t dst_spad_half_size;
33};
34
35static inline void htp_get_rows_vtcm_layout_build(
36    struct htp_get_rows_vtcm_layout * vtcm_layout,
37    int type,
38    uint32_t ne00,
39    uint32_t n_threads) {
40
41    uint32_t src0_row_size = 0;
42    switch (type) {
43        case 0: // HTP_TYPE_F32
44            src0_row_size = ne00 * 4;
45            break;
46        case 1: // HTP_TYPE_F16
47            src0_row_size = ne00 * 2;
48            break;
49        case 8: // HTP_TYPE_Q8_0
50            src0_row_size = (ne00 / 32) * 34;
51            break;
52        default:
53            src0_row_size = 0;
54            break;
55    }
56
57    size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
58    size_t dst_row_size_aligned  = (ne00 * sizeof(float) + 255) & ~255;
59
60    vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
61    vtcm_layout->dst_spad_half_size  = dst_row_size_aligned;
62
63    vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
64    vtcm_layout->dst_bytes_per_thread  = dst_row_size_aligned * 2;
65
66    vtcm_layout->off_src0 = 0;
67    vtcm_layout->off_dst  = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
68    vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
69}
70
71#if defined(__cplusplus)
72static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
73#else
74_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
75#endif
76
77#endif // HTP_GET_ROWS_OPS_H