char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)192067b72

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB74 linesraw
1#ifndef HTP_SET_ROWS_OPS_H
2#define HTP_SET_ROWS_OPS_H
3
4#include "hex-fastdiv.h"
5
6struct htp_set_rows_kernel_params {
7    int32_t  n_threads;
8    int32_t  total_tasks;
9    int32_t  tasks_per_thread;
10    int32_t  vtcm_size;
11
12    // Fastdiv helpers
13    struct fastdiv_values div_ne11;
14    struct fastdiv_values div_ne12;
15    struct fastdiv_values div_tasks_per_thread;
16    struct fastdiv_values div_ne02;
17};
18
19struct htp_set_rows_vtcm_layout {
20    size_t total_bytes;
21    size_t off_src0;
22    size_t off_dst;
23
24    size_t src0_bytes_per_thread;
25    size_t dst_bytes_per_thread;
26
27    size_t src0_spad_half_size;
28    size_t dst_spad_half_size;
29};
30
31static inline void htp_set_rows_vtcm_layout_build(
32    struct htp_set_rows_vtcm_layout * vtcm_layout,
33    int dst_type,
34    uint32_t ne00,
35    uint32_t n_threads) {
36
37    size_t src0_row_size = ne00 * 4;
38    size_t dst_row_size = 0;
39    switch (dst_type) {
40        case 0: // HTP_TYPE_F32
41            dst_row_size = ne00 * 4;
42            break;
43        case 1: // HTP_TYPE_F16
44            dst_row_size = ne00 * 2;
45            break;
46        case 8: // HTP_TYPE_Q8_0
47            dst_row_size = (ne00 / 32) * 34;
48            break;
49        default:
50            dst_row_size = 0;
51            break;
52    }
53
54    size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
55    size_t dst_row_size_aligned  = (dst_row_size + 255) & ~255;
56
57    vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
58    vtcm_layout->dst_spad_half_size  = dst_row_size_aligned;
59
60    vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
61    vtcm_layout->dst_bytes_per_thread  = dst_row_size_aligned * 2;
62
63    vtcm_layout->off_src0 = 0;
64    vtcm_layout->off_dst  = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
65    vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
66}
67
68#if defined(__cplusplus)
69static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
70#else
71_Static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
72#endif
73
74#endif // HTP_SET_ROWS_OPS_H