char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.3 KiB179 linesraw
1#ifndef HTP_CTX_H
2#define HTP_CTX_H
3
4#include "hex-dma.h"
5#include "hmx-queue.h"
6#include "htp-ops.h"
7#include "hex-profile.h"
8#include "work-queue.h"
9#include "hex-fastdiv.h"
10
11#include <assert.h>
12#include <dspqueue.h>
13#include <stdatomic.h>
14#include <stdint.h>
15#include <stdbool.h>
16
17#ifndef HTP_MAX_NTHREADS
18#define HTP_MAX_NTHREADS 10
19#endif
20#define HTP_MAX_MMAPS    16
21
22#define HTP_MAX_DIRTY_RANGES 32
23
24// Memory mapping
25struct htp_mmap {
26    uint64_t size;
27    uint64_t base;
28    uint32_t fd;
29    uint32_t reserved;
30};
31
32struct htp_dirty_range {
33    uint32_t start;
34    uint32_t end;
35};
36
37// Scratchpad state
38struct htp_spad {
39    const struct htp_tensor * src;             // original src of the data (for reuse)
40    uint8_t *                 data;            // pointer to an area in vtcm
41    uint32_t                  stride;          // stride used inside this spad
42    uint32_t                  size;            // total size
43    uint32_t                  size_per_thread; // size per thread
44};
45
46struct htp_mdev_group {
47    uint16_t              idx;
48    uint16_t              count;
49    struct fastdiv_values count_div;
50    uint8_t *             fence_base;
51    uint32_t              fence_seq;
52};
53
54struct htp_context;
55
56// Context while processing an Op
57// TODO: fold this into the main context
58struct htp_ops_context {
59    struct htp_context * ctx;
60
61    enum htp_op_code    op; // FIXME: rename to opcode
62    int32_t             op_params[HTP_OP_MAX_PARAMS];
63    int32_t             kernel_params[HTP_OP_MAX_KERN_PARAMS];
64
65    const struct htp_tensor * src[HTP_OP_MAX_INPUTS];
66    union {
67        const struct htp_tensor * dst;
68        const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS];
69    };
70
71    dma_queue **    src_dma[HTP_OP_MAX_INPUTS];
72    dma_queue **    dst_dma[HTP_OP_MAX_OUTPUTS];
73
74    // TODO convert these to an array
75    struct htp_spad src0_spad;
76    struct htp_spad src1_spad;
77    struct htp_spad src2_spad;
78    struct htp_spad src3_spad;
79    struct htp_spad dst_spad;
80
81    uint32_t              flags;
82    uint32_t              n_threads;
83    struct fastdiv_values n_threads_div;
84    int                   status;
85};
86
87// Main context for htp DSP backend
88struct htp_context {
89    dspqueue_t             dsp_queue;
90
91    struct htp_mmap        mmap[HTP_MAX_MMAPS];
92    dma_queue_t            dma[HTP_MAX_NTHREADS];
93    dma_queue_t            dma_cached[HTP_MAX_NTHREADS];
94    struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
95    work_queue_t           work_queue;
96    hmx_queue_t            hmx_queue;
97
98    uint32_t               n_threads;
99    struct fastdiv_values  n_threads_div;
100
101    int                    thread_id;
102    int                    thread_prio;
103
104    bool                   hmx_enabled;
105    bool                   etm;
106    uint32_t               profiler;
107
108    uint8_t *              vtcm_base;
109    size_t                 vtcm_size;
110    uint32_t               vtcm_rctx;
111    atomic_bool            vtcm_valid;
112    atomic_bool            vtcm_needs_release;
113
114    uint64_t               max_vmem;
115    struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES];
116
117    // Persistent DDR scratchpad for MUL_MAT_ID mappings
118    void *                 ddr_spad_base;
119    size_t                 ddr_spad_size;
120
121    struct htp_mdev_group  mdev;
122    struct htp_ops_context octx;
123
124    qurt_thread_t          main_thread;
125    void *                 main_stack;
126    atomic_bool            killed;
127    size_t                 footprint;
128};
129
130static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
131    if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
132        return false;
133    }
134
135    if (n_threads != octx->n_threads) {
136        octx->n_threads = n_threads;
137        octx->n_threads_div = n_threads == octx->ctx->n_threads
138            ? octx->ctx->n_threads_div
139            : init_fastdiv_values(n_threads);
140    }
141
142    return true;
143}
144
145static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) {
146    if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) {
147        octx->status = status;
148    }
149}
150
151int op_matmul(struct htp_ops_context * octx);
152int op_matmul_id(struct htp_ops_context * octx);
153int op_matmul_nx(struct htp_ops_context * octx);
154int op_matmul_id_nx(struct htp_ops_context * octx);
155int op_binary(struct htp_ops_context * octx);
156int op_unary(struct htp_ops_context * octx);
157int op_sum_rows(struct htp_ops_context * octx);
158int op_activations(struct htp_ops_context * octx);
159int op_softmax(struct htp_ops_context * octx);
160int op_add_id(struct htp_ops_context * octx);
161int op_rope(struct htp_ops_context * octx);
162int op_flash_attn_ext(struct htp_ops_context * octx);
163int op_set_rows(struct htp_ops_context * octx);
164int op_get_rows(struct htp_ops_context * octx);
165int op_cpy(struct htp_ops_context * octx);
166int op_repeat(struct htp_ops_context * octx);
167int op_argsort(struct htp_ops_context * octx);
168int op_ssm_conv(struct htp_ops_context * octx);
169int op_cumsum(struct htp_ops_context * octx);
170int op_fill(struct htp_ops_context * octx);
171int op_concat(struct htp_ops_context * octx);
172int op_diag(struct htp_ops_context * octx);
173int op_solve_tri(struct htp_ops_context * octx);
174int op_gated_delta_net(struct htp_ops_context * octx);
175int op_pad(struct htp_ops_context * octx);
176int op_im2col(struct htp_ops_context * octx);
177int op_allreduce(struct htp_ops_context * octx);
178
179#endif /* HTP_CTX_H */