char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)192067b72

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.6 KiB408 linesraw
1#ifndef HTP_DMA_H
2#define HTP_DMA_H
3
4#include <HAP_farf.h>
5#include <hexagon_types.h>
6#include <stdbool.h>
7#include <stdint.h>
8#include "hex-utils.h"
9
10#include "hex-profile.h"
11
12#ifdef __cplusplus
13extern "C" {
14#endif
15
16// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date
17typedef struct dma_descriptor_1d_s {
18    void *   next;
19    uint32_t size:24;
20    uint32_t desc_size:2;
21    uint32_t dst_comp:1;
22    uint32_t src_comp:1;
23    uint32_t dst_bypass:1;
24    uint32_t src_bypass:1;
25    uint32_t order:1;
26    uint32_t done:1;
27    void *   src;
28    void *   dst;
29} dma_descriptor_1d;
30
31#if __HVX_ARCH__ < 75
32
33typedef struct dma_descriptor_2d_s {
34    void *   next;
35    uint32_t reserved0:24;
36    uint32_t desc_size:2;
37    uint32_t dst_comp:1;
38    uint32_t src_comp:1;
39    uint32_t dst_bypass:1;
40    uint32_t src_bypass:1;
41    uint32_t order:1;
42    uint32_t done:1;
43    void *   src;
44    void *   dst;
45    uint32_t desc_type:8;
46    uint32_t reserved1:24;
47    uint32_t row_size:16;
48    uint32_t nrows:16;
49    uint32_t src_stride:16;
50    uint32_t dst_stride:16;
51    uint32_t src_offset:16;
52    uint32_t dst_offset:16;
53} dma_descriptor_2d;
54
55#else
56
57typedef struct dma_descriptor_2d_s {
58    void *   next;
59    uint32_t dst_stride:24;
60    uint32_t desc_size:2;
61    uint32_t dst_comp:1;
62    uint32_t src_comp:1;
63    uint32_t dst_bypass:1;
64    uint32_t src_bypass:1;
65    uint32_t order:1;
66    uint32_t done:1;
67    void *   src;
68    void *   dst;
69    uint32_t desc_type:8;
70    uint32_t reserved0:24;
71    uint32_t row_size:24;
72    uint32_t nrows_lo:8;
73    uint32_t nrows_hi:8;
74    uint32_t src_stride:24;
75    uint32_t offset:24;
76    uint32_t reserved1:8;
77} dma_descriptor_2d;
78
79#endif
80
81typedef struct {
82    void       *dst;
83    const void *src;
84} dma_ptr;
85
86typedef struct dma_ring_s dma_ring;
87struct dma_ring_s {
88    dma_descriptor_2d * desc;      // descriptor pointers
89    dma_descriptor_2d * tail;      // tail pointer
90    dma_ptr           * dptr;      // dst/src pointers
91    uint32_t            push_idx;
92    uint32_t            pop_idx;
93    uint32_t            capacity;
94    uint32_t            idx_mask;
95    struct htp_thread_trace * trace;
96    uintptr_t           vtcm_base;
97    uintptr_t           vtcm_end;
98};
99
100typedef struct dma_queue_s dma_queue;
101typedef dma_queue * dma_queue_t;
102
103struct dma_queue_s {
104    dma_ring *          ring;      // Points to the descriptor ring state
105    uint8_t             nocache;   // Queue-specific bypass flag
106    bool                alias;     // When set, dma_queue_delete will not free the ring
107};
108
109
110
111size_t      dma_queue_sizeof(size_t capacity);
112size_t      dma_queue_alignof(void);
113dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace);
114void        dma_queue_free(dma_queue_t q);
115
116size_t      dma_queue_alias_sizeof(void);
117dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache);
118void        dma_queue_alias_free(dma_queue_t q);
119
120// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
121// but those do not seem to always compiler properly.
122static inline void dmstart(void * next) {
123    asm volatile(" release(%0):at" : : "r"(next));
124    asm volatile(" dmstart(%0)" : : "r"(next));
125}
126
127static inline void dmlink(void * cur, void * next) {
128    asm volatile(" release(%0):at" : : "r"(next));
129    asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
130}
131
132static inline unsigned int dmpoll(void) {
133    unsigned int ret = 0;
134    asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
135    return ret;
136}
137
138static inline unsigned int dmwait(void) {
139    unsigned int ret = 0;
140    asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
141    return ret;
142}
143
144static inline dma_ptr dma_make_ptr(void *dst, const void *src)
145{
146    dma_ptr p = { dst, src };
147    return p;
148}
149
150static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) {
151    return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end;
152}
153
154static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) {
155    dma_ring * r = q->ring;
156    if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
157        return false;
158    }
159
160    dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx];
161    desc->src  = (void *) dptr.src;
162    desc->dst  = (void *) dptr.dst;
163    desc->size = size;
164
165    r->dptr[r->push_idx] = dptr;
166
167    htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
168
169    if (size) {
170        desc->next       = NULL;
171        desc->desc_size  = 0; // 1D mode
172        desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;
173        desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;
174        desc->order      = 0;
175        desc->done       = 0;
176
177        dmlink(r->tail, desc);
178        r->tail = (dma_descriptor_2d *) desc;
179    } else {
180        desc->desc_size = 0;
181        desc->done      = 1;
182    }
183
184    r->push_idx = (r->push_idx + 1) & r->idx_mask;
185    return true;
186}
187
188static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
189    dma_ring * r = q->ring;
190    if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
191        return false;
192    }
193
194    dma_descriptor_2d * desc = &r->desc[r->push_idx];
195
196    desc->next           = NULL;
197    desc->reserved0      = 0;
198    desc->reserved1      = 0;
199    desc->desc_size      = 1; // 2d mode
200    desc->src_bypass     = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;
201    desc->dst_bypass     = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;
202    desc->src_comp       = 0;
203    desc->dst_comp       = 0;
204    desc->order          = 0;
205    desc->done           = 0;
206    desc->src_stride     = src_stride;
207    desc->dst_stride     = dst_stride;
208    desc->src            = (void *) dptr.src;
209    desc->dst            = (void *) dptr.dst;
210    desc->row_size       = row_size;
211
212#if __HVX_ARCH__ < 75
213    desc->desc_type      = 0; // 2d (16-bit) mode
214    desc->nrows          = nrows;
215    desc->src_offset     = 0;
216    desc->dst_offset     = 0;
217#else
218    desc->desc_type      = 9; // 2d (24-bit) mode
219    desc->nrows_lo       = (nrows & 0xff);
220    desc->nrows_hi       = (nrows >> 8);
221    desc->offset         = 0;
222#endif
223
224    r->dptr[r->push_idx] = dptr;
225
226    htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
227
228    if (nrows) {
229        dmlink(r->tail, desc);
230        r->tail = desc;
231    } else {
232        desc->done = 1;
233    }
234
235    r->push_idx = (r->push_idx + 1) & r->idx_mask;
236    return true;
237}
238
239static inline dma_ptr dma_queue_pop(dma_queue * q) {
240    dma_ring * r = q->ring;
241    dma_ptr dptr  = { NULL };
242
243    if (r->push_idx == r->pop_idx) {
244        return dptr;
245    }
246
247    dptr = r->dptr[r->pop_idx];
248
249    volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx];
250
251    // Wait for desc to complete
252    if (!desc->done) {
253        // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, dptr.dst, dptr.src);
254        while (!desc->done) {
255            dmpoll();
256        }
257    }
258
259    htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
260
261    r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
262    return dptr;
263}
264
265static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {
266    dma_ring * r = q->ring;
267    dma_ptr dptr  = { NULL };
268
269    if (r->push_idx == r->pop_idx) {
270        return dptr;
271    }
272
273    dptr = r->dptr[r->pop_idx];
274
275    htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
276
277    r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
278    return dptr;
279}
280
281static inline bool dma_queue_empty(dma_queue * q) {
282    return q->ring->push_idx == q->ring->pop_idx;
283}
284
285static inline void dma_queue_flush(dma_queue * q) {
286    while (dma_queue_pop(q).dst != NULL) ;
287}
288
289static inline uint32_t dma_queue_depth(dma_queue * q) {
290    return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask;
291}
292
293static inline uint32_t dma_queue_capacity(dma_queue * q) {
294    return q->ring->capacity;
295}
296
297#if __HVX_ARCH__ < 75
298
299// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535.
300// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions.
301
302#define DMA_MAX_FIELD_VAL 65535u
303
304static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
305    // Fast path: everything fits in 16 bits
306    if (nrows == 0 || __builtin_expect(
307            row_size   <= DMA_MAX_FIELD_VAL &&
308            nrows      <= DMA_MAX_FIELD_VAL &&
309            src_stride <= DMA_MAX_FIELD_VAL &&
310            dst_stride <= DMA_MAX_FIELD_VAL, 1)) {
311        return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
312    }
313
314    // Contiguous block
315    // Use 1d DMA mode which supports sizes up to 24-bits (16MB)
316    if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) {
317        size_t total = row_size * nrows;
318        return dma_queue_push_single_1d(q, dptr, total);
319    }
320
321    // Stride overflow - fall back to row-by-row.
322    {
323        const uint8_t *src = (const uint8_t *) dptr.src;
324        uint8_t       *dst = (uint8_t *)       dptr.dst;
325        size_t r = 0;
326        while (r + 1 < nrows) {
327            dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);
328            if (!dma_queue_push_single_1d(q, p, row_size)) {
329                dma_queue_flush(q);
330            } else {
331                r++;
332            }
333        }
334        dma_queue_flush(q);
335        dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);
336        return dma_queue_push_single_1d(q, p, row_size);
337    }
338}
339
340#else // HVX_ARCH >= 75
341
342static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
343    // On v75 and up we always use 2d 24-bit mode
344    return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
345}
346
347#endif
348
349static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
350    return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);
351}
352
353static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
354    return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);
355}
356
357#define DMA_CACHE_MAX_SIZE 256U
358
359typedef struct {
360    uint8_t *base;
361    uint32_t line_size;
362    uint32_t capacity;
363    uint32_t src[DMA_CACHE_MAX_SIZE];
364    uint16_t age[DMA_CACHE_MAX_SIZE];
365} dma_cache;
366
367static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
368{
369    c->capacity  = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
370    c->base      = base;
371    c->line_size = line_size;
372
373    for (unsigned i=0; i < c->capacity; i++) {
374        c->src[i] = 0;
375        c->age[i] = 0;
376    }
377}
378
379static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
380{
381    uint32_t o_idx = 0;
382    uint16_t o_age = 0;
383    uint8_t *  dst = 0;
384
385    for (unsigned i=0; i < c->capacity; i++) {
386        if (c->src[i] == (uint32_t) src) {
387            c->age[i] = 0;
388            dst = c->base + (i * c->line_size); nrows = 0; // dummy dma
389        } else {
390            c->age[i]++;
391            if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }
392        }
393    }
394    if (!dst) {
395        c->age[o_idx] = 0;
396        c->src[o_idx] = (uint32_t) src;
397        dst = c->base + o_idx * c->line_size; // normal nrows dma
398        return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows);
399    }
400
401    return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0);
402}
403
404#ifdef __cplusplus
405}  // extern "C"
406#endif
407
408#endif /* HTP_DMA_H */