char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: L2 cache handling rework (dirty bit tracking with lazy flushing) and more MUL_MAT updates (#25762)b2dd28a3b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.1 KiB160 linesraw
1#ifndef HMX_QUEUE_H
2#define HMX_QUEUE_H
3
4#include <stdbool.h>
5#include <stdint.h>
6#include <stdatomic.h>
7
8#include <hexagon_types.h>
9#include <qurt_thread.h>
10#include <qurt_futex.h>
11#include <HAP_farf.h>
12
13#include "hex-utils.h"
14#include "hex-profile.h"
15
16#ifdef __cplusplus
17extern "C" {
18#endif
19
20#if __HVX_ARCH__ > 79
21#define HMX_QUEUE_POLL_COUNT 2000
22#else
23#define HMX_QUEUE_POLL_COUNT 1
24#endif
25
26typedef void (*hmx_queue_func)(void *);
27
28// Dummy funcs used as signals
29enum hmx_queue_signal {
30    HMX_QUEUE_NOOP = 0, // aka NULL
31    HMX_QUEUE_WAKEUP,
32    HMX_QUEUE_SUSPEND,
33    HMX_QUEUE_KILL
34};
35
36struct hmx_queue_desc {
37    hmx_queue_func   func;
38    void *           data;
39    atomic_uint      done;
40};
41
42struct hmx_queue_s {
43    struct hmx_queue_desc * desc;
44    atomic_uint      idx_write; // updated by producer (push)
45    atomic_uint      idx_read;  // updated by consumer (process)
46    unsigned int     idx_pop;   // updated by producer (pop)
47    uint32_t         idx_mask;
48    uint32_t         capacity;
49
50    atomic_uint      seqn;      // incremented for all pushes, used with futex
51    qurt_thread_t    thread;
52    void *           stack;
53    uint32_t         hap_rctx;
54    bool             hmx_locked;
55    struct htp_thread_trace * trace;
56    bool             external_mem; // memory owned externally
57};
58
59typedef struct hmx_queue_s * hmx_queue_t;
60
61size_t      hmx_queue_sizeof(size_t capacity, uint32_t stack_size);
62size_t      hmx_queue_alignof(void);
63hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace);
64void        hmx_queue_free(hmx_queue_t q);
65
66static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) {
67    struct hmx_queue_desc d = { func, data };
68    return d;
69}
70
71static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) {
72    unsigned int ir = atomic_load(&q->idx_read);
73    unsigned int iw = atomic_load(&q->idx_write);
74
75    if (((iw + 1) & q->idx_mask) == ir) {
76        FARF(HIGH, "hmx-queue-push: queue is full\n");
77        return false;
78    }
79
80    atomic_store(&d.done, 0);
81
82    FARF(HIGH, "hmx-queue-push: iw %u func %p data %p\n", iw, d.func, d.data);
83
84    q->desc[iw] = d;
85    atomic_store(&q->idx_write, (iw + 1) & q->idx_mask);
86    // wake up our thread
87    atomic_fetch_add(&q->seqn, 1);
88    qurt_futex_wake(&q->seqn, 1);
89
90    return true;
91}
92
93static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) {
94    return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL));
95}
96
97static inline bool hmx_queue_empty(hmx_queue_t q) {
98    return q->idx_pop == atomic_load(&q->idx_write);
99}
100
101static inline uint32_t hmx_queue_depth(hmx_queue_t q) {
102    return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask;
103}
104
105static inline uint32_t hmx_queue_capacity(hmx_queue_t q) {
106    return q->capacity;
107}
108
109static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) {
110    unsigned int ip = q->idx_pop;
111    unsigned int iw = atomic_load(&q->idx_write);
112
113    struct hmx_queue_desc rd = { NULL, NULL };
114    if (ip == iw) {
115        return rd;
116    }
117
118    // Wait for desc to complete
119    struct hmx_queue_desc * d = &q->desc[ip];
120    while (!atomic_load(&d->done)) {
121        FARF(HIGH, "hmx-queue-pop: waiting for HMX queue : %u\n", ip);
122        hex_pause();
123    }
124
125    rd = *d;
126    q->idx_pop = (ip + 1) & q->idx_mask;
127
128    FARF(HIGH, "hmx-queue-pop: ip %u func %p data %p\n", ip, rd.func, rd.data);
129    return rd;
130}
131
132static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) {
133    while (1) {
134        struct hmx_queue_desc d = hmx_queue_pop_one(q);
135
136        uint32_t sig = (uint32_t) d.func;
137        if (sig && sig <= HMX_QUEUE_KILL)
138            continue;
139
140        return d;
141    }
142}
143
144static inline void hmx_queue_flush(hmx_queue_t q) {
145    while (hmx_queue_pop_one(q).func != NULL) ;
146}
147
148static inline void hmx_queue_wakeup(hmx_queue_t q) {
149    hmx_queue_signal(q, HMX_QUEUE_WAKEUP);
150}
151
152static inline void hmx_queue_suspend(hmx_queue_t q) {
153    hmx_queue_signal(q, HMX_QUEUE_SUSPEND);
154}
155
156#ifdef __cplusplus
157}  // extern "C"
158#endif
159
160#endif /* HMX_QUEUE_H */