char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
41.6 KiB1304 linesraw
1#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
2#pragma clang diagnostic ignored "-Wunused-function"
3#pragma clang diagnostic ignored "-Wunused-variable"
4#pragma clang diagnostic ignored "-Wunused-but-set-variable"
5
6#include <HAP_farf.h>
7#include <HAP_perf.h>
8#include <AEEStdErr.h>
9#include <dspqueue.h>
10#include <HAP_compute_res.h>
11#include <HAP_etm_config.h>
12#include <HAP_mem.h>
13#include <HAP_power.h>
14#include <HAP_ps.h>
15#include <HAP_dcvs.h>
16#include <qurt.h>
17#include <qurt_thread.h>
18#include <qurt_memory.h>
19#include <remote.h>
20#include <string.h>
21#include <stdatomic.h>
22
23#include "hex-utils.h"
24#include "hex-dma.h"
25#include "hmx-queue.h"
26
27#define GGML_COMMON_DECL_C
28#include "ggml-common.h"
29#include "hex-bitmap.h"
30#include "htp-ctx.h"
31#include "htp-ops.h"
32#include "htp-tensor.h"
33#include "htp_iface.h"
34#include "work-queue.h"
35#include "hex-profile.h"
36#include "allreduce-ops.h"
37#include "htp-fence.h"
38
39#define HMX_QUEUE_CAPACITY     16
40#define HMX_QUEUE_STACK_SIZE   16384
41#define WORK_QUEUE_CAPACITY    16
42#define WORK_QUEUE_STACK_SIZE  16384
43#define MAIN_THREAD_STACK_SIZE 32768
44
45_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS,
46               "work-queue thread cap must be >= HTP_MAX_NTHREADS");
47
48struct htp_handle {
49    struct htp_context * ctx;
50};
51
52static inline void * htp_mmap(uint32_t fd, uint32_t size) {
53    void * va = (void *)-1;
54    for (int retry = 0; retry < 2; retry++) {
55#if __HVX_ARCH__ > 73
56        va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
57#else
58        if (size > HTP_MMAP_MAX_VMEM) {
59            FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size);
60            abort();
61        }
62        va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
63#endif
64        if (va != (void *)-1 && va != NULL) {
65            return va;
66        }
67        if (retry == 0) {
68            FARF(HIGH, "mmap failed first try (va %p fd %u size %u), retrying...", va, fd, size);
69        }
70    }
71    return NULL;
72}
73
74static inline void htp_munmap(void * va, uint32_t size) {
75#if __HVX_ARCH__ > 73
76    HAP_munmap2(va, size);
77#else
78    HAP_munmap(va, size);
79#endif
80}
81
82AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {
83    (void) uri;
84    struct htp_handle * h = calloc(1, sizeof(*h));
85    if (h == NULL) {
86        return AEE_ENOMEMORY;
87    }
88
89    *handle = (remote_handle64) h;
90    return AEE_SUCCESS;
91}
92
93AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) {
94    struct htp_handle * h = (struct htp_handle *) handle;
95    if (!h) {
96        return AEE_EBADPARM;
97    }
98
99    int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable();
100    if (err) {
101        if (err == AEE_EVERSIONNOTSUPPORT) {
102            FARF(ERROR, "API HAP_user_etm_enable/disable is not supported\n");
103        } else {
104            FARF(ERROR, "Error executing HAP_user_etm_enable/disable with error code : 0x%x\n", err);
105        }
106    }
107    return err;
108}
109
110AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) {
111    struct htp_handle * h = (struct htp_handle *) handle;
112    if (!h || !h->ctx) {
113        return AEE_EBADPARM;
114    }
115    struct htp_context * ctx = h->ctx;
116
117    if (mode == HTP_PROF_PMU) {
118        const uint32_t* events = pmu_conf->events;
119
120        // Pack 4 event IDs (low 8 bits) into each 32-bit config register
121        uint32_t evtcfg = 0, evtcfg1 = 0, cfg = 0, i = 0;
122        for (; i < HEX_NUM_PMU_COUNTERS/2; i++) {
123            evtcfg  |= ((events[i + 0] & 0xFF) << (i * 8));
124            evtcfg1 |= ((events[i + 4] & 0xFF) << (i * 8));
125        }
126
127        // For events >255 pack high 2 bits of all 8 event IDs into cfg register
128        // 2 bits per counter: bits [1:0] for counter 0, [3:2] for counter 1, etc.
129        for (i = 0; i < HEX_NUM_PMU_COUNTERS; i++) {
130            cfg |= (((events[i] >> 8) & 3) << (i * 2));
131        }
132
133        FARF(ALWAYS, "Configuring PMU registers: evtcfg = 0x%x, evtcfg1 = 0x%x, pmucfg = 0x%x", evtcfg, evtcfg1, cfg);
134
135        // Configure PMU registers
136        qurt_pmu_set(QURT_PMUCFG,     cfg);
137        qurt_pmu_set(QURT_PMUEVTCFG,  evtcfg);
138        qurt_pmu_set(QURT_PMUEVTCFG1, evtcfg1);
139        qurt_pmu_enable(1);
140    }
141
142    ctx->profiler = mode;
143
144    return AEE_SUCCESS;
145}
146
147AEEResult htp_iface_close(remote_handle64 handle) {
148    struct htp_handle * h = (struct htp_handle *) handle;
149    if (!h) {
150        return AEE_EBADPARM;
151    }
152
153    struct htp_context * ctx = h->ctx;
154    if (ctx) {
155        if (ctx->dsp_queue) {
156            FARF(ERROR, "Closing handle with queue still open");
157            return AEE_EITEMBUSY;
158        }
159
160        // release the mmaps (if any)
161        for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
162            if (ctx->mmap[i].size) {
163                htp_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
164                ctx->mmap[i].size = 0;
165                ctx->mmap[i].base = NULL;
166                ctx->mmap[i].fd   = -1;
167            }
168        }
169
170        if (ctx->profiler) {
171            qurt_pmu_enable(1);
172        }
173
174        if (ctx->etm) {
175            HAP_user_etm_disable();
176        }
177
178        // Free the unified block (ctx is the base address of the block)
179        free(ctx);
180        h->ctx = NULL;
181    }
182
183    free(h);
184    return AEE_SUCCESS;
185}
186
187AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {
188    struct htp_handle * h = (struct htp_handle *) handle;
189    if (!h || !h->ctx) {
190        return AEE_EBADPARM;
191    }
192    struct htp_context * ctx = h->ctx;
193
194    // See if we already have this mapping
195    for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
196        struct htp_mmap *m = &ctx->mmap[i];
197        if (m->fd == fd) {
198            return AEE_SUCCESS;
199        }
200    }
201
202    // Add new mapping
203    for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
204        struct htp_mmap *m = &ctx->mmap[i];
205        if (!m->size) {
206            FARF(HIGH, "mmap : fd %u size %u", fd, size);
207            void *va = htp_mmap(fd, size);
208            if (va == NULL) {
209                FARF(ERROR, "mmap failed : fd %u size %u", fd, (uint32_t) size);
210                return AEE_EFAILED;
211            }
212
213            m->base   = (uint64_t) va;
214            m->fd     = fd;
215            m->size   = size;
216
217            return AEE_SUCCESS;
218        }
219    }
220
221    return AEE_ENOMEMORY;
222}
223
224AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
225    struct htp_handle * h = (struct htp_handle *) handle;
226    if (!h || !h->ctx) {
227        return AEE_EBADPARM;
228    }
229    struct htp_context * ctx = h->ctx;
230
231    for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
232        struct htp_mmap *m = &ctx->mmap[i];
233        if (fd < 0 || m->fd == fd) {
234            FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size);
235            htp_munmap((void *) m->base, m->size);
236            m->size   = 0;
237            m->base   = NULL;
238            m->fd     = -1;
239        }
240    }
241
242    return AEE_SUCCESS;
243}
244
245static void vtcm_acquire(struct htp_context * ctx) {
246    if (!ctx->vtcm_valid) {
247        int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
248        if (err != 0) {
249            FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
250            abort();
251        }
252
253        ctx->vtcm_needs_release = false;
254        ctx->vtcm_valid = true;
255
256        // Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions
257        HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10);
258    }
259}
260
261static void vtcm_release(struct htp_context * ctx) {
262    if (ctx->vtcm_valid) {
263        ctx->vtcm_valid         = false;
264        ctx->vtcm_needs_release = false;
265        HAP_compute_res_release_cached(ctx->vtcm_rctx);
266    }
267}
268
269static int vtcm_release_callback(unsigned int rctx, void * state) {
270    struct htp_context * ctx = (struct htp_context *) state;
271    ctx->vtcm_needs_release = true;
272    return 0;
273}
274
275static int vtcm_alloc(struct htp_context * ctx) {
276    unsigned int vtcm_size = 8 * 1024 * 1024;  // 8MB default
277    HAP_compute_res_query_VTCM(0, &vtcm_size, NULL, NULL, NULL);
278
279    compute_res_attr_t attr;
280    HAP_compute_res_attr_init(&attr);
281    HAP_compute_res_attr_set_serialize(&attr, 0);
282    HAP_compute_res_attr_set_cache_mode(&attr, 1);
283    HAP_compute_res_attr_set_vtcm_param_v2(&attr, vtcm_size, vtcm_size, vtcm_size); // single page
284    HAP_compute_res_attr_set_release_callback(&attr, vtcm_release_callback, (void *) ctx);
285    HAP_compute_res_attr_set_hmx_param(&attr, 1);
286
287    // Allocate VTCM for scratch pads
288    uint32_t rctx = HAP_compute_res_acquire(&attr, 1000000 /* timeout */);
289    if (!rctx) {
290        FARF(ERROR, "failed to allocate %zu bytes VTCM\n", ctx->vtcm_size);
291        return AEE_ENOMEMORY;
292    }
293
294    void * vtcm_ptr;
295    if (HAP_compute_res_attr_get_vtcm_ptr_v2(&attr, &vtcm_ptr, &vtcm_size) != 0) {
296        HAP_compute_res_release(rctx);
297        FARF(ERROR, "failed to allocate %zu bytes VTCM (new)\n", ctx->vtcm_size);
298        return AEE_ENOMEMORY;
299    }
300
301    ctx->vtcm_base          = (uint8_t *) vtcm_ptr;
302    ctx->vtcm_size          = vtcm_size;
303    ctx->vtcm_rctx          = rctx;
304    ctx->vtcm_valid         = false;
305    ctx->vtcm_needs_release = false;
306
307    return 0;
308}
309
310static void vtcm_free(struct htp_context * ctx) {
311    if (ctx->vtcm_rctx) {
312        HAP_compute_res_release(ctx->vtcm_rctx);
313        ctx->vtcm_base = 0;
314        ctx->vtcm_rctx = 0;
315    }
316}
317
318static void htp_main_thread(void * context);
319static void htp_packet_callback(dspqueue_t queue, int error, void * context);
320static void htp_error_callback(dspqueue_t queue, int error, void * context);
321
322AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) {
323    struct htp_handle * h = (struct htp_handle *) handle;
324    if (!h) {
325        return AEE_EBADPARM;
326    }
327
328    if (h->ctx) {
329        FARF(ERROR, "Queue already open");
330        return AEE_EITEMBUSY;
331    }
332
333    // Cache the original FastRPC thread priority, then calculate compute priority
334    int fastrpc_tid  = qurt_thread_get_id();
335    int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid);
336    int main_prio    = fastrpc_prio - 10;
337    if (main_prio < 1) main_prio = 1;
338
339    dspqueue_t dsp_queue = NULL;
340    bool use_callbacks = false;
341
342    // Import queue with NULL callbacks to avoid starting dspueue internal threads
343    int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue);
344    if (err == AEE_EBADPARM) {
345        // Fallback for devices that don't support NULL callbacks
346        FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks");
347        use_callbacks = true;
348        err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue);
349    }
350
351    if (err) {
352        FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err);
353        return err;
354    }
355
356    qurt_sysenv_max_hthreads_t hw_threads;
357    qurt_sysenv_get_max_hw_threads(&hw_threads);
358    uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;
359
360    if (n_hvx == 0) {
361        n_hvx = hw_nhvx;
362    }
363    if (n_hvx > hw_threads.max_hthreads) {
364        n_hvx = hw_threads.max_hthreads;
365    }
366    if (n_hvx > HTP_MAX_NTHREADS) {
367        n_hvx = HTP_MAX_NTHREADS;
368    }
369
370    // layout segments of our contiguous block
371
372    // 1. htp_context : sits at the base (block is 4K-aligned via memalign below)
373    size_t offset = sizeof(struct htp_context);
374
375    // 2. main_stack
376    size_t offset_main_stack = 0;
377    size_t size_main_stack   = 0;
378    if (!use_callbacks) {
379        offset_main_stack = hex_align_up(offset, 4096);
380        size_main_stack   = MAIN_THREAD_STACK_SIZE;
381        offset = offset_main_stack + size_main_stack;
382    }
383
384    // 3. work_queue
385    size_t wq_align  = work_queue_alignof();
386    size_t offset_wq = hex_align_up(offset, wq_align);
387    size_t size_wq   = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
388    offset = offset_wq + size_wq;
389
390    // 4. dma_queue
391    size_t dma_align = dma_queue_alignof();
392    size_t offset_dma = hex_align_up(offset, dma_align);
393    size_t size_dma = 0;
394    for (uint32_t i = 0; i < n_hvx; i++) {
395        size_dma  = hex_align_up(size_dma, dma_queue_alignof());
396        size_dma += dma_queue_sizeof(256);
397        size_dma  = hex_align_up(size_dma, dma_queue_alignof());
398        size_dma += dma_queue_alias_sizeof();
399    }
400    offset = offset_dma + size_dma;
401
402    // 5. hmx_queue
403    size_t offset_hmx = 0;
404    size_t size_hmx = 0;
405    if (n_hmx) {
406        size_t hmx_align = hmx_queue_alignof();
407        offset_hmx = hex_align_up(offset, hmx_align);
408        size_hmx   = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE);
409        offset = offset_hmx + size_hmx;
410    }
411
412    size_t footprint = hex_align_up(offset, 128);
413
414    void * block = memalign(4096, footprint);
415    if (!block) {
416        FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint);
417        dspqueue_close(dsp_queue);
418        return AEE_ENOMEMORY;
419    }
420    memset(block, 0, footprint);
421
422    h->ctx = (struct htp_context *) block;
423    struct htp_context * ctx = h->ctx;
424    ctx->footprint = footprint;
425
426    ctx->thread_id   = fastrpc_tid;
427    ctx->thread_prio = main_prio;
428    ctx->max_vmem    = max_vmem;
429    ctx->dsp_queue   = dsp_queue;
430
431    err = vtcm_alloc(ctx);
432    if (err != AEE_SUCCESS) {
433        FARF(ERROR, "Unable to allocate VTCM");
434        htp_iface_stop(handle);
435        return AEE_ENOMEMORY;
436    }
437
438    HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);
439
440    // Set client class
441    {
442        HAP_power_request_t request;
443        memset(&request, 0, sizeof(HAP_power_request_t));
444        request.type    = HAP_power_set_apptype;
445        request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;
446
447        if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
448            htp_iface_stop(handle);
449            return err;
450        }
451    }
452
453    // DCVS setup
454    {
455        HAP_power_request_t request;
456        memset(&request, 0, sizeof(request));
457
458        request.type                              = HAP_power_set_DCVS_v3;
459        request.dcvs_v3.set_dcvs_enable           = TRUE;
460        request.dcvs_v3.dcvs_enable               = FALSE;
461        request.dcvs_v3.set_bus_params            = TRUE;
462        request.dcvs_v3.bus_params.min_corner     = HAP_DCVS_VCORNER_MAX;
463        request.dcvs_v3.bus_params.max_corner     = HAP_DCVS_VCORNER_MAX;
464        request.dcvs_v3.bus_params.target_corner  = HAP_DCVS_VCORNER_MAX;
465        request.dcvs_v3.set_core_params           = TRUE;
466        request.dcvs_v3.core_params.min_corner    = HAP_DCVS_VCORNER_MAX;
467        request.dcvs_v3.core_params.max_corner    = HAP_DCVS_VCORNER_MAX;
468        request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;
469        request.dcvs_v3.set_sleep_disable         = TRUE;
470        request.dcvs_v3.sleep_disable             = TRUE;
471
472#if (__HEXAGON_ARCH__ >= 79)
473        HAP_set_dcvs_v3_protected_bus_corners(&request, 1);
474#endif
475        if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
476            htp_iface_stop(handle);
477            return err;
478        }
479
480        memset(&request, 0, sizeof(request));
481        request.type         = HAP_power_set_HVX;
482        request.hvx.power_up = TRUE;
483        if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
484            htp_iface_stop(handle);
485            return err;
486        }
487    }
488
489#if __HVX_ARCH__ >= 75
490    {
491        // Power on HMX and set HMX clock
492        HAP_power_request_t request;
493        memset(&request, 0, sizeof(HAP_power_request_t));
494        request.type = HAP_power_set_HMX_v2;
495        request.hmx_v2.set_power     = TRUE;
496        request.hmx_v2.power_up      = TRUE;
497        request.hmx_v2.set_clock     = TRUE;
498        request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;
499        request.hmx_v2.min_corner    = HAP_DCVS_EXP_VCORNER_MAX;
500        request.hmx_v2.max_corner    = HAP_DCVS_EXP_VCORNER_MAX;
501        request.hmx_v2.perf_mode     = HAP_CLK_PERF_HIGH;
502        FARF(ALWAYS, "Setting HMX clock\n");
503        err = HAP_power_set((void *) ctx, &request);
504        if (err != AEE_SUCCESS) {
505            FARF(ERROR, "ggml-hex: error setting HMX clock.");
506            htp_iface_stop(handle);
507            return err;
508        }
509    }
510#else
511    {
512        // Power on HMX
513        HAP_power_request_t request;
514        memset(&request, 0, sizeof(HAP_power_request_t));
515        request.type         = HAP_power_set_HMX;
516        request.hmx.power_up = TRUE;
517        FARF(ALWAYS, "Powering HMX on\n");
518        err = HAP_power_set((void *) ctx, &request);
519        if (err != AEE_SUCCESS) {
520            FARF(ERROR, "ggml-hex: error powering on HMX.");
521            htp_iface_stop(handle);
522            return err;
523        }
524    }
525#endif
526
527    ctx->hmx_enabled = n_hmx;
528    ctx->hmx_queue   = NULL;
529    if (n_hmx) {
530        void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx);
531        ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]);
532    }
533    FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);
534
535    ctx->n_threads = n_hvx;
536    ctx->n_threads_div = init_fastdiv_values(ctx->n_threads);
537
538    // Initialize DMA queues
539    uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma);
540    size_t size_dma_q = dma_queue_sizeof(256);
541    size_t size_dma_alias = dma_queue_alias_sizeof();
542
543    for (int i = 0; i < ctx->n_threads; i++) {
544        dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());
545        ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]);
546        dma_ptr_curr += size_dma_q;
547
548        dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());
549        ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1);
550        dma_ptr_curr += size_dma_alias;
551    }
552
553    ctx->ddr_spad_size = 512 * 1024; // 512 KB
554    ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size);
555
556    void * wq_ptr = (void *) ((uintptr_t) block + offset_wq);
557    ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
558
559    ctx->main_stack = NULL;
560    ctx->main_thread = 0;
561    atomic_store(&ctx->killed, false);
562
563    if (!use_callbacks) {
564        // Start main compute thread
565        ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack);
566
567        qurt_thread_attr_t attr;
568        qurt_thread_attr_init(&attr);
569        qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack);
570        qurt_thread_attr_set_stack_size(&attr, size_main_stack);
571        qurt_thread_attr_set_priority(&attr, main_prio);
572        qurt_thread_attr_set_name(&attr, "htp-main");
573
574        int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx);
575        if (err_thread) {
576            FARF(ERROR, "Unable to create htp main thread: %d", err_thread);
577            htp_iface_stop(handle);
578            return AEE_ENOMEMORY;
579        }
580    }
581
582    FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n",
583         sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio);
584
585    return AEE_SUCCESS;
586}
587
588AEEResult htp_iface_stop(remote_handle64 handle) {
589    struct htp_handle * h = (struct htp_handle *) handle;
590    if (!h || !h->ctx) {
591        return AEE_EBADPARM;
592    }
593    struct htp_context * ctx = h->ctx;
594
595    if (ctx->main_thread) {
596        atomic_store(&ctx->killed, true);
597        int status;
598        (void) qurt_thread_join(ctx->main_thread, &status);
599        ctx->main_thread = 0;
600    }
601
602    int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL;
603    if (err != 0) {
604        FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err);
605        return err;
606    }
607
608    work_queue_free(ctx->work_queue);
609
610    for (int i = 0; i < ctx->n_threads; i++) {
611        dma_queue_alias_free(ctx->dma[i]);
612        dma_queue_free(ctx->dma_cached[i]);
613    }
614
615    if (ctx->hmx_queue) {
616        hmx_queue_free(ctx->hmx_queue);
617        ctx->hmx_queue = NULL;
618    }
619    ctx->hmx_enabled = false;
620
621    vtcm_free(ctx);
622
623    if (ctx->ddr_spad_base) {
624        free(ctx->ddr_spad_base);
625        ctx->ddr_spad_base = NULL;
626        ctx->ddr_spad_size = 0;
627    }
628
629    free(ctx);
630    h->ctx = NULL;
631
632    return AEE_SUCCESS;
633}
634
635AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_t * n_hvx, uint32_t * n_hmx, uint64_t * vtcm_size) {
636    (void)handle;
637    if (!n_threads || !n_hvx || !n_hmx || !vtcm_size) {
638        return AEE_EBADPARM;
639    }
640
641    qurt_sysenv_max_hthreads_t hw_threads;
642    qurt_sysenv_get_max_hw_threads(&hw_threads);
643    uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;
644
645    uint32_t n_hvx_val = hw_nhvx;
646    if (n_hvx_val > hw_threads.max_hthreads) {
647        n_hvx_val = hw_threads.max_hthreads;
648    }
649    if (n_hvx_val > HTP_MAX_NTHREADS) {
650        n_hvx_val = HTP_MAX_NTHREADS;
651    }
652
653    // for now we force n_threads == n_hvx
654    *n_threads = n_hvx_val;
655    *n_hvx     = n_hvx_val;
656    *n_hmx     = 1;
657
658    uint32_t vtcm_sz = 8 * 1024 * 1024; // 8MB default fallback
659    HAP_compute_res_query_VTCM(0, (unsigned int *)&vtcm_sz, NULL, NULL, NULL);
660    *vtcm_size = vtcm_sz;
661
662    return AEE_SUCCESS;
663}
664
665static void htp_error_callback(dspqueue_t queue, int error, void * context) {
666    // No errors expected on the DSP.
667    FARF(ERROR, "Error callback: 0x%08x", (unsigned) error);
668}
669
670struct profile_data {
671    uint64_t usecs;
672    uint64_t cycles_start;
673    uint64_t cycles_stop;
674    uint32_t pmu_counters[HEX_NUM_PMU_COUNTERS];
675};
676
677static inline void profile_start(uint32_t mode, struct profile_data * d) {
678    switch (mode) {
679        case HTP_PROF_PMU:
680            hex_get_pmu(d->pmu_counters);
681            // fallthrough
682        case HTP_PROF_BASIC:
683        case HTP_PROF_TRACE:
684            d->usecs  = HAP_perf_get_qtimer_count();
685            d->cycles_start = hex_get_cycles();
686            break;
687        default:
688            break;
689    }
690}
691
692static inline void profile_stop(uint32_t mode, struct profile_data * d) {
693    uint32_t pmu_counters[HEX_NUM_PMU_COUNTERS];
694    switch (mode) {
695        case HTP_PROF_PMU:
696            hex_get_pmu(pmu_counters);
697            for (int i = 0; i < HEX_NUM_PMU_COUNTERS; i++) {
698                d->pmu_counters[i] = pmu_counters[i] - d->pmu_counters[i];
699            }
700            // fallthrough
701        case HTP_PROF_BASIC:
702        case HTP_PROF_TRACE:
703            d->usecs  = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - d->usecs);
704            d->cycles_stop = hex_get_cycles();
705            break;
706        default:
707            break;
708    }
709}
710
711static int op_fence(struct htp_ops_context * octx) {
712    struct htp_context *ctx = octx->ctx;
713    struct htp_thread_trace * tr = &ctx->trace[0];
714    const uint32_t seq  = (uint32_t) octx->op_params[0];
715    const uint32_t mode = (uint32_t) octx->op_params[1];
716
717    htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
718
719    const struct htp_tensor * sync = octx->src[0];
720    atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;
721
722    if (mode == 1) {
723        htp_flush_dirty_ranges(ctx);
724
725        htp_mdev_group_barrier(octx);
726
727        if (ctx->mdev.idx == 0) {
728            htp_fence_write(sync_fence, seq, octx->status);
729        }
730        htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
731        FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status);
732        return octx->status;
733    }
734
735    int status = HTP_STATUS_OK;
736    uint64_t spins = 0;
737    while (1) {
738        uint32_t sync_seq;
739        uint32_t sync_status;
740        htp_fence_read(sync_fence, &sync_seq, &sync_status);
741        if ((int32_t)(sync_seq - seq) >= 0) {
742            if (sync_status > HTP_STATUS_OK) {
743                FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq);
744                status = sync_status;
745            }
746            break;
747        }
748        if (++spins > HTP_FENCE_TIMEOUT) {
749            FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq);
750            status = HTP_STATUS_INTERNAL_ERR;
751            break;
752        }
753        hex_pause();
754    }
755
756    htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
757
758    FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq);
759    return status;
760}
761
762static int op_mdev_group(struct htp_ops_context * octx) {
763    struct htp_context * ctx = octx->ctx;
764    const struct htp_tensor * sync = octx->src[0];
765    ctx->mdev.idx   = (uint16_t) octx->op_params[0];
766    ctx->mdev.count = (uint16_t) sync->ne[1];
767    if (ctx->mdev.count > 1) {
768        ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count);
769        ctx->mdev.fence_base = (uint8_t *) sync->data;
770    }
771    return HTP_STATUS_OK;
772}
773
774static int execute_op(struct htp_ops_context * octx) {
775    switch (octx->op) {
776        case HTP_OP_MDEV_GROUP:
777            return op_mdev_group(octx);
778
779        case HTP_OP_FENCE:
780            return op_fence(octx);
781
782        case HTP_OP_ALLREDUCE:
783        case HTP_OP_ALLREDUCE_ADD:
784            return op_allreduce(octx);
785
786        case HTP_OP_MUL_MAT:
787        case HTP_OP_MUL_MAT_ADD:
788            return op_matmul(octx);
789
790        case HTP_OP_MUL_MAT_ID:
791            return op_matmul_id(octx);
792
793        case HTP_OP_MUL_MAT_ID_NX:
794            return op_matmul_id_nx(octx);
795
796        case HTP_OP_MUL_MAT_NX:
797            return op_matmul_nx(octx);
798
799        case HTP_OP_MUL:
800        case HTP_OP_ADD:
801        case HTP_OP_SUB:
802        case HTP_OP_DIV:
803        case HTP_OP_ADD_ID:
804            return op_binary(octx);
805
806        case HTP_OP_NORM:
807        case HTP_OP_RMS_NORM:
808        case HTP_OP_RMS_NORM_MUL:
809        case HTP_OP_SCALE:
810        case HTP_OP_CLAMP:
811        case HTP_OP_LEAKY_RELU:
812        case HTP_OP_SQR:
813        case HTP_OP_SQRT:
814        case HTP_OP_UNARY_SOFTPLUS:
815        case HTP_OP_UNARY_SIGMOID:
816        case HTP_OP_UNARY_SILU:
817        case HTP_OP_UNARY_GELU:
818        case HTP_OP_UNARY_NEG:
819        case HTP_OP_UNARY_EXP:
820        case HTP_OP_UNARY_TANH:
821        case HTP_OP_UNARY_ABS:
822        case HTP_OP_UNARY_LOG:
823        case HTP_OP_UNARY_RELU:
824        case HTP_OP_L2_NORM:
825            return op_unary(octx);
826
827        case HTP_OP_GLU_SWIGLU:
828        case HTP_OP_GLU_SWIGLU_OAI:
829        case HTP_OP_GLU_SWIGLU_CLAMP:
830        case HTP_OP_GLU_GEGLU:
831            return op_activations(octx);
832
833        case HTP_OP_SOFTMAX:
834            return op_softmax(octx);
835
836        case HTP_OP_ROPE:
837            return op_rope(octx);
838
839        case HTP_OP_FLASH_ATTN_EXT:
840            return op_flash_attn_ext(octx);
841
842        case HTP_OP_SET_ROWS:
843            return op_set_rows(octx);
844
845        case HTP_OP_GET_ROWS:
846            return op_get_rows(octx);
847
848        case HTP_OP_SUM_ROWS:
849            return op_sum_rows(octx);
850
851        case HTP_OP_CPY:
852        case HTP_OP_CPY_FENCE:
853            return op_cpy(octx);
854
855        case HTP_OP_REPEAT:
856            return op_repeat(octx);
857
858        case HTP_OP_ARGSORT:
859            return op_argsort(octx);
860
861        case HTP_OP_SSM_CONV:
862            return op_ssm_conv(octx);
863
864        case HTP_OP_CUMSUM:
865            return op_cumsum(octx);
866
867        case HTP_OP_FILL:
868            return op_fill(octx);
869
870        case HTP_OP_DIAG:
871            return op_diag(octx);
872
873        case HTP_OP_SOLVE_TRI:
874            return op_solve_tri(octx);
875
876        case HTP_OP_PAD:
877            return op_pad(octx);
878
879        case HTP_OP_IM2COL:
880            return op_im2col(octx);
881
882        case HTP_OP_CONCAT:
883            return op_concat(octx);
884
885        case HTP_OP_GATED_DELTA_NET:
886            return op_gated_delta_net(octx);
887
888        case HTP_OP_TRI:
889            return op_unary(octx);
890
891        case HTP_OP_INVALID:
892            break;
893    }
894
895    FARF(ERROR, "Unknown Op %u", octx->op);
896    return HTP_STATUS_NO_SUPPORT;
897}
898
899static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) {
900    b->base = NULL;
901
902    for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
903        struct htp_mmap *m = ctx->mmap + i;
904        if (m->size && m->fd == b->fd) {
905            b->base   = m->base;
906            *m_reuse |= (1 << i);
907            return true;
908        }
909    }
910
911    return false;
912}
913
914static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) {
915    if (m->size) {
916        FARF(ALWAYS, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
917        htp_munmap((void *) m->base, m->size);
918        m->size = 0;
919        m->base = 0;
920        m->fd   = -1;
921    }
922}
923
924static inline bool mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) {
925    if (b->base) return true; // already mapped
926
927    // find unused mapping
928    for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) {
929        struct htp_mmap *m = &ctx->mmap[i];
930        if (!m->size) {
931            void *va = htp_mmap(b->fd, b->size);
932            if (va == NULL) {
933                FARF(HIGH, "mmap failed (will attempt defrag) : fd %u size %u", b->fd, (uint32_t) b->size);
934                return false;
935            }
936
937            m->base   = b->base = (uint64_t) va;
938            m->fd     = b->fd;
939            m->size   = b->size;
940
941            FARF(ALWAYS, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
942            return true;
943        }
944    }
945
946    FARF(ERROR, "mmap failed : exceeded mapping capacity limit of %u", HTP_MAX_MMAPS);
947    return false;
948}
949
950static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t n_bufs) {
951    uint32_t m_reuse = 0; // mmap reuse mask (index from ctx->mmap array)
952    uint32_t b_reuse = 0; // buf reuse count
953
954    uint64_t m_vmem  = 0; // mapped vmem
955    uint64_t e_vmem  = 0; // extra  vmem
956
957    // See what we can reuse
958    for (uint32_t i=0; i < n_bufs; i++) {
959        struct htp_buf_desc *b = bufs + i;
960        if (reuse_buf(ctx, &m_reuse, b)) { b_reuse++; } else { e_vmem += b->size; }
961        FARF(HIGH, "prep-buf #%u : pass0 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags);
962    }
963
964    if (b_reuse == n_bufs) return; // all bufs reuse existing mappings
965
966    // See how much vmem we have mmaped right now
967    for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) { m_vmem += ctx->mmap[i].size; }
968
969    FARF(HIGH, "prep-bufs : pass1 mmap-vmem %zu extra-vmem %zu max-vmem %zu : n-bufs %u b-reuse %u",
970            (size_t) m_vmem, (size_t) e_vmem, (size_t) ctx->max_vmem, n_bufs, b_reuse);
971
972    if ((m_vmem + e_vmem) > ctx->max_vmem) {
973        // Drop unused mappings
974        for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) {
975            bool used = m_reuse & (1<<i);
976            if (!used) { drop_mmap(ctx, ctx->mmap + i); }
977        }
978    }
979
980    // Create missing mappings (pass 1)
981    bool mmap_ok = true;
982    for (uint32_t i=0; i < n_bufs; i++) {
983        struct htp_buf_desc *b = bufs + i;
984        if (!mmap_buf(ctx, b)) {
985            mmap_ok = false;
986            break;
987        }
988        FARF(HIGH, "prep-buf #%u : pass1 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags);
989    }
990
991    if (!mmap_ok) {
992        // Attempt clean defragmentation: drop all mappings and remap (pass 2)
993        FARF(HIGH, "prep-bufs : dropping all mappings to defragment address space");
994        for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { drop_mmap(ctx, ctx->mmap + i); }
995
996        for (uint32_t i=0; i < n_bufs; i++) {
997            struct htp_buf_desc *b = bufs + i;
998            b->base = 0;
999            if (!mmap_buf(ctx, b)) {
1000                FARF(ERROR, "prep-bufs : mmap failed after defragmentation (fd %u size %u)", b->fd, (uint32_t) b->size);
1001                abort();
1002            }
1003            FARF(HIGH, "prep-buf #%u : pass2 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags);
1004        }
1005    }
1006}
1007
1008static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) {
1009    uint32_t offset = t->data;
1010    uint32_t size   = t->size;
1011    uint32_t bi     = t->bi;
1012
1013    t->data  = (uint32_t) (bufs[bi].base + offset);  // update data to the actual pointer
1014
1015    FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data,
1016        t->ne[0], t->ne[1], t->ne[3], t->ne[3]);
1017}
1018
1019static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) {
1020    for (uint32_t i=0; i < n_tens; i++) {
1021        prep_tensor(ctx, bufs, tens, i, tens + i);
1022    }
1023}
1024
1025static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) {
1026    memset(&ctx->mdev, 0, sizeof(ctx->mdev));
1027    ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12);
1028}
1029
1030static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs,
1031                       struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) {
1032    memcpy(octx->op_params,     op->params, sizeof(octx->op_params));
1033    memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params));
1034    octx->flags         = op->flags;
1035    octx->op            = op->opcode;
1036    octx->n_threads     = octx->ctx->n_threads;
1037    octx->n_threads_div = octx->ctx->n_threads_div;
1038
1039    FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags);
1040
1041    // Prep input tensors
1042    for (uint32_t i=0; i<HTP_OP_MAX_INPUTS; i++) {
1043        uint16_t src_idx = op->src[i];
1044        if (src_idx == 0xffff) {
1045            octx->src[i]     = NULL;
1046            octx->src_dma[i] = NULL;
1047            continue;
1048        }
1049
1050        struct htp_tensor *src = tens + src_idx;
1051        octx->src[i]     = src;
1052        octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;
1053
1054        FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size,
1055            src->ne[0], src->ne[1], src->ne[2], src->ne[3]);
1056    }
1057
1058    htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS);
1059
1060    // Prep output tensors
1061    for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {
1062        uint16_t dst_idx = op->dst[i];
1063        if (dst_idx == 0xffff) {
1064            octx->dsts[i]    = NULL;
1065            octx->dst_dma[i] = NULL;
1066            continue;
1067        }
1068        struct htp_tensor *dst = tens + dst_idx;
1069        octx->dsts[i]    = dst;
1070        octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;
1071
1072        FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size,
1073            dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
1074    }
1075
1076    htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS);
1077
1078    htp_mdev_group_barrier(octx);
1079
1080    int status = execute_op(octx);
1081
1082    htp_ops_context_set_status(octx, status);
1083
1084    octx->src0_spad.src = NULL;
1085    octx->src1_spad.src = NULL;
1086    octx->src2_spad.src = NULL;
1087    octx->src3_spad.src = NULL;
1088    octx->dst_spad.src  = NULL;
1089
1090    return octx->status;
1091}
1092
1093static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) {
1094    dspqueue_t queue = ctx->dsp_queue;
1095    int err;
1096
1097    const uint32_t n_bufs = req->n_bufs;
1098    const uint32_t n_tens = req->n_tensors;
1099    const uint32_t n_ops  = req->n_ops;
1100
1101    const uint32_t b_size = sizeof(struct htp_buf_desc)  * n_bufs;
1102    const uint32_t t_size = sizeof(struct htp_tensor)    * n_tens;
1103    const uint32_t o_size = sizeof(struct htp_op_desc)   * n_ops;
1104    const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;
1105    const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc);
1106
1107    if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) {
1108        FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size);
1109        return;
1110    }
1111
1112    FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq,
1113            n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
1114
1115    // Setup descriptor pointers
1116    uint8_t * m_ptr = dbuf->ptr;
1117    struct htp_buf_desc* bufs = (struct htp_buf_desc*)  m_ptr; m_ptr += b_size;
1118    struct htp_tensor*   tens = (struct htp_tensor*)    m_ptr; m_ptr += t_size;
1119    struct htp_op_desc*   ops = (struct htp_op_desc*)   m_ptr; m_ptr += o_size;
1120    struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr;
1121
1122    struct profile_data batch_prof;
1123    profile_start(HTP_PROF_BASIC, &batch_prof);
1124
1125    memset(ctx->trace, 0, sizeof(ctx->trace));
1126    if (ctx->profiler == HTP_PROF_TRACE) {
1127        struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);
1128        for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
1129            ctx->trace[t].events     = &trace_events[t * req->n_traces];
1130            ctx->trace[t].max_events = req->n_traces;
1131        }
1132    }
1133
1134    // Clean cache at the start of the batch
1135    htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
1136    qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
1137    hex_l2fetch_block(ctx, ctx->footprint);
1138    memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
1139    htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
1140
1141    htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_BUFF, 0);
1142    prep_op_bufs(ctx, bufs, n_bufs);
1143    htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_BUFF, 0);
1144
1145    prep_tensors(ctx, bufs, tens, n_tens);
1146
1147    struct htp_ops_context *octx = &ctx->octx;
1148    memset(octx, 0, sizeof(*octx));
1149    octx->n_threads     = ctx->n_threads;
1150    octx->n_threads_div = ctx->n_threads_div;
1151    octx->ctx           = ctx;
1152
1153    mdev_group_init(ctx, req);
1154
1155    work_queue_wakeup(ctx->work_queue);
1156    if (ctx->hmx_queue) {
1157        hmx_queue_wakeup(ctx->hmx_queue);
1158    }
1159
1160    int op_status = HTP_STATUS_OK;
1161    octx->status  = HTP_STATUS_OK;
1162    for (uint32_t i = 0; i < n_ops; i++) {
1163        struct profile_data prof;
1164
1165        profile_start(ctx->profiler, &prof);
1166
1167        op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]);
1168
1169        profile_stop(ctx->profiler, &prof);
1170
1171        htp_ops_context_set_status(octx, op_status);
1172
1173        if (ctx->profiler) {
1174            pds[i].opcode = ops[i].opcode;
1175            pds[i].usecs  = prof.usecs;
1176            pds[i].cycles_start = prof.cycles_start;
1177            pds[i].cycles_stop = prof.cycles_stop;
1178            for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {
1179                pds[i].pmu[j] = prof.pmu_counters[j];
1180            }
1181        }
1182    }
1183
1184    if (ctx->hmx_queue) {
1185        hmx_queue_suspend(ctx->hmx_queue);
1186        hmx_queue_flush(ctx->hmx_queue);
1187    }
1188    work_queue_suspend(ctx->work_queue);
1189
1190    // Flush remaining dirty tensors at the end of the batch
1191    htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
1192    qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
1193    htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
1194
1195    htp_mdev_group_barrier(octx);
1196
1197    profile_stop(HTP_PROF_BASIC, &batch_prof);
1198
1199    struct htp_opbatch_rsp rsp;
1200    memset(&rsp, 0, sizeof(rsp));
1201    rsp.seq          = req->seq;
1202    rsp.status       = octx->status;
1203    rsp.n_bufs       = n_bufs;
1204    rsp.n_tensors    = n_tens;
1205    rsp.n_ops        = n_ops;
1206    rsp.usecs        = batch_prof.usecs;
1207    rsp.cycles_start = batch_prof.cycles_start;
1208    rsp.cycles_stop  = batch_prof.cycles_stop;
1209
1210    if (ctx->profiler == HTP_PROF_TRACE) {
1211        for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
1212            rsp.n_traces[t] = ctx->trace[t].count;
1213        }
1214    }
1215
1216    struct dspqueue_buffer write_dbuf = *dbuf;
1217    write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
1218
1219    err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE);
1220    if (err != 0) {
1221        FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err);
1222    }
1223}
1224
1225#define DSPQUEUE_READ_TIMEOUT_USEC 5000
1226#define DSPQUEUE_POLL_TIMEOUT_USEC 100
1227#define DSPQUEUE_POLL_COUNT        100
1228
1229static void process_ops(struct htp_context * ctx) {
1230    dspqueue_t queue = ctx->dsp_queue;
1231    int err;
1232
1233    uint32_t poll_count = DSPQUEUE_POLL_COUNT;
1234
1235    vtcm_acquire(ctx);
1236
1237    while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {
1238        struct htp_opbatch_req req;
1239        uint32_t r_size = sizeof(req);
1240
1241        struct dspqueue_buffer dbuf;
1242        uint32_t n_dbufs = 1;
1243        uint32_t flags   = 0;
1244
1245        err = dspqueue_read_noblock(queue, &flags, n_dbufs, &n_dbufs, &dbuf, r_size, &r_size, (uint8_t *) &req);
1246        if (err == AEE_EWOULDBLOCK) {
1247            if (--poll_count) {
1248                qurt_sleep(DSPQUEUE_POLL_TIMEOUT_USEC);
1249                continue;
1250            }
1251            break;
1252        }
1253
1254        if (err != 0) {
1255            FARF(ERROR, "dspqueue_read_noblock failed: 0x%08x", (unsigned) err);
1256            break;
1257        }
1258
1259        if (r_size < sizeof(req) || n_dbufs != 1) {
1260            FARF(ERROR, "invalid request : size %u n-dbufs %u", r_size, n_dbufs);
1261            continue;
1262        }
1263
1264        // Reset poll count for valid requests
1265        poll_count = DSPQUEUE_POLL_COUNT;
1266
1267        process_opbatch(ctx, &req, &dbuf);
1268    }
1269
1270    vtcm_release(ctx);
1271}
1272
1273static void htp_packet_callback(dspqueue_t queue, int error, void * context) {
1274    (void) queue;
1275    (void) error;
1276    struct htp_handle * h = (struct htp_handle *) context;
1277    if (h && h->ctx) {
1278        process_ops(h->ctx);
1279    }
1280}
1281
1282static void htp_main_thread(void * context) {
1283    struct htp_context * ctx = (struct htp_context *) context;
1284
1285    FARF(HIGH, "htp-main-thread: started");
1286
1287    while (!atomic_load(&ctx->killed)) {
1288        uint32_t flags = 0;
1289        uint32_t num_buffers = 0;
1290        uint32_t message_length = 0;
1291
1292        int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000);
1293        if (err == 0) {
1294            process_ops(ctx);
1295        } else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) {
1296            continue;
1297        } else {
1298            FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err);
1299            break;
1300        }
1301    }
1302
1303    FARF(HIGH, "htp-main-thread: stopped");
1304}