char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.2 KiB89 linesraw
1#ifndef HTP_FENCE_H
2#define HTP_FENCE_H
3
4#include <stdatomic.h>
5#include <stdint.h>
6
7#include <HAP_farf.h>
8
9#include "hex-utils.h"
10#include "htp-ops.h"
11#include "htp-ctx.h"
12
13static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) {
14    return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE);
15}
16
17static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) {
18    atomic_uint * fence = (atomic_uint *) fence_ptr;
19    atomic_store(&fence[1], status);
20    atomic_store(&fence[0], seq);
21    asm volatile ("syncht" : : : "memory");
22    Q6_dccleaninva_A((void *) fence);
23}
24
25static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) {
26    const atomic_uint * fence = (const atomic_uint *) fence_ptr;
27    Q6_dccleaninva_A((void *) fence);
28    asm volatile ("syncht" : : : "memory");
29    *seq = atomic_load(&fence[0]);
30    *status = atomic_load(&fence[1]);
31}
32
33static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) {
34    struct htp_context * ctx = octx->ctx;
35    if (ctx->mdev.count <= 1) {
36        return;
37    }
38
39    const uint32_t seq = ++ctx->mdev.fence_seq;
40
41    struct htp_thread_trace * tr = &ctx->trace[0];
42    htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
43
44    const uint32_t mdev_idx   = ctx->mdev.idx;
45    const uint32_t mdev_count = ctx->mdev.count;
46
47    uint8_t * fence_base   = ctx->mdev.fence_base;
48    atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx);
49    htp_fence_write(my_fence, seq, octx->status);
50
51    for (uint32_t d = 0; d < mdev_count; d++) {
52        if (d == mdev_idx) continue;
53        atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d);
54        uint64_t spins = 0;
55        while (1) {
56            uint32_t peer_seq;
57            uint32_t peer_status;
58            htp_fence_read(peer_fence, &peer_seq, &peer_status);
59            if ((int32_t)(peer_seq - seq) >= 0) {
60                if (peer_status > HTP_STATUS_OK) {
61                    FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n",
62                         mdev_idx, d, peer_status, seq);
63                    htp_ops_context_set_status(octx, peer_status);
64                }
65                break;
66            }
67            if (++spins == 10000) {
68                FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n",
69                     mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq));
70            }
71            if (spins > HTP_FENCE_TIMEOUT) {
72                FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n",
73                     mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq);
74                htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR);
75                break;
76            }
77            hex_pause();
78        }
79    }
80    asm volatile ("syncht" : : : "memory");
81
82    if (octx->status > HTP_STATUS_OK) {
83        htp_fence_write(my_fence, seq, octx->status);
84    }
85
86    htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
87}
88
89#endif // HTP_FENCE_H