char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: further improved pipeline of the core bits (L2, DMA, MM, FA) (#26049)0a50d9909

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.7 KiB104 linesraw
1#include "dma-queue.h"
2
3#include <stdbool.h>
4#include <stdlib.h>
5#include <string.h>
6
7#pragma clang diagnostic ignored "-Wunused-function"
8
9static inline uint32_t pow2_ceil(uint32_t x) {
10    if (x <= 1) {
11        return 1;
12    }
13    int p = 2;
14    x--;
15    while (x >>= 1) {
16        p <<= 1;
17    }
18    return p;
19}
20
21static inline uintptr_t align_up(uintptr_t addr, size_t align) {
22    return (addr + align - 1) & ~(align - 1);
23}
24
25size_t dma_queue_sizeof(size_t capacity) {
26    capacity = pow2_ceil(capacity);
27
28    size_t size_q      = sizeof(dma_queue);
29    size_t offset_r    = align_up(size_q, HEX_L2_LINE_SIZE);
30    size_t size_r      = sizeof(dma_ring);
31    size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
32    size_t size_desc   = capacity * sizeof(dma_descriptor_2d);
33    size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
34    size_t size_dptr   = capacity * sizeof(dma_ptr);
35
36    return offset_dptr + size_dptr;
37}
38
39size_t dma_queue_alignof(void) {
40    return HEX_L2_LINE_SIZE;
41}
42
43dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) {
44    capacity = pow2_ceil(capacity);
45
46    size_t size_q      = sizeof(dma_queue);
47    size_t offset_r    = align_up(size_q, HEX_L2_LINE_SIZE);
48    size_t size_r      = sizeof(dma_ring);
49    size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
50    size_t size_desc   = capacity * sizeof(dma_descriptor_2d);
51    size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
52    size_t size_dptr   = capacity * sizeof(dma_ptr);
53
54    size_t total_size = offset_dptr + size_dptr;
55    memset(ptr, 0, total_size);
56
57    dma_queue * q = (dma_queue *) ptr;
58    dma_ring * r  = (dma_ring *) ((uintptr_t) ptr + offset_r);
59
60    q->ring    = r;
61    q->nocache = 0;
62    q->alias   = false;
63
64    r->trace     = trace;
65    r->vtcm_base = vtcm_base;
66    r->vtcm_end  = vtcm_base + vtcm_size;
67    r->capacity  = capacity;
68    r->idx_mask  = capacity - 1;
69    r->push_idx  = 0;
70    r->pop_idx   = 0;
71
72    r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc);
73    r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr);
74    r->tail = &r->desc[capacity - 1];
75
76    FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size);
77
78    return q;
79}
80
81void dma_queue_free(dma_queue_t q) {
82    (void) q;
83}
84
85size_t dma_queue_alias_sizeof(void) {
86    return sizeof(dma_queue);
87}
88
89dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) {
90    dma_queue * q = (dma_queue *) ptr;
91    memset(q, 0, sizeof(dma_queue));
92
93    q->ring    = main_q->ring;
94    q->nocache = nocache;
95    q->alias   = true;
96
97    return q;
98}
99
100void dma_queue_alias_free(dma_queue_t q) {
101    (void) q;
102}
103
104