char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Todor Boinovskihexagon: rope updates (#28628)6d9c82ea2

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.8 KiB56 linesraw
1#ifndef HTP_ROPE_OPS_H
2#define HTP_ROPE_OPS_H
3
4#include "hex-common.h"
5#include "hex-fastdiv.h"
6
7#define HTP_ROPE_SPAD_BLOCK  8
8#define HTP_ROPE_SPAD_NSLOTS 4
9#define HTP_ROPE_SPAD_NROWS  (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS)
10
11struct htp_rope_kernel_params {
12    uint32_t n_threads;
13    uint32_t src0_nrows;
14    uint32_t src0_nrows_per_thread;
15    uint32_t vtcm_size;
16    uint32_t spad_per_thread;
17    uint32_t theta_cache_offset;
18    uint32_t src0_row_size_aligned;
19
20    struct fastdiv_values div_ne2_ne1;
21    struct fastdiv_values div_ne1;
22};
23
24#if defined(__cplusplus)
25static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");
26#else
27_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");
28#endif
29
30struct htp_rope_vtcm_layout {
31    size_t total_bytes;
32    size_t bytes_per_thread;
33    size_t theta_cache_size_aligned;
34    size_t src0_row_size_aligned;
35};
36
37static inline void htp_rope_vtcm_layout_build(
38    struct htp_rope_vtcm_layout * layout,
39    uint32_t ne00,
40    uint32_t n_threads
41) {
42    const size_t src0_row_size            = ne00 * sizeof(float);
43    const size_t src0_row_size_aligned    = hex_round_up((uint32_t) src0_row_size, 128);
44    const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256);
45
46    layout->src0_row_size_aligned    = src0_row_size_aligned;
47    layout->theta_cache_size_aligned = theta_cache_size_aligned;
48    layout->bytes_per_thread         = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;
49    layout->total_bytes              = layout->bytes_per_thread * n_threads;
50}
51
52static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) {
53    return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned;
54}
55
56#endif // HTP_ROPE_OPS_H