char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Kevin Hoppervulkan: add DeepSeek-V4 hyper-connection fused ops (DSV4_HC_COMB/PRE/POST) (#26578)7a333e724

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.3 KiB83 linesraw
1#version 450
2
3#extension GL_EXT_control_flow_attributes : require
4
5// Fan one stream back out to hc streams and add the combination-weighted
6// residuals:
7//
8//   dst[i0, idst, it] = x[i0, it]*post[idst, it]
9//                     + sum_isrc residual[i0, isrc, it]*comb[idst, isrc, it]
10
11layout(constant_id = 0) const uint BLOCK_SIZE = 256;
12
13layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
14
15layout(push_constant) uniform parameter
16{
17    uint n_embd;
18    uint n_tokens;
19
20    uint nbx0; uint nbx1;              // x
21    uint nbr0; uint nbr1; uint nbr2;   // residual
22    uint nbp0; uint nbp1;              // post
23    uint nbc0; uint nbc1; uint nbc2;   // comb
24    uint nbd0; uint nbd1; uint nbd2;   // dst
25
26    uint x_offset;
27    uint r_offset;
28    uint p_offset;
29    uint c_offset;
30    uint d_offset;
31};
32
33layout(binding = 0, std430) readonly buffer X { float data_x[]; };
34layout(binding = 1, std430) readonly buffer R { float data_r[]; };
35layout(binding = 2, std430) readonly buffer P { float data_p[]; };
36layout(binding = 3, std430) readonly buffer C { float data_c[]; };
37layout(binding = 4, std430) writeonly buffer D { float data_d[]; };
38
39const uint hc = 4;
40
41shared float post_s[hc];
42shared float comb_s[hc * hc];
43
44void main() {
45    const uint tid = gl_LocalInvocationID.x;
46    const uint it  = gl_WorkGroupID.y;
47
48    if (tid < hc) {
49        post_s[tid] = data_p[p_offset + tid * nbp0 + it * nbp1];
50    }
51    if (tid < hc * hc) {
52        const uint idst = tid & 3;
53        const uint isrc = tid >> 2;
54        comb_s[tid] = data_c[c_offset + idst * nbc0 + isrc * nbc1 + it * nbc2];
55    }
56    barrier();
57
58    // After the barrier, so every invocation reaches it.
59    const uint i0 = gl_WorkGroupID.x * BLOCK_SIZE + tid;
60    if (i0 >= n_embd) {
61        return;
62    }
63
64    const float xv = data_x[x_offset + i0 * nbx0 + it * nbx1];
65
66    const uint rb = r_offset + i0 * nbr0 + it * nbr2;
67
68    float r[hc];
69    [[unroll]]
70    for (uint isrc = 0; isrc < hc; ++isrc) {
71        r[isrc] = data_r[rb + isrc * nbr1];
72    }
73
74    [[unroll]]
75    for (uint idst = 0; idst < hc; ++idst) {
76        float result = xv * post_s[idst];
77        [[unroll]]
78        for (uint isrc = 0; isrc < hc; ++isrc) {
79            result = fma(r[isrc], comb_s[idst + hc * isrc], result);
80        }
81        data_d[d_offset + i0 * nbd0 + idst * nbd1 + it * nbd2] = result;
82    }
83}