char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Nathanw1014vulkan : dequant q8_0 KV once in coopmat1 (#25494)dc72703fc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.2 KiB42 linesraw
1#version 450
2
3#include "dequant_head.glsl"
4
5layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
6
7layout (binding = 0) readonly buffer A {block_q8_0 data_a[];};
8layout (binding = 1) writeonly buffer D {D_TYPE data_b[];};
9
10void main() {
11    const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64;
12
13    const uint tid = gl_LocalInvocationID.x % 64;
14    const uint il  = tid/32;
15    const uint ir  = tid%32;
16    const uint ib = 32*i + ir;
17    if (ib >= p.nel / 32) {
18        return;
19    }
20
21#ifdef DEQUANT_TRANSPOSE
22    // read [HS, NH, KV, NS], write [HS, KV, NH, NS]
23    const uint HS = p.M, NH = p.K, KVn = p.stride_a;
24    const uint e0 = ib * 32;
25    const uint b_idx = (e0 % HS)
26                     + ((e0 / (HS * NH)) % KVn) * HS
27                     + ((e0 / HS) % NH) * (HS * KVn)
28                     + (e0 / (HS * NH * KVn)) * (HS * KVn * NH)
29                     + 16 * il;
30#else
31    const uint b_idx = 1024*i + 32*ir + 16*il;
32#endif
33
34    const float d = float(data_a[ib].d);
35
36    const uint q_idx = 16*il;
37
38    [[unroll]] for (uint l = 0; l < 16; l += 2) {
39        data_b[b_idx + l    ] = D_TYPE(d * data_a[ib].qs[q_idx + l    ]);
40        data_b[b_idx + l + 1] = D_TYPE(d * data_a[ib].qs[q_idx + l + 1]);
41    }
42}