char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Sigbjørn Skjæretsycl : add Q2_K reordered MMVQ and ESIMD kernels (again) (#27490)867229003

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.1 KiB204 linesraw
1//
2// MIT license
3// Copyright (C) 2025 Codeplay Software Ltd.
4// Copyright (C) 2025 Intel Corporation
5// SPDX-License-Identifier: MIT
6//
7
8//
9// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
10// See https://llvm.org/LICENSE.txt for license information.
11// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
12//
13
14#ifndef GGML_SYCL_QUANTS_HPP
15#define GGML_SYCL_QUANTS_HPP
16
17#include <utility>
18
19#include "ggml-common.h"
20#include "ggml.h"
21
22namespace ggml_sycl_reordered {
23
24// The reordered block moves quants (qs) and  scales(d) to two
25// uniform regions of memory that is contiguous in the same tensor.
26// What this means is that instead of having:
27// [d0, qs0] [d1, qs1] [d2, qs2] ... [dN, qsN]
28// We have:
29// [qs0, qs1, qs2, ..., qsN]  [d0, d1, d2, ..., dN]
30//
31// Notes: out-of-bounds qs will run into d values
32// Alignment relies on the allocated size of qs
33
34template <ggml_type type> struct block_q_t;
35
36// qk number of weights / quants in a block
37// qr number of weights in a byte (described as 'before dequantization')
38//    for quantization types that has low and high bits split, qr is calculated with
39//    using the lower bits, e.g for Q6 quants QR6 is 2
40// qi number of 32 bit integers needed to represent all the quants from a block (`qs` field)
41// See ggml-common.h to see how these are calculated
42template <> struct block_q_t<GGML_TYPE_Q4_0> {
43    struct traits {
44        static constexpr uint32_t qk       = QK4_0;
45        static constexpr uint32_t qi       = QI4_0;
46        static constexpr uint32_t qr       = QR4_0;
47        static constexpr uint32_t vdr_mmvq = 2;
48    };
49
50    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
51        return { block_index * (QK4_0 / QR4_0), 0 };
52    }
53
54    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
55        return { (ncols / QR4_0 * nrows) + block_index * sizeof(ggml_half), 0 };
56    }
57
58    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
59};
60
61template <> struct block_q_t<GGML_TYPE_Q2_K> {
62    struct traits {
63        static constexpr uint32_t qk       = QK_K;
64        static constexpr uint32_t qi       = QI2_K;
65        static constexpr uint32_t qr       = QR2_K;
66        static constexpr uint32_t vdr_mmvq = 1;
67    };
68
69    // Reordered layout: [qs (QK_K/4 per block)] [scales (QK_K/16 per block)] [dm]
70    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* n_blocks */) {
71        return { block_index * (QK_K / 4), 0 };
72    }
73
74    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
75        auto nblocks        = (nrows * (ncols / QK_K));
76        auto total_qs_bytes = nblocks * (QK_K / 4);
77        return { total_qs_bytes + block_index * (QK_K / 16),
78                 total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half2) };
79    }
80
81    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
82};
83
84template <> struct block_q_t<GGML_TYPE_Q3_K> {
85    struct traits {
86        static constexpr uint32_t qk       = QK_K;
87        static constexpr uint32_t qi       = QI3_K;
88        static constexpr uint32_t qr       = QR3_K;
89        static constexpr uint32_t vdr_mmvq = 1;
90    };
91
92    // Reordered layout: [qs (QK_K/4 per block)] [hmask (QK_K/8 per block)] [scales] [d]
93    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
94        auto qs_offset    = block_index * (QK_K / 4);
95        auto hmask_offset = n_blocks * (QK_K / 4) + block_index * (QK_K / 8);
96        return { qs_offset, hmask_offset };
97    }
98
99    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
100        auto nblocks        = (nrows * (ncols / QK_K));
101        auto total_qs_bytes = nblocks * (QK_K / 4) + nblocks * (QK_K / 8);
102        return { total_qs_bytes + block_index * 12,
103                 total_qs_bytes + nblocks * 12 + block_index * sizeof(ggml_half) };
104    }
105
106    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
107};
108
109template <> struct block_q_t<GGML_TYPE_Q4_K> {
110    struct traits {
111        static constexpr uint32_t qk       = QK_K;
112        static constexpr uint32_t qi       = QI4_K;
113        static constexpr uint32_t qr       = QR4_K;
114        static constexpr uint32_t vdr_mmvq = 2;
115    };
116
117    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
118        return { block_index * (traits::qk / traits::qr), 0 };
119    }
120
121    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
122        auto nblocks = (nrows * (ncols / QK_K));
123        return { nblocks * (QK_K / 2) + (block_index * K_SCALE_SIZE),
124                 (nblocks * QK_K / 2) + (nblocks * K_SCALE_SIZE) + (block_index * sizeof(ggml_half2)) };
125    }
126
127    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
128};
129
130template <> struct block_q_t<GGML_TYPE_Q5_K> {
131    struct traits {
132        static constexpr uint32_t qk       = QK_K;
133        static constexpr uint32_t qi       = QI5_K;
134        static constexpr uint32_t qr       = QR5_K;
135        static constexpr uint32_t vdr_mmvq = 2;
136    };
137
138    // Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales] [dm]
139    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
140        auto qs_offset = block_index * (QK_K / 2);
141        auto qh_offset = n_blocks * (QK_K / 2) + block_index * (QK_K / 8);
142        return { qs_offset, qh_offset };
143    }
144
145    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
146        auto nblocks        = (nrows * (ncols / QK_K));
147        auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 8);
148        return { total_qs_bytes + block_index * K_SCALE_SIZE,
149                 total_qs_bytes + nblocks * K_SCALE_SIZE + block_index * sizeof(ggml_half2) };
150    }
151
152    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
153};
154
155template <> struct block_q_t<GGML_TYPE_Q6_K> {
156    struct traits {
157        static constexpr uint32_t qk       = QK_K;
158        static constexpr uint32_t qi       = QI6_K;
159        static constexpr uint32_t qr       = QR6_K;
160        static constexpr uint32_t vdr_mmvq = 1;
161    };
162
163    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
164        auto low_bits_index  = block_index * (QK_K / QR6_K);
165        // the index of high bits it's after all low bits
166        auto high_bits_index = n_blocks * (QK_K / 2) + (block_index * (QK_K / 4));
167        return { low_bits_index, high_bits_index };
168    }
169
170    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
171        auto nblocks        = (nrows * (ncols / QK_K));
172        auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 4);
173        auto block_scales   = total_qs_bytes + block_index * (QK_K / 16);
174        auto sb_scale       = total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half);
175        return { block_scales, sb_scale };
176    }
177
178    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
179};
180
181template <> struct block_q_t<GGML_TYPE_Q8_0> {
182    struct traits {
183        static constexpr uint32_t qk       = QK8_0;      // 32
184        static constexpr uint32_t qi       = QI8_0;      // 8
185        static constexpr uint32_t qr       = QR8_0;      // 1
186        static constexpr uint32_t vdr_mmvq = 4;
187    };
188
189    // Q8_0 reorder layout: [qs0|qs1|...|qsN][d0|d1|...|dN]
190    // Each block has 32 int8 weights (32 bytes) followed by all scales
191    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
192        return { block_index * QK8_0, 0 };
193    }
194
195    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
196        return { (ncols * nrows) + block_index * sizeof(ggml_half), 0 };
197    }
198
199    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }  // 1
200};
201
202}  // namespace ggml_sycl_reordered
203
204#endif  // GGML_SYCL_QUANTS_HPP