hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Qwen3.5, memory optimization, and test-recurrent-state-rollback (#26952)aee56b3ab2026-08-14

1#include "ggml-quants.h"
2
3#include "ggml-common.h"
4#include "ggml-impl.h"
5#include "ggml-openvino-extra.h"
6#include "ggml.h"
7
8#include <algorithm>
9#include <cassert>
10#include <cmath>
11#include <cstddef>
12#include <cstdint>
13#include <limits>
14#include <memory>
15#include <openvino/core/except.hpp>
16#include <openvino/core/node.hpp>
17#include <openvino/core/node_output.hpp>
18#include <openvino/core/parallel.hpp>
19#include <openvino/core/shape.hpp>
20#include <openvino/core/type/element_type.hpp>
21#include <openvino/core/type/element_type_traits.hpp>
22#include <openvino/core/type/float16.hpp>
23#include <openvino/core/type/float4_e2m1.hpp>
24#include <openvino/core/type/float8_e8m0.hpp>
25#include <openvino/op/add.hpp>
26#include <openvino/op/constant.hpp>
27#include <openvino/op/convert.hpp>
28#include <openvino/op/multiply.hpp>
29#include <openvino/op/reshape.hpp>
30#include <openvino/op/subtract.hpp>
31#include <openvino/op/util/attr_types.hpp>
32#include <openvino/pass/constant_folding.hpp>
33#include <openvino/runtime/tensor.hpp>
34#include <string>
35#include <vector>
36
37void unpack_32_4(const uint8_t * data, uint8_t * dst) {
38    std::fill_n(dst, 16, 0);
39    for (int j = 0; j < 16; ++j) {
40        uint8_t x = (data[j] & 0x0F);
41        uint8_t y = (data[j] >> 4);
42        if (j % 2 != 0) {
43            x <<= 4;
44            y <<= 4;
45        }
46        dst[j / 2] |= x;
47        dst[8 + j / 2] |= y;  // Last 16 weights are in the higher bits
48    }
49}
50
51static constexpr size_t MXFP4_BLOCK_SIZE = 32;
52static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
53static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
54
55static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
56    for (int j = 0; j < static_cast<int>(MXFP4_BLOCK_QS_SIZE); j += 2) {
57        const uint8_t v0 = data[j] & 0x0F;
58        const uint8_t v1 = (data[j + 1] & 0x0F) << 4;
59        const uint8_t v16 = data[j] >> 4;
60        const uint8_t v17 = data[j + 1] & 0xF0;
61        dst[j / 2] = v0 | v1;
62        dst[MXFP4_BLOCK_SIZE / 4 + j / 2] = v16 | v17;
63    }
64}
65
66void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) {
67    GGML_ASSERT(tensor->type == GGML_TYPE_MXFP4);
68    GGML_ASSERT(weights_arr.get_element_type() == ov::element::f4e2m1);
69    GGML_ASSERT(scales_arr.get_element_type() == ov::element::f8e8m0);
70
71    const auto * data = static_cast<const uint8_t *>(tensor->data);
72    auto * weights = static_cast<uint8_t *>(weights_arr.data());
73    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f8e8m0>::value_type>();
74    const size_t n_blocks = scales_arr.get_size();
75
76    ov::parallel_for(n_blocks, [&](size_t i) {
77        const uint8_t * block = data + i * MXFP4_BLOCK_BYTES;
78        pack_32_mxfp4_for_openvino(block + sizeof(uint8_t), weights + i * MXFP4_BLOCK_QS_SIZE);
79        scales[i] = ov::float8_e8m0::from_bits(block[0]);
80    });
81}
82
83// Extracts (weight, scales, zp) from Q4_0 tensors.
84// Data layout is: |16 bit scale|32 x 4bit weights|.
85// When zp_arr is empty (symmetric), weights are stored as signed i4 (value - 8).
86void extract_q4_0_data(const ggml_tensor * tensor,
87                       ov::Tensor & weights_arr,
88                       ov::Tensor & scales_arr,
89                       ov::Tensor & zp_arr) {
90    const uint64_t bytes_per_block = 18;  // 2 bytes scale, 32x0.5 byte weights
91
92    auto * data = static_cast<uint8_t *>(tensor->data);
93    auto * weights = static_cast<uint8_t *>(weights_arr.data());
94    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
95
96    bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4);  // Signed i4 path
97
98    if (!is_symmetric) {
99        auto * zp = static_cast<uint8_t *>(zp_arr.data());
100        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
101            scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block)));
102            // Pack two 4-bit zero points per byte
103            if (i % 2 == 0) {
104                zp[i / 2] = 8;          // Lower nibble
105            } else {
106                zp[i / 2] |= (8 << 4);  // Upper nibble
107            }
108            unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16);
109        });
110    } else {
111        // Symmetric: unpack as u4 then convert to i4 by subtracting 8 (XOR each nibble)
112        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
113            scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block)));
114            unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16);
115            // Convert u4 to i4: subtract 8 from each nibble. XOR 0x88 flips each nibble by 8.
116            for (int j = 0; j < 16; ++j) {
117                weights[i * 16 + j] ^= 0x88;
118            }
119        });
120    }
121}
122
123// Extracts (weight, scales, zp) from Q4_1 tensors.
124// Data layout is: |16 bit scale|16 bit min|32 x 4bit weights|.
125void extract_q4_1_data(const ggml_tensor * tensor,
126                       ov::Tensor & weights_arr,
127                       ov::Tensor & scales_arr,
128                       ov::Tensor & zp_arr,
129                       bool use_bias) {
130    const uint64_t bytes_per_block = 20;  // 2 bytes scale, 2 bytes min, 32x0.5 byte weights
131
132    auto * data = static_cast<uint8_t *>(tensor->data);
133    auto * weights = static_cast<uint8_t *>(weights_arr.data());
134    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
135
136    if (use_bias) {
137        // Store bias (min) directly as f16 instead of computing u4 zero points
138        auto * bias = zp_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
139        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
140            float scale = static_cast<float>(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))));
141            float min = static_cast<float>(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block + 2))));
142            scales[i] = ov::float16(scale);
143            bias[i] = ov::float16(min);  // bias = min, dequant: w*s + bias
144            unpack_32_4(data + i * bytes_per_block + 4, weights + i * 16);
145        });
146    } else {
147        auto * zp = static_cast<uint8_t *>(zp_arr.data());
148        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
149            float scale = static_cast<float>(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))));
150            float min = static_cast<float>(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block + 2))));
151            scales[i] = ov::float16(scale);
152            // zp = -min / scale (bias = min, so zp = -bias/scale)
153            uint8_t zp_val = (scale != 0.0f) ? (uint8_t) std::round(-min / scale) : 0;
154            // Pack two 4-bit zero points per byte
155            if (i % 2 == 0) {
156                zp[i / 2] = zp_val & 0x0F;   // Lower nibble
157            } else {
158                zp[i / 2] |= (zp_val << 4);  // Upper nibble
159            }
160            unpack_32_4(data + i * bytes_per_block + 4, weights + i * 16);
161        });
162    }
163}
164
165// Extracts (weight, scales, zp) from Q5_1 tensors.
166// Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|.
167// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m.
168// Weights are stored as u8 (5-bit values do not fit u4), matching make_int8_weights.
169void extract_q5_1_data(const ggml_tensor * tensor,
170                       ov::Tensor & weights_arr,
171                       ov::Tensor & scales_arr,
172                       ov::Tensor & zp_arr,
173                       bool use_bias) {
174    const uint64_t bytes_per_block = 24;  // 2 scale + 2 min + 4 qh + 16 (32x0.5) weights
175    const int qk = 32;
176
177    auto * data = static_cast<uint8_t *>(tensor->data);
178    auto * weights = static_cast<uint8_t *>(weights_arr.data());  // u8 weights, one byte per weight
179    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
180
181    // Read a 16-bit little-endian value without aliasing/const-qual violations.
182    auto read_u16 = [](const uint8_t * p) {
183        uint16_t v;
184        memcpy(&v, p, sizeof(v));
185        return v;
186    };
187
188    auto unpack_block = [&](const uint8_t * block, uint8_t * dst) {
189        uint32_t qh;
190        memcpy(&qh, block + 4, sizeof(uint32_t));
191        const uint8_t * qs = block + 8;
192        for (int j = 0; j < qk / 2; ++j) {
193            const uint8_t lo = qs[j] & 0x0F;
194            const uint8_t hi = qs[j] >> 4;
195            const uint8_t bit_lo = (qh >> j) & 1;
196            const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1;
197            dst[j] = lo | (bit_lo << 4);           // first 16 weights
198            dst[j + qk / 2] = hi | (bit_hi << 4);  // last 16 weights
199        }
200    };
201
202    if (use_bias) {
203        // Store bias (min) directly as f16: dequant w*d + m
204        auto * bias = zp_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
205        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
206            const uint8_t * block = data + i * bytes_per_block;
207            float scale = static_cast<float>(ov::float16::from_bits(read_u16(block)));
208            float min = static_cast<float>(ov::float16::from_bits(read_u16(block + 2)));
209            scales[i] = ov::float16(scale);
210            bias[i] = ov::float16(min);
211            unpack_block(block, weights + i * qk);
212        });
213    } else {
214        auto * zp = static_cast<uint8_t *>(zp_arr.data());  // u8 zero points
215        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
216            const uint8_t * block = data + i * bytes_per_block;
217            float scale = static_cast<float>(ov::float16::from_bits(read_u16(block)));
218            float min = static_cast<float>(ov::float16::from_bits(read_u16(block + 2)));
219            scales[i] = ov::float16(scale);
220            // zp = -min / scale (dequant: (w - zp) * s == w*s + min)
221            zp[i] = (scale != 0.0f) ? (uint8_t) std::lround(-min / scale) : 0;
222            unpack_block(block, weights + i * qk);
223        });
224    }
225}
226
227// Extracts (weight, scales, zp) from Q8_0 tensors.
228// Data layout is: |16 bit scale|32 x 8bit weights|.
229// When zp_arr is empty (symmetric), weights are stored as signed i8 directly.
230void extract_q8_0_data(const ggml_tensor * tensor,
231                       ov::Tensor & weights_arr,
232                       ov::Tensor & scales_arr,
233                       ov::Tensor & zp_arr) {
234    const uint64_t weights_per_block = 32;
235    const uint64_t bytes_per_block = 34;  // 2 bytes scale, 32x1 byte weights
236
237    auto * data = static_cast<uint8_t *>(tensor->data);
238    auto * weights = static_cast<uint8_t *>(weights_arr.data());
239    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
240
241    bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8);  // Signed i8 path
242
243    if (!is_symmetric) {
244        auto * zp = static_cast<uint8_t *>(zp_arr.data());
245        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
246            uint8_t * block_data = data + i * bytes_per_block;
247            scales[i] = ov::float16::from_bits(*(uint16_t *) block_data);
248            zp[i] = 128;
249            for (size_t j = 0; j < weights_per_block; ++j) {
250                uint8_t x = block_data[j + 2];
251                x ^= 1 << 7;  // Convert int8 to uint8 by flipping sign bit
252                weights[i * weights_per_block + j] = x;
253            }
254        });
255    } else {
256        // Symmetric: store original int8 values directly (no unsigned bias)
257        ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
258            uint8_t * block_data = data + i * bytes_per_block;
259            scales[i] = ov::float16::from_bits(*(uint16_t *) block_data);
260            // Copy int8 weights as-is (the tensor element type is i8)
261            memcpy(weights + i * weights_per_block, block_data + 2, weights_per_block);
262        });
263    }
264}
265
266void unpack_256_4(const uint8_t * data, uint8_t * dst) {
267    // Initialize the output array with zeros
268    std::fill_n(dst, 128, 0);
269
270    for (size_t i = 0; i < 4; ++i) {
271        for (int j = 0; j < 32; ++j) {
272            uint8_t x = (data[i * 32 + j] & 0x0F);
273            uint8_t y = (data[i * 32 + j] >> 4);
274            if (j % 2 != 0) {
275                x <<= 4;
276                y <<= 4;
277            }
278            dst[i * 32 + j / 2] |= x;
279            dst[i * 32 + 16 + j / 2] |= y;  // Last 16 weights are in the higher bits
280        }
281    }
282}
283
284void extract_q4_k_data(const ggml_tensor * tensor,
285                       ov::Tensor & weights_arr,
286                       ov::Tensor & scales_arr,
287                       ov::Tensor & zp_arr,
288                       bool use_bias) {
289    const uint64_t bytes_per_block = 2 + 2 + 12 + 128;
290    const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;
291
292    auto * data = static_cast<uint8_t *>(tensor->data);
293    auto * weights = static_cast<uint8_t *>(weights_arr.data());
294    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
295
296    // For bias path, zp_arr holds f16 bias values; for zp path, it holds packed u4 zero points
297    auto * zp_u4 = use_bias ? nullptr : static_cast<uint8_t *>(zp_arr.data());
298    auto * bias_f16 = use_bias ? zp_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() : nullptr;
299
300    ov::parallel_for(n_super_block, [&](size_t i) {
301        uint8_t * block_data = data + i * bytes_per_block;
302
303        // Extract scale factors and offsets
304        float scale_scales = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data)));
305        float scale_mins = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data + 1)));
306
307        // Extract qs1 and qs2
308        uint8_t * qs1 = block_data + 4;
309
310        // Calculate scales
311        float scale_vals[8];
312        scale_vals[0] = scale_scales * static_cast<float>((*(qs1) & 0b111111));
313        scale_vals[1] = scale_scales * static_cast<float>((*(qs1 + 1) & 0b111111));
314        scale_vals[2] = scale_scales * static_cast<float>((*(qs1 + 2) & 0b111111));
315        scale_vals[3] = scale_scales * static_cast<float>((*(qs1 + 3) & 0b111111));
316        scale_vals[4] = scale_scales * static_cast<float>((*(qs1 + 8) & 0b00001111) | ((*(qs1) >> 6) << 4));
317        scale_vals[5] = scale_scales * static_cast<float>((*(qs1 + 9) & 0b00001111) | ((*(qs1 + 1) >> 6) << 4));
318        scale_vals[6] = scale_scales * static_cast<float>((*(qs1 + 10) & 0b00001111) | ((*(qs1 + 2) >> 6) << 4));
319        scale_vals[7] = scale_scales * static_cast<float>((*(qs1 + 11) & 0b00001111) | ((*(qs1 + 3) >> 6) << 4));
320
321        // Calculate min values (bias = -min)
322        float min_vals[8];
323        min_vals[0] = scale_mins * static_cast<float>((*(qs1 + 4) & 0b111111));
324        min_vals[1] = scale_mins * static_cast<float>((*(qs1 + 5) & 0b111111));
325        min_vals[2] = scale_mins * static_cast<float>((*(qs1 + 6) & 0b111111));
326        min_vals[3] = scale_mins * static_cast<float>((*(qs1 + 7) & 0b111111));
327        min_vals[4] = scale_mins * static_cast<float>((*(qs1 + 8) >> 4) | ((*(qs1 + 4) >> 6) << 4));
328        min_vals[5] = scale_mins * static_cast<float>((*(qs1 + 9) >> 4) | ((*(qs1 + 5) >> 6) << 4));
329        min_vals[6] = scale_mins * static_cast<float>((*(qs1 + 10) >> 4) | ((*(qs1 + 6) >> 6) << 4));
330        min_vals[7] = scale_mins * static_cast<float>((*(qs1 + 11) >> 4) | ((*(qs1 + 7) >> 6) << 4));
331
332        // Store scales and compute zero points or bias
333        for (int j = 0; j < 8; j++) {
334            scales[i * 8 + j] = ov::float16(scale_vals[j]);
335            if (use_bias) {
336                // Store bias = -min directly as f16, dequant: w*s + bias
337                bias_f16[i * 8 + j] = ov::float16(-min_vals[j]);
338            } else {
339                // zp = min / scale (since bias = -min and zp = -bias/scale)
340                uint8_t zp_val = (scale_vals[j] != 0.0f) ? (uint8_t) std::round(min_vals[j] / scale_vals[j]) : 0;
341                // Pack two 4-bit zero points per byte
342                size_t idx = i * 8 + j;
343                if (idx % 2 == 0) {
344                    zp_u4[idx / 2] = zp_val & 0x0F;
345                } else {
346                    zp_u4[idx / 2] |= (zp_val << 4);
347                }
348            }
349        }
350        unpack_256_4(block_data + 16, weights + i * 128);
351    });
352}
353
354void extract_q6_k_data(const ggml_tensor * tensor,
355                       ov::Tensor & weights_arr,
356                       ov::Tensor & scales_arr,
357                       ov::Tensor & zp_arr) {
358    const uint64_t bytes_per_block = 128 + 64 + 16 + 2;
359    const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;
360
361    auto * data = static_cast<uint8_t *>(tensor->data);
362    auto * weights = static_cast<uint8_t *>(weights_arr.data());
363    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
364
365    bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8);  // Signed i8 path
366
367    if (!is_symmetric) {
368        auto * zp = static_cast<uint8_t *>(zp_arr.data());
369        ov::parallel_for(n_super_block, [&](size_t i) {
370            uint8_t * block_data = data + i * bytes_per_block;
371            float scale_factor = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data + 104)));
372            for (size_t j = 0; j < 16; j++) {
373                scales[j + i * 16] =
374                    ov::float16(scale_factor * static_cast<float>(*((int8_t *) (block_data + 128 + 64 + j))));
375                zp[j + i * 16] = 32;
376            }
377            uint8_t * ql = block_data;
378            uint8_t * qh = block_data + 128;
379            for (int64_t j = 0; j < 32; ++j) {
380                weights[i * 256 + j] = (ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4);
381                weights[i * 256 + j + 32] = (ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4);
382                weights[i * 256 + j + 64] = (ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4);
383                weights[i * 256 + j + 96] = (ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4);
384                weights[i * 256 + j + 128] = (ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4);
385                weights[i * 256 + j + 160] = (ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4);
386                weights[i * 256 + j + 192] = (ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4);
387                weights[i * 256 + j + 224] = (ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4);
388            }
389        });
390    } else {
391        // Symmetric: subtract 32 from each weight to store as signed i8
392        ov::parallel_for(n_super_block, [&](size_t i) {
393            uint8_t * block_data = data + i * bytes_per_block;
394            float scale_factor = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data + 104)));
395            for (size_t j = 0; j < 16; j++) {
396                scales[j + i * 16] =
397                    ov::float16(scale_factor * static_cast<float>(*((int8_t *) (block_data + 128 + 64 + j))));
398            }
399            uint8_t * ql = block_data;
400            uint8_t * qh = block_data + 128;
401            auto * signed_weights = reinterpret_cast<int8_t *>(weights);
402            for (int64_t j = 0; j < 32; ++j) {
403                signed_weights[i * 256 + j] = static_cast<int8_t>((ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4)) - 32;
404                signed_weights[i * 256 + j + 32] =
405                    static_cast<int8_t>((ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4)) - 32;
406                signed_weights[i * 256 + j + 64] = static_cast<int8_t>((ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4)) - 32;
407                signed_weights[i * 256 + j + 96] =
408                    static_cast<int8_t>((ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4)) - 32;
409                signed_weights[i * 256 + j + 128] =
410                    static_cast<int8_t>((ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4)) - 32;
411                signed_weights[i * 256 + j + 160] =
412                    static_cast<int8_t>((ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4)) - 32;
413                signed_weights[i * 256 + j + 192] =
414                    static_cast<int8_t>((ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4)) - 32;
415                signed_weights[i * 256 + j + 224] =
416                    static_cast<int8_t>((ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4)) - 32;
417            }
418        });
419    }
420}
421
422static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
423    if (j < 4) {
424        *d = q[j] & 63;
425        *m = q[j + 4] & 63;
426    } else {
427        *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
428        *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);
429    }
430}
431
432void extract_q5_k_data(const ggml_tensor * tensor,
433                       ov::Tensor & weights_arr,
434                       ov::Tensor & scales_arr,
435                       ov::Tensor & zp_arr,
436                       bool use_bias) {
437    const uint64_t bytes_per_block = 4 + 12 + 32 + 128;
438    const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;
439
440    auto * data = static_cast<uint8_t *>(tensor->data);
441    auto * weights = static_cast<uint8_t *>(weights_arr.data());
442    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
443
444    // For bias path, zp_arr holds f16 bias values; for zp path, it holds u8 zero points
445    auto * zp_u8 = use_bias ? nullptr : static_cast<uint8_t *>(zp_arr.data());
446    auto * bias_f16 = use_bias ? zp_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() : nullptr;
447
448    ov::parallel_for(n_super_block, [&](size_t i) {
449        uint8_t * block_data = data + i * bytes_per_block;
450
451        const float d = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data)));
452        const float min_factor = static_cast<float>(ov::float16::from_bits(*((uint16_t *) block_data + 1)));
453
454        const uint8_t * scales_data = block_data + 4;   // 12 bytes of scales
455        const uint8_t * qh = block_data + 4 + 12;       // 32 bytes of high bits
456        const uint8_t * ql = block_data + 4 + 12 + 32;  // 128 bytes of low bits
457
458        int is = 0;
459        uint8_t u1 = 1;
460        uint8_t u2 = 2;
461
462        // Process 2 blocks in one iteration
463        for (int j = 0; j < 256; j += 64) {  // 256 = QK_K, so 4 iterations of 64
464            uint8_t sc;
465            uint8_t m;
466
467            // Get scale and min for first 32 elements
468            get_scale_min_k4(is + 0, scales_data, &sc, &m);
469            const float d1 = d * sc;
470            const float m1 = min_factor * m;
471
472            // Get scale and min for second 32 elements
473            get_scale_min_k4(is + 1, scales_data, &sc, &m);
474            const float d2 = d * sc;
475            const float m2 = min_factor * m;
476
477            scales[i * 8 + is] = ov::float16(d1);
478            scales[i * 8 + is + 1] = ov::float16(d2);
479            if (use_bias) {
480                // Store bias = -min directly as f16, dequant: w*s + bias
481                bias_f16[i * 8 + is] = ov::float16(-m1);
482                bias_f16[i * 8 + is + 1] = ov::float16(-m2);
483            } else {
484                // zp = min / scale (since bias = -min and zp = -bias/scale)
485                zp_u8[i * 8 + is] = (d1 != 0.0f) ? (uint8_t) std::round(m1 / d1) : 0;
486                zp_u8[i * 8 + is + 1] = (d2 != 0.0f) ? (uint8_t) std::round(m2 / d2) : 0;
487            }
488
489            // Extract weights for first 32 elements (matching deq formula exactly)
490            for (int l = 0; l < 32; ++l) {
491                weights[i * 256 + j + l] = (ql[l] & 0xF) + ((qh[l] & u1) ? 16 : 0);
492            }
493
494            // Extract weights for second 32 elements
495            for (int l = 0; l < 32; ++l) {
496                weights[i * 256 + j + l + 32] = (ql[l] >> 4) + ((qh[l] & u2) ? 16 : 0);
497            }
498
499            ql += 32;
500            is += 2;
501            u1 <<= 2;
502            u2 <<= 2;
503        }
504    });
505}
506
507// TODO Reorder for make_intX_weights
508
509// If for_gather_matmul is true, weight may be N-D (e.g. 3D MoE expert weights [n_expert, rows, cols]).
510// The dequantization chain below is built as usual but left in f16 (no final Convert to f32) --
511// ov::pass::MarkDequantization (registered in translate_session.cpp) marks the chain so it survives
512// model-build-time ConstantFolding. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul directly
513// on top of the resulting f16 chain.
514ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
515                                       ov::Tensor & scales,
516                                       ov::Tensor & zp,
517                                       size_t group_size,
518                                       bool use_bias,
519                                       bool for_gather_matmul) {
520    ov::Shape orig_shape = weight.get_shape();
521    bool is_signed = (weight.get_element_type() == ov::element::i8);  // Symmetric: signed weights, no ZP
522
523    // Expand dimensions for scales and zp/bias
524    auto scale_shape = scales.get_shape();
525
526    // Group the innermost (last) dimension. For 2D weights [rows, cols] this yields
527    // [rows, cols/group_size, group_size]; for 3D MoE experts [n_expert, rows, cols] this yields
528    // [n_expert, rows, cols/group_size, group_size].
529    ov::Shape packed_shape = orig_shape;
530    packed_shape.back() /= group_size;
531    packed_shape.push_back(group_size);
532    const size_t group_dim = packed_shape.size() - 2;
533
534    if (packed_shape[group_dim] == 1) {
535        // Requantized channel-wise case
536        packed_shape.erase(packed_shape.begin() + group_dim);
537    } else {
538        scale_shape.push_back(1);
539        scales.set_shape(scale_shape);
540        if (!is_signed && zp.get_size() > 0) {
541            auto zp_shape = zp.get_shape();
542            zp_shape.push_back(1);
543            zp.set_shape(zp_shape);
544        }
545    }
546
547    auto scales_f16 = std::make_shared<ov::op::v0::Constant>(scales);
548
549    ov::Output<ov::Node> result;
550    if (is_signed) {
551        // Signed path: q * s (no zero point subtraction needed)
552        auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::i8, packed_shape,
553                                                                   static_cast<uint8_t *>(weight.data()), nullptr);
554        weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
555        auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
556        auto mul = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
557        result = mul;
558    } else {
559        // Unsigned path
560        auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u8, packed_shape,
561                                                                   static_cast<uint8_t *>(weight.data()), nullptr);
562        weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
563        auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
564
565        if (use_bias && zp.get_size() > 0) {
566            // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s, where the zero
567            // point is an exact f16 value zp = -bias/scale (the zp tensor holds bias values
568            // coming in). Algebraically equal to w*s + bias, but unlike an Add(bias) graph this
569            // matches CompressedWeightsBlock's pattern (Constant->Convert->Subtract->Multiply),
570            // so for_gather_matmul weights still fuse into GatherMatmulCompressed. Also avoids
571            // the round(min/scale) error of an integer zero point. Convert bias -> zero-point IN
572            // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation.
573            auto * bias_zp_data = zp.data<ov::float16>();
574            const auto * scale_data = scales.data<ov::float16>();
575            const size_t n = zp.get_size();
576            for (size_t i = 0; i < n; i++) {
577                float s = static_cast<float>(scale_data[i]);
578                float b = static_cast<float>(bias_zp_data[i]);
579                bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f);
580            }
581            auto zero_point_f16 = std::make_shared<ov::op::v0::Constant>(zp);
582            auto w_zp =
583                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY);
584            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
585        } else {
586            // Zero point path: (w - zp) * s
587            auto zero_point = std::make_shared<ov::op::v0::Constant>(zp);
588            float zp_value;
589            if (ov::op::util::get_single_value(zero_point, zp_value)) {
590                zero_point = ov::op::v0::Constant::create(zero_point->get_element_type(), {}, {zp_value});
591            }
592            auto zero_point_f16 = std::make_shared<ov::op::v0::Convert>(zero_point, ov::element::f16);
593            auto w_zp =
594                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY);
595            auto mul = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
596            result = mul;
597        }
598    }
599
600    if (packed_shape.size() != orig_shape.size()) {
601        // If not requantized channel-wise case, reshape back to original shape
602        auto final_shape =
603            std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{orig_shape.size()}, orig_shape);
604        auto reshaped = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
605        result = reshaped;
606    }
607
608    if (for_gather_matmul) {
609        return result;
610    }
611    return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
612}
613
614// See make_int8_weights for the meaning of for_gather_matmul.
615ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
616                                       ov::Tensor & scales,
617                                       ov::Tensor & zp,
618                                       size_t group_size,
619                                       bool use_bias,
620                                       bool for_gather_matmul) {
621    ov::Shape orig_weight_shape = weight.get_shape();
622    bool is_signed = (weight.get_element_type() == ov::element::i4);  // Symmetric: signed weights, no ZP
623
624    // Expand dimensions for scales and zp/bias
625    ov::Shape scale_shape = scales.get_shape();
626
627    // Create INT4 weight tensor. Group the innermost (last) dimension: for 2D weights
628    // [rows, cols] this yields [rows, cols/group_size, group_size]; for 3D MoE experts
629    // [n_expert, rows, cols] this yields [n_expert, rows, cols/group_size, group_size].
630    ov::Shape packed_shape = orig_weight_shape;
631    packed_shape.back() /= group_size;
632    packed_shape.push_back(group_size);
633    const size_t group_dim = packed_shape.size() - 2;
634
635    if (packed_shape[group_dim] == 1) {
636        // Requantized channel-wise case
637        packed_shape.erase(packed_shape.begin() + group_dim);
638    } else {
639        scale_shape.push_back(1);
640        scales.set_shape(scale_shape);
641        if (!is_signed && zp.get_size() > 0) {
642            auto zp_shape = zp.get_shape();
643            zp_shape.push_back(1);
644            zp.set_shape(zp_shape);
645        }
646    }
647
648    auto scales_f16 = std::make_shared<ov::op::v0::Constant>(scales);
649
650    ov::Output<ov::Node> result;
651    if (is_signed) {
652        // Signed path: q * s (no zero point subtraction needed)
653        auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::i4, packed_shape,
654                                                                   static_cast<uint8_t *>(weight.data()), nullptr);
655        weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
656        auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
657        auto mul = std::make_shared<ov::op::v1::Multiply>(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);
658        result = mul;
659    } else {
660        // Unsigned path
661        auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u4, packed_shape,
662                                                                   static_cast<uint8_t *>(weight.data()), nullptr);
663        weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
664        auto weights_f16 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f16);
665
666        if (use_bias && zp.get_size() > 0) {
667            // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s with an exact f16
668            // zp = -bias/scale. Equivalent to w*s + bias but matches CompressedWeightsBlock's
669            // pattern so for_gather_matmul weights still fuse into GatherMatmulCompressed, and
670            // avoids the round(min/scale) error of an integer zp. Convert bias -> zero-point IN
671            // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation.
672            auto * bias_zp_data = zp.data<ov::float16>();
673            const auto * scale_data = scales.data<ov::float16>();
674            const size_t n = zp.get_size();
675            for (size_t i = 0; i < n; i++) {
676                float s = static_cast<float>(scale_data[i]);
677                float b = static_cast<float>(bias_zp_data[i]);
678                bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f);
679            }
680            auto zero_points_f16 = std::make_shared<ov::op::v0::Constant>(zp);
681            auto w_zp =
682                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY);
683            result = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
684        } else {
685            // Zero point path: (w - zp) * s
686            auto zero_points_node = std::make_shared<ov::op::v0::Constant>(zp);
687            float zp_value;
688            if (ov::op::util::get_single_value(zero_points_node, zp_value)) {
689                zero_points_node = ov::op::v0::Constant::create(zero_points_node->get_element_type(), {}, {zp_value});
690            }
691            auto zero_points_f16 = std::make_shared<ov::op::v0::Convert>(zero_points_node, ov::element::f16);
692            auto w_zp =
693                std::make_shared<ov::op::v1::Subtract>(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY);
694            auto mul = std::make_shared<ov::op::v1::Multiply>(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);
695            result = mul;
696        }
697    }
698
699    if (packed_shape.size() != orig_weight_shape.size()) {
700        // If not requantized channel-wise case, reshape back to original shape
701        auto final_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{orig_weight_shape.size()},
702                                                                  orig_weight_shape);
703        auto reshaped = std::make_shared<ov::op::v1::Reshape>(result, final_shape, false);
704        result = reshaped;
705    }
706
707    if (for_gather_matmul) {
708        return result;
709    }
710    return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
711}
712
713ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales) {
714    const ov::Shape final_shape = weight.get_shape();
715    GGML_ASSERT(!final_shape.empty());
716    GGML_ASSERT(final_shape.back() % MXFP4_BLOCK_SIZE == 0);
717
718    ov::Shape packed_shape = final_shape;
719    packed_shape.back() /= MXFP4_BLOCK_SIZE;
720    packed_shape.push_back(MXFP4_BLOCK_SIZE);
721
722    ov::Shape scale_shape = packed_shape;
723    scale_shape.back() = 1;
724    scales.set_shape(scale_shape);
725
726    auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::f4e2m1, packed_shape,
727                                                               static_cast<uint8_t *>(weight.data()), nullptr);
728    weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
729    auto weights_f32 = std::make_shared<ov::op::v0::Convert>(weights_node, ov::element::f32);
730
731    auto scales_node = std::make_shared<ov::op::v0::Constant>(scales);
732    auto scales_f32 = std::make_shared<ov::op::v0::Convert>(scales_node, ov::element::f32);
733    ov::Output<ov::Node> result =
734        std::make_shared<ov::op::v1::Multiply>(weights_f32, scales_f32, ov::op::AutoBroadcastType::NUMPY);
735
736    auto final_shape_node =
737        std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{final_shape.size()}, final_shape);
738    return std::make_shared<ov::op::v1::Reshape>(result, final_shape_node, false);
739}
740
741ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight) {
742    auto weights_node = std::make_shared<ov::op::v0::Constant>(ov::element::u8, weight.get_shape(),
743                                                               static_cast<uint8_t *>(weight.data()), nullptr);
744    weights_node->get_rt_info()["__gguf_tensor_holder"] = weight;
745    weights_node->get_rt_info()["__ggml_openvino_mxfp4_moe_packed"] = true;
746    return weights_node;
747}
748
749// Extract quantized weights from tensor and create weight subgraph
750std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
751                                                    const void * data,
752                                                    ov::Tensor & weights,
753                                                    ov::Tensor & scales,
754                                                    ov::Tensor & zp,
755                                                    bool use_bias) {
756    // Create a temporary tensor for extraction functions that read from tensor->data
757    ggml_tensor temp_tensor = *tensor;
758    temp_tensor.data = const_cast<void *>(data);
759
760    if (tensor->type == GGML_TYPE_MXFP4) {
761        extract_mxfp4_data(&temp_tensor, weights, scales);
762        auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr();
763        result->set_friendly_name(tensor->name);
764        return result;
765    }
766
767    // Determine block size based on tensor type
768    int64_t weights_per_block;
769    bool is_u4;
770    switch (tensor->type) {
771    case GGML_TYPE_Q4_0:
772    case GGML_TYPE_Q4_1:
773    case GGML_TYPE_Q4_K:
774        is_u4 = true;
775        weights_per_block = 32;
776        break;
777    case GGML_TYPE_Q8_0:
778    case GGML_TYPE_Q5_1:
779    case GGML_TYPE_Q5_K:
780        is_u4 = false;
781        weights_per_block = 32;
782        break;
783    case GGML_TYPE_Q6_K:
784        is_u4 = false;
785        weights_per_block = 16;
786        break;
787    default:
788        throw std::runtime_error("Unsupported quantized type for extraction: " +
789                                 std::string(ggml_type_name(tensor->type)));
790    }
791
792    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
793    // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
794    // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
795    // into GatherMatmulCompressed since it stays a Subtract, not an Add.
796    const bool for_gather_matmul = tensor->ne[2] > 1;
797    use_bias = use_bias || for_gather_matmul;
798
799    // Extract quantized data
800    switch (tensor->type) {
801    case GGML_TYPE_Q4_0:
802        extract_q4_0_data(&temp_tensor, weights, scales, zp);
803        break;
804    case GGML_TYPE_Q4_1:
805        extract_q4_1_data(&temp_tensor, weights, scales, zp, use_bias);
806        break;
807    case GGML_TYPE_Q4_K:
808        extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias);
809        break;
810    case GGML_TYPE_Q5_1:
811        extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias);
812        break;
813    case GGML_TYPE_Q8_0:
814        extract_q8_0_data(&temp_tensor, weights, scales, zp);
815        break;
816    case GGML_TYPE_Q6_K:
817        extract_q6_k_data(&temp_tensor, weights, scales, zp);
818        break;
819    case GGML_TYPE_Q5_K:
820        extract_q5_k_data(&temp_tensor, weights, scales, zp, use_bias);
821        break;
822    default:
823        throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type)));
824    }
825
826    // Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the
827    // GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain.
828    ov::Output<ov::Node> weight_node;
829    if (is_u4) {
830        weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
831    } else {
832        weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul);
833    }
834
835    auto result = weight_node.get_node_shared_ptr();
836    result->set_friendly_name(tensor->name);
837    return result;
838}
839
840// Requantize weights to target format, writing to provided buffers
841std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
842                                                const void * data,
843                                                ExtraQuantType requant_type,
844                                                int64_t block_size,
845                                                ov::Tensor & weights,
846                                                ov::Tensor & scales,
847                                                ov::Tensor & zp) {
848    int64_t n_elements = ggml_nelements(tensor);
849    const int64_t ne0 = tensor->ne[0];                 // elements per row
850    const int64_t n_rows = n_elements / ne0;
851    const auto * type_traits = ggml_get_type_traits(tensor->type);
852    const size_t src_row_bytes = ggml_row_size(tensor->type, ne0);
853
854    bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128);
855
856    // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
857    // GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
858    // materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize
859    // a chunk of complete rows into a small scratch and quantize/convert it straight into
860    // the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats.
861    //
862    // Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size
863    // divides a row (channel-wise _C uses block_size == ne0) so no target block straddles
864    // a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two
865    // weights per byte with running zp ORs that assume a single whole-array call, so it is
866    // never streamed. When the flag is off, behavior is identical to the original
867    // full-materialization path.
868    const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 &&
869                                !(block_size > 0 && ne0 % block_size != 0);
870
871    if (!stream_requant) {
872        // Full materialization (original behavior): dequantize the whole tensor to F32,
873        // then convert/quantize in one call.
874        std::vector<float> weights_f32(n_elements);
875        type_traits->to_float(data, weights_f32.data(), n_elements);
876        if (requant_type == ExtraQuantType::F16) {
877            ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);
878            auto result = std::make_shared<ov::op::v0::Constant>(weights);
879            result->set_friendly_name(tensor->name);
880            return result;
881        }
882        if (is_u4) {
883            quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
884        } else if (requant_type == ExtraQuantType::Q8_1_C) {
885            quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
886        } else {
887            quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
888        }
889    } else {
890        // Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight,
891        // and per-layer Q6_K/Q5_K requant — the large transient cases).
892        const int64_t CHUNK_ROWS = std::min<int64_t>(n_rows, 256);
893        std::vector<float> scratch(CHUNK_ROWS * ne0);
894        // F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements.
895        auto * f16_base = static_cast<uint8_t *>(weights.data());
896        for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) {
897            const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0);
898            const int64_t elems = rows * ne0;
899            const auto * src = static_cast<const uint8_t *>(data) + r0 * src_row_bytes;
900            type_traits->to_float(src, scratch.data(), elems);
901
902            if (requant_type == ExtraQuantType::F16) {
903                ggml_get_type_traits(GGML_TYPE_F16)
904                    ->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems);
905            } else {
906                const int64_t block_offset = (r0 * ne0) / block_size;
907                if (requant_type == ExtraQuantType::Q8_1_C) {
908                    quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
909                } else {
910                    quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
911                }
912            }
913        }
914        if (requant_type == ExtraQuantType::F16) {
915            auto result = std::make_shared<ov::op::v0::Constant>(weights);
916            result->set_friendly_name(tensor->name);
917            return result;
918        }
919    }
920
921    // Create the OpenVINO weight subgraph
922    ov::Output<ov::Node> weight_node;
923    if (is_u4) {
924        weight_node = make_int4_weights(weights, scales, zp, block_size);
925    } else {
926        weight_node = make_int8_weights(weights, scales, zp, block_size);
927    }
928
929    auto result = weight_node.get_node_shared_ptr();
930    result->set_friendly_name(tensor->name);
931    return result;
932}
933
934OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) {
935    GGML_ASSERT(tensor != nullptr);
936    GGML_ASSERT(data != nullptr);
937
938    OvWeight result;
939
940    // Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights.
941    ov::Shape node_shape = (tensor->ne[2] > 1) ?
942                               ov::Shape{static_cast<size_t>(tensor->ne[2]), static_cast<size_t>(tensor->ne[1]),
943                                         static_cast<size_t>(tensor->ne[0])} :
944                               ov::Shape{static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
945
946    // Handle F16/F32/BF16 weights
947    if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
948        ov::element::Type element_type;
949        switch (tensor->type) {
950        case GGML_TYPE_F32:
951            element_type = ov::element::f32;
952            break;
953        case GGML_TYPE_F16:
954            element_type = ov::element::f16;
955            break;
956        case GGML_TYPE_BF16:
957            element_type = ov::element::bf16;
958            break;
959        default:
960            OPENVINO_THROW("Unexpected tensor type in F16/F32/BF16 path");
961        }
962
963        if (output_base_ptr && output_base_ptr != data) {
964            // Using external buffer - copy data and create shared-memory constant
965            size_t tensor_bytes = ggml_nbytes(tensor);
966            memcpy(output_base_ptr, data, tensor_bytes);
967            result.weights = ov::Tensor(element_type, node_shape, output_base_ptr);
968        } else {
969            result.weights = ov::Tensor(element_type, node_shape, data);
970        }
971        result.weight_node = std::make_shared<ov::op::v0::Constant>(result.weights);
972        return result;
973    }
974
975    // Handle quantized weights
976    if (!ggml_is_quantized(tensor->type)) {
977        OPENVINO_THROW("Unsupported weight tensor type: ", ggml_type_name(tensor->type));
978    }
979
980    result.layout = ggml_openvino_get_extracted_layout(tensor, use_bias);
981    const auto & layout = result.layout;
982    if (layout.total_size == 0) {
983        OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
984    }
985
986    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
987    // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
988    // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
989    // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
990    // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
991    const bool for_gather_matmul = tensor->ne[2] > 1;
992    const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
993
994    const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
995    if (is_3d_mxfp4_moe) {
996        ov::Shape packed_shape = {static_cast<size_t>(tensor->ne[3]),
997                                  static_cast<size_t>(tensor->ne[2]),
998                                  static_cast<size_t>(tensor->ne[1]),
999                                  static_cast<size_t>(tensor->ne[0] / MXFP4_BLOCK_SIZE),
1000                                  MXFP4_BLOCK_BYTES};
1001        const size_t tensor_bytes = ggml_nbytes(tensor);
1002        if (output_base_ptr) {
1003            auto * buf_base = static_cast<uint8_t *>(output_base_ptr);
1004            memcpy(buf_base + layout.weights_offset, data, tensor_bytes);
1005            result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset);
1006        } else {
1007            result.weights = ov::Tensor(ov::element::u8, packed_shape);
1008            memcpy(result.weights.data(), data, tensor_bytes);
1009        }
1010        result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr();
1011        result.weight_node->set_friendly_name(tensor->name);
1012        return result;
1013    }
1014
1015    if (use_bias) {
1016        OPENVINO_ASSERT(!layout.is_requant,
1017                        "use_bias is only used for test-backend-ops, which should not have requantization");
1018        // bias node will be created on the fly and not use backend buffer
1019        output_base_ptr = nullptr;
1020    }
1021
1022    // F16 requant path - no separate scales/zp needed in result
1023    if (layout.is_requant && layout.requant_type.has_value() && layout.requant_type.value() == ExtraQuantType::F16) {
1024        if (output_base_ptr) {
1025            result.weights = ov::Tensor(ov::element::f16, node_shape,
1026                                        static_cast<uint8_t *>(output_base_ptr) + layout.weights_offset);
1027        } else {
1028            result.weights = ov::Tensor(ov::element::f16, node_shape);
1029        }
1030        ov::Tensor dummy_scales, dummy_zp;  // Not used for F16
1031        result.weight_node =
1032            requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp);
1033        return result;
1034    }
1035
1036    // Quantized path (normal extraction or quantized requant)
1037    // Create weight/scale/zp tensors - shared between both paths
1038    // For symmetric quantization, use signed types (i4/i8) and no ZP tensor
1039    ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ?
1040                                        ov::element::f4e2m1 :
1041                                        (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) :
1042                                                               (layout.is_u4 ? ov::element::u4 : ov::element::u8));
1043    ov::Shape scale_shape = node_shape;
1044    scale_shape.back() /= layout.weights_per_block;
1045
1046    if (tensor->type == GGML_TYPE_MXFP4) {
1047        if (tensor->ne[2] == 1 && tensor->ne[3] == 1) {
1048            node_shape = {static_cast<size_t>(tensor->ne[1]), static_cast<size_t>(tensor->ne[0])};
1049        } else {
1050            node_shape.clear();
1051            for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
1052                node_shape.push_back(static_cast<size_t>(tensor->ne[i]));
1053            }
1054        }
1055
1056        scale_shape = node_shape;
1057        scale_shape.back() /= layout.weights_per_block;
1058    }
1059
1060    if (output_base_ptr) {
1061        uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr);
1062        result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);
1063        const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
1064        result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset);
1065        if (!layout.is_symmetric) {
1066            ov::element::Type zp_type =
1067                zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8);
1068            result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset);
1069        }
1070        // else: result.zp remains default-constructed (empty) for symmetric
1071    } else {
1072        result.weights = ov::Tensor(weight_type, node_shape);
1073        const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
1074        result.scales = ov::Tensor(scale_type, scale_shape);
1075        if (!layout.is_symmetric) {
1076            if (zp_is_f16) {
1077                result.zp = ov::Tensor(ov::element::f16, scale_shape);
1078            } else {
1079                ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
1080                result.zp = ov::Tensor(zp_type, scale_shape);
1081            }
1082        }
1083        // else: result.zp remains default-constructed (empty) for symmetric
1084    }
1085
1086    if (layout.is_requant && layout.requant_type.has_value()) {
1087        result.weight_node = requantize_to_buffers(tensor, data, layout.requant_type.value(), layout.weights_per_block,
1088                                                   result.weights, result.scales, result.zp);
1089    } else {
1090        result.weight_node =
1091            extract_quantized_weights(tensor, data, result.weights, result.scales, result.zp, use_bias);
1092    }
1093
1094    return result;
1095}
1096
1097void quantize_q4_0(const float * x,
1098                   ov::Tensor & weights_arr,
1099                   ov::Tensor & scales_arr,
1100                   ov::Tensor & zp_arr,
1101                   int64_t k,
1102                   int64_t qk) {
1103    assert(k % qk == 0);
1104    const int nb = k / qk;
1105
1106    auto * weights = static_cast<uint8_t *>(weights_arr.data());
1107    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
1108    bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4);  // Signed i4 path
1109
1110    if (!is_symmetric) {
1111        auto * zp = static_cast<uint8_t *>(zp_arr.data());
1112        for (int i = 0; i < nb; i++) {
1113            float amax = 0.0f;
1114            float max = 0.0f;
1115            for (int j = 0; j < qk; j++) {
1116                const float v = x[i * qk + j];
1117                if (amax < fabsf(v)) {
1118                    amax = fabsf(v);
1119                    max = v;
1120                }
1121            }
1122            const float d = max / -8;
1123            if (d == 0) {
1124                scales[i] = ov::float16(1.0f);
1125                if (i % 2 == 0) {
1126                    zp[i / 2] = 8;
1127                } else {
1128                    zp[i / 2] |= (8 << 4);
1129                }
1130                memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2);
1131                continue;
1132            }
1133            const float id = 1.0f / d;
1134            scales[i] = ov::float16(d);
1135            if (i % 2 == 0) {
1136                zp[i / 2] = 8;
1137            } else {
1138                zp[i / 2] |= (8 << 4);
1139            }
1140            for (int j = 0; j < qk / 2; ++j) {
1141                const float x0 = x[i * qk + 2 * j] * id;
1142                const float x1 = x[i * qk + 2 * j + 1] * id;
1143                const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f));
1144                const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f));
1145                weights[i * qk / 2 + j] = xi0 | (xi1 << 4);
1146            }
1147        }
1148    } else {
1149        // Symmetric: produce signed i4 values in [-8, 7]
1150        for (int i = 0; i < nb; i++) {
1151            float amax = 0.0f;
1152            float max = 0.0f;
1153            for (int j = 0; j < qk; j++) {
1154                const float v = x[i * qk + j];
1155                if (amax < fabsf(v)) {
1156                    amax = fabsf(v);
1157                    max = v;
1158                }
1159            }
1160            const float d = max / -8;
1161            if (d == 0) {
1162                scales[i] = ov::float16(1.0f);
1163                // i4 value 0 packed: 0x00
1164                memset(weights + i * qk / 2, 0, qk / 2);
1165                continue;
1166            }
1167            const float id = 1.0f / d;
1168            scales[i] = ov::float16(d);
1169            for (int j = 0; j < qk / 2; ++j) {
1170                const float x0 = x[i * qk + 2 * j] * id;
1171                const float x1 = x[i * qk + 2 * j + 1] * id;
1172                // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement.
1173                int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0)));
1174                int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1)));
1175                weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4);
1176            }
1177        }
1178    }
1179}
1180
1181void quantize_q8_0(const float * x,
1182                   ov::Tensor & weights_arr,
1183                   ov::Tensor & scales_arr,
1184                   ov::Tensor & zp_arr,
1185                   int64_t k,
1186                   int64_t qk,
1187                   int64_t block_offset) {
1188    assert(k % qk == 0);
1189    const int nb = k / qk;
1190
1191    // block_offset lets a caller quantize a chunk of blocks into the right place in the
1192    // output buffers (used for streaming requant). x points at this chunk's first block;
1193    // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no
1194    // nibble packing), so any block boundary is safe.
1195    auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
1196    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
1197    bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8);  // Signed i8 path
1198
1199    if (!is_symmetric) {
1200        auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
1201        for (int i = 0; i < nb; i++) {
1202            float amax = 0.0f;
1203            for (int j = 0; j < qk; j++) {
1204                const float v = x[i * qk + j];
1205                amax = std::max(amax, fabsf(v));
1206            }
1207            const float d = amax / 127.0f;
1208            const float id = d ? 1.0f / d : 0.0f;
1209            scales[i] = ov::float16(d);
1210            zp[i] = 128;
1211            for (int j = 0; j < qk; ++j) {
1212                const float x0 = x[i * qk + j] * id;
1213                const int8_t xi0 = roundf(x0);
1214                weights[i * qk + j] = (uint8_t) (xi0 + 128);
1215            }
1216        }
1217    } else {
1218        // Symmetric: store signed int8 values directly
1219        auto * signed_weights = reinterpret_cast<int8_t *>(weights);
1220        for (int i = 0; i < nb; i++) {
1221            float amax = 0.0f;
1222            for (int j = 0; j < qk; j++) {
1223                const float v = x[i * qk + j];
1224                amax = std::max(amax, fabsf(v));
1225            }
1226            const float d = amax / 127.0f;
1227            const float id = d ? 1.0f / d : 0.0f;
1228            scales[i] = ov::float16(d);
1229            for (int j = 0; j < qk; ++j) {
1230                const float x0 = x[i * qk + j] * id;
1231                signed_weights[i * qk + j] = (int8_t) roundf(x0);
1232            }
1233        }
1234    }
1235}
1236
1237void quantize_q8_1(const float * x,
1238                   ov::Tensor & weights_arr,
1239                   ov::Tensor & scales_arr,
1240                   ov::Tensor & zp_arr,
1241                   int64_t k,
1242                   int64_t qk,
1243                   int64_t block_offset) {
1244    assert(k % qk == 0);
1245    const int nb = k / qk;
1246
1247    // See quantize_q8_0: block_offset places this chunk's output at the right block.
1248    auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
1249    auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
1250    auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
1251    for (int i = 0; i < nb; i++) {
1252        float min = std::numeric_limits<float>::max();
1253        float max = std::numeric_limits<float>::lowest();
1254
1255        for (int j = 0; j < qk; j++) {
1256            const float v = x[i * qk + j];
1257            min = std::min(v, min);
1258            max = std::max(v, max);
1259        }
1260
1261        const float d = (max - min) / ((1 << 8) - 1);
1262        const float id = d ? 1.0f / d : 0.0f;
1263        scales[i] = ov::float16(d);
1264        // zp = -min / scale (Q8_1 is asymmetric)
1265        zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0;
1266
1267        for (int j = 0; j < qk; ++j) {
1268            const float x0 = (x[i * qk + j] - min) * id;
1269            const uint8_t xi0 = roundf(x0);
1270            weights[i * qk + j] = xi0;
1271        }
1272    }
1273}