hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Qwen3.5, memory optimization, and test-recurrent-state-rollback (#26952)aee56b3ab2026-08-14

1#pragma once
2#include "ggml-openvino-extra.h"  // For ExtraQuantType
3#include "ggml.h"
4
5#include <cstdint>
6#include <openvino/op/constant.hpp>
7#include <openvino/core/node_output.hpp>
8#include <openvino/runtime/tensor.hpp>
9
10void unpack_32_4(const uint8_t * data, uint8_t * dst);
11
12void extract_q4_0_data(const ggml_tensor * tensor,
13                       ov::Tensor & weights_arr,
14                       ov::Tensor & scales_arr,
15                       ov::Tensor & zp_arr);
16
17void extract_q4_1_data(const ggml_tensor * tensor,
18                       ov::Tensor & weights_arr,
19                       ov::Tensor & scales_arr,
20                       ov::Tensor & zp_arr,
21                       bool use_bias = false);
22
23void extract_q5_1_data(const ggml_tensor * tensor,
24                       ov::Tensor & weights_arr,
25                       ov::Tensor & scales_arr,
26                       ov::Tensor & zp_arr,
27                       bool use_bias = false);
28
29void extract_q8_0_data(const ggml_tensor * tensor,
30                       ov::Tensor & weights_arr,
31                       ov::Tensor & scales_arr,
32                       ov::Tensor & zp_arr);
33
34void unpack_256_4(const uint8_t * data, uint8_t * dst);
35
36void extract_q4_k_data(const ggml_tensor * tensor,
37                       ov::Tensor & weights_arr,
38                       ov::Tensor & scales_arr,
39                       ov::Tensor & zp_arr,
40                       bool use_bias = false);
41
42void extract_q5_k_data(const ggml_tensor * tensor,
43                       ov::Tensor & weights_arr,
44                       ov::Tensor & scales_arr,
45                       ov::Tensor & zp_arr,
46                       bool use_bias = false);
47
48void extract_q6_k_data(const ggml_tensor * tensor,
49                       ov::Tensor & weights_arr,
50                       ov::Tensor & scales_arr,
51                       ov::Tensor & zp_arr);
52
53void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr);
54
55static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32;
56
57// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
58// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
59// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
60// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
61// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
62// directly from the resulting f16 dequant chain.
63//
64// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
65// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
66// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
67// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
68ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
69                                       ov::Tensor & scales,
70                                       ov::Tensor & zp,
71                                       size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
72                                       bool use_bias = false,
73                                       bool for_gather_matmul = false);
74
75ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
76                                       ov::Tensor & scales,
77                                       ov::Tensor & zp,
78                                       size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
79                                       bool use_bias = false,
80                                       bool for_gather_matmul = false);
81
82ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales);
83
84ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight);
85
86// Extract quantized weights from tensor and create weight subgraph
87// If weights/scales/zp are provided (non-empty), uses them as output buffers
88// Otherwise allocates new ov::Tensors internally
89// Returns the weight node (make_int4_weights or make_int8_weights result)
90std::shared_ptr<ov::Node> extract_quantized_weights(
91    const ggml_tensor * tensor,
92    const void * data,  // Source data pointer (may differ from tensor->data)
93    ov::Tensor & weights,
94    ov::Tensor & scales,
95    ov::Tensor & zp,
96    bool use_bias = false);  // Use an exact f16 zero point (vs. a rounded integer one); always
97                             // used for for_gather_matmul (3D MoE expert) weights regardless of
98                             // this flag, and also settable explicitly for test-backend-ops.
99
100// Requantize weights from tensor to target format, writing to provided buffers
101// For F16 target, only weights buffer is used (scales/zp ignored)
102// Returns the weight node
103std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
104                                                const void * data,  // Source data pointer
105                                                ExtraQuantType requant_type,
106                                                int64_t block_size,
107                                                ov::Tensor & weights,
108                                                ov::Tensor & scales,
109                                                ov::Tensor & zp);
110
111inline const char * extra_quant_type_name(ExtraQuantType t) {
112    switch (t) {
113    case ExtraQuantType::F16:
114        return "F16";
115    case ExtraQuantType::Q4_0_C:
116        return "Q4_0_C";
117    case ExtraQuantType::Q4_0_128:
118        return "Q4_0_128";
119    case ExtraQuantType::Q8_0_C:
120        return "Q8_0_C";
121    case ExtraQuantType::Q8_0_32:
122        return "Q8_0_32";
123    case ExtraQuantType::Q8_1_C:
124        return "Q8_1_C";
125    default:
126        return "unknown";
127    }
128}
129
130// Result from process_weight_tensor containing the weight node and tensors.
131// For quantized weights, also contains the extracted layout and scale/zp tensors.
132struct OvWeight {
133    std::shared_ptr<ov::Node> weight_node;
134    ggml_openvino_extracted_layout layout;  // Only meaningful for quantized (layout.total_size > 0)
135    ov::Tensor weights;
136    ov::Tensor scales;
137    ov::Tensor zp;
138
139    bool is_quantized() const { return layout.scales_size > 0; }
140};
141
142// Process weight tensor and create an OpenVINO weight node
143// Handles F16/F32/BF16 and quantized weights, with optional requantization
144// If output_base_ptr is nullptr, allocates internal buffers (for decoder use)
145// If output_base_ptr is provided, uses pre-allocated buffers at specified offsets (for backend buffer use)
146// Returns OvWeight with the weight node and optional quantized tensors
147OvWeight process_weight_tensor(
148    const ggml_tensor * tensor,
149    const void * data,                 // Source data pointer (may differ from tensor->data)
150    void * output_base_ptr = nullptr,  // Base pointer for output buffers (or nullptr for internal allocation)
151    bool use_bias = false);            // Use an exact f16 zero point (vs. a rounded integer one);
152                                       // always used for for_gather_matmul (3D MoE expert) weights
153                                       // regardless of this flag, and also settable explicitly for
154                                       // test-backend-ops.
155
156void quantize_q4_0(const float * x,
157                   ov::Tensor & weights_arr,
158                   ov::Tensor & scales_arr,
159                   ov::Tensor & zp_arr,
160                   int64_t k,
161                   int64_t qk);
162void quantize_q8_1(const float * x,
163                   ov::Tensor & weights_arr,
164                   ov::Tensor & scales_arr,
165                   ov::Tensor & zp_arr,
166                   int64_t k,
167                   int64_t qk,
168                   int64_t block_offset = 0);
169void quantize_q8_0(const float * x,
170                   ov::Tensor & weights_arr,
171                   ov::Tensor & scales_arr,
172                   ov::Tensor & zp_arr,
173                   int64_t k,
174                   int64_t qk,
175                   int64_t block_offset = 0);
176
177namespace ov {
178namespace op {
179namespace util {
180// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
181bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
182                      float & value,
183                      bool check_value_range = true);
184}  // namespace util
185}  // namespace op
186}  // namespace ov