char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
18.5 KiB493 linesraw
1#include "ggml-openvino-extra.h"
2
3#include "ggml-impl.h"
4#include "ggml.h"
5
6#include <cstdlib>
7#include <cstring>
8#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
9#include <openvino/runtime/intel_npu/level_zero/level_zero.hpp>
10#include <openvino/runtime/properties.hpp>
11#include <optional>
12
13ov::Core & ov_singleton_core() {
14    static ov::Core core;
15    return core;
16}
17
18// =====================================================
19// Device Configuration Implementations
20// =====================================================
21
22void ggml_openvino_device_config::init() {
23    if (initialized) {
24        return;
25    }
26
27    // All recognized GGML_OPENVINO_* env vars. Their values are cached here
28    // once at backend init time and read back via ggml_openvino_getenv_str()
29    // (raw string) or ggml_openvino_getenv_int() (integer / boolean toggle).
30    static constexpr const char * env_var_names[] = {
31        // String values (use ggml_openvino_getenv_str)
32        "GGML_OPENVINO_DEVICE",
33        "GGML_OPENVINO_CACHE_DIR",
34        "GGML_OPENVINO_DEBUG_NODE",
35        "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
36        "GGML_OPENVINO_NPU_COMPILE_CONFIG",
37        // Integer values (use ggml_openvino_getenv_int)
38        "GGML_OPENVINO_PREFILL_CHUNK_SIZE",
39        // Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
40        "GGML_OPENVINO_STATEFUL_EXECUTION",
41        "GGML_OPENVINO_PROFILING",
42        "GGML_OPENVINO_DUMP_CGRAPH",
43        "GGML_OPENVINO_DUMP_IR",
44        "GGML_OPENVINO_DEBUG_INPUT",
45        "GGML_OPENVINO_DEBUG_OUTPUT",
46        // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU,
47        // to test the static-shape translation without NPUW/real NPU hardware in the loop.
48        "GGML_OPENVINO_FORCE_STATIC",
49        "GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS",
50        "GGML_OPENVINO_ENABLE_CACHE",
51        "GGML_OPENVINO_DISABLE_CACHE",
52        "GGML_OPENVINO_DISABLE_KV_SLICE",
53        "GGML_OPENVINO_ENABLE_FALLBACK",
54        "GGML_OPENVINO_MANUAL_GQA_ATTN",
55        "GGML_OPENVINO_MEMORY_OPTIMIZE",
56        "GGML_OPENVINO_RELEASE_WEIGHTS",
57        "GGML_OPENVINO_REDUCE_COMPILE_MEM",
58        "GGML_OPENVINO_LOG_UNSUPPORTED_OPS",
59    };
60
61    for (const char * const & env_var : env_var_names) {
62        auto * env = getenv(env_var);
63        if (env) {
64            environment_variables[env_var] = env;
65        }
66    }
67
68    device_name = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU");
69    auto available_devices = ov_singleton_core().get_available_devices();
70    if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) {
71        GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to CPU\n", device_name.c_str());
72        device_name = "CPU";
73    }
74    is_npu = (device_name == "NPU");
75
76    const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR");
77    if (device_name == "NPU") {
78        compile_config = {
79            {"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES"   },
80            {"NPU_USE_NPUW",                      "YES"   },
81            {"NPUW_DEVICES",                      "NPU"   },
82            {"NPUW_FOLD",                         "YES"   },
83            {"NPUW_WEIGHTS_BANK",                 "shared"},
84            {"NPUW_FUNCALL_FOR_ALL",              "YES"   },
85            {"NPUW_FUNCALL_ASYNC",                "YES"   },
86            {"NPUW_DQ",                           "YES"   },
87            {"NPUW_DQ_FULL",                      "NO"    },
88        };
89        if (cache_dir && strlen(cache_dir) > 0) {
90            compile_config["NPUW_CACHE_DIR"] = cache_dir;
91            compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
92        }
93        const char * compilation_mode_params =
94            ggml_openvino_getenv_str("GGML_OPENVINO_NPU_COMPILE_CONFIG");
95        if (compilation_mode_params && strlen(compilation_mode_params) > 0) {
96            compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params;
97        }
98    } else if (cache_dir && strlen(cache_dir) > 0) {
99        compile_config.insert(ov::cache_dir(cache_dir));
100        compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
101    }
102
103    // Initialize remote context with queue sharing for GPU
104    if (device_name == "GPU") {
105        // Create OpenCL context and queue
106        cl_int err;
107        cl_platform_id platform;
108        err = clGetPlatformIDs(1, &platform, nullptr);
109        if (err != CL_SUCCESS) {
110            GGML_LOG_ERROR("Failed to get OpenCL platform: %d\n", err);
111            return;
112        }
113
114        cl_device_id cl_device;
115        err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &cl_device, nullptr);
116        if (err != CL_SUCCESS) {
117            GGML_LOG_ERROR("Failed to get OpenCL device: %d\n", err);
118            return;
119        }
120
121        cl_context cl_ctx = clCreateContext(nullptr, 1, &cl_device, nullptr, nullptr, &err);
122        if (err != CL_SUCCESS) {
123            GGML_LOG_ERROR("Failed to create OpenCL context: %d\n", err);
124            return;
125        }
126
127        cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, nullptr, &err);
128        if (err != CL_SUCCESS) {
129            GGML_LOG_ERROR("Failed to create OpenCL command queue: %d\n", err);
130            clReleaseContext(cl_ctx);
131            return;
132        }
133
134        // Create OpenVINO remote context with queue sharing
135        remote_context = ov::intel_gpu::ocl::ClContext(ov_singleton_core(), cl_queue);
136
137        // Release the context (queue keeps a reference)
138        clReleaseContext(cl_ctx);
139    } else if (device_name == "NPU") {
140        // remote tensor is not used for NPU yet
141        // remote_context = ov_singleton_core().get_default_context(device_name);
142    }
143
144    initialized = true;
145}
146
147ggml_openvino_device_config::~ggml_openvino_device_config() {
148    if (cl_queue != nullptr) {
149        clReleaseCommandQueue(cl_queue);
150        cl_queue = nullptr;
151    }
152}
153
154// Get the global device config singleton
155ggml_openvino_device_config & ggml_openvino_get_device_config() {
156    static ggml_openvino_device_config config;
157    return config;
158}
159
160// Initialize device config (call during backend init)
161void ggml_openvino_init_device_config() {
162    ggml_openvino_get_device_config().init();
163}
164
165// Get the device name
166const std::string & ggml_openvino_get_device_name() {
167    return ggml_openvino_get_device_config().device_name;
168}
169
170// Get the value of a GGML_OPENVINO_* env var as a string. Returns
171// default_value when the var is unset or set to an empty string.
172const char * ggml_openvino_getenv_str(const char * var, const char * default_value) {
173    auto & env_map = ggml_openvino_get_device_config().environment_variables;
174    auto it = env_map.find(var);
175    return (it == env_map.end() || it->second.empty()) ? default_value : it->second.c_str();
176}
177
178// Get the value of a GGML_OPENVINO_* env var as an int (via std::atoi).
179// Returns default_value (0) when the var is unset or empty. Used for both
180// integer settings (e.g. GGML_OPENVINO_PREFILL_CHUNK_SIZE) and boolean
181// toggles: "0" disables, any non-zero integer enables.
182int ggml_openvino_getenv_int(const char * var, int default_value) {
183    const char * v = ggml_openvino_getenv_str(var, nullptr);
184    return v ? std::atoi(v) : default_value;
185}
186
187bool ggml_openvino_reduce_compile_mem_enabled() {
188    const char * reduce_compile_mem = ggml_openvino_getenv_str("GGML_OPENVINO_REDUCE_COMPILE_MEM");
189    if (reduce_compile_mem != nullptr) {
190        return ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0;
191    }
192    return ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0;
193}
194
195bool ggml_openvino_release_weights_enabled(const std::string & device) {
196    const char * release_weights = ggml_openvino_getenv_str("GGML_OPENVINO_RELEASE_WEIGHTS");
197    if (release_weights != nullptr) {
198        return device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_RELEASE_WEIGHTS") != 0;
199    }
200    return device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0;
201}
202
203// Check if running on NPU
204bool ggml_openvino_is_npu() {
205    return ggml_openvino_get_device_config().is_npu;
206}
207
208// Get the remote context for the current device (returns empty optional for CPU)
209std::optional<ov::RemoteContext> ggml_openvino_get_remote_context() {
210    return ggml_openvino_get_device_config().remote_context;
211}
212
213// Get the compile config for the current device
214const ov::AnyMap & ggml_openvino_get_compile_config() {
215    return ggml_openvino_get_device_config().compile_config;
216}
217
218// Get the OpenCL command queue for GPU operations
219cl_command_queue ggml_openvino_get_cl_queue() {
220    return ggml_openvino_get_device_config().cl_queue;
221}
222
223// Get the clEnqueueMemFillINTEL function pointer (lazy load)
224clEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL() {
225    static clEnqueueMemFillINTEL_fn fn = nullptr;
226    static bool loaded = false;
227    if (!loaded) {
228        loaded = true;
229        cl_platform_id platform;
230        if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) {
231            fn = (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemFillINTEL");
232        }
233    }
234    return fn;
235}
236
237// Get the clEnqueueMemcpyINTEL function pointer (lazy load)
238clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() {
239    static clEnqueueMemcpyINTEL_fn fn = nullptr;
240    static bool loaded = false;
241    if (!loaded) {
242        loaded = true;
243        cl_platform_id platform;
244        if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) {
245            fn = (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemcpyINTEL");
246        }
247    }
248    return fn;
249}
250
251// Get requantization type for a tensor type (returns nullopt if no requant needed)
252std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant) {
253    if (no_requant) {
254        return std::nullopt;
255    }
256    if (strncmp(tensor->name, "token_embd.weight", 17) == 0) {
257        return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 :
258                                                                             ExtraQuantType::Q8_0_C);
259    }
260    if (strncmp(tensor->name, "output.weight", 13) == 0) {
261        return ExtraQuantType::Q8_0_C;
262    }
263    if (ggml_openvino_is_npu()) {
264        return ExtraQuantType::Q4_0_128;
265    }
266    switch (tensor->type) {
267    case GGML_TYPE_Q6_K:
268    case GGML_TYPE_Q5_K:
269        return ExtraQuantType::Q8_0_C;
270    default:
271        return std::nullopt;
272    }
273}
274
275// =====================================================
276// Extracted Layout Calculation
277// =====================================================
278
279ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_tensor * tensor, bool use_bias) {
280    ggml_openvino_extracted_layout layout = {};
281    layout.is_symmetric = false;
282
283    if (!ggml_is_quantized(tensor->type)) {
284        return layout;
285    }
286
287    // Most quantized weights use the existing 2D extraction path. 3D expert weights for
288    // MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the
289    // generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D.
290    if (tensor->ne[3] != 1) {
291        return layout;
292    }
293
294    // 3D MoE expert weights that are not requantized (see below) always use the exact f16
295    // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than
296    // the packed integer zero point -- must be kept in sync with that function so the buffer
297    // sizing here matches what process_weight_tensor actually writes.
298    const bool for_gather_matmul = tensor->ne[2] > 1;
299
300    int64_t n_elements = ggml_nelements(tensor);
301    const size_t alignment = 64;  // Good for SIMD
302
303    if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) {
304        layout.weights_per_block = 32;
305        layout.is_symmetric = true;
306        layout.weights_size = ggml_nbytes(tensor);
307        layout.weights_offset = 0;
308        layout.total_size = layout.weights_size;
309        return layout;
310    }
311
312    // Check if requantization is needed (NPU-specific)
313    auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias);
314    if (requant_type.has_value()) {
315        layout.is_requant = true;
316        layout.requant_type = requant_type;
317
318        // Special case: requant to F16 - just store F16 weights, no scales/zp
319        if (requant_type.value() == ExtraQuantType::F16) {
320            layout.weights_size = n_elements * sizeof(uint16_t);  // F16 = 2 bytes
321            layout.total_size = layout.weights_size;
322            layout.weights_offset = 0;
323            // No scales/zp for F16
324            return layout;
325        }
326
327        // Requant to different quantized format (e.g., Q4_0_128)
328        switch (requant_type.value()) {
329        case ExtraQuantType::Q4_0_128:
330            layout.is_u4 = true;
331            layout.weights_per_block = 128;
332            layout.is_symmetric = true;
333            break;
334        case ExtraQuantType::Q4_0_C:
335            layout.is_u4 = true;
336            layout.weights_per_block = tensor->ne[0];
337            layout.is_symmetric = true;
338            break;
339        case ExtraQuantType::Q8_0_32:
340            layout.is_u4 = false;
341            layout.weights_per_block = 32;
342            layout.is_symmetric = true;
343            break;
344        case ExtraQuantType::Q8_0_C:
345            layout.is_u4 = false;
346            layout.weights_per_block = tensor->ne[0];
347            layout.is_symmetric = true;
348            break;
349        case ExtraQuantType::Q8_1_C:
350            layout.is_u4 = false;
351            layout.weights_per_block = tensor->ne[0];
352            break;
353        default:
354            layout.weights_per_block = -1;
355            GGML_ABORT("Code of re-quantizing to channel-wise is not updated");
356            break;
357        }
358
359        if (layout.is_requant) {
360            // Calculate sizes for requantized format
361            layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;
362            int64_t n_blocks = n_elements / layout.weights_per_block;
363            layout.scales_size = n_blocks * sizeof(uint16_t);
364            // For symmetric quantization, no zp needed (weights stored as signed)
365            if (layout.is_symmetric) {
366                layout.zp_size = 0;
367            } else {
368                layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
369            }
370
371            layout.weights_offset = 0;
372            layout.scales_offset = ((layout.weights_size + alignment - 1) / alignment) * alignment;
373            layout.zp_offset = layout.scales_offset + ((layout.scales_size + alignment - 1) / alignment) * alignment;
374            layout.total_size = layout.zp_offset + layout.zp_size;
375            layout.total_size = std::max(layout.total_size, ggml_nbytes(tensor));
376            return layout;
377        }
378    }
379
380    // Normal extraction (no requant) - determine format based on tensor type
381    layout.is_u4 = false;
382    layout.weights_per_block = 32;
383    layout.is_symmetric = false;
384
385    switch (tensor->type) {
386    case GGML_TYPE_MXFP4:
387        layout.is_u4 = true;
388        layout.is_symmetric = true;
389        break;
390
391    case GGML_TYPE_Q4_0:
392        layout.is_u4 = true;
393        layout.is_symmetric = true;
394        break;
395
396    case GGML_TYPE_Q4_1:
397    case GGML_TYPE_Q4_K:
398        layout.is_u4 = true;
399        break;
400
401    case GGML_TYPE_Q8_0:
402        layout.is_symmetric = true;
403        break;
404
405    case GGML_TYPE_Q5_1:
406        // u8 weights (5-bit values), asymmetric (scale + zero point)
407        break;
408
409    case GGML_TYPE_Q6_K:
410        layout.weights_per_block = 16;
411        layout.is_symmetric = true;
412        break;
413
414    case GGML_TYPE_Q5_K:
415        break;
416
417    default:
418        // Unsupported quantization type
419        return layout;
420    }
421
422    // Calculate sizes
423    // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes
424    layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;
425
426    // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
427    int64_t n_blocks = n_elements / layout.weights_per_block;
428    layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
429    // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric
430    // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see
431    // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per
432    // block instead of a packed u4/u8 integer zero point.
433    if (layout.is_symmetric) {
434        layout.zp_size = 0;
435    } else if (use_bias || for_gather_matmul) {
436        layout.zp_size = n_blocks * sizeof(uint16_t);
437    } else {
438        layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
439    }
440
441    // Layout in buffer: [weights | scales | zp] with alignment
442    layout.weights_offset = 0;
443    layout.scales_offset = ((layout.weights_size + alignment - 1) / alignment) * alignment;
444    layout.zp_offset = layout.scales_offset + ((layout.scales_size + alignment - 1) / alignment) * alignment;
445    layout.total_size = layout.zp_offset + layout.zp_size;
446    layout.total_size = std::max(layout.total_size, ggml_nbytes(tensor));
447
448    return layout;
449}
450
451ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote) {
452    ov::Shape shape;
453    for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
454        shape.push_back(static_cast<size_t>(tensor->ne[i]));
455    }
456
457    ov::element::Type element_type;
458    switch (tensor->type) {
459    case GGML_TYPE_F32:
460        element_type = ov::element::f32;
461        break;
462    case GGML_TYPE_F16:
463        element_type = ov::element::f16;
464        break;
465    case GGML_TYPE_BF16:
466        element_type = ov::element::bf16;
467        break;
468    case GGML_TYPE_I32:
469        element_type = ov::element::i32;
470        break;
471    case GGML_TYPE_I64:
472        element_type = ov::element::i64;
473        break;
474    default:
475        // GGML_LOG_WARN("%s: unsupported tensor type for ov::Tensor: %s\n", __func__, ggml_type_name(tensor->type));
476        return nullptr;
477    }
478
479    const auto & device_name = ggml_openvino_get_device_name();
480    auto remote_context = ggml_openvino_get_remote_context();
481
482    std::shared_ptr<ov::Tensor> ov_tensor;
483    if (is_remote) {
484        GGML_ASSERT(device_name == "GPU");
485        auto gpu_context = remote_context->as<ov::intel_gpu::ocl::ClContext>();
486        auto usm_tensor = gpu_context.create_tensor(element_type, shape, tensor->data);
487        ov_tensor = std::make_shared<ov::intel_gpu::ocl::USMTensor>(std::move(usm_tensor));
488    } else {
489        ov_tensor = std::make_shared<ov::Tensor>(element_type, shape, tensor->data);
490    }
491
492    return new ggml_openvino_tensor_extra(ov_tensor);
493}