char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.9 KiB195 linesraw
1#include "ggml-decoder.h"
2#include "ggml-impl.h"
3
4#include <algorithm>
5#include <atomic>
6#include <cstddef>
7#include <functional>
8#include <memory>
9#include <mutex>
10#include <openvino/runtime/core.hpp>
11#include <openvino/runtime/infer_request.hpp>
12#include <string>
13#include <unordered_map>
14#include <utility>
15#include <vector>
16
17struct graph_key {
18    int n_nodes;
19    std::string first_node_name;
20    std::string last_node_name;
21    std::vector<std::string> input_src_names;
22
23    graph_key(const ggml_cgraph * cgraph) : n_nodes(cgraph->n_nodes) {
24        if (n_nodes > 0) {
25            first_node_name = cgraph->nodes[0]->name;
26            last_node_name = cgraph->nodes[n_nodes - 1]->name;
27        }
28
29        auto get_input_key_name = [](const ggml_cgraph * graph, const ggml_tensor * tensor) {
30            std::string name = tensor->name;
31            const size_t hash_pos = ggml_hash_find(&graph->visited_hash_set, tensor);
32            if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) &&
33                hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, hash_pos)) {
34                name += "#" + std::to_string(hash_pos);
35            }
36            return name;
37        };
38
39        std::vector<std::string> node_names;
40        node_names.reserve(cgraph->n_nodes);
41        for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) {
42            node_names.emplace_back(cgraph->nodes[node_idx]->name);
43        }
44
45        for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) {
46            const ggml_tensor * node = cgraph->nodes[node_idx];
47            for (int src_idx = 0; src_idx < GGML_MAX_SRC; src_idx++) {
48                const ggml_tensor * src = node->src[src_idx];
49                if (src == nullptr || src->name[0] == '\0') {
50                    continue;
51                }
52
53                const std::string src_name = get_input_key_name(cgraph, src);
54                if (std::find(node_names.begin(), node_names.end(), src_name) != node_names.end()) {
55                    continue;
56                }
57                if (src_name.find("weight") != std::string::npos) {
58                    continue;
59                }
60
61                input_src_names.push_back(std::to_string(node_idx) + ":" + std::to_string(src_idx) + ":" + src_name);
62            }
63        }
64    }
65
66    bool operator==(const graph_key & other) const {
67        return n_nodes == other.n_nodes && first_node_name == other.first_node_name &&
68               last_node_name == other.last_node_name && input_src_names == other.input_src_names;
69    }
70};
71
72struct graph_key_hash {
73    size_t operator()(const graph_key & key) const {
74        size_t hash = std::hash<int>{}(key.n_nodes);
75        if (key.n_nodes > 0) {
76            hash ^= std::hash<std::string>{}(key.first_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
77            hash ^= std::hash<std::string>{}(key.last_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
78        }
79        for (const auto & input_src_name : key.input_src_names) {
80            hash ^= std::hash<std::string>{}(input_src_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
81        }
82        return hash;
83    }
84};
85
86struct decoder_runtime_ctx {
87    decoder_runtime_ctx(std::shared_ptr<std::mutex> mutex) : mutex(std::move(mutex)) {}
88
89    std::shared_ptr<std::mutex> mutex;
90    std::shared_ptr<GgmlOvDecoder> ptr;
91};
92
93struct ov_runtime_context {
94    mutable std::mutex ctx_mutex;
95    std::string device;
96    bool stateful;
97    std::unordered_map<graph_key, std::shared_ptr<decoder_runtime_ctx>, graph_key_hash> decoder_cache;
98    std::unordered_map<graph_key, std::shared_ptr<ov::InferRequest>, graph_key_hash> infer_request_cache;
99    std::unordered_map<graph_key, std::shared_ptr<ov::InferRequest>, graph_key_hash> infer_request_cache_prefill;
100    std::unordered_map<graph_key, std::vector<std::string>, graph_key_hash> ov_input_names_cache;
101    std::unordered_map<graph_key, std::vector<std::string>, graph_key_hash> ov_output_names_cache;
102    //TODO: Stateful is only supported for single request at a time.
103    //      Simultanous stateful inference request support to be added.
104    size_t stateful_kv_size;
105    std::map<std::string, std::string> kv_state_input_name_map;
106    std::atomic<int> backend_count;
107
108    ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {}
109
110    void clear_caches_locked() {
111        decoder_cache.clear();
112        infer_request_cache.clear();
113        infer_request_cache_prefill.clear();
114        ov_input_names_cache.clear();
115        ov_output_names_cache.clear();
116        kv_state_input_name_map.clear();
117        stateful_kv_size = 0;
118    }
119
120    void clear_caches() {
121        std::lock_guard<std::mutex> lock(ctx_mutex);
122        clear_caches_locked();
123    }
124};
125
126enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend);
127
128enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
129enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
130
131size_t checksum(const void * data, size_t size);
132
133bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path);
134
135void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor);
136
137void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst);
138
139template <typename T>
140std::vector<T> pad_input(const T * data,
141                         size_t rows,
142                         size_t cols,
143                         size_t padded_rows,
144                         size_t padded_cols,
145                         T pad_value) {
146    std::vector<T> padded(padded_rows * padded_cols, pad_value);
147
148    for (size_t i = 0; i < std::min(rows, padded_rows); ++i) {
149        for (size_t j = 0; j < std::min(cols, padded_cols); ++j) {
150            padded[i * padded_cols + j] = data[i * cols + j];
151        }
152    }
153
154    return padded;
155}
156
157template <typename T>
158std::vector<T> pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t padded_cols, T pad_value) {
159    return pad_input<T>(reinterpret_cast<const T *>(tensor->data),
160                        static_cast<size_t>(tensor->ne[1]),  // rows
161                        static_cast<size_t>(tensor->ne[0]),  // cols
162                        padded_rows, padded_cols, pad_value);
163}
164
165const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph);
166
167int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
168
169bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
170
171ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name);
172ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
173                                             const std::string & param_name);
174ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
175                                              const std::string & param_name,
176                                              int chunk_index);
177
178ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
179                                   std::shared_ptr<ov::InferRequest> infer_request,
180                                   int output_index,
181                                   const ggml_tensor * ggml_tensor);
182
183bool is_naive(struct ggml_cgraph * cgraph);
184
185/**
186 * @brief Heuristically checks whether the given computation graph is a split-model fragment.
187 * @param cgraph Pointer to the GGML computation graph to analyze.
188 * @return true if the graph is identified as split; otherwise false.
189 */
190bool is_model_splitted(struct ggml_cgraph * cgraph);
191
192enum ggml_status naive_compute(struct ggml_cgraph * cgraph,
193                               ov::Core & core,
194                               const std::string & device,
195                               const ov::AnyMap & config);