char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
18.2 KiB417 linesraw
1#pragma once
2
3#include "ggml-backend-impl.h"
4#include "ggml-backend.h"
5#include "ggml.h"
6#include "openvino/decoder.h"
7
8#include <cstdint>
9#include <cstring>
10#include <map>
11#include <memory>
12#include <openvino/core/partial_shape.hpp>
13#include <optional>
14#include <set>
15#include <string>
16#include <vector>
17
18struct ModelParams {
19    int ctx = -1;
20    int ctx_per_seq = -1;
21    int ctx_per_seq_swa = -1;
22    int n_seq = 1;
23    int n_heads_kv = -1;
24    int head_size = -1;
25    int state_size = -1;  // for SSM molels, eg qwen35
26    int32_t rope_params[15];
27    bool mixed_rope_params = false;
28    std::vector<int> swa_layers;
29
30    std::vector<std::string> kv_names;
31    size_t kv_buffer_ctx_id = 0;
32
33    bool same_rope_params(const ModelParams & other) const {
34        return mixed_rope_params == other.mixed_rope_params &&
35               memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;
36    }
37
38    bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); }
39
40    bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx; }
41
42    bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; }
43};
44
45struct ComputeParams {
46    int n_seq_active = 1;
47    int seq_active_start = 0;
48    int attention_size = -1;
49    int attention_size_swa = -1;
50    int attention_size_static = -1;  // encoder/cross-attn KV fill level (whisper)
51    int input_len = -1;
52    int token_len_per_seq = -1;
53    int past_kv_len = -1;
54    int output_len = 1;
55
56    int cache_rs_reset_idx = -1;
57    int cache_rs_reset_len = -1;
58    // SSM/DeltaNet models otionally clear cache_r and cache_s of certain slots in the cgraph
59    // 3: [ 18432,     4,     1,     1] RESHAPE              cache_r_l0 (reshaped)
60    //    [ 18432,     4,     1,     1]            0: NONE        cache_r_l0
61    // 4: [ 18432,     1,     1,     1] VIEW                 cache_r_l0 (reshaped) (view)
62    //    [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
63    // 5: [ 18432,     1,     1,     1] SCALE                cache_r_l0 (reshaped) (view) (view)
64    //    [ 18432,     1,     1,     1]            0: VIEW        cache_r_l0 (reshaped) (view)
65
66    int s_copy_active_slot_len = -1;
67    // SSM/DeltaNet models otionally reorder slots of state cache, to make the active slots contiguous
68    // leaf_5 is the inp->s_copy in llama-graph.cpp, eg if there are 8 slots in total and slot 3 and 7
69    // are active in the current batch, leaf_5 will be [3, 7, 5, 6, 4]
70    //  6: [     2,     1,     1,     1] VIEW                  (view)
71    //      [     2,     1,     1,     1]            0: NONE        leaf_5
72    //  7: [ 18432,     2,     1,     1] GET_ROWS             conv_states-0
73    //      [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
74    //      [     2,     1,     1,     1]            1: VIEW         (view)
75    //  8: [     0,     1,     1,     1] VIEW                  (view)
76    //      [     2,     1,     1,     1]            0: NONE        leaf_5
77    //  9: [ 18432,     0,     1,     1] GET_ROWS             node_9
78    //      [ 18432,     4,     1,     1]            0: RESHAPE     cache_r_l0 (reshaped)
79    //      [     0,     1,     1,     1]            1: VIEW         (view)
80    // 10: [ 18432,     0,     1,     1] VIEW                 cache_r_l0 (view)
81    //      [ 18432,     4,     1,     1]            0: NONE        cache_r_l0
82    // 11: [ 18432,     0,     1,     1] CPY                  cache_r_l0 (view) (copy of )
83    //      [ 18432,     0,     1,     1]            0: GET_ROWS    node_9
84    //      [ 18432,     0,     1,     1]            1: VIEW        cache_r_l0 (view)
85
86    struct RsWriteback {
87        int slot_begin = 0;  // first cache slot written by the CPY
88        int src_begin = 0;   // first source row or column copied by the CPY
89    };
90
91    std::map<std::string, RsWriteback> rs_writebacks;
92    // Destination slot offset of each state cache writeback CPY node, keyed by node name. It
93    // changes with the batch (kv head, active sequence count) and, with rollback enabled
94    // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the
95    // cached model as a runtime input. Dynamic models also receive the source-side offset; static
96    // models use a fixed end-anchored offset in the translator.
97};
98
99class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
100public:
101    struct NodeInfo {
102        ggml_tensor * node;
103        std::string node_name;
104        std::string node_op_type;
105        std::map<std::string, ggml_tensor *> node_inputs;
106        std::map<std::string, std::vector<std::pair<std::string, ggml_tensor *>>> node_inputs_views;
107        std::vector<std::string> node_inputs_names;
108        int node_op_case = 0;
109        void * data_addr;
110    };
111
112    // Graph decoder
113    GgmlOvDecoder(ggml_cgraph * cgraph,
114                  ModelParams & model_params,
115                  ComputeParams & compute_params,
116                  std::map<std::string, std::shared_ptr<ov::Node>> & model_weights,
117                  bool is_static,
118                  bool is_stateful = false,
119                  bool model_is_splitted = false,
120                  bool is_prefill = false,
121                  int prefill_chunk_size = 256);
122
123    // Naive graph decoder
124    GgmlOvDecoder(ggml_cgraph * cgraph, std::map<std::string, std::shared_ptr<ov::Node>> & model_weights);
125
126    virtual ov::Any get_attribute(const std::string & name) const override {
127        return nullptr;
128        GGML_UNUSED(name);
129    }
130
131    virtual ov::PartialShape get_input_shape(int node_idx, const std::string & name) const override;
132
133    virtual std::vector<size_t> get_input_stride(int node_idx, const std::string & name) const override;
134
135    virtual size_t get_view_input_size(int node_idx, const std::string & name) const override;
136
137    virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override;
138
139    virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const override;
140
141    virtual std::vector<size_t> get_view_input_stride(int node_idx,
142                                                      const std::string & name,
143                                                      size_t view_index) const override;
144
145    virtual std::vector<size_t> get_view_input_src_stride(int node_idx,
146                                                          const std::string & name,
147                                                          size_t view_index) const override;
148
149    virtual ov::Shape get_view_input_ggml_shape(int node_idx,
150                                                const std::string & name,
151                                                size_t view_index) const override;
152
153    virtual ov::Shape get_view_input_src_ggml_shape(int node_idx,
154                                                    const std::string & name,
155                                                    size_t view_index) const override;
156
157    virtual ov::PartialShape get_view_input_ov_shape(int node_idx,
158                                                     const std::string & name,
159                                                     size_t view_index) const override;
160
161    virtual ov::PartialShape get_view_input_src_ov_shape(int node_idx,
162                                                         const std::string & name,
163                                                         size_t view_index) const override;
164
165    virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const override;
166
167    virtual std::string get_view_input_src_name(int node_idx,
168                                                const std::string & name,
169                                                size_t view_index) const override;
170
171    virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override;
172
173    virtual size_t get_input_size() const override;
174
175    virtual size_t get_input_size(int node_idx) const override;
176
177    virtual void get_input_node(size_t input_port_idx,
178                                std::string & producer_name,
179                                std::string & producer_output_port_name,
180                                size_t & producer_output_port_index) const override {
181        GGML_UNUSED(input_port_idx);
182        GGML_UNUSED(producer_name);
183        GGML_UNUSED(producer_output_port_name);
184        GGML_UNUSED(producer_output_port_index);
185    }
186
187    virtual std::vector<std::string> get_input_names(int node_idx) const override;
188
189    virtual ov::PartialShape get_output_shape(int node_idx) const override;
190
191    virtual ov::element::Type get_output_type(int node_idx) const override;
192
193    virtual std::vector<size_t> get_output_stride(int node_idx) const override;
194
195    virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override;
196
197    virtual int32_t * get_output_op_params(int node_idx) const override;
198
199    virtual size_t get_output_op_offset(int node_idx) const override;
200
201    virtual std::vector<std::string> get_output_names(int node_idx) const override;
202
203    virtual std::string get_inplace_op_src(int node_idx) const override;
204
205    virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override;
206
207    virtual const std::string & get_op_type() const override;
208
209    virtual const std::string & get_op_type(int node_idx) const override;
210
211    virtual const std::string & get_op_name() const override;
212
213    virtual const std::string & get_op_name(int node_idx) const override;
214
215    virtual int32_t get_op_dynamic_dim(int node_idx) const override;
216
217    virtual void visit_subgraph(
218        std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const override;
219
220    ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); }
221
222    virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }
223
224    virtual const std::map<std::string, ov::frontend::ggml::ModelInputInfo> & get_model_inputs() const override {
225        return m_model_inputs;
226    }
227
228    virtual const std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> & get_model_extra_inputs() const override {
229        return m_model_extra_inputs;
230    }
231
232    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const override {
233        return m_model_weights;
234    }
235
236    virtual std::set<std::string> get_model_output_names() const override { return m_model_output_names; }
237
238    const std::map<std::string, ggml_tensor *> & get_model_outputs() const { return m_model_outputs; }
239
240    virtual int get_ctx_size() const { return m_model_params.ctx; }
241
242    virtual int get_ctx_per_seq() const { return m_model_params.ctx_per_seq; }
243
244    virtual int get_ctx_per_seq_swa() const { return m_model_params.ctx_per_seq_swa; }
245
246    virtual int get_n_seq() const { return m_model_params.n_seq; }
247
248    virtual int is_swa_layer(int layer) const override {
249        return std::find(m_model_params.swa_layers.begin(), m_model_params.swa_layers.end(), layer) !=
250               m_model_params.swa_layers.end();
251    }
252
253    int get_past_kv_len() const { return m_compute_params.past_kv_len; }
254
255    int get_input_len() const { return m_compute_params.input_len; }
256
257    virtual int32_t * get_rope_params() const override { return const_cast<int32_t *>(m_model_params.rope_params); }
258
259    virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; }
260
261    virtual int get_ssm_state_size() const override { return m_model_params.state_size; }
262
263    virtual std::map<std::string, std::string> get_kv_param_res_names() const override;
264
265    virtual bool is_static() const override { return m_is_static; }
266
267    virtual bool is_stateful() const override { return m_is_stateful; }
268
269    int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; }
270
271    virtual bool is_splited_model() const override { return m_model_is_splitted; }
272
273    ov::PartialShape get_graph_input_shape(const ggml_tensor * op,
274                                           const ggml_tensor * input,
275                                           int dynamic_dim_index = -1) const;
276
277    static void dump_cgraph(const ggml_cgraph * cgraph, std::string & filename);
278
279    static std::shared_ptr<ov::Node> create_weight_node(ggml_tensor * tensor, bool naive = false);
280
281    static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(ggml_cgraph * cgraph,
282                                                                                bool naive = false);
283
284    // Collect just the set of weight-tensor names referenced by the graph, without
285    // building (or requantizing) any OV weight nodes. Used by topology checks like
286    // is_model_splitted that only need name membership.
287    static std::set<std::string> collect_weight_names(ggml_cgraph * cgraph);
288
289    const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;
290
291    const ggml_tensor * get_tensor_from_name(const std::string & name) const;
292
293    void clear_model_weights() { m_model_weights.clear(); }
294
295    static std::pair<ModelParams, ComputeParams> compute_llm_params(ggml_cgraph * cgraph, bool is_static);
296
297    ModelParams get_model_params() const { return m_model_params; }
298
299    ComputeParams get_compute_params() const { return m_compute_params; }
300
301    void set_model_params(const ModelParams & model_params) { m_model_params = model_params; }
302
303    void set_compute_params(const ComputeParams & compute_params) { m_compute_params = compute_params; }
304
305    bool m_is_static = false;
306    bool m_is_stateful = false;
307    bool m_is_prefill = false;
308    bool m_naive = false;
309    int m_prefill_chunk_size = 0;
310    bool m_model_is_splitted = false;  // label the cgraph is splited or not
311
312    static ov::Shape get_shape(const ggml_tensor * tensor);
313    static std::vector<size_t> get_stride(const ggml_tensor * tensor);
314    static ov::element::Type get_ov_type(const ggml_tensor * tensor);
315    static std::string compute_op_type(const ggml_tensor * node);
316    void add_extra_inputs();
317
318    void update_io(ggml_cgraph * cgraph);
319
320    inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
321        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;
322    }
323
324    inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
325        return op->op == GGML_OP_ROPE && tensor == op->src[1];
326    }
327
328    // IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length
329    // n_tokens; other modes carry a single position per token.
330    inline static int get_inp_pos_n_planes(const ggml_tensor * op) {
331        return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1;
332    }
333
334    inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
335        return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
336    }
337
338    inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
339        return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) ||
340               (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
341    }
342
343    inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
344        return op->op == GGML_OP_ROPE && tensor == op->src[2];
345    }
346
347    // also returns true for cache_s and cache_r in SSM/DeltaNet models
348    inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
349        if (tensor == nullptr) {
350            return false;
351        }
352        return (tensor->buffer != nullptr && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) ||
353               (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
354    }
355
356    inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
357        return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
358    }
359
360    inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
361        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
362               op->src[1]->op == GGML_OP_NONE;
363    }
364
365    // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
366    inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
367        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
368               op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY;
369    }
370
371    std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const {
372        if (is_inp_pos(tensor, op)) {
373            return "inp_pos";
374        }
375        if (is_inp_emb(tensor, op)) {
376            return "embd";
377        }
378        if (is_stateful() && is_inp_mask(tensor, op)) {
379            return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
380        }
381        return tensor->name;
382    }
383
384private:
385    void set_input_output();
386    int compute_op_case(const ggml_tensor * node) const;
387    bool node_is_used_as_src(const int node_idx);
388    void compute_model_inputs();
389    void compute_model_outputs();
390
391    // True if tensor is the inp->s_copy index leaf gathered by a recurrent state cache GET_ROWS
392    // (possibly through a VIEW), so it gets a dynamic [1,1,1,-1] graph-input shape.
393    bool is_s_copy_leaf(const ggml_tensor * tensor) const;
394
395    // Infer and propagate dynamic-dimension indices for all tensors in the GGML graph.
396    void compute_node_dynamic_dims();
397
398    void validate_cgraph() const;
399
400    ggml_cgraph * m_cgraph = nullptr;
401    std::map<std::string, ggml_tensor *> m_inputs;
402
403    std::map<std::string, ov::frontend::ggml::ModelInputInfo> m_model_inputs;
404    std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> m_model_extra_inputs;
405    std::map<std::string, std::shared_ptr<ov::Node>> m_model_weights;
406    std::map<std::string, ggml_tensor *> m_model_outputs;
407    std::set<std::string> m_model_output_names;
408    std::vector<NodeInfo> m_node_info_list;
409    std::map<ggml_tensor *, int> m_node_dynamic_dims;
410
411    ModelParams m_model_params;
412    ComputeParams m_compute_params;
413};
414
415void print_tensor_address_map(const ggml_cgraph * cgraph);
416
417std::optional<int> extract_layer_from_name(const std::string & name);