char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Qwen3.5, memory optimization, and test-recurrent-state-rollback (#26952)aee56b3ab

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.9 KiB133 linesraw
1#pragma once
2
3#include <cstdint>
4#include <map>
5#include <openvino/core/node.hpp>
6#include <openvino/core/partial_shape.hpp>
7#include <openvino/core/shape.hpp>
8#include <openvino/frontend/decoder.hpp>
9#include <set>
10#include <string>
11
12namespace ov {
13namespace frontend {
14namespace ggml {
15
16struct ModelInputInfo {
17    element::Type type;
18    PartialShape shape;
19};
20
21struct ModelExtraInputInfo {
22    element::Type type;
23    Shape shape;
24    int64_t value;
25    bool is_parameter;
26};
27
28class GgmlDecoder : public DecoderBase {
29public:
30    virtual ov::Any get_attribute(const std::string & name) const = 0;
31
32    virtual PartialShape get_input_shape(int node_idx, const std::string & name) const = 0;
33
34    virtual std::vector<size_t> get_input_stride(int node_idx, const std::string & name) const = 0;
35
36    virtual size_t get_view_input_size(int node_idx, const std::string & name) const = 0;
37
38    virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const = 0;
39
40    virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const = 0;
41
42    virtual std::vector<size_t> get_view_input_stride(int node_idx,
43                                                      const std::string & name,
44                                                      size_t view_index) const = 0;
45
46    virtual std::vector<size_t> get_view_input_src_stride(int node_idx,
47                                                          const std::string & name,
48                                                          size_t view_index) const = 0;
49
50    virtual Shape get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
51
52    virtual Shape get_view_input_src_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
53
54    virtual PartialShape get_view_input_ov_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
55
56    virtual PartialShape get_view_input_src_ov_shape(int node_idx,
57                                                     const std::string & name,
58                                                     size_t view_index) const = 0;
59
60    virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const = 0;
61
62    virtual std::string get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const = 0;
63
64    virtual element::Type get_input_type(int node_idx, const std::string & name) const = 0;
65
66    virtual size_t get_input_size() const = 0;
67
68    virtual size_t get_input_size(int node_idx) const = 0;
69
70    virtual void get_input_node(size_t input_port_idx,
71                                std::string & producer_name,
72                                std::string & producer_output_port_name,
73                                size_t & producer_output_port_index) const = 0;
74
75    virtual std::vector<std::string> get_input_names(int node_idx) const = 0;
76
77    virtual PartialShape get_output_shape(int node_idx) const = 0;
78
79    virtual element::Type get_output_type(const int node_idx) const = 0;
80
81    virtual std::vector<size_t> get_output_stride(int node_idx) const = 0;
82
83    virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const = 0;
84
85    virtual int32_t * get_output_op_params(int node_idx) const = 0;
86
87    virtual size_t get_output_op_offset(int node_idx) const = 0;
88
89    virtual std::vector<std::string> get_output_names(int node_idx) const = 0;
90
91    virtual std::string get_inplace_op_src(int node_idx) const = 0;
92
93    virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0;
94
95    virtual const std::string & get_op_type() const = 0;
96
97    virtual const std::string & get_op_type(int node_idx) const = 0;
98
99    virtual const std::string & get_op_name() const = 0;
100
101    virtual const std::string & get_op_name(int node_idx) const = 0;
102
103    virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const = 0;
104
105    virtual int get_op_case(int node_idx) const = 0;
106
107    virtual const std::map<std::string, ModelInputInfo> & get_model_inputs() const = 0;
108    virtual const std::map<std::string, ModelExtraInputInfo> & get_model_extra_inputs() const = 0;
109    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const = 0;
110    virtual std::set<std::string> get_model_output_names() const = 0;
111
112    virtual int32_t * get_rope_params() const = 0;
113
114    virtual bool has_mixed_rope_params() const = 0;
115
116    virtual int get_ssm_state_size() const = 0;
117
118    virtual std::map<std::string, std::string> get_kv_param_res_names() const = 0;
119
120    virtual bool is_static() const = 0;
121
122    virtual bool is_stateful() const = 0;
123
124    virtual bool is_splited_model() const = 0;
125
126    virtual int is_swa_layer(int layer) const = 0;
127
128    virtual int32_t get_op_dynamic_dim(int node_idx) const = 0;
129};
130
131}  // namespace ggml
132}  // namespace frontend
133}  // namespace ov