char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Qwen3.5, memory optimization, and test-recurrent-state-rollback (#26952)aee56b3ab

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.8 KiB171 linesraw
1#pragma once
2
3#include "decoder.h"
4
5#include <cstdint>
6#include <openvino/frontend/node_context.hpp>
7#include <string>
8
9namespace ov {
10namespace frontend {
11namespace ggml {
12
13class TranslateSession;
14
15typedef std::map<std::string, Output<Node>> TensorMap;
16
17class NodeContext : public frontend::NodeContext {
18public:
19    NodeContext(const std::shared_ptr<GgmlDecoder> & decoder,
20                std::shared_ptr<TensorMap> & tensor_map,
21                int node_idx,
22                TranslateSession * translate_session = nullptr) :
23        ov::frontend::NodeContext(decoder->get_op_type(node_idx)),
24        m_decoder(decoder),
25        m_tensor_map(tensor_map),
26        m_node_idx(node_idx),
27        m_translate_session(translate_session) {
28        m_input_names = decoder->get_input_names(m_node_idx);
29        m_output_names = decoder->get_output_names(m_node_idx);
30    }
31
32    TranslateSession * get_translate_session() const { return m_translate_session; }
33
34    const std::vector<std::string> & get_input_names() const { return m_input_names; }
35
36    size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); }
37
38    ov::element::Type get_input_type(size_t index) const {
39        return m_decoder->get_input_type(m_node_idx, m_input_names[index]);
40    }
41
42    PartialShape get_input_shape(size_t input_index) const {
43        return m_decoder->get_input_shape(m_node_idx, m_input_names[input_index]);
44    }
45
46    std::vector<size_t> get_input_stride(size_t index) const {
47        return m_decoder->get_input_stride(m_node_idx, m_input_names[index]);
48    }
49
50    std::string get_output_name() const { return m_output_names[0]; }
51
52    PartialShape get_output_shape() const { return m_decoder->get_output_shape(m_node_idx); }
53
54    int32_t * get_input_op_params(size_t index) const {
55        return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]);
56    }
57
58    size_t get_view_input_size(size_t index) const {
59        return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]);
60    }
61
62    size_t get_view_input_offset(size_t index, size_t view_index) const {
63        return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index);
64    }
65
66    size_t get_view_input_src_offset(size_t index, size_t view_index) const {
67        return m_decoder->get_view_input_src_offset(m_node_idx, m_input_names[index], view_index);
68    }
69
70    std::vector<size_t> get_view_input_stride(size_t index, size_t view_index) const {
71        return m_decoder->get_view_input_stride(m_node_idx, m_input_names[index], view_index);
72    }
73
74    std::vector<size_t> get_view_input_src_stride(size_t index, size_t view_index) const {
75        return m_decoder->get_view_input_src_stride(m_node_idx, m_input_names[index], view_index);
76    }
77
78    ov::Shape get_view_input_ggml_shape(size_t index, size_t view_index) const {
79        return m_decoder->get_view_input_ggml_shape(m_node_idx, m_input_names[index], view_index);
80    }
81
82    ov::Shape get_view_input_src_ggml_shape(size_t index, size_t view_index) const {
83        return m_decoder->get_view_input_src_ggml_shape(m_node_idx, m_input_names[index], view_index);
84    }
85
86    ov::PartialShape get_view_input_ov_shape(size_t index, size_t view_index) const {
87        return m_decoder->get_view_input_ov_shape(m_node_idx, m_input_names[index], view_index);
88    }
89
90    ov::PartialShape get_view_input_src_ov_shape(size_t index, size_t view_index) const {
91        return m_decoder->get_view_input_src_ov_shape(m_node_idx, m_input_names[index], view_index);
92    }
93
94    std::string get_view_input_name(size_t index, size_t view_index) const {
95        return m_decoder->get_view_input_name(m_node_idx, m_input_names[index], view_index);
96    }
97
98    std::string get_view_input_src_name(size_t index, size_t view_index) const {
99        return m_decoder->get_view_input_src_name(m_node_idx, m_input_names[index], view_index);
100    }
101
102    int32_t get_op_dynamic_dim() const { return m_decoder->get_op_dynamic_dim(m_node_idx); }
103
104    int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); }
105
106    size_t get_output_op_offset() const { return m_decoder->get_output_op_offset(m_node_idx); }
107
108    ov::element::Type get_output_type() const { return m_decoder->get_output_type(m_node_idx); }
109
110    std::vector<size_t> get_output_stride() const { return m_decoder->get_output_stride(m_node_idx); }
111
112    Output<Node> get_input(int idx) const override {
113        // Check if this input is a VIEW
114        size_t view_input_size = m_decoder->get_view_input_size(m_node_idx, m_input_names[idx]);
115        if (view_input_size > 0) {
116            // This is a VIEW input, get the base tensor name (last element in the chain)
117            std::string base_name =
118                m_decoder->get_view_input_src_name(m_node_idx, m_input_names[idx], view_input_size - 1);
119            // Check if the VIEW has been resolved (translate_view produced a Slice)
120            auto view_it = m_tensor_map->find(m_input_names[idx]);
121            if (!base_name.empty() && view_it != m_tensor_map->end()) {
122                auto base_it = m_tensor_map->find(base_name);
123                if (base_it != m_tensor_map->end() &&
124                    view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) {
125                    return view_it->second;
126                }
127                return base_it->second;
128            }
129            if (!base_name.empty()) {
130                return m_tensor_map->at(base_name);
131            }
132        }
133        // Not a VIEW or failed to get base name, use the original logic
134        return m_tensor_map->at(m_input_names[idx]);
135    }
136
137    Output<Node> get_input(const std::string & name) const override {
138        if (m_tensor_map->find(name) == m_tensor_map->end()) {
139            throw std::runtime_error("'" + name + "' not found in tensor map.");
140        }
141        return m_tensor_map->at(name);
142    }
143
144    bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); }
145
146    const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); }
147
148    ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); }
149
150    int get_op_case() const { return m_decoder->get_op_case(m_node_idx); }
151
152    bool is_static() const { return m_decoder->is_static(); }
153
154    bool is_stateful() const { return m_decoder->is_stateful(); }
155
156    int get_ssm_state_size() const { return m_decoder->get_ssm_state_size(); }
157
158private:
159    std::shared_ptr<GgmlDecoder> m_decoder;
160    std::shared_ptr<TensorMap> & m_tensor_map;
161    int m_node_idx;
162    TranslateSession * m_translate_session;
163    std::vector<std::string> m_input_names;
164    std::vector<std::string> m_output_names;
165};
166
167using CreatorFunction = std::function<ov::OutputVector(const ov::frontend::ggml::NodeContext &)>;
168
169}  // namespace ggml
170}  // namespace frontend
171}  // namespace ov