char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
16.5 KiB295 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <climits>
6#include <cstdint>
7#include <cstdio>
8#include <memory>
9#include <numeric>
10#include <openvino/frontend/exception.hpp>
11#include <openvino/op/add.hpp>
12#include <openvino/op/concat.hpp>
13#include <openvino/op/constant.hpp>
14#include <openvino/op/convert.hpp>
15#include <openvino/op/gather.hpp>
16#include <openvino/op/multiply.hpp>
17#include <openvino/op/negative.hpp>
18#include <openvino/op/range.hpp>
19#include <openvino/op/reshape.hpp>
20#include <openvino/op/scatter_update.hpp>
21#include <openvino/op/shape_of.hpp>
22#include <openvino/op/slice.hpp>
23#include <openvino/op/squeeze.hpp>
24#include <openvino/op/subtract.hpp>
25#include <vector>
26
27namespace ov {
28namespace frontend {
29namespace ggml {
30namespace op {
31
32OutputVector translate_cpy(const NodeContext & context) {
33    auto op_case = context.get_op_case();
34    auto input_shape = context.get_input_shape(0);
35    auto output_shape = context.get_input_shape(1);
36
37    if (op_case == 4) {
38        auto src = process_view_input_new(context, 0);
39        auto base = context.get_input(1);
40
41        int64_t n_elems = 1;
42        for (const auto & dim : context.get_output_shape().to_shape()) {
43            n_elems *= static_cast<int64_t>(dim);
44        }
45
46        const auto output_stride = context.get_output_stride();
47        const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back();
48        FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state view update has invalid element size");
49
50        const int64_t begin_val = static_cast<int64_t>(context.get_output_op_offset() / elem_size);
51        const int64_t end_val = begin_val + n_elems;
52
53        auto flat_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, 1, -1});
54        src = std::make_shared<ov::op::v1::Reshape>(src, flat_shape, false);
55        if (src.get_element_type() != context.get_output_type()) {
56            src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
57        }
58
59        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
60        auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val});
61        auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val});
62        auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
63        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
64        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
65
66        auto head_part = std::make_shared<ov::op::v8::Slice>(base, zero, begin, one, axis);
67        auto tail_part = std::make_shared<ov::op::v8::Slice>(base, end, int_max, one, axis);
68        auto res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{head_part, src, tail_part}, 3);
69        return rename_outputs_with_suffix({res}, context.get_name());
70    }
71
72    // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a
73    // runtime input, so the cached model works for any kv head and active sequence count. The
74    // result is the full updated cache.
75    // op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
76    if (op_case == 3) {
77        // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers
78        // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size
79        // tensors, so short-circuit instead of building a degenerate Slice/Concat chain.
80        bool is_empty = false;
81        if (input_shape.rank().is_static()) {
82            for (const auto & d : input_shape) {
83                if (d.is_static() && d.get_length() == 0) {
84                    is_empty = true;
85                    break;
86                }
87            }
88        }
89        if (is_empty) {
90            return {context.get_input(1)};
91        }
92    }
93    const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
94    const bool slice_assign =
95        context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3);
96    if (slice_assign) {
97        const int64_t slot_axis = 2;
98        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
99        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
100        auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
101        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {slot_axis});
102        auto feature = ov::op::v0::Constant::create(ov::element::i64, {4},
103                                                    std::vector<int64_t>{1, 1, -1, output_shape[3].get_length()});
104
105        ov::Output<ov::Node> src;
106        ov::Output<ov::Node> begin = context.get_input(slot_begin_name);
107        auto base = context.get_input(1);
108        if (op_case == 1) {
109            ov::Output<ov::Node> state_begin;
110            const std::string src_begin_name = "rs_src_begin_" + context.get_name();
111            if (context.has_input(src_begin_name)) {
112                state_begin = context.get_input(src_begin_name);
113            } else {
114                auto ssm_state_size = context.get_ssm_state_size();
115                if (context.has_input("s_copy_active_slot_len")) {
116                    auto len = context.get_input("s_copy_active_slot_len");
117                    auto state_rows = std::make_shared<ov::op::v1::Multiply>(
118                        ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len);
119                    state_begin = std::make_shared<ov::op::v0::Negative>(state_rows);
120                } else {
121                    state_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size});
122                }
123            }
124            auto state_part =
125                std::make_shared<ov::op::v8::Slice>(context.get_input(0), state_begin, int_max, one, axis);
126            src = std::make_shared<ov::op::v1::Reshape>(state_part, feature, false);
127        } else if (op_case == 2) {
128            // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1
129            // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output
130            // shape static even though the window start is a runtime value.
131            auto window_size = (int64_t) input_shape[3].get_length();
132            ov::Output<ov::Node> window;
133            auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
134            const std::string src_begin_name = "rs_src_begin_" + context.get_name();
135            if (context.has_input(src_begin_name)) {
136                auto src_begin = context.get_input(src_begin_name);
137                auto src_end = std::make_shared<ov::op::v1::Add>(
138                    src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
139                window = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, src_end, one, col_axis);
140            } else if (context.has_input("chunk_valid_len")) {
141                std::vector<int64_t> offsets(window_size);
142                std::iota(offsets.begin(), offsets.end(), 0);
143                auto indices = std::make_shared<ov::op::v1::Add>(
144                    ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets),
145                    context.get_input("chunk_valid_len"));
146                window = std::make_shared<ov::op::v8::Gather>(context.get_input(0), indices, col_axis);
147            } else {
148                auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size});
149                window =
150                    std::make_shared<ov::op::v8::Slice>(context.get_input(0), window_begin, int_max, one, col_axis);
151            }
152            const auto base_shape = base.get_partial_shape();
153            FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4,
154                                          "CPY conv state cache update requires rank-4 base cache");
155            FRONT_END_OP_CONVERSION_CHECK(base_shape[3].is_static(),
156                                          "CPY conv state cache update requires static feature size");
157            FRONT_END_OP_CONVERSION_CHECK(input_shape.rank().is_static() && input_shape.rank().get_length() == 4 &&
158                                              input_shape[2].is_static() && input_shape[3].is_static(),
159                                          "CPY conv state cache update requires static source feature view");
160
161            const int64_t full_feature_size = base_shape[3].get_length();
162            const int64_t update_feature_size = input_shape[2].get_length() * input_shape[3].get_length();
163            const auto output_stride = context.get_output_stride();
164            const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back();
165            FRONT_END_OP_CONVERSION_CHECK(elem_size > 0,
166                                          "CPY conv state cache update has invalid element size");
167            const int64_t feature_begin = static_cast<int64_t>(context.get_output_op_offset() / elem_size) %
168                                          full_feature_size;
169            const int64_t feature_end = feature_begin + update_feature_size;
170            FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size,
171                                          "CPY conv state cache update feature range is out of bounds");
172
173            auto partial_feature = ov::op::v0::Constant::create(
174                ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, update_feature_size});
175            src = std::make_shared<ov::op::v1::Reshape>(window, partial_feature, false);
176            if (src.get_element_type() != context.get_output_type()) {
177                src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
178            }
179
180            auto src_len = std::make_shared<ov::op::v8::Gather>(
181                std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
182                ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
183            auto slot_end = std::make_shared<ov::op::v1::Add>(begin, src_len);
184            auto active_slots = std::make_shared<ov::op::v8::Slice>(base, begin, slot_end, one, axis);
185
186            auto feature_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
187            auto feature_begin_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_begin});
188            auto feature_end_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_end});
189            auto feature_head = std::make_shared<ov::op::v8::Slice>(active_slots, zero, feature_begin_node, one,
190                                                                    feature_axis);
191            auto feature_tail = std::make_shared<ov::op::v8::Slice>(active_slots, feature_end_node, int_max, one,
192                                                                    feature_axis);
193            src = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{feature_head, src, feature_tail}, 3);
194        } else {
195            // op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature]
196            src = context.get_input(0);
197        }
198
199        if (src.get_element_type() != context.get_output_type()) {
200            src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
201        }
202
203        auto src_len =
204            std::make_shared<ov::op::v8::Gather>(std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
205                                                 ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
206        auto end = std::make_shared<ov::op::v1::Add>(begin, src_len);
207        auto head_part = std::make_shared<ov::op::v8::Slice>(base, zero, begin, one, axis);
208        auto tail_part = std::make_shared<ov::op::v8::Slice>(base, end, int_max, one, axis);
209        auto res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{head_part, src, tail_part}, slot_axis);
210        return rename_outputs_with_suffix({res}, context.get_name());
211    }
212
213    auto input = process_view_input_new(context, 0);
214
215    if (op_case == 5 || op_case == 6) {
216        auto input_shape = context.get_input_shape(0);
217        auto output_shape = context.get_output_shape();
218        auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0);
219        auto dst_stride = context.get_view_input_stride(1, 0);
220        size_t offset_bytes = context.get_view_input_offset(1, 0);
221        auto n_state = (int64_t) context.get_input_shape(0)[3].get_length();
222        auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
223        auto kv_buf = context.get_input(1);  // shape {1,1,1,N}
224
225        Output<Node> token_len_per_seq;
226        Output<Node> n_write_dyn;
227        if (context.has_input("token_len_per_seq")) {
228            token_len_per_seq = context.get_input("token_len_per_seq");
229            n_write_dyn = std::make_shared<ov::op::v1::Multiply>(token_len_per_seq, n_state_c);
230        } else {
231            n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]});
232        }
233        size_t elem_size = dst_stride[3];
234        FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size");
235        int64_t start_elem = (int64_t) (offset_bytes / elem_size);
236        // op_case 5: decoder self-attention – write offset advances each step.
237        // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time.
238        const bool is_decoder_self_attn = (op_case == 5);
239        auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{1, 1, 1});
240        auto new_shape = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{ones_c, n_write_dyn}, 0);
241
242        auto reshaped = std::make_shared<ov::op::v1::Reshape>(input, new_shape, false);
243        auto data = std::make_shared<ov::op::v0::Convert>(reshaped, context.get_output_type());
244        // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N}
245        // For decoder self-attention the write offset advances each step, so compute it
246        // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state.
247        // For encoder self-attn and cross-attn the offset is fixed at graph-compile time.
248        ov::Output<ov::Node> start;
249        if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) {
250            auto attention_size_in = context.get_input("attention_size");
251            auto token_len_in = context.get_input("token_len_per_seq");
252            auto past_tokens = std::make_shared<ov::op::v1::Subtract>(attention_size_in, token_len_in);
253            auto new_start = std::make_shared<ov::op::v1::Multiply>(past_tokens, n_state_c);
254            start = std::make_shared<ov::op::v1::Add>(
255                new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}));
256        } else {
257            start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem});
258        }
259        auto start_squeezed = std::make_shared<ov::op::v0::Squeeze>(start);
260        auto end = std::make_shared<ov::op::v1::Add>(start_squeezed, n_write_dyn);
261        auto end_squeezed = std::make_shared<ov::op::v0::Squeeze>(end);
262        auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
263        auto step_squeezed = std::make_shared<ov::op::v0::Squeeze>(step);
264        auto indices =
265            std::make_shared<ov::op::v4::Range>(start_squeezed, end_squeezed, step_squeezed, ov::element::i64);
266        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
267
268        auto kv_updated = std::make_shared<ov::op::v3::ScatterUpdate>(kv_buf, indices, data, axis);
269        return rename_outputs_with_suffix({kv_updated}, context.get_name());
270    }
271
272    if (input_shape != output_shape) {
273        auto new_shape = ov::op::v0::Constant::create(
274            ov::element::i64, {static_cast<size_t>(output_shape.rank().get_length())}, output_shape.to_shape());
275        input = std::make_shared<ov::op::v1::Reshape>(input, new_shape, false);
276    }
277
278    ov::Output<Node> res;
279    if (context.get_input_type(0) != context.get_output_type()) {
280        res = std::make_shared<ov::op::v0::Convert>(input, context.get_output_type());
281    } else {
282        res = input;
283    }
284
285    if (res.get_node_shared_ptr() == context.get_input(0).get_node_shared_ptr()) {
286        return {res};
287    }
288
289    return rename_outputs_with_suffix({res}, context.get_name());
290}
291
292}  // namespace op
293}  // namespace ggml
294}  // namespace frontend
295}  // namespace ov