char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.3 KiB245 linesraw
1#include "../op_table.h"
2#include "../utils.h"
3
4#include <openvino/op/concat.hpp>
5#include <openvino/op/constant.hpp>
6#include <openvino/op/gather.hpp>
7#include <openvino/op/reshape.hpp>
8#include <openvino/op/shape_of.hpp>
9#include <openvino/op/slice.hpp>
10#include <set>
11
12namespace ov {
13namespace frontend {
14namespace ggml {
15namespace op {
16
17OutputVector translate_view(const NodeContext & context) {
18    num_inputs_check(context, 1, 1);
19
20    if (context.get_op_case() == 1) {
21        // Static-mode identity pass-through for VIEWs over a GATED_DELTA_NET combined output or
22        // the conv_input CONCAT; the consuming op (CPY/RMS_NORM) does its own runtime-correct
23        // slicing on the full tensor (see ggml-decoder.cpp compute_op_case, GGML_OP_VIEW).
24        return {context.get_input(0)};
25    }
26
27    if (!context.is_static()) {
28        // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice).
29        // EXCEPTION: the MoE expert aggregation slices each expert plane out of
30        // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then
31        // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this
32        // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here
33        // makes every plane the full tensor and the expert sum collapses. Materialize the
34        // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't
35        // affect any other view.
36        const std::string & vname = context.get_name();
37        if (vname.find("ffn_moe_weighted") != std::string::npos) {
38            auto src_ps = context.get_input_shape(0);
39            auto dst_ps = context.get_output_shape();
40            if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() &&
41                src_ps.is_static() && dst_ps.is_static()) {
42                auto sst = context.get_input_stride(0);
43                auto dst = context.get_output_stride();
44                size_t voff = context.get_output_op_offset();
45                auto ss = src_ps.to_shape();
46                auto dd = dst_ps.to_shape();
47                const size_t nd = ss.size();
48                if (sst.size() == nd && dst.size() == nd) {
49                    // Map each dst axis of size>1 to a src axis with equal (size,stride);
50                    // the unmatched src axis of size>1 is the indexed expert axis.
51                    // dst_to_src[d] records which src axis each dst axis came from, so we can
52                    // later pull the dynamic (token) dim from the right source axis at runtime.
53                    std::vector<bool> used(nd, false);
54                    std::vector<int> dst_to_src(nd, -1);
55                    bool ok = true;
56                    for (size_t d = 0; d < nd; ++d) {
57                        if (dd[d] == 1) {
58                            continue;
59                        }
60                        int found = -1;
61                        for (size_t s = 0; s < nd; ++s) {
62                            if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) {
63                                found = (int) s;
64                                break;
65                            }
66                        }
67                        if (found < 0) {
68                            ok = false;
69                            break;
70                        }
71                        used[found] = true;
72                        dst_to_src[d] = found;
73                    }
74                    int dropped = -1;
75                    if (ok) {
76                        for (size_t s = 0; s < nd; ++s) {
77                            if (!used[s] && ss[s] > 1) {
78                                if (dropped >= 0) {
79                                    ok = false;
80                                    break;
81                                }
82                                dropped = (int) s;
83                            }
84                        }
85                    }
86                    if (ok && dropped >= 0) {
87                        const size_t dstr = sst[dropped];
88                        const int64_t dsz = (int64_t) ss[dropped];
89                        if (dstr > 0 && voff % dstr == 0) {
90                            const int64_t sel = (int64_t) (voff / dstr);
91                            if (sel >= 0 && sel < dsz) {
92                                ov::Output<ov::Node> sl = std::make_shared<ov::op::v8::Slice>(
93                                    context.get_input(0),
94                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}),
95                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}),
96                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
97                                    ov::op::v0::Constant::create(ov::element::i64, {1}, {dropped}));
98                                // Build the reshape target from the (concrete) dst shape, but
99                                // keep the dynamic token axis dynamic instead of freezing it
100                                // to the captured n_tokens. Without this the constant dst
101                                // shape bakes in the prefill token count and the static value
102                                // flows downstream, turning every later decoder layer static
103                                // (the GPU in-place-concat KV-cache bug). The token axis is
104                                // PERMUTED between the sliced input and the dst (e.g. input
105                                // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero
106                                // (which copies the same-position dim) is not enough: pull the
107                                // dynamic dim from the correct SOURCE axis via ShapeOf+Gather
108                                // and place it at the dst token position.
109                                const int32_t dyn = context.get_op_dynamic_dim();  // output ggml axis, -1 if none
110                                int dst_ov_axis = (dyn != -1) ? (3 - (int) dyn) : -1;  // get_shape() reverses ggml order
111                                int src_ov_axis = (dst_ov_axis >= 0 && dst_ov_axis < (int) nd)
112                                                      ? dst_to_src[dst_ov_axis]
113                                                      : -1;
114                                if (dst_ov_axis >= 0 && src_ov_axis >= 0) {
115                                    // target = concat of per-axis scalars; the token axis is a
116                                    // runtime Gather of the slice's shape, the rest are constants.
117                                    auto sl_shape = std::make_shared<ov::op::v3::ShapeOf>(sl, ov::element::i64);
118                                    auto tok_dim = std::make_shared<ov::op::v8::Gather>(
119                                        sl_shape,
120                                        ov::op::v0::Constant::create(ov::element::i64, {1}, {src_ov_axis}),
121                                        ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
122                                    ov::OutputVector parts;
123                                    for (int a = 0; a < (int) nd; ++a) {
124                                        if (a == dst_ov_axis) {
125                                            parts.push_back(tok_dim);
126                                        } else {
127                                            parts.push_back(ov::op::v0::Constant::create(
128                                                ov::element::i64, {1}, {(int64_t) dd[a]}));
129                                        }
130                                    }
131                                    auto dc = std::make_shared<ov::op::v0::Concat>(parts, 0);
132                                    auto rs = std::make_shared<ov::op::v1::Reshape>(sl, dc, false);
133                                    return rename_outputs_with_suffix({rs}, context.get_name());
134                                }
135                                auto dc = ov::op::v0::Constant::create(
136                                    ov::element::i64, {nd}, std::vector<int64_t>(dd.begin(), dd.end()));
137                                auto rs = std::make_shared<ov::op::v1::Reshape>(sl, dc, false);
138                                return rename_outputs_with_suffix({rs}, context.get_name());
139                            }
140                        }
141                    }
142                }
143            }
144        }
145        return {context.get_input(0)};
146    }
147
148    auto input = context.get_input(0);
149    auto src_shape = context.get_input_shape(0);
150    auto dst_shape = context.get_output_shape();
151
152    if (src_shape.rank().is_dynamic() || dst_shape.rank().is_dynamic()) {
153        return {input};
154    }
155
156    int64_t src_elems = 1, dst_elems = 1;
157    for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) {
158        if (src_shape[i].is_dynamic()) {
159            return {input};
160        }
161        src_elems *= src_shape[i].get_length();
162    }
163    for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) {
164        if (dst_shape[i].is_dynamic()) {
165            return {input};
166        }
167        dst_elems *= dst_shape[i].get_length();
168    }
169
170    if (dst_elems >= src_elems) {
171        return {input};
172    }
173
174    auto src_stride = context.get_input_stride(0);
175    auto dst_stride = context.get_output_stride();
176    size_t view_offset = context.get_output_op_offset();
177
178    bool same_stride = (src_stride.size() == dst_stride.size());
179    if (same_stride) {
180        for (size_t i = 0; i < src_stride.size(); ++i) {
181            if (src_stride[i] != dst_stride[i]) {
182                same_stride = false;
183                break;
184            }
185        }
186    }
187
188    if (!same_stride) {
189        return {input};
190    }
191
192    auto src_ov_shape = src_shape.to_shape();
193    auto dst_ov_shape = dst_shape.to_shape();
194    size_t ndims = src_ov_shape.size();
195    if (dst_ov_shape.size() != ndims) {
196        return {input};
197    }
198
199    std::vector<int> diff_dims;
200    for (size_t i = 0; i < ndims; ++i) {
201        if (src_ov_shape[i] != dst_ov_shape[i]) {
202            diff_dims.push_back(static_cast<int>(i));
203        }
204    }
205
206    if (diff_dims.size() != 1) {
207        return {input};
208    }
209
210    int slice_dim = diff_dims[0];
211    int64_t dim_size = static_cast<int64_t>(src_ov_shape[slice_dim]);
212
213    size_t ov_stride_for_dim = 1;
214    for (size_t i = slice_dim + 1; i < ndims; ++i) {
215        ov_stride_for_dim *= src_ov_shape[i];
216    }
217    size_t elem_size = src_stride.back();
218    if (elem_size == 0) {
219        elem_size = 1;
220    }
221
222    int64_t begin_val = 0;
223    if (ov_stride_for_dim > 0 && elem_size > 0) {
224        begin_val = static_cast<int64_t>((view_offset / elem_size) / ov_stride_for_dim);
225    }
226    int64_t end_val = begin_val + static_cast<int64_t>(dst_ov_shape[slice_dim]);
227
228    if (begin_val < 0 || end_val > dim_size) {
229        return {input};
230    }
231
232    auto sliced =
233        std::make_shared<ov::op::v8::Slice>(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}),
234                                            ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}),
235                                            ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
236                                            ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}));
237
238    sliced->set_friendly_name(context.get_output_name());
239    return {sliced->output(0)};
240}
241
242}  // namespace op
243}  // namespace ggml
244}  // namespace frontend
245}  // namespace ov