char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun Yuopenvino: OV 2026.2, context-shift, Q5_1 support, gemma4 dense/embedding, and -fa off (#24503)890f1a27e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.7 KiB100 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <climits>
6#include <cstdint>
7#include <memory>
8#include <openvino/core/node.hpp>
9#include <openvino/core/node_output.hpp>
10#include <openvino/op/broadcast.hpp>
11#include <openvino/op/concat.hpp>
12#include <openvino/op/constant.hpp>
13#include <openvino/op/convert.hpp>
14#include <openvino/op/matmul.hpp>
15#include <openvino/op/reshape.hpp>
16#include <openvino/op/slice.hpp>
17#include <openvino/op/transpose.hpp>
18#include <openvino/op/unsqueeze.hpp>
19#include <openvino/op/util/op_types.hpp>
20#include <vector>
21
22namespace ov {
23namespace frontend {
24namespace ggml {
25namespace op {
26
27OutputVector translate_mulmat(const NodeContext & context) {
28    num_inputs_check(context, 2, 2);
29
30    int op_case = context.get_op_case();
31
32    ov::Output<Node> res;
33    ov::Output<ov::Node> B;
34    ov::Output<ov::Node> A;
35    if (op_case == 3) {
36        B = process_view_input(context, 0);
37        A = process_view_input(context, 1);
38    } else {
39        B = process_view_input_new(context, 0);
40        A = process_view_input_new(context, 1);
41    }
42
43    if (A.get_element_type() != B.get_element_type()) {
44        B = std::make_shared<ov::op::v0::Convert>(context.get_input(0), context.get_input_type(1));
45    }
46
47    auto B_shape = context.get_input_shape(0).to_shape();
48    auto A_shape = context.get_input_shape(1).to_shape();
49    int64_t A_batch = A_shape[1];
50    int64_t B_batch = B_shape[1];
51
52    auto A_batch_larger = A_batch > B_batch;
53    auto batch_large = A_batch_larger ? A_batch : B_batch;
54    auto batch_small = A_batch_larger ? B_batch : A_batch;
55
56    Output<Node> Z = A_batch_larger ? B : A;
57    auto Z_shape = A_batch_larger ? B_shape : A_shape;
58    int64_t factor = batch_large / batch_small;
59    if (factor > 1 && batch_small > 1) {
60        auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{batch_large});
61        auto batch_small_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{batch_small});
62        auto factor_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{factor});
63
64        auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});
65        auto Z_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(Z, unsqueeze_axes);
66
67        auto broadcast_shape = ov::op::v0::Constant::create(
68            ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});
69        auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
70                                                        {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]});
71        if (op_case == 2) {
72            new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
73                                                       {(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1});
74        }
75
76        auto Z_broadcasted = std::make_shared<ov::op::v3::Broadcast>(Z_unsqueezed, broadcast_shape,
77                                                                     ov::op::BroadcastType::BIDIRECTIONAL);
78        Z = std::make_shared<ov::op::v1::Reshape>(Z_broadcasted, new_Z_shape, true);
79    }
80    if (A_batch_larger) {
81        B = Z;
82    } else {
83        A = Z;
84    }
85
86    bool transpose_b = true;
87    res = std::make_shared<ov::op::v0::MatMul>(A, B, false, transpose_b);
88
89    const auto output_type = context.get_output_type();
90    if (res.get_element_type() != output_type) {
91        res = std::make_shared<ov::op::v0::Convert>(res, output_type);
92    }
93
94    return rename_outputs_with_suffix({res}, context.get_name());
95}
96
97}  // namespace op
98}  // namespace ggml
99}  // namespace frontend
100}  // namespace ov