char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun Yuggml : add OpenVINO backend (#15307)9789c4ecd

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB58 linesraw
1#include "squeeze_matmul.h"
2
3#include <openvino/core/graph_util.hpp>
4#include <openvino/core/rt_info.hpp>
5#include <openvino/op/constant.hpp>
6#include <openvino/op/matmul.hpp>
7#include <openvino/op/squeeze.hpp>
8#include <openvino/op/unsqueeze.hpp>
9#include <openvino/pass/pattern/op/label.hpp>
10#include <openvino/pass/pattern/op/pattern.hpp>
11#include <openvino/pass/pattern/op/wrap_type.hpp>
12
13namespace opp = ov::pass::pattern;
14
15namespace ov {
16namespace frontend {
17namespace ggml {
18namespace pass {
19
20// For quantized models, NPUW expects the activation to be 3d in DQ(DynamicQuantization) opt, e.g. DQMatMulGQ2i
21SqueezeMatmul::SqueezeMatmul() {
22    auto m_act = opp::any_input();
23    auto m_wei = opp::any_input();
24    auto m_matmul = opp::wrap_type<ov::op::v0::MatMul>({m_act, m_wei});
25
26    const auto callback = [=](ov::pass::pattern::Matcher & m) {
27        const auto & pattern_map = m.get_pattern_value_map();
28        auto matmul_node =
29            std::dynamic_pointer_cast<ov::op::v0::MatMul>(pattern_map.at(m_matmul).get_node_shared_ptr());
30        auto act = pattern_map.at(m_act);
31        auto wei = pattern_map.at(m_wei);
32        auto act_shape = act.get_partial_shape();
33        auto wei_shape = wei.get_partial_shape();
34        if (act_shape.rank().is_dynamic() || wei_shape.rank().is_dynamic()) {
35            return false;
36        }
37        if (act_shape.rank().get_length() == 4 && wei_shape.rank().get_length() == 2) {
38            auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0});
39            auto squeezed_act = std::make_shared<ov::op::v0::Squeeze>(act, axis);
40            auto new_matmul = std::make_shared<ov::op::v0::MatMul>(squeezed_act, wei, matmul_node->get_transpose_a(),
41                                                                   matmul_node->get_transpose_b());
42            auto unsqueezed_output = std::make_shared<ov::op::v0::Unsqueeze>(new_matmul, axis);
43            unsqueezed_output->set_friendly_name(matmul_node->get_friendly_name());
44            ov::copy_runtime_info(matmul_node, {squeezed_act, new_matmul, unsqueezed_output});
45            ov::replace_node(matmul_node, unsqueezed_output);
46            return true;
47        }
48        return false;
49    };
50
51    register_matcher(std::make_shared<ov::pass::pattern::Matcher>(m_matmul, "ov::frontend::ggml::pass::SqueezeMatmul"),
52                     callback);
53}
54
55}  // namespace pass
56}  // namespace ggml
57}  // namespace frontend
58}  // namespace ov