char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Qwen3.5, memory optimization, and test-recurrent-state-rollback (#26952)aee56b3ab

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.5 KiB86 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <openvino/core/except.hpp>
6#include <openvino/op/add.hpp>
7#include <openvino/op/concat.hpp>
8#include <openvino/op/constant.hpp>
9#include <openvino/op/convert.hpp>
10#include <openvino/op/equal.hpp>
11#include <openvino/op/gather.hpp>
12#include <openvino/op/greater_eq.hpp>
13#include <openvino/op/if.hpp>
14#include <openvino/op/less.hpp>
15#include <openvino/op/logical_or.hpp>
16#include <openvino/op/multiply.hpp>
17#include <openvino/op/range.hpp>
18#include <openvino/op/reshape.hpp>
19#include <openvino/op/shape_of.hpp>
20#include <openvino/op/slice.hpp>
21#include <openvino/op/squeeze.hpp>
22#include <openvino/op/unsqueeze.hpp>
23#include <vector>
24
25namespace ov {
26namespace frontend {
27namespace ggml {
28namespace op {
29
30OutputVector translate_scale(const NodeContext & context) {
31    num_inputs_check(context, 1, 1);
32
33    float scale;
34    float bias;
35    memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float));
36    memcpy(&bias, (float *) context.get_output_op_params() + 1, sizeof(float));
37
38    auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
39
40    if (context.get_op_case() == 1 && context.has_input("cache_rs_reset_len")) {
41        auto cache_rs_reset_idx = context.get_input("cache_rs_reset_idx");
42        auto cache_rs_reset_len = context.get_input("cache_rs_reset_len");
43
44        auto cache_rs = context.get_input(0);
45
46        auto cache_shape = std::make_shared<ov::op::v3::ShapeOf>(cache_rs, ov::element::i64);
47        auto n_slots_1d = std::make_shared<ov::op::v8::Gather>(
48            cache_shape, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}),
49            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}));
50        auto n_slots = std::make_shared<ov::op::v0::Squeeze>(n_slots_1d);
51
52        auto iota = std::make_shared<ov::op::v4::Range>(
53            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}), n_slots,
54            ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}), ov::element::i64);
55
56        auto idx_plus_len = std::make_shared<ov::op::v1::Add>(cache_rs_reset_idx, cache_rs_reset_len);
57        auto less_than_idx = std::make_shared<ov::op::v1::Less>(iota, cache_rs_reset_idx);
58        auto greater_equal_idx_plus_len = std::make_shared<ov::op::v1::GreaterEqual>(iota, idx_plus_len);
59        auto keep_mask = std::make_shared<ov::op::v1::LogicalOr>(less_than_idx, greater_equal_idx_plus_len);
60
61        auto keep_mask_f32 = std::make_shared<ov::op::v0::Convert>(keep_mask, ov::element::f32);
62        auto keep_mask_reshape = std::make_shared<ov::op::v0::Unsqueeze>(
63            keep_mask_f32, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1}));
64
65        auto cleared_cache_rs = std::make_shared<ov::op::v1::Multiply>(cache_rs, keep_mask_reshape);
66        return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name());
67    }
68
69    auto scaled = std::make_shared<ov::op::v1::Multiply>(context.get_input(0), scale_node);
70
71    std::shared_ptr<ov::Node> res;
72    if (bias != 0.0f) {
73        auto bias_node =
74            std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{bias});
75        res = std::make_shared<ov::op::v1::Add>(scaled, bias_node);
76    } else {
77        res = scaled;
78    }
79
80    return rename_outputs_with_suffix({res}, context.get_name());
81}
82
83}  // namespace op
84}  // namespace ggml
85}  // namespace frontend
86}  // namespace ov