char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Ravi Panchumarthyopenvino: Update to OV 2026.2.1, self-contained release packages, operator improvements (#24974)5397c3619

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.5 KiB166 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <cmath>
6#include <cstdint>
7#include <cstring>
8#include <memory>
9#include <openvino/op/broadcast.hpp>
10#include <openvino/frontend/exception.hpp>
11#include <openvino/op/add.hpp>
12#include <openvino/op/concat.hpp>
13#include <openvino/op/constant.hpp>
14#include <openvino/op/convert.hpp>
15#include <openvino/op/multiply.hpp>
16#include <openvino/op/reshape.hpp>
17#include <openvino/op/shape_of.hpp>
18#include <openvino/op/slice.hpp>
19#include <openvino/op/softmax.hpp>
20#include <vector>
21
22namespace ov {
23namespace frontend {
24namespace ggml {
25namespace op {
26
27static bool is_static_one(const ov::Dimension & dim) {
28    return dim.is_static() && dim.get_length() == 1;
29}
30
31static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) {
32    return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length();
33}
34
35static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) {
36    if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 ||
37        logits_shape.rank().get_length() != 4) {
38        return false;
39    }
40
41    return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) &&
42           same_static_dim(candidate[3], logits_shape[1]);
43}
44
45// Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend:
46// 1) logits = src0 * scale
47// 2) logits += mask (if provided)
48// 3) append attention sinks as hidden logits (if provided)
49// 4) softmax over the last dimension and remove the hidden sink column
50OutputVector translate_soft_max(const NodeContext & context) {
51    num_inputs_check(context, 1, 3);
52
53    float scale = 1.0f;
54    float max_bias = 0.0f;
55    memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float));
56    memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float));
57
58    ov::Output<ov::Node> logits = context.get_input(0);
59    const bool second_input_is_sinks =
60        context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape());
61    const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks;
62    const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2;
63    const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2;
64
65    // Apply scale first: logits = src0 * scale
66    if (scale != 1.0f) {
67        auto scale_const =
68            std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
69        logits = std::make_shared<ov::op::v1::Multiply>(logits, scale_const);
70    }
71
72    FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask),
73                                "OpenVINO softmax ALiBi path requires mask input");
74
75    // Optional mask add: logits += mask
76    // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding.
77    if (has_mask) {
78        ov::Output<ov::Node> mask = context.get_input(1);
79
80        // For stateful
81        std::string mask_name = "KQ_mask_sliced";
82        if (context.get_input_names()[1].find("swa") != std::string::npos) {
83            mask_name = "KQ_mask_swa_sliced";
84        }
85        if (context.has_input(mask_name)) {
86            mask = context.get_input(mask_name);
87        }
88
89        if (mask.get_element_type() != logits.get_element_type()) {
90            mask = std::make_shared<ov::op::v0::Convert>(mask, logits.get_element_type());
91        }
92
93        if (max_bias > 0.0f) {
94            auto out_shape = context.get_output_shape().to_shape();
95            FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor");
96
97            const uint32_t n_head = static_cast<uint32_t>(out_shape[1]);
98            FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0");
99
100            const uint32_t n_head_log2 = 1u << static_cast<uint32_t>(std::floor(std::log2(static_cast<float>(n_head))));
101            const float m0 = std::pow(2.0f, -(max_bias) / static_cast<float>(n_head_log2));
102            const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast<float>(n_head_log2));
103
104            std::vector<float> slopes(n_head);
105            for (uint32_t h = 0; h < n_head; ++h) {
106                slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast<float>(h + 1)) :
107                                              std::pow(m1, static_cast<float>(2 * (h - n_head_log2) + 1));
108            }
109
110            ov::Output<ov::Node> slope_node =
111                std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{n_head}, slopes);
112            if (slope_node.get_element_type() != mask.get_element_type()) {
113                slope_node = std::make_shared<ov::op::v0::Convert>(slope_node, mask.get_element_type());
114            }
115
116            auto slope_shape = std::make_shared<ov::op::v0::Constant>(
117                ov::element::i64, ov::Shape{4}, std::vector<int64_t>{1, static_cast<int64_t>(n_head), 1, 1});
118            auto slope_4d = std::make_shared<ov::op::v1::Reshape>(slope_node, slope_shape, false);
119            mask = std::make_shared<ov::op::v1::Multiply>(mask, slope_4d);
120        }
121
122        logits = std::make_shared<ov::op::v1::Add>(logits, mask);
123    }
124
125    ov::Output<ov::Node> softmax_input = logits;
126    if (has_sinks) {
127        ov::Output<ov::Node> sinks = context.get_input(sinks_input_idx);
128        if (sinks.get_element_type() != logits.get_element_type()) {
129            sinks = std::make_shared<ov::op::v0::Convert>(sinks, logits.get_element_type());
130        }
131
132        auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1});
133        auto sinks_4d = std::make_shared<ov::op::v1::Reshape>(sinks, sink_shape, false);
134
135        auto logits_shape = std::make_shared<ov::op::v3::ShapeOf>(logits, ov::element::i64);
136        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
137        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
138        auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
139        auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4});
140        auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
141
142        auto sink_prefix_shape = std::make_shared<ov::op::v8::Slice>(logits_shape, zero, three, one, shape_axis);
143        auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
144        auto sink_broadcast_shape = std::make_shared<ov::op::v0::Concat>(
145            ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0);
146        auto sink_column = std::make_shared<ov::op::v3::Broadcast>(sinks_4d, sink_broadcast_shape,
147                                                                   ov::op::BroadcastType::BIDIRECTIONAL);
148        softmax_input = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{logits, sink_column}, 3);
149
150        auto softmax_with_sink = std::make_shared<ov::op::v8::Softmax>(softmax_input, -1);
151        auto original_last_dim = std::make_shared<ov::op::v8::Slice>(logits_shape, three, four, one, shape_axis);
152        auto res = std::make_shared<ov::op::v8::Slice>(softmax_with_sink, zero, original_last_dim, one, three);
153
154        return rename_outputs_with_suffix({res}, context.get_name());
155    }
156
157    // Softmax along last dimension (equivalent to ggml softmax over ne[0]).
158    auto res = std::make_shared<ov::op::v8::Softmax>(softmax_input, -1);
159
160    return rename_outputs_with_suffix({res}, context.get_name());
161}
162
163}  // namespace op
164}  // namespace ggml
165}  // namespace frontend
166}  // namespace ov