#include "../node_context.h" #include "../op_table.h" #include "../utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace ov { namespace frontend { namespace ggml { namespace op { static bool is_static_one(const ov::Dimension & dim) { return dim.is_static() && dim.get_length() == 1; } static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) { return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length(); } static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) { if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 || logits_shape.rank().get_length() != 4) { return false; } return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) && same_static_dim(candidate[3], logits_shape[1]); } // Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: // 1) logits = src0 * scale // 2) logits += mask (if provided) // 3) append attention sinks as hidden logits (if provided) // 4) softmax over the last dimension and remove the hidden sink column OutputVector translate_soft_max(const NodeContext & context) { num_inputs_check(context, 1, 3); float scale = 1.0f; float max_bias = 0.0f; memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float)); memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); ov::Output logits = context.get_input(0); const bool second_input_is_sinks = context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape()); const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks; const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; // Apply scale first: logits = src0 * scale if (scale != 1.0f) { auto scale_const = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); logits = std::make_shared(logits, scale_const); } FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask), "OpenVINO softmax ALiBi path requires mask input"); // Optional mask add: logits += mask // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. if (has_mask) { ov::Output mask = context.get_input(1); // For stateful std::string mask_name = "KQ_mask_sliced"; if (context.get_input_names()[1].find("swa") != std::string::npos) { mask_name = "KQ_mask_swa_sliced"; } if (context.has_input(mask_name)) { mask = context.get_input(mask_name); } if (mask.get_element_type() != logits.get_element_type()) { mask = std::make_shared(mask, logits.get_element_type()); } if (max_bias > 0.0f) { auto out_shape = context.get_output_shape().to_shape(); FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor"); const uint32_t n_head = static_cast(out_shape[1]); FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0"); const uint32_t n_head_log2 = 1u << static_cast(std::floor(std::log2(static_cast(n_head)))); const float m0 = std::pow(2.0f, -(max_bias) / static_cast(n_head_log2)); const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast(n_head_log2)); std::vector slopes(n_head); for (uint32_t h = 0; h < n_head; ++h) { slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast(h + 1)) : std::pow(m1, static_cast(2 * (h - n_head_log2) + 1)); } ov::Output slope_node = std::make_shared(ov::element::f32, ov::Shape{n_head}, slopes); if (slope_node.get_element_type() != mask.get_element_type()) { slope_node = std::make_shared(slope_node, mask.get_element_type()); } auto slope_shape = std::make_shared( ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(n_head), 1, 1}); auto slope_4d = std::make_shared(slope_node, slope_shape, false); mask = std::make_shared(mask, slope_4d); } logits = std::make_shared(logits, mask); } ov::Output softmax_input = logits; if (has_sinks) { ov::Output sinks = context.get_input(sinks_input_idx); if (sinks.get_element_type() != logits.get_element_type()) { sinks = std::make_shared(sinks, logits.get_element_type()); } auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1}); auto sinks_4d = std::make_shared(sinks, sink_shape, false); auto logits_shape = std::make_shared(logits, ov::element::i64); auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4}); auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto sink_prefix_shape = std::make_shared(logits_shape, zero, three, one, shape_axis); auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto sink_broadcast_shape = std::make_shared( ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0); auto sink_column = std::make_shared(sinks_4d, sink_broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); softmax_input = std::make_shared(ov::OutputVector{logits, sink_column}, 3); auto softmax_with_sink = std::make_shared(softmax_input, -1); auto original_last_dim = std::make_shared(logits_shape, three, four, one, shape_axis); auto res = std::make_shared(softmax_with_sink, zero, original_last_dim, one, three); return rename_outputs_with_suffix({res}, context.get_name()); } // Softmax along last dimension (equivalent to ggml softmax over ne[0]). auto res = std::make_shared(softmax_input, -1); return rename_outputs_with_suffix({res}, context.get_name()); } } // namespace op } // namespace ggml } // namespace frontend } // namespace ov