char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aman Guptaggml: add SWIGLU_CLAMP (#27930)0190529ec

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.6 KiB110 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <cstdint>
6#include <limits>
7#include <memory>
8#include <openvino/core/node_output.hpp>
9#include <openvino/op/add.hpp>
10#include <openvino/op/clamp.hpp>
11#include <openvino/op/constant.hpp>
12#include <openvino/op/multiply.hpp>
13#include <openvino/op/sigmoid.hpp>
14#include <openvino/op/slice.hpp>
15
16namespace ov {
17namespace frontend {
18namespace ggml {
19namespace op {
20
21static std::pair<ov::Output<ov::Node>, ov::Output<ov::Node>> get_glu_inputs(const NodeContext & context) {
22    num_inputs_check(context, 1, 2);
23
24    ov::Output<ov::Node> src0;
25    ov::Output<ov::Node> src1;
26    if (context.get_input_size() == 2) {
27        // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts):
28        // resolve them so each half has its real sliced shape, not the base tensor.
29        src0 = process_view_input_new(context, 0);
30        src1 = process_view_input_new(context, 1);
31    } else {
32        // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2.
33        // Both halves are nc elements; if the dimension is odd, the last element is dropped.
34        // Use Slice instead of Split to handle odd dimensions correctly.
35        // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first.
36        auto combined = process_view_input_new(context, 0);
37        auto combined_shape = combined.get_partial_shape();
38        int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();
39        int64_t nc = last_dim_val / 2;
40
41        auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
42        auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
43        auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
44        auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
45        auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
46        auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
47
48        src0 = std::make_shared<ov::op::v8::Slice>(combined, start0, stop0, step, axis);
49        src1 = std::make_shared<ov::op::v8::Slice>(combined, start1, stop1, step, axis);
50    }
51
52    int32_t * params = context.get_output_op_params();
53    const int32_t swapped = params[1];
54    if (swapped) {
55        std::swap(src0, src1);
56    }
57
58    return {src0, src1};
59}
60
61OutputVector translate_glu_swiglu(const NodeContext & context) {
62    auto [src0, src1] = get_glu_inputs(context);
63
64    auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(src0);
65    auto silu = std::make_shared<ov::op::v1::Multiply>(src0, sigmoid);
66    auto res = std::make_shared<ov::op::v1::Multiply>(silu, src1);
67
68    return rename_outputs_with_suffix({res}, context.get_name());
69}
70
71OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
72    auto [src0, src1] = get_glu_inputs(context);
73
74    const int32_t * params = context.get_output_op_params();
75    const float alpha = reinterpret_cast<const float *>(params)[2];
76    const float limit = reinterpret_cast<const float *>(params)[3];
77
78    auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
79    auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha});
80    auto scaled_gate = std::make_shared<ov::op::v1::Multiply>(gate, alpha_const);
81    auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(scaled_gate);
82    auto out_glu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid);
83
84    auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
85    auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f});
86    auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one);
87    auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
88
89    return rename_outputs_with_suffix({res}, context.get_name());
90}
91
92OutputVector translate_glu_swiglu_clamp(const NodeContext & context) {
93    auto [src0, src1] = get_glu_inputs(context);
94
95    const int32_t * params = context.get_output_op_params();
96    const float limit = reinterpret_cast<const float *>(params)[3];
97
98    auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
99    auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(gate);
100    auto silu = std::make_shared<ov::op::v1::Multiply>(gate, sigmoid);
101    auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
102    auto res = std::make_shared<ov::op::v1::Multiply>(silu, up);
103
104    return rename_outputs_with_suffix({res}, context.get_name());
105}
106
107}  // namespace op
108}  // namespace ggml
109}  // namespace frontend
110}  // namespace ov