char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun Yuopenvino: OV 2026.2, context-shift, Q5_1 support, gemma4 dense/embedding, and -fa off (#24503)890f1a27e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.6 KiB147 linesraw
1#include "../node_context.h"
2#include "../op_table.h"
3#include "../utils.h"
4
5#include <climits>
6#include <cstdint>
7#include <memory>
8#include <openvino/core/node.hpp>
9#include <openvino/op/add.hpp>
10#include <openvino/op/concat.hpp>
11#include <openvino/op/constant.hpp>
12#include <openvino/op/reshape.hpp>
13#include <openvino/op/slice.hpp>
14#include <openvino/op/transpose.hpp>
15#include <vector>
16
17namespace ov {
18namespace frontend {
19namespace ggml {
20namespace op {
21
22OutputVector translate_permute(const NodeContext & context) {
23    num_inputs_check(context, 1, 1);
24
25    int op_case = context.get_op_case();
26    FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case");
27    // op_case 1 is trivial permute
28    // op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension
29    // op_case 3 4 it to permute KV cache in the default layout
30    // op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true
31
32    ov::Output<Node> res;
33    ov::Output<Node> src;
34    if (op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6) {
35        src = context.get_input(0);
36    } else {
37        src = process_view_input_new(context, 0);
38    }
39    std::vector<int64_t> perm_values{0, 2, 1, 3};
40    const int32_t * op_params = context.get_output_op_params();
41    if (op_params != nullptr) {
42        for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) {
43            const size_t output_axis = static_cast<size_t>(op_params[input_axis]);
44            perm_values[perm_values.size() - 1 - output_axis] =
45                static_cast<int64_t>(perm_values.size() - 1 - input_axis);
46        }
47    }
48    auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
49
50    if (op_case == 1 || context.is_stateful()) {
51        res = std::make_shared<ov::op::v1::Transpose>(src, perm);
52    } else if (op_case == 2) {
53        auto output_shape = context.get_output_shape().to_shape();
54        auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]});
55        auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});
56        auto n_seq_active = context.has_input("n_seq_active") ?
57                                context.get_input("n_seq_active") :
58                                ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[0]});
59        auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
60
61        auto new_shape =
62            std::make_shared<ov::op::v0::Concat>(ov::OutputVector{n_seq_active, neg_one, n_heads, head_size}, 0);
63
64        // // Alternative
65        // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
66        // auto new_shape = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{n_seq_active, neg_one, zero, zero}, 0);
67
68        auto reshaped = std::make_shared<ov::op::v1::Reshape>(src, new_shape, true);
69        res = std::make_shared<ov::op::v1::Transpose>(reshaped, perm);
70    } else {
71        auto cache_shape = src.get_partial_shape();
72        auto output_shape = context.get_output_shape().to_shape();
73        int64_t head_size = output_shape[3];
74        int64_t n_heads = output_shape[1];
75        if (op_case == 5 || op_case == 6) {
76            head_size = output_shape[2];
77            n_heads = output_shape[1];
78        }
79        int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1;
80        int64_t n_seq = cache_shape[1].get_length();
81
82        Output<Node> attention_size;
83        if (!context.has_input("attention_size")) {
84            attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]});
85        } else if (op_case == 3 || op_case == 5) {
86            attention_size = context.get_input("attention_size");
87        } else {
88            attention_size = context.get_input("attention_size_swa");
89        }
90
91        Output<Node> seq_active_start;
92        Output<Node> seq_active_end;
93        if (context.has_input("seq_active_start")) {
94            seq_active_start = context.get_input("seq_active_start");
95            seq_active_end = context.get_input("seq_active_end");
96        } else {
97            int64_t n_seq_active = output_shape[0];
98            size_t offset = *((size_t *) context.get_input_op_params(0));
99            int64_t seq_active_start_val = offset / context.get_input_stride(0)[0];
100            int64_t seq_active_end_val = seq_active_start_val + n_seq_active;
101            seq_active_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_start_val});
102            seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val});
103        }
104
105        // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq])
106        // 2. slice out the active sequences
107        // 3. slice out the attention part in each sequence
108        // 4. permute (skip for `-fa off`)
109        auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
110        auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
111
112        if (op_case == 3 || op_case == 4) {
113            auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
114                src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}),
115                false);
116            ov::Output<ov::Node> after_seq_slice;
117            if (n_seq == 1) {
118                after_seq_slice = src_reshaped;
119            } else {
120                after_seq_slice =
121                    std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
122            }
123            auto slice2 = std::make_shared<ov::op::v8::Slice>(after_seq_slice, zero, attention_size, one, one);
124            res = std::make_shared<ov::op::v1::Transpose>(slice2, perm);
125        } else {
126            auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
127            auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
128                src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}),
129                false);
130            ov::Output<ov::Node> after_seq_slice;
131            if (n_seq == 1) {
132                after_seq_slice = src_reshaped;
133            } else {
134                after_seq_slice =
135                    std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
136            }
137            auto slice2 = std::make_shared<ov::op::v8::Slice>(after_seq_slice, zero, attention_size, one, three);
138            res = slice2;
139        }
140    }
141    return rename_outputs_with_suffix({res}, context.get_name());
142}
143
144}  // namespace op
145}  // namespace ggml
146}  // namespace frontend
147}  // namespace ov