#include "../node_context.h" #include "../op_table.h" #include "../utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace ov { namespace frontend { namespace ggml { namespace op { OutputVector translate_cpy(const NodeContext & context) { auto op_case = context.get_op_case(); auto input_shape = context.get_input_shape(0); auto output_shape = context.get_input_shape(1); if (op_case == 4) { auto src = process_view_input_new(context, 0); auto base = context.get_input(1); int64_t n_elems = 1; for (const auto & dim : context.get_output_shape().to_shape()) { n_elems *= static_cast(dim); } const auto output_stride = context.get_output_stride(); const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state view update has invalid element size"); const int64_t begin_val = static_cast(context.get_output_op_offset() / elem_size); const int64_t end_val = begin_val + n_elems; auto flat_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, -1}); src = std::make_shared(src, flat_shape, false); if (src.get_element_type() != context.get_output_type()) { src = std::make_shared(src, context.get_output_type()); } auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}); auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}); auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); auto head_part = std::make_shared(base, zero, begin, one, axis); auto tail_part = std::make_shared(base, end, int_max, one, axis); auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, 3); return rename_outputs_with_suffix({res}, context.get_name()); } // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a // runtime input, so the cached model works for any kv head and active sequence count. The // result is the full updated cache. // op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder. if (op_case == 3) { // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size // tensors, so short-circuit instead of building a degenerate Slice/Concat chain. bool is_empty = false; if (input_shape.rank().is_static()) { for (const auto & d : input_shape) { if (d.is_static() && d.get_length() == 0) { is_empty = true; break; } } } if (is_empty) { return {context.get_input(1)}; } } const std::string slot_begin_name = "rs_slot_begin_" + context.get_name(); const bool slice_assign = context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3); if (slice_assign) { const int64_t slot_axis = 2; auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {slot_axis}); auto feature = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, -1, output_shape[3].get_length()}); ov::Output src; ov::Output begin = context.get_input(slot_begin_name); auto base = context.get_input(1); if (op_case == 1) { ov::Output state_begin; const std::string src_begin_name = "rs_src_begin_" + context.get_name(); if (context.has_input(src_begin_name)) { state_begin = context.get_input(src_begin_name); } else { auto ssm_state_size = context.get_ssm_state_size(); if (context.has_input("s_copy_active_slot_len")) { auto len = context.get_input("s_copy_active_slot_len"); auto state_rows = std::make_shared( ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len); state_begin = std::make_shared(state_rows); } else { state_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size}); } } auto state_part = std::make_shared(context.get_input(0), state_begin, int_max, one, axis); src = std::make_shared(state_part, feature, false); } else if (op_case == 2) { // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1 // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output // shape static even though the window start is a runtime value. auto window_size = (int64_t) input_shape[3].get_length(); ov::Output window; auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); const std::string src_begin_name = "rs_src_begin_" + context.get_name(); if (context.has_input(src_begin_name)) { auto src_begin = context.get_input(src_begin_name); auto src_end = std::make_shared( src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size})); window = std::make_shared(context.get_input(0), src_begin, src_end, one, col_axis); } else if (context.has_input("chunk_valid_len")) { std::vector offsets(window_size); std::iota(offsets.begin(), offsets.end(), 0); auto indices = std::make_shared( ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets), context.get_input("chunk_valid_len")); window = std::make_shared(context.get_input(0), indices, col_axis); } else { auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size}); window = std::make_shared(context.get_input(0), window_begin, int_max, one, col_axis); } const auto base_shape = base.get_partial_shape(); FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4, "CPY conv state cache update requires rank-4 base cache"); FRONT_END_OP_CONVERSION_CHECK(base_shape[3].is_static(), "CPY conv state cache update requires static feature size"); FRONT_END_OP_CONVERSION_CHECK(input_shape.rank().is_static() && input_shape.rank().get_length() == 4 && input_shape[2].is_static() && input_shape[3].is_static(), "CPY conv state cache update requires static source feature view"); const int64_t full_feature_size = base_shape[3].get_length(); const int64_t update_feature_size = input_shape[2].get_length() * input_shape[3].get_length(); const auto output_stride = context.get_output_stride(); const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state cache update has invalid element size"); const int64_t feature_begin = static_cast(context.get_output_op_offset() / elem_size) % full_feature_size; const int64_t feature_end = feature_begin + update_feature_size; FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size, "CPY conv state cache update feature range is out of bounds"); auto partial_feature = ov::op::v0::Constant::create( ov::element::i64, {4}, std::vector{1, 1, -1, update_feature_size}); src = std::make_shared(window, partial_feature, false); if (src.get_element_type() != context.get_output_type()) { src = std::make_shared(src, context.get_output_type()); } auto src_len = std::make_shared( std::make_shared(src, ov::element::i64), axis, ov::op::v0::Constant::create(ov::element::i64, {}, {0})); auto slot_end = std::make_shared(begin, src_len); auto active_slots = std::make_shared(base, begin, slot_end, one, axis); auto feature_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); auto feature_begin_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_begin}); auto feature_end_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_end}); auto feature_head = std::make_shared(active_slots, zero, feature_begin_node, one, feature_axis); auto feature_tail = std::make_shared(active_slots, feature_end_node, int_max, one, feature_axis); src = std::make_shared(ov::OutputVector{feature_head, src, feature_tail}, 3); } else { // op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature] src = context.get_input(0); } if (src.get_element_type() != context.get_output_type()) { src = std::make_shared(src, context.get_output_type()); } auto src_len = std::make_shared(std::make_shared(src, ov::element::i64), axis, ov::op::v0::Constant::create(ov::element::i64, {}, {0})); auto end = std::make_shared(begin, src_len); auto head_part = std::make_shared(base, zero, begin, one, axis); auto tail_part = std::make_shared(base, end, int_max, one, axis); auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, slot_axis); return rename_outputs_with_suffix({res}, context.get_name()); } auto input = process_view_input_new(context, 0); if (op_case == 5 || op_case == 6) { auto input_shape = context.get_input_shape(0); auto output_shape = context.get_output_shape(); auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0); auto dst_stride = context.get_view_input_stride(1, 0); size_t offset_bytes = context.get_view_input_offset(1, 0); auto n_state = (int64_t) context.get_input_shape(0)[3].get_length(); auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state}); auto kv_buf = context.get_input(1); // shape {1,1,1,N} Output token_len_per_seq; Output n_write_dyn; if (context.has_input("token_len_per_seq")) { token_len_per_seq = context.get_input("token_len_per_seq"); n_write_dyn = std::make_shared(token_len_per_seq, n_state_c); } else { n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]}); } size_t elem_size = dst_stride[3]; FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size"); int64_t start_elem = (int64_t) (offset_bytes / elem_size); // op_case 5: decoder self-attention – write offset advances each step. // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time. const bool is_decoder_self_attn = (op_case == 5); auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{1, 1, 1}); auto new_shape = std::make_shared(ov::OutputVector{ones_c, n_write_dyn}, 0); auto reshaped = std::make_shared(input, new_shape, false); auto data = std::make_shared(reshaped, context.get_output_type()); // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N} // For decoder self-attention the write offset advances each step, so compute it // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state. // For encoder self-attn and cross-attn the offset is fixed at graph-compile time. ov::Output start; if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) { auto attention_size_in = context.get_input("attention_size"); auto token_len_in = context.get_input("token_len_per_seq"); auto past_tokens = std::make_shared(attention_size_in, token_len_in); auto new_start = std::make_shared(past_tokens, n_state_c); start = std::make_shared( new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem})); } else { start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}); } auto start_squeezed = std::make_shared(start); auto end = std::make_shared(start_squeezed, n_write_dyn); auto end_squeezed = std::make_shared(end); auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto step_squeezed = std::make_shared(step); auto indices = std::make_shared(start_squeezed, end_squeezed, step_squeezed, ov::element::i64); auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); auto kv_updated = std::make_shared(kv_buf, indices, data, axis); return rename_outputs_with_suffix({kv_updated}, context.get_name()); } if (input_shape != output_shape) { auto new_shape = ov::op::v0::Constant::create( ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape()); input = std::make_shared(input, new_shape, false); } ov::Output res; if (context.get_input_type(0) != context.get_output_type()) { res = std::make_shared(input, context.get_output_type()); } else { res = input; } if (res.get_node_shared_ptr() == context.get_input(0).get_node_shared_ptr()) { return {res}; } return rename_outputs_with_suffix({res}, context.get_name()); } } // namespace op } // namespace ggml } // namespace frontend } // namespace ov