char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Gaurav GargRemove redundant CUDA copies after gated_delta_net. (#23940)5a460dea9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
689 B14 linesraw
1#include "common.cuh"
2#include "ggml.h"
3
4// fused-kernel recurrent-state output; strides in elements (per-seq stride is always D, set in-kernel)
5struct ggml_cuda_gated_delta_net_fused_cache {
6    float * data;        // rollback slot 0
7    int64_t slot_stride; // between rollback slots (0 when K==1)
8};
9
10void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
11
12// same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion)
13void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
14                                              ggml_cuda_gated_delta_net_fused_cache cache);