char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

ynankaniCUDA: extend MOE fusion to specdec, earlier MOE glu fusion and topk-router fusion were restricted to 1 token (#27621)41ef91f7c

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.1 KiB31 linesraw
1#include "common.cuh"
2#include "ggml.h"
3
4#include <initializer_list>
5
6// Rows that one CUDA block handles.
7#define TOPK_MOE_ROWS_PER_BLOCK 8
8
9struct ggml_cuda_topk_moe_args {
10    bool sigmoid{};
11    bool sqrt_softplus{};
12    bool softmax{};
13    bool delayed_softmax{};
14    bool prob_bias{};
15    bool norm{};
16    bool scale{};
17};
18
19void ggml_cuda_op_topk_moe(ggml_backend_cuda_context &     ctx,
20                           const ggml_tensor *             logits,
21                           ggml_tensor *                   weights,
22                           ggml_tensor *                   ids,
23                           const ggml_tensor *             clamp,
24                           const ggml_tensor *             scale,
25                           const ggml_tensor *             bias,
26                           const ggml_cuda_topk_moe_args & args);
27
28bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op,
29                                   const ggml_tensor * weights,
30                                   const ggml_tensor * logits,
31                                   const ggml_tensor * ids);