char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

fairydreamingggml : process data in smaller chunks in CUDA ggml_top_k() and ggml_argsort() to reduce temporary buffers memory usage (#24776)074944998

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
950 B20 linesraw
1#include "common.cuh"
2
3void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
4
5#ifdef GGML_CUDA_USE_CUB
6int argsort_f32_i32_cuda_cub_chunk_nrows(const size_t nb01, const int64_t nrows);
7void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool,
8                              const float *    x,
9                              int *            dst,
10                              const int        ncols,
11                              const int        nrows,
12                              ggml_sort_order  order,
13                              cudaStream_t     stream);
14#endif  // GGML_CUDA_USE_CUB
15void argsort_f32_i32_cuda_bitonic(const float *   x,
16                                  int *           dst,
17                                  const int       ncols,
18                                  const int       nrows,
19                                  ggml_sort_order order,
20                                  cudaStream_t    stream);