char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Andreas KieslingerProgrammatic Dependent Launch (PDL) for more performance on newer NVIDIA GPUs (Hopper+) (#22522)e94722822

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.5 KiB37 linesraw
1#include "scale.cuh"
2
3#define MAX_GRIDDIM_X 0x7FFFFFFF
4
5static __global__ void scale_f32(const float * x, float * dst, const float scale, const float bias, const int64_t nelements) {
6    ggml_cuda_pdl_lc();
7    int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;
8    int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;
9
10    ggml_cuda_pdl_sync();
11    for (int64_t i = tid; i < nelements; i += stride) {
12        dst[i] = scale * x[i] + bias;
13    }
14}
15
16static void scale_f32_cuda(const float * x, float * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {
17    const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
18    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream);
19    ggml_cuda_kernel_launch(scale_f32, launch_params, x, dst, scale, bias, nelements);
20}
21
22void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
23    const ggml_tensor * src0 = dst->src[0];
24    const float * src0_d = (const float *)src0->data;
25    float * dst_d = (float *)dst->data;
26    cudaStream_t stream = ctx.stream();
27
28    GGML_ASSERT(src0->type == GGML_TYPE_F32);
29    GGML_ASSERT( dst->type == GGML_TYPE_F32);
30
31    float scale;
32    float bias;
33    memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
34    memcpy(&bias,  (float *) dst->op_params + 1, sizeof(float));
35
36    scale_f32_cuda(src0_d, dst_d, scale, bias, ggml_nelements(src0), stream);
37}