char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Johannes Gäßlerggml: backend-agnostic tensor parallelism (experimental) (#19378)d6f303004

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
741 B28 linesraw
1#pragma once
2
3#include <cuda_runtime.h>
4#include <cuda.h>
5#include <cublas_v2.h>
6#include <cuda_bf16.h>
7#include <cuda_fp16.h>
8
9#ifdef GGML_USE_NCCL
10#include <nccl.h>
11#endif // GGML_USE_NCCL
12
13#if CUDART_VERSION >= 11080
14#include <cuda_fp8.h>
15#define FP8_AVAILABLE
16#endif // CUDART_VERSION >= 11080
17
18#if CUDART_VERSION >= 12080
19#include <cuda_fp4.h>
20#endif // CUDART_VERSION >= 12080
21
22#if CUDART_VERSION < 11020
23#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
24#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH
25#define CUBLAS_COMPUTE_16F CUDA_R_16F
26#define CUBLAS_COMPUTE_32F CUDA_R_32F
27#define cublasComputeType_t cudaDataType_t
28#endif // CUDART_VERSION < 11020