char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Piotr Wilkin (ilintar)CUDA: replace GGML_FA_ALL_QUANTS with GGML_FA_QUANTS, more control over what is compiled (#28079)5a4d0feca

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
10.5 KiB258 linesraw
1cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
2
3find_package(CUDAToolkit)
4
5if (CUDAToolkit_FOUND)
6    message(STATUS "CUDA Toolkit found")
7
8    if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
9        # native == GPUs available at build time
10        # 50     == Maxwell, lowest CUDA 12 standard
11        # 60     == P100, FP16 CUDA intrinsics
12        # 61     == Pascal, __dp4a instruction (per-byte integer dot product)
13        # 70     == V100, FP16 tensor cores
14        # 75     == Turing, int8 tensor cores
15        # 80     == Ampere, asynchronous data loading, faster tensor core instructions
16        # 86     == RTX 3000, needs CUDA v11.1
17        # 89     == RTX 4000, needs CUDA v11.8
18        # 90     == Hopper H100/200, needs CUDA v11.8
19        # 120    == Blackwell, needs CUDA v12.8, FP4 tensor cores
20        #
21        # XX-virtual == compile CUDA code as PTX, do JIT compilation to binary code on first run
22        # XX-real    == compile CUDA code as device code for this specific architecture
23        # no suffix  == compile as both PTX and device code
24        #
25        # The default behavior for a non-native is to build virtual architectures as needed to cover all features needed
26        #     for best performance and to also build real architectures for the most commonly used GPUs.
27        if (GGML_NATIVE AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "11.6" AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.24")
28            set(CMAKE_CUDA_ARCHITECTURES "native")
29        else()
30            if (CUDAToolkit_VERSION VERSION_LESS "13")
31                list(APPEND CMAKE_CUDA_ARCHITECTURES 50-virtual 61-virtual 70-virtual)
32            endif ()
33
34            list(APPEND CMAKE_CUDA_ARCHITECTURES 75-virtual 80-virtual 86-real)
35
36            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "11.8")
37                list(APPEND CMAKE_CUDA_ARCHITECTURES 89-real 90-virtual)
38            endif()
39
40            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8")
41                # The CUDA architecture 120f-virtual would in principle work for Blackwell support
42                #     but the newly added "f" suffix conflicted with a preexising regex for validating CUDA architectures in CMake.
43                # So either a recent CMake version or one with the backported fix is needed.
44                # The following versions should work:
45                #   - CMake >= v3.31.8 && CMake < v4.0.0
46                #   - CMake >= v4.0.2
47                # This is NOT documented in the CMake release notes,
48                #     check Modules/Internal/CMakeCUDAArchitecturesValidate.cmake in the CMake git repository instead.
49                # However, the architectures 120a-real and 121a-real should work with basically any CMake version and
50                #     until the release of e.g. Rubin there is no benefit to shipping virtual architectures for Blackwell.
51                list(APPEND CMAKE_CUDA_ARCHITECTURES 120a-real)
52            endif()
53            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.9")
54                list(APPEND CMAKE_CUDA_ARCHITECTURES 121a-real)
55            endif()
56        endif()
57    endif()
58
59    enable_language(CUDA)
60
61    # TODO: Remove once CCCL 3.2 has been released and bundled with CUDA Toolkit
62    if (GGML_CUDA_CUB_3DOT2)
63        include(FetchContent)
64
65        FetchContent_Declare(
66            CCCL
67            GIT_REPOSITORY https://github.com/nvidia/cccl.git
68            GIT_TAG        v3.2.0
69            GIT_SHALLOW    TRUE
70        )
71
72        FetchContent_MakeAvailable(CCCL)
73    endif()
74
75    # Replace any plain 12X CUDA architectures with their "architecture-specific" equivalents 12Xa.
76    # 12X is forwards-compatible, 12Xa is not.
77    # Notably the Blackwell FP4 tensor core instructions are not forwards compatible and therefore need 12Xa.
78    # But while 12X vs. 12Xa can be checked in device code there is (to my knowledge) no easy way to do the same check in host code.
79    # So for now just replace all instances of 12X with 12Xa, this should be fine until Rubin is released.
80    foreach(ARCHS IN ITEMS CMAKE_CUDA_ARCHITECTURES CMAKE_CUDA_ARCHITECTURES_NATIVE)
81        set(FIXED_ARCHS "")
82        foreach(ARCH IN LISTS ${ARCHS})
83            if (ARCH MATCHES "^12[0-9](-real|-virtual)?$")
84                string(REGEX REPLACE "^(12[0-9])((-real|-virtual)?)$" "\\1a\\2" FIXED_ARCH ${ARCH})
85                message(STATUS "Replacing ${ARCH} in ${ARCHS} with ${FIXED_ARCH}")
86                list(APPEND FIXED_ARCHS "${FIXED_ARCH}")
87            else()
88                list(APPEND FIXED_ARCHS "${ARCH}")
89            endif()
90        endforeach()
91        set(${ARCHS} ${FIXED_ARCHS})
92    endforeach()
93
94    # If we try to compile a "native" build it will use the 12X architectures and fail.
95    # So we should instead use the native architectures as determined by CMake after replacing 12X with 12Xa.
96    # But if at the time of the build no GPUs are connected at all CMAKE_CUDA_ARCHITECTURES will contain garbage that we should not use.
97    if (CMAKE_CUDA_ARCHITECTURES STREQUAL "native" AND CMAKE_CUDA_ARCHITECTURES_NATIVE MATCHES "^[0-9]+(a|f)?(-real|-virtual)?(;[0-9]+(a|f)?(-real|-virtual)?|;)*$")
98        set(CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_NATIVE})
99    endif()
100    message(STATUS "Using CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} CMAKE_CUDA_ARCHITECTURES_NATIVE=${CMAKE_CUDA_ARCHITECTURES_NATIVE}")
101
102    file(GLOB   GGML_HEADERS_CUDA "*.cuh")
103    list(APPEND GGML_HEADERS_CUDA "../../include/ggml-cuda.h")
104
105    file(GLOB   GGML_SOURCES_CUDA "*.cu")
106    file(GLOB   SRCS "template-instances/fattn-tile*.cu")
107    list(APPEND GGML_SOURCES_CUDA ${SRCS})
108    file(GLOB   SRCS "template-instances/fattn-mma*.cu")
109    list(APPEND GGML_SOURCES_CUDA ${SRCS})
110    file(GLOB   SRCS "template-instances/mmq*.cu")
111    list(APPEND GGML_SOURCES_CUDA ${SRCS})
112    file(GLOB   SRCS "template-instances/mmf*.cu")
113    list(APPEND GGML_SOURCES_CUDA ${SRCS})
114
115    ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS)
116    list(APPEND GGML_SOURCES_CUDA ${SRCS})
117
118    ggml_add_backend_library(ggml-cuda
119                             ${GGML_HEADERS_CUDA}
120                             ${GGML_SOURCES_CUDA}
121                            )
122
123    if (GGML_CUDA_GRAPHS)
124        add_compile_definitions(GGML_CUDA_USE_GRAPHS)
125    endif()
126
127    if (GGML_CUDA_FORCE_MMQ)
128        add_compile_definitions(GGML_CUDA_FORCE_MMQ)
129    endif()
130
131    if (GGML_CUDA_FORCE_CUBLAS)
132        add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)
133    endif()
134
135    if (GGML_CUDA_NO_VMM)
136        add_compile_definitions(GGML_CUDA_NO_VMM)
137    endif()
138
139    if (NOT GGML_CUDA_FA)
140        add_compile_definitions(GGML_CUDA_NO_FA)
141    endif()
142
143    if (GGML_CUDA_NO_PEER_COPY)
144        add_compile_definitions(GGML_CUDA_NO_PEER_COPY)
145    endif()
146
147    if (GGML_STATIC)
148        if (WIN32)
149            # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library
150            target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas)
151        else ()
152            if (GGML_CUDA_CUB_3DOT2)
153                target_link_libraries(ggml-cuda PRIVATE  CCCL::CCCL)
154            endif()
155            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "10.1")
156                target_link_libraries(ggml-cuda PRIVATE  CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
157            else()
158                target_link_libraries(ggml-cuda PRIVATE  CUDA::cudart_static CUDA::cublas_static)
159            endif()
160        endif()
161    else()
162        if (GGML_CUDA_CUB_3DOT2)
163            target_link_libraries(ggml-cuda PRIVATE  CCCL::CCCL)
164        endif()
165        target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas)
166    endif()
167
168    if (GGML_CUDA_NO_VMM)
169        # No VMM requested, no need to link directly with the cuda driver lib (libcuda.so)
170    else()
171        target_link_libraries(ggml-cuda PRIVATE CUDA::cuda_driver)
172    endif()
173
174    if (GGML_CUDA_NCCL)
175        find_package(NCCL)
176        if (NCCL_FOUND)
177            add_compile_definitions(GGML_USE_NCCL)
178            target_link_libraries(ggml-cuda PRIVATE NCCL::NCCL)
179        else()
180            message(STATUS "Warning: NCCL not found, performance for multiple CUDA GPUs will be suboptimal")
181        endif()
182    endif()
183
184    set(CUDA_CXX_FLAGS "")
185
186    set(CUDA_FLAGS -use_fast_math -extended-lambda)
187
188    if (GGML_CUDA_DEBUG)
189        list(APPEND CUDA_FLAGS -lineinfo)
190        add_compile_definitions(GGML_CUDA_DEBUG)
191    endif()
192
193    if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8")
194        # Options are:
195        # - none (not recommended)
196        # - speed (nvcc's default)
197        # - balance
198        # - size
199        list(APPEND CUDA_FLAGS -compress-mode=${GGML_CUDA_COMPRESSION_MODE})
200    endif()
201
202    if (GGML_FATAL_WARNINGS)
203        list(APPEND CUDA_FLAGS -Werror all-warnings)
204    endif()
205
206    if (GGML_ALL_WARNINGS AND NOT MSVC)
207        set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c)
208        if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL "")
209            list(APPEND NVCC_CMD -ccbin ${CMAKE_CUDA_HOST_COMPILER})
210        endif()
211
212        execute_process(
213            COMMAND ${NVCC_CMD} -Xcompiler --version
214            OUTPUT_VARIABLE CUDA_CCFULLVER
215            ERROR_QUIET
216        )
217
218        if (NOT CUDA_CCFULLVER MATCHES clang)
219            set(CUDA_CCID "GNU")
220            execute_process(
221                COMMAND ${NVCC_CMD} -Xcompiler "-dumpfullversion -dumpversion"
222                OUTPUT_VARIABLE CUDA_CCVER
223                ERROR_QUIET
224                OUTPUT_STRIP_TRAILING_WHITESPACE
225            )
226        else()
227            if (CUDA_CCFULLVER MATCHES Apple)
228                set(CUDA_CCID "AppleClang")
229            else()
230                set(CUDA_CCID "Clang")
231            endif()
232            string(REGEX REPLACE "^.* version ([0-9.]*).*$" "\\1" CUDA_CCVER ${CUDA_CCFULLVER})
233        endif()
234
235        message(STATUS "CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}")
236
237        ggml_get_flags(${CUDA_CCID} ${CUDA_CCVER})
238        list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS})  # This is passed to -Xcompiler later
239    endif()
240
241    if (NOT MSVC)
242        list(APPEND CUDA_CXX_FLAGS -Wno-pedantic)
243    else()
244        # CCCL 3.2 onwards will require a cpp-standard-compliant preprocessor for MSVC
245        # https://github.com/NVIDIA/cccl/pull/6827
246        list(APPEND CUDA_CXX_FLAGS /Zc:preprocessor)
247    endif()
248
249    list(JOIN   CUDA_CXX_FLAGS " " CUDA_CXX_FLAGS_JOINED)  # pass host compiler flags as a single argument
250
251    if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL "")
252        list(APPEND CUDA_FLAGS -Xcompiler ${CUDA_CXX_FLAGS_JOINED})
253    endif()
254
255    target_compile_options(ggml-cuda PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_FLAGS}>")
256else()
257    message(FATAL_ERROR "CUDA Toolkit not found")
258endif()