char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Piotr Wilkin (ilintar)CUDA: replace GGML_FA_ALL_QUANTS with GGML_FA_QUANTS, more control over what is compiled (#28079)5a4d0feca

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.9 KiB121 linesraw
1function(ggml_get_flags CCID CCVER)
2    set(C_FLAGS "")
3    set(CXX_FLAGS "")
4
5    if (CCID MATCHES "Clang")
6        set(C_FLAGS   -Wunreachable-code-break -Wunreachable-code-return)
7        set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
8
9        if (
10            (CCID STREQUAL "Clang"      AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
11            (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
12        )
13            list(APPEND C_FLAGS -Wdouble-promotion)
14        endif()
15    elseif (CCID STREQUAL "GNU")
16        set(C_FLAGS   -Wdouble-promotion)
17        set(CXX_FLAGS -Wno-array-bounds)
18
19        if (CCVER VERSION_GREATER_EQUAL 8.1.0)
20            list(APPEND CXX_FLAGS -Wextra-semi)
21        endif()
22    endif()
23
24    set(GF_C_FLAGS   ${C_FLAGS}   PARENT_SCOPE)
25    set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
26endfunction()
27
28function(ggml_get_system_arch)
29    if (CMAKE_OSX_ARCHITECTURES      STREQUAL "arm64" OR
30        CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
31        (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
32            CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
33        set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
34    elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
35            CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
36            (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
37            CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
38        set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
39    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
40        set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
41    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
42        set(GGML_SYSTEM_ARCH "loongarch64"  PARENT_SCOPE)
43    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
44        set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
45    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
46        set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
47    else()
48        set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
49    endif()
50endfunction()
51
52# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
53function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
54    set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16)
55
56    string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
57    string(STRIP   "${FA_QUANTS}" FA_QUANTS)
58    if (GGML_CUDA_FA_ALL_QUANTS)
59        message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead")
60        set(FA_QUANTS all)
61    endif()
62    if (NOT FA_QUANTS)
63        message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty")
64    endif()
65
66    if (FA_QUANTS STREQUAL "all")
67        set(FA_COMBINATIONS "")
68        foreach (TYPE_V IN LISTS FA_TYPES)
69            foreach (TYPE_K IN LISTS FA_TYPES)
70                list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
71            endforeach()
72        endforeach()
73    else()
74        set(FA_COMBINATIONS f16-f16)
75
76        string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}")
77        foreach (COMBINATION IN LISTS FA_SELECTED)
78            string(STRIP   "${COMBINATION}" COMBINATION)
79            if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$")
80                message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination")
81            endif()
82            set(TYPE_K ${CMAKE_MATCH_1})
83            set(TYPE_V ${CMAKE_MATCH_2})
84            foreach (TYPE ${TYPE_K} ${TYPE_V})
85                if (NOT TYPE IN_LIST FA_TYPES)
86                    message(FATAL_ERROR
87                        "GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}")
88                endif()
89            endforeach()
90            list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
91        endforeach()
92    endif()
93    list(REMOVE_DUPLICATES FA_COMBINATIONS)
94
95    string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}")
96    add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}")
97    foreach (TYPE_V IN LISTS FA_TYPES)
98        foreach (TYPE_K IN LISTS FA_TYPES)
99            if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS)
100                set(COMPILED 1)
101            else()
102                set(COMPILED 0)
103            endif()
104            string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF)
105            add_compile_definitions(${COMBINATION_DEF}=${COMPILED})
106        endforeach()
107    endforeach()
108
109    message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}")
110
111    set(SRCS "")
112    foreach (COMBINATION IN LISTS FA_COMBINATIONS)
113        set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu")
114        if (NOT EXISTS "${SRC}")
115            message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist")
116        endif()
117        list(APPEND SRCS "${SRC}")
118    endforeach()
119
120    set(${OUT_SRCS} ${SRCS} PARENT_SCOPE)
121endfunction()