hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovggml-cpu : disable PCH and fix CACHE_LINE_SIZE ambiguity to fix heap corruption (#28882)2f539596c2026-09-14

1function(ggml_add_cpu_backend_features cpu_name arch)
2    # The feature detection code is compiled as a separate target so that
3    # it can be built without the architecture flags
4    # Since multiple variants of the CPU backend may be included in the same
5    # build, using set_source_files_properties() to set the arch flags is not possible
6    set(GGML_CPU_FEATS_NAME ${cpu_name}-feats)
7    add_library(${GGML_CPU_FEATS_NAME} OBJECT ggml-cpu/arch/${arch}/cpu-feats.cpp)
8    target_include_directories(${GGML_CPU_FEATS_NAME} PRIVATE . ../include)
9    target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE ${ARGN})
10    target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE GGML_BACKEND_DL GGML_BACKEND_BUILD GGML_BACKEND_SHARED)
11    set_target_properties(${GGML_CPU_FEATS_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
12    # Disable LTO for the feature detection code to prevent cross-module optimization
13    # from inlining architecture-specific instructions into the score function.
14    # Without this, LTO can cause SIGILL when loading backends on older CPUs
15    # (e.g., loading power10 backend on power9 crashes before feature check runs).
16    target_compile_options(${GGML_CPU_FEATS_NAME} PRIVATE -fno-lto)
17    target_link_libraries(${cpu_name} PRIVATE ${GGML_CPU_FEATS_NAME})
18endfunction()
19
20function(ggml_add_cpu_backend_variant_impl tag_name)
21    if (tag_name)
22        set(GGML_CPU_NAME ggml-cpu-${tag_name})
23    else()
24        set(GGML_CPU_NAME ggml-cpu)
25    endif()
26
27    ggml_add_backend_library(${GGML_CPU_NAME})
28
29    list (APPEND GGML_CPU_SOURCES
30        ggml-cpu/ggml-cpu.c
31        ggml-cpu/ggml-cpu.cpp
32        ggml-cpu/repack.cpp
33        ggml-cpu/repack.h
34        ggml-cpu/iqp.cpp
35        ggml-cpu/iqp.h
36        ggml-cpu/hbm.cpp
37        ggml-cpu/hbm.h
38        ggml-cpu/quants.c
39        ggml-cpu/quants.h
40        ggml-cpu/traits.cpp
41        ggml-cpu/traits.h
42        ggml-cpu/amx/amx.cpp
43        ggml-cpu/amx/amx.h
44        ggml-cpu/amx/mmq.cpp
45        ggml-cpu/amx/mmq.h
46        ggml-cpu/ggml-cpu-impl.h
47        ggml-cpu/common.h
48        ggml-cpu/binary-ops.h
49        ggml-cpu/binary-ops.cpp
50        ggml-cpu/unary-ops.h
51        ggml-cpu/unary-ops.cpp
52        ggml-cpu/simd-mappings.h
53        ggml-cpu/vec.h
54        ggml-cpu/vec.cpp
55        ggml-cpu/ops.h
56        ggml-cpu/ops.cpp
57        )
58
59    target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)
60    target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)
61
62    if (APPLE AND GGML_ACCELERATE)
63        find_library(ACCELERATE_FRAMEWORK Accelerate)
64        if (ACCELERATE_FRAMEWORK)
65            message(STATUS "Accelerate framework found")
66
67            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_ACCELERATE)
68            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_NEW_LAPACK)
69            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_LAPACK_ILP64)
70
71            target_link_libraries(${GGML_CPU_NAME} PRIVATE ${ACCELERATE_FRAMEWORK})
72        else()
73            message(WARNING "Accelerate framework not found")
74        endif()
75    endif()
76
77    if (GGML_OPENMP_ENABLED)
78        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)
79        target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX})
80    endif()
81
82    if (GGML_LLAMAFILE)
83        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_LLAMAFILE)
84
85        list(APPEND GGML_CPU_SOURCES
86                    ggml-cpu/llamafile/sgemm.cpp
87                    ggml-cpu/llamafile/sgemm.h)
88    endif()
89
90    if (GGML_CPU_HBM)
91        find_library(memkind memkind REQUIRED)
92
93        message(STATUS "Using memkind for CPU HBM")
94
95        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_HBM)
96
97        target_link_libraries(${GGML_CPU_NAME} PUBLIC memkind)
98    endif()
99
100    if (GGML_SYSTEM_ARCH STREQUAL "ARM")
101        message(STATUS "ARM detected")
102        list(APPEND GGML_CPU_SOURCES
103            ggml-cpu/arch/arm/quants.c
104            ggml-cpu/arch/arm/repack.cpp
105            )
106
107        if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
108            message(FATAL_ERROR "MSVC is not supported for ARM, use clang")
109        else()
110            check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
111            if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
112                list(APPEND ARCH_FLAGS -mfp16-format=ieee)
113            endif()
114
115            if (GGML_NATIVE)
116                # -mcpu=native does not always enable all the features in some compilers,
117                # so we check for them manually and enable them if available
118
119                execute_process(
120                    COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v -
121                    INPUT_FILE "/dev/null"
122                    OUTPUT_QUIET
123                    ERROR_VARIABLE ARM_MCPU
124                    RESULT_VARIABLE ARM_MCPU_RESULT
125                )
126                if (NOT ARM_MCPU_RESULT)
127                    string(REGEX MATCH "-mcpu=[^ ']+" ARM_MCPU_FLAG "${ARM_MCPU}")
128                    string(REGEX MATCH "-march=[^ ']+" ARM_MARCH_FLAG "${ARM_MCPU}")
129
130                    # on some old GCC we need to read -march=
131                    if (ARM_MARCH_FLAG AND NOT "${ARM_MARCH_FLAG}" STREQUAL "-march=native")
132                        set(ARM_NATIVE_FLAG "${ARM_MARCH_FLAG}")
133                    elseif(ARM_MCPU_FLAG AND NOT "${ARM_MCPU_FLAG}" STREQUAL "-mcpu=native")
134                        set(ARM_NATIVE_FLAG "${ARM_MCPU_FLAG}")
135                    endif()
136                endif()
137
138                if ("${ARM_NATIVE_FLAG}" STREQUAL "")
139                    set(ARM_NATIVE_FLAG -mcpu=native)
140                    message(WARNING "ARM -march/-mcpu not found, -mcpu=native will be used")
141                else()
142                    message(STATUS "ARM detected flags: ${ARM_NATIVE_FLAG}")
143                endif()
144
145                include(CheckCXXSourceRuns)
146
147                macro(check_arm_feature tag feature code)
148                    set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
149                    set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+${tag}")
150                    check_cxx_source_runs("${code}" GGML_MACHINE_SUPPORTS_${tag})
151                    if (GGML_MACHINE_SUPPORTS_${tag})
152                        set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+${tag}")
153                    else()
154                        set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+no${tag}")
155                        check_cxx_source_compiles("int main() { return 0; }" GGML_MACHINE_SUPPORTS_no${tag})
156                        if (GGML_MACHINE_SUPPORTS_no${tag})
157                            set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+no${tag}")
158                            list(APPEND ARCH_FLAGS -U__ARM_FEATURE_${feature})
159                        endif()
160                    endif()
161                    set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
162                endmacro()
163
164                check_arm_feature(dotprod DOTPROD     "#include <arm_neon.h>\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }")
165                check_arm_feature(i8mm    MATMUL_INT8 "#include <arm_neon.h>\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }")
166                check_arm_feature(sve     SVE         "#include <arm_sve.h>\nint main()  { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }")
167                check_arm_feature(sme     SME         "#include <arm_sme.h>\n__arm_locally_streaming int main() { __asm__ volatile(\"smstart; smstop;\"); return 0; }")
168
169                list(APPEND ARCH_FLAGS "${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}")
170            else()
171                if (GGML_CPU_ARM_ARCH)
172                    list(APPEND ARCH_FLAGS -march=${GGML_CPU_ARM_ARCH})
173                elseif(GGML_CPU_ALL_VARIANTS)
174                    # Begin with the lowest baseline
175                    set(ARM_MCPU "armv8-a")
176                    set(ARCH_TAGS "")
177                    set(ARCH_DEFINITIONS "")
178
179                    # When a feature is selected, bump the MCPU to the first
180                    # version that supported it
181                    if (GGML_INTERNAL_DOTPROD)
182                        set(ARM_MCPU "armv8.2-a")
183                        set(ARCH_TAGS "${ARCH_TAGS}+dotprod")
184                        list(APPEND ARCH_DEFINITIONS GGML_USE_DOTPROD)
185                    endif()
186                    if (GGML_INTERNAL_FP16_VECTOR_ARITHMETIC)
187                        set(ARM_MCPU "armv8.2-a")
188                        set(ARCH_TAGS "${ARCH_TAGS}+fp16")
189                        list(APPEND ARCH_DEFINITIONS GGML_USE_FP16_VECTOR_ARITHMETIC)
190                    endif()
191                    if (GGML_INTERNAL_SVE)
192                        set(ARM_MCPU "armv8.2-a")
193                        set(ARCH_TAGS "${ARCH_TAGS}+sve")
194                        list(APPEND ARCH_DEFINITIONS GGML_USE_SVE)
195                    endif()
196                    if (GGML_INTERNAL_MATMUL_INT8)
197                        set(ARM_MCPU "armv8.6-a")
198                        set(ARCH_TAGS "${ARCH_TAGS}+i8mm")
199                        list(APPEND ARCH_DEFINITIONS GGML_USE_MATMUL_INT8)
200                    endif()
201                    if (GGML_INTERNAL_SVE2)
202                        set(ARM_MCPU "armv8.6-a")
203                        set(ARCH_TAGS "${ARCH_TAGS}+sve2")
204                        list(APPEND ARCH_DEFINITIONS GGML_USE_SVE2)
205                    endif()
206                    if (GGML_INTERNAL_NOSVE)
207                        set(ARCH_TAGS "${ARCH_TAGS}+nosve")
208                    endif()
209                    if (GGML_INTERNAL_SME)
210                        set(ARM_MCPU "armv9.2-a")
211                        set(ARCH_TAGS "${ARCH_TAGS}+sme")
212                        list(APPEND ARCH_DEFINITIONS GGML_USE_SME)
213                    endif()
214                    list(APPEND ARCH_FLAGS "-march=${ARM_MCPU}${ARCH_TAGS}")
215                    ggml_add_cpu_backend_features(${GGML_CPU_NAME} arm ${ARCH_DEFINITIONS})
216                endif()
217            endif()
218
219            message(STATUS "Checking for ARM features using flags:")
220            foreach(flag IN LISTS ARCH_FLAGS)
221                message(STATUS "  ${flag}")
222            endforeach()
223
224            include(CheckCXXSourceCompiles)
225            set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
226            string(REPLACE ";" " " ARCH_FLAGS_STR "${ARCH_FLAGS}")
227            set(CMAKE_REQUIRED_FLAGS "${ARCH_FLAGS_STR}")
228            foreach(feature DOTPROD SVE MATMUL_INT8 FMA FP16_VECTOR_ARITHMETIC SME)
229                set(ARM_FEATURE "HAVE_${feature}")
230                check_cxx_source_compiles(
231                    "
232                    #if !defined(__ARM_FEATURE_${feature})
233                    #  error \"Feature ${feature} is not defined\"
234                    #endif
235                    int main() { return 0; }
236                    "
237                    ${ARM_FEATURE}
238                )
239            endforeach()
240            set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
241        endif()
242    elseif (GGML_SYSTEM_ARCH STREQUAL "x86")
243        message(STATUS "x86 detected")
244        list(APPEND GGML_CPU_SOURCES
245            ggml-cpu/arch/x86/quants.c
246            ggml-cpu/arch/x86/repack.cpp
247            )
248
249        if (MSVC)
250            # instruction set detection for MSVC only
251            if (GGML_NATIVE)
252                include(ggml-cpu/cmake/FindSIMD.cmake)
253            endif ()
254            if (GGML_AVX512)
255                list(APPEND ARCH_FLAGS /arch:AVX512)
256                # /arch:AVX512 includes: __AVX512F__, __AVX512CD__, __AVX512BW__, __AVX512DQ__, and __AVX512VL__
257                # MSVC has no compile-time flags enabling specific
258                # AVX512 extensions, neither it defines the
259                # macros corresponding to the extensions.
260                # Do it manually.
261                list(APPEND ARCH_DEFINITIONS GGML_AVX512)
262                if (GGML_AVX512_VBMI)
263                    list(APPEND ARCH_DEFINITIONS __AVX512VBMI__)
264                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
265                        list(APPEND ARCH_FLAGS -mavx512vbmi)
266                    endif()
267                endif()
268                if (GGML_AVX512_VNNI)
269                    list(APPEND ARCH_DEFINITIONS __AVX512VNNI__ GGML_AVX512_VNNI)
270                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
271                        list(APPEND ARCH_FLAGS -mavx512vnni)
272                    endif()
273                endif()
274                if (GGML_AVX512_BF16)
275                    list(APPEND ARCH_DEFINITIONS __AVX512BF16__ GGML_AVX512_BF16)
276                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
277                        list(APPEND ARCH_FLAGS -mavx512bf16)
278                    endif()
279                endif()
280                if (GGML_AMX_TILE)
281                    list(APPEND ARCH_DEFINITIONS __AMX_TILE__ GGML_AMX_TILE)
282                endif()
283                if (GGML_AMX_INT8)
284                    list(APPEND ARCH_DEFINITIONS __AMX_INT8__ GGML_AMX_INT8)
285                endif()
286                if (GGML_AMX_BF16)
287                    list(APPEND ARCH_DEFINITIONS __AMX_BF16__ GGML_AMX_BF16)
288                endif()
289            elseif (GGML_AVX2)
290                list(APPEND ARCH_FLAGS /arch:AVX2)
291                list(APPEND ARCH_DEFINITIONS GGML_AVX2 GGML_FMA GGML_F16C)
292            elseif (GGML_AVX)
293                list(APPEND ARCH_FLAGS /arch:AVX)
294                list(APPEND ARCH_DEFINITIONS GGML_AVX)
295            elseif (GGML_SSE42)
296                list(APPEND ARCH_FLAGS /arch:SSE4.2)
297                list(APPEND ARCH_DEFINITIONS GGML_SSE42)
298            endif()
299            if (GGML_AVX_VNNI)
300                list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI)
301            endif()
302            if (GGML_BMI2)
303                # MSVC does not define macro __BMI2__
304                list(APPEND ARCH_DEFINITIONS __BMI2__ GGML_BMI2)
305            endif()
306        else ()
307            if (GGML_NATIVE)
308                list(APPEND ARCH_FLAGS -march=native)
309            else ()
310                if (GGML_SSE42)
311                    list(APPEND ARCH_FLAGS -msse4.2)
312                    list(APPEND ARCH_DEFINITIONS GGML_SSE42)
313                endif()
314                if (GGML_F16C)
315                    list(APPEND ARCH_FLAGS -mf16c)
316                    list(APPEND ARCH_DEFINITIONS GGML_F16C)
317                endif()
318                if (GGML_FMA)
319                    list(APPEND ARCH_FLAGS -mfma)
320                    list(APPEND ARCH_DEFINITIONS GGML_FMA)
321                endif()
322                if (GGML_BMI2)
323                    list(APPEND ARCH_FLAGS -mbmi2)
324                    list(APPEND ARCH_DEFINITIONS GGML_BMI2)
325                endif()
326                if (GGML_AVX)
327                    list(APPEND ARCH_FLAGS -mavx)
328                    list(APPEND ARCH_DEFINITIONS GGML_AVX)
329                endif()
330                if (GGML_AVX2)
331                    list(APPEND ARCH_FLAGS -mavx2)
332                    list(APPEND ARCH_DEFINITIONS GGML_AVX2)
333                endif()
334                if (GGML_AVX_VNNI)
335                    list(APPEND ARCH_FLAGS -mavxvnni)
336                    list(APPEND ARCH_DEFINITIONS GGML_AVX_VNNI)
337                endif()
338                if (GGML_AVX512)
339                    list(APPEND ARCH_FLAGS -mavx512f)
340                    list(APPEND ARCH_FLAGS -mavx512cd)
341                    list(APPEND ARCH_FLAGS -mavx512vl)
342                    list(APPEND ARCH_FLAGS -mavx512dq)
343                    list(APPEND ARCH_FLAGS -mavx512bw)
344                    list(APPEND ARCH_DEFINITIONS GGML_AVX512)
345                endif()
346                if (GGML_AVX512_VBMI)
347                    list(APPEND ARCH_FLAGS -mavx512vbmi)
348                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VBMI)
349                endif()
350                if (GGML_AVX512_VNNI)
351                    list(APPEND ARCH_FLAGS -mavx512vnni)
352                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VNNI)
353                endif()
354                if (GGML_AVX512_BF16)
355                    list(APPEND ARCH_FLAGS -mavx512bf16)
356                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_BF16)
357                endif()
358                if (GGML_AMX_TILE)
359                    list(APPEND ARCH_FLAGS -mamx-tile)
360                    list(APPEND ARCH_DEFINITIONS GGML_AMX_TILE)
361                endif()
362                if (GGML_AMX_INT8)
363                    list(APPEND ARCH_FLAGS -mamx-int8)
364                    list(APPEND ARCH_DEFINITIONS GGML_AMX_INT8)
365                endif()
366                if (GGML_AMX_BF16)
367                    list(APPEND ARCH_FLAGS -mamx-bf16)
368                    list(APPEND ARCH_DEFINITIONS GGML_AMX_BF16)
369                endif()
370            endif()
371        endif()
372
373        if (GGML_BACKEND_DL)
374            if (GGML_NATIVE)
375                # the feature check relies on ARCH_DEFINITIONS, but it is not set with GGML_NATIVE
376                message(FATAL_ERROR "GGML_NATIVE is not compatible with GGML_BACKEND_DL, consider using GGML_CPU_ALL_VARIANTS")
377            endif()
378            ggml_add_cpu_backend_features(${GGML_CPU_NAME} x86 ${ARCH_DEFINITIONS})
379        endif()
380    elseif (GGML_SYSTEM_ARCH STREQUAL "PowerPC")
381        message(STATUS "PowerPC detected")
382        list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/powerpc/quants.c)
383        if (GGML_NATIVE)
384            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
385                file(READ "/proc/cpuinfo" POWER10_M)
386            elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc")
387                execute_process(COMMAND bash -c "prtconf |grep 'Implementation' | head -n 1" OUTPUT_VARIABLE POWER10_M)
388            endif()
389
390            string(TOUPPER "${POWER10_M}" POWER10_M_UPPER)
391            string(REGEX MATCHALL "POWER *([0-9]+)" MATCHED_STRING "${POWER10_M_UPPER}")
392            string(REGEX REPLACE "POWER *([0-9]+)" "\\1" EXTRACTED_NUMBER "${MATCHED_STRING}")
393
394            if (EXTRACTED_NUMBER EQUAL 10 OR EXTRACTED_NUMBER EQUAL 11)
395                list(APPEND ARCH_FLAGS -mcpu=power10)
396            elseif (EXTRACTED_NUMBER EQUAL 9)
397                list(APPEND ARCH_FLAGS -mcpu=power9)
398            elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le")
399                list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native)
400            else()
401                list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64)
402            endif()
403        elseif(GGML_CPU_ALL_VARIANTS)
404            # Begin with the lowest baseline
405            set(ARCH_DEFINITIONS "")
406
407            # When a feature is selected, bump the MCPU to the first
408            # version that supported it
409            foreach(PVER RANGE 7 11)
410                if(DEFINED GGML_INTERNAL_POWER${PVER})
411                    set(POWERPC_MCPU "power${PVER}")
412                    list(APPEND ARCH_DEFINITIONS GGML_USE_POWER${PVER})
413                endif()
414            endforeach()
415            if (GGML_INTERNAL_VSX)
416                list(APPEND ARCH_DEFINITIONS GGML_USE_VSX)
417                list(APPEND ARCH_FLAGS -mvsx)
418            endif()
419
420            if (DEFINED POWERPC_MCPU)
421                list(APPEND ARCH_FLAGS -mcpu=${POWERPC_MCPU})
422            endif()
423            ggml_add_cpu_backend_features(${GGML_CPU_NAME} powerpc ${ARCH_DEFINITIONS})
424        else()
425            if (GGML_CPU_POWERPC_CPUTYPE)
426                list(APPEND ARCH_FLAGS -mcpu=${GGML_CPU_POWERPC_CPUTYPE})
427            endif()
428        endif()
429    elseif (GGML_SYSTEM_ARCH STREQUAL "loongarch64")
430        message(STATUS "loongarch64 detected")
431        list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/loongarch/quants.c)
432
433        list(APPEND ARCH_FLAGS -march=loongarch64)
434        if (GGML_LASX)
435            list(APPEND ARCH_FLAGS -mlasx)
436        endif()
437        if (GGML_LSX)
438            list(APPEND ARCH_FLAGS -mlsx)
439        endif()
440    elseif (GGML_SYSTEM_ARCH STREQUAL "riscv64")
441        message(STATUS "riscv64 detected")
442        list(APPEND GGML_CPU_SOURCES
443            ggml-cpu/arch/riscv/quants.c
444            ggml-cpu/arch/riscv/repack.cpp
445            )
446        if (GGML_CPU_RISCV64_SPACEMIT)
447            include(ggml-cpu/cmake/FindSMTIME.cmake)
448            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC})
449            list(APPEND GGML_CPU_SOURCES
450                ggml-cpu/spacemit/ime.cpp
451                ggml-cpu/spacemit/ime.h
452                ggml-cpu/spacemit/spine_mem_pool.cpp
453                ggml-cpu/spacemit/spine_mem_pool.h
454                ggml-cpu/spacemit/repack.cpp
455                ggml-cpu/spacemit/repack.h
456                ggml-cpu/spacemit/ime_env.cpp
457                ggml-cpu/spacemit/ime_env.h
458                ggml-cpu/spacemit/ime_kernels.h
459                ggml-cpu/spacemit/rvv_kernels.cpp
460                ggml-cpu/spacemit/rvv_kernels.h
461            )
462            if ("RISCV64_SPACEMIT_IME1" IN_LIST RISCV64_SPACEMIT_IME_SPEC)
463                list(APPEND GGML_CPU_SOURCES ggml-cpu/spacemit/ime1_kernels.cpp)
464            endif()
465            if ("RISCV64_SPACEMIT_IME2" IN_LIST RISCV64_SPACEMIT_IME_SPEC)
466                list(APPEND GGML_CPU_SOURCES ggml-cpu/spacemit/ime2_kernels.cpp)
467            endif()
468        endif()
469        if(NOT GGML_CPU_ALL_VARIANTS)
470            set(MARCH_STR "rv64gc")
471            if (GGML_RVV)
472                string(APPEND MARCH_STR "v")
473            endif()
474
475            if (GGML_RV_ZFH)
476                string(APPEND MARCH_STR "_zfh")
477            endif()
478
479            if (GGML_XTHEADVECTOR)
480                string(APPEND MARCH_STR "_xtheadvector")
481            elseif (GGML_RVV)
482                if (GGML_RV_ZVFH)
483                    string(APPEND MARCH_STR "_zvfh")
484                endif()
485                if (GGML_RV_ZVFBFWMA)
486                    string(APPEND MARCH_STR "_zvfbfwma")
487                endif()
488            endif()
489
490            if (GGML_RV_ZICBOP)
491                string(APPEND MARCH_STR "_zicbop")
492            endif()
493            if (GGML_RV_ZIHINTPAUSE)
494                string(APPEND MARCH_STR "_zihintpause")
495            endif()
496            if (GGML_RV_ZBA)
497                string(APPEND MARCH_STR "_zba")
498            endif()
499            if (GGML_CPU_RISCV64_SPACEMIT)
500                # `xsmtvdotii' is only required for GCC >= 15.
501                if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND
502                    CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 15)
503                    string(APPEND MARCH_STR "_xsmtvdotii")
504                endif()
505            endif()
506
507            list(APPEND ARCH_FLAGS "-march=${MARCH_STR}" -mabi=lp64d)
508        else()
509            # Begin with the lowest baseline
510            set(ARCH_DEFINITIONS "")
511
512            if (GGML_INTERNAL_RVV)
513                message(STATUS "RVV enabled")
514                list(APPEND ARCH_DEFINITIONS GGML_USE_RVV)
515                list(APPEND ARCH_FLAGS -march=rv64gc_v -mabi=lp64d)
516            endif()
517
518            ggml_add_cpu_backend_features(${GGML_CPU_NAME} riscv ${ARCH_DEFINITIONS})
519        endif()
520    elseif (GGML_SYSTEM_ARCH STREQUAL "s390x")
521        message(STATUS "s390x detected")
522        list(APPEND GGML_CPU_SOURCES
523            ggml-cpu/arch/s390/quants.c
524            ggml-cpu/arch/s390/repack.cpp
525            )
526
527        # for native compilation
528        if (GGML_NATIVE)
529            # check machine level to determine target
530            file(READ "/proc/cpuinfo" CPUINFO_CONTENTS)
531            string(REGEX REPLACE "machine[ \t\r\n]*=[ \t\r\n]*([0-9]+)" "\\1" S390X_M ${CPUINFO_CONTENTS})
532
533            # TODO: Separation to determine activation of VX/VXE/VXE2
534            if (${S390X_M} MATCHES "8561|8562")
535                message(STATUS "z15 target")
536                list(APPEND ARCH_FLAGS -march=z15)
537            elseif (${S390X_M} MATCHES "3931")
538                message(STATUS "z16 target")
539                list(APPEND ARCH_FLAGS -march=z16)
540            elseif (${S390X_M} MATCHES "9175|9176")
541                # NOTE: Only available from GCC 15.1.0 onwards. Any z17 machine with compile issues must first verify their GCC version.
542                #       binutils must also be updated to the latest for the -march=z17 flag to work. Otherwise, use -march=arch15.
543                message(STATUS "z17 target")
544                list(APPEND ARCH_FLAGS -march=arch15)
545            else()
546                message(STATUS "Unknown target")
547                message(WARNING "Unknown target. If you are compiling for z14 and earlier, you might have to add -DGGML_VXE=OFF.")
548                list(APPEND ARCH_FLAGS -march=native -mtune=native)
549            endif()
550        # for cross-compilation
551        elseif(GGML_CPU_ALL_VARIANTS)
552            # range through IBM z15 to z17
553            # NOTE: update when a new hardware level is released
554            foreach (ZHW RANGE 15 17)
555                if(DEFINED GGML_INTERNAL_Z${ZHW})
556                    message(STATUS "z${ZHW} cross-compile target")
557                    list(APPEND ARCH_FLAGS -march=z${ZHW})
558                endif()
559            endforeach()
560        endif()
561
562        if (GGML_VXE OR GGML_INTERNAL_VXE2)
563            message(STATUS "VXE2 enabled")
564            list(APPEND ARCH_FLAGS -mvx -mzvector)
565            list(APPEND ARCH_DEFINITIONS GGML_USE_VXE2)
566        endif()
567
568        if (GGML_INTERNAL_NNPA)
569            message(STATUS "NNPA enabled")
570            list(APPEND ARCH_DEFINITIONS GGML_USE_NNPA)
571        endif()
572
573        ggml_add_cpu_backend_features(${GGML_CPU_NAME} s390 ${ARCH_DEFINITIONS})
574    elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "wasm")
575        message(STATUS "Wasm detected")
576        list (APPEND GGML_CPU_SOURCES ggml-cpu/arch/wasm/quants.c)
577    else()
578        message(WARNING "Unknown CPU architecture. Falling back to generic implementations.")
579        list(APPEND ARCH_FLAGS -DGGML_CPU_GENERIC)
580    endif()
581
582    if (GGML_CPU_REPACK)
583        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_REPACK)
584    endif()
585
586    if (GGML_CPU_KLEIDIAI)
587        # upstream repo requires at least cmake 3.16
588        if (CMAKE_VERSION VERSION_LESS 3.16)
589            message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires CMake >= 3.16")
590        endif()
591
592        set(GGML_CPU_KLEIDIAI_AARCH64 OFF)
593        if (GGML_SYSTEM_ARCH STREQUAL "ARM" AND
594            (APPLE OR WIN32 OR CMAKE_SYSTEM_NAME MATCHES "^(Linux|Android)$") AND
595            (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|arm64-v8a)$" OR
596             CMAKE_OSX_ARCHITECTURES MATCHES "arm64" OR
597             CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
598             CMAKE_ANDROID_ARCH_ABI STREQUAL "arm64-v8a"))
599            set(GGML_CPU_KLEIDIAI_AARCH64 ON)
600        endif()
601        if (NOT GGML_CPU_KLEIDIAI_AARCH64)
602            message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires a Linux, Android, Apple, or Windows AArch64/arm64 target")
603        endif()
604
605        message(STATUS "Using KleidiAI optimized kernels if applicable")
606
607        # Fetch KleidiAI sources:
608        include(FetchContent)
609        set(KLEIDIAI_COMMIT_TAG "v1.24.0")
610        set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")
611        set(KLEIDIAI_RELEASE_ARCHIVE_MD5  "2f02ebe29573d45813e671eb304f2a00")
612
613        set(KLEIDIAI_FETCH_ARGS
614            URL ${KLEIDIAI_DOWNLOAD_URL}
615            URL_HASH MD5=${KLEIDIAI_RELEASE_ARCHIVE_MD5}
616        )
617        if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24")
618            list(APPEND KLEIDIAI_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP NEW)
619        endif()
620
621        FetchContent_Declare(kleidiai
622            ${KLEIDIAI_FETCH_ARGS}
623        )
624
625        # Disable tests and benchmark building
626        set(KLEIDIAI_BUILD_TESTS OFF CACHE BOOL "" FORCE)
627        set(KLEIDIAI_BUILD_BENCHMARK OFF CACHE BOOL "" FORCE)
628
629        # Use the Populate/add_subdirectory flow for compatibility with CMake 3.16.
630        FetchContent_GetProperties(kleidiai
631            SOURCE_DIR KLEIDIAI_SRC
632            BINARY_DIR KLEIDIAI_BIN
633            POPULATED KLEIDIAI_POPULATED
634        )
635        if (NOT KLEIDIAI_POPULATED)
636            FetchContent_Populate(kleidiai)
637            FetchContent_GetProperties(kleidiai
638                SOURCE_DIR KLEIDIAI_SRC
639                BINARY_DIR KLEIDIAI_BIN
640            )
641        endif()
642
643        if (NOT TARGET kleidiai)
644            add_subdirectory(
645                "${CMAKE_CURRENT_SOURCE_DIR}/ggml-cpu/kleidiai"
646                "${CMAKE_CURRENT_BINARY_DIR}/kleidiai-wrapper"
647                EXCLUDE_FROM_ALL
648            )
649            if (NOT CMAKE_SKIP_INSTALL_RULES AND
650                (NOT DEFINED BUILD_SHARED_LIBS OR NOT BUILD_SHARED_LIBS))
651                install(TARGETS kleidiai ARCHIVE)
652            endif()
653        endif()
654
655        if (NOT TARGET kleidiai)
656            message(FATAL_ERROR "KleidiAI target was not created")
657        endif()
658
659        set_target_properties(kleidiai PROPERTIES POSITION_INDEPENDENT_CODE ON)
660
661        target_link_libraries(${GGML_CPU_NAME} PRIVATE kleidiai)
662
663        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_KLEIDIAI)
664
665        list(APPEND GGML_CPU_SOURCES
666            ggml-cpu/kleidiai/kleidiai.cpp
667            ggml-cpu/kleidiai/kernels.cpp
668            ggml-cpu/kleidiai/kleidiai.h
669            ggml-cpu/kleidiai/kernels.h
670            )
671    endif()
672
673    message(STATUS "Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}")
674    target_sources(${GGML_CPU_NAME} PRIVATE ${GGML_CPU_SOURCES})
675    target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS})
676    target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS})
677
678    if (EMSCRIPTEN)
679        set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128")
680    endif()
681
682  if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
683      # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math"
684      target_compile_options(${GGML_CPU_NAME} PRIVATE "$<$<OR:$<COMPILE_LANG_AND_ID:C,IntelLLVM>,$<COMPILE_LANG_AND_ID:CXX,IntelLLVM>>:$<$<BOOL:${WIN32}>:/clang:>-fno-associative-math>")
685  endif()
686
687endfunction()