hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovmetal : single-source fusion table + fusion debug rework (#28164)a2878d30d2026-09-11

1find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
2find_library(METAL_FRAMEWORK    Metal      REQUIRED)
3find_library(METALKIT_FRAMEWORK MetalKit   REQUIRED)
4
5message(STATUS "Metal framework found")
6
7ggml_add_backend_library(ggml-metal
8                         ggml-metal.cpp
9                         ggml-metal-device.m
10                         ggml-metal-device.cpp
11                         ggml-metal-common.cpp
12                         ggml-metal-context.m
13                         ggml-metal-fusion.cpp
14                         ggml-metal-ops.cpp
15                         ggml-metal-tuning.cpp
16                        )
17
18target_link_libraries(ggml-metal PRIVATE
19                      ${FOUNDATION_LIBRARY}
20                      ${METAL_FRAMEWORK}
21                      ${METALKIT_FRAMEWORK}
22                      )
23
24if (GGML_METAL_NDEBUG)
25    add_compile_definitions(GGML_METAL_NDEBUG)
26endif()
27
28set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/../ggml-common.h")
29set(METALLIB_KERNELS_COMMON     "${CMAKE_CURRENT_SOURCE_DIR}/kernels/common.h")
30set(METALLIB_KERNELS_DEQUANTIZE "${CMAKE_CURRENT_SOURCE_DIR}/kernels/dequantize.h")
31set(METALLIB_KERNELS_QUANTIZE   "${CMAKE_CURRENT_SOURCE_DIR}/kernels/quantize.h")
32
33set(METALLIB_KERNEL_SOURCES
34    kernels/fa.metal
35    kernels/mul_mv.metal
36    kernels/mul_mm.metal
37    kernels/quantize.metal
38    kernels/softmax.metal
39    kernels/norm.metal
40    kernels/unary.metal
41    kernels/binbcast.metal
42    kernels/reduce.metal
43    kernels/tri.metal
44    kernels/ssm.metal
45    kernels/wkv.metal
46    kernels/gated_delta_net.metal
47    kernels/solve_tri.metal
48    kernels/rope.metal
49    kernels/conv.metal
50    kernels/upscale.metal
51    kernels/argsort.metal
52    kernels/pool.metal
53    kernels/misc.metal
54)
55
56if (GGML_METAL_EMBED_LIBRARY)
57    enable_language(ASM)
58
59    add_compile_definitions(GGML_METAL_EMBED_LIBRARY)
60
61    set(METALLIB_IMPL "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h")
62
63    file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/autogenerated")
64
65    set(METALLIB_EMBED_ASM_FILES "")
66    foreach(src ${METALLIB_KERNEL_SOURCES})
67        get_filename_component(kind ${src} NAME_WE)
68        # symbol names must be valid C identifiers ('-' is not allowed)
69        string(REPLACE "-" "_" kind_sym ${kind})
70
71        set(SRC   "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${kind}.metal")
72        set(EMBED "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed-${kind}.metal")
73        set(ASM   "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed-${kind}.s")
74
75        # only prepend headers that this source actually includes
76        set(HEADERS_FOR_SRC ${METALLIB_KERNELS_COMMON})
77        file(STRINGS ${SRC} _has_dequantize REGEX "#include \"dequantize\\.h\"")
78        file(STRINGS ${SRC} _has_quantize   REGEX "#include \"quantize\\.h\"")
79        if(_has_dequantize)
80            list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_DEQUANTIZE})
81        endif()
82        if(_has_quantize)
83            list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_QUANTIZE})
84        endif()
85
86        add_custom_command(
87            OUTPUT "${ASM}"
88            # Step 1: concatenate shared headers + this kernel source
89            COMMAND cat ${HEADERS_FOR_SRC} ${SRC} > "${EMBED}.tmp1"
90            # Step 2: remove internal #include and #pragma once
91            COMMAND sed -e "/\#include \"common.h\"/d" -e "/\#include \"dequantize.h\"/d" -e "/\#include \"quantize.h\"/d" -e "/\#pragma once/d" < "${EMBED}.tmp1" > "${EMBED}.tmp2"
92            # Step 3: inline ggml-common.h (replacing __embed_ggml-common.h__ sentinel)
93            COMMAND sed -e "/__embed_ggml-common.h__/r ${METALLIB_COMMON}" -e "/__embed_ggml-common.h__/d" < "${EMBED}.tmp2" > "${EMBED}.tmp3"
94            # Step 4: inline ggml-metal-impl.h
95            COMMAND sed -e "/\#include \"ggml-metal-impl.h\"/r ${METALLIB_IMPL}" -e "/\#include \"ggml-metal-impl.h\"/d" < "${EMBED}.tmp3" > "${EMBED}"
96            # Step 5: emit an asm chunk with kind-specific start/end symbols
97            #   note: '-' is illegal in C symbols, so we use kind_sym; the macOS
98            #   section name is limited to 16 chars so we keep it shared
99            #   across kinds (__ggml_metallib) and only vary the global symbols.
100            COMMAND echo ".section __DATA,__ggml_metallib"                       >  "${ASM}"
101            COMMAND echo ".globl _ggml_metallib_${kind_sym}_start"               >> "${ASM}"
102            COMMAND echo "_ggml_metallib_${kind_sym}_start:"                     >> "${ASM}"
103            COMMAND echo .incbin "\"${EMBED}\""                                  >> "${ASM}"
104            COMMAND echo ".globl _ggml_metallib_${kind_sym}_end"                 >> "${ASM}"
105            COMMAND echo "_ggml_metallib_${kind_sym}_end:"                       >> "${ASM}"
106            DEPENDS ../ggml-common.h ggml-metal-impl.h
107                    kernels/common.h kernels/dequantize.h kernels/quantize.h
108                    kernels/${kind}.metal
109            COMMENT "Generate embedded Metal library for ${kind}"
110            VERBATIM
111        )
112
113        list(APPEND METALLIB_EMBED_ASM_FILES "${ASM}")
114    endforeach()
115
116    target_sources(ggml-metal PRIVATE ${METALLIB_EMBED_ASM_FILES})
117else()
118    # copy header files to bin directory
119    configure_file(../ggml-common.h  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h     COPYONLY)
120    configure_file(ggml-metal-impl.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h COPYONLY)
121
122    file(MAKE_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels")
123    configure_file(kernels/common.h     ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/common.h     COPYONLY)
124    configure_file(kernels/dequantize.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/dequantize.h COPYONLY)
125    configure_file(kernels/quantize.h   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/quantize.h   COPYONLY)
126
127    foreach(src ${METALLIB_KERNEL_SOURCES})
128        configure_file(${src} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} COPYONLY)
129    endforeach()
130
131    # CMAKE_OSX_SYSROOT is an SDK name or path - xcrun accepts both
132    set(METAL_SDK ${CMAKE_OSX_SYSROOT})
133    if (NOT METAL_SDK)
134        set(METAL_SDK macosx)
135    endif()
136
137    if (CMAKE_OSX_SYSROOT MATCHES "[Ss]imulator")
138        set(METAL_TARGET_SIM "-simulator")
139    else()
140        set(METAL_TARGET_SIM "")
141    endif()
142
143    if (GGML_METAL_SHADER_DEBUG)
144        # note: disabling fast math is needed in order to pass tests/test-backend-ops
145        # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
146        # note: unfortunately, we have to call it default.metallib instead of ggml.metallib
147        #       ref: https://github.com/ggml-org/whisper.cpp/issues/1720
148        # note: adding -g causes segmentation fault during compile
149        set(XC_FLAGS -fno-fast-math -fno-inline)
150    else()
151        set(XC_FLAGS -O3)
152    endif()
153
154    execute_process(COMMAND xcrun -sdk ${METAL_SDK} --show-sdk-version OUTPUT_VARIABLE METAL_SDK_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
155    if (METAL_SDK_VERSION VERSION_GREATER_EQUAL 26.0)
156        set(GGML_METAL_HAS_TENSOR_LIB ON)
157    else()
158        message(STATUS "Metal SDK ${METAL_SDK_VERSION} does not support the tensor API, skipping ggml-tensor.metallib")
159    endif()
160
161    if (GGML_METAL_MACOSX_VERSION_MIN)
162        message(STATUS "Adding  -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
163        list   (APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})
164    elseif (NOT GGML_METAL_TARGET_OS STREQUAL "macos" AND CMAKE_OSX_DEPLOYMENT_TARGET)
165        message(STATUS "Adding  -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM} flag to metal compilation")
166        list   (APPEND XC_FLAGS -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM})
167    endif()
168
169    if (GGML_METAL_STD)
170        message(STATUS "Adding  -std=${GGML_METAL_STD} flag to metal compilation")
171        list   (APPEND XC_FLAGS -std=${GGML_METAL_STD})
172    endif()
173
174    # Compile each kernel source to .air, then link into default.metallib
175    set(AIR_FILES "")
176    foreach(src ${METALLIB_KERNEL_SOURCES})
177        get_filename_component(name ${src} NAME_WE)
178        set(AIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${name}.air")
179        list(APPEND AIR_FILES ${AIR})
180        add_custom_command(
181            OUTPUT ${AIR}
182            COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} -o ${AIR}
183            DEPENDS ${src} kernels/common.h kernels/dequantize.h kernels/quantize.h ${METALLIB_COMMON} ggml-metal-impl.h
184            COMMENT "Compiling ${src}"
185            VERBATIM
186        )
187    endforeach()
188
189    set(METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
190
191    # the tensor API kernels go in a separate metallib, loaded only where supported
192    if (GGML_METAL_HAS_TENSOR_LIB)
193        set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")
194        # the tensor API needs OS 26+
195        set(XC_FLAGS_TENSOR ${XC_FLAGS} -mtargetos=${GGML_METAL_TARGET_OS}26.0${METAL_TARGET_SIM})
196        add_custom_command(
197            OUTPUT ${AIR_MM_TENSOR}
198            COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}
199            DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h
200            COMMENT "Compiling kernels/mul_mm.metal (tensor API)"
201            VERBATIM
202        )
203
204        add_custom_command(
205            OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
206            COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
207            DEPENDS ${AIR_MM_TENSOR}
208            COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"
209        )
210
211        list(APPEND METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib)
212    endif()
213
214    add_custom_command(
215        OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
216        COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
217        COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
218        COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h
219        COMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels
220        DEPENDS ${AIR_FILES} ${AIR_MM_TENSOR}
221        COMMENT "Linking Metal kernels into default.metallib"
222    )
223
224    add_custom_target(
225        ggml-metal-lib ALL
226        DEPENDS ${METALLIB_FILES}
227    )
228endif() # GGML_METAL_EMBED_LIBRARY
229
230if (NOT GGML_METAL_EMBED_LIBRARY)
231    install(
232        DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/kernels/
233        DESTINATION ${CMAKE_INSTALL_BINDIR}/kernels
234        FILES_MATCHING PATTERN "*.metal" PATTERN "*.h"
235    )
236
237    install(
238        FILES ${METALLIB_FILES}
239        DESTINATION ${CMAKE_INSTALL_BINDIR}
240    )
241endif()