char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Neo Zhangsycl : Fix get mem error (#28227)b6b003d2c

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.8 KiB224 linesraw
1message(STATUS  "GGML_SYCL_TARGET=${GGML_SYCL_TARGET}")
2
3if (NOT GGML_SYCL_TARGET MATCHES "^(INTEL)$")
4    message(FATAL_ERROR "GGML_SYCL_TARGET: Invalid target, the supported options are [INTEL]")
5endif()
6
7check_cxx_compiler_flag("-fsycl" SUPPORTS_SYCL)
8
9if (DEFINED ENV{ONEAPI_ROOT})
10    message(STATUS "Using oneAPI Release SYCL compiler (icpx).")
11elseif(SUPPORTS_SYCL)
12    message(WARNING "Using open-source SYCL compiler (clang++). Didn't detect ENV {ONEAPI_ROOT}.
13        If you expected the oneAPI Release compiler, please install oneAPI & source it, like:
14        source /opt/intel/oneapi/setvars.sh")
15else()
16    message(FATAL_ERROR "C++ compiler lacks SYCL support.")
17endif()
18message(STATUS "SYCL found")
19#todo: AOT
20
21ggml_add_backend_library(ggml-sycl
22                         ggml-sycl.cpp
23                         ../../include/ggml-sycl.h
24                        )
25
26file(GLOB   GGML_HEADERS_SYCL "*.hpp")
27file(GLOB   GGML_SOURCES_SYCL "*.cpp")
28file(GLOB   SRCS "template-instances/fattn-tile*.cpp")
29list(APPEND GGML_SOURCES_SYCL ${SRCS})
30file(GLOB   SRCS "template-instances/fattn-vec*.cpp")
31list(APPEND GGML_SOURCES_SYCL ${SRCS})
32
33target_sources(ggml-sycl PRIVATE ${GGML_HEADERS_SYCL} ${GGML_SOURCES_SYCL})
34
35if (WIN32)
36    # To generate a Visual Studio solution, using Intel C++ Compiler for ggml-sycl is mandatory
37    if( ${CMAKE_GENERATOR} MATCHES "Visual Studio" AND NOT (${CMAKE_GENERATOR_TOOLSET} MATCHES "Intel C"))
38        set_target_properties(ggml-sycl PROPERTIES VS_PLATFORM_TOOLSET "Intel C++ Compiler 2025")
39        set(CMAKE_CXX_COMPILER "icx")
40        set(CMAKE_CXX_COMPILER_ID "IntelLLVM")
41    endif()
42    # Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO_API is enabled)
43    if(GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
44        if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH})
45            set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH})
46            if(EXISTS "${LEVEL_ZERO_V1_SDK_PATH}")
47                target_include_directories(ggml-sycl PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include")
48                set(LEVEL_ZERO_V1_SDK_LIB_PATH "${LEVEL_ZERO_V1_SDK_PATH}/lib")
49            else()
50                message(WARNING "LEVEL_ZERO_V1_SDK_PATH set but folder not found: ${LEVEL_ZERO_V1_SDK_PATH}")
51            endif()
52        endif()
53    endif()
54endif()
55
56macro(detect_and_find_package package_name)
57    set(test_source "
58    cmake_minimum_required(VERSION ${CMAKE_VERSION})
59    project(check_package LANGUAGES CXX)
60    find_package(${package_name} QUIET)
61    ")
62
63    set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/check_package_${package_name}")
64    file(WRITE "${test_dir}/CMakeLists.txt" "${test_source}")
65
66    set(cmake_args "")
67    if(CMAKE_GENERATOR)
68        list(APPEND cmake_args "-G" "${CMAKE_GENERATOR}")
69    endif()
70    if(CMAKE_GENERATOR_PLATFORM)
71        list(APPEND cmake_args "-A" "${CMAKE_GENERATOR_PLATFORM}")
72    endif()
73    if(CMAKE_GENERATOR_TOOLSET)
74        list(APPEND cmake_args "-T" "${CMAKE_GENERATOR_TOOLSET}")
75    endif()
76    if(CMAKE_CXX_COMPILER)
77        list(APPEND cmake_args "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}")
78    endif()
79
80    execute_process(
81        COMMAND ${CMAKE_COMMAND} ${cmake_args} .
82        WORKING_DIRECTORY "${test_dir}"
83        RESULT_VARIABLE result
84        OUTPUT_QUIET
85        ERROR_QUIET
86    )
87
88    if(result EQUAL 0)
89        find_package(${package_name} ${ARGN})
90    else()
91        message(WARNING "Detection of ${package_name} failed. The package might be broken or incompatible.")
92        set(${package_name}_FOUND FALSE)
93    endif()
94endmacro()
95
96detect_and_find_package(IntelSYCL)
97if (IntelSYCL_FOUND)
98    # Use oneAPI CMake when possible
99    target_link_libraries(ggml-sycl PRIVATE IntelSYCL::SYCL_CXX)
100else()
101    # Fallback to the simplest way of enabling SYCL when using intel/llvm nightly for instance
102    target_compile_options(ggml-sycl PRIVATE "-fsycl")
103    target_link_options(ggml-sycl PRIVATE "-fsycl")
104endif()
105
106target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing")
107
108message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}")
109if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
110    # Link against Level Zero loader for direct device memory allocation.
111    # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging
112    # in the xe kernel driver during multi-GPU inference.
113    find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include)
114    find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH)
115    if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR)
116        target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
117        target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
118        message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")
119        message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}")
120    else()
121        message(WARNING "Level Zero loader or development headers not found, "
122                        "Level Zero API support disabled. "
123                        "Please install the Level Zero SDK/development package "
124                        "to support Level Zero API features. "
125                        "Level Zero API is not mandatory for SYCL backend, "
126                        "but it is required by the special features for better "
127                        "function & performance on Intel GPUs.")
128    endif()
129endif()
130
131# Link against oneDNN
132set(GGML_SYCL_DNNL 0)
133if(GGML_SYCL_DNN)
134    find_package(DNNL)
135    if(DNNL_FOUND)
136        if (NOT DEFINED DNNL_GPU_VENDOR)
137            # default to intel target
138            set(DNNL_GPU_VENDOR "INTEL")
139            if(NOT "${GGML_SYCL_TARGET}" STREQUAL "INTEL")
140                message(WARNING "oneDNN builds bundled with oneapi release only support INTEL target")
141            endif()
142        endif()
143
144        # Verify oneDNN was compiled for the same target as llama
145        if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}")
146            target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl)
147            set(GGML_SYCL_DNNL 1)
148            get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS)
149            foreach(CONFIG ${CONFIGS})
150                get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG})
151                message(STATUS "Found oneDNN: ${DNNL_LIB}")
152            endforeach()
153        else()
154            message(WARNING
155                "oneDNN must be compiled for the same target as llama.cpp.
156                 llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}.
157                 Disabling oneDNN support.")
158        endif()
159    else()
160        message(STATUS "oneDNN not found, disabling oneDNN support")
161    endif()
162else()
163    message(STATUS "oneDNN support disabled by the user")
164endif()
165target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_DNNL=${GGML_SYCL_DNNL})
166
167if (GGML_SYCL_F16)
168    add_compile_definitions(GGML_SYCL_F16)
169endif()
170
171if (GGML_SYCL_TARGET STREQUAL "INTEL")
172    add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
173    if (NOT GGML_SYCL_DEVICE_ARCH)
174        target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required)
175    else()
176        message(STATUS "Skipping -ze-intel-greater-than-4GB-buffer-required for spir64_gen AOT")
177    endif()
178
179    # Link against Intel oneMKL
180    if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
181        set(SYCL_COMPILER ON)
182    endif()
183    find_package(MKL REQUIRED)
184    target_link_libraries(ggml-sycl PRIVATE MKL::MKL_SYCL::BLAS)
185else()
186    # default for other target
187    message(FATAL_ERROR "GGML_SYCL_TARGET is not supported")
188    add_compile_definitions(GGML_SYCL_WARP_SIZE=32)
189endif()
190
191if (GGML_SYCL_GRAPH)
192    message(STATUS "find GGML_SYCL_GRAPH")
193    target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_GRAPH)
194endif()
195
196if (GGML_SYCL_HOST_MEM_FALLBACK)
197    message(STATUS "find GGML_SYCL_HOST_MEM_FALLBACK")
198    target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_HOST_MEM_FALLBACK)
199endif()
200
201if (GGML_SYCL_DEVICE_ARCH)
202    message(STATUS "GGML_SYCL_DEVICE_ARCH=${GGML_SYCL_DEVICE_ARCH} (AOT via spir64_gen)")
203    target_compile_options(
204        ggml-sycl PRIVATE
205        -fsycl-targets=spir64_gen
206        "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\""
207    )
208
209    # Pass through parallel job (process) count for parallelising the
210    # `llvm-foreach -- ocloc` invocation for compiling AOT device images.
211    include(ProcessorCount)
212    ProcessorCount(_ggml_sycl_nproc)
213    if (_ggml_sycl_nproc LESS 1)
214        set(_ggml_sycl_nproc 1)
215    endif()
216    set(GGML_SYCL_MAX_PARALLEL_LINK_JOBS ${_ggml_sycl_nproc} CACHE STRING
217        "Parallel ocloc jobs for spir64_gen AOT device-image lowering")
218    target_link_options(
219        ggml-sycl PRIVATE
220        -fsycl-targets=spir64_gen
221        "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\""
222        -fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS}
223    )
224endif()