char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: flash attention rework (optimizations, accuracy improvements, etc) (#25085)13e673863

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.6 KiB113 linesraw
1file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}"   HEXAGON_SDK_ROOT)
2file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
3
4if (NOT IS_DIRECTORY "${HEXAGON_SDK_ROOT}")
5    message(FATAL_ERROR "Make sure HEXAGON_SDK_ROOT point to the correct Hexagon SDK installation.")
6endif()
7
8if (NOT IS_DIRECTORY "${HEXAGON_TOOLS_ROOT}")
9    message("Try to read HEXAGON_TOOLS_ROOT from hexagon_sdk.json")
10    file(READ "${HEXAGON_SDK_ROOT}/hexagon_sdk.json" HEXAGON_SDK_CONFIG_PATH)
11    string(JSON HEXAGON_TOOLS_PATH GET ${HEXAGON_SDK_CONFIG_PATH} "root" "tools" "info" 0 "path")
12    message("Found HEXAGON_TOOLS_PATH: ${HEXAGON_TOOLS_PATH}")
13    set(HEXAGON_TOOLS_ROOT "${HEXAGON_SDK_ROOT}/${HEXAGON_TOOLS_PATH}")
14    file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
15    if (NOT IS_DIRECTORY "${HEXAGON_TOOLS_ROOT}")
16        message(FATAL_ERROR "Make sure HEXAGON_TOOLS_ROOT point to the correct Hexagon SDK installation.")
17    endif()
18endif()
19
20message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for building libggml-htp skels")
21
22include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
23include(ExternalProject)
24
25option(GGML_HEXAGON_HTP_DEBUG  "ggml-hexagon: enable HTP debug output" OFF)
26set(GGML_HEXAGON_HTP_CERT  "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate")
27
28add_library(htp_iface OBJECT
29    ${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c)
30
31set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON)
32target_include_directories(htp_iface PUBLIC
33    ${HEXAGON_SDK_ROOT}/incs
34    ${HEXAGON_SDK_ROOT}/incs/stddef
35    ${HEXAGON_SDK_ROOT}/utils/examples
36    ${CMAKE_CURRENT_SOURCE_DIR}/htp
37    ${CMAKE_CURRENT_BINARY_DIR})
38
39build_idl(htp/htp_iface.idl htp_iface)
40
41if (CMAKE_SYSTEM_NAME MATCHES Android)
42    target_link_options(htp_iface PUBLIC -llog -ldl)
43elseif (CMAKE_SYSTEM_NAME MATCHES Windows)
44    target_precompile_headers(htp_iface PUBLIC <sal.h>)
45else()
46    target_link_options(htp_iface PUBLIC -ldl)
47endif()
48
49set(TARGET_NAME ggml-hexagon)
50ggml_add_backend_library(${TARGET_NAME}
51    ggml-hexagon.cpp
52    htp-drv.cpp
53    htp-drv.h
54    libdl.h
55    ../../include/ggml-hexagon.h)
56
57target_link_libraries(${TARGET_NAME} PRIVATE htp_iface)
58target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR})
59
60# Build HTP skels
61set(HTP_SKELS)
62function(build_htp_skel V)
63    ExternalProject_Add(htp-${V}
64        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON
65        BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so
66        CMAKE_ARGS
67            -DCMAKE_BUILD_TYPE=Release
68            -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake
69            -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}
70            -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT}
71            -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT}
72            -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG}
73            -DDSP_VERSION=${V}
74            -DPREBUILT_LIB_DIR="toolv19_${V}")
75    list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so)
76    set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE)
77endfunction()
78
79build_htp_skel(v73)
80build_htp_skel(v75)
81build_htp_skel(v79)
82build_htp_skel(v81)
83
84# Install Hexagon skels required at runtime
85install(FILES ${HTP_SKELS} TYPE LIB)
86
87if (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT)
88    file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/arm64"      WINSDK_BIN0_ARM64)
89    file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/x86"        WINSDK_BIN0_X86)
90    file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/arm64" WINSDK_BIN1_ARM64)
91    file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/x86"   WINSDK_BIN1_X86)
92
93    set(WINSDK_PATHS ${WINSDK_BIN0_ARM64} ${WINSDK_BIN0_X86} ${WINSDK_BIN1_ARM64} ${WINSDK_BIN1_X86})
94
95    find_program(INF2CAT  NAMES inf2cat.exe  PATHS ${WINSDK_PATHS} REQUIRED)
96    find_program(SIGNTOOL NAMES signtool.exe PATHS ${WINSDK_PATHS} REQUIRED)
97
98    message(STATUS "hexagon: using ${GGML_HEXAGON_HTP_CERT} to sign libggml-htp skels")
99
100    set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat)
101    add_custom_target(libggml-htp-cat
102        BYPRODUCTS ${LIBGGML_HTP_CAT}
103        DEPENDS libggml-htp.inf ${HTP_SKELS}
104        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR}
105        COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64
106        COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT}
107        COMMENT "generating and signing libggml-htp.cat file"
108        VERBATIM
109    )
110
111    add_dependencies(${TARGET_NAME} libggml-htp-cat)
112    install(FILES ${LIBGGML_HTP_CAT} TYPE LIB)
113endif()