char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Martin Changggml-et: Initial ET backend (#24179)082b326fc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.4 KiB246 linesraw
1
2message(STATUS "Using ET backend")
3
4# Configure ET platform path
5if (DEFINED ENV{ET_PLATFORM})
6    set(ET_PLATFORM_PATH $ENV{ET_PLATFORM})
7else()
8    set(ET_PLATFORM_PATH "/opt/et")
9endif()
10
11# Use sysemu for ET backend if compiled with `-DGGML_ET_SYSEMU=ON`
12if (GGML_ET_SYSEMU)
13    message(STATUS "Using ET backend with sysemu instead of hardware")
14else()
15    message(STATUS "Using ET backend with hardware device")
16endif()
17
18# Add ET platform CMake modules and config files to search paths
19list(APPEND CMAKE_PREFIX_PATH ${ET_PLATFORM_PATH}/lib/cmake)
20list(APPEND CMAKE_MODULE_PATH ${ET_PLATFORM_PATH}/lib/cmake)
21include(aifoundry-utils/ProjectFunctions)
22
23message(STATUS "Using ET Platform at ${ET_PLATFORM_PATH}")
24
25find_package(runtime REQUIRED)
26
27# Kernel list
28set(KERNELS
29    el_map_f32
30    flash_attn_ext_f32
31    glu_f32
32    scale_f32
33    mul_mat_f32
34    mul_mat_f32_matrix_engine
35    mul_mat_id_f32
36    mul_mat_id_Q4_0
37    mul_mat_id_Q8_0
38    mul_mat_Q8_0
39    mul_mat_Q4_0
40    mul_mat_Q4_0_matrix_engine
41    mul_mat_f16
42    mul_mat_f16_matrix_engine
43    rope_f32
44    unary_f32
45    sqr_f32
46    clamp_f32
47    sum_rows_f32
48    mean_f32
49    cumsum_f32
50    norm_f32
51    l2_norm_f32
52    group_norm_f32
53    rms_norm_f32
54    rms_norm_mul_f32
55    softmax_f32
56    im2col
57    get_rows_f32
58    concat_f32
59    repeat_f32
60    rwkv_wkv6_f32
61    rwkv_wkv7_f32
62    gated_delta_net_f32
63    cont_f32
64    cont_f16
65    cpy_f32_f16
66    flash_attn_ext_f16_me
67    set_rows_f32
68    set_f32
69    fill_f32
70    pad_f32
71    diag_f32
72    tri_f32
73    solve_tri_f32
74    ssm_conv_f32
75    ssm_scan_f32
76    conv_2d_f32_me
77    memops
78    uberkernel
79)
80
81# Kernels that we support dispatch form Uberkernel
82set(UBERKERNEL_SUPPORTED_KERNELS
83    el_map_f32
84    # unary_f32
85    # cpy_f32_f16
86    # cont_f32
87    # get_rows_f32
88    concat_f32
89    cont_f16
90    cumsum_f32
91    diag_f32
92    fill_f32
93    flash_attn_ext_f16_me
94    flash_attn_ext_f32
95    gated_delta_net_f32
96    glu_f32
97    group_norm_f32
98    im2col
99    l2_norm_f32
100    mul_mat_f16
101    mul_mat_f16_matrix_engine
102    mul_mat_f32
103    mul_mat_f32_matrix_engine
104    mul_mat_id_f32
105    mul_mat_Q4_0
106    mul_mat_Q8_0
107    norm_f32
108    pad_f32
109    repeat_f32
110    rms_norm_f32
111    rms_norm_mul_f32
112    rope_f32
113    rwkv_wkv6_f32
114    rwkv_wkv7_f32
115    scale_f32
116    set_f32
117    set_rows_f32
118    softmax_f32
119    solve_tri_f32
120    sqr_f32
121    # ssm_conv_f32
122    ssm_scan_f32
123    sum_rows_f32
124    tri_f32
125)
126
127set(UBERKERNEL_MAP_HPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h)
128set(UBERKERNEL_MAP_CPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp)
129
130set(UBERKERNEL_KERNELS_SORTED ${UBERKERNEL_SUPPORTED_KERNELS})
131list(SORT UBERKERNEL_KERNELS_SORTED)
132
133set(UBERKERNEL_ENUM_ENTRIES "")
134set(UBERKERNEL_MAP_ENTRIES "")
135set(_uk_idx 1)
136foreach(KERNEL ${UBERKERNEL_KERNELS_SORTED})
137    string(TOUPPER ${KERNEL} _uk_upper)
138    string(APPEND UBERKERNEL_ENUM_ENTRIES
139        "    GGML_ET_UBERKERNEL_KERNEL_${_uk_upper} = ${_uk_idx},\n")
140    string(APPEND UBERKERNEL_MAP_ENTRIES
141        "        {\"${KERNEL}\", GGML_ET_UBERKERNEL_KERNEL_${_uk_upper}},\n")
142    math(EXPR _uk_idx "${_uk_idx} + 1")
143endforeach()
144
145configure_file(
146    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-uberkernel-kernel-map.h.in
147    ${UBERKERNEL_MAP_HPP}
148    @ONLY)
149configure_file(
150    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-uberkernel-kernel-map.cpp.in
151    ${UBERKERNEL_MAP_CPP}
152    @ONLY)
153
154add_custom_target(et-uberkernel-map
155    DEPENDS ${UBERKERNEL_MAP_HPP} ${UBERKERNEL_MAP_CPP}
156)
157
158# Build ET kernels (cross-compiled in subdirectory scope)
159add_subdirectory(et-kernels)
160
161# Embed kernels into C++ source
162set(EMBED_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_one_kernel.cmake)
163set(EMBED_HPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-kernels-embed.hpp)
164set(EMBED_CPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-kernels-embed.cpp)
165set(EMBED_DIR ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/embed)
166file(MAKE_DIRECTORY ${EMBED_DIR})
167
168set(EMBED_KERNEL_SOURCES)
169set(EMBED_EXTERNS "")
170set(EMBED_MAP_ENTRIES "")
171
172foreach(KERNEL ${KERNELS})
173    set(ELF_PATH ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/${KERNEL}.elf)
174    set(OUT_CPP ${EMBED_DIR}/${KERNEL}.cpp)
175
176    add_custom_command(
177        OUTPUT ${OUT_CPP}
178        COMMAND ${CMAKE_COMMAND}
179            -DELF_FILE=${ELF_PATH}
180            -DOUT_FILE=${OUT_CPP}
181            -DVAR_NAME=${KERNEL}
182            -P ${EMBED_SCRIPT}
183        DEPENDS ${KERNEL}.elf ${EMBED_SCRIPT}
184        COMMENT "Embedding ${KERNEL}.elf"
185        VERBATIM
186    )
187    list(APPEND EMBED_KERNEL_SOURCES ${OUT_CPP})
188
189    string(APPEND EMBED_EXTERNS
190        "extern unsigned char ${KERNEL}_data[];\n"
191        "extern const uint64_t ${KERNEL}_len;\n")
192    string(APPEND EMBED_MAP_ENTRIES
193        "    {\"${KERNEL}\", {${KERNEL}_data, ${KERNEL}_len}},\n")
194endforeach()
195
196configure_file(
197    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-kernels-embed.hpp.in
198    ${EMBED_HPP}
199    @ONLY)
200configure_file(
201    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-kernels-embed.cpp.in
202    ${EMBED_CPP}
203    @ONLY)
204
205add_custom_target(et-kernels-embed ALL
206    DEPENDS ${EMBED_KERNEL_SOURCES} ${EMBED_HPP} ${EMBED_CPP} et-uberkernel-map
207)
208
209ggml_add_backend_library(ggml-et
210                         ggml-et.cpp
211                         ggml-et-kernels.cpp
212                         ggml-et-memops.cpp
213                         ggml-et-ops.cpp
214                         ggml-et-cpu-compare.cpp
215                        )
216
217# Mark generated files as such
218set_source_files_properties(
219    ${EMBED_CPP}
220    ${EMBED_HPP}
221    ${EMBED_KERNEL_SOURCES}
222    ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp
223    ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h
224    PROPERTIES GENERATED TRUE
225)
226
227# Add embedded kernel sources
228target_sources(ggml-et PRIVATE
229    ${EMBED_CPP}
230    ${EMBED_HPP}
231    ${EMBED_KERNEL_SOURCES}
232    ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp
233    ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h
234)
235
236# Include directory for embedded headers
237target_include_directories(ggml-et PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/et-kernels)
238
239target_link_libraries(ggml-et PRIVATE runtime::etrt_static deviceLayer::deviceLayer)
240target_compile_definitions(ggml-et PRIVATE GGML_ET_UBERKERNEL_HOST_LOOKUP)
241if (GGML_ET_SYSEMU)
242    target_compile_definitions(ggml-et PRIVATE GGML_ET_SYSEMU=1)
243endif()
244
245# Ensure kernels are built and embedded before the backend library
246add_dependencies(ggml-et et-kernels-embed et-uberkernel-map)