char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

shaofeiqiopencl: add bin kernel `kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin` (#28677)07fc97716

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.2 KiB249 linesraw
1find_package(OpenCL REQUIRED)
2find_package(Python3 REQUIRED)
3
4set(TARGET_NAME ggml-opencl)
5
6ggml_add_backend_library(${TARGET_NAME}
7                         ggml-opencl.cpp
8                         cl-program-cache.cpp
9                         cl-program-cache.h
10                         ../../include/ggml-opencl.h)
11target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES})
12target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS})
13
14if (GGML_OPENCL_PROFILING)
15    message(STATUS "OpenCL profiling enabled (increases CPU overhead)")
16    add_compile_definitions(GGML_OPENCL_PROFILING)
17endif ()
18
19add_compile_definitions(GGML_OPENCL_SOA_Q)
20add_compile_definitions(GGML_OPENCL_TARGET_VERSION=${GGML_OPENCL_TARGET_VERSION})
21
22if (GGML_OPENCL_USE_ADRENO_KERNELS)
23    message(STATUS "OpenCL will use matmul kernels optimized for Adreno")
24    add_compile_definitions(GGML_OPENCL_USE_ADRENO_KERNELS)
25endif ()
26
27if (GGML_OPENCL_EMBED_KERNELS)
28    add_compile_definitions(GGML_OPENCL_EMBED_KERNELS)
29
30    set(EMBED_KERNEL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kernels/embed_kernel.py")
31    file(MAKE_DIRECTORY     "${CMAKE_CURRENT_BINARY_DIR}/autogenerated")
32
33    target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/autogenerated")
34endif ()
35
36if (GGML_OPENCL_USE_ADRENO_BIN_KERNELS)
37    message(STATUS "OpenCL will use precompiled binary kernels for Adreno (improved performance on some platforms)")
38    add_compile_definitions(GGML_OPENCL_USE_ADRENO_BIN_KERNELS)
39endif ()
40
41function(ggml_opencl_add_kernel KNAME)
42    set(KERN_HDR ${CMAKE_CURRENT_BINARY_DIR}/autogenerated/${KNAME}.cl.h)
43    set(KERN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${KNAME}.cl)
44
45    if (GGML_OPENCL_EMBED_KERNELS)
46        message(STATUS "opencl: embedding kernel ${KNAME}")
47
48        # Python must be accessible from command line
49        add_custom_command(
50            OUTPUT ${KERN_HDR}
51            COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} ${KERN_SRC} ${KERN_HDR}
52            DEPENDS ${KERN_SRC} ${EMBED_KERNEL_SCRIPT}
53            COMMENT "Generate ${KERN_HDR}"
54        )
55
56        target_sources(${TARGET_NAME} PRIVATE ${KERN_HDR})
57    else ()
58        message(STATUS "opencl: adding kernel ${KNAME}")
59        configure_file(${KERN_SRC} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${KNAME}.cl COPYONLY)
60    endif ()
61endfunction()
62
63set(GGML_OPENCL_KERNELS
64    add
65    add_id
66    moe_add_id_glu
67    argsort
68    tri
69    fill
70    clamp
71    cpy
72    cvt
73    diag_mask_inf
74    diag
75    div
76    gelu
77    get_rows
78    glu
79    group_norm
80    solve_tri
81    im2col_f32
82    im2col_f16
83    mean
84    mul_mv_f16_f16
85    mul_mv_f16_f32_1row
86    mul_mv_f16_f32_l4
87    mul_mv_f16_f32
88    mul_mv_f16_f32_mrow
89    mul_mv_f32_f32
90    mul_mv_q1_0_f32
91    mul_mv_q1_0_f32_flat
92    mul_mv_q4_0_f32
93    mul_mv_q4_0_f32_v
94    mul_mv_q4_0_f32_8x_flat
95    mul_mv_q4_0_f32_1d_8x_flat
96    mul_mv_q4_0_f32_1d_16x_flat
97    mul_mv_q4_1_f32
98    mul_mv_q4_1_f32_flat
99    mul_mv_q4_k_f32
100    mul_mv_q4_k_f32_flat
101    mul_mv_q5_0_f32
102    mul_mv_q5_0_f32_flat
103    mul_mv_q5_1_f32
104    mul_mv_q5_1_f32_flat
105    mul_mv_q5_k_f32
106    mul_mv_q5_k_f32_flat
107    mul_mv_q6_k_f32
108    mul_mv_q6_k_f32_flat
109    mul_mv_q8_0_f32
110    mul_mv_q8_0_f32_flat
111    mul_mv_iq4_nl_f32
112    mul_mv_iq4_nl_f32_flat
113    mul_mv_mxfp4_f32
114    mul_mv_mxfp4_f32_flat
115    mul_mv_id_q4_0_f32_8x_flat
116    mul_mv_id_q8_0_f32
117    mul_mv_id_q8_0_f32_flat
118    mul_mv_id_mxfp4_f32
119    mul_mv_id_mxfp4_f32_flat
120    gemm_moe_q4_0_f32_ns
121    gemm_moe_q4_0_q8_1_dp4a
122    gemv_moe_q4_0_f32_ns
123    gemm_moe_q8_0_f32_ns
124    gemm_moe_q4_1_f32_ns
125    gemv_moe_q4_1_f32_ns
126    gemm_moe_q5_0_f32_ns
127    gemv_moe_q5_0_f32_ns
128    gemm_moe_q5_1_f32_ns
129    gemv_moe_q5_1_f32_ns
130    gemm_moe_q4_k_f32_ns
131    gemm_moe_q4_k_q8_1_dp4a
132    gemm_moe_q6_k_q8_1_dp4a
133    gemm_moe_q8_1_dp4a
134    moe_reorder_quant_a_q8_1
135    gemm_noshuffle_q4_k_q8_1_dp4a
136    gemm_noshuffle_q5_k_q8_1_dp4a
137    gemm_noshuffle_q6_k_q8_1_dp4a
138    gemm_noshuffle_q8_0_q8_1_dp4a
139    gemm_noshuffle_q5_0_q8_1_dp4a
140    gemm_noshuffle_iq4_nl_q8_1_dp4a
141    gemm_noshuffle_q4_0_q8_1_dp4a
142    quant_a_q8_1
143    gemv_moe_q4_k_f32_ns
144    gemm_moe_q5_k_f32_ns
145    gemv_moe_q5_k_f32_ns
146    gemm_moe_q6_k_f32_ns
147    gemv_moe_q6_k_f32_ns
148    gemm_moe_mxfp4_f32
149    gemv_moe_mxfp4_f32
150    gemm_moe_mxfp4_f32_ns
151    gemm_moe_mxfp4_q8_1_dp4a
152    gemv_moe_mxfp4_f32_ns
153    moe_reorder_b
154    moe_combine
155    moe_sort_by_expert
156    mul_mm_f32_f32_l4_lm
157    mul_mm_f16_f32_l4_lm
158    mul_mm_q1_0_f32_l4_lm
159    mul_mm_q4_0_f32_l4_lm
160    mul_mm_q4_1_f32_l4_lm
161    mul_mm_q5_0_f32_l4_lm
162    mul_mm_q5_1_f32_l4_lm
163    mul_mm_q8_0_f32_l4_lm
164    mul_mm_iq4_nl_f32_l4_lm
165    mul_mm_q4_k_f32_l4_lm
166    mul_mm_q5_k_f32_l4_lm
167    mul_mm_q6_k_f32_l4_lm
168    gemv_noshuffle_q1_0_f32
169    gemm_noshuffle_q1_0_f32
170    gemv_noshuffle_q4_0_f32
171    gemv_noshuffle_q4_0_f32_spec
172    gemm_noshuffle_q4_0_f32
173    gemv_noshuffle_q4_0_f32_32b_trans
174    gemv_noshuffle_q4_1_f32
175    gemm_noshuffle_q4_1_f32
176    gemv_noshuffle_q5_0_f32
177    gemm_noshuffle_q5_0_f32
178    gemv_noshuffle_q5_1_f32
179    gemm_noshuffle_q5_1_f32
180    gemv_noshuffle_iq4_nl_f32
181    gemm_noshuffle_iq4_nl_f32
182    gemv_noshuffle_q8_0_f32
183    gemm_noshuffle_q8_0_f32
184    gemv_noshuffle_q4_k_f32
185    gemv_noshuffle_q4_k_f32_o4
186    gemv_noshuffle_q4_k_f32_tiled
187    gemm_noshuffle_q4_k_f32
188    gemv_noshuffle_q4_k_f32_32b_trans
189    gemv_noshuffle_q6_k_f32
190    gemv_noshuffle_q6_k_f32_o4
191    gemv_noshuffle_q6_k_f32_tiled
192    gemm_noshuffle_q6_k_f32
193    gemm_noshuffle_q6_k_f32_tiled
194    gemv_noshuffle_q5_k_f32
195    gemm_noshuffle_q5_k_f32
196    mul
197    neg
198    norm
199    relu
200    l2_norm
201    rms_norm
202    rope
203    scale
204    set_rows
205    sigmoid
206    silu
207    softmax_4_f32
208    softmax_4_f16
209    softmax_f32
210    softmax_f16
211    sqr
212    sqrt
213    ssm_conv
214    ssm_scan
215    gated_delta_net
216    sub
217    sum_rows
218    cumsum
219    transpose
220    concat
221    tsembd
222    upscale
223    tanh
224    exp
225    expm1
226    abs
227    unary_ext
228    softplus
229    pad
230    repeat
231    mul_mat_f16_f32
232    mul_mm_f16_f32_kq_kqv
233    conv2d
234    conv2d_f16_f32
235    flash_attn_pre_f16
236    flash_attn_f32_f16
237    flash_attn_f32_q8_0
238    flash_attn_f32_q4_0
239    flash_attn_f16
240    flash_attn_f32
241)
242
243if (GGML_OPENCL_USE_ADRENO_KERNELS)
244    list(APPEND GGML_OPENCL_KERNELS gemm_xmem_f16_f32_os8 sdpa_xmem_f32_f16_os8)
245endif ()
246
247foreach (K ${GGML_OPENCL_KERNELS})
248    ggml_opencl_add_kernel(${K})
249endforeach()