char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovci : remove gg_sum summary logic (#28857)15d8f2d59

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
28.1 KiB729 linesraw
1#!/usr/bin/env bash
2#
3# sample usage:
4#
5# mkdir tmp
6#
7# # CPU-only build
8# bash ./ci/run.sh ./tmp/results ./tmp/mnt
9#
10# # with CUDA support
11# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
12#
13# # with ROCm support
14# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ./tmp/results ./tmp/mnt
15#
16# # with SYCL support
17# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
18#
19# # with VULKAN support
20# GG_BUILD_VULKAN=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
21#
22# # with WebGPU support
23# GG_BUILD_WEBGPU=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
24#
25# # with MUSA support
26# GG_BUILD_MUSA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
27#
28# # with KLEIDIAI support
29# GG_BUILD_KLEIDIAI=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
30#
31# # with BLAS support
32# GG_BUILD_BLAS=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
33#
34# with BLAS support (custom vendor)
35# GG_BUILD_BLAS=1 GG_BUILD_BLAS_VENDOR=Intel10_64lp bash ./ci/run.sh ./tmp/results ./tmp/mnt
36#
37# with OPENVINO support
38# GG_BUILD_OPENVINO=1 GG_BUILD_LOW_PERF=1 GGML_OPENVINO_DEVICE=CPU bash ./ci/run.sh ./tmp/results ./tmp/mnt
39#
40
41if [ -z "$2" ]; then
42    echo "usage: $0 <output-dir> <mnt-dir>"
43    exit 1
44fi
45
46mkdir -p "$1"
47mkdir -p "$2"
48
49OUT=$(realpath "$1")
50MNT=$(realpath "$2")
51
52# gpu-rocm self-hosted runner can't upload logs to blob; keep each run's logs in
53# their own dir keyed by the GitHub run id so an Actions run URL maps to its logs.
54if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then
55    OUT="$OUT/run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
56    mkdir -p "$OUT"
57    echo "ci results dir: $OUT"
58fi
59
60rm -f $OUT/*.log
61
62sd=`dirname $0`
63cd $sd/../
64SRC=`pwd`
65
66CMAKE_EXTRA="-DLLAMA_FATAL_WARNINGS=${LLAMA_FATAL_WARNINGS:-ON} -DLLAMA_OPENSSL=OFF -DGGML_SCHED_NO_REALLOC=ON"
67CTEST_EXTRA=""
68
69# Default to use make unless specified for compatibility
70CMAKE_GENERATOR="Unix Makefiles"
71
72if [ ! -z "${GG_BUILD_NINJA}" ]; then
73    CMAKE_GENERATOR="Ninja"
74fi
75
76if [ ! -z ${GG_BUILD_METAL} ]; then
77    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
78else
79    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF"
80fi
81
82if [ ! -z ${GG_BUILD_CUDA} ]; then
83    # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
84    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON"
85
86    if command -v nvidia-smi >/dev/null 2>&1; then
87        CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
88        if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
89            CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
90        else
91            echo "Warning: Using fallback CUDA architectures"
92            CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
93        fi
94    else
95        echo "Error: nvidia-smi not found, cannot build with CUDA"
96        exit 1
97    fi
98fi
99
100if [ ! -z ${GG_BUILD_ROCM} ]; then
101    CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang -DGGML_HIP=ON"
102    if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
103        echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
104        exit 1
105    fi
106
107    CMAKE_EXTRA="${CMAKE_EXTRA} -DGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
108fi
109
110if [ ! -z ${GG_BUILD_SYCL} ]; then
111    if [ -z ${ONEAPI_ROOT} ]; then
112        echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
113        echo "source /opt/intel/oneapi/setvars.sh"
114        exit 1
115    fi
116    # Use only main GPU
117    export ONEAPI_DEVICE_SELECTOR="level_zero:0"
118    # Enable sysman for correct memory reporting
119    export ZES_ENABLE_SYSMAN=1
120    # to circumvent precision issues on CPY operations
121    export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
122    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
123fi
124
125if [ ! -z ${GG_BUILD_VULKAN} ]; then
126    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
127
128    if [[ "$OSTYPE" == "darwin"* ]]; then
129        MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION="/usr/local/lib/cmake/vulkan"
130        MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
131        if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
132            CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
133        fi
134    fi
135
136    # Build shared libs on Windows
137    # to reduce binary size and avoid errors in library loading unit tests
138    if uname -s | grep -qi nt; then
139        CMAKE_EXTRA="${CMAKE_EXTRA} -DBUILD_SHARED_LIBS=ON"
140    fi
141fi
142
143if [ ! -z ${GG_BUILD_WEBGPU} ]; then
144    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
145
146    if [ ! -z "${GG_BUILD_WEBGPU_DAWN_PREFIX}" ]; then
147        if [ -z "${CMAKE_PREFIX_PATH}" ]; then
148            export CMAKE_PREFIX_PATH="${GG_BUILD_WEBGPU_DAWN_PREFIX}"
149        else
150            export CMAKE_PREFIX_PATH="${GG_BUILD_WEBGPU_DAWN_PREFIX}:${CMAKE_PREFIX_PATH}"
151        fi
152    fi
153
154    # For some systems, Dawn_DIR needs to be set explicitly, e.g., the lib64 path
155    if [ ! -z "${GG_BUILD_WEBGPU_DAWN_DIR}" ]; then
156        CMAKE_EXTRA="${CMAKE_EXTRA} -DDawn_DIR=${GG_BUILD_WEBGPU_DAWN_DIR}"
157    fi
158fi
159
160if [ ! -z ${GG_BUILD_MUSA} ]; then
161    # Use qy1 by default (MTT S80)
162    MUSA_ARCH=${MUSA_ARCH:-21}
163    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
164fi
165
166if [ ! -z ${GG_BUILD_NO_SVE} ]; then
167    # arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
168    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
169fi
170
171if [ -n "${GG_BUILD_KLEIDIAI}" ]; then
172    echo ">>===== Enabling KleidiAI support"
173    CMAKE_EXTRA="${CMAKE_EXTRA:+$CMAKE_EXTRA } -DGGML_CPU_KLEIDIAI=ON"
174fi
175
176if [ ! -z ${GG_BUILD_BLAS} ]; then
177    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=${GG_BUILD_BLAS_VENDOR:-OpenBLAS}"
178else
179    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=OFF"
180fi
181
182if [ ! -z ${GG_BUILD_OPENVINO} ]; then
183    if [ -z ${OpenVINO_DIR} ]; then
184        echo "OpenVINO_DIR not found, please install OpenVINO via archives and enable it by:"
185        echo "source /opt/intel/openvino/setupvars.sh"
186        exit 1
187    fi
188    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
189
190    # TODO: fix failing tests on OpenVINO backend
191    CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
192fi
193
194## helpers
195
196# download a file if it does not exist or if it is outdated
197function gg_wget {
198    local out=$1
199    local url=$2
200
201    local cwd=`pwd`
202
203    mkdir -p $out
204    cd $out
205
206    # should not re-download if file is the same
207    wget -nv -c -N $url
208
209    cd $cwd
210}
211
212function gg_run {
213    ci=$1
214
215    set -o pipefail
216    set -x
217
218    gg_run_$ci | tee $OUT/$ci.log
219    cur=$?
220
221    set +x
222    set +o pipefail
223
224    ret=$((ret | cur))
225}
226
227## ci
228
229# ctest_debug
230
231function gg_run_ctest_debug {
232    cd ${SRC}
233
234    rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
235
236    set -e
237
238    # Check required binaries are installed
239    gg_check_build_requirements
240
241    (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
242    (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
243
244    (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
245
246    set +e
247}
248
249# ctest_release
250
251function gg_run_ctest_release {
252    cd ${SRC}
253
254    rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
255
256    set -e
257
258    # Check required binaries are installed
259    gg_check_build_requirements
260
261    (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
262    (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
263
264    if [ -z ${GG_BUILD_LOW_PERF} ]; then
265        (time ctest -C Release --output-on-failure -L 'main|python' ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
266    else
267        (time ctest -C Release --output-on-failure -L main -E test-opt ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
268    fi
269
270    set +e
271}
272
273# test_llama_archs_tensor_split
274
275function gg_run_test_llama_archs_tensor_split {
276    cd ${SRC}
277
278    set -e
279
280    if [ ! -z ${GG_BUILD_CUDA} ]; then
281        GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
282        GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
283        GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
284        GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
285    fi
286
287    if [ ! -z ${GG_BUILD_METAL} ]; then
288        GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
289        GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
290        GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
291        GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
292    fi
293
294    set +e
295}
296
297# test_llama_archs_models
298
299function gg_run_test_llama_archs_models {
300    cd ${SRC}
301
302    set -e
303
304    # TODO: fix and re-enable `test-llama-archs` on OpenVINO
305    # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
306    if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
307        rm -rf build-ci-models && mkdir -p build-ci-models
308
309        # generate the dummy models used by the model-dependent tests
310        ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
311    fi
312
313    set +e
314}
315
316# test_scripts
317
318function gg_run_test_scripts {
319    cd ${SRC}
320
321    set -e
322
323    (cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
324    (cd ./tools/quantize   && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
325
326    set +e
327}
328
329function gg_get_model {
330    #local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf"
331    local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf"
332    if [[ -s $gguf_0 ]]; then
333        echo -n "$gguf_0"
334    else
335        echo >&2 "No model found. Can't run gg_run_ctest_with_model."
336        exit 1
337    fi
338}
339
340function gg_run_ctest_with_model_debug {
341    cd ${SRC}
342
343    local model; model=$(gg_get_model)
344    cd build-ci-debug
345    set -e
346
347    (LLAMACPP_TEST_MODELFILE="$model" time ctest -C Debug --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
348
349    set +e
350    cd ..
351}
352
353function gg_run_ctest_with_model_release {
354    cd ${SRC}
355
356    local model; model=$(gg_get_model)
357    cd build-ci-release
358    set -e
359
360    (LLAMACPP_TEST_MODELFILE="$model" time ctest -C Release --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
361
362    # test memory leaks
363    #if [[ ! -z ${GG_BUILD_METAL} ]]; then
364    #    # TODO: this hangs for some reason ...
365    #    (time leaks -quiet -atExit -- ./bin/test-thread-safety -m $model --parallel 2 -t 2 -p "hello") 2>&1 | tee -a $OUT/${ci}-leaks.log
366    #fi
367
368    set +e
369    cd ..
370}
371
372# qwen3_0_6b
373
374function gg_run_qwen3_0_6b {
375    cd ${SRC}
376
377    gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/config.json
378    gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/tokenizer.json
379    gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/tokenizer_config.json
380   #gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/special_tokens_map.json
381    gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/resolve/main/model.safetensors
382
383
384    gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
385    unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
386
387    path_models="../models-mnt/qwen3/0.6B"
388    path_wiki="../models-mnt/wikitext/wikitext-2-raw"
389
390    rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
391
392    set -e
393
394    (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
395    (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
396
397    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf  --outtype f16
398    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-bf16.gguf --outtype bf16
399
400    model_f16="${path_models}/ggml-model-f16.gguf"
401    model_bf16="${path_models}/ggml-model-bf16.gguf"
402    model_q8_0="${path_models}/ggml-model-q8_0.gguf"
403    model_q4_0="${path_models}/ggml-model-q4_0.gguf"
404    model_q4_1="${path_models}/ggml-model-q4_1.gguf"
405    model_q5_0="${path_models}/ggml-model-q5_0.gguf"
406    model_q5_1="${path_models}/ggml-model-q5_1.gguf"
407    model_q2_k="${path_models}/ggml-model-q2_k.gguf"
408    model_q3_k="${path_models}/ggml-model-q3_k.gguf"
409    model_q4_k="${path_models}/ggml-model-q4_k.gguf"
410    model_q5_k="${path_models}/ggml-model-q5_k.gguf"
411    model_q6_k="${path_models}/ggml-model-q6_k.gguf"
412
413    wiki_test="${path_wiki}/wiki.test.raw"
414
415    ./bin/llama-quantize ${model_bf16} ${model_q8_0} q8_0 $(nproc)
416    ./bin/llama-quantize ${model_bf16} ${model_q4_0} q4_0 $(nproc)
417    ./bin/llama-quantize ${model_bf16} ${model_q4_1} q4_1 $(nproc)
418    ./bin/llama-quantize ${model_bf16} ${model_q5_0} q5_0 $(nproc)
419    ./bin/llama-quantize ${model_bf16} ${model_q5_1} q5_1 $(nproc)
420    ./bin/llama-quantize ${model_bf16} ${model_q2_k} q2_k $(nproc)
421    ./bin/llama-quantize ${model_bf16} ${model_q3_k} q3_k $(nproc)
422    ./bin/llama-quantize ${model_bf16} ${model_q4_k} q4_k $(nproc)
423    ./bin/llama-quantize ${model_bf16} ${model_q5_k} q5_k $(nproc)
424    ./bin/llama-quantize ${model_bf16} ${model_q6_k} q6_k $(nproc)
425
426    (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
427
428    (time ./bin/llama-completion -no-cnv --model ${model_f16}  -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
429    (time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
430    (time ./bin/llama-completion -no-cnv --model ${model_q8_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
431    (time ./bin/llama-completion -no-cnv --model ${model_q4_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
432    (time ./bin/llama-completion -no-cnv --model ${model_q4_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
433    (time ./bin/llama-completion -no-cnv --model ${model_q5_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
434    (time ./bin/llama-completion -no-cnv --model ${model_q5_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
435    (time ./bin/llama-completion -no-cnv --model ${model_q2_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
436    (time ./bin/llama-completion -no-cnv --model ${model_q3_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
437    (time ./bin/llama-completion -no-cnv --model ${model_q4_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
438    (time ./bin/llama-completion -no-cnv --model ${model_q5_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
439    (time ./bin/llama-completion -no-cnv --model ${model_q6_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
440
441    (time ./bin/llama-perplexity --model ${model_f16}  -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
442    if [ -z ${GG_BUILD_NO_BF16} ]; then
443        (time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
444    fi
445    (time ./bin/llama-perplexity --model ${model_q8_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
446    (time ./bin/llama-perplexity --model ${model_q4_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
447    (time ./bin/llama-perplexity --model ${model_q4_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
448    (time ./bin/llama-perplexity --model ${model_q5_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
449    (time ./bin/llama-perplexity --model ${model_q5_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
450    (time ./bin/llama-perplexity --model ${model_q2_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
451    (time ./bin/llama-perplexity --model ${model_q3_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
452    (time ./bin/llama-perplexity --model ${model_q4_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
453    (time ./bin/llama-perplexity --model ${model_q5_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
454    (time ./bin/llama-perplexity --model ${model_q6_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
455
456    (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
457
458    (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
459    (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
460    (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
461    (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
462
463    function check_ppl {
464        qnt="$1"
465        ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
466
467        if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
468            printf '  - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
469            return 20
470        fi
471
472        printf '  - %s @ %s OK\n' "$qnt" "$ppl"
473        return 0
474    }
475
476    check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")"
477    if [ -z ${GG_BUILD_NO_BF16} ]; then
478        check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
479    fi
480    check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
481    check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
482    check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
483    check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
484    check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
485   #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
486    check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
487    check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
488    check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
489    check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
490
491    set +e
492}
493
494# bge-small
495
496function gg_run_embd_bge_small {
497    cd ${SRC}
498
499    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
500    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer.json
501    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer_config.json
502    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/special_tokens_map.json
503    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/pytorch_model.bin
504    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/sentence_bert_config.json
505    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/vocab.txt
506    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/modules.json
507    gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
508
509    gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json
510
511    path_models="../models-mnt/bge-small"
512
513    rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
514
515    set -e
516
517    (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
518    (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
519
520    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
521
522    model_f16="${path_models}/ggml-model-f16.gguf"
523    model_q8_0="${path_models}/ggml-model-q8_0.gguf"
524
525    ./bin/llama-quantize ${model_f16} ${model_q8_0} q8_0
526
527    (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
528
529    (time ./bin/llama-embedding --model ${model_f16}  -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
530    (time ./bin/llama-embedding --model ${model_q8_0} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
531
532    set +e
533}
534
535# rerank_tiny
536
537function gg_run_rerank_tiny {
538    cd ${SRC}
539
540    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/config.json
541    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/tokenizer.json
542    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/tokenizer_config.json
543    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/special_tokens_map.json
544    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/resolve/main/pytorch_model.bin
545    gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/vocab.json
546
547    path_models="../models-mnt/rerank-tiny"
548
549    rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
550
551    set -e
552
553    (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
554    (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
555
556    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
557
558    model_f16="${path_models}/ggml-model-f16.gguf"
559
560    (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
561
562    # for this model, the SEP token is "</s>"
563    (time ./bin/llama-embedding --model ${model_f16} -p "what is panda?\thi\nwhat is panda?\tit's a bear\nwhat is panda?\tThe giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China." -ngl 99 -c 0 --pooling rank --embd-normalize -1 --no-op-offload --verbose-prompt) 2>&1 | tee -a $OUT/${ci}-rk-f16.log
564
565    # sample output
566    # rerank score 0:    0.029
567    # rerank score 1:    0.029
568    # rerank score 2:    0.135
569
570    # check that the score is in the range [$3, $4]
571    function check_score {
572        qnt="$1"
573        score=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
574
575        if [ $(echo "$score < $3" | bc) -eq 1 ] || [ $(echo "$score > $4" | bc) -eq 1 ]; then
576            printf '  - %s @ %s (FAIL: score not in range [%s, %s])\n' "$qnt" "$score" "$3" "$4"
577            return 20
578        fi
579
580        printf '  - %s @ %s OK\n' "$qnt" "$score"
581        return 0
582    }
583
584    check_score "rerank score 0" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 0")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
585    check_score "rerank score 1" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 1")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
586    check_score "rerank score 2" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 2")" "0.10" "0.30" | tee -a $OUT/${ci}-rk-f16.log
587
588    set +e
589}
590
591function gg_check_build_requirements {
592    if ! command -v git &> /dev/null; then
593        echo 'git not found, please install'
594        exit 1
595    fi
596
597    if ! command -v git-lfs &> /dev/null; then
598        echo 'git-lfs not found, please install'
599        exit 1
600    fi
601
602    if ! git config --get filter.lfs.clean &> /dev/null; then
603        echo 'git-lfs not initialized, please run `git lfs install`'
604        exit 1
605    fi
606
607    if ! command -v wget &> /dev/null; then
608        echo 'wget not found, please install'
609        exit 1
610    fi
611
612    if ! command -v python3 &> /dev/null; then
613        echo 'python3 not found, please install'
614        exit 1
615    fi
616
617    if ! command -v pip3 &> /dev/null; then
618        echo 'pip3 not found, please install'
619        exit 1
620    fi
621
622    if ! python3 -m ensurepip --help &> /dev/null; then
623        echo 'ensurepip not found, please install python3-venv package'
624        exit 1
625    fi
626
627    if ! command -v cmake &> /dev/null; then
628        echo 'cmake not found, please install'
629        exit 1
630    fi
631
632    if ! command -v ccache &> /dev/null; then
633        echo 'ccache not found, please consider installing for faster builds'
634    fi
635
636    if ! command -v ctest &> /dev/null; then
637        echo 'ctest not found, please install'
638        exit 1
639    fi
640
641    if ! command -v unzip &> /dev/null; then
642        echo 'unzip not found, please install'
643        exit 1
644    fi
645}
646
647function gg_run_test_backend_ops {
648    cd ${SRC}
649
650    cd build-ci-release
651
652    set -e
653
654    local n_jobs=$(nproc)
655    if [ "${n_jobs}" -gt 2 ]; then
656        n_jobs=2
657    fi
658    local args_extra="-j ${n_jobs}"
659
660    # TODO: fix multi-threaded for ROCm
661    #       https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
662    if [ ! -z ${GG_BUILD_ROCM} ]; then
663        args_extra=""
664    fi
665
666    # TODO: MoltenVK bug?
667    #       https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
668    if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
669        args_extra=""
670    fi
671
672    # TODO: reduce the test-backend-ops timeout to 1800s
673    if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
674        (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
675    else
676        (time timeout 3600 ./bin/test-backend-ops ${args_extra}       ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
677    fi
678
679    set +e
680}
681
682## main
683
684export LLAMA_ARG_LOG_PREFIX=1
685export LLAMA_ARG_LOG_TIMESTAMPS=1
686
687if [ -z ${GG_BUILD_LOW_PERF} ]; then
688    # Create symlink: ./llama.cpp/models-mnt -> $MNT/models
689    rm -rf ${SRC}/models-mnt
690    mnt_models=${MNT}/models
691    mkdir -p ${mnt_models}
692    ln -sfn ${mnt_models} ${SRC}/models-mnt
693
694    # Create a fresh python3 venv and enter it
695    if ! python3 -m venv "$MNT/venv"; then
696        echo "Error: Failed to create Python virtual environment at $MNT/venv."
697        exit 1
698    fi
699    source "$MNT/venv/bin/activate"
700
701    pip install -r ${SRC}/requirements.txt --disable-pip-version-check
702    pip install --editable gguf-py --disable-pip-version-check
703fi
704
705ret=0
706
707test $ret -eq 0 && gg_run ctest_debug
708test $ret -eq 0 && gg_run ctest_release
709
710test $ret -eq 0 && gg_run test_backend_ops
711
712test $ret -eq 0 && gg_run test_llama_archs_models
713test $ret -eq 0 && gg_run test_llama_archs_tensor_split
714
715if [ -z ${GG_BUILD_LOW_PERF} ]; then
716    test $ret -eq 0 && gg_run embd_bge_small
717    test $ret -eq 0 && gg_run rerank_tiny
718
719    if [ -z ${GG_BUILD_CLOUD} ] || [ ${GG_BUILD_EXTRA_TESTS_0} ]; then
720        test $ret -eq 0 && gg_run test_scripts
721    fi
722
723    test $ret -eq 0 && gg_run qwen3_0_6b
724
725    test $ret -eq 0 && gg_run ctest_with_model_debug
726    test $ret -eq 0 && gg_run ctest_with_model_release
727fi
728
729exit $ret