char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: add batching for mtmd-cli, add video tests (#24778)a6b3260a4

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.8 KiB233 linesraw
1#!/usr/bin/env bash
2
3# make sure we are in the right directory
4SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
5cd $SCRIPT_DIR
6
7#export LLAMA_CACHE="$SCRIPT_DIR/tmp"
8
9set -eux
10
11mkdir -p $SCRIPT_DIR/output
12
13PROJ_ROOT="$SCRIPT_DIR/../.."
14cd $PROJ_ROOT
15
16export MTMD_TEST_RESPONSE_MARKER="<MTMD_TEST_RESPONSE_MARKER>"
17
18# Check if the first argument is "big", then run test with big models
19# This is useful if we're running the script on a larger machine, so we can test the big models
20RUN_BIG_TESTS=false
21if [ "${1:-}" = "big" ]; then
22    RUN_BIG_TESTS=true
23    echo "Include BIG models..."
24fi
25
26RUN_HUGE_TESTS=false
27if [ "${1:-}" = "huge" ]; then
28    RUN_HUGE_TESTS=true
29    RUN_BIG_TESTS=true
30    echo "Include BIG and HUGE models..."
31fi
32
33USE_VIDEO=false
34if [ "${1:-}" = "video" ]; then
35    USE_VIDEO=true
36    echo "Using video as input..."
37    # behavior of USE_VIDEO:
38    # do NOT check if the output contains "new york", only verify if the exit code is 0
39    # when printing the result, print the OK/FAIL line then print the generated text
40fi
41
42# Check if the second argument is "flash", then enable flash attention
43# This is useful to test if flash attention off works correctly
44FLASH_ATTN="on"
45if [ "${2:-}" = "flash_off" ] || [ "${1:-}" = "flash_off" ]; then
46    FLASH_ATTN="off"
47    echo "Flash attention disabled..."
48fi
49
50###############
51
52arr_prefix=()
53arr_hf=()
54arr_extra_args=()
55arr_file=()
56
57add_test_vision() {
58    local hf=$1
59    shift
60    local extra_args=""
61    if [ $# -gt 0 ]; then
62        extra_args=$(printf " %q" "$@")
63    fi
64    if [ "$USE_VIDEO" = true ]; then
65        arr_file+=("test-3.mp4")
66    else
67        arr_file+=("test-1.jpeg")
68    fi
69    arr_prefix+=("[vision]")
70    arr_hf+=("$hf")
71    arr_extra_args+=("$extra_args")
72}
73
74add_test_audio() {
75    if [ "$USE_VIDEO" = true ]; then
76        return 0
77    fi
78    local hf=$1
79    shift
80    local extra_args=""
81    if [ $# -gt 0 ]; then
82        extra_args=$(printf " %q" "$@")
83    fi
84    arr_prefix+=("[audio] ")
85    arr_hf+=("$hf")
86    arr_extra_args+=("$extra_args")
87    arr_file+=("test-2.mp3")
88}
89
90add_test_vision "ggml-org/SmolVLM-500M-Instruct-GGUF:Q8_0"
91add_test_vision "ggml-org/SmolVLM2-2.2B-Instruct-GGUF:Q4_K_M"
92add_test_vision "ggml-org/SmolVLM2-500M-Video-Instruct-GGUF:Q8_0"
93add_test_vision "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M"
94add_test_vision "THUDM/glm-edge-v-5b-gguf:Q4_K_M" -p "name of the newspaper?<__media__>"
95add_test_vision "second-state/Llava-v1.5-7B-GGUF:Q2_K" --chat-template vicuna
96add_test_vision "cjpais/llava-1.6-mistral-7b-gguf:Q3_K_M" --chat-template vicuna
97add_test_vision "ibm-research/granite-vision-3.2-2b-GGUF:Q4_K_M"
98add_test_vision "second-state/MiniCPM-Llama3-V-2_5-GGUF:Q2_K"  # model from openbmb is corrupted
99add_test_vision "openbmb/MiniCPM-V-2_6-gguf:Q2_K"
100add_test_vision "openbmb/MiniCPM-o-2_6-gguf:Q4_0"
101add_test_vision "bartowski/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"
102add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"
103add_test_vision "ggml-org/InternVL2_5-1B-GGUF:Q8_0"
104add_test_vision "ggml-org/InternVL3-1B-Instruct-GGUF:Q8_0"
105add_test_vision "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"
106add_test_vision "ggml-org/LFM2-VL-450M-GGUF:Q8_0"
107add_test_vision "ggml-org/granite-docling-258M-GGUF:Q8_0"
108add_test_vision "ggml-org/LightOnOCR-1B-1025-GGUF:Q8_0"
109add_test_vision "ggml-org/DeepSeek-OCR-GGUF:Q8_0" -p "Free OCR." --chat-template deepseek-ocr
110add_test_vision "ggml-org/dots.ocr-GGUF:Q8_0" -p "OCR"
111add_test_vision "ggml-org/HunyuanOCR-GGUF:Q8_0" -p "OCR"
112add_test_vision "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja
113
114add_test_audio  "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF:Q8_0"
115add_test_audio  "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"
116add_test_audio  "ggml-org/Voxtral-Mini-3B-2507-GGUF:Q4_K_M"
117add_test_audio  "ggml-org/LFM2-Audio-1.5B-GGUF:Q8_0"
118add_test_audio  "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja
119add_test_audio  "ggml-org/Qwen3-ASR-0.6B-GGUF:Q8_0"
120
121# to test the big models, run: ./tests.sh big
122if [ "$RUN_BIG_TESTS" = true ]; then
123    add_test_vision "ggml-org/pixtral-12b-GGUF:Q4_K_M"
124    add_test_vision "ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF" --chat-template mistral-v7
125    add_test_vision "ggml-org/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"
126    add_test_vision "ggml-org/Qwen2-VL-7B-Instruct-GGUF:Q4_K_M"
127    add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"
128    add_test_vision "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M"
129    add_test_vision "ggml-org/Qwen3-VL-2B-Instruct-GGUF:Q8_0"
130    add_test_vision "ggml-org/InternVL3-8B-Instruct-GGUF:Q4_K_M"
131    add_test_vision "ggml-org/InternVL3-14B-Instruct-GGUF:Q4_K_M"
132    add_test_vision "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"
133    # add_test_vision "ggml-org/Qwen2.5-VL-32B-Instruct-GGUF:Q4_K_M" # does not work on my mac M3 Ultra
134    # add_test_vision "ggml-org/Kimi-VL-A3B-Thinking-2506-GGUF:Q4_K_M" # not always working
135    add_test_vision "ggml-org/GLM-4.6V-Flash-GGUF:Q4_K_M" -p "extract all texts from this image"
136
137    add_test_audio  "ggml-org/ultravox-v0_5-llama-3_1-8b-GGUF:Q4_K_M"
138    add_test_audio  "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"
139fi
140
141# to test the huge models, run: ./tests.sh huge
142# this will run both the big and huge models
143# huge models are > 32B parameters
144if [ "$RUN_HUGE_TESTS" = true ]; then
145    add_test_vision "ggml-org/Qwen2.5-VL-72B-Instruct-GGUF:Q4_K_M"
146    add_test_vision "ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:IQ1_S"
147fi
148
149# these models always give the wrong answer, not sure why
150# add_test_vision "ggml-org/SmolVLM-Instruct-GGUF:Q4_K_M"
151# add_test_vision "ggml-org/SmolVLM-256M-Instruct-GGUF:Q8_0"
152# add_test_vision "ggml-org/SmolVLM2-256M-Video-Instruct-GGUF:Q8_0"
153
154# this model has broken chat template, not usable
155# add_test_vision "cmp-nct/Yi-VL-6B-GGUF:Q5_K"
156# add_test_vision "guinmoon/MobileVLM-3B-GGUF:Q4_K_M" "deepseek"
157
158###############
159
160cmake --build build -j --target llama-mtmd-cli
161
162arr_res=()
163
164for i in "${!arr_hf[@]}"; do
165    bin="llama-mtmd-cli"
166    prefix="${arr_prefix[$i]}"
167    hf="${arr_hf[$i]}"
168    extra_args="${arr_extra_args[$i]}"
169    inp_file="${arr_file[$i]}"
170
171    echo "Running test with binary: $bin and HF model: $hf"
172    echo ""
173    echo ""
174
175    cmd="$(printf %q "$PROJ_ROOT/build/bin/$bin") \
176        -hf $(printf %q "$hf") \
177        --image $(printf %q "$SCRIPT_DIR/$inp_file") \
178        --temp 0 -n 128 \
179        --flash-attn $(printf %q "$FLASH_ATTN") \
180        ${extra_args}"
181
182    # if extra_args does not contain -p, we add a default prompt
183    if ! [[ "$extra_args" =~ "-p" ]]; then
184        cmd+=" -p \"what is the publisher name of the newspaper?\""
185    fi
186
187    exit_code=0
188    output=$(eval "$cmd" 2>&1 | tee /dev/tty) || exit_code=$?
189
190    echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log
191
192    if [ "$USE_VIDEO" = true ]; then
193        # for video, only check exit code; do not grep for "new york"
194        if [ $exit_code -eq 0 ]; then
195            result="$prefix \033[32mOK\033[0m:   $hf"
196        else
197            result="$prefix \033[31mFAIL\033[0m: $hf"
198        fi
199        # append generated text (after the response marker)
200        generated_text=$(echo "$output" | sed "1,/${MTMD_TEST_RESPONSE_MARKER}/d" | tail -10)
201        if [ -n "$generated_text" ]; then
202            result+="\n$generated_text"
203        fi
204        echo -e "$result"
205    else
206        # either contains "new york" or both "men" and "walk"
207        if echo "$output" | grep -iq "new york" \
208                || (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk")
209        then
210            result="$prefix \033[32mOK\033[0m:   $hf"
211        else
212            result="$prefix \033[31mFAIL\033[0m: $hf"
213        fi
214        echo -e "$result"
215    fi
216    arr_res+=("$result")
217
218    echo ""
219    echo ""
220    echo ""
221    echo "#################################################"
222    echo "#################################################"
223    echo ""
224    echo ""
225done
226
227set +x
228
229for i in "${!arr_res[@]}"; do
230    echo -e "${arr_res[$i]}"
231done
232echo ""
233echo "Output logs are saved in $SCRIPT_DIR/output"