char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Maximilian Werkmodel : add Jina Embeddings v5 Nano (partial EuroBERT) support (#19826)66287bdaa

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.1 KiB46 linesraw
1#!/usr/bin/env bash
2#
3# Usage:
4#
5#   test-tokenizer-0.sh <name> <input>
6#
7
8if [ $# -ne 2 ]; then
9    printf "Usage: $0 <name> <input>\n"
10    exit 1
11fi
12
13name=$1
14input=$2
15
16# Build using CMake if binary doesn't exist
17if [ ! -f ./build/bin/test-tokenizer-0 ]; then
18    printf "Building test-tokenizer-0 with CMake...\n"
19    cmake -B build -DLLAMA_BUILD_TESTS=ON
20    cmake --build build --target test-tokenizer-0 -j
21fi
22
23printf "Testing %s on %s ...\n" $name $input
24
25set -e
26
27printf "Tokenizing using (py)  Python AutoTokenizer ...\n"
28python3 ./tests/test-tokenizer-0.py ./models/tokenizers/$name --fname-tok $input > /tmp/test-tokenizer-0-$name-py.log 2>&1
29
30printf "Tokenizing using (cpp) llama.cpp ...\n"
31./build/bin/test-tokenizer-0 ./models/ggml-vocab-$name.gguf $input > /tmp/test-tokenizer-0-$name-cpp.log 2>&1
32
33cat /tmp/test-tokenizer-0-$name-py.log | grep "tokenized in"
34cat /tmp/test-tokenizer-0-$name-cpp.log | grep "tokenized in"
35
36set +e
37
38diff $input.tok $input.tokcpp > /dev/null 2>&1
39
40if [ $? -eq 0 ]; then
41    printf "Tokenization is correct!\n"
42else
43    diff $input.tok $input.tokcpp | head -n 32
44
45    printf "Tokenization differs!\n"
46fi