char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

KnightYao[Model] Support for Spark2_5ForCausalLM implementation (#27868)3ad1ba733

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
27.9 KiB500 linesraw
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4import logging
5import os
6import pathlib
7import re
8
9import requests
10import json
11import shutil
12import argparse
13
14from hashlib import sha256
15from enum import IntEnum, auto
16from transformers import AutoTokenizer
17
18logging.basicConfig(level=logging.DEBUG)
19logger = logging.getLogger("convert_hf_to_gguf_update")
20sess = requests.Session()
21
22convert_py_pth = pathlib.Path("conversion/base.py")
23convert_py = convert_py_pth.read_text(encoding="utf-8")
24hf_token_pth = pathlib.Path.home() / ".cache" / "huggingface" / "token"
25hf_token = hf_token_pth.read_text(encoding="utf-8").strip() if hf_token_pth.exists() else None
26
27
28class TOKENIZER_TYPE(IntEnum):
29    SPM = auto()
30    BPE = auto()
31    WPM = auto()
32    UGM = auto()
33
34
35DOC_STRING = """
36This script downloads the tokenizer models of the specified models from Huggingface and
37generates the get_vocab_base_pre() function for convert_hf_to_gguf.py
38
39/!\\ It is intended to be used by contributors and is not meant to be run by end users
40
41This is necessary in order to analyze the type of pre-tokenizer used by the model and
42provide the necessary information to llama.cpp via the GGUF header in order to implement
43the same pre-tokenizer.
44
45ref: https://github.com/ggml-org/llama.cpp/pull/6920
46
47Instructions:
48
49- Add a new model to the "models" list
50- Run the script with your huggingface token
51    By default, token will be read from ~/.cache/huggingface/token
52- The convert_hf_to_gguf.py script will have had its get_vocab_base_pre() function updated
53- Update llama.cpp with the new pre-tokenizer if necessary
54"""
55# TODO: generate tokenizer tests for llama.cpp
56
57parser = argparse.ArgumentParser(description=DOC_STRING, formatter_class=argparse.RawTextHelpFormatter)
58parser.add_argument(
59    "--full", action="store_true",
60    help="download full list of models - make sure you have access to all of them",
61)
62parser.add_argument(
63    "--check-missing", action="store_true",
64    help="only check for missing pre-tokenizer hashes",
65)
66parser.add_argument(
67    "hf_token",
68    help="optional HF token",
69    nargs="?",
70)
71args = parser.parse_args()
72hf_token = args.hf_token if args.hf_token is not None else hf_token
73
74if hf_token is None:
75    logger.warning("HF token not found. You can provide it as an argument or set it in ~/.cache/huggingface/token")
76
77if args.check_missing and args.full:
78    logger.warning("Downloading full list of models requested, ignoring --check-missing!")
79    args.check_missing = False
80
81# TODO: this string has to exercise as much pre-tokenizer functionality as possible
82#       will be updated with time - contributions welcome
83CHK_TXT = '\n \n\n \n\n\n \t \t\t \t\n  \n   \n    \n     \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'
84
85# TODO: add models here, base models preferred
86models = [
87    {"name": "llama-spm",        "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
88    {"name": "llama-bpe",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
89    {"name": "phi-3",            "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
90    {"name": "deepseek-llm",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
91    {"name": "deepseek-coder",   "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
92    {"name": "falcon",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
93    {"name": "bert-bge",         "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
94    {"name": "falcon3",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon3-7B-Base", },
95    {"name": "bert-bge-large",   "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/BAAI/bge-large-zh-v1.5", },
96    {"name": "mpt",              "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
97    {"name": "starcoder",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
98    {"name": "gpt-2",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
99    {"name": "stablelm2",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", },
100    {"name": "refact",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
101    {"name": "command-r",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
102    {"name": "tiny_aya",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },
103    {"name": "cohere2moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", },
104    {"name": "qwen2",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
105    {"name": "olmo",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
106    {"name": "dbrx",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
107    {"name": "jina-v1-en",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-reranker-v1-tiny-en", },
108    {"name": "jina-v2-en",       "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM!
109    {"name": "jina-v2-es",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", },
110    {"name": "jina-v2-de",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", },
111    {"name": "jina-v5-nano",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v5-text-nano", },
112    {"name": "smaug-bpe",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct", },
113    {"name": "poro-chat",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Poro-34B-chat", },
114    {"name": "jina-v2-code",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-code", },
115    {"name": "viking",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B", }, # Also used for Viking 13B and 33B
116    {"name": "gemma",            "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2b", },
117    {"name": "gemma-2",          "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", },
118    {"name": "jais",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", },
119    {"name": "jais-2",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inceptionai/Jais-2-8B-Chat", },
120    {"name": "t5",               "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },
121    {"name": "codeshell",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/WisdomShell/CodeShell-7B", },
122    {"name": "tekken",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mistralai/Mistral-Nemo-Base-2407", },
123    {"name": "smollm",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/HuggingFaceTB/SmolLM-135M", },
124    {'name': "bloom",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigscience/bloom", },
125    {'name': "gpt3-finnish",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/TurkuNLP/gpt3-finnish-small", },
126    {"name": "exaone",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", },
127    {"name": "phi-2",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/microsoft/phi-2", },
128    {"name": "chameleon",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/facebook/chameleon-7b", },
129    {"name": "roberta-bpe",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sentence-transformers/stsb-roberta-base"},
130    {"name": "gigachat",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct"},
131    {"name": "megrez",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Infinigence/Megrez-3B-Instruct"},
132    {"name": "deepseek-v3",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3"},
133    {"name": "deepseek-r1-qwen", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"},
134    {"name": "gpt-4o",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Xenova/gpt-4o", },
135    {"name": "superbpe",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/UW/OLMo2-8B-SuperBPE-t180k", },
136    {"name": "trillion",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/trillionlabs/Trillion-7B-preview", },
137    {"name": "bailingmoe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/Ling-lite", },
138    {"name": "llama4",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", },
139    {"name": "pixtral",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mistral-community/pixtral-12b", },
140    {"name": "seed-coder",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ByteDance-Seed/Seed-Coder-8B-Base", },
141    {"name": "a.x-4.0",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/skt/A.X-4.0", },
142    {"name": "midm-2.0",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/K-intelligence/Midm-2.0-Base-Instruct", },
143    {"name": "lfm2",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-350M", },
144    {"name": "exaone4",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B", },
145    {"name": "mellum",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum-4b-base", },
146    {"name": "modern-bert",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/answerdotai/ModernBERT-base", },
147    {"name": "afmoe",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/arcee-ai/Trinity-Tokenizer", },
148    {"name": "bailingmoe2",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/Ling-mini-base-2.0", },
149    {"name": "granite-docling",  "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-docling-258M", },
150    {"name": "minimax-m2",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/MiniMaxAI/MiniMax-M2", },
151    {"name": "kormo",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/KORMo-Team/KORMo-tokenizer", },
152    {"name": "youtu",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Youtu-LLM-2B", },
153    {"name": "solar-open",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/upstage/Solar-Open-100B", },
154    {"name": "exaone-moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B", },
155    {"name": "qwen35",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3.5-9B-Instruct", },
156    {"name": "joyai-llm",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jdopensource/JoyAI-LLM-Flash", },
157    {"name": "kanana2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601", },
158    {"name": "f2llmv2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/codefuse-ai/F2LLM-v2-4B", },
159    {"name": "sarvam-moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sarvamai/sarvam-30b", },
160    {"name": "talkie",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/lewtun/talkie-1930-13b-it-hf", },
161    {"name": "minicpm5",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM5-1B"},
162    {"name": "granite-embed-multi-97m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-97m-multilingual-r2", },
163    {"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", },
164    {"name": "mellum2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"},
165    {"name": "laguna",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", },
166]
167
168# some models are known to be broken upstream, so we will skip them as exceptions
169pre_computed_hashes = [
170    # chatglm-bpe has 2 hashes, why?
171    {"name": "chatglm-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-chat", "chkhsh": "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b"},
172    {"name": "chatglm-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-chat", "chkhsh": "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516"},
173    {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-hf", "chkhsh": "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2"},
174    {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},
175    {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
176    {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
177    {"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
178    {"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
179    {"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},
180    # falcon-h1 series uses 4 different tokenizers across model sizes (0.5b - 34b), hence we need to define 4 different hashes
181    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base", "chkhsh": "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6"},
182    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-1B-Base", "chkhsh": "60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86"},
183    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-7B-Base", "chkhsh": "3eda48b4c4dc7de733d1a8b3e3b4a85243dbbf704da2ee9d42c6beced8897896"},
184    {"name": "falcon-h1", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon-H1-34B-Base", "chkhsh": "48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b"},
185    {"name": "kimi-k2",   "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/moonshotai/Kimi-K2-Base",   "chkhsh": "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890"},
186    {"name": "qwen2",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "chkhsh": "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c"},
187    {"name": "qwen35",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM-V-4_6", "chkhsh": "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f"},
188    {"name": "grok-2",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
189    # jina-v2-de variants
190    {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
191    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
192    # lfm2 variants
193    {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
194    {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"},
195]
196
197
198def download_file_with_auth(url, token, save_path):
199    headers = {"Authorization": f"Bearer {token}"} if token else None
200    response = sess.get(url, headers=headers)
201    response.raise_for_status()
202    os.makedirs(os.path.dirname(save_path), exist_ok=True)
203    with open(save_path, 'wb') as downloaded_file:
204        downloaded_file.write(response.content)
205    logger.info(f"File {save_path} downloaded successfully")
206
207
208def download_model(model):
209    name = model["name"]
210    repo = model["repo"]
211    tokt = model["tokt"]
212
213    os.makedirs(f"models/tokenizers/{name}", exist_ok=True)
214
215    files = ["config.json", "tokenizer.json", "tokenizer_config.json"]
216
217    if name == "gpt-4o":
218        # Xenova/gpt-4o is tokenizer-only, it does not contain config.json
219        files = ["tokenizer.json", "tokenizer_config.json"]
220
221    if tokt == TOKENIZER_TYPE.SPM:
222        files.append("tokenizer.model")
223
224    if tokt == TOKENIZER_TYPE.UGM:
225        files.append("spiece.model")
226
227    if os.path.isdir(repo):
228        # If repo is a path on the file system, copy the directory
229        for file in files:
230            src_path = os.path.join(repo, file)
231            dst_path = f"models/tokenizers/{name}/{file}"
232            if os.path.isfile(dst_path):
233                logger.info(f"{name}: File {dst_path} already exists - skipping")
234                continue
235            if os.path.isfile(src_path):
236                shutil.copy2(src_path, dst_path)
237                logger.info(f"{name}: Copied {src_path} to {dst_path}")
238            else:
239                logger.warning(f"{name}: Source file {src_path} does not exist")
240    else:
241        # If repo is a URL, download the files
242        for file in files:
243            save_path = f"models/tokenizers/{name}/{file}"
244            if os.path.isfile(save_path):
245                logger.info(f"{name}: File {save_path} already exists - skipping")
246                continue
247            download_file_with_auth(f"{repo}/resolve/main/{file}", hf_token, save_path)
248
249
250# get list of existing models and chkhsh from the convert_hf_to_gguf.py file
251# returns mapping res --> chkhsh
252def get_existing_models(convert_py):
253    pattern = r'if chkhsh == "([a-f0-9]{64})":\s*\n\s*.*\s*res = "([^"]+)"'
254    matches = re.findall(pattern, convert_py)
255    output = {}
256    for chkhsh, res in matches:
257        output[res] = chkhsh
258    return output
259
260
261existing_models = {}
262all_models = models.copy()
263if not args.full:
264    # Filter out models that already exist in convert_hf_to_gguf.py
265    existing_models = get_existing_models(convert_py)
266    all_models = models.copy()
267    models = [model for model in all_models if model["name"] not in existing_models]
268
269if not args.check_missing:
270    logging.info(f"Downloading {len(models)} models...")
271    for model in models:
272        try:
273            download_model(model)
274        except Exception as e:
275            logger.error(f"Failed to download model {model['name']}. Error: {e}")
276
277
278# generate the source code for the convert_hf_to_gguf.py:get_vocab_base_pre() function:
279
280src_ifs = ""
281for model in [*pre_computed_hashes, *all_models]:
282    name = model["name"]
283    tokt = model["tokt"]
284    chkhsh = model.get("chkhsh")
285
286    if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM:
287        continue
288
289    # create the tokenizer
290    if chkhsh is not None:
291        # if the model has a pre-computed hash, use it
292        logger.info(f"Using pre-computed hash for model {name}: {chkhsh}")
293    elif name in existing_models:
294        # if the model already exists in convert_hf_to_gguf.py, skip compute hash
295        chkhsh = existing_models[name]
296    else:
297        # otherwise, compute the hash of the tokenizer
298
299        # Fail if the tokenizer folder with config does not exist or there are other download issues previously
300        if not os.path.isfile(f"models/tokenizers/{name}/tokenizer_config.json"):
301            raise OSError(f"Config for tokenizer {name} not found. The model may not exist or is not accessible with the provided token.")
302
303        try:
304            logger.info(f"Loading tokenizer from {f'models/tokenizers/{name}'}...")
305            if name == "t5":
306                tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)
307            else:
308                tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
309        except Exception as e:
310            raise OSError(f"Error loading tokenizer for model {name}.") from e
311
312        chktok = tokenizer.encode(CHK_TXT)  # ty: ignore[unresolved-attribute]
313        chkhsh = sha256(str(chktok).encode()).hexdigest()
314
315        logger.info(f"model: {name}")
316        logger.info(f"tokt: {tokt}")
317        logger.info(f"repo: {model['repo']}")
318        logger.info(f"chktok: {chktok}")
319        logger.info(f"chkhsh: {chkhsh}")
320
321        # print the "pre_tokenizer" content from the tokenizer.json
322        with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
323            cfg = json.load(f)
324            normalizer = cfg["normalizer"]
325            logger.info("normalizer: " + json.dumps(normalizer, indent=4))
326            pre_tokenizer = cfg["pre_tokenizer"]
327            logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
328            if "ignore_merges" in cfg["model"]:
329                logger.info("ignore_merges: " + json.dumps(cfg["model"]["ignore_merges"], indent=4))
330
331        logger.info("")
332
333    src_ifs += f"        if chkhsh == \"{chkhsh}\":\n"
334    src_ifs += f"            # ref: {model['repo']}\n"
335    src_ifs += f"            res = \"{name}\"\n"
336
337src_func = f"""
338    def get_vocab_base_pre(self, tokenizer) -> str:
339        # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that
340        # is specific for the BPE pre-tokenizer used by the model
341        # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can
342        # use in llama.cpp to implement the same pre-tokenizer
343
344        chktxt = {repr(CHK_TXT)}
345
346        chktok = tokenizer.encode(chktxt)
347        chkhsh = sha256(str(chktok).encode()).hexdigest()
348
349        logger.debug(f"chktok: {{chktok}}")
350        logger.debug(f"chkhsh: {{chkhsh}}")
351
352        res = None
353
354        # NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script
355        #       or pull the latest version of the model from Huggingface
356        #       don't edit the hashes manually!
357{src_ifs}
358        if res is None:
359            logger.warning("\\n")
360            logger.warning("**************************************************************************************")
361            logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")
362            logger.warning("**          There are 2 possible reasons for this:")
363            logger.warning("**          - the model has not been added to convert_hf_to_gguf_update.py yet")
364            logger.warning("**          - the pre-tokenization config has changed upstream")
365            logger.warning("**          Check your model files and convert_hf_to_gguf_update.py and update them accordingly.")
366            logger.warning("** ref:     https://github.com/ggml-org/llama.cpp/pull/6920")
367            logger.warning("**")
368            logger.warning(f"** chkhsh:  {{chkhsh}}")
369            logger.warning("**************************************************************************************")
370            logger.warning("\\n")
371            raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
372
373        logger.debug(f"tokenizer.ggml.pre: {{repr(res)}}")
374        logger.debug(f"chkhsh: {{chkhsh}}")
375
376        return res
377"""
378
379convert_py = re.sub(
380    r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",
381    lambda m: m.group(1) + src_func + m.group(3),
382    convert_py,
383    flags=re.DOTALL | re.MULTILINE,
384)
385
386convert_py_pth.write_text(convert_py, encoding="utf-8")
387
388logger.info(f"+++ {convert_py_pth} was updated")
389
390# generate tests for each tokenizer model
391
392tests = [
393    "ied 4 ½ months",
394    "Äpfel",
395    "",
396    " ",
397    "  ",
398    "   ",
399    "\t",
400    "\n",
401    "\n\n",
402    "\n\n\n",
403    "\t\n",
404    "Hello world",
405    " Hello world",
406    "Hello World",
407    " Hello World",
408    " Hello World!",
409    "Hello, world!",
410    " Hello, world!",
411    " this is 🦙.cpp",
412    "w048 7tuijk dsdfhu",
413    "нещо на Български",
414    "កាន់តែពិសេសអាចខលចេញ",
415    "🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",
416    "Hello",
417    " Hello",
418    "  Hello",
419    "   Hello",
420    "    Hello",
421    "    Hello\n    Hello",
422    " (",
423    "\n =",
424    "' era",
425    "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～",
426    "!!!!!!",
427    "3",
428    "33",
429    "333",
430    "3333",
431    "33333",
432    "333333",
433    "3333333",
434    "33333333",
435    "333333333",
436    "Cửa Việt", # llama-bpe fails on this
437    " discards",
438    CHK_TXT,
439]
440
441# write the tests to ./models/ggml-vocab-{name}.gguf.inp
442# the format is:
443#
444# test0
445# __ggml_vocab_test__
446# test1
447# __ggml_vocab_test__
448# ...
449#
450
451# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out
452# for each test, write the resulting tokens on a separate line
453
454for model in models:
455    name = model["name"]
456    tokt = model["tokt"]
457
458    # Skip if the tokenizer folder does not exist or there are other download issues previously
459    if not os.path.exists(f"models/tokenizers/{name}"):
460        logger.warning(f"Directory for tokenizer {name} not found. Skipping...")
461        continue
462
463    # create the tokenizer
464    try:
465        if name == "t5":
466            tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)
467        else:
468            tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
469    except (OSError, TypeError) as e:
470        logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")
471        continue  # Skip this model and continue with the next one in the loop
472
473    if not os.path.exists(f"models/ggml-vocab-{name}.gguf"):
474        logger.info(f"Skip vocab files for model {name}, no GGUF file found")
475        continue
476
477    with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
478        for text in tests:
479            f.write(f"{text}")
480            f.write("\n__ggml_vocab_test__\n")
481
482    with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
483        for text in tests:
484            res = tokenizer.encode(text, add_special_tokens=False)  # ty: ignore[unresolved-attribute]
485            for r in res:
486                f.write(f" {r}")
487            f.write("\n")
488
489    logger.info(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
490
491# generate commands for creating vocab files
492
493logger.info("\nRun the following commands to generate the vocab files for testing:\n")
494
495for model in models:
496    name = model["name"]
497
498    print(f"python3 convert_hf_to_gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only") # noqa: NP100
499
500logger.info("\n")