char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
17.8 KiB392 linesraw
1from __future__ import annotations
2
3import json
4import math
5
6from typing import Callable, Iterable, TYPE_CHECKING
7
8import torch
9
10if TYPE_CHECKING:
11    from torch import Tensor
12
13from .base import MmprojModel, ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
14
15
16@ModelBase.register("PhiForCausalLM")
17@ModelBase.example("microsoft/phi-2")
18class Phi2Model(TextModel):
19    model_arch = gguf.MODEL_ARCH.PHI2
20
21    def set_gguf_parameters(self):
22        rot_pct = self.rope_parameters["partial_rotary_factor"]
23        n_embd = self.find_hparam(["hidden_size", "n_embd"])
24        n_head = self.find_hparam(["num_attention_heads", "n_head"])
25
26        self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"]))
27
28        self.gguf_writer.add_embedding_length(n_embd)
29        self.gguf_writer.add_feed_forward_length(4 * n_embd)
30        self.gguf_writer.add_block_count(self.block_count)
31        self.gguf_writer.add_head_count(n_head)
32        self.gguf_writer.add_head_count_kv(n_head)
33        self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))
34        self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)
35        self.gguf_writer.add_file_type(self.ftype)
36        self.gguf_writer.add_add_bos_token(False)
37
38
39@ModelBase.register("Phi3ForCausalLM", "Phi4ForCausalLMV")
40@ModelBase.example("microsoft/Phi-3-mini-4k-instruct")
41class Phi3MiniModel(TextModel):
42    model_arch = gguf.MODEL_ARCH.PHI3
43
44    def set_vocab(self):
45        # Phi-4 model uses GPT2Tokenizer
46        tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
47        if tokenizer_config_file.is_file():
48            with open(tokenizer_config_file, "r", encoding="utf-8") as f:
49                tokenizer_config_json = json.load(f)
50                tokenizer_class = tokenizer_config_json['tokenizer_class']
51                if tokenizer_class == 'GPT2Tokenizer':
52                    return self._set_vocab_gpt2()
53
54        from sentencepiece import SentencePieceProcessor
55
56        tokenizer_path = self.dir_model / 'tokenizer.model'
57
58        if not tokenizer_path.is_file():
59            raise ValueError(f'Error: Missing {tokenizer_path}')
60
61        tokenizer = SentencePieceProcessor()
62        tokenizer.LoadFromFile(str(tokenizer_path))
63
64        vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())
65
66        tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]
67        scores: list[float] = [-10000.0] * vocab_size
68        toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size
69
70        for token_id in range(tokenizer.vocab_size()):
71
72            piece = tokenizer.IdToPiece(token_id)
73            text = piece.encode("utf-8")
74            score = tokenizer.GetScore(token_id)
75
76            toktype = SentencePieceTokenTypes.NORMAL
77            if tokenizer.IsUnknown(token_id):
78                toktype = SentencePieceTokenTypes.UNKNOWN
79            elif tokenizer.IsControl(token_id):
80                toktype = SentencePieceTokenTypes.CONTROL
81            elif tokenizer.IsUnused(token_id):
82                toktype = SentencePieceTokenTypes.UNUSED
83            elif tokenizer.IsByte(token_id):
84                toktype = SentencePieceTokenTypes.BYTE
85
86            tokens[token_id] = text
87            scores[token_id] = score
88            toktypes[token_id] = toktype
89
90        added_tokens_file = self.dir_model / 'added_tokens.json'
91        if added_tokens_file.is_file():
92            with open(added_tokens_file, "r", encoding="utf-8") as f:
93                added_tokens_json = json.load(f)
94
95                for key in added_tokens_json:
96                    token_id = added_tokens_json[key]
97                    if token_id >= vocab_size:
98                        logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')
99                        continue
100
101                    tokens[token_id] = key.encode("utf-8")
102                    scores[token_id] = -1000.0
103                    toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED
104
105        tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
106        if tokenizer_config_file.is_file():
107            with open(tokenizer_config_file, "r", encoding="utf-8") as f:
108                tokenizer_config_json = json.load(f)
109                added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})
110                for token_id, foken_data in added_tokens_decoder.items():
111                    token_id = int(token_id)
112                    token = foken_data["content"].encode("utf-8")
113                    if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
114                        if tokens[token_id] != token:
115                            logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token.decode("utf-8")!r}')
116                    tokens[token_id] = token
117                    scores[token_id] = -1000.0
118                    toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED
119                    if foken_data.get("special"):
120                        toktypes[token_id] = SentencePieceTokenTypes.CONTROL
121
122        tokenizer_file = self.dir_model / 'tokenizer.json'
123        if tokenizer_file.is_file():
124            with open(tokenizer_file, "r", encoding="utf-8") as f:
125                tokenizer_json = json.load(f)
126                added_tokens = tokenizer_json.get("added_tokens", [])
127                for foken_data in added_tokens:
128                    token_id = int(foken_data["id"])
129                    token = foken_data["content"].encode("utf-8")
130                    if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
131                        if tokens[token_id] != token:
132                            logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token.decode("utf-8")!r}')
133                    tokens[token_id] = token
134                    scores[token_id] = -1000.0
135                    toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED
136                    if foken_data.get("special"):
137                        toktypes[token_id] = SentencePieceTokenTypes.CONTROL
138
139        self.gguf_writer.add_tokenizer_model("llama")
140        self.gguf_writer.add_tokenizer_pre("default")
141        self.gguf_writer.add_token_list(tokens)
142        self.gguf_writer.add_token_scores(scores)
143        self.gguf_writer.add_token_types(toktypes)
144
145        special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
146        special_vocab.add_to_gguf(self.gguf_writer)
147
148    def set_gguf_parameters(self):
149        n_embd = self.find_hparam(["hidden_size", "n_embd"])
150        n_head = self.find_hparam(["num_attention_heads", "n_head"])
151        n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])
152        rms_eps = self.find_hparam(["rms_norm_eps"])
153        max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])
154        orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"]
155        rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0)
156        rope_dims = int(rot_pct * n_embd) // n_head
157
158        self.gguf_writer.add_context_length(max_pos_embds)
159        self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds)
160        self.gguf_writer.add_embedding_length(n_embd)
161        self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"]))
162        self.gguf_writer.add_block_count(self.block_count)
163        self.gguf_writer.add_head_count(n_head)
164        self.gguf_writer.add_head_count_kv(n_head_kv)
165        self.gguf_writer.add_layer_norm_rms_eps(rms_eps)
166        self.gguf_writer.add_rope_dimension_count(rope_dims)
167        self.gguf_writer.add_rope_freq_base(self.rope_parameters.get("full_attention", self.rope_parameters)["rope_theta"])
168        self.gguf_writer.add_file_type(self.ftype)
169        sliding_window = self.hparams.get("sliding_window")
170        # use zero value of sliding_window to distinguish Phi-4 from other PHI3 models
171        if sliding_window is None:
172            sliding_window = 0
173        self.gguf_writer.add_sliding_window(sliding_window)
174
175    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
176        n_embd = self.find_hparam(["hidden_size", "n_embd"])
177        n_head = self.find_hparam(["num_attention_heads", "n_head"])
178        max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])
179        orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"]
180        rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0)
181        rope_dims = int(rot_pct * n_embd) // n_head
182
183        # write rope scaling for long context (128k) model
184        long_factors = self.rope_parameters.get('long_factor')
185        short_factors = self.rope_parameters.get('short_factor')
186        if not long_factors:
187            return
188
189        scale = max_pos_embds / orig_max_pos_embds
190
191        rope_scaling_type = self.rope_parameters.get('rope_type', '').lower()
192        if len(rope_scaling_type) == 0:
193            raise KeyError('Missing the required key rope_scaling.type')
194
195        if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':
196            attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0
197        elif rope_scaling_type == 'yarn':
198            attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0
199        else:
200            raise NotImplementedError(f'The rope scaling type {rope_scaling_type} is not supported yet')
201
202        self.gguf_writer.add_rope_scaling_attn_factors(attn_factor)
203
204        if long_factors is None or short_factors is None:
205            raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')
206
207        if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:
208            raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}. long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.')
209
210        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG), torch.tensor(long_factors, dtype=torch.float32))
211        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT), torch.tensor(short_factors, dtype=torch.float32))
212
213
214@ModelBase.register("Phi4ForCausalLMV")
215# [TAG_HF_EXAMPLE_MISSING]
216class Phi4VisionMmprojModel(MmprojModel):
217    def __init__(self, *args, **kwargs):
218        super().__init__(*args, **kwargs)
219        assert self.hparams_vision is not None
220
221        self.vision_total_layers = int(self.find_vparam(self.n_block_keys))
222        if self.vision_total_layers < 2:
223            raise ValueError(
224                f"Phi-4 vision mmproj conversion requires at least 2 vision layers, got {self.vision_total_layers}"
225            )
226
227        # Phi-4 uses SigLIP2 hidden_states[-2], so export one fewer encoder block and
228        # drop post-layernorm/head weights. This makes the GGUF runtime output match
229        # the feature map consumed by the patched siglip.cpp Phi-4 projector path.
230        self.vision_export_layers = self.vision_total_layers - 1
231        self.vision_last_layer_idx = self.vision_total_layers - 1
232
233        for key in self.n_block_keys:
234            if key in self.hparams_vision:
235                self.hparams_vision[key] = self.vision_export_layers
236                break
237
238        self.block_count = self.vision_export_layers
239        self.tensor_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.MMPROJ, self.block_count)
240
241        patch_size = self.preprocessor_config.get("patch_size")
242        if patch_size is None:
243            raise KeyError("Phi-4 vision mmproj conversion requires patch_size in preprocessor_config.json")
244
245        self.hparams_vision["patch_size"] = patch_size
246
247        pos_emb_name = next(
248            (
249                name for name in self.model_tensors
250                if name.endswith("vision_model.embeddings.position_embedding.weight")
251            ),
252            None,
253        )
254        if pos_emb_name is None:
255            raise KeyError("Phi-4 vision mmproj conversion could not find position_embedding.weight")
256
257        pos_emb_shape = self.model_tensors[pos_emb_name]().shape
258        base_grid_tokens = int(pos_emb_shape[0])
259        grid_side = math.isqrt(base_grid_tokens)
260        if grid_side * grid_side != base_grid_tokens:
261            raise ValueError(f"Unexpected Phi-4 position embedding shape: {tuple(pos_emb_shape)}")
262
263        self.hparams_vision["image_size"] = grid_side * patch_size
264
265        min_num_patches = self.preprocessor_config.get("min_num_patches", self.global_config.get("min_num_patches"))
266        max_num_patches = self.preprocessor_config.get("max_num_patches", self.global_config.get("max_num_patches"))
267        if min_num_patches is None or max_num_patches is None:
268            raise KeyError("Phi-4 vision mmproj conversion requires min_num_patches and max_num_patches")
269
270        self.min_pixels = int(min_num_patches) * patch_size * patch_size
271        self.max_pixels = int(max_num_patches) * patch_size * patch_size
272
273    def set_gguf_parameters(self):
274        super().set_gguf_parameters()
275        assert self.hparams_vision is not None
276
277        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PHI4)
278        self.gguf_writer.add_vision_min_pixels(self.min_pixels)
279        self.gguf_writer.add_vision_max_pixels(self.max_pixels)
280        self.gguf_writer.add_vision_use_gelu(True)
281        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-6))
282
283    @classmethod
284    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
285        name, gen = item
286
287        name = name.replace("model.vision_tower.vision_tower.", "vision_tower.")
288
289        if not name.startswith(("vision_tower.", "model.mm_projector.", "mm_projector.")):
290            return None
291
292        if ".vision_model.head." in name:
293            return None
294
295        if ".vision_model.post_layernorm." in name:
296            return None
297
298        return super().filter_tensors((name, gen))
299
300    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
301        if name.startswith("vision_tower."):
302            if bid is not None and bid == self.vision_last_layer_idx:
303                return
304
305            if name.endswith("vision_model.embeddings.patch_embedding.weight"):
306                assert self.hparams_vision is not None
307                if data_torch.ndim != 2:
308                    raise ValueError(f"Unexpected Phi-4 patch embedding shape: {tuple(data_torch.shape)}")
309
310                patch_area = self.hparams_vision["patch_size"] ** 2
311                in_features = data_torch.shape[1]
312                if in_features % patch_area != 0:
313                    raise ValueError(
314                        f"Phi-4 patch embedding input dim {in_features} is not divisible by patch area {patch_area}"
315                    )
316
317                num_channels = in_features // patch_area
318                patch_size = self.hparams_vision["patch_size"]
319                data_torch = data_torch.view(data_torch.shape[0], patch_size, patch_size, num_channels)
320                data_torch = data_torch.permute(0, 3, 1, 2)
321
322            yield from super().modify_tensors(data_torch, name, bid)
323            return
324
325        if name.startswith(("model.mm_projector.", "mm_projector.")):
326            local_name = name
327            local_name = local_name.replace("model.mm_projector.", "")
328            local_name = local_name.replace("mm_projector.", "")
329
330            if not (local_name.startswith("0.") or local_name.startswith("2.")):
331                return
332
333            suffix = ".bias" if local_name.endswith(".bias") else ".weight"
334            mm_idx = int(local_name.split(".", maxsplit=1)[0])
335            yield (self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, mm_idx, suffix=suffix), data_torch)
336            return
337
338        return
339
340
341@ModelBase.register("PhiMoEForCausalLM")
342@ModelBase.example("microsoft/Phi-3.5-MoE-instruct")
343class PhiMoeModel(Phi3MiniModel):
344    model_arch = gguf.MODEL_ARCH.PHIMOE
345
346    _experts: list[dict[str, Tensor]] | None = None
347
348    def set_gguf_parameters(self):
349        super().set_gguf_parameters()
350        self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))
351        self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))
352
353    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
354        # process the experts separately
355        if name.find("block_sparse_moe.experts") != -1:
356            n_experts = self.find_hparam(["num_local_experts", "num_experts"])
357            assert bid is not None
358
359            if self._experts is None:
360                self._experts = [{} for _ in range(self.block_count)]
361
362            self._experts[bid][name] = data_torch
363
364            if len(self._experts[bid]) >= n_experts * 3:
365                # merge the experts into a single 3d tensor
366                for w_name in ["w1", "w2", "w3"]:
367                    datas: list[Tensor] = []
368
369                    for xid in range(n_experts):
370                        ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{w_name}.weight"
371                        datas.append(self._experts[bid][ename])
372                        del self._experts[bid][ename]
373
374                    data_torch = torch.stack(datas, dim=0)
375
376                    merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"
377
378                    yield from super().modify_tensors(data_torch, merged_name, bid)
379                return
380            else:
381                return
382
383        yield from super().modify_tensors(data_torch, name, bid)
384
385    def prepare_tensors(self):
386        super().prepare_tensors()
387
388        if self._experts is not None:
389            # flatten `list[dict[str, Tensor]]` into `list[str]`
390            experts = [k for d in self._experts for k in d.keys()]
391            if len(experts) > 0:
392                raise ValueError(f"Unprocessed experts: {experts}")