char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
16.4 KiB368 linesraw
1from __future__ import annotations
2
3import json
4
5from typing import Any, Callable, Iterable, TYPE_CHECKING
6
7if TYPE_CHECKING:
8    from torch import Tensor
9
10from .base import MmprojModel, ModelBase, gguf, logger
11
12from .qwen import Qwen3Model, Qwen3MoeModel
13from .qwenvl import Qwen25AudioModel
14
15
16@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration")
17@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-30B-A3B-Instruct", "Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-35B-A3B")
18class Qwen3VLVisionModel(MmprojModel):
19    def __init__(self, *args, **kwargs):
20        super().__init__(*args, **kwargs)
21        if self.hparams_vision is None:
22            logger.info("No vision config found, skipping vision tensor processing")
23            return
24
25        # Compute image_size if not present
26        if "image_size" not in self.hparams_vision:
27            # For Qwen3VL/Qwen3VLMoe, compute from num_position_embeddings
28            num_pos = self.hparams_vision.get("num_position_embeddings", 2304)
29            patch_size = self.hparams_vision.get("patch_size", 16)
30            # num_position_embeddings = (image_size / patch_size) ** 2
31            # So image_size = sqrt(num_position_embeddings) * patch_size
32            image_size = int(num_pos**0.5 * patch_size)
33            self.hparams_vision["image_size"] = image_size
34
35        # Rename config values for compatibility
36        self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")
37        self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")
38
39        self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0)
40        for idx in self.hparams_vision.get("deepstack_visual_indexes", []):
41            self.is_deepstack_layers[idx] = True
42
43    def set_gguf_parameters(self):
44        super().set_gguf_parameters()
45        # in case mixed modalities, the arch will be handled by subclass
46        if not self.has_audio_encoder:
47            self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3VL)
48        self.gguf_writer.add_vision_use_gelu(True)
49
50        if self.hparams_vision is not None:
51            merge_size = self.hparams_vision.get("spatial_merge_size")
52            if merge_size is not None:
53                self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))
54
55        # Use text config's rms_norm_eps for vision attention layernorm eps
56        rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6)
57        self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)
58
59        if self.is_deepstack_layers:
60            self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers)
61
62    @classmethod
63    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
64        name, gen = item
65
66        # Skip text model tensors
67        if name.startswith("lm_head."):
68            return None
69
70        # Skip MTP tensors
71        if name.startswith("mtp."):
72            return None
73
74        if name.startswith("model.visual."):
75            name = name.replace("model.visual.", "visual.", 1)
76
77        if not name.startswith("visual."):
78            return None
79
80        return super().filter_tensors((name, gen))
81
82    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
83        assert self.hparams_vision is not None
84
85        if name.startswith("visual.deepstack_merger_list."):
86            prefix, rest = name.split(".", maxsplit=3)[2:]
87            # prefix is the layer index, convert to absolute clip layer index!
88            idx = self.hparams_vision.get("deepstack_visual_indexes", [])[int(prefix)]
89            target = rest
90
91            tensor_type: gguf.MODEL_TENSOR
92            if target.startswith("norm."):
93                tensor_type = gguf.MODEL_TENSOR.V_DS_NORM
94                suffix = target.split(".", 1)[1]
95            elif target.startswith("linear_fc1."):
96                tensor_type = gguf.MODEL_TENSOR.V_DS_FC1
97                suffix = target.split(".", 1)[1]
98            elif target.startswith("linear_fc2."):
99                tensor_type = gguf.MODEL_TENSOR.V_DS_FC2
100                suffix = target.split(".", 1)[1]
101            else:
102                raise ValueError(f"Unexpected deepstack tensor: {name}")
103
104            new_name = self.format_tensor_name(tensor_type, idx, suffix=f".{suffix}")
105            yield from super().modify_tensors(data_torch, new_name, bid)
106            return
107
108        if name.startswith("visual.merger."):
109            suffix = name.split(".", 2)[2]
110            if suffix.startswith("linear_fc"):
111                fc_idx_str, tail = suffix.split(".", 1)
112                fc_num = int(fc_idx_str.replace("linear_fc", ""))
113                # Qwen3VL has linear_fc1 and linear_fc2
114                # Map to indices 0 and 2 (matching Qwen2VL which uses indices 0 and 2)
115                if fc_num == 1:
116                    fc_idx = 0
117                elif fc_num == 2:
118                    fc_idx = 2
119                else:
120                    raise ValueError(f"unexpected fc index {fc_num} in {name}")
121                new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, fc_idx, suffix=f".{tail}")
122            elif suffix.startswith("norm."):
123                new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_POST_NORM, suffix=f".{suffix.split('.', 1)[1]}")
124            else:
125                raise ValueError(f"Unexpected merger tensor: {name}")
126            yield (new_name, data_torch)
127            return
128
129        if name == "visual.patch_embed.proj.weight":
130            # split Conv3D into Conv2Ds along temporal dimension
131            c1, c2, kt, _, _ = data_torch.shape
132            del c1, c2
133            if kt != 2:
134                raise ValueError("Current implementation only supports temporal_patch_size of 2")
135            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...])
136            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])
137            return
138
139        if name == "visual.patch_embed.proj.bias":
140            # Include the bias - it's used by the C++ code
141            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".bias", data_torch)
142            return
143
144        yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
145
146
147@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
148@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
149class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):
150    has_audio_encoder = True
151    has_vision_encoder = True
152
153    def get_vision_config(self) -> dict[str, Any] | None:
154        if self.has_vision_encoder:
155            return self.global_config["thinker_config"].get("vision_config")
156        else:
157            return None
158
159    def get_audio_config(self) -> dict[str, Any] | None:
160        if self.has_audio_encoder:
161            return self.global_config["thinker_config"].get("audio_config")
162        else:
163            return None
164
165    def set_gguf_parameters(self):
166        if self.has_vision_encoder:
167            Qwen3VLVisionModel.set_gguf_parameters(self)
168            self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.QWEN3VL)
169        if self.has_audio_encoder:
170            Qwen25AudioModel.set_gguf_parameters(self)
171            self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3A)
172
173    @classmethod
174    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
175        name, gen = item
176
177        # Skip text model tensors
178        if name.startswith("lm_head."):
179            return None
180
181        # Skip MTP tensors
182        if name.startswith("mtp."):
183            return None
184
185        if name.startswith("model.visual."):
186            name = name.replace("model.visual.", "visual.", 1)
187
188        if name.startswith("thinker.audio_tower."):
189            name = name.replace("thinker.audio_tower.", "audio_tower.", 1)
190
191        if "visual." not in name and "audio_tower." not in name:
192            return None
193
194        return MmprojModel.filter_tensors((name, gen))
195
196    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
197        if "visual." in name:
198            if not self.has_vision_encoder:
199                raise ValueError(f"Model does not have vision encoder, but found tensor {name}")
200            # need to transform vision tensor naming, so that modify_tensors() logic can be used correctly
201            name = name.replace("thinker.visual.", "model.visual.")
202            if ".merger_list." in name:
203                name = name.replace(".merger_list.", ".deepstack_merger_list.")
204                name = name.replace(".ln_q", ".norm")
205                name = name.replace(".mlp.0", ".linear_fc1")
206                name = name.replace(".mlp.2", ".linear_fc2")
207            elif ".merger." in name:
208                name = name.replace(".ln_q", ".norm")
209                name = name.replace(".mlp.0", ".linear_fc1")
210                name = name.replace(".mlp.2", ".linear_fc2")
211            yield from Qwen3VLVisionModel.modify_tensors(self, data_torch, name, bid)
212        elif "audio_tower." in name:
213            if not self.has_audio_encoder:
214                raise ValueError(f"Model does not have audio encoder, but found tensor {name}")
215            if "conv2d" in name and name.endswith(".bias"):
216                # transform conv2d bias [n_embd] --> [1, 1, n_embd]
217                data_torch = data_torch.unsqueeze(-1).unsqueeze(-1)
218            yield from Qwen25AudioModel.modify_tensors(self, data_torch, name, bid)
219
220
221@ModelBase.register("Qwen3ASRForConditionalGeneration")
222@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
223class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):
224    has_audio_encoder = True
225    has_vision_encoder = False
226
227
228@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
229@ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")
230class Glm4VVisionModel(Qwen3VLVisionModel):
231    def set_gguf_parameters(self):
232        MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
233        assert self.hparams_vision is not None
234        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLM4V)
235
236        hidden_act = str(self.hparams_vision.get("hidden_act", "")).lower()
237        if hidden_act == "gelu":
238            self.gguf_writer.add_vision_use_gelu(True)
239        elif hidden_act == "silu":
240            self.gguf_writer.add_vision_use_silu(True)
241
242        rms_norm_eps = self.hparams_vision.get("rms_norm_eps", 1e-5)
243        self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)
244
245    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
246        if name.startswith("visual.merger."):
247            yield from ModelBase.modify_tensors(self, data_torch, name, bid)
248            return
249        yield from super().modify_tensors(data_torch, name, bid)
250
251
252@ModelBase.register("Qwen3VLForConditionalGeneration")
253@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct")
254class Qwen3VLTextModel(Qwen3Model):
255    model_arch = gguf.MODEL_ARCH.QWEN3VL
256
257    def set_gguf_parameters(self):
258        super().set_gguf_parameters()
259        if "thinker_config" in self.hparams:
260            vision_config = self.hparams["thinker_config"].get("vision_config", {})
261        else:
262            vision_config = self.hparams.get("vision_config", {})
263        deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))
264        self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)
265
266    @classmethod
267    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
268        name, gen = item
269
270        name = name.replace("thinker.", "")
271
272        return super().filter_tensors((name, gen))
273
274
275@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
276@ModelBase.example("Qwen/Qwen3-VL-30B-A3B-Instruct")
277class Qwen3VLMoeTextModel(Qwen3MoeModel):
278    model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
279
280    def set_gguf_parameters(self):
281        super().set_gguf_parameters()
282        vision_config = self.hparams.get("vision_config", {})
283        deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))
284        self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)
285
286    @classmethod
287    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
288        name, gen = item
289
290        name = name.replace("thinker.", "")
291
292        return super().filter_tensors((name, gen))
293
294    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
295        # Qwen3VL has transposed packed tensors, so we treat it differently from general Qwen2MoE packed tensors
296        if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):
297            mapped = f"{name}.weight" if not name.endswith(".weight") else name
298            permuted = data_torch.permute(0, 2, 1).contiguous()
299            yield from ModelBase.modify_tensors(self, permuted, mapped, bid)
300            return
301
302        if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):
303            if data_torch.ndim < 3 or data_torch.shape[-1] % 2 != 0:
304                raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")
305            split_dim = data_torch.shape[-1] // 2
306            gate = data_torch[..., :split_dim].contiguous()
307            up = data_torch[..., split_dim:].contiguous()
308            # Input gate/up: (n_expert=128, n_embd=2048, n_ff_exp=768)
309            # Want GGML ne: {n_embd, n_ff_exp, n_expert} = {2048, 768, 128}
310            # Need PyTorch: (128, 768, 2048) [reversed of GGML]
311            # So: permute(0, 2, 1): (128, 2048, 768) -> (128, 768, 2048)
312            base_name = name.removesuffix(".weight")
313            base = base_name.rsplit('.', 1)[0]
314            mapped_gate = f"{base}.gate_proj.weight"
315            mapped_up = f"{base}.up_proj.weight"
316            perm_gate = gate.permute(0, 2, 1).contiguous()
317            perm_up = up.permute(0, 2, 1).contiguous()
318            yield from ModelBase.modify_tensors(self, perm_gate, mapped_gate, bid)
319            yield from ModelBase.modify_tensors(self, perm_up, mapped_up, bid)
320            return
321
322        yield from super().modify_tensors(data_torch, name, bid)
323
324
325@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
326@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
327class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):
328    model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
329
330    def set_vocab(self):
331        super().set_vocab()
332        # correct BOS/EOS tokens
333        with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:
334            tokenizer_config = json.load(f)
335            added_tokens = tokenizer_config.get("added_tokens_decoder", {})
336            for token_id, data in added_tokens.items():
337                if data.get("content") == "<|im_end|>":
338                    self.gguf_writer.add_bos_token_id(int(token_id))
339                    self.gguf_writer.add_eos_token_id(int(token_id))
340                    break
341
342    def set_gguf_parameters(self):
343        super().set_gguf_parameters()
344        self.gguf_writer.add_num_deepstack_layers(0)
345
346
347@ModelBase.register("Qwen3ASRForConditionalGeneration")
348@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
349class Qwen3ASRTextModel(Qwen3VLTextModel):
350    model_arch = gguf.MODEL_ARCH.QWEN3VL
351
352    def set_gguf_parameters(self):
353        super().set_gguf_parameters()
354        self.gguf_writer.add_num_deepstack_layers(0)
355
356    def set_vocab(self):
357        super().set_vocab()
358        # fix chat template, use correct chatml format
359        self.gguf_writer.add_chat_template("{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}")
360        # correct BOS/EOS tokens
361        with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:
362            tokenizer_config = json.load(f)
363            added_tokens = tokenizer_config.get("added_tokens_decoder", {})
364            for token_id, data in added_tokens.items():
365                if data.get("content") == "<|im_end|>":
366                    self.gguf_writer.add_bos_token_id(int(token_id))
367                    self.gguf_writer.add_eos_token_id(int(token_id))
368                    break