char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
9.0 KiB210 linesraw
1from __future__ import annotations
2
3from typing import Any, Callable, Iterable, TYPE_CHECKING
4
5if TYPE_CHECKING:
6    from torch import Tensor
7
8from .base import MmprojModel, ModelBase, TextModel, gguf
9
10
11@ModelBase.register("UltravoxModel")
12@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
13class UltravoxModel(TextModel):
14    model_arch = gguf.MODEL_ARCH.LLAMA # dummy
15
16    def __init__(self, *args, **kwargs):
17        super().__init__(*args, **kwargs)
18        raise NotImplementedError("Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to get the audio encoder, please use --mmproj argument")
19
20
21@ModelBase.register("GlmasrModel")
22@ModelBase.example("zai-org/GLM-ASR-Nano-2512")
23class GlmASRWhisperEncoderModel(MmprojModel):
24    has_vision_encoder = False
25    has_audio_encoder = True
26
27    def __init__(self, *args, **kwargs):
28        super().__init__(*args, **kwargs)
29        if "hidden_size" not in self.hparams and "intermediate_size" not in self.hparams:
30            self.hparams["hidden_size"] = self.hparams["d_model"]
31            self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
32            self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]
33
34    def set_gguf_parameters(self):
35        super().set_gguf_parameters()
36        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLMA)
37        self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])
38        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))
39        self.gguf_writer.add_audio_stack_factor(self.global_config["merge_factor"])
40
41    def tensor_force_quant(self, name, new_name, bid, n_dims):
42        if ".conv" in name and ".weight" in name:
43            return gguf.GGMLQuantizationType.F16
44        return super().tensor_force_quant(name, new_name, bid, n_dims)
45
46    @classmethod
47    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
48        name, gen = item
49
50        if name.startswith(("model.", "lm_head.")):
51            # skip language model tensors
52            return None
53
54        if name.startswith("audio_encoder.whisper."):
55            name = name.replace("audio_encoder.whisper.","audio_tower.")
56        if "audio_encoder.layer_norm." in name or "audio_encoder.proj." in name:
57            name = name.replace("audio_encoder.", "audio_encoder.adapting.")
58        if name.startswith("audio_encoder.adapting."):
59            name = name.replace("audio_encoder.adapting.","audio.multi_modal_projector.")
60            if ".layer_norm." in name:
61                name = name.replace(".layer_norm.", ".ln_pre.")
62            if ".0." in name:
63                name = name.replace(".0.", ".linear_1.")
64            if ".2." in name:
65                name = name.replace(".2.", ".linear_2.")
66
67        return super().filter_tensors((name, gen))
68
69    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
70        if name.startswith("audio_encoder.audio_bos_eos_token."):
71            yield from super().modify_tensors(data_torch[0], "model.vision.boi", bid)
72            yield from super().modify_tensors(data_torch[1], "model.vision.eoi", bid)
73            return
74
75        if name.startswith("audio_encoder.adapting."):
76            if ".proj." in name:
77                return
78
79        if "conv1.bias" in name or "conv2.bias" in name:
80            # transpose conv1 and conv2 bias
81            data_torch = data_torch.unsqueeze(-1)
82
83        yield from super().modify_tensors(data_torch, name, bid)
84
85
86@ModelBase.register("Qwen2AudioForConditionalGeneration")
87@ModelBase.example("Qwen/Qwen2-Audio-7B-Instruct")
88class WhisperEncoderModel(MmprojModel):
89    has_vision_encoder = False # no vision encoder
90    has_audio_encoder = True
91
92    def __init__(self, *args, **kwargs):
93        super().__init__(*args, **kwargs)
94        if "hidden_size" not in self.hparams and "intermediate_size" not in self.hparams:
95            self.hparams["hidden_size"] = self.hparams["d_model"]
96            self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
97            self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]
98
99    def set_gguf_parameters(self):
100        super().set_gguf_parameters()
101        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2A)
102        self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])
103        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))
104
105    def tensor_force_quant(self, name, new_name, bid, n_dims):
106        if ".conv" in name and ".weight" in name:
107            return gguf.GGMLQuantizationType.F16
108        return super().tensor_force_quant(name, new_name, bid, n_dims)
109
110    @classmethod
111    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
112        name, gen = item
113
114        # prevent clash naming with vision tensors
115        if name.startswith("multi_modal_projector"):
116            name = "audio." + name
117
118        return super().filter_tensors((name, gen))
119
120    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
121        if "conv1.bias" in name or "conv2.bias" in name:
122            # transpose conv1 and conv2 bias
123            data_torch = data_torch.unsqueeze(-1)
124
125        yield from super().modify_tensors(data_torch, name, bid)
126
127
128@ModelBase.register("UltravoxModel")
129@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
130class UltravoxWhisperEncoderModel(WhisperEncoderModel):
131    has_vision_encoder = False # no vision encoder
132    has_audio_encoder = True
133
134    def set_gguf_parameters(self):
135        super().set_gguf_parameters()
136        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.ULTRAVOX)
137        self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])
138
139
140@ModelBase.register("MERaLiON2ForConditionalGeneration")
141@ModelBase.example("MERaLiON/MERaLiON-2-3B")
142class MERaLiONWhisperEncoderModel(WhisperEncoderModel):
143    has_vision_encoder = False
144    has_audio_encoder = True
145
146    def get_audio_config(self) -> dict[str, Any] | None:
147        return self.global_config.get("speech_config")
148
149    def set_gguf_parameters(self):
150        super().set_gguf_parameters()
151        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MERALION)
152        self.gguf_writer.add_audio_stack_factor(self.global_config.get("speech_mlp_scale_factor", 15))
153
154    @classmethod
155    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
156        name, gen = item
157
158        if name.startswith("text_decoder."):
159            return None
160
161        if name.startswith("speech_encoder."):
162            name = name.replace("speech_encoder.", "audio_tower.")
163
164        return super().filter_tensors((name, gen))
165
166    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
167        suffix = "." + name.rsplit(".", 1)[-1]
168
169        if name.startswith("ln_speech."):
170            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MM_NORM_PRE, suffix=suffix), data_torch)
171            return
172
173        if name.startswith("speech_audio_adapter."):
174            if ".mlp_adapter.0." in name:
175                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 0, suffix=suffix), data_torch)
176            elif ".gate_proj." in name:
177                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 1, suffix=suffix), data_torch)
178            elif ".pool_proj." in name:
179                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 2, suffix=suffix), data_torch)
180            elif ".out_proj." in name:
181                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 3, suffix=suffix), data_torch)
182            return
183
184        yield from super().modify_tensors(data_torch, name, bid)
185
186
187@ModelBase.register("VoxtralForConditionalGeneration")
188@ModelBase.example("mistralai/Voxtral-Mini-3B-2507")
189class VoxtralWhisperEncoderModel(WhisperEncoderModel):
190    has_vision_encoder = False # no vision encoder
191    has_audio_encoder = True
192
193    def set_gguf_parameters(self):
194        super().set_gguf_parameters()
195        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VOXTRAL)
196        self.gguf_writer.add_audio_stack_factor(4) # == intermediate_size // hidden_size
197
198
199@ModelBase.register("AudioFlamingo3ForConditionalGeneration")
200@ModelBase.example("nvidia/audio-flamingo-3-hf")
201class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
202    def set_gguf_parameters(self):
203        super().set_gguf_parameters()
204        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSIC_FLAMINGO)
205
206    def tensor_force_quant(self, name, new_name, bid, n_dims):
207        if ".conv" in name and ".weight" in name:
208            # Was trained in BF16, being safe, avoiding quantizing to FP16
209            return gguf.GGMLQuantizationType.F32
210        return super().tensor_force_quant(name, new_name, bid, n_dims)