char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB48 linesraw
1from __future__ import annotations
2
3from typing import Callable, TYPE_CHECKING
4
5if TYPE_CHECKING:
6    from torch import Tensor
7
8from .base import MmprojModel, ModelBase, gguf
9
10
11@ModelBase.register("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")
12@ModelBase.example("HuggingFaceTB/SmolVLM-Instruct", "HuggingFaceM4/Idefics3-8B-Llama3")
13class SmolVLMModel(MmprojModel):
14    def __init__(self, *args, **kwargs):
15        super().__init__(*args, **kwargs)
16        if self.hparams["model_type"] == "smolvlm_vision":
17            # fix for SmolVLM2, missing some keys in config.json
18            # default values are taken from transformers code
19            self.hparams["hidden_size"] = self.hparams.get("hidden_size", 1152)
20            self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 16)
21            self.hparams["intermediate_size"] = self.hparams.get("intermediate_size", 3072)
22
23    def set_gguf_parameters(self):
24        super().set_gguf_parameters()
25        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.IDEFICS3)
26        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))
27        self.gguf_writer.add_vision_projector_scale_factor(self.global_config.get("scale_factor", 2))
28        self.gguf_writer.add_vision_use_gelu(True)
29
30        # Add the preprocessor longest edge size
31        preproc_image_size = self.preprocessor_config.get("size", {}).get("longest_edge", self.image_size)
32        self.gguf_writer.add_vision_preproc_image_size(preproc_image_size)
33
34    def tensor_force_quant(self, name, new_name, bid, n_dims):
35        if ".embeddings." in name:
36            return gguf.GGMLQuantizationType.F32
37        return super().tensor_force_quant(name, new_name, bid, n_dims)
38
39    @classmethod
40    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
41        name, gen = item
42
43        is_vision_tensor = "vision_tower" in name or "vision_model" in name or "model.connector" in name
44
45        if not is_vision_tensor:
46            return None
47
48        return super().filter_tensors(item)