char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
933 B30 linesraw
1from __future__ import annotations
2
3from typing import Callable, TYPE_CHECKING
4
5if TYPE_CHECKING:
6    from torch import Tensor
7
8from .base import ModelBase, gguf
9
10from .llava import LlavaVisionModel
11
12
13@ModelBase.register("LightOnOCRForConditionalGeneration")
14@ModelBase.example("lightonai/LightOnOCR-1B-1025")
15class LightOnOCRVisionModel(LlavaVisionModel):
16    is_mistral_format = False
17    use_break_tok = False
18
19    def set_gguf_parameters(self):
20        super().set_gguf_parameters()
21        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LIGHTONOCR)
22
23    @classmethod
24    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
25        name, gen = item
26
27        name = name.replace("model.vision_encoder.", "vision_tower.")
28        name = name.replace("model.vision_projection.", "multi_modal_projector.")
29
30        return super().filter_tensors((name, gen))