char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB68 linesraw
1from __future__ import annotations
2
3from typing import TYPE_CHECKING
4
5if TYPE_CHECKING:
6    from torch import Tensor
7
8from .base import ModelBase, TextModel, gguf
9
10from .deepseek import DeepseekV2Model
11from .llama import LlamaModel
12
13
14@ModelBase.register(
15    "Mistral3ForConditionalGeneration",
16    "Ministral3ForCausalLM",
17)
18@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM")
19class Mistral3Model(TextModel):
20    class Ministral3Model(LlamaModel):
21        model_arch = gguf.MODEL_ARCH.MISTRAL3
22
23        def set_gguf_parameters(self):
24            super().set_gguf_parameters()
25            rope_params = self.rope_parameters
26            if self.hparams.get("model_type") == "ministral3":
27                assert rope_params, "ministral3 must have 'rope_parameters' config"
28                assert rope_params["rope_type"] == "yarn", "ministral3 rope_type must be 'yarn'"
29                self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
30                self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])
31
32    class Mistral4Model(DeepseekV2Model):
33        model_arch = gguf.MODEL_ARCH.MISTRAL4
34        skip_mtp = False # model contains no MTP layers, so no need to skip
35        merge_expert = False # experts are already stacked as 3D
36
37        def modify_tensors(self, data_torch, name, bid):
38            if name.endswith(".down_proj") or name.endswith(".gate_up_proj"):
39                name = name + ".weight"
40            yield from super().modify_tensors(data_torch, name, bid)
41
42    model_arch = gguf.MODEL_ARCH.MISTRAL3 # unused
43    impl: TextModel
44
45    def __init__(self, *args, **kwargs):
46        super().__init__(*args, **kwargs)
47        if self.hparams.get("model_type") == "mistral4":
48            self.impl = Mistral3Model.Mistral4Model(*args, **kwargs)
49        else:
50            self.impl = Mistral3Model.Ministral3Model(*args, **kwargs)
51
52    def set_vocab(self):
53        self.impl.set_vocab()
54
55    def set_gguf_parameters(self):
56        self.impl.set_gguf_parameters()
57
58    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
59        yield from self.impl.modify_tensors(data_torch, name, bid)
60
61    def prepare_tensors(self):
62        self.impl.prepare_tensors()
63
64    def write_vocab(self):
65        self.impl.write_vocab()
66
67    def write(self):
68        self.impl.write()