char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

KnightYao[Model] Support for Spark2_5ForCausalLM implementation (#27868)3ad1ba733

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.8 KiB65 linesraw
1from __future__ import annotations
2
3from collections.abc import Iterable
4from typing import TYPE_CHECKING
5
6if TYPE_CHECKING:
7    from torch import Tensor
8
9from .base import ModelBase, TextModel, gguf
10
11
12@ModelBase.register("Spark2_5ForCausalLM")
13@ModelBase.example("XHToken/Spark-X2.5-1.7B")
14class Spark2_5Model(TextModel):
15    model_arch = gguf.MODEL_ARCH.SPARK2_5
16
17    def set_gguf_parameters(self) -> None:
18        super().set_gguf_parameters()
19
20        hparams = self.hparams
21        layer_types = hparams["layer_types"]
22        if len(layer_types) != self.block_count:
23            raise ValueError(
24                f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}"
25            )
26        if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types):
27            raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}")
28        if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True:
29            raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates")
30        if hparams.get("hidden_act") != "gelu":
31            raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}")
32
33        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
34        self.gguf_writer.add_sliding_window(hparams["sliding_window"])
35        self.gguf_writer.add_sliding_window_pattern(
36            [layer_type == "sliding_attention" for layer_type in layer_types]
37        )
38
39        head_dim = hparams["head_dim"]
40        full_rope = self.rope_parameters["full_attention"]
41        swa_rope = self.rope_parameters["sliding_attention"]
42        self.gguf_writer.add_rope_dimension_count(
43            int(head_dim * float(full_rope["partial_rotary_factor"]))
44        )
45        self.gguf_writer.add_rope_dimension_count_swa(
46            int(head_dim * float(swa_rope["partial_rotary_factor"]))
47        )
48
49    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
50        if name.endswith(".self_attn.q_k_v_proj.weight"):
51            if bid is None:
52                raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}")
53            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch
54            return
55
56        if name.endswith(".self_attn.g_proj.weight"):
57            if bid is None:
58                raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}")
59            expected = self.hparams["num_attention_heads"]
60            if data_torch.shape[0] != expected:
61                raise ValueError(
62                    f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}"
63                )
64
65        yield from super().modify_tensors(data_torch, name, bid)