char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.8 KiB91 linesraw
1from __future__ import annotations
2
3import re
4
5from typing import Iterable, TYPE_CHECKING
6
7if TYPE_CHECKING:
8    from torch import Tensor
9
10from .base import ModelBase, TextModel, gguf
11
12
13@ModelBase.register("XverseForCausalLM")
14@ModelBase.example("xverse/XVERSE-7B")
15class XverseModel(TextModel):
16    model_arch = gguf.MODEL_ARCH.XVERSE
17
18    def set_vocab(self):
19        assert (self.dir_model / "tokenizer.json").is_file()
20        dir_model = self.dir_model
21        hparams = self.hparams
22
23        tokens: list[bytes] = []
24        toktypes: list[int] = []
25
26        from transformers import AutoTokenizer
27        tokenizer = AutoTokenizer.from_pretrained(dir_model)
28        vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))  # ty: ignore[unresolved-attribute]
29        # Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
30        # because vocab_size is the count of items, and indexes start at 0.
31        max_vocab_index = max(tokenizer.get_vocab().values())  # ty: ignore[unresolved-attribute]
32        if max_vocab_index >= vocab_size:
33            raise ValueError("Vocabulary size exceeds expected maximum size.")
34
35        reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
36        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
37
38        for token_id in range(vocab_size):
39            token_text = reverse_vocab[token_id].encode('utf-8')
40            # replace "\x00" to string with length > 0
41            if token_text == b"\x00":
42                toktype = gguf.TokenType.BYTE  # special
43                token_text = f"<{token_text}>".encode('utf-8')
44            elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):
45                toktype = gguf.TokenType.BYTE  # special
46            elif reverse_vocab[token_id] in added_vocab:
47                if tokenizer.added_tokens_decoder[token_id].special:  # ty: ignore[unresolved-attribute]
48                    toktype = gguf.TokenType.CONTROL
49                else:
50                    toktype = gguf.TokenType.USER_DEFINED
51            else:
52                toktype = gguf.TokenType.NORMAL
53
54            tokens.append(token_text)
55            toktypes.append(toktype)
56
57        self.gguf_writer.add_tokenizer_model("llama")
58        self.gguf_writer.add_tokenizer_pre("default")
59        self.gguf_writer.add_token_list(tokens)
60        self.gguf_writer.add_token_types(toktypes)
61
62        special_vocab = gguf.SpecialVocab(dir_model, n_vocab=len(tokens))
63        special_vocab.add_to_gguf(self.gguf_writer)
64
65    def set_gguf_parameters(self):
66        super().set_gguf_parameters()
67
68        self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
69        self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])
70
71    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
72        head_count = self.hparams["num_attention_heads"]
73        head_count_kv = self.hparams.get("num_key_value_heads", head_count)
74
75        # HF models permute some of the tensors, so we need to undo that
76        if name.endswith("q_proj.weight"):
77            data_torch = self._reverse_hf_permute(data_torch, head_count, head_count)
78        if name.endswith("k_proj.weight"):
79            data_torch = self._reverse_hf_permute(data_torch, head_count, head_count_kv)
80
81        yield from super().modify_tensors(data_torch, name, bid)
82
83    def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:
84        if n_kv_head is not None and n_head != n_kv_head:
85            n_head //= n_kv_head
86
87        return (
88            weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
89            .swapaxes(1, 2)
90            .reshape(weights.shape)
91        )