char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.8 KiB47 linesraw
1from __future__ import annotations
2
3import json
4
5from typing import Iterable, TYPE_CHECKING
6
7if TYPE_CHECKING:
8    from torch import Tensor
9
10from .base import ModelBase, TextModel, gguf, logger
11
12
13@ModelBase.register("PanguEmbeddedForCausalLM")
14@ModelBase.example("FreedomIntelligence/openPangu-Embedded-7B-V1.1")
15class PanguEmbeddedModel(TextModel):
16    model_arch = gguf.MODEL_ARCH.PANGU_EMBED
17
18    def set_vocab(self):
19        self._set_vocab_sentencepiece()
20
21        tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
22        if tokenizer_config_file.is_file():
23            with open(tokenizer_config_file, "r", encoding="utf-8") as f:
24                tokenizer_config_json = json.load(f)
25                if "add_prefix_space" in tokenizer_config_json:
26                    self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
27
28    def set_gguf_parameters(self):
29        super().set_gguf_parameters()
30        hparams = self.hparams
31        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
32
33        # PanguEmbedded's hparam loaded from config.json without head_dim
34        if (rope_dim := hparams.get("head_dim")) is None:
35            rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
36        self.gguf_writer.add_rope_dimension_count(rope_dim)
37
38        if hparams.get("head_dim") is None:
39            self.gguf_writer.add_key_length(rope_dim)
40            self.gguf_writer.add_value_length(rope_dim)
41
42    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
43        if name == "lm_head.weight":
44            if self.hparams.get("tie_word_embeddings", False):
45                logger.info("Skipping tied output layer 'lm_head.weight'")
46                return
47        yield from super().modify_tensors(data_torch, name, bid)