char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenconvert: add @ModelBase.example (#27208)fa88ae936

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.7 KiB38 linesraw
1from __future__ import annotations
2
3from .base import ModelBase, TextModel, gguf
4
5
6@ModelBase.register("OrionForCausalLM")
7@ModelBase.example("OrionStarAI/Orion-14B-Base")
8class OrionModel(TextModel):
9    model_arch = gguf.MODEL_ARCH.ORION
10
11    def set_vocab(self):
12        self._set_vocab_sentencepiece()
13
14    def set_gguf_parameters(self):
15        head_count = self.hparams["num_attention_heads"]
16        head_count_kv = self.hparams.get("num_key_value_heads", head_count)
17
18        ctx_length = 0
19        if "max_sequence_length" in self.hparams:
20            ctx_length = self.hparams["max_sequence_length"]
21        elif "max_position_embeddings" in self.hparams:
22            ctx_length = self.hparams["max_position_embeddings"]
23        elif "model_max_length" in self.hparams:
24            ctx_length = self.hparams["model_max_length"]
25        else:
26            raise ValueError("gguf: can not find ctx length parameter.")
27
28        self.gguf_writer.add_file_type(self.ftype)
29        self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
30        self.gguf_writer.add_context_length(ctx_length)
31        self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
32        self.gguf_writer.add_block_count(self.block_count)
33        self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
34        self.gguf_writer.add_head_count(head_count)
35        self.gguf_writer.add_head_count_kv(head_count_kv)
36        # note: config provides rms norm but it is actually layer norm
37        # ref:  https://huggingface.co/OrionStarAI/Orion-14B-Chat/blob/276a17221ce42beb45f66fac657a41540e71f4f5/modeling_orion.py#L570-L571
38        self.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"])