char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
98a01afd9
char/aliceai-t5-moe
1#include "llama-arch.h" 2 3#include "llama-impl.h" 4 5#include <map> 6#include <vector> 7 8static const std::map < llm_arch ,const char *> LLM_ARCH_NAMES = { 9 {LLM_ARCH_CLIP ,"clip" },// dummy, only used by llama-quantize 10 {LLM_ARCH_LLAMA ,"llama" }, 11 {LLM_ARCH_LLAMA4 ,"llama4" }, 12 {LLM_ARCH_DECI ,"deci" }, 13 {LLM_ARCH_FALCON ,"falcon" }, 14 {LLM_ARCH_GROK ,"grok" }, 15 {LLM_ARCH_GPT2 ,"gpt2" }, 16 {LLM_ARCH_GPTJ ,"gptj" }, 17 {LLM_ARCH_GPTNEOX ,"gptneox" }, 18 {LLM_ARCH_MPT ,"mpt" }, 19 {LLM_ARCH_BAICHUAN ,"baichuan" }, 20 {LLM_ARCH_STARCODER ,"starcoder" }, 21 {LLM_ARCH_REFACT ,"refact" }, 22 {LLM_ARCH_BERT ,"bert" }, 23 {LLM_ARCH_MODERN_BERT ,"modern-bert" }, 24 {LLM_ARCH_NOMIC_BERT ,"nomic-bert" }, 25 {LLM_ARCH_NOMIC_BERT_MOE ,"nomic-bert-moe" }, 26 {LLM_ARCH_NEO_BERT ,"neo-bert" }, 27 {LLM_ARCH_JINA_BERT_V2 ,"jina-bert-v2" }, 28 {LLM_ARCH_JINA_BERT_V3 ,"jina-bert-v3" }, 29 {LLM_ARCH_EUROBERT ,"eurobert" }, 30 {LLM_ARCH_BLOOM ,"bloom" }, 31 {LLM_ARCH_STABLELM ,"stablelm" }, 32 {LLM_ARCH_QWEN ,"qwen" }, 33 {LLM_ARCH_QWEN2 ,"qwen2" }, 34 {LLM_ARCH_QWEN2MOE ,"qwen2moe" }, 35 {LLM_ARCH_QWEN2VL ,"qwen2vl" }, 36 {LLM_ARCH_QWEN3 ,"qwen3" }, 37 {LLM_ARCH_QWEN3MOE ,"qwen3moe" }, 38 {LLM_ARCH_QWEN3NEXT ,"qwen3next" }, 39 {LLM_ARCH_QWEN3VL ,"qwen3vl" }, 40 {LLM_ARCH_QWEN3VLMOE ,"qwen3vlmoe" }, 41 {LLM_ARCH_QWEN35 ,"qwen35" }, 42 {LLM_ARCH_QWEN35MOE ,"qwen35moe" }, 43 {LLM_ARCH_QWEN4EXP ,"qwen4exp" }, 44 {LLM_ARCH_PHI2 ,"phi2" }, 45 {LLM_ARCH_PHI3 ,"phi3" }, 46 {LLM_ARCH_PHIMOE ,"phimoe" }, 47 {LLM_ARCH_PLAMO ,"plamo" }, 48 {LLM_ARCH_PLAMO2 ,"plamo2" }, 49 {LLM_ARCH_PLAMO3 ,"plamo3" }, 50 {LLM_ARCH_CODESHELL ,"codeshell" }, 51 {LLM_ARCH_ORION ,"orion" }, 52 {LLM_ARCH_INTERNLM2 ,"internlm2" }, 53 {LLM_ARCH_MINICPM ,"minicpm" }, 54 {LLM_ARCH_MINICPM3 ,"minicpm3" }, 55 {LLM_ARCH_GEMMA ,"gemma" }, 56 {LLM_ARCH_GEMMA2 ,"gemma2" }, 57 {LLM_ARCH_GEMMA3 ,"gemma3" }, 58 {LLM_ARCH_GEMMA3N ,"gemma3n" }, 59 {LLM_ARCH_GEMMA4 ,"gemma4" }, 60 {LLM_ARCH_GEMMA4_ASSISTANT ,"gemma4-assistant" }, 61 {LLM_ARCH_GEMMA_EMBEDDING ,"gemma-embedding" }, 62 {LLM_ARCH_STARCODER2 ,"starcoder2" }, 63 {LLM_ARCH_MAMBA ,"mamba" }, 64 {LLM_ARCH_MAMBA2 ,"mamba2" }, 65 {LLM_ARCH_MAPLE ,"maple" }, 66 {LLM_ARCH_JAMBA ,"jamba" }, 67 {LLM_ARCH_FALCON_H1 ,"falcon-h1" }, 68 {LLM_ARCH_XVERSE ,"xverse" }, 69 {LLM_ARCH_COMMAND_R ,"command-r" }, 70 {LLM_ARCH_COHERE2 ,"cohere2" }, 71 {LLM_ARCH_COHERE2MOE ,"cohere2moe" }, 72 {LLM_ARCH_DBRX ,"dbrx" }, 73 {LLM_ARCH_OLMO ,"olmo" }, 74 {LLM_ARCH_OLMO2 ,"olmo2" }, 75 {LLM_ARCH_OLMOE ,"olmoe" }, 76 {LLM_ARCH_MUSE_GLIMMER ,"muse-glimmer" }, 77 {LLM_ARCH_OPENELM ,"openelm" }, 78 {LLM_ARCH_ARCTIC ,"arctic" }, 79 {LLM_ARCH_DEEPSEEK ,"deepseek" }, 80 {LLM_ARCH_DEEPSEEK2 ,"deepseek2" }, 81 {LLM_ARCH_DEEPSEEK2OCR ,"deepseek2-ocr" }, 82 {LLM_ARCH_DEEPSEEK32 ,"deepseek32" }, 83 {LLM_ARCH_DEEPSEEK4 ,"deepseek4" }, 84 {LLM_ARCH_CHATGLM ,"chatglm" }, 85 {LLM_ARCH_GLM4 ,"glm4" }, 86 {LLM_ARCH_GLM4_MOE ,"glm4moe" }, 87 {LLM_ARCH_GLM_DSA ,"glm-dsa" }, 88 {LLM_ARCH_BITNET ,"bitnet" }, 89 {LLM_ARCH_T5 ,"t5" }, 90 {LLM_ARCH_ALICEAI_T5_MOE ,"aliceai_t5_moe" }, 91 {LLM_ARCH_T5ENCODER ,"t5encoder" }, 92 {LLM_ARCH_JAIS ,"jais" }, 93 {LLM_ARCH_JAIS2 ,"jais2" }, 94 {LLM_ARCH_NEMOTRON ,"nemotron" }, 95 {LLM_ARCH_NEMOTRON_H ,"nemotron_h" }, 96 {LLM_ARCH_NEMOTRON_H_MOE ,"nemotron_h_moe" }, 97 {LLM_ARCH_EXAONE ,"exaone" }, 98 {LLM_ARCH_EXAONE4 ,"exaone4" }, 99 {LLM_ARCH_EXAONE_MOE ,"exaone-moe" }, 100 {LLM_ARCH_RWKV6 ,"rwkv6" }, 101 {LLM_ARCH_RWKV6QWEN2 ,"rwkv6qwen2" }, 102 {LLM_ARCH_RWKV7 ,"rwkv7" }, 103 {LLM_ARCH_ARWKV7 ,"arwkv7" }, 104 {LLM_ARCH_GRANITE ,"granite" }, 105 {LLM_ARCH_GRANITE_MOE ,"granitemoe" }, 106 {LLM_ARCH_GRANITE_HYBRID ,"granitehybrid" }, 107 {LLM_ARCH_GRANITE_SWITCH ,"graniteswitch" }, 108 {LLM_ARCH_GRANITE_SWA ,"granite_swa" }, 109 {LLM_ARCH_CHAMELEON ,"chameleon" }, 110 {LLM_ARCH_WAVTOKENIZER_DEC ,"wavtokenizer-dec" }, 111 {LLM_ARCH_PLM ,"plm" }, 112 {LLM_ARCH_BAILINGMOE ,"bailingmoe" }, 113 {LLM_ARCH_BAILINGMOE2 ,"bailingmoe2" }, 114 {LLM_ARCH_BAILINGMOE3 ,"bailingmoe3" }, 115 {LLM_ARCH_DOTS1 ,"dots1" }, 116 {LLM_ARCH_DOTS3NOTE ,"dots3note" }, 117 {LLM_ARCH_ARCEE ,"arcee" }, 118 {LLM_ARCH_AFMOE ,"afmoe" }, 119 {LLM_ARCH_LAGUNA ,"laguna" }, 120 {LLM_ARCH_ERNIE4_5 ,"ernie4_5" }, 121 {LLM_ARCH_ERNIE4_5_MOE ,"ernie4_5-moe" }, 122 {LLM_ARCH_HUNYUAN_MOE ,"hunyuan-moe" }, 123 {LLM_ARCH_HUNYUAN_DENSE ,"hunyuan-dense" }, 124 {LLM_ARCH_HUNYUAN_VL ,"hunyuan_vl" }, 125 {LLM_ARCH_HY_V3 ,"hy_v3" }, 126 {LLM_ARCH_HY_V4 ,"hy_v4" }, 127 {LLM_ARCH_SMOLLM3 ,"smollm3" }, 128 {LLM_ARCH_OPENAI_MOE ,"gpt-oss" }, 129 {LLM_ARCH_LFM2 ,"lfm2" }, 130 {LLM_ARCH_LFM2MOE ,"lfm2moe" }, 131 {LLM_ARCH_DREAM ,"dream" }, 132 {LLM_ARCH_SMALLTHINKER ,"smallthinker" }, 133 {LLM_ARCH_LLADA ,"llada" }, 134 {LLM_ARCH_LLADA_MOE ,"llada-moe" }, 135 {LLM_ARCH_SEED_OSS ,"seed_oss" }, 136 {LLM_ARCH_GROVEMOE ,"grovemoe" }, 137 {LLM_ARCH_APERTUS ,"apertus" }, 138 {LLM_ARCH_MINIMAX_01 ,"minimax-01" }, 139 {LLM_ARCH_MINIMAX_M2 ,"minimax-m2" }, 140 {LLM_ARCH_MINIMAX_M3 ,"minimax-m3" }, 141 {LLM_ARCH_COGVLM ,"cogvlm" }, 142 {LLM_ARCH_RND1 ,"rnd1" }, 143 {LLM_ARCH_PANGU_EMBED ,"pangu-embedded" }, 144 {LLM_ARCH_MISTRAL3 ,"mistral3" }, 145 {LLM_ARCH_EAGLE3 ,"eagle3" }, 146 {LLM_ARCH_DFLASH ,"dflash" }, 147 {LLM_ARCH_MISTRAL4 ,"mistral4" }, 148 {LLM_ARCH_PADDLEOCR ,"paddleocr" }, 149 {LLM_ARCH_MIMO2 ,"mimo2" }, 150 {LLM_ARCH_STEP35 ,"step35" }, 151 {LLM_ARCH_SPARK2_5 ,"spark2_5" }, 152 {LLM_ARCH_LLAMA_EMBED ,"llama-embed" }, 153 {LLM_ARCH_MAINCODER ,"maincoder" }, 154 {LLM_ARCH_KIMI_LINEAR ,"kimi-linear" }, 155 {LLM_ARCH_KIMI_K3 ,"kimi-k3" }, 156 {LLM_ARCH_TALKIE ,"talkie" }, 157 {LLM_ARCH_MELLUM ,"mellum" }, 158 {LLM_ARCH_NANBEIGE ,"nanbeige" }, 159 {LLM_ARCH_QWEN3TTS ,"qwen3tts" }, 160 {LLM_ARCH_POCKETTTS ,"pockettts" }, 161 {LLM_ARCH_UNKNOWN ,"(unknown)" }, 162}; 163 164static const std::map < llm_kv ,const char *> LLM_KV_NAMES = { 165 {LLM_KV_GENERAL_TYPE ,"general.type" }, 166 {LLM_KV_GENERAL_ARCHITECTURE ,"general.architecture" }, 167 {LLM_KV_GENERAL_QUANTIZATION_VERSION ,"general.quantization_version" }, 168 {LLM_KV_GENERAL_ALIGNMENT ,"general.alignment" }, 169 {LLM_KV_GENERAL_FILE_TYPE ,"general.file_type" }, 170 {LLM_KV_GENERAL_SAMPLING_SEQUENCE ,"general.sampling.sequence" }, 171 {LLM_KV_GENERAL_SAMPLING_TOP_K ,"general.sampling.top_k" }, 172 {LLM_KV_GENERAL_SAMPLING_TOP_P ,"general.sampling.top_p" }, 173 {LLM_KV_GENERAL_SAMPLING_MIN_P ,"general.sampling.min_p" }, 174 {LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY ,"general.sampling.xtc_probability" }, 175 {LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD ,"general.sampling.xtc_threshold" }, 176 {LLM_KV_GENERAL_SAMPLING_TEMP ,"general.sampling.temp" }, 177 {LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N ,"general.sampling.penalty_last_n" }, 178 {LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT ,"general.sampling.penalty_repeat" }, 179 {LLM_KV_GENERAL_SAMPLING_MIROSTAT ,"general.sampling.mirostat" }, 180 {LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU ,"general.sampling.mirostat_tau" }, 181 {LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA ,"general.sampling.mirostat_eta" }, 182 {LLM_KV_GENERAL_NAME ,"general.name" }, 183 {LLM_KV_GENERAL_AUTHOR ,"general.author" }, 184 {LLM_KV_GENERAL_VERSION ,"general.version" }, 185 {LLM_KV_GENERAL_URL ,"general.url" }, 186 {LLM_KV_GENERAL_DESCRIPTION ,"general.description" }, 187 {LLM_KV_GENERAL_LICENSE ,"general.license" }, 188 {LLM_KV_GENERAL_SOURCE_URL ,"general.source.url" }, 189 {LLM_KV_GENERAL_SOURCE_HF_REPO ,"general.source.huggingface.repository" }, 190 191 {LLM_KV_VOCAB_SIZE ,"%s.vocab_size" }, 192 {LLM_KV_CONTEXT_LENGTH ,"%s.context_length" }, 193 {LLM_KV_EMBEDDING_LENGTH ,"%s.embedding_length" }, 194 {LLM_KV_EMBEDDING_LENGTH_OUT ,"%s.embedding_length_out" }, 195 {LLM_KV_EMBEDDING_LENGTH_PER_LAYER ,"%s.embedding_length_per_layer_input" }, 196 {LLM_KV_FEATURES_LENGTH ,"%s.features_length" }, 197 {LLM_KV_BLOCK_COUNT ,"%s.block_count" }, 198 {LLM_KV_LEADING_DENSE_BLOCK_COUNT ,"%s.leading_dense_block_count" }, 199 {LLM_KV_ATTN_RES_BLOCK_SIZE ,"%s.attn_res.block_size" }, 200 {LLM_KV_ACTIVATION_SITU_BETA ,"%s.activation.situ_beta" }, 201 {LLM_KV_ACTIVATION_SITU_LINEAR_BETA ,"%s.activation.situ_linear_beta" }, 202 {LLM_KV_FEED_FORWARD_LENGTH ,"%s.feed_forward_length" }, 203 {LLM_KV_EXPERT_FEED_FORWARD_LENGTH ,"%s.expert_feed_forward_length" }, 204 {LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH ,"%s.expert_shared_feed_forward_length" }, 205 {LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH ,"%s.expert_chunk_feed_forward_length" }, 206 {LLM_KV_SWIGLU_CLAMP_EXP ,"%s.swiglu_clamp_exp" }, 207 {LLM_KV_SWIGLU_CLAMP_SHEXP ,"%s.swiglu_clamp_shexp" }, 208 {LLM_KV_USE_PARALLEL_RESIDUAL ,"%s.use_parallel_residual" }, 209 {LLM_KV_TENSOR_DATA_LAYOUT ,"%s.tensor_data_layout" }, 210 {LLM_KV_EXPERT_COUNT ,"%s.expert_count" }, 211 {LLM_KV_EXPERT_USED_COUNT ,"%s.expert_used_count" }, 212 {LLM_KV_EXPERT_SHARED_COUNT ,"%s.expert_shared_count" }, 213 {LLM_KV_EXPERT_GROUP_COUNT ,"%s.expert_group_count" }, 214 {LLM_KV_EXPERT_GROUP_USED_COUNT ,"%s.expert_group_used_count" }, 215 {LLM_KV_EXPERT_WEIGHTS_SCALE ,"%s.expert_weights_scale" }, 216 {LLM_KV_EXPERT_WEIGHTS_NORM ,"%s.expert_weights_norm" }, 217 {LLM_KV_EXPERT_LATENT_LENGTH ,"%s.expert_latent_length" }, 218 {LLM_KV_EXPERT_GATING_FUNC ,"%s.expert_gating_func" }, 219 {LLM_KV_EXPERT_GROUP_SCALE ,"%s.expert_group_scale" }, 220 {LLM_KV_EXPERTS_PER_GROUP ,"%s.experts_per_group" }, 221 {LLM_KV_MOE_EVERY_N_LAYERS ,"%s.moe_every_n_layers" }, 222 {LLM_KV_MOE_LATENT_SIZE ,"%s.moe_latent_size" }, 223 {LLM_KV_NEXTN_PREDICT_LAYERS ,"%s.nextn_predict_layers" }, 224 {LLM_KV_NUM_DEEPSTACK_LAYERS ,"%s.n_deepstack_layers" }, 225 {LLM_KV_DEEPSTACK_MAPPING ,"%s.deepstack_mapping" }, 226 {LLM_KV_HIDDEN_ACT ,"%s.hidden_activation" }, 227 {LLM_KV_POOLING_TYPE ,"%s.pooling_type" }, 228 {LLM_KV_LOGIT_SCALE ,"%s.logit_scale" }, 229 {LLM_KV_DECODER_START_TOKEN_ID ,"%s.decoder_start_token_id" }, 230 {LLM_KV_DECODER_BLOCK_COUNT ,"%s.decoder_block_count" }, 231 {LLM_KV_ENCODER_ATTENTION_HEAD_COUNT_KV ,"%s.encoder.attention.head_count_kv" }, 232 {LLM_KV_ATTN_LOGIT_SOFTCAPPING ,"%s.attn_logit_softcapping" }, 233 {LLM_KV_ROUTER_LOGIT_SOFTCAPPING ,"%s.router_logit_softcapping" }, 234 {LLM_KV_FINAL_LOGIT_SOFTCAPPING ,"%s.final_logit_softcapping" }, 235 {LLM_KV_SWIN_NORM ,"%s.swin_norm" }, 236 {LLM_KV_RESCALE_EVERY_N_LAYERS ,"%s.rescale_every_n_layers" }, 237 {LLM_KV_TIME_MIX_EXTRA_DIM ,"%s.time_mix_extra_dim" }, 238 {LLM_KV_TIME_DECAY_EXTRA_DIM ,"%s.time_decay_extra_dim" }, 239 {LLM_KV_RESIDUAL_SCALE ,"%s.residual_scale" }, 240 {LLM_KV_EMBEDDING_SCALE ,"%s.embedding_scale" }, 241 {LLM_KV_ADAPTER_COUNT ,"%s.adapters.count" }, 242 {LLM_KV_ADAPTER_TOKEN_IDS_ACTIVATE ,"%s.adapters.token_ids_activate" }, 243 {LLM_KV_ADAPTER_TOKEN_IDS_SUBSTITUTE ,"%s.adapters.token_ids_substitute" }, 244 {LLM_KV_ADAPTER_LORA_RANK ,"%s.adapters.lora_rank" }, 245 {LLM_KV_ADAPTER_ROUTER_GAIN ,"%s.adapters.router_gain" }, 246 {LLM_KV_TOKEN_SHIFT_COUNT ,"%s.token_shift_count" }, 247 {LLM_KV_INTERLEAVE_MOE_LAYER_STEP ,"%s.interleave_moe_layer_step" }, 248 {LLM_KV_FULL_ATTENTION_INTERVAL ,"%s.full_attention_interval" }, 249 {LLM_KV_NUM_LOOPS ,"%s.num_loops" }, 250 {LLM_KV_SKIP_LOOP_FINAL_NORM ,"%s.skip_loop_final_norm" }, 251 252 {LLM_KV_ATTENTION_HEAD_COUNT ,"%s.attention.head_count" }, 253 {LLM_KV_ATTENTION_HEAD_COUNT_KV ,"%s.attention.head_count_kv" }, 254 {LLM_KV_ATTENTION_MAX_ALIBI_BIAS ,"%s.attention.max_alibi_bias" }, 255 {LLM_KV_ATTENTION_CLAMP_KQV ,"%s.attention.clamp_kqv" }, 256 {LLM_KV_ATTENTION_KEY_LENGTH ,"%s.attention.key_length" }, 257 {LLM_KV_ATTENTION_VALUE_LENGTH ,"%s.attention.value_length" }, 258 {LLM_KV_ATTENTION_LAYERNORM_EPS ,"%s.attention.layer_norm_epsilon" }, 259 {LLM_KV_ATTENTION_LAYERNORM_RMS_EPS ,"%s.attention.layer_norm_rms_epsilon" }, 260 {LLM_KV_ATTENTION_GROUPNORM_EPS ,"%s.attention.group_norm_epsilon" }, 261 {LLM_KV_ATTENTION_GROUPNORM_GROUPS ,"%s.attention.group_norm_groups" }, 262 {LLM_KV_ATTENTION_CAUSAL ,"%s.attention.causal" }, 263 {LLM_KV_ATTENTION_Q_LORA_RANK ,"%s.attention.q_lora_rank" }, 264 {LLM_KV_ATTENTION_KV_LORA_RANK ,"%s.attention.kv_lora_rank" }, 265 {LLM_KV_ATTENTION_DECAY_LORA_RANK ,"%s.attention.decay_lora_rank" }, 266 {LLM_KV_ATTENTION_ICLR_LORA_RANK ,"%s.attention.iclr_lora_rank" }, 267 {LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK ,"%s.attention.value_residual_mix_lora_rank" }, 268 {LLM_KV_ATTENTION_GATE_LORA_RANK ,"%s.attention.gate_lora_rank" }, 269 {LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT ,"%s.attention.relative_buckets_count" }, 270 {LLM_KV_ATTENTION_SLIDING_WINDOW ,"%s.attention.sliding_window" }, 271 {LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN ,"%s.attention.sliding_window_pattern" }, 272 {LLM_KV_ATTENTION_ROPE_PATTERN ,"%s.attention.rope_pattern" }, 273 274 {LLM_KV_ATTENTION_SCALE ,"%s.attention.scale" }, 275 {LLM_KV_ATTENTION_OUTPUT_SCALE ,"%s.attention.output_scale" }, 276 {LLM_KV_ATTENTION_VALUE_SCALE ,"%s.attention.value_scale" }, 277 {LLM_KV_ATTENTION_TEMPERATURE_LENGTH ,"%s.attention.temperature_length" }, 278 {LLM_KV_ATTENTION_TEMPERATURE_SCALE ,"%s.attention.temperature_scale" }, 279 {LLM_KV_ATTENTION_KEY_LENGTH_MLA ,"%s.attention.key_length_mla" }, 280 {LLM_KV_ATTENTION_VALUE_LENGTH_MLA ,"%s.attention.value_length_mla" }, 281 {LLM_KV_ATTENTION_KEY_LENGTH_SWA ,"%s.attention.key_length_swa" }, 282 {LLM_KV_ATTENTION_VALUE_LENGTH_SWA ,"%s.attention.value_length_swa" }, 283 {LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA ,"%s.attention.key_length_mla_swa" }, 284 {LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA ,"%s.attention.value_length_mla_swa" }, 285 {LLM_KV_ATTENTION_KV_LORA_RANK_SWA ,"%s.attention.kv_lora_rank_swa" }, 286 {LLM_KV_ATTENTION_INDEXER_HEAD_COUNT ,"%s.attention.indexer.head_count" }, 287 {LLM_KV_ATTENTION_INDEXER_KEY_LENGTH ,"%s.attention.indexer.key_length" }, 288 {LLM_KV_ATTENTION_INDEXER_TOP_K ,"%s.attention.indexer.top_k" }, 289 {LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE ,"%s.attention.indexer.block_size" }, 290 {LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS ,"%s.attention.indexer.local_blocks" }, 291 {LLM_KV_ATTENTION_INDEXER_TYPES ,"%s.attention.indexer.types" }, 292 {LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT ,"%s.attention.output_group_count" }, 293 {LLM_KV_ATTENTION_OUTPUT_LORA_RANK ,"%s.attention.output_lora_rank" }, 294 {LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE ,"%s.attention.compress_rope_freq_base" }, 295 {LLM_KV_ATTENTION_COMPRESS_RATIOS ,"%s.attention.compress_ratios" }, 296 {LLM_KV_ATTENTION_SHARED_KV_LAYERS ,"%s.attention.shared_kv_layers" }, 297 {LLM_KV_ATTENTION_RECURRENT_LAYERS ,"%s.attention.recurrent_layers" }, 298 299 {LLM_KV_HYPER_CONNECTION_COUNT ,"%s.hyper_connection.count" }, 300 {LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS ,"%s.hyper_connection.sinkhorn_iterations" }, 301 {LLM_KV_HYPER_CONNECTION_EPSILON ,"%s.hyper_connection.epsilon" }, 302 {LLM_KV_HYPER_CONNECTION_MAGNITUDE ,"%s.hyper_connection.magnitude" }, 303 {LLM_KV_HYPER_CONNECTION_LOW_RANK ,"%s.hyper_connection.low_rank" }, 304 305 {LLM_KV_PLE_LAYERS ,"%s.ple.layers" }, 306 {LLM_KV_PLE_NGRAM_SIZE ,"%s.ple.ngram_size" }, 307 {LLM_KV_PLE_HEADS_PER_NGRAM ,"%s.ple.heads_per_ngram" }, 308 {LLM_KV_PLE_CONV_KERNEL ,"%s.ple.conv_kernel" }, 309 {LLM_KV_PLE_LAYER_MULTIPLIERS ,"%s.ple.layer_multipliers" }, 310 {LLM_KV_PLE_HEAD_OFFSETS ,"%s.ple.head_offsets" }, 311 {LLM_KV_PLE_HEAD_VOCAB_SIZES ,"%s.ple.head_vocab_sizes" }, 312 {LLM_KV_PLE_EOS_TOKEN_ID ,"%s.ple.eos_token_id" }, 313 {LLM_KV_PLE_IMAGE_TOKEN_ID ,"%s.ple.image_token_id" }, 314 315 {LLM_KV_HASH_LAYER_COUNT ,"%s.hash_layer_count" }, 316 317 {LLM_KV_ROPE_DIMENSION_COUNT ,"%s.rope.dimension_count" }, 318 {LLM_KV_ROPE_DIMENSION_COUNT_SWA ,"%s.rope.dimension_count_swa" }, 319 {LLM_KV_ROPE_DIMENSION_SECTIONS ,"%s.rope.dimension_sections" }, 320 {LLM_KV_ROPE_FREQ_BASE ,"%s.rope.freq_base" }, 321 {LLM_KV_ROPE_FREQ_BASE_SWA ,"%s.rope.freq_base_swa" }, 322 {LLM_KV_ROPE_SCALE_LINEAR ,"%s.rope.scale_linear" }, 323 {LLM_KV_ROPE_SCALING_TYPE ,"%s.rope.scaling.type" }, 324 {LLM_KV_ROPE_SCALING_FACTOR ,"%s.rope.scaling.factor" }, 325 {LLM_KV_ROPE_SCALING_ALPHA ,"%s.rope.scaling.alpha" }, 326 {LLM_KV_ROPE_SCALING_ATTN_FACTOR ,"%s.rope.scaling.attn_factor" }, 327 {LLM_KV_ROPE_SCALING_ORIG_CTX_LEN ,"%s.rope.scaling.original_context_length" }, 328 {LLM_KV_ROPE_SCALING_FINETUNED ,"%s.rope.scaling.finetuned" }, 329 {LLM_KV_ROPE_SCALING_YARN_LOG_MUL ,"%s.rope.scaling.yarn_log_multiplier" }, 330 {LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR ,"%s.rope.scaling.yarn_ext_factor" }, 331 {LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR ,"%s.rope.scaling.yarn_attn_factor" }, 332 {LLM_KV_ROPE_SCALING_YARN_BETA_FAST ,"%s.rope.scaling.yarn_beta_fast" }, 333 {LLM_KV_ROPE_SCALING_YARN_BETA_SLOW ,"%s.rope.scaling.yarn_beta_slow" }, 334 335 {LLM_KV_SPLIT_NO ,"split.no" }, 336 {LLM_KV_SPLIT_COUNT ,"split.count" }, 337 {LLM_KV_SPLIT_TENSORS_COUNT ,"split.tensors.count" }, 338 339 {LLM_KV_SSM_CONV_KERNEL ,"%s.ssm.conv_kernel" }, 340 {LLM_KV_SSM_INNER_SIZE ,"%s.ssm.inner_size" }, 341 {LLM_KV_SSM_STATE_SIZE ,"%s.ssm.state_size" }, 342 {LLM_KV_SSM_TIME_STEP_RANK ,"%s.ssm.time_step_rank" }, 343 {LLM_KV_SSM_GROUP_COUNT ,"%s.ssm.group_count" }, 344 {LLM_KV_SSM_DT_B_C_RMS ,"%s.ssm.dt_b_c_rms" }, 345 346 {LLM_KV_KDA_HEAD_DIM ,"%s.kda.head_dim" }, 347 {LLM_KV_KDA_SAFE_GATE ,"%s.kda.safe_gate" }, 348 {LLM_KV_KDA_GATE_LOWER_BOUND ,"%s.kda.gate_lower_bound" }, 349 350 {LLM_KV_WKV_HEAD_SIZE ,"%s.wkv.head_size" }, 351 352 {LLM_KV_POSNET_EMBEDDING_LENGTH ,"%s.posnet.embedding_length" }, 353 {LLM_KV_POSNET_BLOCK_COUNT ,"%s.posnet.block_count" }, 354 355 {LLM_KV_CONVNEXT_EMBEDDING_LENGTH ,"%s.convnext.embedding_length" }, 356 {LLM_KV_CONVNEXT_BLOCK_COUNT ,"%s.convnext.block_count" }, 357 358 {LLM_KV_CLASSIFIER_OUTPUT_LABELS ,"%s.classifier.output_labels" }, 359 360 {LLM_KV_TARGET_LAYERS ,"%s.target_layers" }, 361 {LLM_KV_TARGET_HIDDEN_SIZE ,"%s.target_hidden_size" }, 362 {LLM_KV_NORM_BEFORE_RESIDUAL ,"%s.norm_before_residual" }, 363 {LLM_KV_NORM_BEFORE_FC ,"%s.norm_before_fc" }, 364 365 {LLM_KV_DFLASH_BLOCK_SIZE ,"%s.block_size" }, 366 {LLM_KV_DFLASH_CONV_KERNEL_SIZE ,"%s.conv_kernel_size" }, 367 {LLM_KV_DFLASH_CONV_GROUP_SIZE ,"%s.conv_group_size" }, 368 {LLM_KV_DFLASH_SELECTOR_RANK ,"%s.selector_rank" }, 369 {LLM_KV_DFLASH_SELECTOR_TOP_K ,"%s.selector_top_k" }, 370 371 {LLM_KV_SHORTCONV_L_CACHE ,"%s.shortconv.l_cache" }, 372// sentence-transformers dense modules feature dims 373 {LLM_KV_DENSE_2_FEAT_IN ,"%s.dense_2_feat_in" }, 374 {LLM_KV_DENSE_2_FEAT_OUT ,"%s.dense_2_feat_out" }, 375 {LLM_KV_DENSE_3_FEAT_IN ,"%s.dense_3_feat_in" }, 376 {LLM_KV_DENSE_3_FEAT_OUT ,"%s.dense_3_feat_out" }, 377 378 {LLM_KV_TOKENIZER_MODEL ,"tokenizer.ggml.model" }, 379 {LLM_KV_TOKENIZER_PRE ,"tokenizer.ggml.pre" }, 380 {LLM_KV_TOKENIZER_LIST ,"tokenizer.ggml.tokens" }, 381 {LLM_KV_TOKENIZER_TOKEN_TYPE ,"tokenizer.ggml.token_type" }, 382 {LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT ,"tokenizer.ggml.token_type_count" }, 383 {LLM_KV_TOKENIZER_SCORES ,"tokenizer.ggml.scores" }, 384 {LLM_KV_TOKENIZER_MERGES ,"tokenizer.ggml.merges" }, 385 {LLM_KV_TOKENIZER_BOS_ID ,"tokenizer.ggml.bos_token_id" }, 386 {LLM_KV_TOKENIZER_EOS_ID ,"tokenizer.ggml.eos_token_id" }, 387 {LLM_KV_TOKENIZER_EOT_ID ,"tokenizer.ggml.eot_token_id" }, 388 {LLM_KV_TOKENIZER_EOM_ID ,"tokenizer.ggml.eom_token_id" }, 389 {LLM_KV_TOKENIZER_UNK_ID ,"tokenizer.ggml.unknown_token_id" }, 390 {LLM_KV_TOKENIZER_SEP_ID ,"tokenizer.ggml.seperator_token_id" }, 391 {LLM_KV_TOKENIZER_PAD_ID ,"tokenizer.ggml.padding_token_id" }, 392 {LLM_KV_TOKENIZER_CLS_ID ,"tokenizer.ggml.cls_token_id" }, 393 {LLM_KV_TOKENIZER_MASK_ID ,"tokenizer.ggml.mask_token_id" }, 394 {LLM_KV_TOKENIZER_ADD_BOS ,"tokenizer.ggml.add_bos_token" }, 395 {LLM_KV_TOKENIZER_ADD_EOS ,"tokenizer.ggml.add_eos_token" }, 396 {LLM_KV_TOKENIZER_ADD_SEP ,"tokenizer.ggml.add_sep_token" }, 397 {LLM_KV_TOKENIZER_ADD_PREFIX ,"tokenizer.ggml.add_space_prefix" }, 398 {LLM_KV_TOKENIZER_REMOVE_EXTRA_WS ,"tokenizer.ggml.remove_extra_whitespaces" }, 399 {LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP ,"tokenizer.ggml.precompiled_charsmap" }, 400 {LLM_KV_TOKENIZER_HF_JSON ,"tokenizer.huggingface.json" }, 401 {LLM_KV_TOKENIZER_RWKV ,"tokenizer.rwkv.world" }, 402 {LLM_KV_TOKENIZER_CHAT_TEMPLATE ,"tokenizer.chat_template" }, 403 {LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE ,"tokenizer.ggml.normalizer.lowercase" }, 404 {LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS ,"tokenizer.ggml.normalizer.strip_accents" }, 405 {LLM_KV_TOKENIZER_FIM_PRE_ID ,"tokenizer.ggml.fim_pre_token_id" }, 406 {LLM_KV_TOKENIZER_FIM_SUF_ID ,"tokenizer.ggml.fim_suf_token_id" }, 407 {LLM_KV_TOKENIZER_FIM_MID_ID ,"tokenizer.ggml.fim_mid_token_id" }, 408 {LLM_KV_TOKENIZER_FIM_PAD_ID ,"tokenizer.ggml.fim_pad_token_id" }, 409 {LLM_KV_TOKENIZER_FIM_REP_ID ,"tokenizer.ggml.fim_rep_token_id" }, 410 {LLM_KV_TOKENIZER_FIM_SEP_ID ,"tokenizer.ggml.fim_sep_token_id" }, 411 {LLM_KV_TOKENIZER_SUPPRESS_TOKENS ,"tokenizer.ggml.suppress_tokens" }, 412 413 {LLM_KV_ADAPTER_TYPE ,"adapter.type" }, 414 {LLM_KV_ADAPTER_LORA_ALPHA ,"adapter.lora.alpha" }, 415 {LLM_KV_ADAPTER_LORA_TASK_NAME ,"adapter.lora.task_name" }, 416 {LLM_KV_ADAPTER_LORA_PROMPT_PREFIX ,"adapter.lora.prompt_prefix" }, 417 {LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS ,"adapter.alora.invocation_tokens" }, 418 419 {LLM_KV_XIELU_ALPHA_N ,"xielu.alpha_n" }, 420 {LLM_KV_XIELU_ALPHA_P ,"xielu.alpha_p" }, 421 {LLM_KV_XIELU_BETA ,"xielu.beta" }, 422 {LLM_KV_XIELU_EPS ,"xielu.eps" }, 423 424// deprecated 425 {LLM_KV_TOKENIZER_PREFIX_ID ,"tokenizer.ggml.prefix_token_id" }, 426 {LLM_KV_TOKENIZER_SUFFIX_ID ,"tokenizer.ggml.suffix_token_id" }, 427 {LLM_KV_TOKENIZER_MIDDLE_ID ,"tokenizer.ggml.middle_token_id" }, 428}; 429 430static const std::map < llm_tensor ,const char *> LLM_TENSOR_NAMES = { 431 {LLM_TENSOR_TOKEN_EMBD ,"token_embd" }, 432 {LLM_TENSOR_OUTPUT_NORM ,"output_norm" }, 433 {LLM_TENSOR_OUTPUT_NORM_LFM2 ,"token_embd_norm" },// fix for wrong tensor name 434 {LLM_TENSOR_OUTPUT ,"output" }, 435 {LLM_TENSOR_ROPE_FREQS ,"rope_freqs" }, 436 {LLM_TENSOR_ATTN_NORM ,"blk.%d.attn_norm" }, 437 {LLM_TENSOR_ATTN_Q ,"blk.%d.attn_q" }, 438 {LLM_TENSOR_ATTN_K ,"blk.%d.attn_k" }, 439 {LLM_TENSOR_ATTN_V ,"blk.%d.attn_v" }, 440 {LLM_TENSOR_ATTN_OUT ,"blk.%d.attn_output" }, 441 {LLM_TENSOR_ATTN_ROT_EMBD ,"blk.%d.attn_rot_embd" }, 442 {LLM_TENSOR_FFN_GATE_INP ,"blk.%d.ffn_gate_inp" }, 443 {LLM_TENSOR_FFN_NORM ,"blk.%d.ffn_norm" }, 444 {LLM_TENSOR_FFN_GATE ,"blk.%d.ffn_gate" }, 445 {LLM_TENSOR_FFN_DOWN ,"blk.%d.ffn_down" }, 446 {LLM_TENSOR_FFN_UP ,"blk.%d.ffn_up" }, 447 {LLM_TENSOR_FFN_GATE_EXP ,"blk.%d.ffn_gate.%d" }, 448 {LLM_TENSOR_FFN_DOWN_EXP ,"blk.%d.ffn_down.%d" }, 449 {LLM_TENSOR_FFN_UP_EXP ,"blk.%d.ffn_up.%d" }, 450 {LLM_TENSOR_FFN_GATE_EXPS ,"blk.%d.ffn_gate_exps" }, 451 {LLM_TENSOR_FFN_GATE_UP_EXPS ,"blk.%d.ffn_gate_up_exps" }, 452 {LLM_TENSOR_FFN_DOWN_EXPS ,"blk.%d.ffn_down_exps" }, 453 {LLM_TENSOR_FFN_UP_EXPS ,"blk.%d.ffn_up_exps" }, 454 {LLM_TENSOR_ATTN_POST_NORM ,"blk.%d.post_attention_norm" }, 455 {LLM_TENSOR_ATTN_Q_NORM ,"blk.%d.attn_q_norm" }, 456 {LLM_TENSOR_ATTN_K_NORM ,"blk.%d.attn_k_norm" }, 457 {LLM_TENSOR_ATTN_GATE ,"blk.%d.attn_gate" }, 458 {LLM_TENSOR_FFN_POST_NORM ,"blk.%d.post_ffw_norm" }, 459 {LLM_TENSOR_FFN_POST_NORM_1 ,"blk.%d.post_ffw_norm_1" }, 460 {LLM_TENSOR_FFN_POST_NORM_2 ,"blk.%d.post_ffw_norm_2" }, 461 {LLM_TENSOR_FFN_PRE_NORM_2 ,"blk.%d.pre_ffw_norm_2" }, 462 {LLM_TENSOR_FFN_GATE_SHEXP ,"blk.%d.ffn_gate_shexp" }, 463 {LLM_TENSOR_FFN_UP_SHEXP ,"blk.%d.ffn_up_shexp" }, 464 {LLM_TENSOR_FFN_DOWN_SHEXP ,"blk.%d.ffn_down_shexp" }, 465 {LLM_TENSOR_FFN_EXP_PROBS_B ,"blk.%d.exp_probs_b" }, 466 {LLM_TENSOR_FFN_EXP_PROBS_B_VL ,"blk.%d.exp_probs_b_vl" }, 467 {LLM_TENSOR_FFN_LATENT_DOWN ,"blk.%d.ffn_latent_down" }, 468 {LLM_TENSOR_FFN_LATENT_UP ,"blk.%d.ffn_latent_up" }, 469 {LLM_TENSOR_ATTN_NORM_2 ,"blk.%d.attn_norm_2" }, 470 {LLM_TENSOR_ATTN_QKV ,"blk.%d.attn_qkv" }, 471 {LLM_TENSOR_LAYER_OUT_NORM ,"blk.%d.layer_output_norm" }, 472 {LLM_TENSOR_LAYER_OUT_SCALE ,"blk.%d.layer_output_scale" }, 473 {LLM_TENSOR_ATTN_OUT_NORM ,"blk.%d.attn_output_norm" }, 474 {LLM_TENSOR_POS_EMBD ,"position_embd" }, 475 {LLM_TENSOR_FFN_ACT ,"blk.%d.ffn.act" }, 476 {LLM_TENSOR_TOKEN_EMBD_NORM ,"token_embd_norm" }, 477 {LLM_TENSOR_TOKEN_TYPES ,"token_types" }, 478 {LLM_TENSOR_CLS ,"cls" }, 479 {LLM_TENSOR_CLS_OUT ,"cls.output" }, 480 {LLM_TENSOR_CLS_NORM ,"cls.norm" }, 481 {LLM_TENSOR_ENC_OUTPUT_NORM ,"enc.output_norm" }, 482 {LLM_TENSOR_FFN_GATE_INP_SHEXP ,"blk.%d.ffn_gate_inp_shexp" }, 483 {LLM_TENSOR_SSM_A_NOSCAN ,"blk.%d.ssm_a" }, 484 {LLM_TENSOR_SSM_CONV1D ,"blk.%d.ssm_conv1d" }, 485 {LLM_TENSOR_SSM_DT ,"blk.%d.ssm_dt" }, 486 {LLM_TENSOR_SSM_BETA_ALPHA ,"blk.%d.ssm_ba" }, 487 {LLM_TENSOR_SSM_ALPHA ,"blk.%d.ssm_alpha" }, 488 {LLM_TENSOR_SSM_IN ,"blk.%d.ssm_in" }, 489 {LLM_TENSOR_SSM_NORM ,"blk.%d.ssm_norm" }, 490 {LLM_TENSOR_SSM_OUT ,"blk.%d.ssm_out" }, 491 {LLM_TENSOR_ROPE_FACTORS_LONG ,"rope_factors_long" }, 492 {LLM_TENSOR_ROPE_FACTORS_SHORT ,"rope_factors_short" }, 493 {LLM_TENSOR_SSM_X ,"blk.%d.ssm_x" }, 494 {LLM_TENSOR_SSM_A ,"blk.%d.ssm_a" }, 495 {LLM_TENSOR_SSM_D ,"blk.%d.ssm_d" }, 496 {LLM_TENSOR_SSM_DT_NORM ,"blk.%d.ssm_dt_norm" }, 497 {LLM_TENSOR_SSM_B_NORM ,"blk.%d.ssm_b_norm" }, 498 {LLM_TENSOR_SSM_C_NORM ,"blk.%d.ssm_c_norm" }, 499 {LLM_TENSOR_SSM_CONV1D_Q ,"blk.%d.ssm_conv1d_q" }, 500 {LLM_TENSOR_SSM_CONV1D_K ,"blk.%d.ssm_conv1d_k" }, 501 {LLM_TENSOR_SSM_CONV1D_V ,"blk.%d.ssm_conv1d_v" }, 502 {LLM_TENSOR_SSM_F_A ,"blk.%d.ssm_f_a" }, 503 {LLM_TENSOR_SSM_F_B ,"blk.%d.ssm_f_b" }, 504 {LLM_TENSOR_SSM_BETA ,"blk.%d.ssm_beta" }, 505 {LLM_TENSOR_SSM_G_A ,"blk.%d.ssm_g_a" }, 506 {LLM_TENSOR_SSM_G ,"blk.%d.ssm_g" }, 507 {LLM_TENSOR_ATTN_RES_SCORE ,"blk.%d.attn_res_score" }, 508 {LLM_TENSOR_FFN_RES_SCORE ,"blk.%d.ffn_res_score" }, 509 {LLM_TENSOR_OUTPUT_RES_SCORE ,"output_res_score" }, 510 {LLM_TENSOR_FFN_ROUTED_DOWN ,"blk.%d.ffn_routed_down" }, 511 {LLM_TENSOR_FFN_ROUTED_UP ,"blk.%d.ffn_routed_up" }, 512 {LLM_TENSOR_FFN_ROUTED_NORM ,"blk.%d.ffn_routed_norm" }, 513 {LLM_TENSOR_SSM_G_B ,"blk.%d.ssm_g_b" }, 514 {LLM_TENSOR_SSM_NORM ,"blk.%d.ssm_norm" }, 515 {LLM_TENSOR_ATTN_Q_A_NORM ,"blk.%d.attn_q_a_norm" }, 516 {LLM_TENSOR_ATTN_KV_A_NORM ,"blk.%d.attn_kv_a_norm" }, 517 {LLM_TENSOR_ATTN_Q_A ,"blk.%d.attn_q_a" }, 518 {LLM_TENSOR_ATTN_Q_B ,"blk.%d.attn_q_b" }, 519 {LLM_TENSOR_ATTN_KV_A_MQA ,"blk.%d.attn_kv_a_mqa" }, 520 {LLM_TENSOR_ATTN_KV_B ,"blk.%d.attn_kv_b" }, 521 {LLM_TENSOR_ATTN_KV ,"blk.%d.attn_kv" }, 522 {LLM_TENSOR_ATTN_KV_NORM ,"blk.%d.attn_kv_a_norm" }, 523 {LLM_TENSOR_ATTN_OUT_A ,"blk.%d.attn_output_a" }, 524 {LLM_TENSOR_ATTN_OUT_B ,"blk.%d.attn_output_b" }, 525 {LLM_TENSOR_HC_HEAD_FN ,"output_hc_fn" }, 526 {LLM_TENSOR_HC_HEAD_BASE ,"output_hc_base" }, 527 {LLM_TENSOR_HC_HEAD_SCALE ,"output_hc_scale" }, 528 {LLM_TENSOR_HC_HEAD_NORM ,"output_hc_norm" }, 529 {LLM_TENSOR_HC_HEAD_DOWN ,"output_hc_down" }, 530 {LLM_TENSOR_HC_HEAD_UP ,"output_hc_up" }, 531 {LLM_TENSOR_HC_ATTN_FN ,"blk.%d.hc_attn_fn" }, 532 {LLM_TENSOR_HC_ATTN_BASE ,"blk.%d.hc_attn_base" }, 533 {LLM_TENSOR_HC_ATTN_SCALE ,"blk.%d.hc_attn_scale" }, 534 {LLM_TENSOR_HC_FFN_FN ,"blk.%d.hc_ffn_fn" }, 535 {LLM_TENSOR_HC_FFN_BASE ,"blk.%d.hc_ffn_base" }, 536 {LLM_TENSOR_HC_FFN_SCALE ,"blk.%d.hc_ffn_scale" }, 537 {LLM_TENSOR_HC_ATTN_NORM ,"blk.%d.hc_attn_norm" }, 538 {LLM_TENSOR_HC_ATTN_DOWN ,"blk.%d.hc_attn_down" }, 539 {LLM_TENSOR_HC_ATTN_UP ,"blk.%d.hc_attn_up" }, 540 {LLM_TENSOR_HC_ATTN_INJECT ,"blk.%d.hc_attn_inject" }, 541 {LLM_TENSOR_HC_FFN_NORM ,"blk.%d.hc_ffn_norm" }, 542 {LLM_TENSOR_HC_FFN_DOWN ,"blk.%d.hc_ffn_down" }, 543 {LLM_TENSOR_HC_FFN_UP ,"blk.%d.hc_ffn_up" }, 544 {LLM_TENSOR_HC_FFN_INJECT ,"blk.%d.hc_ffn_inject" }, 545 {LLM_TENSOR_PLE_KEY ,"blk.%d.ple_key" }, 546 {LLM_TENSOR_PLE_VALUE ,"blk.%d.ple_value" }, 547 {LLM_TENSOR_PLE_NORM_KEY ,"blk.%d.ple_norm_key" }, 548 {LLM_TENSOR_PLE_NORM_QUERY ,"blk.%d.ple_norm_query" }, 549 {LLM_TENSOR_PLE_NORM_CONV ,"blk.%d.ple_norm_conv" }, 550 {LLM_TENSOR_PLE_CONV1D ,"blk.%d.ple_conv1d" }, 551 {LLM_TENSOR_ATTN_COMPRESSOR_WKV ,"blk.%d.attn_compressor_kv" }, 552 {LLM_TENSOR_ATTN_COMPRESSOR_WGATE ,"blk.%d.attn_compressor_gate" }, 553 {LLM_TENSOR_ATTN_COMPRESSOR_APE ,"blk.%d.attn_compressor_ape" }, 554 {LLM_TENSOR_ATTN_COMPRESSOR_NORM ,"blk.%d.attn_compressor_norm" }, 555 {LLM_TENSOR_PER_LAYER_TOKEN_EMBD ,"per_layer_token_embd" }, 556 {LLM_TENSOR_PER_LAYER_MODEL_PROJ ,"per_layer_model_proj" }, 557 {LLM_TENSOR_PER_LAYER_PROJ_NORM ,"per_layer_proj_norm" }, 558 {LLM_TENSOR_ALTUP_UNEMBD_PROJ ,"altup_unembd_proj" }, 559 {LLM_TENSOR_ALTUP_PROJ ,"altup_proj" }, 560 {LLM_TENSOR_PER_LAYER_INP_GATE ,"blk.%d.inp_gate" }, 561 {LLM_TENSOR_PER_LAYER_PROJ ,"blk.%d.proj" }, 562 {LLM_TENSOR_PER_LAYER_POST_NORM ,"blk.%d.post_norm" }, 563 {LLM_TENSOR_ALTUP_CORRECT_COEF ,"blk.%d.altup_correct_coef" }, 564 {LLM_TENSOR_ALTUP_CORRECT_SCALE ,"blk.%d.altup_correct_scale" }, 565 {LLM_TENSOR_ALTUP_PREDICT_COEF ,"blk.%d.altup_predict_coef" }, 566 {LLM_TENSOR_ALTUP_ROUTER ,"blk.%d.altup_router" }, 567 {LLM_TENSOR_ALTUP_ROUTER_NORM ,"blk.%d.altup_router_norm" }, 568 {LLM_TENSOR_LAUREL_L ,"blk.%d.laurel_l" }, 569 {LLM_TENSOR_LAUREL_R ,"blk.%d.laurel_r" }, 570 {LLM_TENSOR_LAUREL_POST_NORM ,"blk.%d.laurel_post_norm" }, 571 {LLM_TENSOR_DENSE_2_OUT ,"dense_2" }, 572 {LLM_TENSOR_DENSE_3_OUT ,"dense_3" }, 573 {LLM_TENSOR_FFN_NORM_EXPS ,"blk.%d.ffn_norm_exps" }, 574 {LLM_TENSOR_ATTN_K_B ,"blk.%d.attn_k_b" }, 575 {LLM_TENSOR_ATTN_V_B ,"blk.%d.attn_v_b" }, 576 {LLM_TENSOR_NEXTN_PROJ_PRE ,"nextn.pre_projection" }, 577 {LLM_TENSOR_NEXTN_PROJ_POST ,"nextn.post_projection" }, 578 {LLM_TENSOR_NEXTN_EH_PROJ ,"blk.%d.nextn.eh_proj" }, 579 {LLM_TENSOR_NEXTN_EMBED_TOKENS ,"blk.%d.nextn.embed_tokens" }, 580 {LLM_TENSOR_NEXTN_ENORM ,"blk.%d.nextn.enorm" }, 581 {LLM_TENSOR_NEXTN_HNORM ,"blk.%d.nextn.hnorm" }, 582 {LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD ,"blk.%d.nextn.shared_head_head" }, 583 {LLM_TENSOR_NEXTN_SHARED_HEAD_NORM ,"blk.%d.nextn.shared_head_norm" }, 584 {LLM_TENSOR_ATTN_SUB_NORM ,"blk.%d.attn_sub_norm" }, 585 {LLM_TENSOR_FFN_SUB_NORM ,"blk.%d.ffn_sub_norm" }, 586 {LLM_TENSOR_DEC_OUTPUT_NORM ,"dec.output_norm" }, 587 {LLM_TENSOR_DEC_ATTN_NORM ,"dec.blk.%d.attn_norm" }, 588 {LLM_TENSOR_DEC_ATTN_Q ,"dec.blk.%d.attn_q" }, 589 {LLM_TENSOR_DEC_ATTN_K ,"dec.blk.%d.attn_k" }, 590 {LLM_TENSOR_DEC_ATTN_V ,"dec.blk.%d.attn_v" }, 591 {LLM_TENSOR_DEC_ATTN_OUT ,"dec.blk.%d.attn_o" }, 592 {LLM_TENSOR_DEC_ATTN_REL_B ,"dec.blk.%d.attn_rel_b" }, 593 {LLM_TENSOR_DEC_CROSS_ATTN_NORM ,"dec.blk.%d.cross_attn_norm" }, 594 {LLM_TENSOR_DEC_CROSS_ATTN_Q ,"dec.blk.%d.cross_attn_q" }, 595 {LLM_TENSOR_DEC_CROSS_ATTN_K ,"dec.blk.%d.cross_attn_k" }, 596 {LLM_TENSOR_DEC_CROSS_ATTN_V ,"dec.blk.%d.cross_attn_v" }, 597 {LLM_TENSOR_DEC_CROSS_ATTN_OUT ,"dec.blk.%d.cross_attn_o" }, 598 {LLM_TENSOR_DEC_CROSS_ATTN_REL_B ,"dec.blk.%d.cross_attn_rel_b" }, 599 {LLM_TENSOR_DEC_FFN_NORM ,"dec.blk.%d.ffn_norm" }, 600 {LLM_TENSOR_DEC_FFN_GATE ,"dec.blk.%d.ffn_gate" }, 601 {LLM_TENSOR_DEC_FFN_DOWN ,"dec.blk.%d.ffn_down" }, 602 {LLM_TENSOR_DEC_FFN_UP ,"dec.blk.%d.ffn_up" }, 603 {LLM_TENSOR_ENC_ATTN_NORM ,"enc.blk.%d.attn_norm" }, 604 {LLM_TENSOR_ENC_ATTN_Q ,"enc.blk.%d.attn_q" }, 605 {LLM_TENSOR_ENC_ATTN_K ,"enc.blk.%d.attn_k" }, 606 {LLM_TENSOR_ENC_ATTN_V ,"enc.blk.%d.attn_v" }, 607 {LLM_TENSOR_ENC_ATTN_OUT ,"enc.blk.%d.attn_o" }, 608 {LLM_TENSOR_ENC_ATTN_REL_B ,"enc.blk.%d.attn_rel_b" }, 609 {LLM_TENSOR_ENC_FFN_NORM ,"enc.blk.%d.ffn_norm" }, 610 {LLM_TENSOR_ENC_FFN_GATE ,"enc.blk.%d.ffn_gate" }, 611 {LLM_TENSOR_ENC_FFN_DOWN ,"enc.blk.%d.ffn_down" }, 612 {LLM_TENSOR_ENC_FFN_UP ,"enc.blk.%d.ffn_up" }, 613// Keep the tensor names used by the published AliceAI GGUFs. 614 {LLM_TENSOR_ENC_FFN_GATE_INP ,"alice.enc.blk.%d.ffn_gate_inp" }, 615 {LLM_TENSOR_ENC_FFN_GATE_EXPS ,"enc.blk.%d.ffn_gate" }, 616 {LLM_TENSOR_ENC_FFN_DOWN_EXPS ,"enc.blk.%d.ffn_down" }, 617 {LLM_TENSOR_ENC_FFN_UP_EXPS ,"enc.blk.%d.ffn_up" }, 618 {LLM_TENSOR_ENC_EXP_PROBS_B ,"alice.enc.blk.%d.exp_probs_b" }, 619 {LLM_TENSOR_ENC_FFN_OUTPUT_B ,"alice.enc.blk.%d.ffn_output_b" }, 620 {LLM_TENSOR_DEC_FFN_GATE_INP ,"alice.dec.blk.%d.ffn_gate_inp" }, 621 {LLM_TENSOR_DEC_FFN_GATE_EXPS ,"dec.blk.%d.ffn_gate" }, 622 {LLM_TENSOR_DEC_FFN_DOWN_EXPS ,"dec.blk.%d.ffn_down" }, 623 {LLM_TENSOR_DEC_FFN_UP_EXPS ,"dec.blk.%d.ffn_up" }, 624 {LLM_TENSOR_DEC_EXP_PROBS_B ,"alice.dec.blk.%d.exp_probs_b" }, 625 {LLM_TENSOR_DEC_FFN_OUTPUT_B ,"alice.dec.blk.%d.ffn_output_b" }, 626 {LLM_TENSOR_TIME_MIX_W1 ,"blk.%d.time_mix_w1" }, 627 {LLM_TENSOR_TIME_MIX_W2 ,"blk.%d.time_mix_w2" }, 628 {LLM_TENSOR_TIME_MIX_LERP_X ,"blk.%d.time_mix_lerp_x" }, 629 {LLM_TENSOR_TIME_MIX_LERP_W ,"blk.%d.time_mix_lerp_w" }, 630 {LLM_TENSOR_TIME_MIX_LERP_K ,"blk.%d.time_mix_lerp_k" }, 631 {LLM_TENSOR_TIME_MIX_LERP_V ,"blk.%d.time_mix_lerp_v" }, 632 {LLM_TENSOR_TIME_MIX_LERP_R ,"blk.%d.time_mix_lerp_r" }, 633 {LLM_TENSOR_TIME_MIX_LERP_G ,"blk.%d.time_mix_lerp_g" }, 634 {LLM_TENSOR_TIME_MIX_LERP_FUSED ,"blk.%d.time_mix_lerp_fused" }, 635 {LLM_TENSOR_TIME_MIX_FIRST ,"blk.%d.time_mix_first" }, 636 {LLM_TENSOR_TIME_MIX_DECAY ,"blk.%d.time_mix_decay" }, 637 {LLM_TENSOR_TIME_MIX_DECAY_W1 ,"blk.%d.time_mix_decay_w1" }, 638 {LLM_TENSOR_TIME_MIX_DECAY_W2 ,"blk.%d.time_mix_decay_w2" }, 639 {LLM_TENSOR_TIME_MIX_KEY ,"blk.%d.time_mix_key" }, 640 {LLM_TENSOR_TIME_MIX_VALUE ,"blk.%d.time_mix_value" }, 641 {LLM_TENSOR_TIME_MIX_RECEPTANCE ,"blk.%d.time_mix_receptance" }, 642 {LLM_TENSOR_TIME_MIX_GATE ,"blk.%d.time_mix_gate" }, 643 {LLM_TENSOR_TIME_MIX_LN ,"blk.%d.time_mix_ln" }, 644 {LLM_TENSOR_TIME_MIX_OUTPUT ,"blk.%d.time_mix_output" }, 645 {LLM_TENSOR_CHANNEL_MIX_LERP_K ,"blk.%d.channel_mix_lerp_k" }, 646 {LLM_TENSOR_CHANNEL_MIX_LERP_R ,"blk.%d.channel_mix_lerp_r" }, 647 {LLM_TENSOR_CHANNEL_MIX_KEY ,"blk.%d.channel_mix_key" }, 648 {LLM_TENSOR_CHANNEL_MIX_VALUE ,"blk.%d.channel_mix_value" }, 649 {LLM_TENSOR_CHANNEL_MIX_RECEPTANCE ,"blk.%d.channel_mix_receptance" }, 650 {LLM_TENSOR_TIME_MIX_W0 ,"blk.%d.time_mix_w0" }, 651 {LLM_TENSOR_TIME_MIX_A0 ,"blk.%d.time_mix_a0" }, 652 {LLM_TENSOR_TIME_MIX_A1 ,"blk.%d.time_mix_a1" }, 653 {LLM_TENSOR_TIME_MIX_A2 ,"blk.%d.time_mix_a2" }, 654 {LLM_TENSOR_TIME_MIX_V0 ,"blk.%d.time_mix_v0" }, 655 {LLM_TENSOR_TIME_MIX_V1 ,"blk.%d.time_mix_v1" }, 656 {LLM_TENSOR_TIME_MIX_V2 ,"blk.%d.time_mix_v2" }, 657 {LLM_TENSOR_TIME_MIX_G1 ,"blk.%d.time_mix_g1" }, 658 {LLM_TENSOR_TIME_MIX_G2 ,"blk.%d.time_mix_g2" }, 659 {LLM_TENSOR_TIME_MIX_K_K ,"blk.%d.time_mix_k_k" }, 660 {LLM_TENSOR_TIME_MIX_K_A ,"blk.%d.time_mix_k_a" }, 661 {LLM_TENSOR_TIME_MIX_R_K ,"blk.%d.time_mix_r_k" }, 662 {LLM_TENSOR_CONV1D ,"conv1d" }, 663 {LLM_TENSOR_CONVNEXT_DW ,"convnext.%d.dw" }, 664 {LLM_TENSOR_CONVNEXT_NORM ,"convnext.%d.norm" }, 665 {LLM_TENSOR_CONVNEXT_PW1 ,"convnext.%d.pw1" }, 666 {LLM_TENSOR_CONVNEXT_PW2 ,"convnext.%d.pw2" }, 667 {LLM_TENSOR_CONVNEXT_GAMMA ,"convnext.%d.gamma" }, 668 {LLM_TENSOR_POS_NET_CONV1 ,"posnet.%d.conv1" }, 669 {LLM_TENSOR_POS_NET_CONV2 ,"posnet.%d.conv2" }, 670 {LLM_TENSOR_POS_NET_NORM ,"posnet.%d.norm" }, 671 {LLM_TENSOR_POS_NET_NORM1 ,"posnet.%d.norm1" }, 672 {LLM_TENSOR_POS_NET_NORM2 ,"posnet.%d.norm2" }, 673 {LLM_TENSOR_POS_NET_ATTN_NORM ,"posnet.%d.attn_norm" }, 674 {LLM_TENSOR_POS_NET_ATTN_Q ,"posnet.%d.attn_q" }, 675 {LLM_TENSOR_POS_NET_ATTN_K ,"posnet.%d.attn_k" }, 676 {LLM_TENSOR_POS_NET_ATTN_V ,"posnet.%d.attn_v" }, 677 {LLM_TENSOR_POS_NET_ATTN_OUT ,"posnet.%d.attn_output" }, 678 {LLM_TENSOR_ATTN_SINKS ,"blk.%d.attn_sinks" }, 679 {LLM_TENSOR_SHORTCONV_CONV ,"blk.%d.shortconv.conv" }, 680 {LLM_TENSOR_SHORTCONV_INPROJ ,"blk.%d.shortconv.in_proj" }, 681 {LLM_TENSOR_SHORTCONV_OUTPROJ ,"blk.%d.shortconv.out_proj" }, 682 {LLM_TENSOR_FFN_GATE_CHEXPS ,"blk.%d.ffn_gate_chexps" }, 683 {LLM_TENSOR_FFN_DOWN_CHEXPS ,"blk.%d.ffn_down_chexps" }, 684 {LLM_TENSOR_FFN_UP_CHEXPS ,"blk.%d.ffn_up_chexps" }, 685 {LLM_TENSOR_VISEXP_ATTN_QKV ,"blk.%d.vis_attn_qkv" }, 686 {LLM_TENSOR_VISEXP_ATTN_OUT ,"blk.%d.vis_attn_output" }, 687 {LLM_TENSOR_VISEXP_FFN_GATE ,"blk.%d.vis_gate" }, 688 {LLM_TENSOR_VISEXP_FFN_DOWN ,"blk.%d.vis_down" }, 689 {LLM_TENSOR_VISEXP_FFN_UP ,"blk.%d.vis_up" }, 690 {LLM_TENSOR_INDEXER_K_NORM ,"blk.%d.indexer.k_norm" }, 691 {LLM_TENSOR_INDEXER_PROJ ,"blk.%d.indexer.proj" }, 692 {LLM_TENSOR_INDEXER_ATTN_K ,"blk.%d.indexer.attn_k" }, 693 {LLM_TENSOR_INDEXER_ATTN_Q_B ,"blk.%d.indexer.attn_q_b" }, 694 {LLM_TENSOR_INDEXER_Q_PROJ ,"blk.%d.indexer.q_proj" }, 695 {LLM_TENSOR_INDEXER_K_PROJ ,"blk.%d.indexer.k_proj" }, 696 {LLM_TENSOR_INDEXER_Q_NORM ,"blk.%d.indexer.q_norm" }, 697 {LLM_TENSOR_INDEXER_COMPRESSOR_WKV ,"blk.%d.indexer_compressor_kv" }, 698 {LLM_TENSOR_INDEXER_COMPRESSOR_WGATE ,"blk.%d.indexer_compressor_gate" }, 699 {LLM_TENSOR_INDEXER_COMPRESSOR_APE ,"blk.%d.indexer_compressor_ape" }, 700 {LLM_TENSOR_INDEXER_COMPRESSOR_NORM ,"blk.%d.indexer_compressor_norm" }, 701 {LLM_TENSOR_FFN_GATE_TID2EID ,"blk.%d.ffn_gate_tid2eid" }, 702 {LLM_TENSOR_MASKED_EMBD_CENTROIDS ,"masked_embd_centroids" }, 703 {LLM_TENSOR_MASKED_EMBD_ORDERING ,"masked_embd_ordering" }, 704 {LLM_TENSOR_FC ,"fc" }, 705 {LLM_TENSOR_D2T ,"d2t" }, 706 {LLM_TENSOR_DSPARK_MARKOV_W1 ,"markov_w1" }, 707 {LLM_TENSOR_DSPARK_MARKOV_W2 ,"markov_w2" }, 708 {LLM_TENSOR_DSPARK_CONF_PROJ ,"conf_proj" }, 709 {LLM_TENSOR_DFLASH_ATTN_CONV_BASE ,"blk.%d.attn_conv_base" }, 710 {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ ,"blk.%d.attn_conv_proj" }, 711 {LLM_TENSOR_DFLASH_FFN_CONV_BASE ,"blk.%d.ffn_conv_base" }, 712 {LLM_TENSOR_DFLASH_FFN_CONV_PROJ ,"blk.%d.ffn_conv_proj" }, 713 {LLM_TENSOR_DFLASH_SELECTOR_PREV ,"selector_predecessor" }, 714 {LLM_TENSOR_DFLASH_SELECTOR_NEXT ,"selector_successor" }, 715 {LLM_TENSOR_DFLASH_SELECTOR_HIDDEN ,"selector_hidden" }, 716}; 717 718// declare information about the model weight tensors: 719// - the layer in which the tensor is going to be used. this is needed in order to assign the correct buffer type for the weight 720// - the operator which is going to use the weight. this is needed to determine if the respective backend supports the operator 721// 722// for example, input layers are usually assigned to CPU/host buffer types 723// 724// a mismatch between the declared information and the actual layer/op in which the tensor is used can lead to sub-optimal 725// assignment of the buffer types and extra overhead during computation 726// example: https://github.com/ggml-org/llama.cpp/pull/17548 727// 728static const std::map < llm_tensor ,llm_tensor_info > LLM_TENSOR_INFOS = { 729 {LLM_TENSOR_TOKEN_EMBD , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_GET_ROWS }}, 730 {LLM_TENSOR_POS_EMBD , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_GET_ROWS }}, 731 {LLM_TENSOR_TOKEN_TYPES , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_GET_ROWS }}, 732 {LLM_TENSOR_TOKEN_EMBD_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }},// do the norms on the first layer (not the input layer) 733 {LLM_TENSOR_OUTPUT , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 734 {LLM_TENSOR_CLS , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 735 {LLM_TENSOR_CLS_OUT , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 736 {LLM_TENSOR_CLS_NORM , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 737 {LLM_TENSOR_DENSE_2_OUT , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }},// Dense layer output 738 {LLM_TENSOR_DENSE_3_OUT , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }},// Dense layer output 739 {LLM_TENSOR_OUTPUT_NORM , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 740 {LLM_TENSOR_OUTPUT_NORM_LFM2 , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 741 {LLM_TENSOR_DEC_OUTPUT_NORM , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 742 {LLM_TENSOR_ENC_OUTPUT_NORM , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 743 {LLM_TENSOR_ROPE_FREQS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ROPE }}, 744 {LLM_TENSOR_ROPE_FACTORS_LONG , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ROPE }}, 745 {LLM_TENSOR_ROPE_FACTORS_SHORT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ROPE }}, 746 {LLM_TENSOR_ATTN_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 747 {LLM_TENSOR_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 748 {LLM_TENSOR_ATTN_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 749 {LLM_TENSOR_ATTN_QKV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 750 {LLM_TENSOR_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 751 {LLM_TENSOR_ATTN_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 752 {LLM_TENSOR_FFN_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 753 {LLM_TENSOR_FFN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 754 {LLM_TENSOR_FFN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 755 {LLM_TENSOR_FFN_DOWN_SHEXP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 756 {LLM_TENSOR_FFN_GATE_SHEXP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 757 {LLM_TENSOR_FFN_UP_SHEXP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 758 {LLM_TENSOR_ATTN_Q_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 759 {LLM_TENSOR_ATTN_Q_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 760 {LLM_TENSOR_ATTN_KV_A_MQA , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 761 {LLM_TENSOR_ATTN_KV_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 762 {LLM_TENSOR_ATTN_KV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 763 {LLM_TENSOR_ATTN_KV_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 764 {LLM_TENSOR_ATTN_OUT_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 765 {LLM_TENSOR_ATTN_OUT_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 766 {LLM_TENSOR_HC_HEAD_FN , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 767 {LLM_TENSOR_HC_HEAD_BASE , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_ADD }}, 768 {LLM_TENSOR_HC_HEAD_SCALE , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 769 {LLM_TENSOR_HC_HEAD_NORM , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 770 {LLM_TENSOR_HC_HEAD_DOWN , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 771 {LLM_TENSOR_HC_HEAD_UP , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 772 {LLM_TENSOR_HC_ATTN_FN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 773 {LLM_TENSOR_HC_ATTN_BASE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 774 {LLM_TENSOR_HC_ATTN_SCALE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 775 {LLM_TENSOR_HC_FFN_FN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 776 {LLM_TENSOR_HC_FFN_BASE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 777 {LLM_TENSOR_HC_FFN_SCALE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 778 {LLM_TENSOR_HC_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 779 {LLM_TENSOR_HC_ATTN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 780 {LLM_TENSOR_HC_ATTN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 781 {LLM_TENSOR_HC_ATTN_INJECT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 782 {LLM_TENSOR_HC_FFN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 783 {LLM_TENSOR_HC_FFN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 784 {LLM_TENSOR_HC_FFN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 785 {LLM_TENSOR_HC_FFN_INJECT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 786 {LLM_TENSOR_PLE_KEY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 787 {LLM_TENSOR_PLE_VALUE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 788 {LLM_TENSOR_PLE_NORM_KEY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 789 {LLM_TENSOR_PLE_NORM_QUERY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 790 {LLM_TENSOR_PLE_NORM_CONV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 791 {LLM_TENSOR_PLE_CONV1D , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_SSM_CONV }}, 792 {LLM_TENSOR_ATTN_COMPRESSOR_WKV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 793 {LLM_TENSOR_ATTN_COMPRESSOR_WGATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 794 {LLM_TENSOR_ATTN_COMPRESSOR_APE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 795 {LLM_TENSOR_ATTN_COMPRESSOR_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 796 {LLM_TENSOR_ATTN_K_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 797 {LLM_TENSOR_ATTN_V_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 798 {LLM_TENSOR_ATTN_SINKS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_SCALE }}, 799 {LLM_TENSOR_DEC_ATTN_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 800 {LLM_TENSOR_DEC_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 801 {LLM_TENSOR_DEC_ATTN_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 802 {LLM_TENSOR_DEC_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 803 {LLM_TENSOR_DEC_CROSS_ATTN_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 804 {LLM_TENSOR_DEC_CROSS_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 805 {LLM_TENSOR_DEC_CROSS_ATTN_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 806 {LLM_TENSOR_DEC_CROSS_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 807 {LLM_TENSOR_DEC_FFN_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 808 {LLM_TENSOR_DEC_FFN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 809 {LLM_TENSOR_DEC_FFN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 810 {LLM_TENSOR_ENC_ATTN_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 811 {LLM_TENSOR_ENC_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 812 {LLM_TENSOR_ENC_ATTN_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 813 {LLM_TENSOR_ENC_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 814 {LLM_TENSOR_ENC_FFN_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 815 {LLM_TENSOR_ENC_FFN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 816 {LLM_TENSOR_ENC_FFN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 817 {LLM_TENSOR_ENC_FFN_GATE_INP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 818 {LLM_TENSOR_ENC_FFN_GATE_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 819 {LLM_TENSOR_ENC_FFN_DOWN_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 820 {LLM_TENSOR_ENC_FFN_UP_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 821 {LLM_TENSOR_ENC_EXP_PROBS_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 822 {LLM_TENSOR_ENC_FFN_OUTPUT_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 823 {LLM_TENSOR_DEC_FFN_GATE_INP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 824 {LLM_TENSOR_DEC_FFN_GATE_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 825 {LLM_TENSOR_DEC_FFN_DOWN_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 826 {LLM_TENSOR_DEC_FFN_UP_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 827 {LLM_TENSOR_DEC_EXP_PROBS_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 828 {LLM_TENSOR_DEC_FFN_OUTPUT_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 829 {LLM_TENSOR_FFN_GATE_INP_SHEXP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 830 {LLM_TENSOR_FFN_GATE_INP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 831 {LLM_TENSOR_SSM_IN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 832 {LLM_TENSOR_SSM_X , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 833 {LLM_TENSOR_SSM_DT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 834 {LLM_TENSOR_SSM_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 835 {LLM_TENSOR_SSM_ALPHA , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 836 {LLM_TENSOR_SSM_BETA_ALPHA , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 837 {LLM_TENSOR_TIME_MIX_W1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 838 {LLM_TENSOR_TIME_MIX_W2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 839 {LLM_TENSOR_TIME_MIX_A1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 840 {LLM_TENSOR_TIME_MIX_A2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 841 {LLM_TENSOR_TIME_MIX_V1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 842 {LLM_TENSOR_TIME_MIX_V2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 843 {LLM_TENSOR_TIME_MIX_G1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 844 {LLM_TENSOR_TIME_MIX_G2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 845 {LLM_TENSOR_TIME_MIX_DECAY_W1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 846 {LLM_TENSOR_TIME_MIX_DECAY_W2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 847 {LLM_TENSOR_TIME_MIX_KEY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 848 {LLM_TENSOR_TIME_MIX_VALUE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 849 {LLM_TENSOR_TIME_MIX_RECEPTANCE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 850 {LLM_TENSOR_TIME_MIX_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 851 {LLM_TENSOR_TIME_MIX_OUTPUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 852 {LLM_TENSOR_CHANNEL_MIX_KEY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 853 {LLM_TENSOR_CHANNEL_MIX_RECEPTANCE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 854 {LLM_TENSOR_CHANNEL_MIX_VALUE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 855 {LLM_TENSOR_FFN_ACT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_DIV }}, 856 {LLM_TENSOR_SSM_CONV1D , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_SSM_CONV }}, 857 {LLM_TENSOR_SSM_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_SSM_SCAN }}, 858 {LLM_TENSOR_SSM_A_NOSCAN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }},// a version of SSM_A used for MUL instead of SSM_SCAN 859 {LLM_TENSOR_SSM_DT_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 860 {LLM_TENSOR_SSM_B_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 861 {LLM_TENSOR_SSM_C_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 862 {LLM_TENSOR_SSM_D , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 863 {LLM_TENSOR_SSM_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 864// Kimi KDA - Conv tensors are 4D [d_conv, 1, d_inner, 1], reshaped to 2D at runtime 865 {LLM_TENSOR_SSM_CONV1D_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 866 {LLM_TENSOR_SSM_CONV1D_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 867 {LLM_TENSOR_SSM_CONV1D_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 868 {LLM_TENSOR_SSM_F_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 869 {LLM_TENSOR_SSM_F_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 870 {LLM_TENSOR_SSM_BETA , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 871 {LLM_TENSOR_SSM_G_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 872 {LLM_TENSOR_SSM_G , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 873 {LLM_TENSOR_ATTN_RES_SCORE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 874 {LLM_TENSOR_FFN_RES_SCORE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 875 {LLM_TENSOR_OUTPUT_RES_SCORE , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL }}, 876 {LLM_TENSOR_FFN_ROUTED_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 877 {LLM_TENSOR_FFN_ROUTED_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 878 {LLM_TENSOR_FFN_ROUTED_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 879 {LLM_TENSOR_SSM_G_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 880 {LLM_TENSOR_TIME_MIX_LERP_X , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 881 {LLM_TENSOR_TIME_MIX_LN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 882 {LLM_TENSOR_CHANNEL_MIX_LERP_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 883 {LLM_TENSOR_CHANNEL_MIX_LERP_R , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 884 {LLM_TENSOR_TIME_MIX_K_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 885 {LLM_TENSOR_TIME_MIX_K_A , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 886 {LLM_TENSOR_TIME_MIX_R_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 887 {LLM_TENSOR_TIME_MIX_LERP_W , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 888 {LLM_TENSOR_TIME_MIX_LERP_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 889 {LLM_TENSOR_TIME_MIX_LERP_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 890 {LLM_TENSOR_TIME_MIX_LERP_R , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 891 {LLM_TENSOR_TIME_MIX_LERP_G , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 892 {LLM_TENSOR_TIME_MIX_LERP_FUSED , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 893 {LLM_TENSOR_TIME_MIX_DECAY , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 894 {LLM_TENSOR_TIME_MIX_W0 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 895 {LLM_TENSOR_TIME_MIX_A0 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 896 {LLM_TENSOR_TIME_MIX_V0 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 897 {LLM_TENSOR_TIME_MIX_FIRST , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_RWKV_WKV6 }}, 898 {LLM_TENSOR_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 899 {LLM_TENSOR_ATTN_NORM_2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 900 {LLM_TENSOR_ATTN_OUT_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 901 {LLM_TENSOR_ATTN_POST_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 902 {LLM_TENSOR_FFN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 903 {LLM_TENSOR_FFN_PRE_NORM_2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 904 {LLM_TENSOR_FFN_POST_NORM_1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 905 {LLM_TENSOR_FFN_POST_NORM_2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 906 {LLM_TENSOR_FFN_POST_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 907 {LLM_TENSOR_FFN_NORM_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 908 {LLM_TENSOR_ATTN_Q_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 909 {LLM_TENSOR_ATTN_K_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 910 {LLM_TENSOR_LAYER_OUT_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 911 {LLM_TENSOR_LAYER_OUT_SCALE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 912 {LLM_TENSOR_ATTN_Q_A_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 913 {LLM_TENSOR_ATTN_KV_A_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 914 {LLM_TENSOR_ATTN_SUB_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 915 {LLM_TENSOR_FFN_SUB_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 916 {LLM_TENSOR_DEC_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 917 {LLM_TENSOR_DEC_CROSS_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 918 {LLM_TENSOR_DEC_FFN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 919 {LLM_TENSOR_ENC_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 920 {LLM_TENSOR_ENC_FFN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 921 {LLM_TENSOR_DEC_ATTN_REL_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 922 {LLM_TENSOR_ENC_ATTN_REL_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 923 {LLM_TENSOR_FFN_DOWN_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 924 {LLM_TENSOR_FFN_GATE_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 925 {LLM_TENSOR_FFN_UP_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 926 {LLM_TENSOR_FFN_GATE_UP_EXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 927 {LLM_TENSOR_FFN_DOWN_CHEXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 928 {LLM_TENSOR_FFN_GATE_CHEXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 929 {LLM_TENSOR_FFN_UP_CHEXPS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT_ID }}, 930 {LLM_TENSOR_FFN_EXP_PROBS_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 931 {LLM_TENSOR_FFN_EXP_PROBS_B_VL , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_ADD }}, 932// altup / laurel (gemma 3n) 933 {LLM_TENSOR_PER_LAYER_TOKEN_EMBD , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_GET_ROWS }}, 934 {LLM_TENSOR_PER_LAYER_MODEL_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 935 {LLM_TENSOR_PER_LAYER_PROJ_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 936 {LLM_TENSOR_ALTUP_PROJ , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 937 {LLM_TENSOR_ALTUP_UNEMBD_PROJ , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 938 {LLM_TENSOR_PER_LAYER_INP_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 939 {LLM_TENSOR_PER_LAYER_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 940 {LLM_TENSOR_PER_LAYER_POST_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 941 {LLM_TENSOR_ALTUP_CORRECT_COEF , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 942 {LLM_TENSOR_ALTUP_CORRECT_SCALE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 943 {LLM_TENSOR_ALTUP_PREDICT_COEF , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 944 {LLM_TENSOR_ALTUP_ROUTER , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 945 {LLM_TENSOR_ALTUP_ROUTER_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 946 {LLM_TENSOR_LAUREL_L , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 947 {LLM_TENSOR_LAUREL_R , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 948 {LLM_TENSOR_LAUREL_POST_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 949// this tensor is loaded for T5, but never used 950 {LLM_TENSOR_DEC_CROSS_ATTN_REL_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_NONE }}, 951 {LLM_TENSOR_CONV1D , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_IM2COL }}, 952 {LLM_TENSOR_POS_NET_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 953 {LLM_TENSOR_POS_NET_NORM1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 954 {LLM_TENSOR_POS_NET_NORM2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 955 {LLM_TENSOR_POS_NET_CONV1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_IM2COL }}, 956 {LLM_TENSOR_POS_NET_CONV2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_IM2COL }}, 957 {LLM_TENSOR_POS_NET_ATTN_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 958 {LLM_TENSOR_POS_NET_ATTN_Q , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 959 {LLM_TENSOR_POS_NET_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 960 {LLM_TENSOR_POS_NET_ATTN_V , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 961 {LLM_TENSOR_POS_NET_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 962 {LLM_TENSOR_CONVNEXT_DW , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_IM2COL }}, 963 {LLM_TENSOR_CONVNEXT_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 964 {LLM_TENSOR_CONVNEXT_PW1 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 965 {LLM_TENSOR_CONVNEXT_PW2 , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 966 {LLM_TENSOR_CONVNEXT_GAMMA , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 967 {LLM_TENSOR_SHORTCONV_CONV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_SSM_CONV }}, 968 {LLM_TENSOR_SHORTCONV_INPROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 969 {LLM_TENSOR_SHORTCONV_OUTPROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 970 {LLM_TENSOR_VISEXP_ATTN_QKV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 971 {LLM_TENSOR_VISEXP_ATTN_OUT , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 972 {LLM_TENSOR_VISEXP_FFN_GATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 973 {LLM_TENSOR_VISEXP_FFN_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 974 {LLM_TENSOR_VISEXP_FFN_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 975 {LLM_TENSOR_INDEXER_K_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 976 {LLM_TENSOR_INDEXER_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 977 {LLM_TENSOR_INDEXER_ATTN_K , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 978 {LLM_TENSOR_INDEXER_ATTN_Q_B , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 979 {LLM_TENSOR_INDEXER_Q_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 980 {LLM_TENSOR_INDEXER_K_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 981 {LLM_TENSOR_INDEXER_Q_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 982 {LLM_TENSOR_INDEXER_COMPRESSOR_WKV , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 983 {LLM_TENSOR_INDEXER_COMPRESSOR_WGATE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 984 {LLM_TENSOR_INDEXER_COMPRESSOR_APE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 985 {LLM_TENSOR_INDEXER_COMPRESSOR_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 986 {LLM_TENSOR_FFN_GATE_TID2EID , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 987 {LLM_TENSOR_NEXTN_PROJ_PRE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 988 {LLM_TENSOR_NEXTN_PROJ_POST , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 989// NextN/MTP tensors are stored per-block (blk.%d.nextn.*) even though only the 990// last nextn_predict_layers blocks carry them. Classify as LAYER_REPEATING so 991// the model loader doesn't fault on the block index. 992 {LLM_TENSOR_NEXTN_EH_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 993 {LLM_TENSOR_NEXTN_EMBED_TOKENS , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_GET_ROWS }}, 994 {LLM_TENSOR_NEXTN_ENORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 995 {LLM_TENSOR_NEXTN_HNORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 996 {LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 997 {LLM_TENSOR_NEXTN_SHARED_HEAD_NORM , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 998// Nemotron 3 Super 999// latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU 1000 {LLM_TENSOR_FFN_LATENT_DOWN , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 1001 {LLM_TENSOR_FFN_LATENT_UP , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 1002 {LLM_TENSOR_MASKED_EMBD_CENTROIDS , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_NONE }}, 1003 {LLM_TENSOR_MASKED_EMBD_ORDERING , {LLM_TENSOR_LAYER_INPUT ,GGML_OP_NONE }}, 1004// eagle3 1005 {LLM_TENSOR_FC , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 1006 {LLM_TENSOR_D2T , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_GET_ROWS }}, 1007// dspark 1008 {LLM_TENSOR_DSPARK_MARKOV_W1 , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_GET_ROWS }}, 1009 {LLM_TENSOR_DSPARK_MARKOV_W2 , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 1010 {LLM_TENSOR_DSPARK_CONF_PROJ , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 1011 {LLM_TENSOR_DFLASH_ATTN_CONV_BASE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 1012 {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 1013 {LLM_TENSOR_DFLASH_FFN_CONV_BASE , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL }}, 1014 {LLM_TENSOR_DFLASH_FFN_CONV_PROJ , {LLM_TENSOR_LAYER_REPEATING ,GGML_OP_MUL_MAT }}, 1015 {LLM_TENSOR_DFLASH_SELECTOR_PREV , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_GET_ROWS }}, 1016 {LLM_TENSOR_DFLASH_SELECTOR_NEXT , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_GET_ROWS }}, 1017 {LLM_TENSOR_DFLASH_SELECTOR_HIDDEN , {LLM_TENSOR_LAYER_OUTPUT ,GGML_OP_MUL_MAT }}, 1018}; 1019 1020LLM_KV ::LLM_KV (llm_arch arch ,const char * suffix ) :arch (arch ),suffix (suffix ) {} 1021 1022std::string LLM_KV ::operator()(llm_kv kv )const { 1023 std::string name = ::format (LLM_KV_NAMES .at (kv ),LLM_ARCH_NAMES .at (arch )); 1024 1025if (suffix != nullptr ) { 1026name += "." ; 1027name += suffix ; 1028 } 1029 1030return name ; 1031} 1032 1033LLM_TN_IMPL ::LLM_TN_IMPL (llm_arch arch ,llm_tensor tensor ,const char * suffix ,int bid ,int xid ) 1034 :arch (arch ),tensor (tensor ),suffix (suffix ),bid (bid ),xid (xid ) {} 1035 1036std::string LLM_TN_IMPL ::str ()const { 1037if (LLM_TENSOR_NAMES .find (tensor )== LLM_TENSOR_NAMES .end ()) { 1038GGML_ABORT ("unknown tensor name for tensor id %d" ,static_cast < int > (tensor )); 1039 } 1040 1041 std::string name = ::format (LLM_TENSOR_NAMES .at (tensor ),bid ,xid ); 1042if (suffix != nullptr ) { 1043name += "." ; 1044name += suffix ; 1045 } 1046 1047return name ; 1048} 1049 1050std::vector < llm_arch > llm_arch_all () { 1051 std::vector < llm_arch > ret ; 1052ret .reserve (LLM_ARCH_NAMES .size ()); 1053for (const auto & [arch ,_ ] :LLM_ARCH_NAMES ) { 1054ret .push_back (arch ); 1055 } 1056return ret ; 1057} 1058 1059const char * llm_arch_name (llm_arch arch ) { 1060auto it = LLM_ARCH_NAMES .find (arch ); 1061if (it == LLM_ARCH_NAMES .end ()) { 1062return "unknown" ; 1063 } 1064return it -> second ; 1065} 1066 1067llm_arch llm_arch_from_string (const std::string & name ) { 1068for (const auto & kv :LLM_ARCH_NAMES ) {// NOLINT 1069if (kv .second == name ) { 1070return kv .first ; 1071 } 1072 } 1073 1074return LLM_ARCH_UNKNOWN ; 1075} 1076 1077const llm_tensor_info & llm_tensor_info_for (llm_tensor tensor ) { 1078return LLM_TENSOR_INFOS .at (tensor ); 1079} 1080 1081bool llm_arch_is_recurrent (const llm_arch & arch ) { 1082switch (arch ) { 1083case LLM_ARCH_MAMBA : 1084case LLM_ARCH_MAMBA2 : 1085case LLM_ARCH_RWKV6 : 1086case LLM_ARCH_RWKV6QWEN2 : 1087case LLM_ARCH_RWKV7 : 1088case LLM_ARCH_ARWKV7 : 1089return true; 1090default : 1091return false; 1092 } 1093} 1094 1095bool llm_arch_is_hybrid (const llm_arch & arch ) { 1096switch (arch ) { 1097case LLM_ARCH_JAMBA : 1098case LLM_ARCH_FALCON_H1 : 1099case LLM_ARCH_PLAMO2 : 1100case LLM_ARCH_GRANITE_HYBRID : 1101case LLM_ARCH_LFM2 : 1102case LLM_ARCH_LFM2MOE : 1103case LLM_ARCH_NEMOTRON_H : 1104case LLM_ARCH_NEMOTRON_H_MOE : 1105case LLM_ARCH_QWEN3NEXT : 1106case LLM_ARCH_KIMI_LINEAR : 1107case LLM_ARCH_BAILINGMOE3 : 1108case LLM_ARCH_KIMI_K3 : 1109case LLM_ARCH_QWEN35 : 1110case LLM_ARCH_QWEN35MOE : 1111case LLM_ARCH_QWEN4EXP : 1112case LLM_ARCH_DEEPSEEK4 : 1113case LLM_ARCH_MINIMAX_01 : 1114return true; 1115default : 1116return false; 1117 } 1118} 1119 1120bool llm_arch_is_diffusion (const llm_arch & arch ) { 1121switch (arch ) { 1122case LLM_ARCH_DREAM : 1123case LLM_ARCH_LLADA : 1124case LLM_ARCH_LLADA_MOE : 1125case LLM_ARCH_RND1 : 1126return true; 1127default : 1128return false; 1129 } 1130} 1131 1132bool llm_arch_supports_rs_rollback (const llm_arch & arch ) { 1133switch (arch ) { 1134case LLM_ARCH_KIMI_K3 : 1135case LLM_ARCH_QWEN35 : 1136case LLM_ARCH_QWEN35MOE : 1137case LLM_ARCH_QWEN4EXP : 1138case LLM_ARCH_DEEPSEEK4 : 1139case LLM_ARCH_NEMOTRON_H : 1140case LLM_ARCH_NEMOTRON_H_MOE : 1141case LLM_ARCH_LFM2 : 1142case LLM_ARCH_LFM2MOE : 1143case LLM_ARCH_BAILINGMOE3 : 1144return true; 1145default : 1146return false; 1147 } 1148} 1149 1150bool llm_arch_supports_sm_tensor (const llm_arch & arch ) { 1151switch (arch ) { 1152case LLM_ARCH_GROK : 1153case LLM_ARCH_MPT : 1154case LLM_ARCH_PLAMO2 : 1155case LLM_ARCH_MINICPM3 : 1156case LLM_ARCH_GEMMA3N : 1157case LLM_ARCH_MAMBA : 1158case LLM_ARCH_MAMBA2 : 1159case LLM_ARCH_JAMBA : 1160case LLM_ARCH_FALCON_H1 : 1161case LLM_ARCH_OLMO2 : 1162case LLM_ARCH_OLMOE : 1163case LLM_ARCH_DEEPSEEK2 : 1164case LLM_ARCH_DEEPSEEK32 : 1165case LLM_ARCH_HY_V4 : 1166case LLM_ARCH_DOTS3NOTE : 1167case LLM_ARCH_GLM_DSA : 1168case LLM_ARCH_BITNET : 1169case LLM_ARCH_T5 : 1170case LLM_ARCH_ALICEAI_T5_MOE : 1171case LLM_ARCH_NEMOTRON_H : 1172case LLM_ARCH_NEMOTRON_H_MOE : 1173case LLM_ARCH_GRANITE_HYBRID : 1174case LLM_ARCH_MINIMAX_01 : 1175case LLM_ARCH_MINIMAX_M2 : 1176case LLM_ARCH_MINIMAX_M3 : 1177case LLM_ARCH_MISTRAL4 : 1178case LLM_ARCH_KIMI_LINEAR : 1179case LLM_ARCH_BAILINGMOE3 : 1180case LLM_ARCH_KIMI_K3 : 1181case LLM_ARCH_QWEN3TTS : 1182case LLM_ARCH_QWEN4EXP :// TODO: fix test-llama-archs 1183return false; 1184default : 1185return true; 1186 } 1187}