char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Charlotte Somsupport aliceai_t5_moe98a01afd9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
23.4 KiB805 linesraw
1#pragma once
2
3#include "ggml.h" // ggml_op
4
5#include <string>
6#include <set>
7#include <vector>
8
9//
10// gguf constants (sync with gguf.py)
11//
12
13enum llm_arch {
14    LLM_ARCH_CLIP,
15    LLM_ARCH_LLAMA,
16    LLM_ARCH_LLAMA4,
17    LLM_ARCH_DECI,
18    LLM_ARCH_FALCON,
19    LLM_ARCH_BAICHUAN,
20    LLM_ARCH_GROK,
21    LLM_ARCH_GPT2,
22    LLM_ARCH_GPTJ,
23    LLM_ARCH_GPTNEOX,
24    LLM_ARCH_MPT,
25    LLM_ARCH_STARCODER,
26    LLM_ARCH_REFACT,
27    LLM_ARCH_BERT,
28    LLM_ARCH_MODERN_BERT,
29    LLM_ARCH_NOMIC_BERT,
30    LLM_ARCH_NOMIC_BERT_MOE,
31    LLM_ARCH_NEO_BERT,
32    LLM_ARCH_JINA_BERT_V2,
33    LLM_ARCH_JINA_BERT_V3,
34    LLM_ARCH_EUROBERT,
35    LLM_ARCH_BLOOM,
36    LLM_ARCH_STABLELM,
37    LLM_ARCH_QWEN,
38    LLM_ARCH_QWEN2,
39    LLM_ARCH_QWEN2MOE,
40    LLM_ARCH_QWEN2VL,
41    LLM_ARCH_QWEN3,
42    LLM_ARCH_QWEN3MOE,
43    LLM_ARCH_QWEN3NEXT,
44    LLM_ARCH_QWEN3VL,
45    LLM_ARCH_QWEN3VLMOE,
46    LLM_ARCH_QWEN35,
47    LLM_ARCH_QWEN35MOE,
48    LLM_ARCH_QWEN4EXP,
49    LLM_ARCH_PHI2,
50    LLM_ARCH_PHI3,
51    LLM_ARCH_PHIMOE,
52    LLM_ARCH_PLAMO,
53    LLM_ARCH_PLAMO2,
54    LLM_ARCH_PLAMO3,
55    LLM_ARCH_CODESHELL,
56    LLM_ARCH_ORION,
57    LLM_ARCH_INTERNLM2,
58    LLM_ARCH_MINICPM,
59    LLM_ARCH_MINICPM3,
60    LLM_ARCH_GEMMA,
61    LLM_ARCH_GEMMA2,
62    LLM_ARCH_GEMMA3,
63    LLM_ARCH_GEMMA3N,
64    LLM_ARCH_GEMMA4,
65    LLM_ARCH_GEMMA4_ASSISTANT,
66    LLM_ARCH_GEMMA_EMBEDDING,
67    LLM_ARCH_STARCODER2,
68    LLM_ARCH_MAMBA,
69    LLM_ARCH_MAMBA2,
70    LLM_ARCH_MAPLE,
71    LLM_ARCH_JAMBA,
72    LLM_ARCH_FALCON_H1,
73    LLM_ARCH_XVERSE,
74    LLM_ARCH_COMMAND_R,
75    LLM_ARCH_COHERE2,
76    LLM_ARCH_COHERE2MOE,
77    LLM_ARCH_DBRX,
78    LLM_ARCH_OLMO,
79    LLM_ARCH_OLMO2,
80    LLM_ARCH_OLMOE,
81    LLM_ARCH_MUSE_GLIMMER,
82    LLM_ARCH_OPENELM,
83    LLM_ARCH_ARCTIC,
84    LLM_ARCH_DEEPSEEK,
85    LLM_ARCH_DEEPSEEK2,
86    LLM_ARCH_DEEPSEEK2OCR,
87    LLM_ARCH_DEEPSEEK32,
88    LLM_ARCH_DEEPSEEK4,
89    LLM_ARCH_CHATGLM,
90    LLM_ARCH_GLM4,
91    LLM_ARCH_GLM4_MOE,
92    LLM_ARCH_GLM_DSA,
93    LLM_ARCH_BITNET,
94    LLM_ARCH_T5,
95    LLM_ARCH_T5ENCODER,
96    LLM_ARCH_ALICEAI_T5_MOE,
97    LLM_ARCH_JAIS,
98    LLM_ARCH_JAIS2,
99    LLM_ARCH_NEMOTRON,
100    LLM_ARCH_NEMOTRON_H,
101    LLM_ARCH_NEMOTRON_H_MOE,
102    LLM_ARCH_EXAONE,
103    LLM_ARCH_EXAONE4,
104    LLM_ARCH_EXAONE_MOE,
105    LLM_ARCH_RWKV6,
106    LLM_ARCH_RWKV6QWEN2,
107    LLM_ARCH_RWKV7,
108    LLM_ARCH_ARWKV7,
109    LLM_ARCH_GRANITE,
110    LLM_ARCH_GRANITE_MOE,
111    LLM_ARCH_GRANITE_HYBRID,
112    LLM_ARCH_GRANITE_SWITCH,
113    LLM_ARCH_GRANITE_SWA,
114    LLM_ARCH_CHAMELEON,
115    LLM_ARCH_WAVTOKENIZER_DEC,
116    LLM_ARCH_PLM,
117    LLM_ARCH_BAILINGMOE,
118    LLM_ARCH_BAILINGMOE2,
119    LLM_ARCH_BAILINGMOE3,
120    LLM_ARCH_DOTS1,
121    LLM_ARCH_DOTS3NOTE,
122    LLM_ARCH_ARCEE,
123    LLM_ARCH_AFMOE,
124    LLM_ARCH_LAGUNA,
125    LLM_ARCH_ERNIE4_5,
126    LLM_ARCH_ERNIE4_5_MOE,
127    LLM_ARCH_HUNYUAN_MOE,
128    LLM_ARCH_HUNYUAN_DENSE,
129    LLM_ARCH_HUNYUAN_VL,
130    LLM_ARCH_HY_V3,
131    LLM_ARCH_HY_V4,
132    LLM_ARCH_SMOLLM3,
133    LLM_ARCH_OPENAI_MOE,
134    LLM_ARCH_LFM2,
135    LLM_ARCH_LFM2MOE,
136    LLM_ARCH_DREAM,
137    LLM_ARCH_SMALLTHINKER,
138    LLM_ARCH_LLADA,
139    LLM_ARCH_LLADA_MOE,
140    LLM_ARCH_SEED_OSS,
141    LLM_ARCH_GROVEMOE,
142    LLM_ARCH_APERTUS,
143    LLM_ARCH_MINIMAX_M2,
144    LLM_ARCH_COGVLM,
145    LLM_ARCH_RND1,
146    LLM_ARCH_PANGU_EMBED,
147    LLM_ARCH_MISTRAL3,
148    LLM_ARCH_MISTRAL4,
149    LLM_ARCH_PADDLEOCR,
150    LLM_ARCH_MIMO2,
151    LLM_ARCH_STEP35,
152    LLM_ARCH_SPARK2_5,
153    LLM_ARCH_LLAMA_EMBED,
154    LLM_ARCH_MAINCODER,
155    LLM_ARCH_KIMI_LINEAR,
156    LLM_ARCH_KIMI_K3,
157    LLM_ARCH_TALKIE,
158    LLM_ARCH_MELLUM,
159    LLM_ARCH_EAGLE3,
160    LLM_ARCH_MINIMAX_M3,
161    LLM_ARCH_DFLASH,
162    LLM_ARCH_NANBEIGE,
163    LLM_ARCH_QWEN3TTS,
164    LLM_ARCH_POCKETTTS,
165    LLM_ARCH_MINIMAX_01,
166    LLM_ARCH_UNKNOWN,
167};
168
169enum llm_kv {
170    LLM_KV_GENERAL_TYPE,
171    LLM_KV_GENERAL_ARCHITECTURE,
172    LLM_KV_GENERAL_QUANTIZATION_VERSION,
173    LLM_KV_GENERAL_ALIGNMENT,
174    LLM_KV_GENERAL_FILE_TYPE,
175    LLM_KV_GENERAL_SAMPLING_SEQUENCE,
176    LLM_KV_GENERAL_SAMPLING_TOP_K,
177    LLM_KV_GENERAL_SAMPLING_TOP_P,
178    LLM_KV_GENERAL_SAMPLING_MIN_P,
179    LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY,
180    LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD,
181    LLM_KV_GENERAL_SAMPLING_TEMP,
182    LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N,
183    LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT,
184    LLM_KV_GENERAL_SAMPLING_MIROSTAT,
185    LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU,
186    LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA,
187    LLM_KV_GENERAL_NAME,
188    LLM_KV_GENERAL_AUTHOR,
189    LLM_KV_GENERAL_VERSION,
190    LLM_KV_GENERAL_URL,
191    LLM_KV_GENERAL_DESCRIPTION,
192    LLM_KV_GENERAL_LICENSE,
193    LLM_KV_GENERAL_SOURCE_URL,
194    LLM_KV_GENERAL_SOURCE_HF_REPO,
195
196    LLM_KV_VOCAB_SIZE,
197    LLM_KV_CONTEXT_LENGTH,
198    LLM_KV_EMBEDDING_LENGTH,
199    LLM_KV_EMBEDDING_LENGTH_OUT,
200    LLM_KV_EMBEDDING_LENGTH_PER_LAYER,
201    LLM_KV_FEATURES_LENGTH,
202    LLM_KV_BLOCK_COUNT,
203    LLM_KV_LEADING_DENSE_BLOCK_COUNT,
204    LLM_KV_ATTN_RES_BLOCK_SIZE,
205    LLM_KV_ACTIVATION_SITU_BETA,
206    LLM_KV_ACTIVATION_SITU_LINEAR_BETA,
207    LLM_KV_FEED_FORWARD_LENGTH,
208    LLM_KV_EXPERT_FEED_FORWARD_LENGTH,
209    LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,
210    LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH,
211    LLM_KV_SWIGLU_CLAMP_EXP,
212    LLM_KV_SWIGLU_CLAMP_SHEXP,
213    LLM_KV_USE_PARALLEL_RESIDUAL,
214    LLM_KV_TENSOR_DATA_LAYOUT,
215    LLM_KV_EXPERT_COUNT,
216    LLM_KV_EXPERT_USED_COUNT,
217    LLM_KV_EXPERT_SHARED_COUNT,
218    LLM_KV_EXPERT_GROUP_COUNT,
219    LLM_KV_EXPERT_GROUP_USED_COUNT,
220    LLM_KV_EXPERT_WEIGHTS_SCALE,
221    LLM_KV_EXPERT_WEIGHTS_NORM,
222    LLM_KV_EXPERT_LATENT_LENGTH,
223    LLM_KV_EXPERT_GATING_FUNC,
224    LLM_KV_EXPERT_GROUP_SCALE,
225    LLM_KV_EXPERTS_PER_GROUP,
226    LLM_KV_MOE_EVERY_N_LAYERS,
227    LLM_KV_MOE_LATENT_SIZE,
228    LLM_KV_NEXTN_PREDICT_LAYERS,
229    LLM_KV_NUM_DEEPSTACK_LAYERS,
230    LLM_KV_DEEPSTACK_MAPPING,
231    LLM_KV_HIDDEN_ACT,
232    LLM_KV_POOLING_TYPE,
233    LLM_KV_LOGIT_SCALE,
234    LLM_KV_DECODER_START_TOKEN_ID,
235    LLM_KV_DECODER_BLOCK_COUNT,
236    LLM_KV_ENCODER_ATTENTION_HEAD_COUNT_KV,
237    LLM_KV_ATTN_LOGIT_SOFTCAPPING,
238    LLM_KV_ROUTER_LOGIT_SOFTCAPPING,
239    LLM_KV_FINAL_LOGIT_SOFTCAPPING,
240    LLM_KV_SWIN_NORM,
241    LLM_KV_RESCALE_EVERY_N_LAYERS,
242    LLM_KV_TIME_MIX_EXTRA_DIM,
243    LLM_KV_TIME_DECAY_EXTRA_DIM,
244    LLM_KV_RESIDUAL_SCALE,
245    LLM_KV_EMBEDDING_SCALE,
246    LLM_KV_ADAPTER_COUNT,
247    LLM_KV_ADAPTER_TOKEN_IDS_ACTIVATE,
248    LLM_KV_ADAPTER_TOKEN_IDS_SUBSTITUTE,
249    LLM_KV_ADAPTER_LORA_RANK,
250    LLM_KV_ADAPTER_ROUTER_GAIN,
251    LLM_KV_TOKEN_SHIFT_COUNT,
252    LLM_KV_INTERLEAVE_MOE_LAYER_STEP,
253    LLM_KV_FULL_ATTENTION_INTERVAL,
254    LLM_KV_NUM_LOOPS,
255    LLM_KV_SKIP_LOOP_FINAL_NORM,
256
257    LLM_KV_ATTENTION_HEAD_COUNT,
258    LLM_KV_ATTENTION_HEAD_COUNT_KV,
259    LLM_KV_ATTENTION_MAX_ALIBI_BIAS,
260    LLM_KV_ATTENTION_CLAMP_KQV,
261    LLM_KV_ATTENTION_KEY_LENGTH,
262    LLM_KV_ATTENTION_VALUE_LENGTH,
263    LLM_KV_ATTENTION_LAYERNORM_EPS,
264    LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,
265    LLM_KV_ATTENTION_GROUPNORM_EPS,
266    LLM_KV_ATTENTION_GROUPNORM_GROUPS,
267    LLM_KV_ATTENTION_CAUSAL,
268    LLM_KV_ATTENTION_Q_LORA_RANK,
269    LLM_KV_ATTENTION_KV_LORA_RANK,
270    LLM_KV_ATTENTION_DECAY_LORA_RANK,
271    LLM_KV_ATTENTION_ICLR_LORA_RANK,
272    LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK,
273    LLM_KV_ATTENTION_GATE_LORA_RANK,
274    LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,
275    LLM_KV_ATTENTION_SLIDING_WINDOW,
276    LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,
277    LLM_KV_ATTENTION_SCALE,
278    LLM_KV_ATTENTION_ROPE_PATTERN,
279
280    LLM_KV_ATTENTION_OUTPUT_SCALE,
281    LLM_KV_ATTENTION_VALUE_SCALE,
282    LLM_KV_ATTENTION_TEMPERATURE_LENGTH,
283    LLM_KV_ATTENTION_TEMPERATURE_SCALE,
284    LLM_KV_ATTENTION_KEY_LENGTH_MLA,
285    LLM_KV_ATTENTION_VALUE_LENGTH_MLA,
286    LLM_KV_ATTENTION_KEY_LENGTH_SWA,
287    LLM_KV_ATTENTION_VALUE_LENGTH_SWA,
288    LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA,
289    LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA,
290    LLM_KV_ATTENTION_KV_LORA_RANK_SWA,
291    LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,
292    LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
293    LLM_KV_ATTENTION_INDEXER_TOP_K,
294    LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE,
295    LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS,
296    LLM_KV_ATTENTION_INDEXER_TYPES,
297    LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,
298    LLM_KV_ATTENTION_OUTPUT_LORA_RANK,
299    LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,
300    LLM_KV_ATTENTION_COMPRESS_RATIOS,
301    LLM_KV_ATTENTION_SHARED_KV_LAYERS,
302    LLM_KV_ATTENTION_RECURRENT_LAYERS,
303
304    LLM_KV_HYPER_CONNECTION_COUNT,
305    LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS,
306    LLM_KV_HYPER_CONNECTION_EPSILON,
307    LLM_KV_HYPER_CONNECTION_MAGNITUDE,
308    LLM_KV_HYPER_CONNECTION_LOW_RANK,
309
310    LLM_KV_PLE_LAYERS,
311    LLM_KV_PLE_NGRAM_SIZE,
312    LLM_KV_PLE_HEADS_PER_NGRAM,
313    LLM_KV_PLE_CONV_KERNEL,
314    LLM_KV_PLE_LAYER_MULTIPLIERS,
315    LLM_KV_PLE_HEAD_OFFSETS,
316    LLM_KV_PLE_HEAD_VOCAB_SIZES,
317    LLM_KV_PLE_EOS_TOKEN_ID,
318    LLM_KV_PLE_IMAGE_TOKEN_ID,
319
320    LLM_KV_HASH_LAYER_COUNT,
321
322    LLM_KV_ROPE_DIMENSION_COUNT,
323    LLM_KV_ROPE_DIMENSION_COUNT_SWA,
324    LLM_KV_ROPE_DIMENSION_SECTIONS,
325    LLM_KV_ROPE_FREQ_BASE,
326    LLM_KV_ROPE_FREQ_BASE_SWA,
327    LLM_KV_ROPE_SCALE_LINEAR,
328    LLM_KV_ROPE_SCALING_TYPE,
329    LLM_KV_ROPE_SCALING_FACTOR,
330    LLM_KV_ROPE_SCALING_ALPHA,
331    LLM_KV_ROPE_SCALING_ATTN_FACTOR,
332    LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,
333    LLM_KV_ROPE_SCALING_FINETUNED,
334    LLM_KV_ROPE_SCALING_YARN_LOG_MUL,
335    LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR,
336    LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR,
337    LLM_KV_ROPE_SCALING_YARN_BETA_FAST,
338    LLM_KV_ROPE_SCALING_YARN_BETA_SLOW,
339
340    LLM_KV_SPLIT_NO,
341    LLM_KV_SPLIT_COUNT,
342    LLM_KV_SPLIT_TENSORS_COUNT,
343
344    LLM_KV_SSM_INNER_SIZE,
345    LLM_KV_SSM_CONV_KERNEL,
346    LLM_KV_SSM_STATE_SIZE,
347    LLM_KV_SSM_TIME_STEP_RANK,
348    LLM_KV_SSM_GROUP_COUNT,
349    LLM_KV_SSM_DT_B_C_RMS,
350
351    LLM_KV_KDA_HEAD_DIM,
352    LLM_KV_KDA_SAFE_GATE,
353    LLM_KV_KDA_GATE_LOWER_BOUND,
354
355    LLM_KV_WKV_HEAD_SIZE,
356
357    LLM_KV_TOKENIZER_MODEL,
358    LLM_KV_TOKENIZER_PRE,
359    LLM_KV_TOKENIZER_LIST,
360    LLM_KV_TOKENIZER_TOKEN_TYPE,
361    LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,
362    LLM_KV_TOKENIZER_SCORES,
363    LLM_KV_TOKENIZER_MERGES,
364    LLM_KV_TOKENIZER_BOS_ID,
365    LLM_KV_TOKENIZER_EOS_ID,
366    LLM_KV_TOKENIZER_EOT_ID,
367    LLM_KV_TOKENIZER_EOM_ID,
368    LLM_KV_TOKENIZER_UNK_ID,
369    LLM_KV_TOKENIZER_SEP_ID,
370    LLM_KV_TOKENIZER_PAD_ID,
371    LLM_KV_TOKENIZER_CLS_ID,
372    LLM_KV_TOKENIZER_MASK_ID,
373    LLM_KV_TOKENIZER_ADD_BOS,
374    LLM_KV_TOKENIZER_ADD_EOS,
375    LLM_KV_TOKENIZER_ADD_SEP,
376    LLM_KV_TOKENIZER_ADD_PREFIX,
377    LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
378    LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
379    LLM_KV_TOKENIZER_HF_JSON,
380    LLM_KV_TOKENIZER_RWKV,
381    LLM_KV_TOKENIZER_CHAT_TEMPLATE,
382    LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
383    LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
384    LLM_KV_TOKENIZER_FIM_PRE_ID,
385    LLM_KV_TOKENIZER_FIM_SUF_ID,
386    LLM_KV_TOKENIZER_FIM_MID_ID,
387    LLM_KV_TOKENIZER_FIM_PAD_ID,
388    LLM_KV_TOKENIZER_FIM_REP_ID,
389    LLM_KV_TOKENIZER_FIM_SEP_ID,
390    LLM_KV_TOKENIZER_SUPPRESS_TOKENS,
391
392    LLM_KV_ADAPTER_TYPE,
393    LLM_KV_ADAPTER_LORA_ALPHA,
394    LLM_KV_ADAPTER_LORA_TASK_NAME,
395    LLM_KV_ADAPTER_LORA_PROMPT_PREFIX,
396    LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS,
397
398    LLM_KV_POSNET_EMBEDDING_LENGTH,
399    LLM_KV_POSNET_BLOCK_COUNT,
400
401    LLM_KV_CONVNEXT_EMBEDDING_LENGTH,
402    LLM_KV_CONVNEXT_BLOCK_COUNT,
403
404    LLM_KV_CLASSIFIER_OUTPUT_LABELS,
405
406    LLM_KV_TARGET_LAYERS,
407    LLM_KV_TARGET_HIDDEN_SIZE,
408    LLM_KV_DFLASH_BLOCK_SIZE,
409    LLM_KV_DFLASH_CONV_KERNEL_SIZE,
410    LLM_KV_DFLASH_CONV_GROUP_SIZE,
411    LLM_KV_DFLASH_SELECTOR_RANK,
412    LLM_KV_DFLASH_SELECTOR_TOP_K,
413    LLM_KV_NORM_BEFORE_RESIDUAL,
414    LLM_KV_NORM_BEFORE_FC,
415
416    LLM_KV_SHORTCONV_L_CACHE,
417
418    LLM_KV_XIELU_ALPHA_N,
419    LLM_KV_XIELU_ALPHA_P,
420    LLM_KV_XIELU_BETA,
421    LLM_KV_XIELU_EPS,
422
423    // deprecated:
424    LLM_KV_TOKENIZER_PREFIX_ID,
425    LLM_KV_TOKENIZER_SUFFIX_ID,
426    LLM_KV_TOKENIZER_MIDDLE_ID,
427
428    // sentence-transformers dense layers in and out features
429    LLM_KV_DENSE_2_FEAT_IN,
430    LLM_KV_DENSE_2_FEAT_OUT,
431    LLM_KV_DENSE_3_FEAT_IN,
432    LLM_KV_DENSE_3_FEAT_OUT,
433};
434
435enum llm_tensor {
436    LLM_TENSOR_TOKEN_EMBD,
437    LLM_TENSOR_TOKEN_EMBD_NORM,
438    LLM_TENSOR_TOKEN_TYPES,
439    LLM_TENSOR_POS_EMBD,
440    LLM_TENSOR_DENSE_2_OUT,
441    LLM_TENSOR_DENSE_3_OUT,
442    LLM_TENSOR_OUTPUT,
443    LLM_TENSOR_OUTPUT_NORM,
444    LLM_TENSOR_OUTPUT_NORM_LFM2, // fix for wrong tensor name
445    LLM_TENSOR_ROPE_FREQS,
446    LLM_TENSOR_ROPE_FACTORS_LONG,
447    LLM_TENSOR_ROPE_FACTORS_SHORT,
448    LLM_TENSOR_ATTN_Q,
449    LLM_TENSOR_ATTN_K,
450    LLM_TENSOR_ATTN_V,
451    LLM_TENSOR_ATTN_QKV,
452    LLM_TENSOR_ATTN_OUT,
453    LLM_TENSOR_ATTN_NORM,
454    LLM_TENSOR_ATTN_NORM_2,
455    LLM_TENSOR_ATTN_OUT_NORM,
456    LLM_TENSOR_ATTN_POST_NORM,
457    LLM_TENSOR_ATTN_ROT_EMBD,
458    LLM_TENSOR_ATTN_SINKS,
459    LLM_TENSOR_ATTN_GATE,
460    LLM_TENSOR_FFN_GATE_INP,
461    LLM_TENSOR_FFN_GATE_INP_SHEXP,
462    LLM_TENSOR_FFN_NORM,
463    LLM_TENSOR_FFN_POST_NORM,
464    LLM_TENSOR_FFN_POST_NORM_1,
465    LLM_TENSOR_FFN_POST_NORM_2,
466    LLM_TENSOR_FFN_PRE_NORM_2,
467    LLM_TENSOR_FFN_GATE,
468    LLM_TENSOR_FFN_DOWN,
469    LLM_TENSOR_FFN_UP,
470    LLM_TENSOR_FFN_ACT,
471    LLM_TENSOR_FFN_DOWN_EXP,  // split experts for backward compatibility
472    LLM_TENSOR_FFN_GATE_EXP,
473    LLM_TENSOR_FFN_UP_EXP,
474    LLM_TENSOR_FFN_NORM_EXPS,
475    LLM_TENSOR_FFN_DOWN_EXPS, // merged experts
476    LLM_TENSOR_FFN_GATE_EXPS,
477    LLM_TENSOR_FFN_UP_EXPS,
478    LLM_TENSOR_FFN_GATE_UP_EXPS,
479    LLM_TENSOR_FFN_DOWN_SHEXP,
480    LLM_TENSOR_FFN_GATE_SHEXP,
481    LLM_TENSOR_FFN_UP_SHEXP,
482    LLM_TENSOR_FFN_DOWN_CHEXPS,
483    LLM_TENSOR_FFN_GATE_CHEXPS,
484    LLM_TENSOR_FFN_UP_CHEXPS,
485    LLM_TENSOR_FFN_EXP_PROBS_B,
486    LLM_TENSOR_FFN_EXP_PROBS_B_VL,
487    LLM_TENSOR_FFN_LATENT_DOWN,
488    LLM_TENSOR_FFN_LATENT_UP,
489    LLM_TENSOR_ATTN_Q_NORM,
490    LLM_TENSOR_ATTN_K_NORM,
491    LLM_TENSOR_LAYER_OUT_NORM,
492    LLM_TENSOR_LAYER_OUT_SCALE,
493    LLM_TENSOR_POST_ATTN_NORM,
494    LLM_TENSOR_POST_MLP_NORM,
495    LLM_TENSOR_PER_LAYER_TOKEN_EMBD, // gemma3n
496    LLM_TENSOR_PER_LAYER_MODEL_PROJ, // gemma3n
497    LLM_TENSOR_PER_LAYER_INP_GATE,   // gemma3n
498    LLM_TENSOR_PER_LAYER_PROJ,       // gemma3n
499    LLM_TENSOR_PER_LAYER_PROJ_NORM,  // gemma3n
500    LLM_TENSOR_PER_LAYER_POST_NORM,  // gemma3n
501    LLM_TENSOR_ALTUP_PROJ,           // gemma3n
502    LLM_TENSOR_ALTUP_UNEMBD_PROJ,    // gemma3n
503    LLM_TENSOR_ALTUP_CORRECT_COEF,   // gemma3n
504    LLM_TENSOR_ALTUP_CORRECT_SCALE,  // gemma3n
505    LLM_TENSOR_ALTUP_PREDICT_COEF,   // gemma3n
506    LLM_TENSOR_ALTUP_ROUTER,         // gemma3n
507    LLM_TENSOR_ALTUP_ROUTER_NORM,    // gemma3n
508    LLM_TENSOR_LAUREL_L,             // gemma3n
509    LLM_TENSOR_LAUREL_R,             // gemma3n
510    LLM_TENSOR_LAUREL_POST_NORM,     // gemma3n
511    LLM_TENSOR_SSM_IN,
512    LLM_TENSOR_SSM_CONV1D,
513    LLM_TENSOR_SSM_X,
514    LLM_TENSOR_SSM_DT,
515    LLM_TENSOR_SSM_DT_NORM,
516    LLM_TENSOR_SSM_A,
517    LLM_TENSOR_SSM_A_NOSCAN,        // qwen3next special case with MUL instead of SSM_SCAN
518    LLM_TENSOR_SSM_B_NORM,
519    LLM_TENSOR_SSM_C_NORM,
520    LLM_TENSOR_SSM_D,
521    LLM_TENSOR_SSM_NORM,
522    LLM_TENSOR_SSM_OUT,
523    LLM_TENSOR_SSM_BETA_ALPHA,      // qwen3next
524    LLM_TENSOR_SSM_ALPHA,           // qwen3.5
525    // Kimi Linear KDA (using SSM_ prefix for consistency)
526    LLM_TENSOR_SSM_CONV1D_Q,        // kimi: Q conv1d weight
527    LLM_TENSOR_SSM_CONV1D_K,        // kimi: K conv1d weight
528    LLM_TENSOR_SSM_CONV1D_V,        // kimi: V conv1d weight
529    LLM_TENSOR_SSM_F_A,             // kimi: forget gate projection A
530    LLM_TENSOR_SSM_F_B,             // kimi: forget gate projection B
531    LLM_TENSOR_SSM_BETA,            // kimi: beta mixing coefficient and qwen3.5
532    LLM_TENSOR_SSM_G_A,             // kimi: output gate projection A
533    LLM_TENSOR_SSM_G_B,             // kimi: output gate projection B
534    LLM_TENSOR_SSM_G,               // kimi-k3: full-rank KDA gate
535    LLM_TENSOR_ATTN_RES_SCORE,      // kimi-k3: fused res_norm*res_proj (pre-attn)
536    LLM_TENSOR_FFN_RES_SCORE,       // kimi-k3: fused res_norm*res_proj (pre-ffn)
537    LLM_TENSOR_OUTPUT_RES_SCORE,    // kimi-k3: fused res_norm*res_proj (final)
538    LLM_TENSOR_FFN_ROUTED_DOWN,     // kimi-k3: latent MoE down
539    LLM_TENSOR_FFN_ROUTED_UP,       // kimi-k3: latent MoE up
540    LLM_TENSOR_FFN_ROUTED_NORM,     // kimi-k3: latent MoE norm
541    LLM_TENSOR_TIME_MIX_W0,
542    LLM_TENSOR_TIME_MIX_W1,
543    LLM_TENSOR_TIME_MIX_W2,
544    LLM_TENSOR_TIME_MIX_A0,
545    LLM_TENSOR_TIME_MIX_A1,
546    LLM_TENSOR_TIME_MIX_A2,
547    LLM_TENSOR_TIME_MIX_V0,
548    LLM_TENSOR_TIME_MIX_V1,
549    LLM_TENSOR_TIME_MIX_V2,
550    LLM_TENSOR_TIME_MIX_G1,
551    LLM_TENSOR_TIME_MIX_G2,
552    LLM_TENSOR_TIME_MIX_K_K,
553    LLM_TENSOR_TIME_MIX_K_A,
554    LLM_TENSOR_TIME_MIX_R_K,
555    LLM_TENSOR_TIME_MIX_LERP_X,
556    LLM_TENSOR_TIME_MIX_LERP_W,
557    LLM_TENSOR_TIME_MIX_LERP_K,
558    LLM_TENSOR_TIME_MIX_LERP_V,
559    LLM_TENSOR_TIME_MIX_LERP_R,
560    LLM_TENSOR_TIME_MIX_LERP_G,
561    LLM_TENSOR_TIME_MIX_LERP_FUSED,
562    LLM_TENSOR_TIME_MIX_FIRST,
563    LLM_TENSOR_TIME_MIX_DECAY,
564    LLM_TENSOR_TIME_MIX_DECAY_W1,
565    LLM_TENSOR_TIME_MIX_DECAY_W2,
566    LLM_TENSOR_TIME_MIX_KEY,
567    LLM_TENSOR_TIME_MIX_VALUE,
568    LLM_TENSOR_TIME_MIX_RECEPTANCE,
569    LLM_TENSOR_TIME_MIX_GATE,
570    LLM_TENSOR_TIME_MIX_LN,
571    LLM_TENSOR_TIME_MIX_OUTPUT,
572    LLM_TENSOR_CHANNEL_MIX_LERP_K,
573    LLM_TENSOR_CHANNEL_MIX_LERP_R,
574    LLM_TENSOR_CHANNEL_MIX_KEY,
575    LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,
576    LLM_TENSOR_CHANNEL_MIX_VALUE,
577    LLM_TENSOR_ATTN_Q_A,
578    LLM_TENSOR_ATTN_Q_B,
579    LLM_TENSOR_ATTN_KV_A_MQA,
580    LLM_TENSOR_ATTN_KV_B,
581    LLM_TENSOR_ATTN_KV,
582    LLM_TENSOR_ATTN_KV_NORM,
583    LLM_TENSOR_ATTN_OUT_A,
584    LLM_TENSOR_ATTN_OUT_B,
585    LLM_TENSOR_ATTN_K_B,
586    LLM_TENSOR_ATTN_V_B,
587    LLM_TENSOR_ATTN_Q_A_NORM,
588    LLM_TENSOR_ATTN_KV_A_NORM,
589    LLM_TENSOR_HC_HEAD_FN,
590    LLM_TENSOR_HC_HEAD_BASE,
591    LLM_TENSOR_HC_HEAD_SCALE,
592    LLM_TENSOR_HC_HEAD_NORM,   // qwen4exp
593    LLM_TENSOR_HC_HEAD_DOWN,   // qwen4exp
594    LLM_TENSOR_HC_HEAD_UP,     // qwen4exp
595    LLM_TENSOR_HC_ATTN_FN,
596    LLM_TENSOR_HC_ATTN_BASE,
597    LLM_TENSOR_HC_ATTN_SCALE,
598    LLM_TENSOR_HC_FFN_FN,
599    LLM_TENSOR_HC_FFN_BASE,
600    LLM_TENSOR_HC_FFN_SCALE,
601    LLM_TENSOR_HC_ATTN_NORM,   // qwen4exp
602    LLM_TENSOR_HC_ATTN_DOWN,   // qwen4exp
603    LLM_TENSOR_HC_ATTN_UP,     // qwen4exp
604    LLM_TENSOR_HC_ATTN_INJECT, // qwen4exp
605    LLM_TENSOR_HC_FFN_NORM,    // qwen4exp
606    LLM_TENSOR_HC_FFN_DOWN,    // qwen4exp
607    LLM_TENSOR_HC_FFN_UP,      // qwen4exp
608    LLM_TENSOR_HC_FFN_INJECT,  // qwen4exp
609    LLM_TENSOR_PLE_KEY,        // qwen4exp
610    LLM_TENSOR_PLE_VALUE,      // qwen4exp
611    LLM_TENSOR_PLE_NORM_KEY,   // qwen4exp
612    LLM_TENSOR_PLE_NORM_QUERY, // qwen4exp
613    LLM_TENSOR_PLE_NORM_CONV,  // qwen4exp
614    LLM_TENSOR_PLE_CONV1D,     // qwen4exp
615    LLM_TENSOR_ATTN_COMPRESSOR_WKV,
616    LLM_TENSOR_ATTN_COMPRESSOR_WGATE,
617    LLM_TENSOR_ATTN_COMPRESSOR_APE,
618    LLM_TENSOR_ATTN_COMPRESSOR_NORM,
619    LLM_TENSOR_ATTN_SUB_NORM,
620    LLM_TENSOR_FFN_SUB_NORM,
621    LLM_TENSOR_DEC_ATTN_NORM,
622    LLM_TENSOR_DEC_ATTN_Q,
623    LLM_TENSOR_DEC_ATTN_K,
624    LLM_TENSOR_DEC_ATTN_V,
625    LLM_TENSOR_DEC_ATTN_OUT,
626    LLM_TENSOR_DEC_ATTN_REL_B,
627    LLM_TENSOR_DEC_CROSS_ATTN_NORM,
628    LLM_TENSOR_DEC_CROSS_ATTN_Q,
629    LLM_TENSOR_DEC_CROSS_ATTN_K,
630    LLM_TENSOR_DEC_CROSS_ATTN_V,
631    LLM_TENSOR_DEC_CROSS_ATTN_OUT,
632    LLM_TENSOR_DEC_CROSS_ATTN_REL_B,
633    LLM_TENSOR_DEC_FFN_NORM,
634    LLM_TENSOR_DEC_FFN_GATE,
635    LLM_TENSOR_DEC_FFN_DOWN,
636    LLM_TENSOR_DEC_FFN_UP,
637    LLM_TENSOR_DEC_OUTPUT_NORM,
638    LLM_TENSOR_ENC_ATTN_NORM,
639    LLM_TENSOR_ENC_ATTN_Q,
640    LLM_TENSOR_ENC_ATTN_K,
641    LLM_TENSOR_ENC_ATTN_V,
642    LLM_TENSOR_ENC_ATTN_OUT,
643    LLM_TENSOR_ENC_ATTN_REL_B,
644    LLM_TENSOR_ENC_FFN_NORM,
645    LLM_TENSOR_ENC_FFN_GATE,
646    LLM_TENSOR_ENC_FFN_DOWN,
647    LLM_TENSOR_ENC_FFN_UP,
648    LLM_TENSOR_ENC_OUTPUT_NORM,
649    LLM_TENSOR_ENC_FFN_GATE_INP,
650    LLM_TENSOR_ENC_FFN_GATE_EXPS,
651    LLM_TENSOR_ENC_FFN_DOWN_EXPS,
652    LLM_TENSOR_ENC_FFN_UP_EXPS,
653    LLM_TENSOR_ENC_EXP_PROBS_B,
654    LLM_TENSOR_ENC_FFN_OUTPUT_B,
655    LLM_TENSOR_DEC_FFN_GATE_INP,
656    LLM_TENSOR_DEC_FFN_GATE_EXPS,
657    LLM_TENSOR_DEC_FFN_DOWN_EXPS,
658    LLM_TENSOR_DEC_FFN_UP_EXPS,
659    LLM_TENSOR_DEC_EXP_PROBS_B,
660    LLM_TENSOR_DEC_FFN_OUTPUT_B,
661    LLM_TENSOR_CLS,
662    LLM_TENSOR_CLS_OUT,
663    LLM_TENSOR_CLS_NORM,
664    LLM_TENSOR_CONV1D,
665    LLM_TENSOR_CONVNEXT_DW,
666    LLM_TENSOR_CONVNEXT_NORM,
667    LLM_TENSOR_CONVNEXT_PW1,
668    LLM_TENSOR_CONVNEXT_PW2,
669    LLM_TENSOR_CONVNEXT_GAMMA,
670    LLM_TENSOR_POS_NET_CONV1,
671    LLM_TENSOR_POS_NET_CONV2,
672    LLM_TENSOR_POS_NET_NORM,
673    LLM_TENSOR_POS_NET_NORM1,
674    LLM_TENSOR_POS_NET_NORM2,
675    LLM_TENSOR_POS_NET_ATTN_NORM,
676    LLM_TENSOR_POS_NET_ATTN_Q,
677    LLM_TENSOR_POS_NET_ATTN_K,
678    LLM_TENSOR_POS_NET_ATTN_V,
679    LLM_TENSOR_POS_NET_ATTN_OUT,
680    LLM_TENSOR_SHORTCONV_CONV,
681    LLM_TENSOR_SHORTCONV_INPROJ,
682    LLM_TENSOR_SHORTCONV_OUTPROJ,
683    LLM_TENSOR_VISEXP_ATTN_QKV,
684    LLM_TENSOR_VISEXP_ATTN_OUT,
685    LLM_TENSOR_VISEXP_FFN_GATE,
686    LLM_TENSOR_VISEXP_FFN_DOWN,
687    LLM_TENSOR_VISEXP_FFN_UP,
688    LLM_TENSOR_INDEXER_K_NORM,
689    LLM_TENSOR_INDEXER_PROJ,
690    LLM_TENSOR_INDEXER_ATTN_K,
691    LLM_TENSOR_INDEXER_ATTN_Q_B,
692    LLM_TENSOR_INDEXER_Q_PROJ,
693    LLM_TENSOR_INDEXER_K_PROJ,
694    LLM_TENSOR_INDEXER_Q_NORM,
695    LLM_TENSOR_INDEXER_COMPRESSOR_WKV,
696    LLM_TENSOR_INDEXER_COMPRESSOR_WGATE,
697    LLM_TENSOR_INDEXER_COMPRESSOR_APE,
698    LLM_TENSOR_INDEXER_COMPRESSOR_NORM,
699    LLM_TENSOR_FFN_GATE_TID2EID,
700    LLM_TENSOR_NEXTN_PROJ_PRE,
701    LLM_TENSOR_NEXTN_PROJ_POST,
702    LLM_TENSOR_NEXTN_EH_PROJ,
703    LLM_TENSOR_NEXTN_EMBED_TOKENS,
704    LLM_TENSOR_NEXTN_ENORM,
705    LLM_TENSOR_NEXTN_HNORM,
706    LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
707    LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
708    LLM_TENSOR_MASKED_EMBD_CENTROIDS,
709    LLM_TENSOR_MASKED_EMBD_ORDERING,
710    LLM_TENSOR_FC,
711    LLM_TENSOR_D2T,
712    LLM_TENSOR_DSPARK_MARKOV_W1,
713    LLM_TENSOR_DSPARK_MARKOV_W2,
714    LLM_TENSOR_DSPARK_CONF_PROJ,
715    LLM_TENSOR_DFLASH_ATTN_CONV_BASE,
716    LLM_TENSOR_DFLASH_ATTN_CONV_PROJ,
717    LLM_TENSOR_DFLASH_FFN_CONV_BASE,
718    LLM_TENSOR_DFLASH_FFN_CONV_PROJ,
719    LLM_TENSOR_DFLASH_SELECTOR_PREV,
720    LLM_TENSOR_DFLASH_SELECTOR_NEXT,
721    LLM_TENSOR_DFLASH_SELECTOR_HIDDEN,
722};
723
724
725enum llm_tensor_layer {
726    LLM_TENSOR_LAYER_INPUT,
727    LLM_TENSOR_LAYER_REPEATING,
728    LLM_TENSOR_LAYER_OUTPUT,
729};
730
731struct LLM_KV {
732    LLM_KV(llm_arch arch, const char * suffix = nullptr);
733
734    llm_arch arch;
735    const char * suffix;
736
737    std::string operator()(llm_kv kv) const;
738};
739
740// helper to handle gguf constants
741// usage:
742//
743//   const auto tn = LLM_TN(LLM_ARCH_LLAMA);
744//
745//   std::string name = tn(LLM_TENSOR_OUTPUT);                     -> "output"
746//   std::string name = tn(LLM_TENSOR_TOKEN_EMBD, "bias");         -> "token_embd.bias"
747//   std::string name = tn(LLM_TENSOR_ATTN_NORM, "weight", 3);     -> "blk.3.attn_norm.weight"
748//
749struct LLM_TN_IMPL {
750    const llm_arch arch;
751    const llm_tensor tensor;
752    const char * const suffix;
753    const int bid;
754    const int xid;
755
756    LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid);
757
758    std::string str() const;
759
760    operator std::string() const {
761        return str();
762    }
763
764    friend bool operator==(const std::string & str, const LLM_TN_IMPL & tn) {
765        return str == tn.str();
766    }
767
768    friend bool operator!=(const std::string & str, const LLM_TN_IMPL & tn) {
769        return str != tn.str();
770    }
771};
772
773struct LLM_TN {
774    LLM_TN(llm_arch arch) : arch(arch) {}
775
776    llm_arch arch;
777
778    LLM_TN_IMPL operator()(llm_tensor tensor, const char * suffix, int bid = -1, int xid = -1) const {
779        return LLM_TN_IMPL(arch, tensor, suffix, bid, xid);
780    }
781
782    LLM_TN_IMPL operator()(llm_tensor tensor, int bid = -1, int xid = -1) const {
783        return LLM_TN_IMPL(arch, tensor, nullptr, bid, xid);
784    }
785};
786
787
788struct llm_tensor_info {
789    llm_tensor_layer layer;
790    ggml_op op;
791};
792
793std::vector<llm_arch> llm_arch_all();
794
795const char * llm_arch_name(llm_arch arch);
796
797llm_arch llm_arch_from_string(const std::string & name);
798
799const llm_tensor_info & llm_tensor_info_for(llm_tensor tensor);
800
801bool llm_arch_is_recurrent      (const llm_arch & arch);
802bool llm_arch_is_hybrid         (const llm_arch & arch);
803bool llm_arch_is_diffusion      (const llm_arch & arch);
804bool llm_arch_supports_sm_tensor(const llm_arch & arch);
805bool llm_arch_supports_rs_rollback(const llm_arch & arch);