hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Charlotte Somsupport aliceai_t5_moe98a01afd92026-09-15

1#include "llama-model.h"
2
3#include "llama-arch.h"
4#include "llama-ext.h"
5#include "llama-hparams.h"
6#include "llama-impl.h"
7#include "llama-mmap.h"
8#include "llama-cparams.h"
9#include "llama-model-loader.h"
10
11#include "llama-kv-cache.h"
12#include "llama-kv-cache-iswa.h"
13#include "llama-kv-cache-dsa.h"
14#include "llama-kv-cache-dsa-iswa.h"
15#include "llama-kv-cache-msa.h"
16#include "llama-kv-cache-dsv4.h"
17#include "llama-memory-hybrid.h"
18#include "llama-memory-hybrid-iswa.h"
19#include "llama-memory-hybrid-idx.h"
20#include "llama-memory-recurrent.h"
21
22#include "llama.h"
23#include "models/models.h"
24
25#include "ggml.h"
26#include "ggml-cpp.h"
27
28#include <algorithm>
29#include <cassert>
30#include <cfloat>
31#include <cstdint>
32#include <cstring>
33#include <cmath>
34#include <functional>
35#include <map>
36#include <numeric>
37#include <regex>
38#include <sstream>
39#include <stdexcept>
40#include <string>
41#include <vector>
42
43static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params & params) {
44    switch (arch) {
45        case LLM_ARCH_CLIP:
46            return new llama_model_clip(params);
47        case LLM_ARCH_LLAMA:
48            return new llama_model_llama(params);
49        case LLM_ARCH_LLAMA4:
50            return new llama_model_llama4(params);
51        case LLM_ARCH_LLAMA_EMBED:
52            return new llama_model_llama_embed(params);
53        case LLM_ARCH_MAINCODER:
54            return new llama_model_maincoder(params);
55        case LLM_ARCH_TALKIE:
56            return new llama_model_talkie(params);
57        case LLM_ARCH_DECI:
58            return new llama_model_deci(params);
59        case LLM_ARCH_BAICHUAN:
60            return new llama_model_baichuan(params);
61        case LLM_ARCH_FALCON:
62            return new llama_model_falcon(params);
63        case LLM_ARCH_GROK:
64            return new llama_model_grok(params);
65        case LLM_ARCH_STARCODER:
66            return new llama_model_starcoder(params);
67        case LLM_ARCH_REFACT:
68            return new llama_model_refact(params);
69        case LLM_ARCH_BERT:
70            return new llama_model_bert(params);
71        case LLM_ARCH_JINA_BERT_V2:
72            return new llama_model_jina_bert_v2(params);
73        case LLM_ARCH_JINA_BERT_V3:
74            return new llama_model_jina_bert_v3(params);
75        case LLM_ARCH_NOMIC_BERT:
76            return new llama_model_nomic_bert(params);
77        case LLM_ARCH_NOMIC_BERT_MOE:
78            return new llama_model_nomic_bert_moe(params);
79        case LLM_ARCH_MODERN_BERT:
80            return new llama_model_modern_bert(params);
81        case LLM_ARCH_NEO_BERT:
82            return new llama_model_neo_bert(params);
83        case LLM_ARCH_EUROBERT:
84            return new llama_model_eurobert(params);
85        case LLM_ARCH_BLOOM:
86            return new llama_model_bloom(params);
87        case LLM_ARCH_MPT:
88            return new llama_model_mpt(params);
89        case LLM_ARCH_STABLELM:
90            return new llama_model_stablelm(params);
91        case LLM_ARCH_MELLUM:
92            return new llama_model_mellum(params);
93        case LLM_ARCH_NANBEIGE:
94            return new llama_model_nanbeige(params);
95        case LLM_ARCH_QWEN:
96            return new llama_model_qwen(params);
97        case LLM_ARCH_QWEN2:
98            return new llama_model_qwen2(params);
99        case LLM_ARCH_DREAM:
100            return new llama_model_dream(params);
101        case LLM_ARCH_LLADA:
102            return new llama_model_llada(params);
103        case LLM_ARCH_LLADA_MOE:
104            return new llama_model_llada_moe(params);
105        case LLM_ARCH_RND1:
106            return new llama_model_rnd1(params);
107        case LLM_ARCH_QWEN2VL:
108            return new llama_model_qwen2vl(params);
109        case LLM_ARCH_QWEN2MOE:
110            return new llama_model_qwen2moe(params);
111        case LLM_ARCH_QWEN3:
112            return new llama_model_qwen3(params);
113        case LLM_ARCH_QWEN3MOE:
114            return new llama_model_qwen3moe(params);
115        case LLM_ARCH_QWEN3VL:
116            return new llama_model_qwen3vl(params);
117        case LLM_ARCH_QWEN3VLMOE:
118            return new llama_model_qwen3vlmoe(params);
119        case LLM_ARCH_QWEN3TTS:
120            return new llama_model_qwen3tts(params);
121        case LLM_ARCH_POCKETTTS:
122            return new llama_model_pockettts(params);
123        case LLM_ARCH_PHI2:
124            return new llama_model_phi2(params);
125        case LLM_ARCH_PHI3:
126            return new llama_model_phi3(params);
127        case LLM_ARCH_PHIMOE:
128            return new llama_model_phimoe(params);
129        case LLM_ARCH_PLAMO:
130            return new llama_model_plamo(params);
131        case LLM_ARCH_PLAMO2:
132            return new llama_model_plamo2(params);
133        case LLM_ARCH_PLAMO3:
134            return new llama_model_plamo3(params);
135        case LLM_ARCH_GPT2:
136            return new llama_model_gpt2(params);
137        case LLM_ARCH_CODESHELL:
138            return new llama_model_codeshell(params);
139        case LLM_ARCH_ORION:
140            return new llama_model_orion(params);
141        case LLM_ARCH_INTERNLM2:
142            return new llama_model_internlm2(params);
143        case LLM_ARCH_MINICPM3:
144            return new llama_model_minicpm3(params);
145        case LLM_ARCH_GEMMA:
146            return new llama_model_gemma(params);
147        case LLM_ARCH_GEMMA2:
148            return new llama_model_gemma2(params);
149        case LLM_ARCH_GEMMA3:
150            return new llama_model_gemma3(params);
151        case LLM_ARCH_GEMMA3N:
152            return new llama_model_gemma3n(params);
153        case LLM_ARCH_GEMMA4:
154            return new llama_model_gemma4(params);
155        case LLM_ARCH_GEMMA4_ASSISTANT:
156            return new llama_model_gemma4_assistant(params);
157        case LLM_ARCH_GEMMA_EMBEDDING:
158            return new llama_model_gemma_embedding(params);
159        case LLM_ARCH_STARCODER2:
160            return new llama_model_starcoder2(params);
161        case LLM_ARCH_MAMBA:
162            return new llama_model_mamba(params);
163        case LLM_ARCH_MAMBA2:
164            return new llama_model_mamba2(params);
165        case LLM_ARCH_MAPLE:
166            return new llama_model_maple(params);
167        case LLM_ARCH_JAMBA:
168            return new llama_model_jamba(params);
169        case LLM_ARCH_XVERSE:
170            return new llama_model_xverse(params);
171        case LLM_ARCH_COMMAND_R:
172            return new llama_model_command_r(params);
173        case LLM_ARCH_COHERE2:
174            return new llama_model_cohere2(params);
175        case LLM_ARCH_COHERE2MOE:
176            return new llama_model_cohere2moe(params);
177        case LLM_ARCH_DBRX:
178            return new llama_model_dbrx(params);
179        case LLM_ARCH_OLMO:
180            return new llama_model_olmo(params);
181        case LLM_ARCH_OLMO2:
182            return new llama_model_olmo2(params);
183        case LLM_ARCH_OLMOE:
184            return new llama_model_olmoe(params);
185        case LLM_ARCH_MUSE_GLIMMER:
186            return new llama_model_muse_glimmer(params);
187        case LLM_ARCH_OPENELM:
188            return new llama_model_openelm(params);
189        case LLM_ARCH_GPTNEOX:
190            return new llama_model_gptneox(params);
191        case LLM_ARCH_ARCTIC:
192            return new llama_model_arctic(params);
193        case LLM_ARCH_DEEPSEEK:
194            return new llama_model_deepseek(params);
195        case LLM_ARCH_DEEPSEEK2:
196            return new llama_model_deepseek2(params);
197        case LLM_ARCH_DEEPSEEK2OCR:
198            return new llama_model_deepseek2ocr(params);
199        case LLM_ARCH_DEEPSEEK32:
200            return new llama_model_deepseek32(params);
201        case LLM_ARCH_DOTS3NOTE:
202            return new llama_model_dots3note(params);
203        case LLM_ARCH_DEEPSEEK4:
204            return new llama_model_deepseek4(params);
205        case LLM_ARCH_GLM_DSA:
206            return new llama_model_glm_dsa(params);
207        case LLM_ARCH_MISTRAL4:
208            return new llama_model_mistral4(params);
209        case LLM_ARCH_CHATGLM:
210            return new llama_model_chatglm(params);
211        case LLM_ARCH_GLM4:
212            return new llama_model_glm4(params);
213        case LLM_ARCH_GLM4_MOE:
214            return new llama_model_glm4_moe(params);
215        case LLM_ARCH_BITNET:
216            return new llama_model_bitnet(params);
217        case LLM_ARCH_T5:
218            return new llama_model_t5(params);
219        case LLM_ARCH_ALICEAI_T5_MOE:
220            return new llama_model_aliceai_t5_moe(params);
221        case LLM_ARCH_T5ENCODER:
222            return new llama_model_t5encoder(params);
223        case LLM_ARCH_JAIS:
224            return new llama_model_jais(params);
225        case LLM_ARCH_JAIS2:
226            return new llama_model_jais2(params);
227        case LLM_ARCH_NEMOTRON:
228            return new llama_model_nemotron(params);
229        case LLM_ARCH_NEMOTRON_H:
230            return new llama_model_nemotron_h(params);
231        case LLM_ARCH_NEMOTRON_H_MOE:
232            return new llama_model_nemotron_h_moe(params);
233        case LLM_ARCH_EXAONE:
234            return new llama_model_exaone(params);
235        case LLM_ARCH_EXAONE4:
236            return new llama_model_exaone4(params);
237        case LLM_ARCH_EXAONE_MOE:
238            return new llama_model_exaone_moe(params);
239        case LLM_ARCH_RWKV6:
240            return new llama_model_rwkv6(params);
241        case LLM_ARCH_RWKV6QWEN2:
242            return new llama_model_rwkv6qwen2(params);
243        case LLM_ARCH_RWKV7:
244            return new llama_model_rwkv7(params);
245        case LLM_ARCH_ARWKV7:
246            return new llama_model_arwkv7(params);
247        case LLM_ARCH_GRANITE:
248            return new llama_model_granite(params);
249        case LLM_ARCH_GRANITE_MOE:
250            return new llama_model_granite_moe(params);
251        case LLM_ARCH_GRANITE_SWITCH:
252            return new llama_model_granite_switch(params);
253        case LLM_ARCH_MINICPM:
254            return new llama_model_minicpm(params);
255        case LLM_ARCH_GRANITE_HYBRID:
256            return new llama_model_granite_hybrid(params);
257        case LLM_ARCH_GRANITE_SWA:
258            return new llama_model_granite_swa(params);
259        case LLM_ARCH_CHAMELEON:
260            return new llama_model_chameleon(params);
261        case LLM_ARCH_WAVTOKENIZER_DEC:
262            return new llama_model_wavtokenizer_dec(params);
263        case LLM_ARCH_PLM:
264            return new llama_model_plm(params);
265        case LLM_ARCH_BAILINGMOE:
266            return new llama_model_bailingmoe(params);
267        case LLM_ARCH_BAILINGMOE2:
268            return new llama_model_bailingmoe2(params);
269        case LLM_ARCH_BAILINGMOE3:
270            return new llama_model_bailingmoe3(params);
271        case LLM_ARCH_SEED_OSS:
272            return new llama_model_seed_oss(params);
273        case LLM_ARCH_DOTS1:
274            return new llama_model_dots1(params);
275        case LLM_ARCH_ARCEE:
276            return new llama_model_arcee(params);
277        case LLM_ARCH_AFMOE:
278            return new llama_model_afmoe(params);
279        case LLM_ARCH_LAGUNA:
280            return new llama_model_laguna(params);
281        case LLM_ARCH_ERNIE4_5:
282            return new llama_model_ernie4_5(params);
283        case LLM_ARCH_ERNIE4_5_MOE:
284            return new llama_model_ernie4_5_moe(params);
285        case LLM_ARCH_PADDLEOCR:
286            return new llama_model_paddleocr(params);
287        case LLM_ARCH_HUNYUAN_MOE:
288            return new llama_model_hunyuan_moe(params);
289        case LLM_ARCH_HUNYUAN_VL:
290            return new llama_model_hunyuan_vl(params);
291        case LLM_ARCH_HUNYUAN_DENSE:
292            return new llama_model_hunyuan_dense(params);
293        case LLM_ARCH_HY_V3:
294            return new llama_model_hy_v3(params);
295        case LLM_ARCH_HY_V4:
296            return new llama_model_hy_v4(params);
297        case LLM_ARCH_SMOLLM3:
298            return new llama_model_smollm3(params);
299        case LLM_ARCH_OPENAI_MOE:
300            return new llama_model_openai_moe(params);
301        case LLM_ARCH_FALCON_H1:
302            return new llama_model_falcon_h1(params);
303        case LLM_ARCH_LFM2:
304            return new llama_model_lfm2(params);
305        case LLM_ARCH_LFM2MOE:
306            return new llama_model_lfm2moe(params);
307        case LLM_ARCH_SMALLTHINKER:
308            return new llama_model_smallthinker(params);
309        case LLM_ARCH_GROVEMOE:
310            return new llama_model_grovemoe(params);
311        case LLM_ARCH_APERTUS:
312            return new llama_model_apertus(params);
313        case LLM_ARCH_MINIMAX_01:
314            return new llama_model_minimax_01(params);
315        case LLM_ARCH_MINIMAX_M2:
316            return new llama_model_minimax_m2(params);
317        case LLM_ARCH_MINIMAX_M3:
318            return new llama_model_minimax_m3(params);
319        case LLM_ARCH_COGVLM:
320            return new llama_model_cogvlm(params);
321        case LLM_ARCH_PANGU_EMBED:
322            return new llama_model_pangu_embed(params);
323        case LLM_ARCH_QWEN3NEXT:
324            return new llama_model_qwen3next(params);
325        case LLM_ARCH_QWEN35:
326            return new llama_model_qwen35(params);
327        case LLM_ARCH_QWEN35MOE:
328            return new llama_model_qwen35moe(params);
329        case LLM_ARCH_QWEN4EXP:
330            return new llama_model_qwen4exp(params);
331        case LLM_ARCH_MISTRAL3:
332            return new llama_model_mistral3(params);
333        case LLM_ARCH_EAGLE3:
334            return new llama_model_eagle3(params);
335        case LLM_ARCH_DFLASH:
336            return new llama_model_dflash(params);
337        case LLM_ARCH_MIMO2:
338            return new llama_model_mimo2(params);
339        case LLM_ARCH_KIMI_LINEAR:
340            return new llama_model_kimi_linear(params);
341        case LLM_ARCH_KIMI_K3:
342            return new llama_model_kimi_k3(params);
343        case LLM_ARCH_STEP35:
344            return new llama_model_step35(params);
345        case LLM_ARCH_SPARK2_5:
346            return new llama_model_spark2_5(params);
347        default:
348            throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");
349    }
350
351}
352
353llama_model * llama_model_create(llm_arch arch, const llama_model_params & params) {
354    llama_model * model = llama_model_mapping(arch, params);
355
356    if (model != nullptr) {
357        model->arch = arch;
358        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR && !llm_arch_supports_sm_tensor(arch)) {
359            throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");
360        }
361    }
362
363    return model;
364}
365
366llama_model * llama_model_create(llama_model_loader & ml, const llama_model_params & params) {
367    llm_arch arch = ml.get_arch();
368    if (arch == LLM_ARCH_UNKNOWN) {
369        throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");
370    }
371
372    return llama_model_create(arch, params);
373}
374
375struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata) {
376    const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;
377    const llama_hparams & hparams = ud->model->hparams;
378    const std::string tensor_name = tensor->name;
379    const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 ||
380        (ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0);
381
382    static const std::regex pattern_q_weight        ("blk\\.\\d*\\.attn_q.weight");
383    static const std::regex pattern_kv_weight       ("blk\\.\\d*\\.attn_(k|v).weight");
384    static const std::regex pattern_qkv_weight      ("blk\\.\\d*\\.attn_qkv.weight");
385    static const std::regex pattern_q_bias          ("blk\\.\\d*\\.attn_q\\.bias");
386    static const std::regex pattern_kv_bias         ("blk\\.\\d*\\.attn_(k|v)\\.bias");
387    static const std::regex pattern_qkv_bias        ("blk\\.\\d*\\.attn_qkv.bias");
388    static const std::regex pattern_qk_norm         ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
389    static const std::regex pattern_kv_cache        ("cache_(k|v)_l\\d*");
390    static const std::regex pattern_idx_cache       ("cache_idx_(k|v)_l\\d*");
391    static const std::regex pattern_dsv4_state      ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");
392    static const std::regex pattern_attn_sinks      ("blk\\.\\d*\\.attn_sinks.weight");
393    static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
394    static const std::regex pattern_attn_out_bias   ("blk\\.\\d*\\.attn_output.bias");
395    static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
396    static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight");
397    static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight");
398    static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
399
400    static const std::regex pattern_ssm_dt          ("blk\\.\\d*\\.ssm_dt.bias");
401    static const std::regex pattern_ssm_a           ("blk\\.\\d*\\.ssm_a");
402    static const std::regex pattern_ssm_alpha       ("blk\\.\\d*\\.ssm_alpha.weight");
403    static const std::regex pattern_ssm_beta        ("blk\\.\\d*\\.ssm_beta.weight");
404    static const std::regex pattern_ssm_beta_alpha  ("blk\\.\\d*\\.ssm_ba.weight");
405    static const std::regex pattern_r_cache         ("cache_r_l\\d*");
406    static const std::regex pattern_ple_r_cache     ("cache_ple_r_l\\d*");
407    static const std::regex pattern_s_cache         ("cache_s_l\\d*");
408    static const std::regex pattern_ssm_conv1d      ("blk\\.\\d*\\.ssm_conv1d.weight");
409    static const std::regex pattern_ssm_out_weight  ("blk\\.\\d*\\.ssm_out.weight");
410
411    static const std::regex pattern_ffn_up_weight     ("blk\\.\\d*\\.ffn_up(_exps)?.weight");
412    static const std::regex pattern_ffn_up_bias       ("blk\\.\\d*\\.ffn_up(_exps)?.bias");
413    static const std::regex pattern_ffn_gate_weight   ("blk\\.\\d*\\.ffn_gate(_exps)?.weight");
414    static const std::regex pattern_ffn_gate_bias     ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");
415    static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
416    static const std::regex pattern_ffn_down_weight   ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
417    static const std::regex pattern_ffn_down_bias         ("blk\\.\\d*\\.ffn_down.bias");
418    static const std::regex pattern_ffn_down_exps_bias    ("blk\\.\\d*\\.ffn_down_exps.bias");
419    static const std::regex pattern_ffn_up_shexp_weight   ("blk\\.\\d*\\.ffn_up_shexp.weight");
420    static const std::regex pattern_ffn_gate_shexp_weight ("blk\\.\\d*\\.ffn_gate_shexp.weight");
421    static const std::regex pattern_ffn_down_shexp_weight ("blk\\.\\d*\\.ffn_down_shexp.weight");
422
423    static const std::regex pattern_output_weight("output\\.weight");
424    static const std::regex pattern_output_bias  ("output\\.bias");
425
426    struct tensor_config {
427        ggml_backend_meta_split_axis axis;
428
429        const ggml_tensor * tensor_axis_0;
430
431        uint32_t il;
432        size_t   rotation; // when assigning tensor slices, rotate how the rounding is done for more even allocation
433    };
434
435    auto get_tensor_config_impl = [&](
436                const ggml_backend_meta_split_axis axis, const std::string & suffix = "", const std::string & suffix_fallback = "") -> tensor_config {
437        // the layers in a tensor can be inhomogeneous, if the pattern is cleanly divided by the number of GPUs there can be aliasing effects,
438        //     count only the same type of previous layers to avoid this
439        auto get_il_eff = [&](const size_t il){
440            size_t ret = 0;
441            const bool il_is_recr = hparams.is_recr(il);
442            const bool il_is_swa  = hparams.is_swa(il);
443            for (size_t il_prev = 0; il_prev < il; il_prev++) {
444                ret += hparams.is_recr(il_prev) == il_is_recr && hparams.is_swa(il_prev) == il_is_swa;
445            }
446            return ret;
447        };
448
449        uint32_t il;
450        std::string prefix;
451        size_t rotation;
452        if (tensor_name.substr(0, 4) == "blk.") {
453            const size_t length_prefix = tensor_name.find('.', 4);
454            GGML_ASSERT(length_prefix != std::string::npos);
455            prefix = tensor_name.substr(0, length_prefix + 1);
456            il = std::stoull(tensor_name.substr(4, length_prefix));
457            rotation = get_il_eff(il) % ud->n_devices;
458        } else if (tensor_name.substr(0, 6) == "cache_") {
459            const size_t layer_index_start = tensor_name.find("_l", 6);
460            GGML_ASSERT(layer_index_start != std::string::npos);
461            il = std::stoull(tensor_name.substr(layer_index_start + 2));
462            prefix = "blk." + std::to_string(il) + ".";
463            rotation = get_il_eff(il) % ud->n_devices;
464        } else {
465            il = 0;
466            rotation = hparams.n_layer() % ud->n_devices;
467        }
468        const ggml_tensor * tensor_axis_0 = suffix.empty() ? tensor : ud->model->get_tensor((prefix + suffix).c_str());
469        if (tensor_axis_0 == nullptr) {
470            GGML_ASSERT(!suffix_fallback.empty());
471            tensor_axis_0 = ud->model->get_tensor((prefix + suffix_fallback).c_str());
472        }
473        GGML_ASSERT(tensor_axis_0 != nullptr);
474        return {axis, tensor_axis_0, il, rotation};
475    };
476
477    auto get_tensor_config = [&]() -> tensor_config {
478        if (is_dsv4) {
479            if (std::regex_match(tensor_name, pattern_kv_cache) ||
480                    std::regex_match(tensor_name, pattern_dsv4_state)) {
481                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
482            }
483            if (std::regex_match(tensor_name, pattern_attn_sinks)) {
484                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output_a.weight");
485            }
486            if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {
487                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output_a.weight");
488            }
489            if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {
490                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2);
491            }
492            if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {
493                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
494            }
495            if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
496                    std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) {
497                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight");
498            }
499            if (std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {
500                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down_shexp.weight");
501            }
502        }
503
504        // the qsa indexer has one key head and its projections are mirrored, so its cache cannot be split
505        if (std::regex_match(tensor_name, pattern_idx_cache)) {
506            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
507        }
508
509        // the PLE table is model-level and its conv is mirrored, so every device runs the whole conv and needs the whole history
510        if (std::regex_match(tensor_name, pattern_ple_r_cache)) {
511            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
512        }
513
514        // standard attention
515        if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {
516            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
517        }
518        if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {
519            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");
520        }
521        if (std::regex_match(tensor_name, pattern_qkv_weight)) {
522            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
523        }
524        if ( std::regex_match(tensor_name, pattern_qkv_bias)) {
525            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");
526        }
527        if (std::regex_match(tensor_name, pattern_qk_norm)) {
528            return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
529        }
530        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {
531            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
532        }
533        if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
534            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
535        }
536        if (std::regex_match(tensor_name, pattern_attn_out_bias)) {
537            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
538        }
539
540        if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {
541            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");
542        }
543        if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {
544            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
545        }
546        if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||
547                std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
548            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
549        }
550        if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
551            if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {
552                // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too
553                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");
554            }
555            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
556        }
557        if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {
558            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
559        }
560        if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {
561            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
562        }
563
564        // FFN
565        if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) {
566            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");
567        }
568        if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) {
569            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");
570        }
571        if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
572            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");
573        }
574        if (std::regex_match(tensor_name, pattern_ffn_down_weight)) {
575            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");
576        }
577        if (std::regex_match(tensor_name, pattern_ffn_down_bias)) {
578            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
579        }
580        if (std::regex_match(tensor_name, pattern_ffn_down_exps_bias)) {
581            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_PARTIAL, "ffn_down_exps.weight");
582        }
583
584        // output
585        if (std::regex_match(tensor_name, pattern_output_weight)) {
586            if (is_dsv4) {
587                return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
588            }
589            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);
590        }
591        if (std::regex_match(tensor_name, pattern_output_bias)) {
592            const ggml_tensor * output_weight = ud->model->get_tensor("output.weight");
593            GGML_ASSERT(output_weight != nullptr);
594            return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
595        }
596
597        // everything else
598        return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
599    };
600
601    auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {
602        if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
603                ud->model->arch == LLM_ARCH_QWEN4EXP) {
604            const int64_t head_k_dim = hparams.ssm_d_state;
605            const int64_t head_v_dim = hparams.ssm_d_state;
606            const int64_t n_k_heads  = hparams.ssm_n_group;
607            const int64_t n_v_heads  = hparams.ssm_dt_rank;
608            const int64_t key_dim    = head_k_dim * n_k_heads;
609            const int64_t value_dim  = head_v_dim * n_v_heads;
610
611            // both Qwen 3 Next and Qwen 3.5 support n_v_heads > n_k_heads but the broadcasting pattern is different:
612            //   - Qwen 3 Next: [k0_v0, k0_v1, k1_v2, k1_v3] (this is the default split pattern)
613            //   - Qwen 3.5:    [k0_v0, k1_v1, k0_v2, k1_v3] (needs segmenting of V on the scale of K to get the correct pattern)
614            if (ud->model->arch == LLM_ARCH_QWEN3NEXT) {
615                if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {
616                    GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
617                    return {{key_dim, 2}, {value_dim, 1}};
618                }
619                if (std::regex_match(tensor_name, pattern_r_cache)) {
620                    return {{key_dim * (hparams.ssm_d_conv - 1), 2}, {value_dim * (hparams.ssm_d_conv - 1), 1}};
621                }
622            } else {
623                const int64_t head_ratio = n_v_heads / n_k_heads;
624                if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {
625                    GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
626                    return {{key_dim, 2 + head_ratio}};
627                }
628                if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
629                    return {{key_dim, head_ratio}};
630                }
631                if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
632                        std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
633                    return {{n_k_heads, head_ratio}};
634                }
635                if (std::regex_match(tensor_name, pattern_r_cache)) {
636                    return {{key_dim * (hparams.ssm_d_conv - 1), 2 + head_ratio}};
637                }
638                if (std::regex_match(tensor_name, pattern_s_cache)) {
639                    return {{n_k_heads * head_v_dim * head_v_dim, head_ratio}};
640                }
641            }
642
643            // the FFN is the same for Qwen 3 Next and Qwen 3.5:
644            if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
645                const int64_t n_ff_exp = hparams.n_ff_exp(il);
646                GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
647                return {{n_ff_exp, 2}};
648            }
649            return {{tensor->ne[axis], 1}};
650        }
651
652        if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
653            const int64_t n_embd      = hparams.n_embd;
654            const int64_t n_embd_gqa  = hparams.n_embd_v_gqa(il);
655            GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa);
656            GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);
657            return {{n_embd, 1}, {n_embd_gqa, 2}};
658        }
659        if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) {
660            const int64_t n_ff = hparams.n_ff(il);
661            // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented
662            if (tensor->ne[axis] == 2*n_ff) {
663                return {{n_ff, 2}};
664            }
665            return {{tensor->ne[axis], 1}};
666        }
667        if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
668            const int64_t n_ff_exp = hparams.n_ff_exp(il);
669            GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
670            return {{n_ff_exp, 2}};
671        }
672        return {{tensor->ne[axis], 1}};
673    };
674
675    auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<std::pair<int64_t, uint32_t>> & segments) -> std::vector<int64_t> {
676        // for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used
677        if (hparams.is_recr(il)) {
678            // linear attention
679            const int64_t head_dim        = hparams.ssm_d_state;
680            const int64_t blck_size_perf  = std::lcm(blck_size, 128);
681            const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim);
682            if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||
683                    std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
684                return std::vector<int64_t>(segments.size(), granularity_qkv);
685            }
686            if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
687                    std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
688                return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);
689            }
690            if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
691                return std::vector<int64_t>(segments.size(), 2 * (granularity_qkv / head_dim));
692            }
693            if (std::regex_match(tensor_name, pattern_r_cache)) {
694                return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));
695            }
696            if (std::regex_match(tensor_name, pattern_s_cache)) {
697                return std::vector<int64_t>(segments.size(), granularity_qkv * head_dim);
698            }
699        } else {
700            // regular attention
701            const uint32_t n_gqa    = hparams.n_gqa(il);
702            const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);
703
704            // to handle head sizes like 80, only increase granularity while it doesn't cause underutilization
705            int64_t blck_size_perf = blck_size;
706            while (blck_size_perf < 128 && blck_size_perf*ud->n_devices < n_embd_q) {
707                blck_size_perf *= 2;
708            }
709
710            const int64_t granularity_q    = std::lcm(n_embd_q, blck_size_perf);
711            const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head
712            if (std::regex_match(tensor_name, pattern_attn_sinks)) {
713                GGML_ASSERT(segments.size() == 1);
714                if (is_dsv4) {
715                    return {hparams.n_head(il) / hparams.dsv4_o_group_count};
716                }
717                return {granularity_head};
718            }
719
720            if (is_dsv4) {
721                if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) {
722                    GGML_ASSERT(segments.size() == 1);
723                    // the grouped output projection requires each device to hold whole groups of heads
724                    const int64_t n_head_group = hparams.n_head(il) / hparams.dsv4_o_group_count;
725                    return {n_head_group * hparams.n_embd_head_k(il)};
726                }
727                if (std::regex_match(tensor_name, pattern_attn_out_a_weight)) {
728                    GGML_ASSERT(segments.size() == 1);
729                    return {1};
730                }
731                if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) {
732                    GGML_ASSERT(segments.size() == 1);
733                    // the boundaries must align with wo_a's per-group split, so quant blocks must not straddle groups
734                    GGML_ASSERT(hparams.dsv4_o_lora_rank % blck_size == 0);
735                    return {hparams.dsv4_o_lora_rank};
736                }
737            }
738            if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {
739                GGML_ASSERT(segments.size() == 1);
740                // some models have Q gate tensors, for those cases the granularity needs to be doubled:
741                if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
742                        ud->model->arch == LLM_ARCH_QWEN4EXP) {
743                    return {std::lcm(2*n_embd_q, blck_size_perf)};
744                }
745                return {granularity_q};
746            }
747            if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
748                GGML_ASSERT(segments.size() == 1);
749                return {granularity_q};
750            }
751            if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {
752                GGML_ASSERT(segments.size() == 1);
753                if (tensor->ne[1] == hparams.n_head(il)) {
754                    return {granularity_head};
755                }
756                return {granularity_q};
757            }
758
759            const int64_t granularity_kv = granularity_q / n_gqa;
760            if (std::regex_match(tensor_name, pattern_kv_weight) ||
761                std::regex_match(tensor_name, pattern_kv_bias) ||
762                std::regex_match(tensor_name, pattern_kv_cache)) {
763                GGML_ASSERT(segments.size() == 1);
764                return {granularity_kv};
765            }
766            if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
767                GGML_ASSERT(segments.size() == 2);
768                return {granularity_q, granularity_kv};
769            }
770        }
771
772        // FFN
773        if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||
774                std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||
775                std::regex_match(tensor_name, pattern_ffn_gate_up_weight) ||
776                std::regex_match(tensor_name, pattern_ffn_down_weight) ||
777                std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) ||
778                std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight) ||
779                std::regex_match(tensor_name, pattern_ffn_down_shexp_weight)) {
780            const int64_t blck_size_perf = std::lcm(blck_size, 128);
781            GGML_ASSERT(segments.size() == 1);
782            return {blck_size_perf};
783        }
784
785        // everything else
786        GGML_ASSERT(segments.size() == 1);
787        return {1};
788    };
789
790    ggml_backend_meta_split_state split_state;
791    memset(&split_state, 0, sizeof(split_state));
792    tensor_config tc = get_tensor_config();
793    split_state.axis = tc.axis;
794    if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {
795        const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);
796        const float * tensor_split = ud->model->tensor_split();
797        std::vector<float> tensor_split_scan;
798        tensor_split_scan.reserve(ud->n_devices);
799        for (size_t j = 0; j < ud->n_devices; j++) {
800            tensor_split_scan.push_back(tensor_split == nullptr ? 0.0f : tensor_split[(j + tc.rotation) % ud->n_devices]);
801            if (j > 0) {
802                tensor_split_scan[j] += tensor_split_scan[j - 1];
803            }
804        }
805        const std::vector<std::pair<int64_t, uint32_t>> segments = get_split_segments(split_state.axis, tc.il);
806        const std::vector<int64_t> granularity = get_split_granularity(blck_size, tc.il, segments);
807        for (size_t is = 0; is < segments.size(); is++) {
808            const int64_t  ne_s = segments[is].first;
809            const uint32_t nr_s = segments[is].second;
810            const int64_t  g_s  = granularity[is];
811            int64_t low = 0;
812            size_t j = 0;
813            for (; j < ud->n_devices - 1; j++) {
814                int64_t high = tensor_split_scan.back() == 0.0f ?
815                    ne_s * (j+1)/ud->n_devices : ne_s * tensor_split_scan[j]/tensor_split_scan.back();
816                if (high % g_s != 0) {
817                    high -= high % g_s;
818                }
819                split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = high - low;
820                low = high;
821            }
822            split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = ne_s - low;
823            split_state.nr[is] = nr_s;
824        }
825        split_state.n_segments = segments.size();
826    } else {
827        memset(split_state.ne, 0, sizeof(split_state.ne));
828        split_state.nr[0] = 1;
829        split_state.n_segments = 1;
830        if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) {
831            GGML_ASSERT(tc.tensor_axis_0 != tensor);
832            const ggml_backend_meta_split_state source_split_state = llama_meta_device_get_split_state(tc.tensor_axis_0, userdata);
833            GGML_ASSERT(source_split_state.axis >= 0 && source_split_state.axis < GGML_MAX_DIMS);
834            for (size_t j = 0; j < ud->n_devices; j++) {
835                for (size_t is = 0; is < source_split_state.n_segments; is++) {
836                    split_state.ne[j] += source_split_state.ne[is*ud->n_devices + j] * source_split_state.nr[is];
837                }
838            }
839        }
840    }
841    return split_state;
842    GGML_UNUSED(userdata);
843}
844
845const char * llm_type_name(llm_type type) {
846    switch (type) {
847        case LLM_TYPE_14M:           return "14M";
848        case LLM_TYPE_17M:           return "17M";
849        case LLM_TYPE_22M:           return "22M";
850        case LLM_TYPE_33M:           return "33M";
851        case LLM_TYPE_47M:           return "47M";
852        case LLM_TYPE_60M:           return "60M";
853        case LLM_TYPE_70M:           return "70M";
854        case LLM_TYPE_80M:           return "80M";
855        case LLM_TYPE_109M:          return "109M";
856        case LLM_TYPE_137M:          return "137M";
857        case LLM_TYPE_140M:          return "140M";
858        case LLM_TYPE_149M:          return "149M";
859        case LLM_TYPE_160M:          return "160M";
860        case LLM_TYPE_190M:          return "190M";
861        case LLM_TYPE_220M:          return "220M";
862        case LLM_TYPE_230M:          return "230M";
863        case LLM_TYPE_250M:          return "250M";
864        case LLM_TYPE_256M:          return "256M";
865        case LLM_TYPE_270M:          return "270M";
866        case LLM_TYPE_335M:          return "335M";
867        case LLM_TYPE_350M:          return "350M";
868        case LLM_TYPE_360M:          return "360M";
869        case LLM_TYPE_395M:          return "395M";
870        case LLM_TYPE_410M:          return "410M";
871        case LLM_TYPE_450M:          return "450M";
872        case LLM_TYPE_475M:          return "475M";
873        case LLM_TYPE_558M:          return "558M";
874        case LLM_TYPE_700M:          return "700M";
875        case LLM_TYPE_770M:          return "770M";
876        case LLM_TYPE_780M:          return "780M";
877        case LLM_TYPE_950M:          return "950M";
878        case LLM_TYPE_0_3B:          return "0.3B";
879        case LLM_TYPE_0_5B:          return "0.5B";
880        case LLM_TYPE_0_6B:          return "0.6B";
881        case LLM_TYPE_0_8B:          return "0.8B";
882        case LLM_TYPE_1B:            return "1B";
883        case LLM_TYPE_1_2B:          return "1.2B";
884        case LLM_TYPE_1_3B:          return "1.3B";
885        case LLM_TYPE_1_4B:          return "1.4B";
886        case LLM_TYPE_1_5B:          return "1.5B";
887        case LLM_TYPE_1_6B:          return "1.6B";
888        case LLM_TYPE_1_7B:          return "1.7B";
889        case LLM_TYPE_1_8B:          return "1.8B";
890        case LLM_TYPE_2B:            return "2B";
891        case LLM_TYPE_2_6B:          return "2.6B";
892        case LLM_TYPE_2_8B:          return "2.8B";
893        case LLM_TYPE_2_9B:          return "2.9B";
894        case LLM_TYPE_3B:            return "3B";
895        case LLM_TYPE_4B:            return "4B";
896        case LLM_TYPE_6B:            return "6B";
897        case LLM_TYPE_6_9B:          return "6.9B";
898        case LLM_TYPE_7B:            return "7B";
899        case LLM_TYPE_8B:            return "8B";
900        case LLM_TYPE_9B:            return "9B";
901        case LLM_TYPE_11B:           return "11B";
902        case LLM_TYPE_12B:           return "12B";
903        case LLM_TYPE_13B:           return "13B";
904        case LLM_TYPE_14B:           return "14B";
905        case LLM_TYPE_15B:           return "15B";
906        case LLM_TYPE_16B:           return "16B";
907        case LLM_TYPE_20B:           return "20B";
908        case LLM_TYPE_26B:           return "26B";
909        case LLM_TYPE_27B:           return "27B";
910        case LLM_TYPE_30B:           return "30B";
911        case LLM_TYPE_31B:           return "31B";
912        case LLM_TYPE_32B:           return "32B";
913        case LLM_TYPE_34B:           return "34B";
914        case LLM_TYPE_35B:           return "35B";
915        case LLM_TYPE_36B:           return "36B";
916        case LLM_TYPE_40B:           return "40B";
917        case LLM_TYPE_65B:           return "65B";
918        case LLM_TYPE_70B:           return "70B";
919        case LLM_TYPE_120B:          return "120B";
920        case LLM_TYPE_142B:          return "142B";
921        case LLM_TYPE_236B:          return "236B";
922        case LLM_TYPE_290B:          return "290B";
923        case LLM_TYPE_314B:          return "314B";
924        case LLM_TYPE_405B:          return "405B";
925        case LLM_TYPE_456B:          return "456B";
926        case LLM_TYPE_671B:          return "671B";
927        case LLM_TYPE_SMALL:         return "0.1B";
928        case LLM_TYPE_MEDIUM:        return "0.4B";
929        case LLM_TYPE_LARGE:         return "0.8B";
930        case LLM_TYPE_XL:            return "1.5B";
931        case LLM_TYPE_A1_7B:         return "A1.7B";
932        case LLM_TYPE_A2_7B:         return "A2.7B";
933        case LLM_TYPE_8x7B:          return "8x7B";
934        case LLM_TYPE_8x22B:         return "8x22B";
935        case LLM_TYPE_16x12B:        return "16x12B";
936        case LLM_TYPE_16x3_8B:       return "16x3.8B";
937        case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";
938        case LLM_TYPE_57B_A14B:      return "57B.A14B";
939        case LLM_TYPE_17B_16E:       return "17Bx16E (Scout)";
940        case LLM_TYPE_17B_128E:      return "17Bx128E (Maverick)";
941        case LLM_TYPE_A13B:          return "A13B";
942        case LLM_TYPE_1B_A400M:      return "1B.A400M";
943        case LLM_TYPE_3B_A800M:      return "3B.A800M";
944        case LLM_TYPE_7B_A1B:        return "7B.A1B";
945        case LLM_TYPE_8B_A1B:        return "8B.A1B";
946        case LLM_TYPE_7_9B_A1_3B:    return "7.9B.A1.3B";
947        case LLM_TYPE_12B_A2_5B:     return "12B.A2.5B";
948        case LLM_TYPE_16B_A1B:       return "16B.A1B";
949        case LLM_TYPE_21B_A3B:       return "21B.A3B";
950        case LLM_TYPE_24B_A2B:       return "24B.A2B";
951        case LLM_TYPE_26B_A4B:       return "26B.A4B";
952        case LLM_TYPE_30B_A3B:       return "30B.A3B";
953        case LLM_TYPE_31B_A3_5B:     return "31B.A3.5B";
954        case LLM_TYPE_32B_A9B:       return "32B.A9B";
955        case LLM_TYPE_35B_A3B:       return "35B.A3B";
956        case LLM_TYPE_48B_A3B:       return "48B.A3B";
957        case LLM_TYPE_75B_A9B:       return "75B.A9B";
958        case LLM_TYPE_80B_A3B:       return "80B.A3B";
959        case LLM_TYPE_A3B:           return "A3B";
960        case LLM_TYPE_100B_A6B:      return "100B.A6B";
961        case LLM_TYPE_102B_A12B:     return "102B.A12B";
962        case LLM_TYPE_106B_A12B:     return "106B.A12B";
963        case LLM_TYPE_118B_A8B:      return "118B.A8B";
964        case LLM_TYPE_120B_A12B:     return "120B.A12B";
965        case LLM_TYPE_122B_A10B:     return "122B.A10B";
966        case LLM_TYPE_124B_A5_1B:    return "124B.A5.1B";
967        case LLM_TYPE_196B_A11B:     return "196B.A11B";
968        case LLM_TYPE_230B_A10B:     return "230B.A10B";
969        case LLM_TYPE_428B_A23B:     return "428B.A23B";
970        case LLM_TYPE_235B_A22B:     return "235B.A22B";
971        case LLM_TYPE_288B_A19B:     return "288B.A19B";
972        case LLM_TYPE_300B_A47B:     return "300B.A47B";
973        case LLM_TYPE_310B_A15B:     return "310B.A15B";
974        case LLM_TYPE_355B_A32B:     return "355B.A32B";
975        case LLM_TYPE_397B_A17B:     return "397B.A17B";
976        case LLM_TYPE_685B_A37B:     return "685B.A37B";
977        case LLM_TYPE_744B_A40B:     return "744B.A40B";
978        case LLM_TYPE_2_8T_A50B:     return "2.8T.A50B";
979        case LLM_TYPE_E2B:           return "E2B";
980        case LLM_TYPE_E4B:           return "E4B";
981        default:                     return "?B";
982    }
983}
984
985static const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {
986    switch (type) {
987        case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";
988        case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";
989        case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: return "sqrtsoftplus";
990        default:                                    return "unknown";
991    }
992}
993
994static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {
995    { LLAMA_ROPE_SCALING_TYPE_NONE,       "none"       },
996    { LLAMA_ROPE_SCALING_TYPE_LINEAR,     "linear"     },
997    { LLAMA_ROPE_SCALING_TYPE_YARN,       "yarn"       },
998    { LLAMA_ROPE_SCALING_TYPE_LONGROPE,   "longrope"   },
999};
1000
1001std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type) {
1002    return LLAMA_ROPE_SCALING_TYPES.at(rope_scaling_type);
1003}
1004
1005static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {
1006    for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {
1007        if (kv.second == name) {
1008            return (llama_rope_scaling_type) kv.first;
1009        }
1010    }
1011
1012    return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
1013}
1014
1015// Maps the GGUF `<arch>.hidden_activation` string to the FFN op type used by the
1016// graph builders. Only gated activations that map cleanly to llm_ffn_op_type are
1017// listed; unrecognized values fall back to GeGLU, which matches the historical
1018// default for ModernBert-style architectures.
1019static const std::map<std::string, llm_ffn_op_type> LLM_FFN_OP_TYPES_FROM_STRING = {
1020    { "gelu",   LLM_FFN_GEGLU  },
1021    { "geglu",  LLM_FFN_GEGLU  },
1022    { "silu",   LLM_FFN_SWIGLU },
1023    { "swish",  LLM_FFN_SWIGLU },
1024    { "swiglu", LLM_FFN_SWIGLU },
1025    { "relu",   LLM_FFN_RELU   },
1026    { "reglu",  LLM_FFN_REGLU  },
1027};
1028
1029llm_ffn_op_type llm_ffn_op_type_from_string(const std::string & name, llm_ffn_op_type fallback) {
1030    const auto it = LLM_FFN_OP_TYPES_FROM_STRING.find(name);
1031    if (it != LLM_FFN_OP_TYPES_FROM_STRING.end()) {
1032        return it->second;
1033    }
1034    return fallback;
1035}
1036
1037// CPU: ACCEL -> GPU host -> CPU extra -> CPU
1038static buft_list_t make_cpu_buft_list(const std::vector<llama_device> & devices, bool use_extra_bufts, bool no_host) {
1039    buft_list_t buft_list;
1040
1041    // add ACCEL buffer types
1042    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
1043        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
1044        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
1045            auto * buft = ggml_backend_dev_buffer_type(dev);
1046            // skip
1047            if (buft != ggml_backend_cpu_buffer_type()) {
1048                buft_list.emplace_back(dev, buft);
1049            }
1050        }
1051    }
1052
1053    // add a host buffer type
1054    // storing the tensors in a host buffer is useful when the processing of large batches
1055    // is offloaded to a GPU device, since it reduces the time spent on data transfers
1056    // generally, this will be done using the first device in the list
1057    // a better approach would be to handle this on a weight-by-weight basis using the offload_op
1058    // function of the device to determine if it would benefit from being stored in a host buffer
1059    if (!no_host) {
1060        for (const auto & dev : devices) {
1061            ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev.dev);
1062            if (buft) {
1063                buft_list.emplace_back(dev.dev, buft);
1064                break;
1065            }
1066        }
1067    }
1068
1069    // add extra buffer types
1070    if (use_extra_bufts) {
1071        auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
1072        if (cpu_dev == nullptr) {
1073            throw std::runtime_error(format("%s: no CPU backend found", __func__));
1074        }
1075
1076        auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);
1077        auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)
1078            ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");
1079        if (ggml_backend_dev_get_extra_bufts_fn) {
1080            ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);
1081            while (extra_bufts && *extra_bufts) {
1082                buft_list.emplace_back(cpu_dev, *extra_bufts);
1083                ++extra_bufts;
1084            }
1085        }
1086    }
1087
1088    // add the CPU buffer type
1089    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
1090        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
1091        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
1092            buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));
1093        }
1094    }
1095
1096    return buft_list;
1097}
1098
1099// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU
1100static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) {
1101    buft_list_t buft_list;
1102
1103    // add the device split buffer type if requested and available
1104    if (split_mode == LLAMA_SPLIT_MODE_ROW) {
1105        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
1106        auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)
1107            ggml_backend_reg_get_proc_address(reg, "ggml_backend_split_buffer_type");
1108        if (ggml_backend_split_buffer_type_fn) {
1109            size_t dev_index = [&]() {
1110                auto * reg = ggml_backend_dev_backend_reg(dev);
1111                for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {
1112                    if (ggml_backend_reg_dev_get(reg, i) == dev) {
1113                        return i;
1114                    }
1115                }
1116                throw std::runtime_error(format("device %s not found in its backend reg", ggml_backend_dev_name(dev)));
1117            }();
1118            auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);
1119            if (buft != nullptr) {
1120                buft_list.emplace_back(dev, buft);
1121            }
1122        } else {
1123            throw std::runtime_error(format("device %s does not support split buffers", ggml_backend_dev_name(dev)));
1124        }
1125    }
1126
1127    // add the device default buffer type
1128    buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));
1129
1130    // add the device extra buffer type (if any)
1131    ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
1132    if (reg) {
1133        auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)
1134            ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");
1135
1136        if (ggml_backend_dev_get_extra_bufts_fn) {
1137            ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);
1138            while (extra_bufts && *extra_bufts) {
1139                buft_list.emplace_back(dev, *extra_bufts);
1140                ++extra_bufts;
1141            }
1142        }
1143    }
1144
1145    return buft_list;
1146}
1147
1148struct llama_model::impl {
1149    impl() = default;
1150    ~impl() = default;
1151
1152    uint64_t n_elements = 0;
1153
1154    size_t n_bytes = 0;
1155
1156    std::string desc_str;
1157
1158    llama_ftype ftype = LLAMA_FTYPE_ALL_F32;
1159
1160    // model memory mapped files
1161    llama_mmaps mappings;
1162
1163    // objects representing data potentially being locked in memory
1164    llama_mlocks mlock_bufs;
1165    llama_mlocks mlock_mmaps;
1166
1167    // contexts where the model tensors metadata is stored as well as the corresponding buffers:
1168    std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
1169
1170    buft_list_t cpu_buft_list;
1171    std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
1172
1173    struct layer_dev {
1174        ggml_backend_dev_t dev;
1175        buft_list_t * buft_list;
1176    };
1177
1178    layer_dev dev_input = {};
1179    layer_dev dev_output = {};
1180    std::vector<layer_dev> dev_layer;
1181
1182    bool has_tensor_overrides;
1183
1184    std::vector<float> tensor_split_owned;
1185};
1186
1187llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {
1188    if (params.tensor_split != nullptr) {
1189        // llama_model_params stores tensor_split as a borrowed pointer, but the model
1190        // may need it later for tensor-parallel KV-cache split metadata.
1191        pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());
1192        this->params.tensor_split = pimpl->tensor_split_owned.data();
1193    }
1194    pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;
1195}
1196
1197llama_model::~llama_model() {
1198    for (auto * lora : loras) {
1199        delete lora;
1200    }
1201}
1202
1203void llama_model_base::load_stats(llama_model_loader & ml) {
1204    pimpl->n_elements = ml.n_elements;
1205    pimpl->n_bytes = ml.n_bytes;
1206}
1207
1208void llama_model_base::load_hparams(llama_model_loader & ml) {
1209    const gguf_context * ctx = ml.metadata;
1210
1211    // get metadata as string
1212    for (int i = 0; i < gguf_get_n_kv(ctx); i++) {
1213        gguf_type type = gguf_get_kv_type(ctx, i);
1214        if (type == GGUF_TYPE_ARRAY) {
1215            continue;
1216        }
1217        const char * name = gguf_get_key(ctx, i);
1218        const std::string value = gguf_kv_to_str(ctx, i);
1219        gguf_kv.emplace(name, value);
1220    }
1221
1222    // get general kv
1223    ml.get_key(LLM_KV_GENERAL_NAME, name, false);
1224
1225    // everything past this point is not vocab-related
1226    // for CLIP models, we only need to load tensors, no hparams
1227    if (hparams.vocab_only || ml.get_arch() == LLM_ARCH_CLIP) {
1228        return;
1229    }
1230
1231    ml.get_key(LLM_KV_CONTEXT_LENGTH,          hparams.n_ctx_train);
1232    ml.get_key(LLM_KV_EMBEDDING_LENGTH,        hparams.n_embd);
1233    ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT,    hparams.n_embd_out_impl, false);
1234    ml.get_key(LLM_KV_ATTENTION_CAUSAL,        hparams.causal_attn,     false);
1235    ml.get_key(LLM_KV_POOLING_TYPE,            hparams.pooling_type,    false);
1236    ml.get_key(LLM_KV_BLOCK_COUNT,             hparams.n_layer_all);
1237    GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
1238    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,    hparams.n_layer_nextn,   false);
1239    GGML_ASSERT(hparams.n_layer_nextn <= hparams.n_layer_all);
1240    ml.get_key(LLM_KV_EXPERT_COUNT,            hparams.n_expert,        false);
1241    std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);
1242    ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false);
1243    ml.get_key(LLM_KV_EXPERT_GROUP_COUNT,      hparams.n_expert_groups, false);
1244    ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used,    false);
1245
1246    if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) {
1247        if (hparams.n_expert <= 1) {
1248            hparams.n_expert = 0;
1249            std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);
1250        }
1251    }
1252
1253    if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
1254        ml.get_key(LLM_KV_FEATURES_LENGTH,  hparams.n_embd);
1255        ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd_out_impl);
1256
1257        ml.get_key(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);
1258        ml.get_key(LLM_KV_POSNET_BLOCK_COUNT,      hparams.posnet.n_layer);
1259
1260        ml.get_key(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);
1261        ml.get_key(LLM_KV_CONVNEXT_BLOCK_COUNT,      hparams.convnext.n_layer);
1262
1263        GGML_ASSERT(hparams.posnet.n_layer   <= hparams.n_layer_all);
1264        GGML_ASSERT(hparams.convnext.n_layer <= hparams.n_layer_all);
1265    }
1266
1267    // models may route a different number of experts per layer, so validate the maximum
1268    uint32_t n_expert_used_max = hparams.n_expert_used_max();
1269
1270    GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);
1271    GGML_ASSERT(n_expert_used_max <= hparams.n_expert);
1272    if (hparams.n_expert > 0) {
1273        GGML_ASSERT(n_expert_used_max > 0);
1274        GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);
1275        if (hparams.n_expert_groups > 1) {
1276            GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);
1277            GGML_ASSERT(hparams.n_group_used > 0);
1278            GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);
1279        }
1280    } else {
1281        GGML_ASSERT(n_expert_used_max == 0);
1282        GGML_ASSERT(hparams.n_expert_groups == 0);
1283    }
1284
1285    std::fill(hparams.n_head_arr.begin(),       hparams.n_head_arr.end(),       0);
1286    std::fill(hparams.n_head_kv_arr.begin(),    hparams.n_head_kv_arr.end(),    0);
1287    std::fill(hparams.n_ff_arr.begin(),         hparams.n_ff_arr.end(),         0);
1288    std::fill(hparams.n_ff_exp_arr.begin(),     hparams.n_ff_exp_arr.end(),     0);
1289
1290    std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);
1291    std::fill(hparams.rope_pattern.begin(),  hparams.rope_pattern.end(), 1);
1292    std::fill(hparams.is_swa_impl.begin(),   hparams.is_swa_impl.end(), 0);
1293    std::fill(hparams.is_recr_impl.begin(),  hparams.is_recr_impl.end(),  llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0);
1294    std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0);
1295
1296    std::fill(hparams.xielu_alpha_n.begin(), hparams.xielu_alpha_n.end(), 0.0f);
1297    std::fill(hparams.xielu_alpha_p.begin(), hparams.xielu_alpha_p.end(), 0.0f);
1298    std::fill(hparams.xielu_beta.begin(),    hparams.xielu_beta.end(), 0.0f);
1299    std::fill(hparams.xielu_eps.begin(),     hparams.xielu_eps.end(), 0.0f);
1300
1301    std::fill(hparams.swiglu_clamp_exp.begin(),   hparams.swiglu_clamp_exp.end(),   0.0f);
1302    std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f);
1303
1304    ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH,  hparams.n_ff_arr,   hparams.n_layer_all, false);
1305    ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer_all, false);
1306
1307    // Populate deepstack_mapping_arr - initialized to -1 (no deepstack)
1308    std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1);
1309
1310    // n_head_kv is optional, default to n_head
1311    hparams.n_head_kv_arr = hparams.n_head_arr;
1312
1313    ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer_all, false);
1314
1315    bool rope_finetuned = false;
1316    ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
1317    hparams.rope_finetuned = rope_finetuned;
1318
1319    hparams.n_ctx_orig_yarn = hparams.n_ctx_train;
1320    ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);
1321
1322    // rope_freq_base (optional)
1323    hparams.rope_freq_base_train = 10000.0f;
1324    ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);
1325
1326    std::string rope_scaling("linear");
1327    ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);
1328    hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);
1329    GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);
1330
1331    // TODO: Handle SWA metadata similarly when models start implementing it
1332    // rope_freq_scale (inverse of the kv) is optional
1333    float ropescale = 0.0f;
1334    if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {
1335        // try the old key name
1336        ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);
1337    }
1338    hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;
1339
1340    ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);
1341    ml.get_key(LLM_KV_ROPE_SCALING_ALPHA,       hparams.rope_scaling_alpha, false);
1342
1343    // non-transformer models do not have attention heads
1344    if (hparams.n_head() > 0) {
1345        // gpt-neox n_rot = rotary_pct * (n_embd / n_head)
1346        // gpt-j n_rot = rotary_dim
1347
1348        hparams.n_embd_head_k_full = hparams.n_embd / hparams.n_head();
1349        ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full, false);
1350
1351        hparams.n_embd_head_v_full = hparams.n_embd / hparams.n_head();
1352        ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full, false);
1353
1354        // sanity check for n_rot (optional)
1355        hparams.n_rot_full = hparams.n_embd_head_k_full;
1356
1357        ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full, false);
1358
1359        if (arch == LLM_ARCH_LLAMA || arch == LLM_ARCH_DECI || arch == LLM_ARCH_FALCON || arch == LLM_ARCH_LLAMA_EMBED) {
1360            if (hparams.n_rot_full != hparams.n_embd_head_k_full) {
1361                throw std::runtime_error(format("invalid n_rot: %u, expected %u", hparams.n_rot_full, hparams.n_embd_head_k_full));
1362            }
1363        }
1364    } else {
1365        hparams.n_rot_full = 0;
1366        hparams.n_embd_head_k_full = 0;
1367        hparams.n_embd_head_v_full = 0;
1368    }
1369
1370    // head size and n_rot for SWA layers
1371    {
1372        hparams.n_embd_head_k_swa = hparams.n_embd_head_k_full;
1373        hparams.n_embd_head_v_swa = hparams.n_embd_head_v_full;
1374        ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa, false);
1375        ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa, false);
1376
1377        hparams.n_rot_swa = hparams.n_rot_full;
1378        ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa, false);
1379    }
1380
1381    // for classifier models
1382    ml.get_arr(LLM_KV_CLASSIFIER_OUTPUT_LABELS, classifier_labels, false);
1383    if (!classifier_labels.empty()) {
1384        hparams.n_cls_out = classifier_labels.size();
1385    }
1386
1387    // per-arch hparams
1388    load_arch_hparams(ml);
1389
1390    pimpl->n_bytes = ml.n_bytes;
1391
1392    pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name();
1393
1394    pimpl->ftype = ml.ftype;
1395
1396    if (hparams.f_max_alibi_bias > 0.0f) {
1397        hparams.use_alibi = true;
1398    }
1399
1400    hparams.rope_type = llama_model_rope_type(this);
1401}
1402
1403void llama_model_base::load_vocab(llama_model_loader & ml) {
1404    const auto kv = LLM_KV(arch);
1405
1406    vocab.load(ml, kv);
1407}
1408
1409bool llama_model_base::load_tensors(llama_model_loader & ml) {
1410    const auto & split_mode   = params.split_mode;
1411    const bool use_mlock      = params.load_mode == LLAMA_LOAD_MODE_MLOCK || params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;
1412    const auto & tensor_split = params.tensor_split;
1413
1414    const int n_layer_all = hparams.n_layer_all;
1415    const int n_gpu_layers = this->n_gpu_layers();
1416
1417    const bool use_mmap_buffer = true;
1418
1419    this->ml = &ml; // to be used by create_tensor() and load_arch_tensors()
1420
1421    if (ml.use_mmap && params.load_mode == LLAMA_LOAD_MODE_AUTO) {
1422        for (const auto & dev : devices) {
1423            ggml_backend_dev_props props;
1424            ggml_backend_dev_get_props(dev.dev, &props);
1425            if (!props.caps.mmap_support) {
1426                ml.use_mmap = false;
1427                break;
1428            }
1429        }
1430    }
1431
1432    // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF; see #28160
1433    if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) {
1434        for (const auto & dev : devices) {
1435            ggml_backend_dev_props props;
1436            ggml_backend_dev_get_props(dev.dev, &props);
1437            if (!props.caps.mmap_support) {
1438                ml.lazy.mode = LLAMA_LAZY_MODE_OFF;
1439                break;
1440            }
1441        }
1442    }
1443
1444    const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO
1445        ? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE)
1446        : llama_load_mode_name(params.load_mode);
1447
1448    LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (load_mode = %s)\n",
1449        __func__, load_mode_name);
1450
1451    // build a list of buffer types for the CPU and GPU devices
1452    pimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);
1453    for (const auto & dev : devices) {
1454        buft_list_t buft_list = make_gpu_buft_list(dev.dev, split_mode, tensor_split);
1455        // add CPU buffer types as a fallback
1456        buft_list.insert(buft_list.end(), pimpl->cpu_buft_list.begin(), pimpl->cpu_buft_list.end());
1457        pimpl->gpu_buft_list.emplace(dev.dev, std::move(buft_list));
1458    }
1459
1460    ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
1461    if (cpu_dev == nullptr) {
1462        throw std::runtime_error(format("%s: no CPU backend found", __func__));
1463    }
1464
1465    // calculate the split points
1466    bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + n_devices(), [](float x) { return x == 0.0f; });
1467    std::vector<float> splits(n_devices());
1468    if (all_zero) {
1469        // default split, by free memory
1470        for (size_t i = 0; i < n_devices(); ++i) {
1471            ggml_backend_dev_t dev = devices[i].dev;
1472            size_t total;
1473            size_t free;
1474            ggml_backend_dev_memory(dev, &free, &total);
1475
1476            // devices can return 0 bytes for free and total memory if they do not
1477            // have any to report. in this case, we will use the host memory as a fallback
1478            // fixes: https://github.com/ggml-org/llama.cpp/issues/18577
1479            if (free == 0 && total == 0) {
1480                ggml_backend_dev_memory(cpu_dev, &free, &total);
1481            }
1482            splits[i] = free;
1483        }
1484    } else {
1485        std::copy(tensor_split, tensor_split + n_devices(), splits.begin());
1486    }
1487
1488    // sum and normalize the splits to get the split points
1489    float split_sum = 0.0f;
1490    for (size_t i = 0; i < n_devices(); ++i) {
1491        split_sum += splits[i];
1492        splits[i] = split_sum;
1493    }
1494    for (size_t i = 0; i < n_devices(); ++i) {
1495        splits[i] /= split_sum;
1496    }
1497
1498    const int i_gpu_start = std::max(n_layer_all + 1 - n_gpu_layers, 0);
1499    const int act_gpu_layers = devices.empty() ? 0 : std::min(n_gpu_layers, n_layer_all + 1);
1500    auto get_layer_buft_list = [&](int il) -> llama_model::impl::layer_dev {
1501        const bool is_swa = il < n_layer_all && hparams.is_swa(il);
1502        if (il < i_gpu_start || (il - i_gpu_start) >= act_gpu_layers) {
1503            LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(cpu_dev), is_swa);
1504            return {cpu_dev, &pimpl->cpu_buft_list};
1505        }
1506        const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin();
1507        auto * dev = devices.at(layer_gpu).dev;
1508        LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(dev), is_swa);
1509        return {dev, &pimpl->gpu_buft_list.at(dev)};
1510    };
1511
1512    // assign the input layer
1513    // there is very little benefit to offloading the input layer, so always keep it on the CPU
1514    pimpl->dev_input = { cpu_dev, &pimpl->cpu_buft_list };
1515
1516    // assign the repeating layers to the devices according to the splits
1517    pimpl->dev_layer.resize(n_layer_all);
1518    for (int il = 0; il < n_layer_all; ++il) {
1519        pimpl->dev_layer[il] = get_layer_buft_list(il);
1520    }
1521
1522    // assign the output layer
1523    pimpl->dev_output = get_layer_buft_list(n_layer_all);
1524
1525    const auto TENSOR_NOT_REQUIRED = llama_model_loader::TENSOR_NOT_REQUIRED;
1526
1527    // create tensors for the weights
1528    {
1529        // TODO: move to a separate function
1530        const auto tn = LLM_TN(arch);
1531
1532        const int64_t n_expert = hparams.n_expert;
1533
1534        if (n_expert > 0 && hparams.n_expert_used_max() == 0) {
1535            throw std::runtime_error("model has expert layers but no expert layers are used");
1536        }
1537
1538        layers.resize(n_layer_all);
1539
1540        // call the per-model loading function
1541        load_arch_tensors(ml);
1542
1543        // generic pass: load optional per-tensor/per-expert ".scale" tensors (e.g. NVFP4 scale2)
1544        // this avoids having to add scale loading to every architecture
1545        for (int i = 0; i < n_layer_all; ++i) {
1546            auto & layer = layers[i];
1547
1548            // attention weight scales (per-tensor, shape {1})
1549            if (!layer.wq_s && layer.wq) {
1550                layer.wq_s = create_tensor(tn(LLM_TENSOR_ATTN_Q,   "scale", i), {1}, TENSOR_NOT_REQUIRED);
1551            }
1552            if (!layer.wk_s && layer.wk) {
1553                layer.wk_s = create_tensor(tn(LLM_TENSOR_ATTN_K,   "scale", i), {1}, TENSOR_NOT_REQUIRED);
1554            }
1555            if (!layer.wv_s && layer.wv) {
1556                layer.wv_s = create_tensor(tn(LLM_TENSOR_ATTN_V,   "scale", i), {1}, TENSOR_NOT_REQUIRED);
1557            }
1558            if (!layer.wo_s && layer.wo) {
1559                layer.wo_s = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1560            }
1561            if (!layer.wqkv_s && layer.wqkv) {
1562                layer.wqkv_s = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1563            }
1564            if (!layer.wqkv_gate_s && layer.wqkv_gate) {
1565                layer.wqkv_gate_s = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1566            }
1567
1568            // dense FFN weight scales (per-tensor, shape {1})
1569            if (!layer.ffn_gate_s && layer.ffn_gate) {
1570                layer.ffn_gate_s = create_tensor(tn(LLM_TENSOR_FFN_GATE, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1571            }
1572            if (!layer.ffn_down_s && layer.ffn_down) {
1573                layer.ffn_down_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1574            }
1575            if (!layer.ffn_up_s && layer.ffn_up) {
1576                layer.ffn_up_s = create_tensor(tn(LLM_TENSOR_FFN_UP, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1577            }
1578            if (!layer.ffn_gate_shexp_s && layer.ffn_gate_shexp) {
1579                layer.ffn_gate_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1580            }
1581            if (!layer.ffn_down_shexp_s && layer.ffn_down_shexp) {
1582                layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1583            }
1584            if (!layer.ffn_up_shexp_s && layer.ffn_up_shexp) {
1585                layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1586            }
1587
1588            // MoE expert weight scales (per-expert, shape {n_expert})
1589            if (!layer.ffn_gate_exps_s && layer.ffn_gate_exps) {
1590                layer.ffn_gate_exps_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1591            }
1592            if (!layer.ffn_down_exps_s && layer.ffn_down_exps) {
1593                layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1594            }
1595            if (!layer.ffn_up_exps_s && layer.ffn_up_exps) {
1596                layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1597            }
1598
1599            // recurrent / linear-attention weight scales (per-tensor, shape {1})
1600            if (!layer.ssm_in_s && layer.ssm_in) {
1601                layer.ssm_in_s = create_tensor(tn(LLM_TENSOR_SSM_IN, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1602            }
1603            if (!layer.ssm_out_s && layer.ssm_out) {
1604                layer.ssm_out_s = create_tensor(tn(LLM_TENSOR_SSM_OUT, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1605            }
1606            if (!layer.ssm_alpha_s && layer.ssm_alpha) {
1607                layer.ssm_alpha_s = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1608            }
1609            if (!layer.ssm_beta_s && layer.ssm_beta) {
1610                layer.ssm_beta_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1611            }
1612            if (!layer.nextn.eh_proj_s && layer.nextn.eh_proj) {
1613                layer.nextn.eh_proj_s = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1614            }
1615            if (!layer.nextn.shared_head_head_s && layer.nextn.shared_head_head) {
1616                layer.nextn.shared_head_head_s = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "scale", i), {1}, TENSOR_NOT_REQUIRED);
1617            }
1618
1619            // input scales
1620            if (!layer.wq_in_s && layer.wq) {
1621                layer.wq_in_s = create_tensor(tn(LLM_TENSOR_ATTN_Q,   "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1622            }
1623            if (!layer.wk_in_s && layer.wk) {
1624                layer.wk_in_s = create_tensor(tn(LLM_TENSOR_ATTN_K,   "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1625            }
1626            if (!layer.wv_in_s && layer.wv) {
1627                layer.wv_in_s = create_tensor(tn(LLM_TENSOR_ATTN_V,   "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1628            }
1629            if (!layer.wo_in_s && layer.wo) {
1630                layer.wo_in_s = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1631            }
1632            if (!layer.wqkv_in_s && layer.wqkv) {
1633                layer.wqkv_in_s = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1634            }
1635            if (!layer.wqkv_gate_in_s && layer.wqkv_gate) {
1636                layer.wqkv_gate_in_s = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1637            }
1638            if (!layer.ffn_gate_in_s && layer.ffn_gate) {
1639                layer.ffn_gate_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1640            }
1641            if (!layer.ffn_down_in_s && layer.ffn_down) {
1642                layer.ffn_down_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1643            }
1644            if (!layer.ffn_up_in_s && layer.ffn_up) {
1645                layer.ffn_up_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1646            }
1647            if (!layer.ffn_gate_exps_in_s && layer.ffn_gate_exps) {
1648                layer.ffn_gate_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1649            }
1650            if (!layer.ffn_down_exps_in_s && layer.ffn_down_exps) {
1651                layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1652            }
1653            if (!layer.ffn_up_exps_in_s && layer.ffn_up_exps) {
1654                layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
1655            }
1656            if (!layer.ffn_gate_shexp_in_s && layer.ffn_gate_shexp) {
1657                layer.ffn_gate_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1658            }
1659            if (!layer.ffn_down_shexp_in_s && layer.ffn_down_shexp) {
1660                layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1661            }
1662            if (!layer.ffn_up_shexp_in_s && layer.ffn_up_shexp) {
1663                layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1664            }
1665            if (!layer.ssm_in_in_s && layer.ssm_in) {
1666                layer.ssm_in_in_s = create_tensor(tn(LLM_TENSOR_SSM_IN, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1667            }
1668            if (!layer.ssm_out_in_s && layer.ssm_out) {
1669                layer.ssm_out_in_s = create_tensor(tn(LLM_TENSOR_SSM_OUT, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1670            }
1671            if (!layer.ssm_alpha_in_s && layer.ssm_alpha) {
1672                layer.ssm_alpha_in_s = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1673            }
1674            if (!layer.ssm_beta_in_s && layer.ssm_beta) {
1675                layer.ssm_beta_in_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1676            }
1677            if (!layer.nextn.eh_proj_in_s && layer.nextn.eh_proj) {
1678                layer.nextn.eh_proj_in_s = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1679            }
1680            if (!layer.nextn.shared_head_head_in_s && layer.nextn.shared_head_head) {
1681                layer.nextn.shared_head_head_in_s = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
1682            }
1683        }
1684        // output scales
1685        if (output && output->type == GGML_TYPE_NVFP4) {
1686            // weight scale
1687            if (!output_s) {
1688                output_s = create_tensor(tn(LLM_TENSOR_OUTPUT, "scale"), {1}, TENSOR_NOT_REQUIRED);
1689            }
1690            // input scale
1691            if (!output_in_s) {
1692                output_in_s = create_tensor(tn(LLM_TENSOR_OUTPUT, "input_scale"), {1}, TENSOR_NOT_REQUIRED);
1693            }
1694        }
1695    }
1696    ml.done_getting_tensors();
1697
1698    // Tied NVFP4 output is valid when no separate LM-head scale tensors are present.
1699    // If sidecar scales exist, the output weight must be an actual output tensor.
1700    GGML_ASSERT(!(output && tok_embd &&
1701            strcmp(output->name, tok_embd->name) == 0 &&
1702            output->type == GGML_TYPE_NVFP4 &&
1703            (output_s || output_in_s)));
1704    // populate tensors_by_name
1705    for (auto & [_, ctx_ptr] : ml.ctx_map) {
1706        for (auto * cur = ggml_get_first_tensor(ctx_ptr.get()); cur != NULL; cur = ggml_get_next_tensor(ctx_ptr.get(), cur)) {
1707            tensors_by_name.emplace_back(ggml_get_name(cur), cur);
1708        }
1709    }
1710
1711    ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
1712    pimpl->mappings.reserve(ml.mappings.size());
1713
1714    // create the backend buffers
1715    std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;
1716    ctx_buf_maps.reserve(ml.ctx_map.size());
1717
1718    // Ensure we have enough capacity for the maximum backend buffer we will potentially create
1719    const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();
1720    pimpl->ctxs_bufs.reserve(n_max_backend_buffer);
1721
1722    for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) {
1723        ggml_backend_buffer_type_t buft = ctx_key.buft;
1724        ggml_context * ctx = ctx_ptr.get();
1725
1726        // skip contexts without tensors
1727        if (ggml_get_first_tensor(ctx) == nullptr) {
1728            continue;
1729        }
1730
1731        llama_buf_map buf_map;
1732        buf_map.reserve(n_max_backend_buffer);
1733
1734        // check if it is possible to use buffer_from_host_ptr with this buffer type
1735        ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);
1736        if (!dev) {
1737            // FIXME: workaround for CPU backend buft having a NULL device
1738            dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
1739            if (!dev) {
1740                throw std::runtime_error(format("%s: no CPU backend found", __func__));
1741            }
1742        }
1743        ggml_backend_dev_props props;
1744        ggml_backend_dev_get_props(dev, &props);
1745        bool buffer_from_host_ptr_supported = props.caps.buffer_from_host_ptr;
1746        bool is_default_buft = buft == ggml_backend_dev_buffer_type(dev);
1747
1748        std::vector<ggml_backend_buffer_ptr> bufs;
1749
1750        // a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
1751        const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
1752
1753        if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
1754            GGML_ASSERT(!ml.no_alloc);
1755            for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
1756                // only the mmap region containing the tensors in the model is mapped to the backend buffer
1757                // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer,
1758                //     then we could just use metal for all layers
1759                // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size
1760                void * addr = nullptr;
1761                size_t first, last; // NOLINT
1762                ml.get_mapping_range(&first, &last, &addr, idx, ctx);
1763                if (first >= last) {
1764                    continue;
1765                }
1766                const size_t max_size = ggml_get_max_tensor_size(ctx);
1767                ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);
1768                if (buf == nullptr) {
1769                    throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
1770                }
1771                bufs.emplace_back(buf);
1772                buf_map.emplace(idx, buf);
1773            }
1774        } else {
1775            ggml_backend_buffer_t buf;
1776            if (ml.no_alloc) {
1777                buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
1778                for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
1779                    t->buffer = buf; // set dummy buffer for weights so that the backend scheduler won't try to allocate them
1780                }
1781            } else {
1782                buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); // real buffer
1783            }
1784            if (buf == nullptr) {
1785                throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
1786            }
1787            if (use_mlock && ggml_backend_buffer_is_host(buf)) {
1788                pimpl->mlock_bufs.emplace_back(new llama_mlock);
1789                auto & mlock_buf = pimpl->mlock_bufs.back();
1790                mlock_buf->init   (ggml_backend_buffer_get_base(buf));
1791                mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));
1792            }
1793            bufs.emplace_back(buf);
1794            for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
1795                buf_map.emplace(idx, buf);
1796            }
1797        }
1798
1799        for (auto & buf : bufs) {
1800            // indicate that this buffer contains weights
1801            // this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight
1802            ggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
1803        }
1804
1805        pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
1806
1807        ctx_buf_maps.emplace_back(ctx, buf_map);
1808    }
1809
1810    if (llama_supports_gpu_offload()) {
1811        const int n_gpu = std::min(n_gpu_layers, n_layer_all);
1812
1813        int n_repeating = n_gpu;
1814        if (n_repeating > 0) {
1815            LLAMA_LOG_INFO("%s: offloading output layer to GPU\n", __func__);
1816            n_repeating--;
1817        }
1818        LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_repeating);
1819
1820        const int max_backend_supported_layers = n_layer_all + 1;
1821        const int max_offloadable_layers       = n_layer_all + 1;
1822
1823        LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);
1824    }
1825
1826    // print memory requirements per buffer type
1827    for (auto & [_, bufs] : pimpl->ctxs_bufs) {
1828        for (auto & buf: bufs) {
1829            LLAMA_LOG_INFO("%s: %12s model buffer size = %8.2f MiB\n",
1830                __func__, ggml_backend_buffer_name(buf.get()), ggml_backend_buffer_get_size(buf.get()) / 1024.0 / 1024.0);
1831        }
1832    }
1833
1834    if (ml.no_alloc) {
1835        return true;
1836    }
1837
1838    // without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are resident
1839    if (!ml.use_mmap) {
1840        std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) {
1841            const auto & buf_map = ctx_buf_map.second;
1842            return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second);
1843        });
1844    }
1845
1846    // load tensor data
1847    for (auto & [ctx, buf_map] : ctx_buf_maps) {
1848        if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
1849            return false;
1850        }
1851    }
1852
1853    if (use_mmap_buffer) {
1854        for (auto & mapping : ml.mappings) {
1855            pimpl->mappings.emplace_back(std::move(mapping));
1856        }
1857    }
1858
1859    return true;
1860}
1861
1862ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
1863    const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;
1864    return ml.create_tensor(
1865        hparams, &pimpl->cpu_buft_list, pimpl->dev_input.buft_list, pimpl->dev_output.buft_list, buft_list_layer,
1866        tn, ne, flags);
1867}
1868
1869std::string llama_model::arch_name() const {
1870    return llm_arch_name(arch);
1871}
1872
1873std::string llama_model::type_name() const {
1874    return llm_type_name(type);
1875}
1876
1877std::string llama_model::desc() const {
1878    return pimpl->desc_str;
1879}
1880
1881llama_ftype llama_model::ftype() const {
1882    return pimpl->ftype;
1883}
1884
1885size_t llama_model::size() const {
1886    return pimpl->n_bytes;
1887}
1888
1889size_t llama_model::n_tensors() const {
1890    return tensors_by_name.size();
1891}
1892
1893size_t llama_model::n_devices() const {
1894    return devices.size();
1895}
1896
1897const float * llama_model::tensor_split() const {
1898    return params.tensor_split;
1899}
1900
1901uint32_t llama_model::n_gpu_layers() const {
1902    // note: plus 1 for the "output" layer
1903    return params.n_gpu_layers >= 0 ? params.n_gpu_layers : hparams.n_layer_all + 1;
1904}
1905
1906llama_split_mode llama_model::split_mode() const {
1907    return params.split_mode;
1908}
1909
1910std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() const {
1911    std::map<ggml_backend_buffer_type_t, size_t> ret;
1912    for (const auto & [ctx, bufs] : pimpl->ctxs_bufs) {
1913        if (hparams.no_alloc) {
1914            GGML_ASSERT(bufs.size() == 1);
1915            ggml_backend_buffer_t buf = bufs[0].get();
1916            GGML_ASSERT(ggml_backend_buffer_get_base(buf) == nullptr);
1917            ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf);
1918            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
1919        } else {
1920            for (const auto & buf : bufs) {
1921                // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
1922                ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
1923            }
1924        }
1925    }
1926    return ret;
1927}
1928
1929uint64_t llama_model::n_elements() const {
1930    return pimpl->n_elements;
1931}
1932
1933void llama_model::print_info() const {
1934    const std::string rope_scaling_type = llama_rope_scaling_type_name(hparams.rope_scaling_type_train);
1935
1936    auto print_f = [](const std::function<int32_t(uint32_t)> & f, uint32_t n) {
1937        bool is_var = false;
1938
1939        std::vector<int32_t> v;
1940        for (uint32_t i = 0; i < n; ++i) {
1941            v.push_back(f(i));
1942            if (v[i] != v[0]) {
1943                is_var = true;
1944            }
1945        }
1946
1947        std::stringstream ss;
1948
1949        if (is_var) {
1950            ss << "[";
1951            for (uint32_t i = 0; i < n; ++i) {
1952                ss << v[i];
1953                if (i < n - 1) {
1954                    ss << ", ";
1955                }
1956            }
1957            ss << "]";
1958        } else {
1959            ss << v[0];
1960        }
1961
1962        return ss.str();
1963    };
1964
1965    // hparams
1966    LLAMA_LOG_INFO("%s: arch                  = %s\n",     __func__, arch_name().c_str());
1967    LLAMA_LOG_INFO("%s: vocab_only            = %d\n",     __func__, hparams.vocab_only);
1968    LLAMA_LOG_INFO("%s: no_alloc              = %d\n",     __func__, hparams.no_alloc);
1969
1970    if (!hparams.vocab_only) {
1971        LLAMA_LOG_INFO("%s: n_ctx_train           = %u\n",     __func__, hparams.n_ctx_train);
1972        LLAMA_LOG_INFO("%s: n_embd_inp            = %u\n",     __func__, hparams.n_embd_inp());
1973        LLAMA_LOG_INFO("%s: n_embd                = %u\n",     __func__, hparams.n_embd);
1974        LLAMA_LOG_INFO("%s: n_embd_out            = %u\n",     __func__, hparams.n_embd_out());
1975        LLAMA_LOG_INFO("%s: n_layer               = %u\n",     __func__, hparams.n_layer());
1976        LLAMA_LOG_INFO("%s: n_layer_all           = %u\n",     __func__, hparams.n_layer_all);
1977        LLAMA_LOG_INFO("%s: n_head                = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_head(il);    }, hparams.n_layer_all).c_str());
1978        LLAMA_LOG_INFO("%s: n_head_kv             = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_head_kv(il); }, hparams.n_layer_all).c_str());
1979        LLAMA_LOG_INFO("%s: n_rot                 = %u\n",     __func__, hparams.n_rot_full);
1980        LLAMA_LOG_INFO("%s: n_swa                 = %u\n",     __func__, hparams.n_swa);
1981        LLAMA_LOG_INFO("%s: is_swa_any            = %u\n",     __func__, hparams.is_swa_any());
1982        LLAMA_LOG_INFO("%s: non_causal_type       = %d\n",     __func__, hparams.non_causal_type);
1983        LLAMA_LOG_INFO("%s: n_embd_head_k         = %u\n",     __func__, hparams.n_embd_head_k_full);
1984        LLAMA_LOG_INFO("%s: n_embd_head_v         = %u\n",     __func__, hparams.n_embd_head_v_full);
1985        LLAMA_LOG_INFO("%s: n_gqa                 = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_gqa(il);        }, hparams.n_layer_all).c_str());
1986        LLAMA_LOG_INFO("%s: n_embd_k_gqa          = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_embd_k_gqa(il); }, hparams.n_layer_all).c_str());
1987        LLAMA_LOG_INFO("%s: n_embd_v_gqa          = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_embd_v_gqa(il); }, hparams.n_layer_all).c_str());
1988        LLAMA_LOG_INFO("%s: f_norm_eps            = %.1e\n",   __func__, hparams.f_norm_eps);
1989        LLAMA_LOG_INFO("%s: f_norm_rms_eps        = %.1e\n",   __func__, hparams.f_norm_rms_eps);
1990        LLAMA_LOG_INFO("%s: f_clamp_kqv           = %.1e\n",   __func__, hparams.f_clamp_kqv);
1991        LLAMA_LOG_INFO("%s: f_max_alibi_bias      = %.1e\n",   __func__, hparams.f_max_alibi_bias);
1992        LLAMA_LOG_INFO("%s: f_logit_scale         = %.1e\n",   __func__, hparams.f_logit_scale);
1993        LLAMA_LOG_INFO("%s: f_attn_scale          = %.1e\n",   __func__, hparams.f_attention_scale);
1994        LLAMA_LOG_INFO("%s: f_attn_value_scale    = %.4f\n",   __func__, hparams.f_attn_value_scale);
1995        LLAMA_LOG_INFO("%s: n_ff                  = %s\n",     __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer_all).c_str());
1996        LLAMA_LOG_INFO("%s: n_expert              = %u\n",     __func__, hparams.n_expert);
1997        LLAMA_LOG_INFO("%s: n_expert_used         = %u\n",     __func__, hparams.n_expert_used());
1998        LLAMA_LOG_INFO("%s: n_expert_groups       = %d\n",     __func__, hparams.n_expert_groups);
1999        LLAMA_LOG_INFO("%s: n_group_used          = %d\n",     __func__, hparams.n_group_used);
2000        LLAMA_LOG_INFO("%s: causal attn           = %d\n",     __func__, hparams.causal_attn);
2001        LLAMA_LOG_INFO("%s: pooling type          = %d\n",     __func__, hparams.pooling_type);
2002        LLAMA_LOG_INFO("%s: rope type             = %d\n",     __func__, hparams.rope_type);
2003        LLAMA_LOG_INFO("%s: rope scaling          = %s\n",     __func__, rope_scaling_type.c_str());
2004        LLAMA_LOG_INFO("%s: freq_base_train       = %.1f\n",   __func__, hparams.rope_freq_base_train);
2005        LLAMA_LOG_INFO("%s: freq_scale_train      = %g\n",     __func__, hparams.rope_freq_scale_train);
2006        if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
2007            LLAMA_LOG_INFO("%s: freq_base_swa         = %.1f\n",   __func__, hparams.rope_freq_base_train_swa);
2008            LLAMA_LOG_INFO("%s: freq_scale_swa        = %g\n",     __func__, hparams.rope_freq_scale_train_swa);
2009            LLAMA_LOG_INFO("%s: n_embd_head_k_swa     = %u\n",     __func__, hparams.n_embd_head_k_swa);
2010            LLAMA_LOG_INFO("%s: n_embd_head_v_swa     = %u\n",     __func__, hparams.n_embd_head_v_swa);
2011            LLAMA_LOG_INFO("%s: n_rot_swa             = %u\n",     __func__, hparams.n_rot_swa);
2012        }
2013        LLAMA_LOG_INFO("%s: n_ctx_orig_yarn       = %u\n",     __func__, hparams.n_ctx_orig_yarn);
2014        LLAMA_LOG_INFO("%s: rope_yarn_log_mul     = %.4f\n",   __func__, hparams.rope_yarn_log_mul);
2015        LLAMA_LOG_INFO("%s: rope_finetuned        = %s\n",     __func__, hparams.rope_finetuned ? "yes" : "unknown");
2016        if (arch == LLM_ARCH_GRANITE &&
2017            std::any_of(hparams.deepstack_mapping_arr.begin(),
2018                        hparams.deepstack_mapping_arr.end(),
2019                        [](const auto & entry) { return entry >= 0; })) {
2020            LLAMA_LOG_INFO("%s: deepstack_mapping_arr = %s\n", __func__,
2021                           print_f([&](uint32_t il) { return hparams.deepstack_mapping_arr[il]; },
2022                           hparams.n_layer_all).c_str());
2023        }
2024        // MRoPE (Multi-axis Rotary Position Embedding) sections
2025        if (const auto & s = hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) {
2026            LLAMA_LOG_INFO("%s: mrope sections        = [%d, %d, %d, %d]\n", __func__, s[0], s[1], s[2], s[3]);
2027        }
2028        if (!classifier_labels.empty()) {
2029            LLAMA_LOG_INFO("%s: n_cls_out             = %u\n", __func__, hparams.n_cls_out);
2030
2031            size_t i = 0;
2032            for (const auto & label : classifier_labels) {
2033                LLAMA_LOG_INFO("%s: cls_label[%2zu]         = %s\n", __func__, i++, label.c_str());
2034            }
2035        }
2036
2037        if (arch == LLM_ARCH_MAMBA ||
2038                arch == LLM_ARCH_MAMBA2 ||
2039                arch == LLM_ARCH_JAMBA ||
2040                arch == LLM_ARCH_FALCON_H1 ||
2041                arch == LLM_ARCH_PLAMO2 ||
2042                arch == LLM_ARCH_GRANITE_HYBRID ||
2043                arch == LLM_ARCH_QWEN3NEXT ||
2044                arch == LLM_ARCH_QWEN35 ||
2045                arch == LLM_ARCH_QWEN35MOE ||
2046                arch == LLM_ARCH_NEMOTRON_H ||
2047                arch == LLM_ARCH_NEMOTRON_H_MOE) {
2048            LLAMA_LOG_INFO("%s: ssm_d_conv            = %u\n",     __func__, hparams.ssm_d_conv);
2049            LLAMA_LOG_INFO("%s: ssm_d_inner           = %u\n",     __func__, hparams.ssm_d_inner);
2050            LLAMA_LOG_INFO("%s: ssm_d_state           = %u\n",     __func__, hparams.ssm_d_state);
2051            LLAMA_LOG_INFO("%s: ssm_dt_rank           = %u\n",     __func__, hparams.ssm_dt_rank);
2052            LLAMA_LOG_INFO("%s: ssm_n_group           = %u\n",     __func__, hparams.ssm_n_group);
2053            LLAMA_LOG_INFO("%s: ssm_dt_b_c_rms        = %d\n",     __func__, hparams.ssm_dt_b_c_rms);
2054        }
2055
2056        LLAMA_LOG_INFO("%s: model type            = %s\n",     __func__, type_name().c_str());
2057        if (pimpl->n_elements >= 1e12) {
2058            LLAMA_LOG_INFO("%s: model params          = %.2f T\n", __func__, pimpl->n_elements*1e-12);
2059        } else if (pimpl->n_elements >= 1e9) {
2060            LLAMA_LOG_INFO("%s: model params          = %.2f B\n", __func__, pimpl->n_elements*1e-9);
2061        } else if (pimpl->n_elements >= 1e6) {
2062            LLAMA_LOG_INFO("%s: model params          = %.2f M\n", __func__, pimpl->n_elements*1e-6);
2063        } else {
2064            LLAMA_LOG_INFO("%s: model params          = %.2f K\n", __func__, pimpl->n_elements*1e-3);
2065        }
2066
2067        // general kv
2068        LLAMA_LOG_INFO("%s: general.name          = %s\n",    __func__, name.c_str());
2069
2070        if (arch == LLM_ARCH_DEEPSEEK) {
2071            LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
2072            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2073            LLAMA_LOG_INFO("%s: n_expert_shared       = %d\n",     __func__, hparams.n_expert_shared);
2074            LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
2075        }
2076
2077        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR ||
2078                arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA ||
2079                arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4 ||
2080                arch == LLM_ARCH_HY_V4) {
2081            LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
2082            LLAMA_LOG_INFO("%s: n_lora_q              = %d\n",     __func__, hparams.n_lora_q);
2083            LLAMA_LOG_INFO("%s: n_lora_kv             = %d\n",     __func__, hparams.n_lora_kv);
2084            LLAMA_LOG_INFO("%s: n_embd_head_k_mla     = %d\n",     __func__, hparams.n_embd_head_k_mla());
2085            LLAMA_LOG_INFO("%s: n_embd_head_v_mla     = %d\n",     __func__, hparams.n_embd_head_v_mla());
2086            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2087            LLAMA_LOG_INFO("%s: n_expert_shared       = %d\n",     __func__, hparams.n_expert_shared);
2088            LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
2089            LLAMA_LOG_INFO("%s: expert_weights_norm   = %d\n",     __func__, hparams.expert_weights_norm);
2090            LLAMA_LOG_INFO("%s: expert_gating_func    = %s\n",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
2091        }
2092
2093        if (arch == LLM_ARCH_QWEN2MOE) {
2094            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2095            LLAMA_LOG_INFO("%s: n_ff_shexp            = %d\n",     __func__, hparams.n_ff_shexp);
2096        }
2097
2098        if (arch == LLM_ARCH_MELLUM ||
2099                arch == LLM_ARCH_COHERE2MOE ||
2100                arch == LLM_ARCH_QWEN3MOE ||
2101                arch == LLM_ARCH_OPENAI_MOE ||
2102                arch == LLM_ARCH_QWEN3VLMOE ||
2103                arch == LLM_ARCH_RND1) {
2104            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2105        }
2106
2107        if (arch == LLM_ARCH_MINICPM ||
2108                arch == LLM_ARCH_GRANITE ||
2109                arch == LLM_ARCH_GRANITE_MOE ||
2110                arch == LLM_ARCH_GRANITE_HYBRID ||
2111                arch == LLM_ARCH_GRANITE_SWITCH ||
2112                arch == LLM_ARCH_NEMOTRON_H_MOE) {
2113            LLAMA_LOG_INFO("%s: f_embedding_scale     = %f\n", __func__, hparams.f_embedding_scale);
2114            LLAMA_LOG_INFO("%s: f_residual_scale      = %f\n", __func__, hparams.f_residual_scale);
2115            LLAMA_LOG_INFO("%s: f_attention_scale     = %f\n", __func__, hparams.f_attention_scale);
2116            LLAMA_LOG_INFO("%s: n_ff_shexp            = %d\n", __func__, hparams.n_ff_shexp);
2117        }
2118
2119        if (arch == LLM_ARCH_BAILINGMOE) {
2120            LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
2121            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2122            LLAMA_LOG_INFO("%s: n_expert_shared       = %d\n",     __func__, hparams.n_expert_shared);
2123            LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
2124            LLAMA_LOG_INFO("%s: expert_weights_norm   = %d\n",     __func__, hparams.expert_weights_norm);
2125        }
2126
2127        if (arch == LLM_ARCH_BAILINGMOE2 || arch == LLM_ARCH_BAILINGMOE3) {
2128            LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
2129            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2130            LLAMA_LOG_INFO("%s: n_ff_shexp            = %d\n",     __func__, hparams.n_ff_shexp);
2131            LLAMA_LOG_INFO("%s: n_expert_shared       = %d\n",     __func__, hparams.n_expert_shared);
2132            LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
2133            LLAMA_LOG_INFO("%s: expert_weights_norm   = %d\n",     __func__, hparams.expert_weights_norm);
2134            LLAMA_LOG_INFO("%s: expert_gating_func    = %s\n",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
2135            LLAMA_LOG_INFO("%s: n_layer_nextn         = %d\n",     __func__, hparams.n_layer_nextn);
2136        }
2137
2138        if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {
2139            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2140            LLAMA_LOG_INFO("%s: expert_gating_func    = %s\n",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
2141        }
2142
2143        if (arch == LLM_ARCH_GROVEMOE) {
2144            LLAMA_LOG_INFO("%s: n_ff_exp              = %d\n",     __func__, hparams.n_ff_exp());
2145            LLAMA_LOG_INFO("%s: n_ff_chexp            = %d\n",     __func__, hparams.n_ff_chexp);
2146            LLAMA_LOG_INFO("%s: n_group_experts       = %d\n",     __func__, hparams.n_group_experts);
2147            LLAMA_LOG_INFO("%s: expert_group_scale    = %.2f\n",   __func__, hparams.expert_group_scale);
2148        }
2149    }
2150
2151    vocab.print_info();
2152}
2153
2154ggml_backend_dev_t llama_model::dev_layer(int il) const {
2155    return pimpl->dev_layer.at(il).dev;
2156}
2157
2158ggml_backend_dev_t llama_model::dev_output() const {
2159    return pimpl->dev_output.dev;
2160}
2161
2162template<typename F>
2163static bool buft_supported(ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev, F & fn) {
2164    ggml_init_params params = {
2165        /*.mem_size   =*/ ggml_tensor_overhead()*8,
2166        /*.mem_buffer =*/ NULL,
2167        /*.no_alloc   =*/ true,
2168    };
2169
2170    ggml_context_ptr ctx { ggml_init(params) };
2171    if (!ctx) {
2172        throw std::runtime_error(format("failed to create ggml context"));
2173    }
2174
2175    ggml_backend_buffer_ptr buf { ggml_backend_buft_alloc_buffer(buft, 0) };
2176    ggml_tensor * op_tensor = fn(ctx.get());
2177    for (int i = 0; i < GGML_MAX_SRC; i++) {
2178        if (op_tensor->src[i] != nullptr) {
2179            assert(op_tensor->src[i]->buffer == nullptr);
2180            op_tensor->src[i]->buffer = buf.get();
2181        }
2182    }
2183
2184    bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);
2185
2186    return op_supported;
2187}
2188
2189template<typename F>
2190static ggml_backend_buffer_type_t select_buft(const buft_list_t & buft_list, const F & fn) {
2191    for (const auto & cur : buft_list) {
2192        ggml_backend_dev_t cur_dev = cur.first;
2193        ggml_backend_buffer_type_t cur_buft = cur.second;
2194        if (buft_supported(cur_buft, cur_dev, fn)) {
2195            return cur_buft;
2196        }
2197    }
2198
2199    throw std::runtime_error(format("no suitable buffer type found"));
2200}
2201
2202ggml_backend_buffer_type_t llama_model::select_buft(int il) const {
2203    return ::select_buft(
2204            *pimpl->dev_layer.at(il).buft_list,
2205            [&](ggml_context * ctx) {
2206                ggml_tensor * cur = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);
2207                ggml_tensor * layer_dir = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);
2208                return ggml_add(ctx, cur, layer_dir);
2209            });
2210}
2211
2212bool llama_model::has_tensor_overrides() const {
2213    return pimpl->has_tensor_overrides;
2214}
2215
2216const ggml_tensor * llama_model::get_tensor(const char * name) const {
2217    auto it = std::find_if(tensors_by_name.begin(), tensors_by_name.end(),
2218            [name](const std::pair<std::string, ggml_tensor *> & it) {
2219                return it.first == name;
2220            });
2221    if (it == tensors_by_name.end()) {
2222        return nullptr;
2223    }
2224
2225    return it->second;
2226}
2227
2228float llama_model::get_rope_freq_base (const llama_cparams & cparams, int il) const {
2229    return hparams.is_swa(il) ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base;
2230}
2231
2232float llama_model::get_rope_freq_scale(const llama_cparams & cparams, int il) const {
2233    return hparams.is_swa(il) ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale;
2234}
2235
2236ggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int il) const {
2237    const uint32_t n_ctx_seq = cparams.n_ctx_seq;
2238
2239    // choose long/short freq factors based on the context size
2240    if (layers[il].rope_freqs != nullptr) {
2241        return layers[il].rope_freqs;
2242    }
2243
2244    if (n_ctx_seq > hparams.n_ctx_orig_yarn) {
2245        return layers[il].rope_long;
2246    }
2247
2248    return layers[il].rope_short;
2249}
2250
2251llama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const {
2252    llama_memory_i * res;
2253
2254    switch (arch) {
2255        // Models that need specific instantiation should be handled in the
2256        // switch statement
2257        case LLM_ARCH_BERT:
2258        case LLM_ARCH_JINA_BERT_V2:
2259        case LLM_ARCH_JINA_BERT_V3:
2260        case LLM_ARCH_NOMIC_BERT:
2261        case LLM_ARCH_NOMIC_BERT_MOE:
2262        case LLM_ARCH_NEO_BERT:
2263        case LLM_ARCH_EUROBERT:
2264        case LLM_ARCH_WAVTOKENIZER_DEC:
2265        case LLM_ARCH_MODERN_BERT:
2266        case LLM_ARCH_GEMMA_EMBEDDING:
2267        case LLM_ARCH_DREAM:
2268        case LLM_ARCH_LLADA:
2269        case LLM_ARCH_LLADA_MOE:
2270        case LLM_ARCH_RND1:
2271            {
2272                res = nullptr;
2273            } break;
2274        case LLM_ARCH_MINIMAX_M3:
2275            {
2276                // sparse (MSA) layers carry an indexer key cache, but leading dense layers do not
2277                llama_kv_cache::layer_filter_cb filter_idx =
2278                    [&](int32_t il) { return (uint32_t) il >= hparams.n_layer_dense_lead; };
2279
2280                res = new llama_kv_cache_msa(
2281                        *this,
2282                        params.type_k,
2283                        params.type_v,
2284                        !cparams.flash_attn,
2285                        cparams.offload_kqv,
2286                        cparams.kv_unified,
2287                        cparams.n_ctx_seq,
2288                        cparams.n_seq_max,
2289                        1,
2290                        hparams.n_swa,
2291                        hparams.swa_type,
2292                        nullptr,
2293                        filter_idx,
2294                        nullptr);
2295            } break;
2296        case LLM_ARCH_GLM_DSA:
2297        case LLM_ARCH_DEEPSEEK32:
2298            {
2299                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
2300                    // The NextN/MTP draft head runs dense MLA (no DSA indexer), so the
2301                    // MTP context uses a plain attention KV cache holding only the
2302                    // nextn layer(s) - same pattern as the hybrid Qwen3.5 MTP context.
2303                    llama_kv_cache::layer_filter_cb filter =
2304                        [&](uint32_t il) { return il >= hparams.n_layer(); };
2305
2306                    res = new llama_kv_cache(
2307                            *this,
2308                            hparams,
2309                            params.type_k,
2310                            params.type_v,
2311                            !cparams.flash_attn,
2312                            cparams.offload_kqv,
2313                            cparams.kv_unified,
2314                            cparams.n_ctx_seq,
2315                            cparams.n_seq_max,
2316                            1,
2317                            hparams.n_swa,
2318                            hparams.swa_type,
2319                            nullptr,
2320                            filter,
2321                            nullptr,
2322                            nullptr);
2323                } else {
2324                    // Main context: DSA cache for the trunk layers only - the nextn
2325                    // layer(s) are never attended by the trunk graph.
2326                    llama_kv_cache::layer_filter_cb filter_mla = nullptr;
2327                    if (hparams.n_layer_nextn > 0) {
2328                        filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
2329                    }
2330                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && (arch != LLM_ARCH_GLM_DSA || hparams.is_indexer_full(il)); };
2331
2332                    res = new llama_kv_cache_dsa(
2333                            *this,
2334                            params.type_k,
2335                            params.type_v,
2336                            !cparams.flash_attn,
2337                            cparams.offload_kqv,
2338                            cparams.kv_unified,
2339                            cparams.n_ctx_seq,
2340                            cparams.n_seq_max,
2341                            1,
2342                            hparams.n_swa,
2343                            hparams.swa_type,
2344                            filter_mla,
2345                            filter_lid,
2346                            nullptr);
2347                }
2348            } break;
2349        case LLM_ARCH_HY_V4:
2350            {
2351                if (hparams.indexer_top_k == 0) {
2352                    // full-attention checkpoint: no indexer, so no indexer key cache
2353                    res = new llama_kv_cache(
2354                            *this,
2355                            hparams,
2356                            params.type_k,
2357                            params.type_v,
2358                            !cparams.flash_attn,
2359                            cparams.offload_kqv,
2360                            cparams.kv_unified,
2361                            cparams.n_ctx_seq,
2362                            cparams.n_seq_max,
2363                            1,
2364                            hparams.n_swa,
2365                            hparams.swa_type,
2366                            nullptr,
2367                            nullptr,
2368                            nullptr,
2369                            nullptr);
2370                } else {
2371                    // only "full" layers own an indexer, so the shared layers need no indexer cache
2372                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return hparams.is_indexer_full(il); };
2373
2374                    res = new llama_kv_cache_dsa(
2375                            *this,
2376                            params.type_k,
2377                            params.type_v,
2378                            !cparams.flash_attn,
2379                            cparams.offload_kqv,
2380                            cparams.kv_unified,
2381                            cparams.n_ctx_seq,
2382                            cparams.n_seq_max,
2383                            1,
2384                            hparams.n_swa,
2385                            hparams.swa_type,
2386                            nullptr,
2387                            filter_lid,
2388                            nullptr);
2389                }
2390            } break;
2391        case LLM_ARCH_DOTS3NOTE:
2392            {
2393                GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
2394
2395                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
2396                    // MTP draft context: plain attention KV cache holding only the nextn layer
2397                    llama_kv_cache::layer_filter_cb filter =
2398                        [&](uint32_t il) { return il >= hparams.n_layer(); };
2399
2400                    res = new llama_kv_cache(
2401                            *this,
2402                            hparams,
2403                            params.type_k,
2404                            params.type_v,
2405                            !cparams.flash_attn,
2406                            cparams.offload_kqv,
2407                            cparams.kv_unified,
2408                            cparams.n_ctx_seq,
2409                            cparams.n_seq_max,
2410                            1,
2411                            hparams.n_swa,
2412                            hparams.swa_type,
2413                            nullptr,
2414                            filter,
2415                            nullptr,
2416                            nullptr);
2417                } else {
2418                    // main context: DSA cache for the trunk full-attention layers plus a window-sized SWA cache
2419                    llama_kv_cache::layer_filter_cb filter_mla = nullptr;
2420                    if (hparams.n_layer_nextn > 0) {
2421                        filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
2422                    }
2423                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && hparams.is_indexer_full(il); };
2424
2425                    res = new llama_kv_cache_dsa_iswa(
2426                            *this,
2427                            params.type_k,
2428                            params.type_v,
2429                            !cparams.flash_attn,
2430                            cparams.offload_kqv,
2431                            params.swa_full,
2432                            cparams.kv_unified,
2433                            cparams.n_ctx_seq,
2434                            cparams.n_seq_max,
2435                            cparams.n_ubatch,
2436                            1,
2437                            filter_mla,
2438                            filter_lid,
2439                            nullptr);
2440                }
2441            } break;
2442        case LLM_ARCH_DEEPSEEK4:
2443            {
2444                GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
2445
2446                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
2447                    const llama_memory_i::layer_filter_cb filter_mtp = [&](int32_t il) {
2448                        return il >= (int32_t) hparams.n_layer();
2449                    };
2450
2451                    res = new llama_kv_cache_iswa(
2452                            *this,
2453                            params.type_k,
2454                            params.type_v,
2455                            !cparams.flash_attn,
2456                            cparams.offload_kqv,
2457                            params.swa_full,
2458                            cparams.kv_unified,
2459                            cparams.n_ctx_seq,
2460                            cparams.n_seq_max,
2461                            cparams.n_ubatch,
2462                            1,
2463                            nullptr,
2464                            filter_mtp,
2465                            nullptr,
2466                            nullptr);
2467                } else {
2468                    res = new llama_kv_cache_dsv4(
2469                            *this,
2470                            params.type_k,
2471                            params.type_v,
2472                            !cparams.flash_attn,
2473                            cparams.offload_kqv,
2474                            params.swa_full,
2475                            cparams.kv_unified,
2476                            cparams.n_ctx_seq,
2477                            cparams.n_seq_max,
2478                            cparams.n_ubatch,
2479                            1,
2480                            cparams.n_rs_seq,
2481                            nullptr,
2482                            nullptr);
2483                }
2484            } break;
2485        case LLM_ARCH_DFLASH:
2486            {
2487                // DSV4 DSpark stages store a single MLA-style K per position (window = the draft ring)
2488                if (hparams.dsv4_hc_mult > 0) {
2489                    GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
2490
2491                    res = new llama_kv_cache_iswa(
2492                            *this,
2493                            params.type_k,
2494                            params.type_v,
2495                            !cparams.flash_attn,
2496                            cparams.offload_kqv,
2497                            params.swa_full,
2498                            cparams.kv_unified,
2499                            cparams.n_ctx_seq,
2500                            cparams.n_seq_max,
2501                            cparams.n_ubatch,
2502                            1,
2503                            nullptr,
2504                            nullptr,
2505                            nullptr,
2506                            nullptr);
2507                    break;
2508                }
2509            }
2510            [[fallthrough]];
2511        // Models that need standard caching should rely on recurrent/hybrid
2512        // checks
2513        default:
2514            {
2515                // Dense MTP heads use a plain attention KV cache instead of the hybrid wrapper.
2516                const bool mtp_on_hybrid_qwen =
2517                    params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
2518                    (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE ||
2519                     arch == LLM_ARCH_BAILINGMOE3);
2520
2521                const bool mtp_on_hybrid_nemotron =
2522                    params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && arch == LLM_ARCH_NEMOTRON_H_MOE;
2523
2524                if (llm_arch_is_recurrent(arch)) {
2525                    res = new llama_memory_recurrent(
2526                            *this,
2527                            GGML_TYPE_F32,
2528                            GGML_TYPE_F32,
2529                            cparams.offload_kqv,
2530                            std::max((uint32_t) 1, cparams.n_seq_max),
2531                            cparams.n_seq_max,
2532                            cparams.n_rs_seq,
2533                            nullptr);
2534                } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen && !mtp_on_hybrid_nemotron) {
2535                    // The main difference between hybrid architectures is the
2536                    // layer filters, so pick the right one here
2537                    llama_memory_hybrid::layer_filter_cb filter_attn = nullptr;
2538                    llama_memory_hybrid::layer_filter_cb filter_recr = nullptr;
2539                    // only the sparse-attention architectures use llama_memory_hybrid_idx
2540                    // a null filter_idx means the GGUF has no indexer tensors
2541                    llama_memory_hybrid::layer_filter_cb filter_idx  = nullptr;
2542                    const bool needs_mem_idx = (arch == LLM_ARCH_QWEN4EXP);
2543                    if (arch == LLM_ARCH_FALCON_H1) {
2544                        filter_attn = [&](uint32_t) { return true; };
2545                        filter_recr = [&](uint32_t) { return true; };
2546                    } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
2547                        filter_attn = [&](uint32_t il) {
2548                            return !hparams.is_recr(il) && hparams.n_ff(il) == 0;
2549                        };
2550                        filter_recr = [&](uint32_t il) {
2551                            return hparams.is_recr(il) && hparams.n_ff(il) == 0;
2552                        };
2553                    } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
2554                        filter_attn = [&](uint32_t il) {
2555                            return il < hparams.n_layer() && !hparams.is_recr(il);
2556                        };
2557                        filter_recr = [&](uint32_t il) {
2558                            return il < hparams.n_layer() && hparams.is_recr(il);
2559                        };
2560
2561                        if (arch == LLM_ARCH_QWEN4EXP && hparams.indexer_head_size > 0) {
2562                            // QSA runs on the dense-attention layers only
2563                            filter_idx = [&](uint32_t il) {
2564                                return il < hparams.n_layer() && !hparams.is_recr(il);
2565                            };
2566                        }
2567                    }
2568
2569                    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
2570                        // Use hybrid-iswa for hybrid models with SWA
2571                        res = new llama_memory_hybrid_iswa(
2572                            /* model             */ *this,
2573                            /* attn_type_k       */ params.type_k,
2574                            /* attn_type_v       */ params.type_v,
2575                            /* attn_v_trans      */ !cparams.flash_attn,
2576                            /* attn_swa_full     */ params.swa_full,
2577                            /* attn_kv_size      */ cparams.n_ctx_seq,
2578                            /* attn_n_ubatch     */ cparams.n_ubatch,
2579                            /* attn_n_pad        */ 1,
2580                            /* recurrent_type_r  */ GGML_TYPE_F32,
2581                            /* recurrent_type_s  */ GGML_TYPE_F32,
2582                            /* recurrent_rs_size */ std::max((uint32_t) 1, cparams.n_seq_max),
2583                            /* n_seq_max         */ cparams.n_seq_max,
2584                            /* n_rs_seq          */ cparams.n_rs_seq,
2585                            /* offload           */ cparams.offload_kqv,
2586                            /* unified           */ cparams.kv_unified,
2587                            /* filter_attn       */ std::move(filter_attn),
2588                            /* filter_recr       */ std::move(filter_recr));
2589                    } else if (needs_mem_idx) {
2590                        // sparse attention over a per-token indexer cache, in its own memory type
2591                        res = new llama_memory_hybrid_idx(
2592                            /* model             */ *this,
2593                            /* attn_type_k       */ params.type_k,
2594                            /* attn_type_v       */ params.type_v,
2595                            /* attn_v_trans      */ !cparams.flash_attn,
2596                            /* attn_kv_size      */ cparams.n_ctx_seq,
2597                            /* attn_n_pad        */ 1,
2598                            /* attn_n_swa        */ hparams.n_swa,
2599                            /* attn_swa_type     */ hparams.swa_type,
2600                            /* recurrent_type_k  */ GGML_TYPE_F32,
2601                            /* recurrent_type_v  */ GGML_TYPE_F32,
2602                            /* recurrent_kv_size */ std::max((uint32_t) 1, cparams.n_seq_max),
2603                            /* n_seq_max         */ cparams.n_seq_max,
2604                            /* n_rs_seq          */ cparams.n_rs_seq,
2605                            /* offload           */ cparams.offload_kqv,
2606                            /* unified           */ cparams.kv_unified,
2607                            /* filter_attn       */ std::move(filter_attn),
2608                            /* filter_recr       */ std::move(filter_recr),
2609                            /* filter_idx        */ std::move(filter_idx));
2610                    } else {
2611                        res = new llama_memory_hybrid(
2612                            /* model             */ *this,
2613                            /* attn_type_k       */ params.type_k,
2614                            /* attn_type_v       */ params.type_v,
2615                            /* attn_v_trans      */ !cparams.flash_attn,
2616                            /* attn_kv_size      */ cparams.n_ctx_seq,
2617                            /* attn_n_pad        */ 1,
2618                            /* attn_n_swa        */ hparams.n_swa,
2619                            /* attn_swa_type     */ hparams.swa_type,
2620                            /* recurrent_type_k  */ GGML_TYPE_F32,
2621                            /* recurrent_type_v  */ GGML_TYPE_F32,
2622                            /* recurrent_kv_size */ std::max((uint32_t) 1, cparams.n_seq_max),
2623                            /* n_seq_max         */ cparams.n_seq_max,
2624                            /* n_rs_seq          */ cparams.n_rs_seq,
2625                            /* offload           */ cparams.offload_kqv,
2626                            /* unified           */ cparams.kv_unified,
2627                            /* filter_attn       */ std::move(filter_attn),
2628                            /* filter_recr       */ std::move(filter_recr));
2629                    }
2630                } else {
2631                    llama_kv_cache::layer_filter_cb filter = nullptr;
2632                    llama_memory_i::layer_reuse_cb reuse = nullptr;
2633                    llama_kv_cache::layer_share_cb share = nullptr;
2634
2635                    if (arch == LLM_ARCH_GEMMA3N || arch == LLM_ARCH_GEMMA4) {
2636                        reuse = [&](uint32_t il) {
2637                            GGML_ASSERT(hparams.n_layer_kv_from_start >= 2);
2638
2639                            if (il >= (uint32_t)hparams.n_layer_kv_from_start) {
2640                                return hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);
2641                            }
2642
2643                            return -1;
2644                        };
2645                    }
2646
2647                    if (arch == LLM_ARCH_ALICEAI_T5_MOE) {
2648                        filter = [&](uint32_t il) { return il < hparams.dec_n_layer; };
2649                    }
2650
2651                    if (mtp_on_hybrid_qwen || mtp_on_hybrid_nemotron) {
2652                        filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
2653                    }
2654
2655                    // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends
2656                    // or when a model is entirely n_layer_nextn layers and has no trunk
2657                    if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) {
2658                        if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
2659                            filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
2660                        } else {
2661                            filter = [&](uint32_t il) { return il <  hparams.n_layer(); };
2662                        }
2663                    }
2664
2665                    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
2666                        GGML_ASSERT(hparams.is_swa_any());
2667
2668                        if (arch == LLM_ARCH_GEMMA4_ASSISTANT) {
2669                            llama_memory_t mem_other = llama_get_memory(cparams.ctx_other);
2670
2671                            share = [&](int32_t il) {
2672                                const llama_model * model_other = llama_get_model(cparams.ctx_other);
2673
2674                                if (hparams.is_swa(il)) {
2675                                    return llama_model_n_layer(model_other) - 2;
2676                                }
2677
2678                                return llama_model_n_layer(model_other) - 1;
2679                            };
2680
2681                            res = new llama_kv_cache_iswa(
2682                                    *this,
2683                                    params.type_k,
2684                                    params.type_v,
2685                                    !cparams.flash_attn,
2686                                    cparams.offload_kqv,
2687                                    params.swa_full,
2688                                    cparams.kv_unified,
2689                                    cparams.n_ctx_seq,
2690                                    cparams.n_seq_max,
2691                                    cparams.n_ubatch,
2692                                    1,
2693                                    mem_other,
2694                                    filter,
2695                                    reuse,
2696                                    share);
2697                        } else {
2698                            res = new llama_kv_cache_iswa(
2699                                    *this,
2700                                    params.type_k,
2701                                    params.type_v,
2702                                    !cparams.flash_attn,
2703                                    cparams.offload_kqv,
2704                                    params.swa_full,
2705                                    cparams.kv_unified,
2706                                    cparams.n_ctx_seq,
2707                                    cparams.n_seq_max,
2708                                    cparams.n_ubatch,
2709                                    1,
2710                                    nullptr,
2711                                    filter,
2712                                    reuse,
2713                                    share);
2714                        }
2715                    } else {
2716                        GGML_ASSERT(!hparams.is_swa_any());
2717
2718                        res = new llama_kv_cache(
2719                                *this,
2720                                hparams,
2721                                params.type_k,
2722                                params.type_v,
2723                                !cparams.flash_attn,
2724                                cparams.offload_kqv,
2725                                cparams.kv_unified,
2726                                cparams.n_ctx_seq,
2727                                cparams.n_seq_max,
2728                                1,
2729                                hparams.n_swa,
2730                                hparams.swa_type,
2731                                nullptr,
2732                                filter,
2733                                nullptr,
2734                                nullptr);
2735                    }
2736                }
2737            }
2738    }
2739
2740    return res;
2741}
2742
2743ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
2744    std::unique_ptr<llm_graph_context> llm = build_arch_graph(params);
2745
2746    // add on pooling layer
2747    llm->build_pooling(cls, cls_b, cls_out, cls_out_b, cls_norm);
2748
2749    // add backend sampling layers (if any)
2750    llm->build_sampling();
2751
2752    // if the gguf model was converted with --sentence-transformers-dense-modules
2753    // there will be two additional dense projection layers
2754    // dense linear projections are applied after pooling
2755    // TODO: move reranking logic here and generalize
2756    llm->build_dense_out(dense_2_out_layers, dense_2_out_layers_b, dense_3_out_layers);
2757
2758    llm->res->set_outputs(params);
2759
2760    return llm->res->get_gf();
2761}
2762
2763
2764//
2765// interface implementation
2766//
2767
2768llama_model_params llama_model_default_params() {
2769    llama_model_params result = {
2770        /*.devices                     =*/ nullptr,
2771        /*.tensor_buft_overrides       =*/ nullptr,
2772        /*.n_gpu_layers                =*/ -1,
2773        /*.split_mode                  =*/ LLAMA_SPLIT_MODE_LAYER,
2774        /*.load_mode                   =*/ LLAMA_LOAD_MODE_AUTO,
2775        /*.lazy_mode                   =*/ LLAMA_LAZY_MODE_AUTO,
2776        /*.main_gpu                    =*/ 0,
2777        /*.tensor_split                =*/ nullptr,
2778        /*.progress_callback           =*/ nullptr,
2779        /*.progress_callback_user_data =*/ nullptr,
2780        /*.kv_overrides                =*/ nullptr,
2781        /*.vocab_only                  =*/ false,
2782        /*.check_tensors               =*/ false,
2783        /*.use_extra_bufts             =*/ true,
2784        /*.no_host                     =*/ false,
2785        /*.no_alloc                    =*/ false,
2786        /*.load_mtp                    =*/ false,
2787    };
2788
2789    return result;
2790}
2791
2792const llama_vocab * llama_model_get_vocab(const llama_model * model) {
2793    return &model->vocab;
2794}
2795
2796void llama_free_model(llama_model * model) {
2797    llama_model_free(model);
2798}
2799
2800void llama_model_free(llama_model * model) {
2801    delete model;
2802}
2803
2804int32_t llama_model_n_ctx_train(const llama_model * model) {
2805    return model->hparams.n_ctx_train;
2806}
2807
2808int32_t llama_model_n_embd(const llama_model * model) {
2809    return model->hparams.n_embd;
2810}
2811
2812int32_t llama_model_n_embd_inp(const llama_model * model) {
2813    return model->hparams.n_embd_inp();
2814}
2815
2816int32_t llama_model_n_embd_out(const llama_model * model) {
2817    return model->hparams.n_embd_out();
2818}
2819
2820int32_t llama_model_n_layer(const llama_model * model) {
2821    return model->hparams.n_layer();
2822}
2823
2824int32_t llama_model_n_layer_nextn(const llama_model * model) {
2825    return model->hparams.n_layer_nextn;
2826}
2827
2828int32_t llama_model_dflash_selector_top_k(const llama_model * model) {
2829    return model->hparams.dflash_selector_top_k;
2830}
2831
2832int32_t llama_model_n_head(const llama_model * model) {
2833    return model->hparams.n_head();
2834}
2835
2836int32_t llama_model_n_head_kv(const llama_model * model) {
2837    return model->hparams.n_head_kv();
2838}
2839
2840int32_t llama_model_n_swa(const llama_model * model) {
2841    // dsv4 kv-cache has SWA but it cannot be used as a rollback because of
2842    // other compression ratios, so we return 0 here
2843    if (model->arch == LLM_ARCH_DEEPSEEK4) {
2844        return 0;
2845    }
2846    return model->hparams.n_swa;
2847}
2848
2849
2850uint32_t llama_model_n_cls_out(const struct llama_model * model) {
2851    return model->hparams.n_cls_out;
2852}
2853
2854const char * llama_model_cls_label(const struct llama_model * model, uint32_t i) {
2855    if (i < model->classifier_labels.size()) {
2856        return model->classifier_labels[i].c_str();
2857    }
2858
2859    return nullptr;
2860}
2861
2862// deprecated
2863int32_t llama_n_ctx_train(const llama_model * model) {
2864    return llama_model_n_ctx_train(model);
2865}
2866
2867// deprecated
2868int32_t llama_n_embd(const llama_model * model) {
2869    return llama_model_n_embd(model);
2870}
2871
2872// deprecated
2873int32_t llama_n_layer(const llama_model * model) {
2874    return llama_model_n_layer(model);
2875}
2876
2877// deprecated
2878int32_t llama_n_head(const llama_model * model) {
2879    return llama_model_n_head(model);
2880}
2881
2882llama_rope_type llama_model_rope_type(const llama_model * model) {
2883    switch (model->arch) {
2884        // these models do not use RoPE
2885        case LLM_ARCH_CLIP:
2886        case LLM_ARCH_GPT2:
2887        case LLM_ARCH_GPTJ:
2888        case LLM_ARCH_MPT:
2889        case LLM_ARCH_REFACT:
2890        case LLM_ARCH_BLOOM:
2891        case LLM_ARCH_MAMBA:
2892        case LLM_ARCH_MAMBA2:
2893        case LLM_ARCH_JAMBA:
2894        case LLM_ARCH_JINA_BERT_V2:
2895        case LLM_ARCH_T5:
2896        case LLM_ARCH_T5ENCODER:
2897        case LLM_ARCH_JAIS:
2898        case LLM_ARCH_RWKV6:
2899        case LLM_ARCH_RWKV6QWEN2:
2900        case LLM_ARCH_RWKV7:
2901        case LLM_ARCH_ARWKV7:
2902        case LLM_ARCH_WAVTOKENIZER_DEC:
2903        case LLM_ARCH_NEMOTRON_H:
2904        case LLM_ARCH_NEMOTRON_H_MOE:
2905        case LLM_ARCH_KIMI_LINEAR:
2906        case LLM_ARCH_KIMI_K3:
2907            return LLAMA_ROPE_TYPE_NONE;
2908
2909        // use what we call a normal RoPE, operating on pairs of consecutive head values
2910        case LLM_ARCH_LLAMA:
2911        case LLM_ARCH_LLADA:
2912        case LLM_ARCH_LLAMA4:
2913        case LLM_ARCH_DECI:
2914        case LLM_ARCH_BAICHUAN:
2915        case LLM_ARCH_STARCODER:
2916        case LLM_ARCH_INTERNLM2:
2917        case LLM_ARCH_MINICPM:
2918        case LLM_ARCH_XVERSE:
2919        case LLM_ARCH_COMMAND_R:
2920        case LLM_ARCH_COHERE2:
2921        case LLM_ARCH_COHERE2MOE:
2922        case LLM_ARCH_OLMO:
2923        case LLM_ARCH_ARCTIC:
2924        case LLM_ARCH_DEEPSEEK:
2925        case LLM_ARCH_DEEPSEEK2:
2926        case LLM_ARCH_DEEPSEEK2OCR:
2927        case LLM_ARCH_DEEPSEEK32:
2928        case LLM_ARCH_DEEPSEEK4:
2929        case LLM_ARCH_MUSE_GLIMMER:
2930        case LLM_ARCH_PLM:
2931        case LLM_ARCH_CHATGLM:
2932        case LLM_ARCH_GRANITE:
2933        case LLM_ARCH_GRANITE_MOE:
2934        case LLM_ARCH_GRANITE_HYBRID:
2935        case LLM_ARCH_GRANITE_SWITCH:
2936        case LLM_ARCH_GRANITE_SWA:
2937        case LLM_ARCH_CHAMELEON:
2938        case LLM_ARCH_BAILINGMOE:
2939        case LLM_ARCH_BAILINGMOE3:
2940        case LLM_ARCH_NEO_BERT:
2941        case LLM_ARCH_SMOLLM3:
2942        case LLM_ARCH_ARCEE:
2943        case LLM_ARCH_ERNIE4_5:
2944        case LLM_ARCH_ERNIE4_5_MOE:
2945        case LLM_ARCH_MISTRAL3:
2946        case LLM_ARCH_EAGLE3:
2947        case LLM_ARCH_MISTRAL4:
2948        case LLM_ARCH_LLAMA_EMBED:
2949        case LLM_ARCH_MAINCODER:
2950        case LLM_ARCH_GLM_DSA:
2951        case LLM_ARCH_DOTS3NOTE:
2952        case LLM_ARCH_NANBEIGE:
2953        case LLM_ARCH_POCKETTTS:
2954        // HY_V4 rotates consecutive pairs, matching the reference implementation
2955        case LLM_ARCH_HY_V4:
2956            return LLAMA_ROPE_TYPE_NORM;
2957
2958        // the pairs of head values are offset by n_rot/2
2959        case LLM_ARCH_FALCON:
2960        case LLM_ARCH_FALCON_H1:
2961        case LLM_ARCH_GROK:
2962        case LLM_ARCH_DBRX:
2963        case LLM_ARCH_BERT:
2964        case LLM_ARCH_JINA_BERT_V3:
2965        case LLM_ARCH_MODERN_BERT:
2966        case LLM_ARCH_NOMIC_BERT:
2967        case LLM_ARCH_NOMIC_BERT_MOE:
2968        case LLM_ARCH_EUROBERT:
2969        case LLM_ARCH_STABLELM:
2970        case LLM_ARCH_BITNET:
2971        case LLM_ARCH_QWEN:
2972        case LLM_ARCH_QWEN2:
2973        case LLM_ARCH_DREAM:
2974        case LLM_ARCH_QWEN2MOE:
2975        case LLM_ARCH_QWEN3:
2976        case LLM_ARCH_QWEN3MOE:
2977        case LLM_ARCH_LLADA_MOE:
2978        case LLM_ARCH_RND1:
2979        case LLM_ARCH_OLMO2:
2980        case LLM_ARCH_OLMOE:
2981        case LLM_ARCH_PHI2:
2982        case LLM_ARCH_PHI3:
2983        case LLM_ARCH_PHIMOE:
2984        case LLM_ARCH_PLAMO:
2985        case LLM_ARCH_PLAMO2:
2986        case LLM_ARCH_PLAMO3:
2987        case LLM_ARCH_GEMMA:
2988        case LLM_ARCH_GEMMA2:
2989        case LLM_ARCH_GEMMA3:
2990        case LLM_ARCH_GEMMA3N:
2991        case LLM_ARCH_GEMMA4:
2992        case LLM_ARCH_GEMMA4_ASSISTANT:
2993        case LLM_ARCH_GEMMA_EMBEDDING:
2994        case LLM_ARCH_STARCODER2:
2995        case LLM_ARCH_OPENELM:
2996        case LLM_ARCH_GPTNEOX:
2997        case LLM_ARCH_CODESHELL:
2998        case LLM_ARCH_ORION:
2999        case LLM_ARCH_NEMOTRON:
3000        case LLM_ARCH_EXAONE:
3001        case LLM_ARCH_EXAONE4:
3002        case LLM_ARCH_EXAONE_MOE:
3003        case LLM_ARCH_MINICPM3:
3004        case LLM_ARCH_BAILINGMOE2:
3005        case LLM_ARCH_DOTS1:
3006        case LLM_ARCH_HUNYUAN_MOE:
3007        case LLM_ARCH_JAIS2:
3008        case LLM_ARCH_OPENAI_MOE:
3009        case LLM_ARCH_HUNYUAN_DENSE:
3010        case LLM_ARCH_HY_V3:
3011        case LLM_ARCH_LFM2:
3012        case LLM_ARCH_LFM2MOE:
3013        case LLM_ARCH_SMALLTHINKER:
3014        case LLM_ARCH_SEED_OSS:
3015        case LLM_ARCH_GROVEMOE:
3016        case LLM_ARCH_APERTUS:
3017        case LLM_ARCH_MINIMAX_01:
3018        case LLM_ARCH_MINIMAX_M2:
3019        case LLM_ARCH_MINIMAX_M3:
3020        case LLM_ARCH_COGVLM:
3021        case LLM_ARCH_PANGU_EMBED:
3022        case LLM_ARCH_AFMOE:
3023        case LLM_ARCH_LAGUNA:
3024        case LLM_ARCH_QWEN3NEXT:
3025        case LLM_ARCH_MIMO2:
3026        case LLM_ARCH_STEP35:
3027        case LLM_ARCH_SPARK2_5:
3028        case LLM_ARCH_TALKIE:
3029        case LLM_ARCH_MELLUM:
3030        case LLM_ARCH_MAPLE:
3031        case LLM_ARCH_ALICEAI_T5_MOE:
3032            return LLAMA_ROPE_TYPE_NEOX;
3033
3034        case LLM_ARCH_DFLASH:
3035            // drafts for M-RoPE targets carry rope sections and follow the target's temporal dim
3036            if (const auto & s = model->hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) {
3037                return LLAMA_ROPE_TYPE_MROPE;
3038            }
3039            // DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX
3040            return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX;
3041
3042        case LLM_ARCH_QWEN2VL:
3043        case LLM_ARCH_PADDLEOCR:
3044            return LLAMA_ROPE_TYPE_MROPE;
3045        case LLM_ARCH_QWEN3VL:
3046        case LLM_ARCH_QWEN3VLMOE:
3047        case LLM_ARCH_QWEN35:
3048        case LLM_ARCH_QWEN35MOE:
3049        case LLM_ARCH_QWEN4EXP:
3050        case LLM_ARCH_QWEN3TTS:
3051            return LLAMA_ROPE_TYPE_IMROPE;
3052
3053        case LLM_ARCH_GLM4:
3054            return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM;
3055        case LLM_ARCH_GLM4_MOE:
3056            return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NEOX;
3057
3058        case LLM_ARCH_HUNYUAN_VL:
3059            return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NEOX;
3060
3061        // all model arches should be listed explicitly here
3062        case LLM_ARCH_UNKNOWN:
3063            GGML_ABORT("unknown architecture");
3064    }
3065
3066    return LLAMA_ROPE_TYPE_NONE;
3067}
3068
3069float llama_model_rope_freq_scale_train(const llama_model * model) {
3070    return model->hparams.rope_freq_scale_train;
3071}
3072
3073int32_t llama_model_meta_val_str(const llama_model * model, const char * key, char * buf, size_t buf_size) {
3074    const auto & it = model->gguf_kv.find(key);
3075    if (it == model->gguf_kv.end()) {
3076        if (buf_size > 0) {
3077            buf[0] = '\0';
3078        }
3079        return -1;
3080    }
3081    return snprintf(buf, buf_size, "%s", it->second.c_str());
3082}
3083
3084int32_t llama_model_meta_count(const llama_model * model) {
3085    return (int)model->gguf_kv.size();
3086}
3087
3088const char * llama_model_meta_key_str(llama_model_meta_key key) {
3089    switch (key) {
3090        case LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE:        return "general.sampling.sequence";
3091        case LLAMA_MODEL_META_KEY_SAMPLING_TOP_K:           return "general.sampling.top_k";
3092        case LLAMA_MODEL_META_KEY_SAMPLING_TOP_P:           return "general.sampling.top_p";
3093        case LLAMA_MODEL_META_KEY_SAMPLING_MIN_P:           return "general.sampling.min_p";
3094        case LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY: return "general.sampling.xtc_probability";
3095        case LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD:   return "general.sampling.xtc_threshold";
3096        case LLAMA_MODEL_META_KEY_SAMPLING_TEMP:            return "general.sampling.temp";
3097        case LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N:  return "general.sampling.penalty_last_n";
3098        case LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT:  return "general.sampling.penalty_repeat";
3099        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT:        return "general.sampling.mirostat";
3100        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU:    return "general.sampling.mirostat_tau";
3101        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA:    return "general.sampling.mirostat_eta";
3102        default:                                            return nullptr;
3103    }
3104}
3105
3106int32_t llama_model_meta_key_by_index(const llama_model * model, int i, char * buf, size_t buf_size) {
3107    if (i < 0 || i >= (int)model->gguf_kv.size()) {
3108        if (buf_size > 0) {
3109            buf[0] = '\0';
3110        }
3111        return -1;
3112    }
3113    auto it = model->gguf_kv.begin();
3114    std::advance(it, i);
3115    return snprintf(buf, buf_size, "%s", it->first.c_str());
3116}
3117
3118int32_t llama_model_meta_val_str_by_index(const llama_model * model, int32_t i, char * buf, size_t buf_size) {
3119    if (i < 0 || i >= (int)model->gguf_kv.size()) {
3120        if (buf_size > 0) {
3121            buf[0] = '\0';
3122        }
3123        return -1;
3124    }
3125    auto it = model->gguf_kv.begin();
3126    std::advance(it, i);
3127    return snprintf(buf, buf_size, "%s", it->second.c_str());
3128}
3129
3130int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) {
3131    return snprintf(buf, buf_size, "%s", model->desc().c_str());
3132}
3133
3134llama_ftype llama_model_ftype(const llama_model * model) {
3135    return model->ftype();
3136}
3137
3138uint64_t llama_model_size(const llama_model * model) {
3139    return model->size();
3140}
3141
3142const char * llama_model_chat_template(const llama_model * model, const char * name) {
3143    const auto key = name ? LLM_KV(model->arch, name)(LLM_KV_TOKENIZER_CHAT_TEMPLATE)
3144        : LLM_KV(model->arch)(LLM_KV_TOKENIZER_CHAT_TEMPLATE);
3145    const auto & it = model->gguf_kv.find(key);
3146    if (it == model->gguf_kv.end()) {
3147        // one-off fix for very popular models (so we are not flooded with issues)
3148        // do not extend this list unless absolutely necessary
3149        // Mistral-Small-2503 does not have built-in chat template
3150        llama_vocab_pre_type pre_type = model->vocab.get_pre_type();
3151        if (!name && pre_type == LLAMA_VOCAB_PRE_TYPE_TEKKEN && model->layers.size() == 40) {
3152            return "mistral-v7-tekken";
3153        }
3154
3155        return nullptr;
3156    }
3157
3158    return it->second.c_str();
3159}
3160
3161uint64_t llama_model_n_params(const llama_model * model) {
3162    return model->n_elements();
3163}
3164
3165bool llama_model_has_encoder(const llama_model * model) {
3166    switch (model->arch) {
3167        case LLM_ARCH_ALICEAI_T5_MOE:
3168        case LLM_ARCH_T5:
3169        case LLM_ARCH_T5ENCODER:
3170        case LLM_ARCH_EAGLE3:
3171        case LLM_ARCH_DFLASH:    return true;
3172        default:                 return false;
3173    }
3174}
3175
3176bool llama_model_has_decoder(const llama_model * model) {
3177    switch (model->arch) {
3178        case LLM_ARCH_T5ENCODER: return false;
3179        default:                 return true;
3180    }
3181}
3182
3183llama_token llama_model_decoder_start_token(const llama_model * model) {
3184    return model->hparams.dec_start_token_id;
3185}
3186
3187bool llama_model_is_recurrent(const llama_model * model) {
3188    return llm_arch_is_recurrent(model->arch);
3189}
3190
3191bool llama_model_is_hybrid(const llama_model * model) {
3192    return llm_arch_is_hybrid(model->arch);
3193}
3194
3195bool llama_model_is_diffusion(const llama_model * model) {
3196    return llm_arch_is_diffusion(model->arch);
3197}
3198
3199const std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model) {
3200    return model->tensors_by_name;
3201}
3202
3203int32_t llama_model_n_expert(const struct llama_model * model) {
3204    return model->hparams.n_expert;
3205}
3206
3207int32_t llama_model_n_devices(const struct llama_model * model) {
3208    return (int32_t)model->devices.size();
3209}
3210
3211ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i) {
3212    if (i < 0 || i >= (int)model->devices.size()) {
3213        return nullptr;
3214    }
3215    return model->devices[i].dev;
3216}
3217
3218//
3219// llama_model_base
3220//
3221
3222llama_model_base::llama_model_base(const struct llama_model_params & params) : llama_model(params), model(this), tn(model->arch),
3223    TENSOR_DUPLICATED     (llama_model_loader::TENSOR_DUPLICATED),
3224    TENSOR_NOT_REQUIRED   (llama_model_loader::TENSOR_NOT_REQUIRED),
3225    TENSOR_SKIP           (llama_model_loader::TENSOR_SKIP),
3226    TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL),
3227    TENSOR_ALLOW_RESHAPE  (llama_model_loader::TENSOR_ALLOW_RESHAPE),
3228    TENSOR_READ_LAZY      (llama_model_loader::TENSOR_READ_LAZY) {}
3229
3230ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
3231    GGML_ASSERT(ml != nullptr);
3232    return create_tensor(*ml, tn, ne, flags);
3233}
3234
3235void llama_model_base::create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) {
3236    layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED);
3237    if (layer.ffn_gate_up_exps == nullptr) {
3238        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, flags);
3239        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", bid), {n_embd_, n_ff_, n_expert_}, flags);
3240    }
3241}
3242
3243void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
3244        int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
3245        int flags) {
3246    const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
3247
3248    if (flags & TENSOR_SKIP) {
3249        const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
3250
3251        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
3252        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias",   bid), {n_embd_qkv},          skip | TENSOR_SKIP_IF_VIRTUAL);
3253        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "weight", bid), {n_embd_, n_embd_q_},  skip);
3254        create_tensor(tn(LLM_TENSOR_ATTN_K,   "weight", bid), {n_embd_, n_embd_k_},  skip);
3255        create_tensor(tn(LLM_TENSOR_ATTN_V,   "weight", bid), {n_embd_, n_embd_v_},  skip);
3256        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "bias",   bid), {n_embd_q_},           skip);
3257        create_tensor(tn(LLM_TENSOR_ATTN_K,   "bias",   bid), {n_embd_k_},           skip);
3258        create_tensor(tn(LLM_TENSOR_ATTN_V,   "bias",   bid), {n_embd_v_},           skip);
3259        return;
3260    }
3261
3262    layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
3263    if (layer.wqkv) {
3264        layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
3265        // Fused weights may coexist with separate Q/K/V biases in legacy or custom GGUFs.
3266        if (!layer.wqkv_b) {
3267            layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, TENSOR_NOT_REQUIRED);
3268            layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, TENSOR_NOT_REQUIRED);
3269            layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, TENSOR_NOT_REQUIRED);
3270        }
3271    } else {
3272        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, flags);
3273        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, flags);
3274        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, flags);
3275        layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, TENSOR_NOT_REQUIRED);
3276        layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, TENSOR_NOT_REQUIRED);
3277        layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, TENSOR_NOT_REQUIRED);
3278    }
3279}
3280
3281const int32_t * llama_model_target_layer_ids(const struct llama_model * model) {
3282    const auto & v = model->target_layer_ids;
3283    return v.empty() ? nullptr : v.data();
3284}
3285
3286uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) {
3287    return (uint32_t) model->target_layer_ids.size();
3288}
3289
3290uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) {
3291    if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) {
3292        return 0;
3293    }
3294
3295    const ggml_tensor * tensor = model->tok_embd;
3296    const size_t nelements = ggml_nelements(tensor);
3297    GGML_ASSERT(nelements <= UINT32_MAX); // for the return type
3298
3299    if (out == nullptr) {
3300        return (uint32_t) nelements;
3301    }
3302
3303    if (tensor->type == GGML_TYPE_F32) {
3304        ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float));
3305        return (uint32_t) nelements;
3306    }
3307
3308    std::vector<uint8_t> buf(ggml_nbytes(tensor));
3309    ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());
3310
3311    const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
3312    if (tensor->type == GGML_TYPE_F16) {
3313        ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
3314    } else if (tensor->type == GGML_TYPE_BF16) {
3315        ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
3316    } else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
3317        traits->to_float(buf.data(), out, nelements);
3318    } else {
3319        GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
3320    }
3321
3322    return (uint32_t) nelements;
3323}