char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)7798007a2

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
44.0 KiB1083 linesraw
1#pragma once
2
3#include "ggml.h"
4#include "gguf.h"
5#include "clip.h"
6
7#include <array>
8#include <climits>
9#include <cmath>
10#include <cstdarg>
11#include <cinttypes>
12#include <string>
13#include <map>
14#include <sstream>
15#include <vector>
16#include <memory>
17#include <fstream>
18
19#ifdef _WIN32
20#ifndef NOMINMAX
21#define NOMINMAX
22#endif
23#include <windows.h>
24#endif
25
26// Internal header for clip.cpp
27
28#define MTMD_INTERNAL_HEADER
29
30#define KEY_FTYPE               "general.file_type"
31#define KEY_NAME                "general.name"
32#define KEY_DESCRIPTION         "general.description"
33#define KEY_PROJ_TYPE           "clip.projector_type"
34#define KEY_HAS_AUDIO_ENC       "clip.has_audio_encoder"
35#define KEY_HAS_VISION_ENC      "clip.has_vision_encoder"
36#define KEY_HAS_GEN_AUDIO_ENC   "clip.has_gen_audio_encoder"
37#define KEY_USE_GELU            "clip.use_gelu"
38#define KEY_USE_SILU            "clip.use_silu"
39
40#define KEY_N_EMBD              "clip.%s.embedding_length"
41#define KEY_N_FF                "clip.%s.feed_forward_length"
42#define KEY_N_BLOCK             "clip.%s.block_count"
43#define KEY_PROJ_DIM            "clip.%s.projection_dim"
44#define KEY_N_HEAD              "clip.%s.attention.head_count"
45#define KEY_N_HEAD_KV           "clip.%s.attention.head_count_kv"
46#define KEY_N_EMBD_HEAD         "clip.%s.attention.head_dim"
47#define KEY_LAYER_NORM_EPS      "clip.%s.attention.layer_norm_epsilon"
48#define KEY_FEATURE_LAYERS      "clip.%s.feature_layer"
49
50// vision-specific
51#define KEY_VISION_PROJ_TYPE        "clip.vision.projector_type" // for models with mixed modalities
52#define KEY_IMAGE_SIZE              "clip.vision.image_size"
53#define KEY_IMAGE_MIN_PIXELS        "clip.vision.image_min_pixels"
54#define KEY_IMAGE_MAX_PIXELS        "clip.vision.image_max_pixels"
55#define KEY_PREPROC_MIN_TILES       "clip.vision.preproc_min_tiles"
56#define KEY_PREPROC_MAX_TILES       "clip.vision.preproc_max_tiles"
57#define KEY_PREPROC_IMAGE_SIZE      "clip.vision.preproc_image_size"
58#define KEY_PATCH_SIZE              "clip.vision.patch_size"
59#define KEY_IMAGE_MEAN              "clip.vision.image_mean"
60#define KEY_IMAGE_STD               "clip.vision.image_std"
61#define KEY_PROJ_SCALE_FACTOR       "clip.vision.projector.scale_factor"
62#define KEY_PROJ_SAMPLE_QUERY_SIDE  "clip.vision.projector.query_side"
63#define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side"
64#define KEY_PROJ_SPATIAL_OFFSETS    "clip.vision.projector.spatial_offsets"
65#define KEY_SPATIAL_MERGE_SIZE      "clip.vision.spatial_merge_size"
66
67#define KEY_MM_PATCH_MERGE_TYPE    "clip.vision.mm_patch_merge_type"
68#define KEY_IMAGE_GRID_PINPOINTS   "clip.vision.image_grid_pinpoints"
69#define KEY_WIN_ATTN_PATTERN       "clip.vision.n_wa_pattern"
70#define KEY_WIN_ATTN_LAYER_INDEXES "clip.vision.wa_layer_indexes"
71#define KEY_WA_PATTERN_MODE        "clip.vision.wa_pattern_mode"
72#define KEY_ATTN_WINDOW_SIZE       "clip.vision.window_size"
73#define KEY_MINICPMV_VERSION       "clip.minicpmv_version"
74#define KEY_MINICPMV_QUERY_NUM     "clip.minicpmv_query_num"
75#define KEY_SAM_N_HEAD             "clip.vision.sam.head_count"
76#define KEY_SAM_N_BLOCK            "clip.vision.sam.block_count"
77#define KEY_SAM_N_EMBD             "clip.vision.sam.embedding_length"
78#define KEY_VISION_N_EXPERT_USED   "clip.vision.expert_used_count"
79// audio-specific
80#define KEY_AUDIO_PROJ_TYPE        "clip.audio.projector_type" // for models with mixed modalities
81#define KEY_A_NUM_MEL_BINS         "clip.audio.num_mel_bins"
82#define KEY_A_PROJ_STACK_FACTOR    "clip.audio.projector.stack_factor"
83#define KEY_A_CHUNK_SIZE           "clip.audio.chunk_size"
84#define KEY_A_CONV_KERNEL_SIZE     "clip.audio.conv_kernel_size"
85#define KEY_A_MAX_POS_EMB          "clip.audio.max_pos_emb"
86#define KEY_A_PROJ_WINDOW_SIZE     "clip.audio.projector.window_size"
87#define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate"
88#define KEY_A_PROJ_HEAD_COUNT      "clip.audio.projector.head_count"
89#define KEY_A_RVQ_NUM_QUANTIZERS   "clip.audio.rvq.num_quantizers"   // mimo-audio-tokenizer
90#define KEY_A_RVQ_CODEBOOK_SIZE    "clip.audio.rvq.codebook_size"    // mimo-audio-tokenizer: per-quantizer bin count
91#define KEY_A_WA_PATTERN_MODE      "clip.audio.wa_pattern_mode"      // mimo-audio-tokenizer, per-layer -1 (full) / 0 (windowed)
92#define KEY_A_ATTN_WINDOW_SIZE     "clip.audio.window_size"          // mimo-audio-tokenizer: sliding-window radius
93#define KEY_A_LOCAL_BLOCK_COUNT    "clip.audio.local_block_count"    // mimo-v2.5: input_local_transformer layer count
94#define KEY_A_LOCAL_GROUP_SIZE     "clip.audio.local_group_size"     // mimo-v2.5: input_local_transformer grouping size
95// audio generation (gen-audio)-specific
96#define KEY_GEN_AUDIO_PROJ_TYPE    "clip.gen.audio.projector_type" // for models with mixed modalities
97// name of the weight variant, for settings that are not in the checkpoint
98#define KEY_GEN_AUDIO_VARIANT      "clip.gen.audio.model_variant"
99#define KEY_AUDIO_SUBSMPL_FACTOR   "clip.audio.subsampling_factor"
100
101//
102// tensor name constants
103//
104
105#define TN_POS_EMBD        "%s.position_embd.weight"
106#define TN_CLASS_EMBD      "v.class_embd"
107#define TN_PATCH_EMBD      "v.patch_embd.weight"  // not rename tensor with ".0" postfix for backward compat
108#define TN_PATCH_EMBD_1    "v.patch_embd.weight.1"
109#define TN_PATCH_BIAS      "v.patch_embd.bias"
110#define TN_NORM_EMBD       "v.norm_embd.%s"
111#define TN_PATCH_NORM      "v.patch_norm.%d.%s"
112#define TN_ATTN_QKV        "%s.blk.%d.attn_qkv.%s"
113#define TN_ATTN_K          "%s.blk.%d.attn_k.%s"
114#define TN_ATTN_Q          "%s.blk.%d.attn_q.%s"
115#define TN_ATTN_V          "%s.blk.%d.attn_v.%s"
116#define TN_ATTN_OUTPUT     "%s.blk.%d.attn_out.%s"
117#define TN_ATTN_SINKS      "%s.blk.%d.attn_sinks"
118#define TN_ATTN_K_NORM     "%s.blk.%d.attn_k_norm.%s"
119#define TN_ATTN_Q_NORM     "%s.blk.%d.attn_q_norm.%s"
120#define TN_FFN_DOWN        "%s.blk.%d.ffn_down.%s"
121#define TN_FFN_GATE        "%s.blk.%d.ffn_gate.%s"
122#define TN_FFN_UP          "%s.blk.%d.ffn_up.%s"
123#define TN_FFN_GATE_INP    "%s.blk.%d.ffn_gate_inp.%s"    // MoE router (dots3note)
124#define TN_FFN_GATE_EXPS   "%s.blk.%d.ffn_gate_exps.%s"
125#define TN_FFN_UP_EXPS     "%s.blk.%d.ffn_up_exps.%s"
126#define TN_FFN_DOWN_EXPS   "%s.blk.%d.ffn_down_exps.%s"
127#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s"
128#define TN_LN_1            "%s.blk.%d.ln1.%s" // layer norm
129#define TN_LN_2            "%s.blk.%d.ln2.%s" // layer norm
130#define TN_LS_1            "%s.blk.%d.ls1.%s"         // layer scale
131#define TN_LS_2            "%s.blk.%d.ls2.%s"         // layer scale
132#define TN_LS_OUT          "%s.blk.%d.out_scale.%s"      // layer out scale (gemma4)
133#define TN_ATTN_POST_NORM  "%s.blk.%d.attn_post_norm.%s" // post-attn norm (gemma4)
134#define TN_FFN_POST_NORM   "%s.blk.%d.ffn_post_norm.%s"  // post-FFN norm (gemma4)
135#define TN_LN_PRE          "%s.pre_ln.%s"
136#define TN_LN_POST         "%s.post_ln.%s"
137#define TN_LLAVA_PROJ      "mm.%d.%s"
138#define TN_MM_UP           "mm.up.%s"
139#define TN_MM_GATE         "mm.gate.%s"
140#define TN_MM_DOWN         "mm.down.%s"
141#define TN_MM_POST_NORM    "mm.post_norm.%s"
142#define TN_MVLM_PROJ_MLP   "mm.model.mlp.%d.%s"
143#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
144#define TN_MVLM_PROJ_PEG   "mm.model.peg.%d.%s"
145#define TN_IMAGE_NEWLINE   "v.image_newline"
146#define TN_IMAGE_SEPERATOR "v.view_seperator"
147#define TN_MM_INP_NORM     "mm.input_norm.weight"
148#define TN_MM_INP_NORM_B   "mm.input_norm.bias"
149#define TN_MM_INP_PROJ     "mm.input_projection.weight" // gemma3
150#define TN_MM_SOFT_EMB_N   "mm.soft_emb_norm.weight"    // gemma3
151#define TN_MM_PROJECTOR    "mm.model.fc.%s"             // idefics3, deepseekocr
152#define TN_MM_PATCH_MERGER "mm.patch_merger.%s"         // mistral small 3.1, glm4v
153#define TN_MM_MERGER_FC1   "mm.merger.fc1.%s"            // minimax-m3 patch-merge MLP
154#define TN_MM_MERGER_FC2   "mm.merger.fc2.%s"
155#define TN_TOK_IMG_BREAK   "v.token_embd.img_break"     // pixtral
156#define TN_TOK_IMG_START   "v.token_embd.img_start"     // deepseek4v
157#define TN_TOK_IMG_END     "v.token_embd.img_end"       // deepseek4v
158#define TN_TOK_IMG_PAD     "v.token_embd.img_pad"       // deepseek4v
159#define TN_TOK_GLM_BOI     "adapter.boi"                // glm-edge (these embeddings are not in text model)
160#define TN_TOK_GLM_EOI     "adapter.eoi"                // glm-edge (these embeddings are not in text model)
161#define TN_DEEPSTACK_NORM  "v.deepstack.%d.norm.%s"     // qwen3vl deepstack
162#define TN_DEEPSTACK_FC1   "v.deepstack.%d.fc1.%s"      // qwen3vl deepstack
163#define TN_DEEPSTACK_FC2   "v.deepstack.%d.fc2.%s"      // qwen3vl deepstack
164
165// mimicpmv
166#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
167#define TN_MINICPMV_QUERY      "resampler.query"
168#define TN_MINICPMV_PROJ       "resampler.proj.weight"
169#define TN_MINICPMV_KV_PROJ    "resampler.kv.weight"
170#define TN_MINICPMV_ATTN       "resampler.attn.%s.%s"
171#define TN_MINICPMV_LN         "resampler.ln_%s.%s"
172
173// MiniCPM-V 4.6 ViT merger (window attention + MLP downsample),
174// matching the upstream `vit_merger` module name in transformers.
175#define TN_VIT_MERGER_LN1      "v.vit_merger.ln1.%s"
176#define TN_VIT_MERGER_ATTN_Q   "v.vit_merger.attn_q.%s"
177#define TN_VIT_MERGER_ATTN_K   "v.vit_merger.attn_k.%s"
178#define TN_VIT_MERGER_ATTN_V   "v.vit_merger.attn_v.%s"
179#define TN_VIT_MERGER_ATTN_O   "v.vit_merger.attn_out.%s"
180#define TN_VIT_MERGER_DS_LN    "v.vit_merger.ds_ln.%s"
181#define TN_VIT_MERGER_DS_UP    "v.vit_merger.ds_ffn_up.%s"
182#define TN_VIT_MERGER_DS_DOWN  "v.vit_merger.ds_ffn_down.%s"
183
184#define TN_GLM_ADAPER_CONV      "adapter.conv.%s"
185#define TN_GLM_ADAPTER_LINEAR   "adapter.linear.linear.%s"
186#define TN_GLM_ADAPTER_NORM_1   "adapter.linear.norm1.%s"
187#define TN_GLM_ADAPTER_D_H_2_4H "adapter.linear.dense_h_to_4h.%s"
188#define TN_GLM_ADAPTER_GATE     "adapter.linear.gate.%s"
189#define TN_GLM_ADAPTER_D_4H_2_H "adapter.linear.dense_4h_to_h.%s"
190
191// ultravox
192#define TN_CONV1D       "a.conv1d.%d.%s"
193#define TN_CONV2D       "a.conv2d.%d.%s"
194#define TN_CONV_OUT     "a.conv_out.%s"
195#define TN_MM_AUDIO_MLP "mm.a.mlp.%d.%s"
196#define TN_MM_AUDIO_FC  "mm.a.fc.%s" // fully connected layer
197#define TN_MM_NORM_PRE  "mm.a.norm_pre.%s"
198#define TN_MM_NORM_MID  "mm.a.norm_mid.%s"
199
200// mimo-audio-tokenizer
201#define TN_A_DOWNSAMPLE_CONV "a.downsample.conv.%s"
202#define TN_A_DOWNSAMPLE_NORM "a.downsample.norm.%s"
203#define TN_A_RVQ_CODEBOOK    "a.rvq.codebook.%s"
204// mimo-v2.5: text-side RVQ code embedding ("text codebook")
205#define TN_MM_A_CODE_EMBD    "mm.a.code_embd.%s"
206// mimo-v2.5: LLM-side connector (input_local_transformer)
207#define TN_MM_A_LOCAL_ATTN_Q   "mm.a.local_blk.%d.attn_q.%s"
208#define TN_MM_A_LOCAL_ATTN_K   "mm.a.local_blk.%d.attn_k.%s"
209#define TN_MM_A_LOCAL_ATTN_V   "mm.a.local_blk.%d.attn_v.%s"
210#define TN_MM_A_LOCAL_ATTN_OUT "mm.a.local_blk.%d.attn_out.%s"
211#define TN_MM_A_LOCAL_FFN_GATE "mm.a.local_blk.%d.ffn_gate.%s"
212#define TN_MM_A_LOCAL_FFN_UP   "mm.a.local_blk.%d.ffn_up.%s"
213#define TN_MM_A_LOCAL_FFN_DOWN "mm.a.local_blk.%d.ffn_down.%s"
214#define TN_MM_A_LOCAL_LN1      "mm.a.local_blk.%d.ln1.%s"
215#define TN_MM_A_LOCAL_LN2      "mm.a.local_blk.%d.ln2.%s"
216#define TN_MM_A_LOCAL_NORM     "mm.a.local_norm.%s"
217
218// qwen3tts speaker encoder (ECAPA-TDNN)
219#define TN_A_SE_CONV1  "a.blk.%d.se_conv1.%s"
220#define TN_A_SE_CONV2  "a.blk.%d.se_conv2.%s"
221#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s"
222#define TN_A_ASP_ATTN  "a.asp_attn.%s"
223#define TN_A_ASP_TDNN  "a.asp_tdnn.%s"
224
225// qwen3tts code_predictor
226#define TN_A_GEN_CODE_PROJ_IN  "a.gen.code.proj_in.%s"
227#define TN_A_GEN_CODE_EMBD     "a.gen.code.embd.%s"
228#define TN_A_GEN_CODE_HEAD     "a.gen.code.head.%s"
229#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s"
230#define TN_A_GEN_CODE_NORM     "a.gen.code.output_norm.%s"
231
232// qwen3tts code2wav (RVQ codes -> raw PCM)
233// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
234#define TN_A_GEN_WAV_QUANT_FIRST_IN  "a.gen.wav.quant.first.in_proj.%s"
235#define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s"
236#define TN_A_GEN_WAV_QUANT_FIRST_CB  "a.gen.wav.quant.first.codebook.%s"
237#define TN_A_GEN_WAV_QUANT_REST_IN   "a.gen.wav.quant.rest.in_proj.%s"
238#define TN_A_GEN_WAV_QUANT_REST_OUT  "a.gen.wav.quant.rest.out_proj.%s"
239#define TN_A_GEN_WAV_QUANT_REST_CB   "a.gen.wav.quant.rest.codebook.%s"
240#define TN_A_GEN_WAV_PRE_CONV        "a.gen.wav.pre_conv.%s"
241#define TN_A_GEN_WAV_TFM_IN_PROJ     "a.gen.wav.tfm.in_proj.%s"
242#define TN_A_GEN_WAV_TFM_OUT_PROJ    "a.gen.wav.tfm.out_proj.%s"
243#define TN_A_GEN_WAV_TFM_OUT_NORM    "a.gen.wav.tfm.output_norm.%s"
244#define TN_A_GEN_WAV_UP_CONV         "a.gen.wav.up.blk.%d.conv.%s"
245#define TN_A_GEN_WAV_UP_DWCONV       "a.gen.wav.up.blk.%d.dwconv.%s"
246#define TN_A_GEN_WAV_UP_NORM         "a.gen.wav.up.blk.%d.norm.%s"
247#define TN_A_GEN_WAV_UP_PW1          "a.gen.wav.up.blk.%d.pw1.%s"
248#define TN_A_GEN_WAV_UP_PW2          "a.gen.wav.up.blk.%d.pw2.%s"
249#define TN_A_GEN_WAV_UP_GAMMA        "a.gen.wav.up.blk.%d.gamma"
250#define TN_A_GEN_WAV_DAC_ENTRY       "a.gen.wav.dac.entry.%s"
251#define TN_A_GEN_WAV_DAC_SNAKE       "a.gen.wav.dac.blk.%d.snake.%s"
252#define TN_A_GEN_WAV_DAC_CONV        "a.gen.wav.dac.blk.%d.conv.%s"
253#define TN_A_GEN_WAV_DAC_RES_ACT1    "a.gen.wav.dac.blk.%d.res.%d.act1.%s"
254#define TN_A_GEN_WAV_DAC_RES_CONV1   "a.gen.wav.dac.blk.%d.res.%d.conv1.%s"
255#define TN_A_GEN_WAV_DAC_RES_ACT2    "a.gen.wav.dac.blk.%d.res.%d.act2.%s"
256#define TN_A_GEN_WAV_DAC_RES_CONV2   "a.gen.wav.dac.blk.%d.res.%d.conv2.%s"
257#define TN_A_GEN_WAV_DAC_POST_SNAKE  "a.gen.wav.dac.post_snake.%s"
258#define TN_A_GEN_WAV_DAC_POST_CONV   "a.gen.wav.dac.post_conv.%s"
259
260// pocket-tts
261#define TN_A_SEANET_CONV_IN      "a.seanet.conv_in.%s"
262#define TN_A_SEANET_CONV_OUT     "a.seanet.conv_out.%s"
263#define TN_A_SEANET_RES_CONV1    "a.seanet.blk.%d.res_conv1.%s"
264#define TN_A_SEANET_RES_CONV2    "a.seanet.blk.%d.res_conv2.%s"
265#define TN_A_SEANET_SCALE_CONV   "a.seanet.blk.%d.scale_conv.%s"
266#define TN_A_SPEAKER_PROJ        "a.speaker_proj.%s"
267#define TN_A_DOWNSAMPLE_CONV     "a.downsample.conv.%s"
268#define TN_A_GEN_FLOW_INPUT_PROJ "a.gen.flow.input_proj.%s"
269#define TN_A_GEN_FLOW_COND_EMBD  "a.gen.flow.cond_embd.%s"
270#define TN_A_GEN_FLOW_TIME_FREQS "a.gen.flow.time.%d.freqs"
271#define TN_A_GEN_FLOW_TIME_UP    "a.gen.flow.time.%d.up.%s"
272#define TN_A_GEN_FLOW_TIME_DOWN  "a.gen.flow.time.%d.down.%s"
273#define TN_A_GEN_FLOW_TIME_NORM  "a.gen.flow.time.%d.norm"
274#define TN_A_GEN_FLOW_BLK_NORM   "a.gen.flow.blk.%d.norm.%s"
275#define TN_A_GEN_FLOW_BLK_UP     "a.gen.flow.blk.%d.up.%s"
276#define TN_A_GEN_FLOW_BLK_DOWN   "a.gen.flow.blk.%d.down.%s"
277#define TN_A_GEN_FLOW_BLK_ADA    "a.gen.flow.blk.%d.ada.%s"
278#define TN_A_GEN_FLOW_FINAL_ADA  "a.gen.flow.final.ada.%s"
279#define TN_A_GEN_FLOW_FINAL_PROJ "a.gen.flow.final.proj.%s"
280#define TN_A_GEN_OUT_EOS         "a.gen.out_eos.%s"
281#define TN_A_GEN_INPUT_LINEAR    "a.gen.input_linear.%s"
282#define TN_A_GEN_EMB_MEAN        "a.gen.emb_mean"
283#define TN_A_GEN_EMB_STD         "a.gen.emb_std"
284#define TN_A_GEN_WAV_QUANT_OUT   "a.gen.wav.quant_out.%s"
285#define TN_A_GEN_WAV_UPSAMPLE    "a.gen.wav.upsample.%s"
286#define TN_A_GEN_WAV_SEANET_CONV_IN    "a.gen.wav.seanet.conv_in.%s"
287#define TN_A_GEN_WAV_SEANET_CONV_OUT   "a.gen.wav.seanet.conv_out.%s"
288#define TN_A_GEN_WAV_SEANET_RES_CONV1  "a.gen.wav.seanet.blk.%d.res_conv1.%s"
289#define TN_A_GEN_WAV_SEANET_RES_CONV2  "a.gen.wav.seanet.blk.%d.res_conv2.%s"
290#define TN_A_GEN_WAV_SEANET_SCALE_CONV "a.gen.wav.seanet.blk.%d.scale_conv.%s"
291
292// cogvlm
293#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
294#define TN_MM_H_TO_4H      "mm.up.%s"
295#define TN_MM_GATE         "mm.gate.%s"
296#define TN_MM_4H_TO_H      "mm.down.%s"
297#define TN_TOK_BOI         "v.boi"
298#define TN_TOK_EOI         "v.eoi"
299
300// hunyuanvl (shared GGUF tensor names)
301#define TN_MM_PRE_NORM     "mm.pre_norm.%s"
302#define TN_MM_IMG_BEGIN    "mm.image_begin" // note: legacy name, new models should use v.token_embd.*
303#define TN_MM_IMG_END      "mm.image_end"   // note: legacy name, new models should use v.token_embd.*
304
305// deepseek-ocr
306#define TN_SAM_POS_EMBD   "v.sam.pos_embd.%s"
307#define TN_SAM_PATCH_EMBD "v.sam.patch_embd.%s"
308#define TN_SAM_PRE_NORM   "v.sam.blk.%d.pre_ln.%s"
309#define TN_SAM_POST_NORM  "v.sam.blk.%d.post_ln.%s"
310#define TN_SAM_ATTN_POS_H "v.sam.blk.%d.attn.pos_h.%s"
311#define TN_SAM_ATTN_POS_W "v.sam.blk.%d.attn.pos_w.%s"
312#define TN_SAM_ATTN_QKV   "v.sam.blk.%d.attn.qkv.%s"
313#define TN_SAM_ATTN_OUT   "v.sam.blk.%d.attn.out.%s"
314#define TN_SAM_FFN_UP     "v.sam.blk.%d.mlp.lin1.%s"
315#define TN_SAM_FFN_DOWN   "v.sam.blk.%d.mlp.lin2.%s"
316#define TN_SAM_NECK       "v.sam.neck.%d.%s"
317#define TN_SAM_NET        "v.sam.net_%d.%s"
318// deepseek-ocr-2
319#define TN_RESMPL_QUERY  "v.resample_query_%d.%s"
320// (conformer) lfm2
321#define TN_PRE_ENCODE_OUT  "a.pre_encode.out.%s"
322#define TN_FFN_NORM        "%s.blk.%d.ffn_norm.%s"
323#define TN_FFN_NORM_1      "%s.blk.%d.ffn_norm_1.%s"
324#define TN_FFN_UP_1        "%s.blk.%d.ffn_up_1.%s"
325#define TN_FFN_DOWN_1      "%s.blk.%d.ffn_down_1.%s"
326#define TN_POS_BIAS_U      "%s.blk.%d.pos_bias_u"
327#define TN_POS_BIAS_V      "%s.blk.%d.pos_bias_v"
328#define TN_NORM_CONV       "%s.blk.%d.norm_conv.%s"
329#define TN_LINEAR_POS      "%s.blk.%d.linear_pos.%s"
330#define TN_CONV_DW         "%s.blk.%d.conv_dw.%s"
331#define TN_CONV_NORM       "%s.blk.%d.conv_norm.%s"
332#define TN_CONV_PW1        "%s.blk.%d.conv_pw1.%s"
333#define TN_CONV_PW2        "%s.blk.%d.conv_pw2.%s"
334#define TN_INP_PROJ        "a.input_projection.%s"
335#define TN_CTC_OUT         "a.enc_ctc_out.%s"
336#define TN_CTC_OUT_MID     "a.enc_ctc_out_mid.%s"
337#define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb"
338// qformer projector
339#define TN_QF_PROJ_QUERY   "%s.proj_query"
340#define TN_QF_PROJ_NORM    "%s.proj_norm.%s"
341#define TN_QF_PROJ_LINEAR  "%s.proj_linear.%s"
342#define TN_QF_SELF_ATTN_Q  "%s.proj_blk.%d.self_attn_q.%s"
343#define TN_QF_SELF_ATTN_K  "%s.proj_blk.%d.self_attn_k.%s"
344#define TN_QF_SELF_ATTN_V  "%s.proj_blk.%d.self_attn_v.%s"
345#define TN_QF_SELF_ATTN_O  "%s.proj_blk.%d.self_attn_out.%s"
346#define TN_QF_SELF_ATTN_N  "%s.proj_blk.%d.self_attn_norm.%s"
347#define TN_QF_CROSS_ATTN_Q "%s.proj_blk.%d.cross_attn_q.%s"
348#define TN_QF_CROSS_ATTN_K "%s.proj_blk.%d.cross_attn_k.%s"
349#define TN_QF_CROSS_ATTN_V "%s.proj_blk.%d.cross_attn_v.%s"
350#define TN_QF_CROSS_ATTN_O "%s.proj_blk.%d.cross_attn_out.%s"
351#define TN_QF_CROSS_ATTN_N "%s.proj_blk.%d.cross_attn_norm.%s"
352#define TN_QF_FFN_UP       "%s.proj_blk.%d.ffn_up.%s"
353#define TN_QF_FFN_DOWN     "%s.proj_blk.%d.ffn_down.%s"
354#define TN_QF_FFN_NORM     "%s.proj_blk.%d.ffn_norm.%s"
355// multi-projector qformer (bid => projector ID)
356#define TN_MULTI_PROJ_IMG_POS   "v.proj_blk.%d.img_pos"
357#define TN_MULTI_PROJ_QUERY     "%s.proj_blk.%d.query"
358#define TN_MULTI_PROJ_LINEAR    "%s.proj_blk.%d.linear.%s"
359#define TN_MULTI_PROJ_NORM      "%s.proj_blk.%d.norm.%s"
360#define TN_MULTI_PROJ_POST_NORM "%s.proj_blk.%d.post_norm.%s"
361
362// gemma4 audio conformer
363#define TN_A_MM_INP_PROJ     "mm.a.input_projection.%s"
364#define TN_A_MM_SOFT_EMB_N   "mm.a.soft_emb_norm.%s"
365#define TN_A_INP_PROJ        "a.input_projection.%s"
366#define TN_A_CONV1D          "a.conv1d.%d.%s"
367#define TN_A_CONV1D_NORM     "a.conv1d.%d.norm.%s"
368#define TN_A_OUT_PROJ        "a.pre_encode.out.%s"
369#define TN_A_ATTN_PRE_NORM   "%s.blk.%d.attn_pre_norm.%s"
370#define TN_A_ATTN_POST_NORM  "%s.blk.%d.attn_post_norm.%s"
371#define TN_A_ATTN_K_REL      "%s.blk.%d.attn_k_rel.%s"
372#define TN_A_PER_DIM_SCALE   "%s.blk.%d.per_dim_scale.%s"
373#define TN_A_PER_DIM_K_SCALE "%s.blk.%d.per_dim_k_scale.%s"
374#define TN_A_FFN_POST_NORM   "%s.blk.%d.ffn_post_norm.%s"
375#define TN_A_FFN_POST_NORM_1 "%s.blk.%d.ffn_post_norm_1.%s"
376
377// mobilenetv5 (gemma3n) definitions
378#define TN_MNV5_STEM_CONV        "v.conv_stem.conv.weight"
379#define TN_MNV5_STEM_BIAS        "v.conv_stem.conv.bias"
380#define TN_MNV5_STEM_BN          "v.conv_stem.bn.weight"
381
382// Stage 0 Block (Edge Residual)
383#define TN_MNV5_BLK_S0_EXP_W     "v.blk.%d.%d.conv_exp.weight"
384#define TN_MNV5_BLK_S0_BN1_W     "v.blk.%d.%d.bn1.weight"
385#define TN_MNV5_BLK_S0_PWL_W     "v.blk.%d.%d.conv_pwl.weight"
386#define TN_MNV5_BLK_S0_BN2_W     "v.blk.%d.%d.bn2.weight"
387
388// Stage 1+ Block (Universal Inverted Residual)
389#define TN_MNV5_BLK_DW_START_W   "v.blk.%d.%d.dw_start.conv.weight"
390#define TN_MNV5_BLK_DW_START_BN  "v.blk.%d.%d.dw_start.bn.weight"
391#define TN_MNV5_BLK_DW_MID_W     "v.blk.%d.%d.dw_mid.conv.weight"
392#define TN_MNV5_BLK_DW_MID_BN    "v.blk.%d.%d.dw_mid.bn.weight"
393#define TN_MNV5_BLK_PW_EXP_W     "v.blk.%d.%d.pw_exp.conv.weight"
394#define TN_MNV5_BLK_PW_EXP_BN    "v.blk.%d.%d.pw_exp.bn.weight"
395#define TN_MNV5_BLK_PW_PROJ_W    "v.blk.%d.%d.pw_proj.conv.weight"
396#define TN_MNV5_BLK_PW_PROJ_BN   "v.blk.%d.%d.pw_proj.bn.weight"
397#define TN_MNV5_BLK_LAYER_SCALE  "v.blk.%d.%d.layer_scale.gamma"
398
399// Attention Components
400#define TN_MNV5_ATTN_Q_W         "v.blk.%d.%d.attn.query.proj.weight"
401#define TN_MNV5_ATTN_K_W         "v.blk.%d.%d.attn.key.proj.weight"
402#define TN_MNV5_ATTN_V_W         "v.blk.%d.%d.attn.value.proj.weight"
403#define TN_MNV5_ATTN_O_W         "v.blk.%d.%d.attn.output.proj.weight"
404#define TN_MNV5_ATTN_K_DW        "v.blk.%d.%d.attn.key.down_conv.weight"
405#define TN_MNV5_ATTN_K_NORM      "v.blk.%d.%d.attn.key.norm.weight"
406#define TN_MNV5_ATTN_V_DW        "v.blk.%d.%d.attn.value.down_conv.weight"
407#define TN_MNV5_ATTN_V_NORM      "v.blk.%d.%d.attn.value.norm.weight"
408#define TN_MNV5_ATTN_NORM        "v.blk.%d.%d.norm.weight" // Block norm used in attn blocks
409
410// MSFA
411#define TN_MNV5_MSFA_FFN_EXP_W   "v.msfa.ffn.pw_exp.conv.weight"
412#define TN_MNV5_MSFA_FFN_EXP_BN  "v.msfa.ffn.pw_exp.bn.weight"
413#define TN_MNV5_MSFA_FFN_PROJ_W  "v.msfa.ffn.pw_proj.conv.weight"
414#define TN_MNV5_MSFA_FFN_PROJ_BN "v.msfa.ffn.pw_proj.bn.weight"
415#define TN_MNV5_MSFA_NORM        "v.msfa.norm.weight"
416
417// gemma4
418#define TN_STD_BIAS              "v.std_bias"
419#define TN_STD_SCALE             "v.std_scale"
420
421// yasa2
422#define TN_YASA_PATCH_LN_W       "v.patch_ln.weight"
423#define TN_YASA_PATCH_LN_B       "v.patch_ln.bias"
424#define TN_YASA_BACKBONE_LN_W    "v.backbone_ln.weight"
425#define TN_YASA_BACKBONE_LN_B    "v.backbone_ln.bias"
426#define TN_YASA_POS_EMBD         "v.vision_pos_embed"
427#define TN_YASA_STAGE_DOWN_LN    "v.stage.%d.down.ln.%s"
428#define TN_YASA_STAGE_DOWN_CONV  "v.stage.%d.down.conv.%s"
429#define TN_YASA_STAGE_BLK        "v.stage.%d.blk.%d.%s.%s"
430
431// parakeet
432#define TN_MEL_FILTERS           "a.mel_filters"
433#define TN_WINDOW                "a.window"
434#define TN_CONV_NORM_MEAN        "%s.blk.%d.conv_norm_mean"
435#define TN_CONV_NORM_VAR         "%s.blk.%d.conv_norm_var"
436
437// align x to upper multiple of n
438#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
439
440// forward declaration
441// TODO: improve this later
442struct clip_ctx;
443
444enum projector_type {
445    PROJECTOR_TYPE_MLP,
446    PROJECTOR_TYPE_MLP_NORM,
447    PROJECTOR_TYPE_LDP,
448    PROJECTOR_TYPE_LDPV2,
449    PROJECTOR_TYPE_MINICPMV,
450    PROJECTOR_TYPE_GLM_EDGE,
451    PROJECTOR_TYPE_QWEN2VL,
452    PROJECTOR_TYPE_QWEN3VL,
453    PROJECTOR_TYPE_STEP3VL,
454    PROJECTOR_TYPE_GEMMA3,
455    PROJECTOR_TYPE_GEMMA3NV,
456    PROJECTOR_TYPE_GEMMA3NA,
457    PROJECTOR_TYPE_GEMMA4V,
458    PROJECTOR_TYPE_GEMMA4A,
459    PROJECTOR_TYPE_GEMMA4UV,
460    PROJECTOR_TYPE_GEMMA4UA,
461    PROJECTOR_TYPE_PHI4,
462    PROJECTOR_TYPE_IDEFICS3,
463    PROJECTOR_TYPE_PIXTRAL,
464    PROJECTOR_TYPE_QWEN25VL,
465    PROJECTOR_TYPE_ULTRAVOX,
466    PROJECTOR_TYPE_INTERNVL,
467    PROJECTOR_TYPE_LLAMA4,
468    PROJECTOR_TYPE_QWEN2A,
469    PROJECTOR_TYPE_QWEN3A,
470    PROJECTOR_TYPE_GLMA,
471    PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx
472    PROJECTOR_TYPE_VOXTRAL,
473    PROJECTOR_TYPE_MERALION,
474    PROJECTOR_TYPE_MUSIC_FLAMINGO,
475    PROJECTOR_TYPE_LFM2,
476    PROJECTOR_TYPE_KIMIVL,
477    PROJECTOR_TYPE_PADDLEOCR,
478    PROJECTOR_TYPE_LIGHTONOCR,
479    PROJECTOR_TYPE_COGVLM,
480    PROJECTOR_TYPE_JANUS_PRO,
481    PROJECTOR_TYPE_DOTS_OCR,
482    PROJECTOR_TYPE_DOTS3NOTE_V,
483    PROJECTOR_TYPE_DOTS3NOTE_A,
484    PROJECTOR_TYPE_DEEPSEEKOCR,
485    PROJECTOR_TYPE_DEEPSEEKOCR2,
486    PROJECTOR_TYPE_DEEPSEEK4V,
487    PROJECTOR_TYPE_LFM2A,
488    PROJECTOR_TYPE_GLM4V,
489    PROJECTOR_TYPE_YOUTUVL,
490    PROJECTOR_TYPE_YASA2,
491    PROJECTOR_TYPE_KIMIK25,
492    PROJECTOR_TYPE_NEMOTRON_V2_VL,
493    PROJECTOR_TYPE_HUNYUANVL,
494    PROJECTOR_TYPE_PARAKEET,
495    PROJECTOR_TYPE_EXAONE4_5,
496    PROJECTOR_TYPE_MINICPMV4_6,
497    PROJECTOR_TYPE_GRANITE_SPEECH,
498    PROJECTOR_TYPE_MIMOVL,
499    PROJECTOR_TYPE_MINIMAX_M3,
500    PROJECTOR_TYPE_GRANITE4_VISION,
501    PROJECTOR_TYPE_MIMO_AUDIO,
502    PROJECTOR_TYPE_QWEN3TTS_SPKENC,
503    PROJECTOR_TYPE_QWEN3TTS_GEN,
504    PROJECTOR_TYPE_POCKETTTS_SPKENC,
505    PROJECTOR_TYPE_POCKETTTS_GEN,
506    PROJECTOR_TYPE_MUSE_GLIMMER,
507    PROJECTOR_TYPE_UNKNOWN,
508};
509
510static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
511    { PROJECTOR_TYPE_MLP,               "mlp" },
512    { PROJECTOR_TYPE_LDP,               "ldp" },
513    { PROJECTOR_TYPE_LDPV2,             "ldpv2"},
514    { PROJECTOR_TYPE_MINICPMV,          "resampler"},
515    { PROJECTOR_TYPE_GLM_EDGE,          "adapter"},
516    { PROJECTOR_TYPE_QWEN2VL,           "qwen2vl_merger"},
517    { PROJECTOR_TYPE_QWEN25VL,          "qwen2.5vl_merger"},
518    { PROJECTOR_TYPE_QWEN3VL,           "qwen3vl_merger"},
519    { PROJECTOR_TYPE_STEP3VL,           "step3vl"},
520    { PROJECTOR_TYPE_GEMMA3,            "gemma3"},
521    { PROJECTOR_TYPE_GEMMA3NV,          "gemma3nv"},
522    { PROJECTOR_TYPE_GEMMA3NA,          "gemma3na"},
523    { PROJECTOR_TYPE_GEMMA4V,           "gemma4v"},
524    { PROJECTOR_TYPE_GEMMA4A,           "gemma4a"},
525    { PROJECTOR_TYPE_GEMMA4UV,          "gemma4uv"},
526    { PROJECTOR_TYPE_GEMMA4UA,          "gemma4ua"},
527    { PROJECTOR_TYPE_PHI4,              "phi4"},
528    { PROJECTOR_TYPE_IDEFICS3,          "idefics3"},
529    { PROJECTOR_TYPE_PIXTRAL,           "pixtral"},
530    { PROJECTOR_TYPE_ULTRAVOX,          "ultravox"},
531    { PROJECTOR_TYPE_INTERNVL,          "internvl"},
532    { PROJECTOR_TYPE_LLAMA4,            "llama4"},
533    { PROJECTOR_TYPE_QWEN2A,            "qwen2a"},
534    { PROJECTOR_TYPE_QWEN3A,            "qwen3a"},
535    { PROJECTOR_TYPE_GLMA,              "glma"},
536    { PROJECTOR_TYPE_QWEN25O,           "qwen2.5o"},
537    { PROJECTOR_TYPE_VOXTRAL,           "voxtral"},
538    { PROJECTOR_TYPE_MERALION,          "meralion"},
539    { PROJECTOR_TYPE_MUSIC_FLAMINGO,    "musicflamingo"},
540    { PROJECTOR_TYPE_LFM2,              "lfm2"},
541    { PROJECTOR_TYPE_KIMIVL,            "kimivl"},
542    { PROJECTOR_TYPE_PADDLEOCR,         "paddleocr"},
543    { PROJECTOR_TYPE_LIGHTONOCR,        "lightonocr"},
544    { PROJECTOR_TYPE_COGVLM,            "cogvlm"},
545    { PROJECTOR_TYPE_JANUS_PRO,         "janus_pro"},
546    { PROJECTOR_TYPE_DOTS_OCR,          "dots_ocr"},
547    { PROJECTOR_TYPE_DOTS3NOTE_V,       "dots3note_v"},
548    { PROJECTOR_TYPE_DOTS3NOTE_A,       "dots3note_a"},
549    { PROJECTOR_TYPE_DEEPSEEKOCR,       "deepseekocr"},
550    { PROJECTOR_TYPE_DEEPSEEKOCR2,      "deepseekocr2"},
551    { PROJECTOR_TYPE_DEEPSEEK4V,        "deepseek4v"},
552    { PROJECTOR_TYPE_LFM2A,             "lfm2a"},
553    { PROJECTOR_TYPE_GLM4V,             "glm4v"},
554    { PROJECTOR_TYPE_YOUTUVL,           "youtuvl"},
555    { PROJECTOR_TYPE_YASA2,             "yasa2"},
556    { PROJECTOR_TYPE_KIMIK25,           "kimik25"},
557    { PROJECTOR_TYPE_NEMOTRON_V2_VL,    "nemotron_v2_vl"},
558    { PROJECTOR_TYPE_EXAONE4_5,         "exaone4_5"},
559    { PROJECTOR_TYPE_HUNYUANVL,         "hunyuanvl"},
560    { PROJECTOR_TYPE_MINICPMV4_6,       "minicpmv4_6"},
561    { PROJECTOR_TYPE_GRANITE_SPEECH,    "granite_speech"},
562    { PROJECTOR_TYPE_MIMOVL,            "mimovl"},
563    { PROJECTOR_TYPE_MINIMAX_M3,        "minimax_m3"},
564    { PROJECTOR_TYPE_GRANITE4_VISION,   "granite4_vision"},
565    { PROJECTOR_TYPE_MIMO_AUDIO,        "mimo_audio"},
566    { PROJECTOR_TYPE_PARAKEET,          "parakeet"},
567    { PROJECTOR_TYPE_QWEN3TTS_SPKENC,   "qwen3tts_spkenc"},
568    { PROJECTOR_TYPE_QWEN3TTS_GEN,      "qwen3tts_gen"},
569    { PROJECTOR_TYPE_POCKETTTS_SPKENC,  "pockettts_spkenc"},
570    { PROJECTOR_TYPE_POCKETTTS_GEN,     "pockettts_gen"},
571    { PROJECTOR_TYPE_MUSE_GLIMMER,      "muse-glimmer"},
572};
573
574static projector_type clip_projector_type_from_string(const std::string & str) {
575    for (const auto & pair : PROJECTOR_TYPE_NAMES) {
576        if (pair.second == str) {
577            return pair.first;
578        }
579    }
580    return PROJECTOR_TYPE_UNKNOWN;
581}
582
583// RGB uint8 image
584struct clip_image_u8 {
585    clip_image_size get_size() const {
586        return { nx, ny };
587    }
588
589    void set_size(clip_image_size size, bool is_placeholder) {
590        nx = size.width;
591        ny = size.height;
592        if (is_placeholder) {
593            buf.clear();
594        } else {
595            buf.resize((size_t) nx * (size_t) ny * 3);
596        }
597    }
598
599    void cpy_buf(const std::vector<uint8_t> & new_buf) {
600        buf = new_buf;
601    }
602
603    const std::vector<uint8_t> & get_ro_buf() const {
604        if (is_placeholder()) {
605            throw std::runtime_error("this clip_image_u8 is a placeholder");
606        }
607        return buf;
608    }
609
610    // note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern. always use get_pixel / set_pixel for buffer manipulation
611
612    bool is_placeholder() const {
613        return buf.empty();
614    }
615
616    std::array<uint8_t, 3> get_pixel(int x, int y) const {
617        if (is_placeholder()) {
618            // return a dummy value, so that legacy code can still process image without errors
619            return { 0, 0, 0 };
620        }
621        size_t idx = ((size_t) y * (size_t) nx + (size_t) x) * 3;
622        return { buf[idx], buf[idx + 1], buf[idx + 2] };
623    }
624
625    void set_pixel(int x, int y, const std::array<uint8_t, 3> & rgb) {
626        if (is_placeholder()) {
627            return; // no-op
628        }
629        size_t idx = ((size_t) y * (size_t) nx + (size_t) x) * 3;
630        buf[idx]     = rgb[0];
631        buf[idx + 1] = rgb[1];
632        buf[idx + 2] = rgb[2];
633    }
634
635    size_t n_elements() const {
636        return n_pixels() * 3;
637    }
638
639  private:
640    std::vector<uint8_t> buf;
641    int nx = 0;
642    int ny = 0;
643
644    size_t n_pixels() const {
645        return (size_t) nx * (size_t) ny;
646    }
647};
648
649struct mtmd_serialization; // forward declaration
650
651// For images, buf.size() == nx*ny*3
652//     Memory layout: RGBRGBRGB...
653// For seq, buf.size() == nx*ny*3*nt
654//     Memory layout: RGBRGB...RGBRGB... (nt times)
655// For audio, only one channel is used, buf.size() == nx*ny
656//     nx will be n_frames and ny will be n_mel
657struct clip_image_f32 {
658    // marks the global view in e.g., DeepSeek-OCR Models
659    bool add_viewsep = false;
660    // appends a learned newline (or EOI) token after the image
661    // no model uses it now (Granite4 Vision moved to anyres), kept for future models
662    bool add_newline = false;
663    // deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
664    // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
665    int32_t lead_pad = 0;
666
667    // llava-next "anyres" tiling, used by Granite4 Vision
668    // the whole grid is encoded and assembled in a single graph
669    // NOTE: excluded from serialized: a deserialized image is always a placeholder, which is never encoded
670    struct anyres_info {
671        int grid_x = 0; // tiles per row, 0 means the image is not tiled
672        int grid_y = 0; // tiles per column
673        int orig_nx = 0; // size of the source image, used to drop the padding tokens
674        int orig_ny = 0;
675
676        bool is_tiled() const {
677            return grid_x > 0 && grid_y > 0;
678        }
679    };
680    anyres_info anyres;
681
682    clip_image_size get_size() const {
683        return { nx_, ny_ };
684    }
685
686    int nx() const { return nx_; }
687    int ny() const { return ny_; }
688
689    void set_size(clip_image_size size, bool is_placeholder, bool is_audio) {
690        nx_ = size.width;
691        ny_ = size.height;
692        if (is_placeholder) {
693            buf.clear();
694        } else {
695            if (is_audio) {
696                buf.resize((size_t) nx_ * (size_t) ny_);
697            } else {
698                buf.resize((size_t) nx_ * (size_t) ny_ * 3);
699            }
700        }
701    }
702
703    void cpy_buf(const std::vector<float> & new_buf) {
704        buf = new_buf;
705    }
706
707    void from_u8(const clip_image_u8 & img) {
708        auto size = img.get_size();
709        nx_ = size.width;
710        ny_ = size.height;
711        if (img.is_placeholder()) {
712            buf.clear();
713            return; // no-op
714        }
715        buf.resize(img.n_elements());
716        const auto & u8_buf = img.get_ro_buf();
717        for (size_t i = 0; i < img.n_elements(); ++i) {
718            buf[i] = (float) u8_buf[i] / 255.0f;
719        }
720    }
721
722    size_t n_elements() const {
723        return n_pixels() * 3;
724    }
725
726    void normalize(const float mean[3], const float std[3]) {
727        if (is_placeholder()) {
728            return; // no-op
729        }
730        for (size_t i = 0; i < n_pixels(); ++i) {
731            buf[i * 3 + 0] = (buf[i * 3 + 0] - mean[0]) / std[0];
732            buf[i * 3 + 1] = (buf[i * 3 + 1] - mean[1]) / std[1];
733            buf[i * 3 + 2] = (buf[i * 3 + 2] - mean[2]) / std[2];
734        }
735    }
736
737    const std::vector<float> & get_ro_buf() const {
738        if (is_placeholder()) {
739            throw std::runtime_error("this clip_image_f32 is a placeholder");
740        }
741        return buf;
742    }
743
744    // note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern
745
746    bool is_placeholder() const {
747        return buf.empty();
748    }
749
750    void serialize(struct mtmd_serialization & ser) const;
751    void deserialize(struct mtmd_serialization & ser);
752
753  private:
754    std::vector<float> buf;
755    int nx_ = 0;
756    int ny_ = 0;
757
758    size_t n_pixels() const {
759        return (size_t) nx_ * (size_t) ny_;
760    }
761};
762
763// token area kept after removing the padding added by the anyres resize
764// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109
765static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
766                                     int & off_x, int & off_y, int & out_w, int & out_h) {
767    off_x = 0;
768    off_y = 0;
769    out_w = cur_w;
770    out_h = cur_h;
771    if ((float) orig_w / orig_h > (float) cur_w / cur_h) {
772        const int new_h = (int) std::floor((double) orig_h * cur_w / orig_w + 1e-7);
773        off_y = (cur_h - new_h) / 2;
774        out_h = cur_h - 2 * off_y;
775    } else {
776        const int new_w = (int) std::floor((double) orig_w * cur_h / orig_h + 1e-7);
777        off_x = (cur_w - new_w) / 2;
778        out_w = cur_w - 2 * off_x;
779    }
780}
781
782// deepseek4v: layout of the LLM token block built from the aligner grid
783struct dsv4_block_layout {
784    int rows;     // grid rows, padded to an even count
785    int row_len;  // grid width + 1 newline
786    int pad_last; // trailing pads
787    int n_out;    // total block size, including lead pads and the start/end sentinels
788};
789static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
790    dsv4_block_layout bl;
791    bl.rows     = n_llm_h + (n_llm_h % 2);
792    bl.row_len  = n_llm_w + 1;
793    bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
794    bl.n_out    = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
795    return bl;
796}
797
798//
799// logging
800//
801
802static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
803    (void) level;
804    (void) user_data;
805    fputs(text, stderr);
806    fflush(stderr);
807}
808
809struct clip_logger_state {
810    ggml_log_callback log_callback;
811    void * log_callback_user_data;
812};
813
814extern struct clip_logger_state g_logger_state;
815
816static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
817    if (format == NULL) {
818        return;
819    }
820    va_list args_copy;
821    va_copy(args_copy, args);
822    char buffer[128];
823    int len = vsnprintf(buffer, 128, format, args);
824    if (len < 128) {
825        g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);
826    } else {
827        char * buffer2 = (char *) calloc(len + 1, sizeof(char));
828        vsnprintf(buffer2, len + 1, format, args_copy);
829        buffer2[len] = 0;
830        g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);
831        free(buffer2);
832    }
833    va_end(args_copy);
834}
835
836static void clip_log_internal(enum ggml_log_level level, const char * format, ...) {
837    va_list args;
838    va_start(args, format);
839    clip_log_internal_v(level, format, args);
840    va_end(args);
841}
842
843#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
844#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
845#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO,  __VA_ARGS__)
846#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN,  __VA_ARGS__)
847#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
848#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT,  __VA_ARGS__)
849
850//
851// cpp wrappers
852//
853
854struct clip_image_f32_batch {
855    std::vector<clip_image_f32> entries;
856    bool is_audio = false;
857
858    clip_image_f32_batch clone() const {
859        clip_image_f32_batch new_batch{
860            /* entries  */ {},
861            /* is_audio */ is_audio,
862        };
863        new_batch.entries.reserve(entries.size());
864        for (const auto & entry : entries) {
865            new_batch.entries.emplace_back(entry); // copy
866        }
867        return new_batch;
868    }
869
870    void serialize(struct mtmd_serialization & ser) const;
871    void deserialize(struct mtmd_serialization & ser);
872};
873
874//
875// common utils
876//
877
878#ifdef _WIN32
879static std::ifstream open_ifstream_binary(const std::string & fname) {
880    int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);
881    if (!wlen) {
882        throw std::runtime_error("failed to convert filename to UTF-16: " + fname);
883    }
884    std::vector<wchar_t> wfname(wlen);
885    (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);
886    return std::ifstream(wfname.data(), std::ios::binary);
887}
888#else
889static std::ifstream open_ifstream_binary(const std::string & fname) {
890    return std::ifstream(fname, std::ios::binary);
891}
892#endif
893
894// in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated
895// this is a quick fix to avoid compilation errors
896#ifndef DIRECTORY_SEPARATOR
897static std::string string_format(const char * fmt, ...) {
898    va_list ap;
899    va_list ap2;
900    va_start(ap, fmt);
901    va_copy(ap2, ap);
902    int size = vsnprintf(NULL, 0, fmt, ap);
903    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
904    std::vector<char> buf(size + 1);
905    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
906    GGML_ASSERT(size2 == size);
907    va_end(ap2);
908    va_end(ap);
909    return std::string(buf.data(), buf.size());
910}
911
912static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
913    if (search.empty()) {
914        return;
915    }
916    std::string builder;
917    builder.reserve(s.length());
918    size_t pos = 0;
919    size_t last_pos = 0;
920    while ((pos = s.find(search, last_pos)) != std::string::npos) {
921        builder.append(s, last_pos, pos - last_pos);
922        builder.append(replace);
923        last_pos = pos + search.length();
924    }
925    builder.append(s, last_pos, std::string::npos);
926    s = std::move(builder);
927}
928
929// split string by a `std::string delim` instead of `char delim`
930static std::vector<std::string> string_split_str(std::string s, const std::string & delimiter) {
931    std::vector<std::string> tokens;
932    size_t pos = 0;
933    std::string token;
934    while ((pos = s.find(delimiter)) != std::string::npos) {
935        token = s.substr(0, pos);
936        tokens.push_back(token);
937        s.erase(0, pos + delimiter.length());
938    }
939    tokens.push_back(s);
940    return tokens;
941}
942
943// remove when moving to c++20
944inline bool string_starts_with(std::string_view str, std::string_view prefix) {
945    return str.size() >= prefix.size() &&
946           str.compare(0, prefix.size(), prefix) == 0;
947}
948
949// remove when moving to c++20
950inline bool string_ends_with(std::string_view str, std::string_view suffix) {
951    return str.size() >= suffix.size() &&
952           str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
953}
954#endif
955
956//
957// gguf utils
958//
959
960static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {
961    switch (type) {
962        case GGUF_TYPE_UINT8:   return std::to_string(((const uint8_t  *)data)[i]);
963        case GGUF_TYPE_INT8:    return std::to_string(((const int8_t   *)data)[i]);
964        case GGUF_TYPE_UINT16:  return std::to_string(((const uint16_t *)data)[i]);
965        case GGUF_TYPE_INT16:   return std::to_string(((const int16_t  *)data)[i]);
966        case GGUF_TYPE_UINT32:  return std::to_string(((const uint32_t *)data)[i]);
967        case GGUF_TYPE_INT32:   return std::to_string(((const int32_t  *)data)[i]);
968        case GGUF_TYPE_UINT64:  return std::to_string(((const uint64_t *)data)[i]);
969        case GGUF_TYPE_INT64:   return std::to_string(((const int64_t  *)data)[i]);
970        case GGUF_TYPE_FLOAT32: return std::to_string(((const float    *)data)[i]);
971        case GGUF_TYPE_FLOAT64: return std::to_string(((const double   *)data)[i]);
972        case GGUF_TYPE_BOOL:    return ((const int8_t *)data)[i] != 0 ? "true" : "false";
973        default:                return string_format("unknown type %d", type);
974    }
975}
976
977static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
978    const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);
979
980    switch (type) {
981        case GGUF_TYPE_STRING:
982            return gguf_get_val_str(ctx_gguf, i);
983        case GGUF_TYPE_ARRAY:
984            {
985                const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);
986                int arr_n = gguf_get_arr_n(ctx_gguf, i);
987                const void * data = arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx_gguf, i);
988                std::stringstream ss;
989                ss << "[";
990                for (int j = 0; j < arr_n; j++) {
991                    if (arr_type == GGUF_TYPE_STRING) {
992                        std::string val = gguf_get_arr_str(ctx_gguf, i, j);
993                        // escape quotes
994                        string_replace_all(val, "\\", "\\\\");
995                        string_replace_all(val, "\"", "\\\"");
996                        ss << '"' << val << '"';
997                    } else if (arr_type == GGUF_TYPE_ARRAY) {
998                        ss << "???";
999                    } else {
1000                        ss << gguf_data_to_str(arr_type, data, j);
1001                    }
1002                    if (j < arr_n - 1) {
1003                        ss << ", ";
1004                    }
1005                }
1006                ss << "]";
1007                return ss.str();
1008            }
1009        default:
1010            return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);
1011    }
1012}
1013
1014//
1015// debugging
1016//
1017
1018static void print_tensor_shape(ggml_tensor * t) {
1019    printf("%s.shape = [", t->name);
1020    for (int i = 0; i < ggml_n_dims(t); ++i) {
1021        printf("%" PRId64, t->ne[i]);
1022        if (i < ggml_n_dims(t) - 1) {
1023            printf(", ");
1024        }
1025    }
1026    printf("]\n");
1027}
1028
1029static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
1030    ggml_type type = t->type;
1031    int64_t * ne = t->ne;
1032    size_t * nb = t->nb;
1033    for (int64_t i3 = 0; i3 < ne[3]; i3++) {
1034        printf("%s.data: [\n", t->name);
1035        for (int64_t i2 = 0; i2 < ne[2]; i2++) {
1036            if (i2 == n && ne[2] > 2*n) {
1037                printf("     ..., \n");
1038                i2 = ne[2] - n;
1039            }
1040            printf("     [\n");
1041            for (int64_t i1 = 0; i1 < ne[1]; i1++) {
1042                if (i1 == n && ne[1] > 2*n) {
1043                    printf("      ..., \n");
1044                    i1 = ne[1] - n;
1045                }
1046                printf("      [");
1047                for (int64_t i0 = 0; i0 < ne[0]; i0++) {
1048                    if (i0 == n && ne[0] > 2*n) {
1049                        printf("..., ");
1050                        i0 = ne[0] - n;
1051                    }
1052                    size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0];
1053                    float v;
1054                    if (type == GGML_TYPE_F16) {
1055                        v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]);
1056                    } else if (type == GGML_TYPE_F32) {
1057                        v = *(float *) &data[i];
1058                    } else if (type == GGML_TYPE_I32) {
1059                        v = (float) *(int32_t *) &data[i];
1060                    } else if (type == GGML_TYPE_I16) {
1061                        v = (float) *(int16_t *) &data[i];
1062                    } else if (type == GGML_TYPE_I8) {
1063                        v = (float) *(int8_t *) &data[i];
1064                    } else {
1065                        GGML_ABORT("fatal error");
1066                    }
1067                    printf("%8.4f", v);
1068                    if (i0 < ne[0] - 1) printf(", ");
1069                }
1070                printf("],\n");
1071            }
1072            printf("     ],\n");
1073        }
1074        printf("    ]\n");
1075    }
1076}
1077
1078//
1079// API used internally with mtmd
1080//
1081
1082projector_type clip_get_projector_type(const struct clip_ctx * ctx);
1083void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug);