char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: propagate const to preproc class (#28310)de8656bd9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.8 KiB257 linesraw
1#pragma once
2
3#include "ggml.h"
4#include "clip-model.h"
5
6#include <vector>
7#include <string>
8
9#define MTMD_INTERNAL_HEADER
10
11struct mtmd_image_preproc_out {
12    std::vector<clip_image_f32> entries;
13    // grid size is required for llava-uhd style models
14
15    clip_image_f32 overview; // overview image (downscaled image)
16    int grid_x = 0;
17    int grid_y = 0;
18
19    void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
20    void append(const clip_hparams & hparams, const std::vector<clip_image_u8> & imgs, bool normalized = true);
21    void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true);
22
23    void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
24    bool has_overview() const {
25        return overview.nx() > 0 || overview.ny() > 0;
26    }
27};
28
29// base class, models must inherit from this class
30struct mtmd_image_preprocessor {
31    const clip_hparams & hparams;
32
33    mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
34
35    virtual ~mtmd_image_preprocessor() = default;
36    virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0;
37};
38
39/**
40 * implementation of LLaVA-UHD:
41 *  - https://arxiv.org/pdf/2403.11703
42 *  - https://github.com/thunlp/LLaVA-UHD
43 *  - https://github.com/thunlp/LLaVA-UHD/blob/302301bc2175f7e717fb8548516188e89f649753/llava_uhd/train/llava-uhd/slice_logic.py#L118
44 *
45 * overview:
46 *   - an image always have a single overview (downscaled image)
47 *   - an image can have 0 or multiple slices, depending on the image size
48 *   - each slice can then be considered as a separate image
49 *
50 * note: the term "slice" and "tile" are used interchangeably
51 *
52 * for example:
53 *
54 * [overview] --> [slice 1] --> [slice 2]
55 *           |                |
56 *           +--> [slice 3] --> [slice 4]
57 *
58 * NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context
59 */
60struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
61    mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
62    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
63
64    struct slice_coordinates {
65        int x;
66        int y;
67        clip_image_size size;
68    };
69
70    struct slice_instructions {
71        clip_image_size overview_size; // size of downscaled image
72        clip_image_size refined_size;  // size of image right before slicing (must be multiple of slice size)
73        clip_image_size grid_size;     // grid_size.width * grid_size.height = number of slices
74        std::vector<slice_coordinates> slices;
75    };
76
77    virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const;
78
79    struct slice_output {
80        clip_image_u8 overview;
81        std::vector<clip_image_u8> slices;
82    };
83    slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const;
84
85protected:
86    clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const;
87
88    /**
89     * Selects the best resolution from a list of possible resolutions based on the original size.
90     *
91     * For example, when given a list of resolutions:
92     *  - 100x100
93     *  - 200x100
94     *  - 100x200
95     *  - 200x200
96     *
97     * And an input image of size 111x200, then 100x200 is the best fit (least wasted resolution).
98     *
99     * @param original_size The original size of the image
100     * @param possible_resolutions A list of possible resolutions
101     * @return The best fit resolution
102     */
103    clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const;
104
105private:
106    clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const;
107    int ensure_divide(int length, int patch_size) const;
108    clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const;
109    clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const;
110};
111
112// downscale or upscale the input image to fixed size
113struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
114    mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
115    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
116};
117
118// resize image to multiple of patch_size*n_merge, while preserving aspect ratio
119// if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad
120// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
121struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
122    mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
123    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
124};
125
126// similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio
127struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
128    mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
129    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
130};
131
132// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
133struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
134    mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
135    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
136
137private:
138    struct grid_info {
139        int n_llm_h;
140        int n_llm_w;
141        int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)
142    };
143    static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);
144    static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);
145    static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
146};
147
148// custom llava-uhd slicing logic for MiniCPM-V
149struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
150    using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
151    slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
152};
153
154// custom llava-uhd slicing logic for LFM2
155// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
156struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
157    // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
158    static constexpr int   min_tiles            = 2;
159    static constexpr int   max_tiles            = 10;
160    static constexpr float max_pixels_tolerance = 2.0f;
161    static constexpr int   tile_size            = 512;
162
163    using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
164    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
165    slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
166
167    static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
168
169private:
170    clip_image_size find_closest_aspect_ratio(
171            float aspect_ratio,
172            const std::vector<clip_image_size> & target_ratios,
173            int width, int height) const;
174    std::vector<clip_image_size> get_target_ratios() const;
175    clip_image_size get_grid_layout(int height, int width) const;
176};
177
178struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {
179    mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
180    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
181};
182
183struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
184    mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
185    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
186};
187
188// DeepSeek-OCR (v1/v2) global view + optional local tile grid
189struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
190    mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
191        : mtmd_image_preprocessor(ctx),
192            fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
193          base_size(hparams.image_size),
194          tile_size(hparams.preproc_tile_size),
195          min_tiles(hparams.preproc_min_tiles),
196          max_tiles(hparams.preproc_max_tiles) {}
197    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
198
199private:
200    bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
201    int base_size; // global view
202    int tile_size; // each tile
203    int min_tiles;
204    int max_tiles;
205
206    std::vector<clip_image_size> get_target_ratios() const;
207    clip_image_size find_closest_aspect_ratio(
208            float aspect_ratio,
209            const std::vector<clip_image_size> & target_ratios,
210            int width, int height) const;
211};
212
213// custom image preprocessing for Step3VL
214// ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py
215struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd {
216    mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
217    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
218    static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);
219
220private:
221    static constexpr int   default_image_longest_edge = 3024;
222    static constexpr int   default_image_crop_size    = 504;
223    static constexpr float small_aspect_ratio_limit   = 1.5f;
224    static constexpr float wide_aspect_ratio_limit    = 4.0f;
225    static constexpr float crop_rounding_threshold    = 0.2f;
226
227    void img_u8_resize_bilinear_to_f32(
228            const clip_image_u8 & src,
229            clip_image_f32 & dst,
230            int target_width,
231            int target_height,
232            const float mean[3],
233            const float std[3]) const;
234    static int get_image_longest_edge(const clip_hparams & params);
235    static int determine_window_size(const clip_hparams & params, int longer, int shorter);
236    static int calc_crop_extent(int length, int window_size);
237    static std::vector<int> calc_grid(int length, int window_size);
238    static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params);
239    static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h);
240};
241
242struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
243    mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
244    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
245};
246
247// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
248struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
249    mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
250    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
251};
252
253// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize.
254struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor {
255    mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
256    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
257};