char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

John-Henry Limmtmd: add --mmproj-device argument (#23255)681c29d36

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.8 KiB139 linesraw
1#pragma once
2
3#include "ggml.h"
4#include "mtmd.h"
5
6#include <stddef.h>
7#include <stdint.h>
8
9#include <map>
10
11// !!! Internal header, to be used by mtmd only !!!
12
13#define MTMD_INTERNAL_HEADER
14
15struct clip_ctx;
16
17struct clip_image_size {
18    int width;
19    int height;
20    bool operator==(const clip_image_size & other) const {
21        return width == other.width && height == other.height;
22    }
23    bool operator!=(const clip_image_size & other) const {
24        return !(*this == other);
25    }
26    int area() const {
27        // avoid overflow when computing area
28        GGML_ASSERT(width  >= 0 && width  <= 46000);
29        GGML_ASSERT(height >= 0 && height <= 46000);
30        return width * height;
31    }
32};
33
34struct clip_image_f32;
35struct clip_image_f32_batch;
36
37enum clip_modality {
38    CLIP_MODALITY_VISION,
39    CLIP_MODALITY_AUDIO,
40    CLIP_MODALITY_GEN_AUDIO,
41};
42
43enum clip_flash_attn_type {
44    CLIP_FLASH_ATTN_TYPE_AUTO     = -1,
45    CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
46    CLIP_FLASH_ATTN_TYPE_ENABLED  = 1,
47};
48
49struct clip_context_params {
50    bool use_gpu;
51    ggml_backend_dev_t device;
52    enum clip_flash_attn_type flash_attn_type;
53    int image_min_tokens;
54    int image_max_tokens;
55    bool warmup;
56    ggml_backend_sched_eval_callback cb_eval;
57    void * cb_eval_user_data;
58    bool no_alloc;
59    mtmd_progress_callback progress_callback;
60    void * progress_callback_user_data;
61};
62
63struct clip_init_result {
64    struct clip_ctx * ctx_v; // vision context
65    struct clip_ctx * ctx_a; // audio context
66    struct clip_ctx * ctx_gen_a; // audio generation context
67};
68
69struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
70
71void clip_free(struct clip_ctx * ctx);
72
73// TODO: should be enum, not string
74const char * clip_patch_merge_type(const struct clip_ctx * ctx);
75
76int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img);
77
78// for M-RoPE, this will be the number of token positions in X and Y directions
79// for other models, X will be the total number of tokens and Y will be 1
80int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img);
81int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);
82
83// this should be equal to the embedding dimension of the text model
84int clip_n_mmproj_embd(const struct clip_ctx * ctx);
85
86// TODO: remove clip_image_encode() and always use batched version
87bool clip_image_encode      (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
88bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
89
90enum clip_gen_process_type {
91    CLIP_GEN_PROCESS_GEN_UNKNOWN,
92    CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes
93    CLIP_GEN_PROCESS_GEN_WAV,  // codes to raw PCM audio
94};
95struct clip_encode_params {
96    int n_threads = 1;
97    const clip_image_f32_batch * imgs = nullptr;
98    std::vector<float> * out_embd = nullptr;
99
100    // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
101    clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
102
103    // GEN_CODE: out_embd receives the embd to feed back to the backbone
104    int32_t code0 = 0; // semantic code sampled by the backbone
105    int32_t top_k = 50;
106    float   top_p = 1.0f;
107    std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
108    std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
109    uint32_t seed = UINT32_MAX;               // UINT32_MAX for random
110    float   temp = 0.0f;                      // sampling temperature, noise scale for flow-matching decoders
111    bool * out_is_eos = nullptr;
112
113    // GEN_WAV
114    const std::vector<int32_t> * codes = nullptr;     // this frame's 16 RVQ codes
115    const std::vector<float> *   feats = nullptr;     // continuous counterpart of codes
116    std::vector<float> * out_audio = nullptr;         // decoded PCM samples, F32
117    const std::vector<uint8_t> * state_in  = nullptr; // state from previous call, null or wrong size means cold start
118    std::vector<uint8_t> *       state_out = nullptr; // state for the next call
119};
120bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
121
122bool clip_is_llava(const struct clip_ctx * ctx);
123// note for contributor: this clip_is_(model) pattern is deprecated
124//                       do NOT add new functions like this
125
126bool clip_has_vision_encoder(const struct clip_ctx * ctx);
127bool clip_has_audio_encoder(const struct clip_ctx * ctx);
128
129bool clip_support_batch(const struct clip_ctx * ctx);
130
131int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this
132
133std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
134
135struct clip_cap {
136    bool has_vision;
137    bool has_audio;
138};
139struct clip_cap clip_get_cap(const char * fname);