char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: use ggml_rope_set_offset (#27521)b21e4de74

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.3 KiB165 linesraw
1#pragma once
2
3#include "ggml.h"
4#include "ggml-cpp.h"
5#include "clip.h"
6#include "clip-impl.h"
7#include "clip-model.h"
8
9#include <vector>
10#include <functional>
11
12#define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS)
13
14struct build_vit_opts {
15    ggml_tensor * attn_mask = nullptr;
16    // TODO @ngxson : merge attn_mask and attn_mask_layers into one call
17    std::vector<ggml_tensor *> attn_mask_layers; // one per layer
18
19    // hook at layer output embeddings
20    std::function<void(ggml_tensor * cur, int il)> callback_layer_out = nullptr;
21
22    // whether to skip the automatic post-layernorm (model.post_ln_w) applied at the end
23    bool skip_post_ln = false;
24};
25
26struct clip_graph {
27    const clip_model & model;
28    const clip_hparams & hparams;
29    projector_type proj_type;
30
31    const clip_image_f32 & img; // for backward compat
32    const clip_image_f32_batch * img_batch = nullptr;
33
34    const int patch_size;
35    const int n_patches_x;
36    const int n_patches_y;
37    const int n_patches;
38    const int n_embd;
39    const int n_head;
40    const int n_head_kv;
41    const int d_head;
42    const int n_layer;
43    const int n_mmproj_embd;
44    const float eps;
45    float kq_scale; // TODO: maybe move this to hparams
46    const clip_flash_attn_type flash_attn_type;
47
48    // TODO [QWEN_VIDEO]: improve this in the future
49    int n_batch = 1;
50
51    ggml_context_ptr ctx0_ptr;
52    ggml_context * ctx0;
53    ggml_cgraph * gf;
54
55    clip_graph(clip_ctx * ctx, const clip_image_f32 & img);
56
57    // build sub-graph, reuse buf from parent
58    clip_graph(const clip_graph & parent);
59
60    virtual ~clip_graph() = default;
61    virtual ggml_cgraph * build() = 0;
62
63    // wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model
64    // tensor w should be the weight matrix, and tensor x should be the input
65    virtual ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const;
66    // TODO: build_mm(w, b, x) to support bias
67
68    virtual bool support_batch() const {
69        return false;
70    }
71
72    //
73    // utility functions
74    //
75    void cb(ggml_tensor * cur0, const char * name, int il) const;
76
77    const clip_image_f32 & get_img(size_t idx) const {
78        GGML_ASSERT(img_batch);
79        GGML_ASSERT(idx < img_batch->entries.size());
80        return img_batch->entries[idx];
81    }
82
83    // siglip2 naflex
84    ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);
85
86    // build vision transformer (ViT) cgraph
87    // this function should cover most of the models
88    // if your model has specific features, you should probably duplicate this function
89    ggml_tensor * build_vit(
90                ggml_tensor * inp,
91                int64_t n_pos,
92                norm_type norm_t,
93                ffn_op_type ffn_t,
94                ggml_tensor * learned_pos_embd,
95                std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
96                const build_vit_opts & opts = {});
97
98    // build the input after conv2d (inp_raw --> patches)
99    // returns tensor with shape [n_embd, n_patches]
100    ggml_tensor * build_inp();
101
102    ggml_tensor * build_inp_raw(int channels = 3);
103
104    ggml_tensor * build_norm(
105            ggml_tensor * cur,
106            ggml_tensor * mw,
107            ggml_tensor * mb,
108            norm_type type,
109            float norm_eps,
110            int il) const;
111
112    ggml_tensor * build_ffn(
113            ggml_tensor * cur,
114            ggml_tensor * up,
115            ggml_tensor * up_b,
116            ggml_tensor * gate,
117            ggml_tensor * gate_b,
118            ggml_tensor * down,
119            ggml_tensor * down_b,
120            ffn_op_type type_op,
121            int il) const;
122
123    ggml_tensor * build_moe_ffn(
124            ggml_tensor * cur,
125            const clip_layer & layer,
126            ffn_op_type type_op,
127            int il) const;
128
129    ggml_tensor * build_attn(
130            ggml_tensor * wo,
131            ggml_tensor * wo_b,
132            ggml_tensor * q_cur,
133            ggml_tensor * k_cur,
134            ggml_tensor * v_cur,
135            ggml_tensor * kq_mask,
136            float kq_scale,
137            int il,
138            ggml_tensor * sinks = nullptr) const;
139
140    // implementation of the 2D RoPE using two ggml_rope_ext calls
141    //
142    // unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering)
143    //
144    // example:
145    //  given a single head with size = 8 --> [00000000]
146    //  dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb]
147    //  interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION)
148    //  interleave_freq = true  --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral)
149    ggml_tensor * build_rope_2d(
150        ggml_context * ctx0,
151        ggml_tensor * cur,
152        ggml_tensor * pos_a, // first half
153        ggml_tensor * pos_b, // second half
154        const float freq_base,
155        const bool interleave_freq
156    );
157
158    // aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
159    // support dynamic resolution
160    ggml_tensor * build_patch_merge_permute(ggml_tensor * cur, int scale_factor);
161
162    // Generic function to stack frames for audio processing
163    // Abstracts out the StackAudioFrames logic used by ultravox
164    ggml_tensor * build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed);
165};