char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

wendadawenmtmd, model : merge HunyuanOCR into HunyuanVL and fix OCR vision precision (#23329)6a257d446

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB63 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_hunyuanvl::build() {
4    const int merge = hparams.n_merge;
5    const int pw    = n_patches_x;
6    const int ph    = n_patches_y;
7
8    // position embedding: declared as a graph input, filled on CPU
9    // by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there).
10    ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw);
11    ggml_set_name(pos_embd, "hunyuanvl_pos_embd");
12    ggml_set_input(pos_embd);
13
14    ggml_tensor * inp = build_inp();
15    ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, pos_embd, nullptr);
16
17    // perceiver projector
18    cur = build_norm(cur, model.mm_pre_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
19
20    // [C, W*H] -> [W, H, C] for conv2d
21    cur = ggml_reshape_3d(ctx0, cur, n_embd, pw, ph);
22    cur = ggml_permute(ctx0, cur, 2, 0, 1, 3);
23    cur = ggml_cont(ctx0, cur);
24
25    // Conv2d(1152->2304, k=2, s=2) + GELU + Conv2d(2304->4608, k=1, s=1)
26    cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, merge, merge, 0, 0, 1, 1);
27    if (model.mm_0_b) {
28        cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_0_b, 1, 1, model.mm_0_b->ne[0]));
29    }
30    cur = ggml_gelu(ctx0, cur);
31    cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 1, 1, 0, 0, 1, 1);
32    if (model.mm_1_b) {
33        cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_1_b, 1, 1, model.mm_1_b->ne[0]));
34    }
35
36    const int ow   = pw / merge;
37    const int oh   = ph / merge;
38    const int idim = (int)cur->ne[2]; // OC = 4608
39
40    // append newline along W (dim 0)
41    ggml_tensor * nl = ggml_reshape_4d(ctx0, model.image_newline, 1, 1, idim, 1);
42    nl = ggml_repeat_4d(ctx0, nl, 1, oh, idim, 1);
43    cur = ggml_concat(ctx0, cur, nl, 0);
44
45    // [OW+1, OH, OC] -> [OC, (OW+1)*OH]
46    cur = ggml_permute(ctx0, cur, 1, 2, 0, 3);
47    cur = ggml_cont_2d(ctx0, cur, idim, (ow + 1) * oh);
48
49    // project to LLM hidden size
50    cur = build_mm(model.mm_model_proj, cur);
51    if (model.mm_model_proj_b) {
52        cur = ggml_add(ctx0, cur, model.mm_model_proj_b);
53    }
54
55    // wrap with begin/end tokens
56    cur = ggml_concat(ctx0, ggml_reshape_2d(ctx0, model.mm_img_begin, model.mm_img_begin->ne[0], 1), cur, 1);
57    cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, model.mm_img_end, model.mm_img_end->ne[0], 1), 1);
58
59    cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
60
61    ggml_build_forward_expand(gf, cur);
62    return gf;
63}