char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Anav Prasadmtmd : Add Nemotron Nano 12B v2 VL support (#19547)01d8eaa28

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.1 KiB35 linesraw
1#include "models.h"
2
3ggml_cgraph * clip_graph_nemotron_v2_vl::build() {
4    GGML_ASSERT(model.class_embedding != nullptr);
5    GGML_ASSERT(model.position_embeddings != nullptr);
6
7    const int n_registers = model.class_embedding->ne[1];
8    const int n_pos = n_patches + n_registers;
9
10    ggml_tensor * inp = build_inp();
11
12    // add position embeddings (pre-downsampled during GGUF conversion for fixed 512x512 input)
13    inp = ggml_add(ctx0, inp, model.position_embeddings);
14    cb(inp, "inp_pos", -1);
15
16    inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
17
18    ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
19
20    cur = ggml_view_2d(ctx0, cur,
21        n_embd, n_patches,
22        ggml_row_size(cur->type, n_embd),
23        n_registers * ggml_row_size(cur->type, n_embd));
24
25    cur = build_patch_merge_permute(cur, model.hparams.n_merge);
26
27    {
28        cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-6, -1);
29        cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
30    }
31
32    ggml_build_forward_expand(gf, cur);
33
34    return gf;
35}