char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: add mtmd_tokenize_from_parts() (#28250)733905474

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
576 B21 linesraw
1#pragma once
2
3#include "mtmd.h"
4
5#include <string>
6#include <vector>
7
8// !!! Internal header, to be used by mtmd and its unit tests only !!!
9
10#define MTMD_INTERNAL_HEADER
11
12// bitmap is null for text parts
13struct mtmd_internal_part {
14    std::string text;
15    const mtmd_bitmap * bitmap;
16    // only used for text parts
17    bool parse_special = false;
18};
19
20// [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part
21std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge);