char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan Son Nguyenllama : reduce compile time and binary size (#9712)a39ab216a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
630 B20 linesraw
1#pragma once
2
3#include <cstdint>
4#include <vector>
5#include <unordered_map>
6#include <unordered_set>
7
8struct range_nfd {
9    uint32_t first;
10    uint32_t last;
11    uint32_t nfd;
12};
13
14static const uint32_t MAX_CODEPOINTS = 0x110000;
15
16extern const std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;
17extern const std::unordered_set<uint32_t> unicode_set_whitespace;
18extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase;
19extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase;
20extern const std::initializer_list<range_nfd> unicode_ranges_nfd;