char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Piotr Wilkin (ilintar)common/parser: handle reasoning budget (#20297)acb7c7906

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.1 KiB124 linesraw
1#include "unicode.h"
2
3#include <algorithm>
4#include <cassert>
5#include <stdexcept>
6#include <string>
7#include <vector>
8
9// implementation adopted from src/unicode.cpp
10
11size_t common_utf8_sequence_length(unsigned char first_byte) {
12    const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
13    uint8_t highbits = static_cast<uint8_t>(first_byte) >> 4;
14    return lookup[highbits];
15}
16
17utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t offset) {
18    if (offset >= input.size()) {
19        return utf8_parse_result(utf8_parse_result::INCOMPLETE);
20    }
21
22    // ASCII fast path
23    if (!(input[offset] & 0x80)) {
24        return utf8_parse_result(utf8_parse_result::SUCCESS, input[offset], 1);
25    }
26
27    // Invalid: continuation byte as first byte
28    if (!(input[offset] & 0x40)) {
29        return utf8_parse_result(utf8_parse_result::INVALID);
30    }
31
32    // 2-byte sequence
33    if (!(input[offset] & 0x20)) {
34        if (offset + 1 >= input.size()) {
35            return utf8_parse_result(utf8_parse_result::INCOMPLETE);
36        }
37        if ((input[offset + 1] & 0xc0) != 0x80) {
38            return utf8_parse_result(utf8_parse_result::INVALID);
39        }
40        auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f);
41        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2);
42    }
43
44    // 3-byte sequence
45    if (!(input[offset] & 0x10)) {
46        if (offset + 2 >= input.size()) {
47            return utf8_parse_result(utf8_parse_result::INCOMPLETE);
48        }
49        if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) {
50            return utf8_parse_result(utf8_parse_result::INVALID);
51        }
52        auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f);
53        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3);
54    }
55
56    // 4-byte sequence
57    if (!(input[offset] & 0x08)) {
58        if (offset + 3 >= input.size()) {
59            return utf8_parse_result(utf8_parse_result::INCOMPLETE);
60        }
61        if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) {
62            return utf8_parse_result(utf8_parse_result::INVALID);
63        }
64        auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f);
65        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4);
66    }
67
68    // Invalid first byte
69    return utf8_parse_result(utf8_parse_result::INVALID);
70}
71
72bool common_utf8_is_complete(const std::string & s) {
73    if (s.empty()) {
74        return true;
75    }
76    for (int i = 1; i <= std::min(4, (int)s.size()); i++) {
77        unsigned char c = s[s.size() - i];
78        if ((c & 0xC0) != 0x80) {
79            int expected = (c >= 0xF0) ? 4 : (c >= 0xE0) ? 3 : (c >= 0xC0) ? 2 : 1;
80            return i >= expected;
81        }
82    }
83    return false;
84}
85
86std::string common_unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {
87    std::string result;
88    for (size_t i = 0; i < cps.size(); ++i) {
89        result.append(common_unicode_cpt_to_utf8(cps[i]));
90    }
91    return result;
92}
93
94std::string common_unicode_cpt_to_utf8(uint32_t cpt) {
95    std::string result;
96
97    if (/* 0x00 <= cpt && */ cpt <= 0x7f) {
98        result.push_back(cpt);
99        return result;
100    }
101    if (0x80 <= cpt && cpt <= 0x7ff) {
102        result.push_back(0xc0 | ((cpt >> 6) & 0x1f));
103        result.push_back(0x80 | (cpt & 0x3f));
104        return result;
105    }
106    if (0x800 <= cpt && cpt <= 0xffff) {
107        result.push_back(0xe0 | ((cpt >> 12) & 0x0f));
108        result.push_back(0x80 | ((cpt >> 6) & 0x3f));
109        result.push_back(0x80 | (cpt & 0x3f));
110        return result;
111    }
112    if (0x10000 <= cpt && cpt <= 0x10ffff) {
113        result.push_back(0xf0 | ((cpt >> 18) & 0x07));
114        result.push_back(0x80 | ((cpt >> 12) & 0x3f));
115        result.push_back(0x80 | ((cpt >> 6) & 0x3f));
116        result.push_back(0x80 | (cpt & 0x3f));
117        return result;
118    }
119
120    throw std::invalid_argument("invalid codepoint");
121}
122
123
124