char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Thiago Padilhaunicode : include '~' in collapsed symbol class (#26972)afd439df1

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
589 B24 linesraw
1#include "../src/unicode.h"
2
3#include <cstdio>
4#include <string>
5#include <vector>
6
7int main() {
8    const std::vector<std::string> regex_exprs = {
9        "[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
10    };
11    const std::vector<std::string> expected = { " ~", "foo" };
12    const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
13
14    if (actual != expected) {
15        fprintf(stderr, "unexpected split:");
16        for (const auto & piece : actual) {
17            fprintf(stderr, " [%s]", piece.c_str());
18        }
19        fprintf(stderr, "\n");
20        return 1;
21    }
22
23    return 0;
24}