char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon : introduce composable PEG parser combinators for chat parsing (#17136)0a8026e76

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
863 B37 linesraw
1#include "simple-tokenize.h"
2
3std::vector<std::string> simple_tokenize(const std::string & input) {
4    std::vector<std::string> result;
5    std::string              current;
6
7    for (size_t i = 0; i < input.size(); i++) {
8        switch (input[i]) {
9            case ' ':
10            case '\n':
11            case '\t':
12            case '{':
13            case '}':
14            case ',':
15            case '[':
16            case '"':
17            case ']':
18            case '.':
19            case '<':
20            case '>':
21            case '=':
22            case '/':
23                if (!current.empty()) {
24                    result.push_back(current);
25                    current.clear();
26                }
27            default:;
28        }
29        current += input[i];
30    }
31
32    if (!current.empty()) {
33        result.push_back(current);
34    }
35
36    return result;
37}