char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

0jinja : fix quadratic cost in gather_string_parts (#27034)9e40df63b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.7 KiB61 linesraw
1#pragma once
2
3#include <optional>
4#include <string>
5#include <vector>
6
7#include "utils.h"
8
9namespace jinja {
10
11// allow differentiate between user input strings and template strings
12// transformations should handle this information as follows:
13// - one-to-one (e.g., uppercase, lowercase): preserve is_input flag
14// - one-to-many (e.g., strip): if input string is marked as is_input, all resulting parts should be marked as is_input
15// - many-to-one (e.g., concat): if ALL input parts are marked as is_input, resulting part should be marked as is_input
16struct string_part {
17    bool is_input = false; // may skip parsing special tokens if true
18    std::string val;
19
20    bool is_uppercase() const;
21    bool is_lowercase() const;
22};
23
24struct string {
25    std::vector<string_part> parts;
26    string() = default;
27    string(const std::string & v, bool user_input = false) {
28        parts.push_back({user_input, v});
29    }
30    string(int v) {
31        parts.push_back({false, std::to_string(v)});
32    }
33    string(double v) {
34        parts.push_back({false, std::to_string(v)});
35    }
36
37    // mark all parts as user input
38    void mark_input();
39
40    std::string str() const;
41    size_t length() const;
42    void hash_update(hasher & hash) const noexcept;
43    bool all_parts_are_input() const;
44    bool is_uppercase() const;
45    bool is_lowercase() const;
46
47    // mark this string as input if other has ALL parts as input
48    void mark_input_based_on(const string & other);
49
50    string & append(const string & other);
51
52    // in-place transformations
53
54    string uppercase();
55    string lowercase();
56    string capitalize();
57    string titlecase();
58    string strip(bool left, bool right, std::optional<const std::string_view> chars = std::nullopt);
59};
60
61} // namespace jinja