char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon: consolidate PEG string parsers (#20263)c96f608d9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs β†’
12.8 KiB318 linesraw
1#include "tests.h"
2
3void test_python_dict_parser(testing &t) {
4    // Test parsing a simple Python dict object with single quotes
5    t.test("simple Python dict object parsing", [](testing &t) {
6        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
7
8        std::string    input = "{'name': 'test', 'value': 42, 'flag': True}";
9        common_peg_parse_context ctx(input);
10
11        auto result = parser.parse(ctx);
12
13        t.assert_equal("result_is_success", true, result.success());
14        t.assert_equal("result_end", input.size(), result.end);
15    });
16
17    // Test parsing a Python array with mixed types
18    t.test("Python array with mixed types", [](testing &t) {
19        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
20
21        std::string    input = "[1, 'hello', True, None, 3.14]";
22        common_peg_parse_context ctx(input);
23
24        auto result = parser.parse(ctx);
25
26        t.assert_equal("result_is_success", true, result.success());
27        t.assert_equal("result_end", input.size(), result.end);
28    });
29
30    // Test parsing nested Python dict with objects and arrays
31    t.test("nested Python dict with objects and arrays", [](testing &t) {
32        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
33
34        std::string input =
35            "{'users': [{'id': 1, 'name': 'Alice'}, {'id': 2, 'name': 'Bob'}], 'count': 2, 'metadata': {'version': '1.0', 'tags': ['admin', 'user']}}";
36        common_peg_parse_context ctx(input);
37
38        auto result = parser.parse(ctx);
39
40        t.assert_equal("result_is_success", true, result.success());
41        t.assert_equal("result_end", input.size(), result.end);
42    });
43
44    // Test parsing Python dict with escaped single quotes
45    t.test("Python dict with escaped single quotes", [](testing &t) {
46        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
47
48        std::string    input = "{'message': 'It\\'s working!'}";
49        common_peg_parse_context ctx(input);
50
51        auto result = parser.parse(ctx);
52
53        t.assert_equal("result_is_success", true, result.success());
54        t.assert_equal("result_end", input.size(), result.end);
55    });
56
57    // Test parsing Python dict with double quotes inside single quotes
58    t.test("Python dict with double quotes inside single quotes", [](testing &t) {
59        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
60
61        std::string    input = "{'quote': 'He said \"Hello\"'}";
62        common_peg_parse_context ctx(input);
63
64        auto result = parser.parse(ctx);
65
66        t.assert_equal("result_is_success", true, result.success());
67        t.assert_equal("result_end", input.size(), result.end);
68    });
69
70    // Test the example from the requirements
71    t.test("complex Python dict example from requirements", [](testing &t) {
72        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
73
74        std::string    input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
75        common_peg_parse_context ctx(input);
76
77        auto result = parser.parse(ctx);
78
79        t.assert_equal("result_is_success", true, result.success());
80        t.assert_equal("result_end", input.size(), result.end);
81    });
82
83    // Test need_more_input() parsing - incomplete object
84    t.test("need_more_input() parsing - incomplete object", [](testing &t) {
85        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
86
87        std::string    input = "{'name': 'test', 'value': ";
88        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
89
90        auto result = parser.parse(ctx);
91
92        t.assert_equal("result_is_need_more_input", true, result.need_more_input());
93    });
94
95    // Test need_more_input() parsing - incomplete single-quoted string
96    t.test("need_more_input() parsing - incomplete single-quoted string", [](testing &t) {
97        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
98
99        std::string    input = "{'name': 'test";
100        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
101
102        auto result = parser.parse(ctx);
103
104        t.assert_equal("result_is_need_more_input", true, result.need_more_input());
105    });
106
107    // Test unicode in Python dict strings
108    t.test("unicode in Python dict strings", [](testing &t) {
109        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
110
111        std::string    input = "{'message': 'Hello, δΈ–η•Œ!'}";
112        common_peg_parse_context ctx(input);
113
114        auto result = parser.parse(ctx);
115
116        t.assert_equal("result_is_success", true, result.success());
117        t.assert_equal("result_end", input.size(), result.end);
118    });
119
120    // Test Python dict with unicode escapes
121    t.test("Python dict with unicode escapes", [](testing &t) {
122        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
123
124        std::string    input = "{'unicode': 'Hello\\u0041'}";
125        common_peg_parse_context ctx(input);
126
127        auto result = parser.parse(ctx);
128
129        t.assert_equal("result_is_success", true, result.success());
130        t.assert_equal("result_end", input.size(), result.end);
131    });
132
133    // Test that Python parser accepts double-quoted strings too
134    t.test("Python parser accepts double-quoted strings", [](testing &t) {
135        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
136
137        std::string    input = "{\"name\": \"test\"}";
138        common_peg_parse_context ctx(input);
139
140        auto result = parser.parse(ctx);
141
142        t.assert_equal("result_is_success", true, result.success());
143        t.assert_equal("result_end", input.size(), result.end);
144    });
145
146    // Test Python parser with mixed quote styles
147    t.test("Python parser with mixed quote styles", [](testing &t) {
148        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
149
150        std::string    input = "{\"name\": 'test', 'value': \"hello\"}";
151        common_peg_parse_context ctx(input);
152
153        auto result = parser.parse(ctx);
154
155        t.assert_equal("result_is_success", true, result.success());
156        t.assert_equal("result_end", input.size(), result.end);
157    });
158
159    // Test Python True/False/None
160    t.test("Python True/False/None", [](testing &t) {
161        auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
162
163        t.test("True", [&](testing &t) {
164            std::string input = "True";
165            common_peg_parse_context ctx(input);
166            auto result = parser.parse(ctx);
167            t.assert_true("success", result.success());
168            t.assert_equal("end", input.size(), result.end);
169        });
170
171        t.test("False", [&](testing &t) {
172            std::string input = "False";
173            common_peg_parse_context ctx(input);
174            auto result = parser.parse(ctx);
175            t.assert_true("success", result.success());
176            t.assert_equal("end", input.size(), result.end);
177        });
178
179        t.test("None", [&](testing &t) {
180            std::string input = "None";
181            common_peg_parse_context ctx(input);
182            auto result = parser.parse(ctx);
183            t.assert_true("success", result.success());
184            t.assert_equal("end", input.size(), result.end);
185        });
186
187        t.test("rejects JSON-style true/false/null", [&](testing &t) {
188            for (const auto & kw : {"true", "false", "null"}) {
189                std::string input = kw;
190                common_peg_parse_context ctx(input);
191                auto result = parser.parse(ctx);
192                t.assert_true(std::string("rejects ") + kw, result.fail());
193            }
194        });
195    });
196
197    // Test single-quoted string content parser directly
198    t.test("single-quoted string content parser", [](testing &t) {
199        auto parser = build_peg_parser([](common_peg_parser_builder & p) {
200            return p.sequence({ p.literal("'"), p.string_content('\''), p.literal("'"), p.space() });
201        });
202
203        t.test("simple string", [&](testing &t) {
204            std::string input = "'hello'";
205            common_peg_parse_context ctx(input);
206
207            auto result = parser.parse(ctx);
208            t.assert_true("success", result.success());
209            t.assert_equal("end", input.size(), result.end);
210        });
211
212        t.test("string with escaped single quote", [&](testing &t) {
213            std::string input = "'it\\'s'";
214            common_peg_parse_context ctx(input);
215
216            auto result = parser.parse(ctx);
217            t.assert_true("success", result.success());
218            t.assert_equal("end", input.size(), result.end);
219        });
220
221        t.test("string with double quotes", [&](testing &t) {
222            std::string input = "'say \"hello\"'";
223            common_peg_parse_context ctx(input);
224
225            auto result = parser.parse(ctx);
226            t.assert_true("success", result.success());
227            t.assert_equal("end", input.size(), result.end);
228        });
229
230        t.test("incomplete string", [&](testing &t) {
231            std::string input = "'hello";
232            common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
233
234            auto result = parser.parse(ctx);
235            t.assert_true("need_more_input", result.need_more_input());
236        });
237    });
238
239    // Test json() with pre-registered flexible json-string rule (python dict support)
240    t.test("json() parser with flexible json-string rule", [](testing &t) {
241        t.test("json() rejects single quotes by default", [&](testing &t) {
242            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
243                return p.json();
244            });
245
246            std::string input = "{'name': 'test'}";
247            common_peg_parse_context ctx(input);
248
249            auto result = parser.parse(ctx);
250            t.assert_true("fail", result.fail());
251        });
252
253        t.test("json() accepts single quotes with pre-registered flexible json-string rule", [&](testing &t) {
254            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
255                // Pre-register json-string rule with both quote styles
256                p.rule("json-string", [&]() {
257                    return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
258                });
259                return p.json();
260            });
261
262            std::string input = "{'name': 'test'}";
263            common_peg_parse_context ctx(input);
264
265            auto result = parser.parse(ctx);
266            t.assert_true("success", result.success());
267            t.assert_equal("end", input.size(), result.end);
268        });
269
270        t.test("json() still accepts double quotes with flexible json-string rule", [&](testing &t) {
271            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
272                p.rule("json-string", [&]() {
273                    return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
274                });
275                return p.json();
276            });
277
278            std::string input = "{\"name\": \"test\"}";
279            common_peg_parse_context ctx(input);
280
281            auto result = parser.parse(ctx);
282            t.assert_true("success", result.success());
283            t.assert_equal("end", input.size(), result.end);
284        });
285
286        t.test("json() accepts mixed quote styles with flexible json-string rule", [&](testing &t) {
287            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
288                p.rule("json-string", [&]() {
289                    return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
290                });
291                return p.json();
292            });
293
294            std::string input = "{\"name\": 'test', 'value': \"hello\"}";
295            common_peg_parse_context ctx(input);
296
297            auto result = parser.parse(ctx);
298            t.assert_true("success", result.success());
299            t.assert_equal("end", input.size(), result.end);
300        });
301
302        t.test("complex nested structure with flexible json-string rule", [&](testing &t) {
303            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
304                p.rule("json-string", [&]() {
305                    return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
306                });
307                return p.json();
308            });
309
310            std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
311            common_peg_parse_context ctx(input);
312
313            auto result = parser.parse(ctx);
314            t.assert_true("success", result.success());
315            t.assert_equal("end", input.size(), result.end);
316        });
317    });
318}