hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon : gracefully handle incomplete output (#20191)451ef08432026-03-08

1#include "peg-parser.h"
2#include "tests.h"
3
4void test_basic(testing & t) {
5    t.test("chars", [](testing & t) {
6        // Test common escape sequences - newline
7        t.test("escape_sequence_newline", [](testing &t) {
8            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
9
10            common_peg_parse_context ctx;
11            common_peg_parse_result  result;
12
13            ctx    = common_peg_parse_context("\n");
14            result = common_chat_combinator_parser.parse(ctx);
15            t.assert_equal("escape_sequence_newline", true, result.success());
16        });
17
18        // Test common escape sequences - tab
19        t.test("escape_sequence_tab", [](testing &t) {
20            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
21
22            common_peg_parse_context ctx;
23            common_peg_parse_result  result;
24
25            ctx    = common_peg_parse_context("\t");
26            result = common_chat_combinator_parser.parse(ctx);
27            t.assert_equal("escape_sequence_tab", true, result.success());
28        });
29
30        // Test common escape sequences - backslash
31        t.test("escape_sequence_backslash", [](testing &t) {
32            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
33
34            common_peg_parse_context ctx;
35            common_peg_parse_result  result;
36
37            ctx    = common_peg_parse_context("\\");
38            result = common_chat_combinator_parser.parse(ctx);
39            t.assert_equal("escape_sequence_backslash", true, result.success());
40        });
41
42        // Test common escape sequences - space (should ())
43        t.test("escape_sequence_space_fail", [](testing &t) {
44            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
45
46            common_peg_parse_context ctx;
47            common_peg_parse_result  result;
48
49            ctx    = common_peg_parse_context(" ");
50            result = common_chat_combinator_parser.parse(ctx);
51            t.assert_equal("escape_sequence_space_fail", true, result.fail());
52        });
53
54        // Test escaped dash - 'a' should succeed
55        t.test("escaped_dash_a", [](testing &t) {
56            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
57
58            common_peg_parse_context ctx;
59            common_peg_parse_result  result;
60
61            ctx    = common_peg_parse_context("a");
62            result = common_chat_combinator_parser.parse(ctx);
63            t.assert_equal("escaped_dash_a", true, result.success());
64        });
65
66        // Test escaped dash - '-' should succeed (literal dash)
67        t.test("escaped_dash_literal", [](testing &t) {
68            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
69
70            common_peg_parse_context ctx;
71            common_peg_parse_result  result;
72
73            ctx    = common_peg_parse_context("-");
74            result = common_chat_combinator_parser.parse(ctx);
75            t.assert_equal("escaped_dash_literal", true, result.success());
76        });
77
78        // Test escaped dash - 'z' should succeed
79        t.test("escaped_dash_z", [](testing &t) {
80            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
81
82            common_peg_parse_context ctx;
83            common_peg_parse_result  result;
84
85            ctx    = common_peg_parse_context("z");
86            result = common_chat_combinator_parser.parse(ctx);
87            t.assert_equal("escaped_dash_z", true, result.success());
88        });
89
90        // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range)
91        t.test("escaped_dash_b_fail", [](testing &t) {
92            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
93
94            common_peg_parse_context ctx;
95            common_peg_parse_result  result;
96
97            ctx    = common_peg_parse_context("b");
98            result = common_chat_combinator_parser.parse(ctx);
99            t.assert_equal("escaped_dash_b_fail", true, result.fail());
100        });
101    });
102
103
104    t.test("optional", [](testing & t) {
105        // Full match with optional part present
106        t.test("optional_present", [](testing &t) {
107            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
108                return p.literal("hello") + p.optional(p.literal(" world"));
109            });
110
111            auto ctx    = common_peg_parse_context("hello world");
112            auto result = parser.parse(ctx);
113            t.assert_equal("optional_present", true, result.success());
114            t.assert_equal("optional_present_end", 11u, result.end);
115        });
116
117        // Full match with optional part absent
118        t.test("optional_absent", [](testing &t) {
119            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
120                return p.literal("hello") + p.optional(p.literal(" world"));
121            });
122
123            auto ctx    = common_peg_parse_context("hello");
124            auto result = parser.parse(ctx);
125            t.assert_equal("optional_absent", true, result.success());
126            t.assert_equal("optional_absent_end", 5u, result.end);
127        });
128
129        // Partial match - waiting for more input to determine if optional matches
130        t.test("partial_match_need_more", [](testing &t) {
131            auto parser = build_peg_parser([](common_peg_parser_builder & p) {
132                return p.literal("hello") + p.optional(p.literal(" world"));
133            });
134
135            auto ctx    = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT);
136            auto result = parser.parse(ctx);
137            t.assert_equal("partial_match_need_more", true, result.need_more_input());
138        });
139    });
140
141    t.test("partial parsing", [](testing & t) {
142        // Literals - Basic Success
143        t.test("literal_success", [&](testing & t) {
144            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); });
145
146            common_peg_parse_context ctx;
147            common_peg_parse_result  result;
148
149            ctx    = common_peg_parse_context("hello");
150            result = parser.parse(ctx);
151            t.assert_equal("literal_success", true, result.success());
152        });
153
154        // Char Classes - Basic Lowercase Success
155        t.test("char_class_lowercase_success", [&](testing & t) {
156            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
157
158            common_peg_parse_context ctx;
159            common_peg_parse_result  result;
160
161            ctx    = common_peg_parse_context("a");
162            result = parser.parse(ctx);
163            t.assert_equal("char_class_lowercase_success", true, result.success());
164        });
165
166        // Char Classes - Uppercase Fail
167        t.test("char_class_uppercase_fail", [&](testing & t) {
168            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
169
170            common_peg_parse_context ctx;
171            common_peg_parse_result  result;
172
173            ctx    = common_peg_parse_context("A");
174            result = parser.parse(ctx);
175            t.assert_equal("char_class_uppercase_fail", true, result.fail());
176        });
177
178        // Char Classes with Dash - Lowercase Success
179        t.test("char_class_with_dash_lowercase", [&](testing & t) {
180            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
181
182            common_peg_parse_context ctx;
183            common_peg_parse_result  result;
184
185            ctx    = common_peg_parse_context("f");
186            result = parser.parse(ctx);
187            t.assert_equal("char_class_with_dash_lowercase", true, result.success());
188        });
189
190        // Char Classes with Dash - Literal Dash Success
191        t.test("char_class_with_dash_literal_dash", [&](testing & t) {
192            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
193
194            common_peg_parse_context ctx;
195            common_peg_parse_result  result;
196
197            ctx    = common_peg_parse_context("-");
198            result = parser.parse(ctx);
199            t.assert_equal("char_class_with_dash_literal_dash", true, result.success());
200        });
201
202        // Char Classes with Dash - Uppercase Fail
203        t.test("char_class_with_dash_uppercase_fail", [&](testing & t) {
204            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
205
206            common_peg_parse_context ctx;
207            common_peg_parse_result  result;
208
209            ctx    = common_peg_parse_context("A");
210            result = parser.parse(ctx);
211            t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail());
212        });
213
214        // Sequences - Partial Match 1
215        t.test("sequence_partial_match_1", [&](testing & t) {
216            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
217
218            auto ctx    = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT);
219            auto result = parser.parse(ctx);
220            t.assert_equal("sequence_partial_match_1", true, result.need_more_input());
221        });
222
223        // Sequences - Partial Match 2
224        t.test("sequence_partial_match_2", [&](testing & t) {
225            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });
226
227            auto ctx    = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT);
228            auto result = parser.parse(ctx);
229            t.assert_equal("sequence_partial_match_2", true, result.need_more_input());
230        });
231
232        // Sequences - Partial Match 3
233        t.test("sequence_partial_match_3", [&](testing & t) {
234            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
235
236            auto ctx    = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT);
237            auto result = parser.parse(ctx);
238            t.assert_equal("sequence_partial_match_3", true, result.need_more_input());
239        });
240
241        // Sequences - Full Match
242        t.test("sequence_full_match", [&](testing & t) {
243            auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });
244
245            auto ctx    = common_peg_parse_context("helloworld");
246            auto result = common_chat_combinator_parser.parse(ctx);
247            t.assert_equal("sequence_full_match", true, result.success());
248        });
249
250        // Sequences - No Match
251        t.test("sequence_no_match", [&](testing & t) {
252            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
253
254            auto ctx    = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT);
255            auto result = parser.parse(ctx);
256            t.assert_equal("sequence_no_match", true, result.fail());
257        });
258
259        // Choices - Partial Match 1
260        t.test("choices_partial_match_1", [&](testing & t) {
261            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });
262
263            auto ctx    = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT);
264            auto result = parser.parse(ctx);
265            t.assert_equal("choices_partial_match_1", true, result.need_more_input());
266        });
267
268        // Choices - Partial Match 2
269        t.test("choices_partial_match_2", [&](testing & t) {
270            auto parser =
271                build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });
272
273            auto ctx    = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT);
274            auto result = parser.parse(ctx);
275            t.assert_equal("choices_partial_match_2", true, result.need_more_input());
276        });
277
278        // Choices - Full Match 1
279        t.test("choices_full_match_1", [&](testing & t) {
280            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });
281
282            auto ctx    = common_peg_parse_context("first");
283            auto result = parser.parse(ctx);
284            t.assert_equal("choices_full_match_1", true, result.success());
285        });
286
287        // Choices - Full Match 2
288        t.test("choices_full_match_2", [&](testing & t) {
289            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });
290
291            auto ctx    = common_peg_parse_context("beta");
292            auto result = parser.parse(ctx);
293            t.assert_equal("choices_full_match_2", true, result.success());
294        });
295
296        // Choices - No Match
297        t.test("choices_no_match", [&](testing & t) {
298            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });
299
300            auto ctx    = common_peg_parse_context("best");
301            auto result = parser.parse(ctx);
302            t.assert_equal("choices_no_match", true, result.fail());
303        });
304
305        // Zero or More - Partial Match 1
306        t.test("zero_or_more_partial_match_1", [&](testing & t) {
307            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });
308
309            auto ctx    = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT);
310            auto result = parser.parse(ctx);
311            t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input());
312        });
313
314        // Zero or More - Partial Match 2
315        t.test("zero_or_more_partial_match_2", [&](testing & t) {
316            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });
317
318            auto ctx    = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT);
319            auto result = parser.parse(ctx);
320            t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input());
321        });
322
323        // Zero or More - Full Match
324        t.test("zero_or_more_full_match", [&](testing & t) {
325            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });
326
327            auto ctx    = common_peg_parse_context("test");
328            auto result = parser.parse(ctx);
329            t.assert_equal("zero_or_more_full_match", true, result.success());
330        });
331
332        // One or More - Partial Match 1
333        t.test("one_or_more_partial_match_1", [&](testing & t) {
334            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });
335
336            auto ctx    = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT);
337            auto result = parser.parse(ctx);
338            t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input());
339        });
340
341        // One or More - Partial Match 2
342        t.test("one_or_more_partial_match_2", [&](testing & t) {
343            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });
344
345            auto ctx    = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT);
346            auto result = parser.parse(ctx);
347            t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input());
348        });
349
350        // One or More - Full Match
351        t.test("one_or_more_full_match", [&](testing & t) {
352            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });
353
354            auto ctx    = common_peg_parse_context("single");
355            auto result = parser.parse(ctx);
356            t.assert_equal("one_or_more_full_match", true, result.success());
357        });
358
359        // One or More - No Match
360        t.test("one_or_more_no_match", [&](testing & t) {
361            auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });
362
363            auto ctx    = common_peg_parse_context("success");
364            auto result = parser.parse(ctx);
365            t.assert_equal("one_or_more_no_match", true, result.fail());
366        });
367    });
368
369
370    t.test("recursive rules", [](testing &t) {
371        // Test simple number
372        t.test("simple_number", [](testing &t) {
373            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
374                p.rule("number", p.chars("0-9"));
375                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
376                return p.rule("value", p.ref("number") | p.ref("list"));
377            });
378
379            common_peg_parse_context ctx("1");
380            auto           result = value_parser.parse(ctx);
381
382            t.assert_equal("result_is_success", true, result.success());
383        });
384
385        // Test simple list
386        t.test("simple_list", [](testing &t) {
387            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
388                p.rule("number", p.chars("0-9"));
389                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
390                return p.rule("value", p.ref("number") | p.ref("list"));
391            });
392
393            common_peg_parse_context ctx("[1]");
394            auto           result = value_parser.parse(ctx);
395
396            t.assert_equal("result_is_success", true, result.success());
397        });
398
399        // Test nested list
400        t.test("nested_list", [](testing &t) {
401            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
402                p.rule("number", p.chars("0-9"));
403                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
404                return p.rule("value", p.ref("number") | p.ref("list"));
405            });
406
407            common_peg_parse_context ctx("[[2]]");
408            auto           result = value_parser.parse(ctx);
409
410            t.assert_equal("result_is_success", true, result.success());
411        });
412
413        // Test deeply nested list
414        t.test("deeply_nested_list", [](testing &t) {
415            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
416                p.rule("number", p.chars("0-9"));
417                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
418                return p.rule("value", p.ref("number") | p.ref("list"));
419            });
420
421            common_peg_parse_context ctx("[[[3]]]");
422            auto           result = value_parser.parse(ctx);
423
424            t.assert_equal("result_is_success", true, result.success());
425        });
426
427        // Test need_more_input match
428        t.test("need_more_input_match", [](testing &t) {
429            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
430                p.rule("number", p.chars("0-9"));
431                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
432                return p.rule("value", p.ref("number") | p.ref("list"));
433            });
434
435            common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT);
436            auto           result = value_parser.parse(ctx);
437
438            t.assert_equal("result_is_need_more_input", true, result.need_more_input());
439        });
440
441        // Test no match
442        t.test("no_match", [](testing &t) {
443            auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
444                p.rule("number", p.chars("0-9"));
445                p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
446                return p.rule("value", p.ref("number") | p.ref("list"));
447            });
448
449            common_peg_parse_context ctx("[a]");
450            auto           result = value_parser.parse(ctx);
451
452            t.assert_equal("result_is_fail", true, result.fail());
453        });
454
455        // Test markers
456        t.test("marker", [](testing &t) {
457            auto bracket_parser = build_peg_parser([](common_peg_parser_builder & p) {
458                return p.marker();
459            });
460
461            common_peg_parse_context ctx_square("[marker]");
462            common_peg_parse_context ctx_sharp("<marker>");
463
464            auto result_square = bracket_parser.parse(ctx_square);
465            auto result_sharp = bracket_parser.parse(ctx_sharp);
466
467            t.assert_true("result_square_is_success", result_square.success());
468            t.assert_true("result_sharp_is_success", result_sharp.success());
469        });
470    });
471}