char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon/peg : suppress incomplete escape sequences (#26780)48d22e295

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.1 KiB133 linesraw
1#include "tests.h"
2
3void test_json_parser(testing &t) {
4    // Test parsing a simple JSON object
5    t.test("simple JSON object parsing", [](testing &t) {
6        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
7
8        std::string    input = R"({"name": "test", "value": 42, "flag": true})";
9        common_peg_parse_context ctx(input);
10
11        auto result = json.parse(ctx);
12
13        t.assert_equal("result_is_success", true, result.success());
14        t.assert_equal("result_end", input.size(), result.end);
15    });
16
17    // Test parsing a JSON array with mixed types
18    t.test("JSON array with mixed types", [](testing &t) {
19        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
20
21        std::string    input = R"([1, "hello", true, null, 3.14])";
22        common_peg_parse_context ctx(input);
23
24        auto result = json.parse(ctx);
25
26        t.assert_equal("result_is_success", true, result.success());
27        t.assert_equal("result_end", input.size(), result.end);
28    });
29
30    // Test parsing nested JSON with objects and arrays
31    t.test("nested JSON with objects and arrays", [](testing &t) {
32        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
33
34        std::string input =
35            R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})";
36        common_peg_parse_context ctx(input);
37
38        auto result = json.parse(ctx);
39
40        t.assert_equal("result_is_success", true, result.success());
41        t.assert_equal("result_end", input.size(), result.end);
42    });
43
44    // Test need_more_input() parsing - incomplete object
45    t.test("need_more_input() parsing - incomplete object", [](testing &t) {
46        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
47
48        std::string    input = R"({"name": "test", "value": )";
49        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
50
51        auto result = json.parse(ctx);
52
53        t.assert_equal("result_is_need_more_input", true, result.need_more_input());
54    });
55
56    // Test need_more_input() parsing - incomplete array
57    t.test("need_more_input() parsing - incomplete array", [](testing &t) {
58        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
59
60        std::string    input = R"([1, 2, 3, )";
61        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
62
63        auto result = json.parse(ctx);
64
65        t.assert_equal("result_is_need_more_input", true, result.need_more_input());
66    });
67
68    // Test need_more_input() parsing - incomplete nested structure
69    t.test("need_more_input() parsing - incomplete nested structure", [](testing &t) {
70        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
71
72        std::string    input = R"({"data": {"nested": )";
73        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
74
75        auto result = json.parse(ctx);
76
77        t.assert_equal("result_is_need_more_input", true, result.need_more_input());
78    });
79
80    // Test need_more_input() parsing - incomplete escape sequence in a string value
81    t.test("need_more_input() parsing - incomplete escape sequence", [](testing &t) {
82        auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
83
84        std::vector<std::string> inputs {
85            R"({"text": "hello\)",     // dangling backslash
86            R"({"text": "hello\u)",    // incomplete unicode escape sequence
87            R"({"text": "hello\u00)",
88        };
89
90        for (const auto & input : inputs) {
91            t.test(input, [&](testing &t) {
92                common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
93
94                auto result = json.parse(ctx);
95
96                t.assert_equal("result_is_need_more_input", true, result.need_more_input());
97
98                // the incomplete escape sequence is not part of the partial value
99                t.assert_equal("result_end", input.find('\\'), result.end);
100            });
101        }
102    });
103
104    t.test("object member", [](testing &t) {
105        auto parser = build_peg_parser([](common_peg_parser_builder & p) {
106            return p.json_member("name", "\"" + p.chars("[a-z]") + "\"");
107        });
108
109        t.test("success", [&](testing &t) {
110            std::string input = R"("name": "bob")";
111            common_peg_parse_context ctx(input);
112
113            auto result = parser.parse(ctx);
114            t.assert_true("success", result.success());
115        });
116
117        t.test("partial", [&](testing &t) {
118            std::string input = R"("name": "bo)";
119            common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
120
121            auto result = parser.parse(ctx);
122            t.assert_true("need more input", result.need_more_input());
123        });
124
125        t.test("failed", [&](testing &t) {
126            std::string input = R"([])";
127            common_peg_parse_context ctx(input);
128
129            auto result = parser.parse(ctx);
130            t.assert_true("fail", result.fail());
131        });
132    });
133}