char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon : implement common_schema internal representation for JSON schemas (#28736)acecd5603

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
22.1 KiB478 linesraw
1#include "chat-auto-parser-helpers.h"
2#include "chat-auto-parser.h"
3#include "chat-peg-parser.h"
4#include "chat.h"
5#include "common.h"
6#include "json-schema-to-grammar.h"
7#include "log.h"
8#include "parsers/parsers.h"
9#include "peg-parser.h"
10
11#include <stdexcept>
12#include <string>
13
14using json = common_json;
15
16namespace autoparser {
17
18parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
19    p(p),
20    inputs(inputs),
21    reasoning_parser(p.eps()) {}
22
23common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,
24                                                  const struct generation_params & inputs) {
25    // Run differential analysis to extract template structure
26    struct autoparser autoparser;
27    autoparser.analyze_template(tmpl);
28    return generate_parser(tmpl, inputs, autoparser);
29}
30
31common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,
32                                                  const struct generation_params & inputs,
33                                                  const autoparser &              autoparser) {
34    // Create the result structure
35    common_chat_params data;
36    data.prompt            = common_chat_template_direct_apply(tmpl, inputs);
37    data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs);
38    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
39    data.preserved_tokens  = autoparser.preserved_tokens;
40    data.additional_stops.insert(data.additional_stops.end(),
41        autoparser.additional_stops.begin(), autoparser.additional_stops.end());
42
43    std::string parser_generation_prompt = data.generation_prompt;
44
45    if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !inputs.continue_msg.empty()) {
46        // Build up generation prompt manually
47        const auto & msg = inputs.continue_msg;
48
49        if (!autoparser.reasoning.start.empty()) {
50            data.generation_prompt = data.generation_prompt.substr(0, data.generation_prompt.find(autoparser.reasoning.start));
51            data.generation_prompt += autoparser.reasoning.start + msg.reasoning_content;
52            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
53                data.generation_prompt += autoparser.reasoning.end;
54            }
55        }
56
57        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
58            data.generation_prompt += msg.render_content();
59        }
60
61        data.prompt += data.generation_prompt;
62    }
63
64    auto parser = autoparser.build_parser(inputs, parser_generation_prompt);
65    data.parser = parser.save();
66
67    // Build grammar if tools are present
68    bool has_tools =
69        autoparser.tools.format.mode != tool_format::NONE && inputs.tools.is_array() && !inputs.tools.empty();
70    std::string trigger_marker = !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start :
71                                                                                  autoparser.tools.format.per_call_start;
72
73    bool has_response_format = !inputs.json_schema.empty() && inputs.json_schema.is_object();
74    bool include_grammar = has_response_format || (has_tools &&
75            ((inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO && !trigger_marker.empty()) ||
76              inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
77
78    if (include_grammar) {
79        data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
80        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
81            parser.build_grammar(builder, data.grammar_lazy);
82        });
83
84        // Set grammar triggers based on tool section markers (fall back to per-call markers)
85        if (data.grammar_lazy) {
86            data.grammar_triggers = {
87                { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }
88            };
89            if (autoparser.tools.format.openai_wrapper_trigger) {
90                // model emits the OpenAI function wrapper, trigger on it
91                data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," });
92            }
93        }
94    }
95
96    return data;
97}
98
99common_peg_arena autoparser::build_parser(const generation_params & inputs, const std::string & generation_prompt) const {
100    if (!analysis_complete) {
101        throw std::invalid_argument("Cannot call build_parser on autoparser without performing analysis first, call analyze_template(...)");
102    }
103    return build_chat_peg_parser([&](common_chat_peg_builder & p) {
104        parser_build_context ctx(p, inputs);
105        bool                 extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
106
107        ctx.extracting_reasoning = extract_reasoning && reasoning.mode != reasoning_mode::NONE;
108        ctx.content              = &content;
109        ctx.reasoning            = &reasoning;
110
111        // Build reasoning parser
112        ctx.reasoning_parser = reasoning.build_parser(ctx);
113
114        auto parser = p.eps();
115
116        bool has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
117        bool has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
118        bool pure_content        = reasoning.mode == reasoning_mode::NONE;
119
120        if (has_response_format) {
121            auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
122            parser = ctx.reasoning_parser + p.space() + p.choice({
123                p.literal("```json") + p.space() + response_format + p.space() + p.literal("```"),
124                p.space() + response_format  + p.space()
125            }) + p.end();
126            pure_content = false;
127        } else if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && jinja_caps.supports_tool_calls) {
128            parser = tools.build_parser(ctx);
129            pure_content = false;
130        } else {
131            parser = content.build_parser(ctx);
132        }
133        const std::string reasoning_start = trim_whitespace(reasoning.start);
134        return pure_content ? p.prefix(generation_prompt, reasoning_start) + parser : p.prefix(generation_prompt, reasoning_start) << parser;
135    });
136}
137
138common_peg_parser analyze_reasoning::build_parser(parser_build_context & ctx) const {
139    auto & p = ctx.p;
140
141    if (!ctx.extracting_reasoning) {
142        return p.eps();
143    }
144
145    if (mode == reasoning_mode::TAG_BASED || mode == reasoning_mode::TOOLS_ONLY) {
146        if (!end.empty()) {
147            if (!start.empty()) {
148                // Standard tag-based: optional(<think>reasoning</think>)
149                return p.optional(p.optspace(start) + p.reasoning(p.until(trim_whitespace(end))) + p.optspace(end));
150            }
151            // Delimiter-style (empty start)
152            return p.optional(p.reasoning(p.until(trim_whitespace(end))) + p.optspace(end));
153        }
154    }
155
156    return p.eps();
157}
158
159common_peg_parser analyze_content::build_parser(parser_build_context & ctx) const {
160    auto & p = ctx.p;
161
162    if (is_always_wrapped()) {
163        if (ctx.extracting_reasoning) {
164            return ctx.reasoning_parser + start + p.content(p.until(end)) + end + p.end();
165        }
166        return p.content(p.until(start)) + start + p.content(p.until(end)) + end + p.end();
167    }
168    return ctx.reasoning_parser + p.content(p.rest()) + p.end();
169}
170
171common_peg_parser analyze_content::build_optional_wrapped(parser_build_context & ctx) const {
172    auto & p = ctx.p;
173
174    if (is_always_wrapped()) {
175        return p.optional(start + p.content(p.until(end)) + end);
176    }
177    return p.eps();
178}
179
180common_peg_parser analyze_tools::build_parser(parser_build_context & ctx) const {
181    switch (format.mode) {
182        case tool_format::JSON_NATIVE:
183            return build_tool_parser_json_native(ctx);
184        case tool_format::TAG_WITH_JSON:
185            return build_tool_parser_tag_json(ctx);
186        case tool_format::TAG_WITH_TAGGED:
187            return build_tool_parser_tag_tagged(ctx);
188        default:
189            LOG_ERR("[ERROR] Template seems to support tool calls, but failed to determine tool format. Tool calling will not work properly. "
190                "Check for a fixed template for your model in the models/templates directory of your llama.cpp installation or "
191                "report an issue at https://github.com/ggml-org/llama.cpp/issues\n");
192            return ctx.p.eps();
193    }
194}
195
196common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_context & ctx) const {
197    auto &       p           = ctx.p;
198    const auto & inputs      = ctx.inputs;
199
200    // Build effective field names with dot notation if function_field is set
201    std::string name_field = format.name_field;
202    std::string args_field = format.args_field;
203
204    if (!format.function_field.empty() && format.function_field != "function" &&
205        name_field.find('.') == std::string::npos) {
206        name_field = format.function_field + "." + name_field;
207        args_field = format.function_field + "." + args_field;
208    }
209
210    auto tools_parser = p.eps();
211    if (format.section_start.empty() && !format.per_call_start.empty()) {
212        auto single_tool_parser = p.standard_json_tools(
213            format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls,
214            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,
215            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);
216        tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space()));
217    } else {
218        tools_parser = p.standard_json_tools(
219            format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls,
220            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,
221            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);
222    }
223
224    // Handle content wrappers if present
225    if (ctx.content && ctx.content->is_always_wrapped()) {
226        auto wrapped_content = ctx.content->build_optional_wrapped(ctx);
227        return ctx.reasoning_parser + wrapped_content + tools_parser + p.end();
228    }
229
230    std::string tool_start = "{";
231    if (!format.section_start.empty()) {
232        tool_start = format.section_start;
233    } else if (!format.per_call_start.empty()) {
234        tool_start = format.per_call_start;
235    }
236
237    return ctx.reasoning_parser + p.optional(p.content(p.until(tool_start))) + tools_parser + p.end();
238}
239
240common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p, const std::string & name,
241                                                    const common_peg_parser & call_id_section, bool have_call_id,
242                                                    const common_peg_parser & args,
243                                                    std::optional<common_peg_parser> atomic_peek) const {
244    auto              open           = p.tool_open(function.name_prefix + p.tool_name(p.literal(name)) + function.name_suffix);
245    bool              matched_atomic = false;
246    common_peg_parser func_parser    = p.eps();
247
248    if (!function.args_separator.empty()) {
249        open = open + p.space() + p.literal(function.args_separator);
250    }
251
252    if (!function.name_suffix.empty()) {
253        func_parser    = open + call_id_section + p.space() + args;
254        matched_atomic = true;
255    } else if (have_call_id) {
256        func_parser    = p.atomic(open + call_id_section) + p.space() + args;
257        matched_atomic = true;
258    } else if (atomic_peek.has_value()) {
259        func_parser    = p.atomic(open + call_id_section + p.space() + *atomic_peek) + args;
260        matched_atomic = true;
261    } else {
262        func_parser = open + call_id_section + p.space() + args;
263    }
264
265    if (!function.close.empty()) {
266        func_parser = func_parser + p.space() + p.tool_close(p.literal(function.close));
267    } else if (!format.per_call_end.empty()) {
268        // When there's no func_close but there is a per_call_end marker, use peek() to ensure
269        // we only emit tool_close when we can actually see the closing marker. This prevents
270        // premature closing during partial parsing when we've seen e.g. "</" which could be
271        // either "</tool_call>" (end) or "<arg_key>" prefix that failed to match.
272        // Laguna (v4): the model may emit whitespace between the last </arg_value> and
273        // </tool_call> even though the template renders them tight. Tolerate optional
274        // leading space in the close lookahead so the tool call still closes.
275        auto close_peek = arguments.tolerate_intertag_whitespace
276                              ? p.peek(p.space() + p.literal(format.per_call_end))
277                              : p.peek(p.literal(format.per_call_end));
278        func_parser = func_parser + p.tool_close(close_peek);
279    } else {
280        func_parser = func_parser + p.tool_close(p.space());  // force this to process tool closing callbacks in mapper
281    }
282    if (!matched_atomic) {
283        func_parser = p.atomic(func_parser);
284    }
285    return func_parser;
286}
287
288common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context & ctx) const {
289    auto &       p           = ctx.p;
290    const auto & inputs      = ctx.inputs;
291
292    common_peg_parser tool_choice = p.choice();
293
294    foreach_function(inputs.tools, [&](const json & tool) {
295        const auto & func   = tool.at("function");
296        std::string  name   = func.at("name");
297        const auto   schema = common_chat_tool_parameters(func);
298
299        // Build call_id parser based on position (if supported)
300        bool have_call_id = false;
301        common_peg_parser call_id_section = p.eps();
302        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&
303            (!call_id.suffix.empty() || !arguments.start.empty())) {
304            if (!call_id.suffix.empty()) {
305                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix))) + call_id.suffix;
306            } else {
307                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));
308            }
309            have_call_id = true;
310        }
311        auto args_parser = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema));
312        if (!arguments.start.empty()) {
313            args_parser = p.literal(arguments.start) + args_parser;
314        }
315        if (!arguments.end.empty()) {
316            args_parser = args_parser + p.literal(arguments.end);
317        }
318
319        auto atomic_peek = !arguments.start.empty() ? std::optional(p.peek(p.literal(arguments.start))) : std::nullopt;
320        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_parser, atomic_peek);
321        tool_choice |= p.rule("tool-" + name, func_parser);
322    });
323
324    auto require_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
325
326    common_peg_parser tool_calls = p.eps();
327
328    if (!format.per_call_start.empty()) {
329        auto wrapped_call = format.per_call_start + tool_choice + format.per_call_end;
330        if (inputs.parallel_tool_calls) {
331            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call));
332        } else {
333            tool_calls = p.trigger_rule("tool-call", wrapped_call);
334        }
335        if (!format.section_start.empty()) {
336            tool_calls = p.trigger_rule("tool-calls",
337                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +
338                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end)));
339        }
340    } else {
341        std::string separator = ", ";  // Default
342        if (inputs.parallel_tool_calls) {
343            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice +
344                                                         p.zero_or_more(separator + tool_choice) + format.section_end);
345        } else {
346            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice + format.section_end);
347        }
348    }
349
350    if (!require_calls) {
351        tool_calls = p.optional(tool_calls);
352    }
353
354    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;
355    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);
356    return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end();
357}
358
359common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_context & ctx) const {
360    auto &       p           = ctx.p;
361    const auto & inputs      = ctx.inputs;
362
363    auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));
364
365    common_peg_parser tool_choice = p.choice();
366
367    foreach_function(inputs.tools, [&](const json & tool) {
368        const auto & func = tool.at("function");
369        std::string  name = func.at("name");
370
371        // Build parser for each argument, separating required and optional
372        std::vector<common_peg_parser> required_parsers;
373        std::vector<common_peg_parser> optional_parsers;
374        foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
375            auto arg =
376                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
377                                           arguments.name_suffix) +
378                           arguments.value_prefix +
379                           (param.schema->may_be_string() ?
380                                p.ac(p.tool_arg_string_value(until_suffix) +
381                                    p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
382                                (p.tool_arg_json_value(p.schema(
383                                    p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
384                                    p.tool_arg_close(p.literal(arguments.value_suffix)))));
385
386            auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
387            if (param.required) {
388                required_parsers.push_back(named_arg);
389            } else {
390                optional_parsers.push_back(named_arg);
391            }
392        });
393
394        // Build required arg sequence in definition order
395        common_peg_parser args_seq = p.eps();
396        for (size_t i = 0; i < required_parsers.size(); i++) {
397            if (i > 0) {
398                args_seq = args_seq + p.space();
399            }
400            args_seq = args_seq + required_parsers[i];
401        }
402
403        // Build optional args with flexible ordering
404        if (!optional_parsers.empty()) {
405            common_peg_parser any_opt = p.choice();
406            for (const auto & opt : optional_parsers) {
407                any_opt |= opt;
408            }
409            args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
410        }
411
412        if (!arguments.start.empty()) {
413            args_seq = p.literal(arguments.start) + args_seq;
414        }
415        if (!arguments.end.empty()) {
416            args_seq = args_seq + p.literal(arguments.end);
417        }
418
419        // Build call_id parser based on position (if supported)
420        common_peg_parser call_id_section = p.eps();
421        bool have_call_id = false;
422        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&
423            (!call_id.suffix.empty() || !arguments.start.empty())) {
424            have_call_id = true;
425            if (!call_id.suffix.empty()) {
426                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix)) + call_id.suffix);
427            } else {
428                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));
429            }
430        }
431
432        // Only peek for an arg tag when there are required args that must follow.
433        // When all args are optional, the model may emit no arg tags at all (#20650).
434        auto atomic_peek = (!arguments.name_prefix.empty() && !required_parsers.empty()) ?
435            std::optional(p.peek(p.literal(arguments.name_prefix))) : std::nullopt;
436        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_seq, atomic_peek);
437        tool_choice |= p.rule("tool-" + name, func_parser);
438    });
439
440    auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
441
442    common_peg_parser tool_calls = p.eps();
443
444    if (!format.per_call_start.empty()) {
445        auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end;
446        if (inputs.parallel_tool_calls) {
447            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space());
448        } else {
449            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.space());
450        }
451        if (!format.section_start.empty()) {
452            tool_calls = p.trigger_rule("tool-calls",
453                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +
454                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end) + p.space()));
455        }
456    } else {
457        std::string separator = ", ";  // Default
458
459        if (inputs.parallel_tool_calls) {
460            tool_calls = p.trigger_rule("tool-call", format.section_start + p.space() + tool_choice +
461                                                         p.zero_or_more(separator + tool_choice) + p.space() +
462                                                         format.section_end);
463        } else {
464            tool_calls = p.trigger_rule(
465                "tool-call", format.section_start + p.space() + tool_choice + p.space() + format.section_end);
466        }
467    }
468
469    if (!require_tools) {
470        tool_calls = p.optional(tool_calls);
471    }
472
473    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;
474    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);
475    return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end();
476}
477
478}  // namespace autoparser