char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojaschat : improve parsing of complex types in qwen3-coder (#28742)790cf51aa

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
9.0 KiB193 linesraw
1#include "parsers.h"
2
3common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template &          tmpl,
4                                                              const autoparser::generation_params & inputs) {
5    common_chat_params data;
6
7    const std::string GEN_PREFIX = "<|im_start|>assistant\n";
8
9    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
10    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
11    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
12
13    auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
14
15    data.supports_thinking = supports_reasoning;
16    data.preserved_tokens  = {
17        "<tool_call>",
18        "</tool_call>",
19    };
20
21    auto is_qwen3_coder  = !supports_reasoning;
22
23    if (supports_reasoning) {
24        data.thinking_start_tag = "<think>";
25        // Support both </think> and <tool_call> as reasoning end sequences.
26        // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
27        data.thinking_end_tags = { "</think>", "<tool_call>" };
28        data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
29    }
30
31    data.message_delimiters = {
32        { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant"             },
33        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
34        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>tool_response"         }, // StepFun-3.5-Flash
35        { COMMON_CHAT_ROLE_USER,      "<|im_start|>user"                  },
36        { COMMON_CHAT_ROLE_SYSTEM,    "<|im_start|>system"                },
37    };
38
39    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
40    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
41    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
42    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
43
44    if (inputs.has_continuation()) {
45        const auto & msg = inputs.continue_msg;
46
47        data.generation_prompt = GEN_PREFIX;
48        if (supports_reasoning) {
49            data.generation_prompt += "<think>\n" + msg.reasoning_content;
50            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
51                data.generation_prompt += "\n</think>\n\n";
52            }
53        }
54        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
55            data.generation_prompt += msg.render_content();
56        }
57
58        data.prompt += data.generation_prompt;
59    }
60
61    std::vector<std::string> tool_call_starts = { "<tool_call>" };
62
63    if (is_qwen3_coder) {
64        // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
65        // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
66        // constraining on <function which may occur in valid content generation, e.g. #include <functional>
67        foreach_function(inputs.tools, [&](const json & tool) {
68            const std::string name = tool.at("function").at("name");
69            tool_call_starts.push_back("<function=" + name + ">");
70        });
71    }
72
73    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
74        auto generation_prompt = p.literal(GEN_PREFIX);
75
76        auto reasoning = p.eps();
77        if (supports_reasoning && extract_reasoning) {
78            reasoning = p.optional("<think>" + p.space() +
79                                   p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
80                                   (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
81        }
82
83        // Response format parser
84        if (has_response_format) {
85            return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
86        }
87
88        // Tool call parser
89        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
90            auto arg_close  = p.tool_arg_close(p.literal("\n</parameter>\n"));
91            auto arg_string = p.rule("xml-arg-string",
92                p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
93
94            auto tool_choice = p.choice();
95            foreach_function(inputs.tools, [&](const json & tool) {
96                const auto & function = tool.at("function");
97                std::string  name     = function.at("name");
98
99                std::vector<common_peg_parser> required_args;
100                std::vector<common_peg_parser> optional_args;
101
102                foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
103                    auto rule_name = "tool-" + name + "-arg-" + param.name;
104
105                    auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n");
106
107                    auto types = param.schema->value_types();
108
109                    auto arg_value = p.eps();
110                    if (!types.has(common_chat_schema::TYPE_STRING)) {
111                        arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
112                    } else if (types.is_only(common_chat_schema::TYPE_STRING)) {
113                        arg_value = arg_string;
114                    } else {
115                        // The string alternative accepts any text, so the grammar only keeps the raw string
116                        // rule. The parser still tries the JSON alternatives first to type the value.
117                        auto json_value = p.choice();
118                        if (types.has(common_chat_schema::TYPE_OBJECT)) {
119                            json_value |= p.json_object();
120                        }
121                        if (types.has(common_chat_schema::TYPE_ARRAY)) {
122                            json_value |= p.json_array();
123                        }
124                        if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
125                            json_value |= p.json_number();
126                        }
127                        if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
128                            json_value |= p.json_bool();
129                        }
130                        if (types.has(common_chat_schema::TYPE_NULL)) {
131                            json_value |= p.json_null();
132                        }
133                        arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string");
134                    }
135
136                    auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
137
138                    (param.required ? required_args : optional_args).push_back(arg_rule);
139                });
140
141                // Accept required arguments in any order, as Qwen does not always adhere to the
142                // order provided.
143                auto args = p.permute("tool-" + name + "-args", required_args);
144                if (!optional_args.empty()) {
145                    args = args + p.zero_or_more(p.choice(optional_args));
146                }
147
148                auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
149                                   p.tool_args(args) +
150                                   p.tool_close(p.literal("</function>\n")));
151
152                tool_choice |= p.rule("tool-" + name, func);
153            });
154
155            auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
156
157            auto tool_call_body = tool_choice + "</tool_call>" + p.space();
158            auto tool_call      = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
159
160            // Qwen3-Coder models may occasionally omit the <tool_call> token.
161            auto tool_call_first = is_qwen3_coder ?
162                p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
163                tool_call;
164
165            auto calls      = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
166            auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
167
168            return generation_prompt +
169                   (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
170        }
171
172        // Content only parser
173        return generation_prompt + (reasoning << p.content(p.rest()));
174    });
175
176    data.parser = parser.save();
177
178    if (include_grammar) {
179        data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
180
181        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
182            parser.build_grammar(builder, data.grammar_lazy);
183        });
184
185        if (data.grammar_lazy) {
186            for (const auto & start : tool_call_starts) {
187                data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
188            }
189        }
190    }
191
192    return data;
193}