char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aldehir Rojascommon : implement common_schema internal representation for JSON schemas (#28736)acecd5603

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
10.5 KiB229 linesraw
1#include "parsers.h"
2
3common_chat_params common_chat_params_init_minimax_m3(const common_chat_template &          tmpl,
4                                                             const autoparser::generation_params & inputs) {
5    common_chat_params data;
6
7    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
8    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
9    data.format             = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;
10    data.supports_thinking  = true;
11    data.thinking_start_tag = "<mm:think>";
12    data.thinking_end_tags  = {"</mm:think>"};
13
14    // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
15    // params use the parameter name as the tag (<file_path>...</file_path>).
16    const std::string NS          = "]<]minimax[>[";
17    const std::string THINK_START = "<mm:think>";
18    const std::string THINK_END   = "</mm:think>";
19    const std::string FC_START    = NS + "<tool_call>";
20    const std::string FC_END      = NS + "</tool_call>";
21    const std::string INVOKE_END  = NS + "</invoke>";
22
23    data.preserved_tokens = {
24        NS,
25        "<tool_call>",
26        "</tool_call>",
27        THINK_START,
28        THINK_END,
29    };
30
31    data.message_delimiters = {
32        { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai"        },
33        { COMMON_CHAT_ROLE_USER,      "]~b]user"      },
34        { COMMON_CHAT_ROLE_TOOL,      "]~b]tool"      },
35        { COMMON_CHAT_ROLE_SYSTEM,    "]~b]developer" },
36        { COMMON_CHAT_ROLE_SYSTEM,    "]~b]system"    },
37    };
38
39    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
40    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
41    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
42    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
43
44    const std::string GEN_PROMPT = data.generation_prompt;
45
46    using mm3 = common_chat_peg_minimax_m3_mapper;
47
48    if (inputs.has_continuation()) {
49        const auto & msg = inputs.continue_msg;
50
51        data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
52        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
53            data.generation_prompt += THINK_END + msg.render_content();
54        }
55
56        data.prompt += data.generation_prompt;
57    }
58
59    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
60        auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
61        auto end = p.end();
62
63        auto reasoning = p.eps();
64        if (extract_reasoning) {
65            auto block = inputs.enable_thinking
66                             ? p.literal(THINK_START) + p.space() +
67                                   p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)
68                             : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);
69
70            // A turn without reasoning is prefixed with a bare </mm:think>, written either by the
71            // generation prompt (thinking_mode = "disabled") or by the model itself.
72            reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));
73        }
74
75        if (has_response_format) {
76            auto response_format = p.rule("response-format",
77                p.literal("```json") + p.space() +
78                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
79                p.space() + p.literal("```"));
80            return generation_prompt + reasoning + response_format + end;
81        }
82
83        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
84            return generation_prompt + reasoning + p.content(p.rest()) + end;
85        }
86
87        auto tool_choice = p.choice();
88        foreach_function(inputs.tools, [&](const json & tool) {
89            const auto & function = tool.at("function");
90            std::string  name     = function.at("name");
91            auto         params   = common_chat_tool_parameters(function);
92            auto         doc      = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
93
94            // The template expands argument values recursively in XML (see the to_xml() macro)
95            std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of;
96            std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)>              members_of;
97
98            auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) {
99                const std::string close = NS + "</" + tag + ">";
100                return p.rule(rule_name,
101                    p.tool_arg(
102                        p.tool_arg_open(
103                            p.literal(NS + "<") +
104                            p.tool_arg_name(p.literal(tag)) +
105                            p.literal(">")) +
106                        value_of(schema, rule_name, close)));
107            };
108
109            value_of = [&](const common_chat_schema & schema,
110                           const std::string & rule_name,
111                           const std::string & close) -> common_peg_parser {
112                auto close_tag = p.tool_arg_close(p.literal(close));
113
114                // A string accepts anything, so a union with a string alternative is a string
115                if (schema.may_be_string()) {
116                    return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
117                }
118
119                if (schema.kind() == common_chat_schema::KIND_ANY_OF) {
120                    std::vector<common_peg_parser> choices;
121
122                    size_t index = 0;
123                    for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) {
124                        const std::string alt_name = rule_name + "-" + std::to_string(index++);
125
126                        // There is a risk that this breaks streaming deltas, but that's a risk we
127                        // assume to provide tool arg streaming.
128                        choices.push_back(value_of(*alternative, alt_name, close));
129                    }
130
131                    return p.choice(choices);
132                }
133
134                if (schema.kind() == common_chat_schema::KIND_OBJECT) {
135                    const auto & object = static_cast<const common_chat_schema_object &>(schema);
136                    if (!object.properties.empty()) {
137                        return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag;
138                    }
139                }
140
141                if (schema.kind() == common_chat_schema::KIND_ARRAY) {
142                    const std::string item_close = NS + "</item>";
143                    auto item = p.rule(rule_name + "-item",
144                        p.tag(mm3::TOOL_ARG_ITEM,
145                              p.literal(NS + "<item>") +
146                                  value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close)));
147                    return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
148                }
149
150                return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag;
151            };
152
153            // Required properties in schema order, then any number of optional ones in any order.
154            members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
155                std::vector<common_peg_parser> required_elements;
156                std::vector<common_peg_parser> optional_elements;
157                for (const auto & prop : object.properties) {
158                    auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + prop.name);
159                    (prop.required ? required_elements : optional_elements).push_back(element);
160                }
161
162                common_peg_parser members = p.eps();
163                for (size_t i = 0; i < required_elements.size(); i++) {
164                    if (i > 0) {
165                        members = members + p.space();
166                    }
167                    members = members + required_elements[i];
168                }
169
170                if (!optional_elements.empty()) {
171                    common_peg_parser any_optional = p.choice();
172                    for (const auto & element : optional_elements) {
173                        any_optional |= element;
174                    }
175                    members = members + p.repeat(p.space() + any_optional, 0, -1);
176                }
177
178                return members;
179            };
180
181            common_peg_parser invoke_body = p.eps();
182            if (doc->root->kind() == common_chat_schema::KIND_OBJECT) {
183                invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + name + "-arg");
184            }
185
186            auto func_parser = p.tool(
187                p.tool_open(p.literal(NS + "<invoke name=\"") +
188                            p.tool_name(p.literal(name)) + p.literal("\">")) +
189                p.space() + invoke_body + p.space() +
190                p.tool_close(p.literal(INVOKE_END)));
191
192            tool_choice |= p.rule("tool-" + name, func_parser);
193        });
194
195        auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
196
197        common_peg_parser tool_calls = p.eps();
198        if (inputs.parallel_tool_calls) {
199            tool_calls = p.trigger_rule("tool-call",
200                p.literal(FC_START) + p.space() + tool_choice +
201                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
202        } else {
203            tool_calls = p.trigger_rule("tool-call",
204                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
205        }
206
207        if (!require_tools) {
208            tool_calls = p.optional(tool_calls);
209        }
210
211        auto content_before_tools = p.content(p.until(FC_START));
212        return generation_prompt + reasoning + content_before_tools + tool_calls + end;
213    });
214
215    data.parser = parser.save();
216
217    if (include_grammar) {
218        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
219        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
220            parser.build_grammar(builder, data.grammar_lazy);
221        });
222
223        data.grammar_triggers = {
224            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
225        };
226    }
227
228    return data;
229}