char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyentest: move tools/parser to tests (#27548)4a08fa297

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
25.2 KiB615 linesraw
1#include "chat-auto-parser.h"
2#include "chat-auto-parser-helpers.h"
3#include "chat.h"
4#include "log.h"
5#include "jinja/caps.h"
6#include "jinja/runtime.h"
7
8#include <fstream>
9#include <sstream>
10#include <string>
11#include <vector>
12#include <algorithm>
13
14#include "json.h"
15
16using json = common_json;
17
18// ANSI color codes - using 256-color palette for brighter colors (all bold)
19#define ANSI_RESET       "\033[0m"
20#define ANSI_PURPLE      "\033[1m\x1b[38;5;126m"  // Bold bright purple for main headers
21#define ANSI_CYAN        "\033[1m\x1b[38;5;81m"   // Bold bright cyan for section headers
22#define ANSI_BLUE        "\033[1m\x1b[38;5;12m"   // Bold bright blue for labels
23#define ANSI_ORANGE      "\033[1m\x1b[38;5;209m"  // Bold orange for right differences
24#define ANSI_GREEN       "\033[1m\x1b[38;5;83m"   // Bold bright green for left differences
25#define ANSI_GRAY        "\033[1m\x1b[38;5;240m"  // Bold gray (used for "no variables" message)
26#define ANSI_BOLD        "\033[1m"                // Standalone bold
27#define ANSI_PREFIX      "\033[1m\x1b[38;5;176m"  // Bold color for common prefix
28#define ANSI_SUFFIX      "\033[1m\x1b[38;5;61m"   // Bold color for common suffix
29
30// All template paths extracted from tests/test-chat.cpp
31static const std::vector<std::string> ALL_TEMPLATE_PATHS = {
32    "models/templates/Apertus-8B-Instruct.jinja",
33    "models/templates/Apriel-1.6-15b-Thinker-fixed.jinja",
34    "models/templates/ByteDance-Seed-OSS.jinja",
35    "models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja",
36    "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",
37    "models/templates/GLM-4.6.jinja",
38    "models/templates/GLM-4.7-Flash.jinja",
39    "models/templates/Kimi-K2-Instruct.jinja",
40    "models/templates/Kimi-K2-Thinking.jinja",
41    "models/templates/MiMo-VL.jinja",
42    "models/templates/MiniMax-M2.jinja",
43    "models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja",
44    "models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja",
45    "models/templates/NVIDIA-Nemotron-Nano-v2.jinja",
46    "models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja",
47    "models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja",
48    "models/templates/Qwen-QwQ-32B.jinja",
49    "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja",
50    "models/templates/Qwen3-Coder.jinja",
51    "models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja",
52    "models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja",
53    "models/templates/deepseek-ai-DeepSeek-V3.1.jinja",
54    "models/templates/fireworks-ai-llama-3-firefunction-v2.jinja",
55    "models/templates/google-gemma-2-2b-it.jinja",
56    "models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja",
57    "models/templates/llama-cpp-deepseek-r1.jinja",
58    "models/templates/meetkai-functionary-medium-v3.1.jinja",
59    "models/templates/meetkai-functionary-medium-v3.2.jinja",
60    "models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja",
61    "models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja",
62    "models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja",
63    "models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja",
64    "models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja",
65    "models/templates/moonshotai-Kimi-K2.jinja",
66    "models/templates/openai-gpt-oss-120b.jinja",
67    "models/templates/unsloth-Apriel-1.5.jinja",
68    "models/templates/unsloth-mistral-Devstral-Small-2507.jinja",
69};
70
71struct analysis_options {
72    std::vector<std::string> template_paths;
73    bool                     analyze_all = false;
74};
75
76static std::string read_file(const std::string & path) {
77    std::ifstream fin(path, std::ios::binary);
78    if (!fin.is_open()) {
79        throw std::runtime_error("Could not open file: " + path);
80    }
81    std::ostringstream buf;
82    buf << fin.rdbuf();
83    return buf.str();
84}
85
86static void print_usage(const char * program_name) {
87    LOG_ERR("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n");
88    LOG_ERR("\nUsage: %s [options]\n", program_name);
89    LOG_ERR("\nOptions:\n");
90    LOG_ERR("  --template <name>       Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");
91    LOG_ERR("  --template-file <path>  Analyze custom template file\n");
92    LOG_ERR("  --all                   Analyze all templates from test suite (default when no arguments are given)\n");
93    LOG_ERR("\nExamples:\n");
94    LOG_ERR("  %s --all\n", program_name);
95    LOG_ERR("  %s --template deepseek\n", program_name);
96    LOG_ERR("  %s --template-file my-template.jinja\n", program_name);
97}
98
99static bool parse_options(int argc, char ** argv, analysis_options & opts) {
100    if (argc < 2) {
101        // default mode: analyze all templates from the test suite
102        opts.analyze_all = true;
103    }
104
105    for (int i = 1; i < argc; ++i) {
106        std::string arg = argv[i];
107
108        if (arg == "-h" || arg == "--help") {
109            print_usage(argv[0]);
110            return false;
111        } else if (arg == "--all") {
112            opts.analyze_all = true;
113        } else if (arg == "--template") {
114            if (i + 1 >= argc) {
115                LOG_ERR("--template requires an argument\n");
116                return false;
117            }
118            std::string pattern = argv[++i];
119            std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
120
121            // Find matching templates
122            bool found = false;
123            for (const auto & path : ALL_TEMPLATE_PATHS) {
124                std::string path_lower = path;
125                std::transform(path_lower.begin(), path_lower.end(), path_lower.begin(), ::tolower);
126                if (path_lower.find(pattern) != std::string::npos) {
127                    opts.template_paths.push_back(path);
128                    found = true;
129                }
130            }
131
132            if (!found) {
133                LOG_ERR("No templates found matching: %s\n", pattern.c_str());
134                return false;
135            }
136        } else if (arg == "--template-file") {
137            if (i + 1 >= argc) {
138                LOG_ERR("--template-file requires an argument\n");
139                return false;
140            }
141            opts.template_paths.push_back(argv[++i]);
142        } else {
143            LOG_ERR("Unknown option: %s\n", arg.c_str());
144            print_usage(argv[0]);
145            return false;
146        }
147    }
148
149    if (opts.analyze_all) {
150        opts.template_paths = ALL_TEMPLATE_PATHS;
151    }
152
153    if (opts.template_paths.empty()) {
154        LOG_ERR("No templates specified\n");
155        print_usage(argv[0]);
156        return false;
157    }
158
159    return true;
160}
161
162static json build_tools_definition() {
163    json parameters_schema                    = json::object();
164    parameters_schema["type"]                 = "object";
165    parameters_schema["properties"]           = json::object();
166    parameters_schema["properties"]["param1"] = json::object({
167        { "type",        "string"          },
168        { "description", "First parameter" }
169    });
170    parameters_schema["properties"]["param2"] = json::object({
171        { "type",        "string"           },
172        { "description", "Second parameter" }
173    });
174    parameters_schema["required"]             = json::array({ "param1", "param2" });
175
176    return json::array({
177        json{ { "type", "function" },
178             { "function", json{ { "name", "test_function_name" },
179                                  { "description", "A test function for debugging" },
180                                  { "parameters", parameters_schema } } } }
181    });
182}
183
184// Helper to create a tool call with arguments as JSON object
185static json build_tool_call(const std::string & name, const json & args_object, const std::string & id = "call_001") {
186    return json{
187        {"id", id},
188        {"type", "function"},
189        {"function", json{
190            {"name", name},
191            {"arguments", args_object}  // Pass as JSON object, not serialized string
192        }}
193    };
194}
195
196// Helper functions to create repeating message definitions
197static json make_user_msg() {
198    return json{
199        {"role", "user"},
200        {"content", "Hello, please help me."}
201    };
202}
203
204static json make_user_msg2() {
205    return json{
206        {"role", "user"},
207        {"content", "Thank you."}
208    };
209}
210
211static json make_user_msg2_continue() {
212    return json{
213        {"role", "user"},
214        {"content", "Continue."}
215    };
216}
217
218static json make_assistant_no_tool() {
219    return json{
220        {"role", "assistant"},
221        {"content", "Let me help you."}
222    };
223}
224
225static json make_assistant_one_tool() {
226    return json{
227        {"role", "assistant"},
228        {"content", nullptr},
229        {"tool_calls", json::array({
230            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
231        })}
232    };
233}
234
235static json make_assistant_two_tools() {
236    return json{
237        {"role", "assistant"},
238        {"content", nullptr},
239        {"tool_calls", json::array({
240            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})),
241            build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002")
242        })}
243    };
244}
245
246static json make_assistant_no_reasoning() {
247    return json{
248        {"role", "assistant"},
249        {"content", "I can help you with that."}
250    };
251}
252
253static json make_assistant_with_reasoning() {
254    return json{
255        {"role", "assistant"},
256        {"content", "I can help you with that."},
257        {"reasoning_content", "The user is asking for help. I should respond positively."}
258    };
259}
260
261static json make_assistant_one_tool_with_reasoning() {
262    return json{
263        {"role", "assistant"},
264        {"content", nullptr},
265        {"tool_calls", json::array({
266            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
267        })},
268        {"reasoning_content", "I need to call the tool first."}
269    };
270}
271
272static void print_diff_split(const std::string & title, const diff_split & diff) {
273    LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);
274    LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());
275    LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());
276    LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());
277    LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());
278}
279
280static void check_reasoning_variables(const common_chat_template & tmpl) {
281    LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);
282
283    try {
284        // Create a list of candidate reasoning/thinking variable names to probe
285        std::vector<std::string> candidate_vars = {
286            "enable_reasoning",
287            "use_reasoning",
288            "reasoning_enabled",
289            "has_reasoning",
290            "reasoning_mode",
291            "reasoning_format",
292            "reasoning_active",
293            "with_reasoning",
294            "use_thinking",
295            "thinking_enabled",
296            "has_thinking",
297            "thinking_mode",
298            "thinking_format",
299            "thinking_active",
300            "with_thinking",
301            "enable_reason",
302            "reason_enabled",
303            "enable_think",
304            "think_enabled",
305        };
306
307        jinja::context ctx;
308        ctx.is_get_stats = true;
309
310        json messages = json::array({
311            json{
312                {"role", "user"},
313                {"content", "Test message"}
314            },
315            json{
316                {"role", "assistant"},
317                {"content", "Response"},
318                {"reasoning_content", "Some reasoning"}
319            }
320        });
321
322        // Set up base context
323        jinja::global_from_json(ctx, json{
324            {"messages", messages},
325            {"tools", json::array()},
326            {"bos_token", ""},
327            {"eos_token", ""},
328            {"add_generation_prompt", false},
329            {"enable_thinking", true}  // Already passed, so we'll exclude this from results
330        }, true);
331
332        // Add candidate variables as undefined to probe which ones are accessed
333        for (const auto & var_name : candidate_vars) {
334            ctx.set_val(var_name, jinja::mk_val<jinja::value_undefined_t>(var_name));
335        }
336
337        try {
338            jinja::runtime runtime(ctx);
339            runtime.execute(tmpl.prog);
340        } catch (const std::exception & e) {
341            // Execution may fail, that's okay - we just want to see what variables were accessed
342        }
343
344        // Check which candidate variables were accessed (stats.used = true)
345        std::vector<std::string> accessed_vars;
346        for (const auto & var_name : candidate_vars) {
347            auto val = ctx.get_val(var_name);
348            if (!val->is_undefined()) {
349                // Variable was overwritten, skip it
350                continue;
351            }
352            if (val->stats.used) {
353                accessed_vars.push_back(var_name);
354            }
355        }
356
357        if (accessed_vars.empty()) {
358            LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);
359        } else {
360            LOG_ERR("Template queries the following reasoning/thinking-related variables:\n");
361            for (const auto & var : accessed_vars) {
362                LOG_ERR("  %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);
363            }
364        }
365
366    } catch (const std::exception & e) {
367        LOG_ERR("Error checking reasoning variables: %s\n", e.what());
368    }
369}
370
371static void analyze_template(const std::string & template_path) {
372    LOG_ERR("\n");
373    LOG_ERR("%s", ANSI_PURPLE);
374    LOG_ERR("================================================================================\n");
375    LOG_ERR("                    ANALYZING TEMPLATE: %s\n", template_path.c_str());
376    LOG_ERR("================================================================================\n");
377    LOG_ERR("%s", ANSI_RESET);
378
379    std::string template_source;
380    try {
381        template_source = read_file(template_path);
382    } catch (const std::exception & e) {
383        LOG_ERR("Error reading template: %s\n", e.what());
384        return;
385    }
386
387    try {
388        common_chat_template chat_template(template_source, "", "");
389        json tools = build_tools_definition();
390
391        // ===== CAPABILITIES ANALYSIS =====
392        LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);
393        auto caps = chat_template.original_caps();
394        LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");
395        LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");
396        LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");
397        LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");
398        LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");
399        LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");
400
401        // ===== DIFFERENTIAL ANALYSIS =====
402
403        // Test 1: With and without tools (single user message)
404        {
405            json user_msg = make_user_msg();
406
407            autoparser::generation_params params_no_tools;
408            params_no_tools.messages = json::array({ user_msg });
409            params_no_tools.add_generation_prompt = false;
410            params_no_tools.tools = json::array();
411
412            autoparser::generation_params params_with_tools = params_no_tools;
413            params_with_tools.tools = tools;
414
415            std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools);
416            std::string output_with_tools = common_chat_template_direct_apply(chat_template, params_with_tools);
417
418            auto diff = calculate_diff_split(output_no_tools, output_with_tools);
419            print_diff_split("Diff: With vs Without Tools (single user message)", diff);
420        }
421
422        // Test 2: With and without add_generation_prompt (single user message)
423        {
424            json user_msg = make_user_msg();
425
426            autoparser::generation_params params_no_prompt;
427            params_no_prompt.messages = json::array({ user_msg });
428            params_no_prompt.add_generation_prompt = false;
429            params_no_prompt.tools = json::array();
430
431            autoparser::generation_params params_with_prompt = params_no_prompt;
432            params_with_prompt.add_generation_prompt = true;
433
434            std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt);
435            std::string output_with_prompt = common_chat_template_direct_apply(chat_template, params_with_prompt);
436
437            auto diff = calculate_diff_split(output_no_prompt, output_with_prompt);
438            print_diff_split("Diff: With vs Without add_generation_prompt (single user message)", diff);
439        }
440
441        // Test 3: Assistant with reasoning_content (user, assistant)
442        {
443            json user_msg = make_user_msg();
444
445            autoparser::generation_params params_no_reasoning;
446            params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() });
447            params_no_reasoning.add_generation_prompt = false;
448            params_no_reasoning.enable_thinking = true;
449
450            autoparser::generation_params params_with_reasoning = params_no_reasoning;
451            params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() });
452
453            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);
454            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);
455
456            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
457            print_diff_split("Diff: With vs Without reasoning_content (user, assistant)", diff);
458        }
459
460        // Test 4: Assistant with reasoning_content (user, assistant, user)
461        {
462            json user_msg = make_user_msg();
463            json user_msg2 = make_user_msg2();
464
465            autoparser::generation_params params_no_reasoning;
466            params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 });
467            params_no_reasoning.add_generation_prompt = false;
468            params_no_reasoning.enable_thinking = true;
469
470            autoparser::generation_params params_with_reasoning = params_no_reasoning;
471            params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 });
472
473            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);
474            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);
475
476            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
477            print_diff_split("Diff: With vs Without reasoning_content (user, assistant, user)", diff);
478        }
479
480        // Test 5: Tool call in last assistant message (user, assistant)
481        {
482            json user_msg = make_user_msg();
483
484            autoparser::generation_params params_no_tool;
485            params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() });
486            params_no_tool.add_generation_prompt = false;
487            params_no_tool.tools = tools;
488
489            autoparser::generation_params params_with_tool = params_no_tool;
490            params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() });
491
492            std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);
493            std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);
494
495            auto diff = calculate_diff_split(output_no_tool, output_with_tool);
496            print_diff_split("Diff: With vs Without tool call (user, assistant)", diff);
497        }
498
499        // Test 6: Tool call in last assistant message (user, assistant, user)
500        {
501            json user_msg = make_user_msg();
502            json user_msg2 = make_user_msg2_continue();
503
504            autoparser::generation_params params_no_tool;
505            params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 });
506            params_no_tool.add_generation_prompt = false;
507            params_no_tool.tools = tools;
508
509            autoparser::generation_params params_with_tool = params_no_tool;
510            params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });
511
512            std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);
513            std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);
514
515            auto diff = calculate_diff_split(output_no_tool, output_with_tool);
516            print_diff_split("Diff: With vs Without tool call (user, assistant, user)", diff);
517        }
518
519        // Test 7: One vs two tool calls (user, assistant)
520        {
521            json user_msg = make_user_msg();
522
523            autoparser::generation_params params_one_tool;
524            params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() });
525            params_one_tool.add_generation_prompt = false;
526            params_one_tool.tools = tools;
527
528            autoparser::generation_params params_two_tools = params_one_tool;
529            params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() });
530
531            std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);
532            std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);
533
534            auto diff = calculate_diff_split(output_one_tool, output_two_tools);
535            print_diff_split("Diff: One vs Two tool calls (user, assistant)", diff);
536        }
537
538        // Test 8: One vs two tool calls (user, assistant, user)
539        {
540            json user_msg = make_user_msg();
541            json user_msg2 = make_user_msg2_continue();
542
543            autoparser::generation_params params_one_tool;
544            params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });
545            params_one_tool.add_generation_prompt = false;
546            params_one_tool.tools = tools;
547
548            autoparser::generation_params params_two_tools = params_one_tool;
549            params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 });
550
551            std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);
552            std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);
553
554            auto diff = calculate_diff_split(output_one_tool, output_two_tools);
555            print_diff_split("Diff: One vs Two tool calls (user, assistant, user)", diff);
556        }
557
558        // Test 9: Tool call with vs without reasoning_content (user, assistant)
559        {
560            json user_msg = make_user_msg();
561
562            autoparser::generation_params params_no_reasoning;
563            params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() });
564            params_no_reasoning.add_generation_prompt = false;
565            params_no_reasoning.tools = tools;
566            params_no_reasoning.enable_thinking = true;
567
568            autoparser::generation_params params_with_reasoning = params_no_reasoning;
569            params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() });
570
571            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);
572            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);
573
574            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
575            print_diff_split("Diff: Tool call with vs without reasoning_content (user, assistant)", diff);
576        }
577
578        // Check reasoning variables
579        check_reasoning_variables(chat_template);
580
581    } catch (const std::exception & e) {
582        LOG_ERR("Analysis failed: %s\n", e.what());
583    }
584}
585
586int main(int argc, char ** argv) {
587    // Set log level to capture all output
588    common_log_set_verbosity_thold(99);
589
590    analysis_options opts;
591    if (!parse_options(argc, argv, opts)) {
592        return 1;
593    }
594
595    LOG_ERR("\n");
596    LOG_ERR("%s", ANSI_PURPLE);
597    LOG_ERR("================================================================================\n");
598    LOG_ERR("                      TEMPLATE ANALYSIS TOOL\n");
599    LOG_ERR("================================================================================\n");
600    LOG_ERR("%s", ANSI_RESET);
601    LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);
602
603    for (const auto & path : opts.template_paths) {
604        analyze_template(path);
605    }
606
607    LOG_ERR("\n");
608    LOG_ERR("%s", ANSI_GREEN);
609    LOG_ERR("================================================================================\n");
610    LOG_ERR("                      ANALYSIS COMPLETE\n");
611    LOG_ERR("================================================================================\n");
612    LOG_ERR("%s", ANSI_RESET);
613
614    return 0;
615}