char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
d9f918d2d
char/aliceai-t5-moe
1#include "chat-auto-parser.h" 2#include "chat-auto-parser-helpers.h" 3#include "chat-peg-parser.h" 4#include "chat.h" 5#include "common.h" 6#include "log.h" 7#include "peg-parser.h" 8 9#include <algorithm> 10#include <cctype> 11#include <numeric> 12#include <ostream> 13#include <sstream> 14 15#define ANSI_RESET "\033[0m" 16#define ANSI_PURPLE "\033[1m\x1b[38;5;126m" 17#define ANSI_ORANGE "\033[1m\x1b[38;5;214m" 18#define ANSI_RED "\033[1m\x1b[38;5;196m" 19 20using json = common_json ; 21 22namespace autoparser { 23 24static const std::string FUN_FIRST = "FFF_FIRST_FUN_F" ; 25static const std::string FUN_SECOND = "SSS_SECOND_FUN_S" ; 26static const std::string ARG_FIRST = "AA_ARG_FST_AA" ; 27static const std::string ARG_SECOND = "BB_ARG_SND_BB" ; 28static const std::string USER_MSG = "U_USER_MSG Hello END_U" ; 29static const std::string USER_MSG_TWO = "V_USER_MSG Hello END_V" ; 30static const std::string ASSISTANT_MSG = "A_ASST_MSG I can help END_A" ; 31static const std::string THINKING_CONTENT = "REASON_PART I am thinking END_R" ; 32static const std::string CALL_ID_001 = "call00001" ; 33static const std::string CALL_ID_002 = "call00002" ; 34static const std::string CALL_ID_999 = "call99999" ; 35 36static std::vector < std::function < void (const common_chat_template & tmpl ,autoparser & )>> workarounds ( 37 {// Old reasoning Qwen templates - they don't really display reasoning content, but we still want to 38// support reasoning on them 39 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 40if (tmpl .src .find ("content.split('</think>')" )!= std::string::npos && 41tmpl .src .find ("reasoning_content" )== std::string::npos && 42tmpl .src .find ("<SPECIAL_12>" )== std::string::npos && 43analysis .reasoning .mode == reasoning_mode::NONE ) { 44analysis .reasoning .mode = reasoning_mode::TAG_BASED ; 45analysis .reasoning .start = "<think>" ; 46analysis .reasoning .end = "</think>" ; 47analysis .preserved_tokens .push_back ("<think>" ); 48analysis .preserved_tokens .push_back ("</think>" ); 49LOG_DBG (ANSI_ORANGE "[Patch: old Qwen/Deepseek thinking template]\n" ANSI_RESET ); 50 } 51 }, 52// Granite 3.3, with separate reasoning and content markers 53 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 54if (tmpl .src .find ("Write your thoughts between <think></think> and write your response between " 55"<response></response>" )!= std::string::npos ) { 56analysis .reasoning .mode = reasoning_mode::TAG_BASED ; 57analysis .reasoning .start = "<think>" ; 58analysis .reasoning .end = "</think>" ; 59analysis .preserved_tokens .push_back ("<think>" ); 60analysis .preserved_tokens .push_back ("</think>" ); 61analysis .content .mode = content_mode::WRAPPED_WITH_REASONING ; 62analysis .content .start = "<response>" ; 63analysis .content .end = "</response>" ; 64analysis .preserved_tokens .push_back ("<response>" ); 65analysis .preserved_tokens .push_back ("</response>" ); 66LOG_DBG (ANSI_ORANGE "[Patch: Granite 3.3]\n" ANSI_RESET ); 67 } 68 }, 69// Cohere Command R+ - content wrapped in <|CHATBOT_TOKEN|>...<|END_OF_TURN_TOKEN|> 70 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 71if (tmpl .src .find ("<|CHATBOT_TOKEN|>" )!= std::string::npos && 72tmpl .src .find ("<|END_OF_TURN_TOKEN|>" )!= std::string::npos && analysis .content .start .empty ()) { 73analysis .content .mode = content_mode::ALWAYS_WRAPPED ; 74analysis .content .start = "<|CHATBOT_TOKEN|>" ; 75analysis .content .end = "<|END_OF_TURN_TOKEN|>" ; 76analysis .preserved_tokens .push_back ("<|CHATBOT_TOKEN|>" ); 77analysis .preserved_tokens .push_back ("<|END_OF_TURN_TOKEN|>" ); 78analysis .user_start = "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>" ; 79LOG_DBG (ANSI_ORANGE "[Patch: Cohere Command R+]\n" ANSI_RESET ); 80 } 81 }, 82// Functionary - no tool call section delimiter 83 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 84if (tmpl .src .find ("set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", " 85"\"code_interpreter\") | list | length > 0" )!= std::string::npos ) { 86analysis .content .mode = content_mode::PLAIN ; 87analysis .content .end = "" ; 88analysis .tools .function .name_prefix = "" ; 89analysis .tools .format .section_start = "" ; 90analysis .tools .format .section_end = "" ; 91analysis .tools .format .per_call_start = "<function=" ; 92analysis .tools .format .per_call_end = "</function>" ; 93analysis .tools .function .close = "" ; 94analysis .preserved_tokens .clear (); 95analysis .preserved_tokens .push_back ("<|eot_id|>" ); 96analysis .preserved_tokens .push_back ("<|eom_id|>" ); 97analysis .preserved_tokens .push_back ("<function=" ); 98analysis .preserved_tokens .push_back (">" ); 99analysis .preserved_tokens .push_back ("</function>" ); 100LOG_DBG (ANSI_ORANGE "[Patch: Functionary 3.1]\n" ANSI_RESET ); 101 } 102 }, 103// DeepSeek-R1-Distill-Qwen 104 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 105if (tmpl .src .find ( 106"{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'" )!= 107 std::string::npos ) { 108analysis .tools .format .section_start = "<｜tool▁calls▁begin｜>" ; 109analysis .tools .format .section_end = "<｜tool▁calls▁end｜>" ; 110analysis .tools .format .per_call_start = "<｜tool▁call▁begin｜>function" ; 111analysis .tools .function .name_prefix = "<｜tool▁sep｜>" ; 112analysis .tools .format .per_call_end = "<｜tool▁call▁end｜>" ; 113analysis .tools .function .close = "```" ; 114LOG_DBG (ANSI_ORANGE "[Patch: DeepSeek-R1-Distill-Qwen]\n" ANSI_RESET ); 115 } 116 }, 117// Nemotron Nano v2 118 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 119if (tmpl .src .find ("<SPECIAL_10>" )!= std::string::npos && tmpl .src .find ("<SPECIAL_11>" )!= std::string::npos && 120tmpl .src .find ("<SPECIAL_12>" )!= std::string::npos && tmpl .src .find ("<TOOL_RESPONSE>" )!= std::string::npos ) { 121 122analysis .tools .format .mode = tool_format::JSON_NATIVE ; 123analysis .tools .format .section_start = "" ; 124analysis .tools .format .section_end = "" ; 125analysis .tools .format .per_call_start = "<TOOLCALL>" ; 126analysis .tools .format .per_call_end = "</TOOLCALL>" ; 127analysis .tools .format .tools_array_wrapped = true; 128analysis .content .mode = content_mode::PLAIN ; 129analysis .content .start = "" ; 130analysis .content .end = "" ; 131analysis .reasoning .mode = reasoning_mode::TAG_BASED ; 132analysis .reasoning .start = "<think>\n" ; 133analysis .reasoning .end = "</think>" ; 134analysis .assistant_start = "<SPECIAL_11>Assistant" ; 135analysis .user_start = "<SPECIAL_11>User" ; 136analysis .preserved_tokens .clear (); 137analysis .preserved_tokens .push_back ("<SPECIAL_11>" ); 138analysis .preserved_tokens .push_back ("</think>" ); 139analysis .preserved_tokens .push_back ("<TOOLCALL>" ); 140analysis .preserved_tokens .push_back ("</TOOLCALL>" ); 141LOG_DBG (ANSI_ORANGE "[Patch: Nemotron Nano v2]\n" ANSI_RESET ); 142 } 143 }, 144// Fireworks 145 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 146if (tmpl .src .find ("{%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\n'" 147" + message['content'] | trim + '\\n' + system_prompt_suffix + '<|eot_id|>' -%}" )!= std::string::npos ) { 148analysis .assistant_start = "<|start_header_id|>assistant<|end_header_id|>" ; 149analysis .user_start = "<|start_header_id|>user<|end_header_id|>" ; 150LOG_DBG (ANSI_ORANGE "[Patch: Fireworks v2]\n" ANSI_RESET ); 151 } 152 }, 153// Solar Open 154 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 155if (tmpl .src .find ("<|begin|>assistant<|think|><|end|>" )!= std::string::npos ) { 156analysis .assistant_start = "<|begin|>assistant" ; 157LOG_DBG (ANSI_ORANGE "[Patch: Solar Open]\n" ANSI_RESET ); 158 } 159 }, 160// Apriel 1.6 161 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 162if (tmpl .src .find ("if not loop.last and '[BEGIN FINAL RESPONSE]' in asst_text" )!= std::string::npos ) { 163analysis .user_start = "<|begin_user|>" ; 164analysis .assistant_start = "<|begin_assistant|>" ; 165LOG_DBG (ANSI_ORANGE "[Patch: Apriel 1.6]\n" ANSI_RESET ); 166 } 167 }, 168// template uses the JSON {name, parameters} tool instruction, emits the OpenAI function wrapper 169 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 170if (tmpl .src .find ("Respond in the format {\"name\": function name" )!= std::string::npos && 171tmpl .src .find ("Do not use variables." )!= std::string::npos ) { 172analysis .tools .format .openai_wrapper_trigger = true; 173LOG_DBG (ANSI_ORANGE "[Patch: JSON name/parameters tool instruction]\n" ANSI_RESET ); 174 } 175 }, 176// Laguna (poolside) - the v4 chat template renders reasoning and tool-arg 177// delimiters with formatting whitespace ("<think>\n", "</arg_value>\n") that 178// the model does not emit, so the inferred delimiters carry a spurious 179// newline and never match the model output. Trim to the bare tag. (v8 180// renders without the whitespace, so this is a no-op there.) 181 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 182if (tmpl .src .find ("laguna_glm_thinking" )!= std::string::npos ) { 183analysis .reasoning .start = trim_whitespace (analysis .reasoning .start ); 184analysis .reasoning .end = trim_whitespace (analysis .reasoning .end ); 185analysis .tools .arguments .value_prefix = trim_whitespace (analysis .tools .arguments .value_prefix ); 186analysis .tools .arguments .value_suffix = trim_whitespace (analysis .tools .arguments .value_suffix ); 187analysis .tools .arguments .separator = trim_whitespace (analysis .tools .arguments .separator ); 188analysis .tools .arguments .tolerate_intertag_whitespace = true; 189// The CONTROL/eot </assistant> token only halts generation when emitted as the 190// single token; after tool calls the model can spell it out as text tokens. 191// A literal stop string catches it either way. 192analysis .additional_stops .push_back ("</assistant>" ); 193LOG_DBG (ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET ); 194 } 195 }, 196// Bailing V3 197 [](const common_chat_template & tmpl ,autoparser & analysis )-> void { 198if (tmpl .src .find ("Bailing V3 chat template" )!= std::string::npos ) { 199analysis .tools .arguments .value_suffix = trim_whitespace (analysis .tools .arguments .value_suffix ); 200analysis .tools .arguments .tolerate_intertag_whitespace = true; 201LOG_DBG (ANSI_ORANGE "[Patch: Bailing V3]\n" ANSI_RESET ); 202 } 203 }, 204 205 }); 206 207// Common JSON structures 208static json params_schema = { 209 {"type" ,"object" }, 210 {"properties" , 211 { {ARG_FIRST , { {"type" ,"string" }, {"description" ,"First argument" } } }, 212 {ARG_SECOND , { {"type" ,"string" }, {"description" ,"Second argument" } } } } }, 213 {"required" , json::array ({}) } 214}; 215 216static json tools = json::array ({ 217 { {"type" ,"function" }, 218 {"function" , 219json { {"name" ,FUN_FIRST }, {"description" ,"Test function foo" }, {"parameters" ,params_schema } } } }, 220 { {"type" ,"function" }, 221 {"function" , 222json { {"name" ,FUN_SECOND }, {"description" ,"Test function bar" }, {"parameters" ,params_schema } } } } 223}); 224 225static json user_msg = json { 226 {"role" ,"user" }, 227 {"content" ,USER_MSG } 228}; 229 230static json build_tool_call (const std::string & name ,const json & args ,const std::string & id = CALL_ID_001 ) { 231return json { 232 {"id" ,id }, 233 {"type" ,"function" }, 234 {"function" ,json { {"name" ,name }, {"arguments" ,args } } } 235 }; 236} 237 238static json first_tool_call_zero_args = build_tool_call (FUN_FIRST , json::object (),CALL_ID_001 ); 239static json first_tool_call_one_arg = build_tool_call (FUN_FIRST , {{ARG_FIRST ,"XXXX" }},CALL_ID_001 ); 240static json first_tool_call_one_arg_other_val = build_tool_call (FUN_FIRST , {{ARG_FIRST ,"YYYY" }},CALL_ID_001 ); 241static json first_tool_call_other_arg = build_tool_call (FUN_FIRST , {{ARG_SECOND ,"YYYY" }},CALL_ID_001 ); 242 243static json first_tool_call = 244build_tool_call (FUN_FIRST ,json {{ARG_FIRST ,"XXXX" }, {ARG_SECOND ,"YYYY" }},CALL_ID_001 ); 245static json second_tool_call = 246build_tool_call (FUN_SECOND ,json { {ARG_FIRST ,"XXXX" }, {ARG_SECOND ,"YYYY" }},CALL_ID_002 ); 247static json first_tool_call_alt_id = 248build_tool_call (FUN_FIRST ,json {{ARG_FIRST ,"XXXX" }, {ARG_SECOND ,"YYYY" }},CALL_ID_999 ); 249 250template < typename T > 251static std::string mode_to_str (T mode ) { 252 std::ostringstream os ; 253os <<mode ; 254return os .str (); 255} 256 257void autoparser::analyze_template (const common_chat_template & tmpl ) { 258jinja_caps = tmpl .original_caps (); 259reasoning = analyze_reasoning (tmpl ,jinja_caps .supports_tool_calls ); 260content = analyze_content (tmpl ,reasoning ); 261tools = analyze_tools (jinja_caps .supports_tool_calls ?analyze_tools (tmpl ,jinja_caps ,reasoning ) :analyze_tools ()); 262assistant_start = detect_assistant_start_marker (tmpl ); 263user_start = detect_user_start_marker (tmpl ); 264collect_preserved_tokens (); 265 266for (auto & workaround :workarounds ) { 267workaround (tmpl ,* this ); 268 } 269 270LOG_DBG ("\n--- Reasoning & Content Structure ---\n" ); 271LOG_DBG ("user_msg_start: %s\n" ,user_start .c_str ()); 272LOG_DBG ("assistant_msg_start: %s\n" ,assistant_start .c_str ()); 273LOG_DBG ("reasoning_mode: %s\n" ,mode_to_str (reasoning .mode ).c_str ()); 274LOG_DBG ("reasoning_start: '%s'\n" ,reasoning .start .c_str ()); 275LOG_DBG ("reasoning_end: '%s'\n" ,reasoning .end .c_str ()); 276LOG_DBG ("content_mode: %s\n" ,mode_to_str (content .mode ).c_str ()); 277LOG_DBG ("content_start: '%s'\n" ,content .start .c_str ()); 278LOG_DBG ("content_end: '%s'\n" ,content .end .c_str ()); 279 280LOG_DBG ("\n--- Tool Call Structure ---\n" ); 281LOG_DBG ("tool_mode: %s\n" ,mode_to_str (tools .format .mode ).c_str ()); 282LOG_DBG ("supports_tools: %s\n" ,jinja_caps .supports_tools ?"true" :"false" ); 283LOG_DBG ("supports_parallel_calls: %s\n" ,jinja_caps .supports_parallel_tool_calls ?"true" :"false" ); 284LOG_DBG ("tool_section_start: '%s'\n" ,tools .format .section_start .c_str ()); 285LOG_DBG ("tool_section_end: '%s'\n" ,tools .format .section_end .c_str ()); 286LOG_DBG ("per_call_start: '%s'\n" ,tools .format .per_call_start .c_str ()); 287LOG_DBG ("per_call_end: '%s'\n" ,tools .format .per_call_end .c_str ()); 288LOG_DBG ("func_name_prefix: '%s'\n" ,tools .function .name_prefix .c_str ()); 289LOG_DBG ("func_name_suffix: '%s'\n" ,tools .function .name_suffix .c_str ()); 290LOG_DBG ("func_args_separator: '%s'\n" ,tools .function .args_separator .c_str ()); 291LOG_DBG ("func_close: '%s'\n" ,tools .function .close .c_str ()); 292LOG_DBG ("call_id_prefix: '%s'\n" ,tools .call_id .prefix .c_str ()); 293LOG_DBG ("call_id_suffix: '%s'\n" ,tools .call_id .suffix .c_str ()); 294LOG_DBG ("call_id_pos: '%s'\n" ,mode_to_str (tools .call_id .pos ).c_str ()); 295LOG_DBG ("args_start: '%s'\n" ,tools .arguments .start .c_str ()); 296LOG_DBG ("args_end: '%s'\n" ,tools .arguments .end .c_str ()); 297LOG_DBG ("arg_name_prefix: '%s'\n" ,tools .arguments .name_prefix .c_str ()); 298LOG_DBG ("arg_name_suffix: '%s'\n" ,tools .arguments .name_suffix .c_str ()); 299LOG_DBG ("arg_value_prefix: '%s'\n" ,tools .arguments .value_prefix .c_str ()); 300LOG_DBG ("arg_value_suffix: '%s'\n" ,tools .arguments .value_suffix .c_str ()); 301LOG_DBG ("name_field: '%s'\n" ,tools .format .name_field .c_str ()); 302LOG_DBG ("args_field: '%s'\n" ,tools .format .args_field .c_str ()); 303LOG_DBG ("id_field: '%s'\n" ,tools .format .id_field .c_str ()); 304LOG_DBG ("gen_id_field: '%s'\n" ,tools .format .gen_id_field .c_str ()); 305LOG_DBG ("parameter_order: '%s'\n" , std::accumulate (tools .format .parameter_order .begin (),tools .format .parameter_order .end (), 306 std::string ("" ), [] (const std::string & a ,const std::string & b ) {return a .empty () ?b :a + ", " + b ; } 307 ).c_str ()); 308 309LOG_DBG (ANSI_PURPLE "=== Differential analysis complete ===\n" ANSI_RESET ); 310analysis_complete = true; 311} 312 313void autoparser::collect_preserved_tokens () { 314auto add_token = [this ](const std::string & org_token ) { 315 std::string token = trim_whitespace (org_token ); 316if (!token .empty ()) { 317// Avoid duplicates 318if (std::find (preserved_tokens .begin (),preserved_tokens .end (),token )== preserved_tokens .end ()) { 319preserved_tokens .push_back (token ); 320 } 321 } 322 }; 323 324add_token (reasoning .start ); 325add_token (reasoning .end ); 326add_token (content .start ); 327add_token (content .end ); 328add_token (tools .format .section_start ); 329add_token (tools .format .section_end ); 330add_token (tools .format .per_call_start ); 331add_token (tools .format .per_call_end ); 332add_token (tools .function .name_prefix ); 333add_token (tools .function .name_suffix ); 334add_token (tools .function .args_separator ); 335add_token (tools .function .close ); 336add_token (tools .arguments .start ); 337add_token (tools .arguments .end ); 338add_token (tools .arguments .name_prefix ); 339add_token (tools .arguments .name_suffix ); 340add_token (tools .arguments .separator ); 341add_token (tools .arguments .value_prefix ); 342add_token (tools .arguments .value_suffix ); 343add_token (tools .call_id .prefix ); 344add_token (tools .call_id .suffix ); 345} 346 347std::string autoparser::detect_assistant_start_marker (const common_chat_template & tmpl ) { 348json user_msg = json { 349 {"role" ,"user" }, 350 {"content" ,USER_MSG } 351 }; 352 353json assistant_no_reasoning = json { 354 {"role" ,"assistant" }, 355 {"content" ,ASSISTANT_MSG } 356 }; 357 358template_params params ; 359params .messages = json::array ({user_msg }); 360params .add_generation_prompt = false; 361params .enable_thinking = true; 362 363auto comparison = compare_variants ( 364tmpl ,params , [& ](template_params & p ) { 365p .messages = json::array ({user_msg ,assistant_no_reasoning }); 366 } 367 ); 368 369if (!comparison ) { 370LOG_DBG (ANSI_ORANGE "%s: Template application failed, skipping assistant start detection\n" ANSI_RESET ,__func__ ); 371return "" ; 372 } 373 374auto usermsg = comparison -> diff .right ; 375if (usermsg .find (ASSISTANT_MSG )== std::string::npos ) { 376LOG_DBG (ANSI_ORANGE "%s: Did not find assistant message in assistant message block, skipping detection\n" ANSI_RESET ,__func__ ); 377 } 378 379auto ast_prefix = usermsg .substr (0 ,usermsg .find (ASSISTANT_MSG )); 380if (!reasoning .start .empty ()&& ast_prefix .find (trim_whitespace (reasoning .start ))!= std::string::npos ) { 381ast_prefix = ast_prefix .substr (0 ,ast_prefix .find (trim_whitespace (reasoning .start ))); 382 } 383if (!reasoning .end .empty ()&& ast_prefix .find (trim_whitespace (reasoning .end ))!= std::string::npos ) { 384ast_prefix = ast_prefix .substr (0 ,ast_prefix .find (trim_whitespace (reasoning .end ))); 385 } 386return trim_whitespace (ast_prefix ); 387} 388 389std::string autoparser::detect_user_start_marker (const common_chat_template & tmpl ) { 390json user_msg = json { 391 {"role" ,"user" }, 392 {"content" ,USER_MSG } 393 }; 394 395json assistant = json { 396 {"role" ,"assistant" }, 397 {"content" ,ASSISTANT_MSG } 398 }; 399 400json user_msg_two = json { 401 {"role" ,"user" }, 402 {"content" ,USER_MSG_TWO } 403 }; 404 405template_params params ; 406params .messages = json::array ({}); 407params .add_generation_prompt = false; 408params .enable_thinking = true; 409 410auto comparison = compare_variants ( 411tmpl ,params , [& ](template_params & p ) { 412p .messages = json::array ({user_msg }); 413 } 414 ); 415 416if (!comparison ) { 417LOG_DBG (ANSI_ORANGE "%s: Template application failed, unsupported empty messages? trying complex variant\n" ANSI_RESET ,__func__ ); 418params .messages = json::array ({user_msg_two ,assistant }); 419comparison = compare_variants ( 420tmpl ,params , [& ](template_params & p ) { 421p .messages = json::array ({user_msg_two ,assistant ,user_msg }); 422 } 423 ); 424if (!comparison ) { 425LOG_DBG (ANSI_ORANGE "%s: Template application failed for reserve variant, aborting\n" ANSI_RESET ,__func__ ); 426return "" ; 427 } 428 } 429 430auto usermsg = comparison -> diff .right ; 431if (usermsg .find (USER_MSG )== std::string::npos ) { 432LOG_DBG (ANSI_ORANGE "%s: Did not find user message in user message block, aborting detection\n" ANSI_RESET ,__func__ ); 433 } 434 435if (usermsg .find (ASSISTANT_MSG )!= std::string::npos ) { 436usermsg = usermsg .substr (usermsg .find (ASSISTANT_MSG )+ ASSISTANT_MSG .size ()); 437 } 438 439auto candidate = usermsg .substr (0 ,usermsg .find (USER_MSG )); 440auto candidate_split = segmentize_markers (candidate ); 441 std::stringstream result ; 442bool encountered_marker = false; 443for (const auto & mrk :candidate_split ) { 444 std::string lower_mrk = std::string (mrk .value ); 445 std::transform (lower_mrk .begin (),lower_mrk .end (),lower_mrk .begin (), 446 [](unsigned char c ) {return std::tolower (c ); }); 447// heuristic to weed out potential end markers, but only at the start 448if (mrk .type == segment_type::MARKER && !encountered_marker && 449 (lower_mrk .find ("end" )!= std::string::npos || lower_mrk .find ("close" )!= std::string::npos )) { 450continue ; 451 } 452if (mrk .type == segment_type::TEXT && !encountered_marker && trim_whitespace (mrk .value ).empty ()) { 453continue ; 454 } 455encountered_marker |=mrk .type == segment_type::MARKER ; 456result <<mrk .value ; 457 } 458return trim_whitespace (result .str ()); 459} 460 461analyze_reasoning::analyze_reasoning (const common_chat_template & tmpl ,bool supports_tools ) 462 :analyze_base (tmpl ) { 463LOG_DBG (ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET ); 464LOG_DBG (ANSI_ORANGE "Phase 1: Reasoning analysis\n" ANSI_RESET ); 465 466compare_reasoning_presence (); 467compare_thinking_enabled (); 468if (supports_tools ) { 469compare_reasoning_scope (); 470 } 471} 472 473void analyze_reasoning::compare_reasoning_presence () { 474json user_msg = json { 475 {"role" ,"user" }, 476 {"content" ,USER_MSG } 477 }; 478 479json assistant_no_reasoning = json { 480 {"role" ,"assistant" }, 481 {"content" ,ASSISTANT_MSG } 482 }; 483 484json assistant_with_reasoning = json { 485 {"role" ,"assistant" }, 486 {"content" ,ASSISTANT_MSG }, 487 {"reasoning_content" ,THINKING_CONTENT } 488 }; 489 490template_params params ; 491params .messages = json::array ({user_msg ,assistant_no_reasoning }); 492params .add_generation_prompt = false; 493params .enable_thinking = true; 494 495auto comparison = compare_variants ( 496* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_with_reasoning }); }); 497 498if (!comparison ) { 499LOG_DBG (ANSI_ORANGE "%s: Template application failed, skipping reasoning detection\n" ANSI_RESET ,__func__ ); 500return ; 501 } 502 503const auto & diff = comparison -> diff ; 504 505const std::string reasoning_content = THINKING_CONTENT ; 506 507if (!diff .right .empty ()&& diff .right .find (reasoning_content )!= std::string::npos ) { 508auto parser_delimiter = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 509return p .literal (reasoning_content )+ p .space ()+ p .optional (p .tag ("post" , (p .marker ()+ p .space ()))+ p .rest ()); 510 }); 511auto parser_wrapped = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 512return p .tag ("pre" ,p .marker ()+ p .space ())+ p .literal (reasoning_content )+ p .tag ("post" , (p .space ()+ p .marker ()+ p .space ()))+ p .rest (); 513 }); 514// try the more aggressive parse first, if it fails, fall back to the delimiter one 515auto result = parser_wrapped .parse_anywhere_and_extract (comparison -> output_B ); 516if (!result .result .success ()) { 517result = parser_delimiter .parse_anywhere_and_extract (comparison -> output_B ); 518 } 519if (result .result .success ()) { 520if (!result .tags ["pre" ].empty ()&& !result .tags ["post" ].empty ()) { 521mode = reasoning_mode::TAG_BASED ; 522start = result .tags ["pre" ]; 523end = result .tags ["post" ]; 524 }else if (!result .tags ["post" ].empty ()) { 525mode = reasoning_mode::TAG_BASED ; 526end = result .tags ["post" ]; 527 } 528 } 529 } 530} 531 532void analyze_reasoning::compare_thinking_enabled () { 533json user_msg = json { 534 {"role" ,"user" }, 535 {"content" ,USER_MSG } 536 }; 537 538template_params params ; 539params .messages = json::array ({user_msg }); 540params .add_generation_prompt = true; 541params .enable_thinking = false; 542 543auto comparison = compare_variants (* tmpl ,params , [& ](template_params & p ) {p .enable_thinking = true; }); 544 545if (!comparison ) { 546LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 547return ; 548 } 549 550const auto & diff = comparison -> diff ; 551 552 std::string left_trimmed = trim_whitespace (diff .left ); 553 std::string right_trimmed = trim_whitespace (diff .right ); 554 555if (left_trimmed .empty ()&& !diff .right .empty ()) { 556if (!right_trimmed .empty ()&& string_ends_with (comparison -> output_B ,right_trimmed )) { 557if (start .empty ()) { 558start = diff .right ; 559mode = reasoning_mode::TAG_BASED ; 560 } 561 } 562 }else if (right_trimmed .empty ()&& !diff .left .empty ()) { 563if (!left_trimmed .empty ()&& string_ends_with (comparison -> output_A ,left_trimmed )) { 564if (end .empty ()) { 565auto seg = prune_whitespace_segments (segmentize_markers (comparison -> output_A )); 566if (seg .size () >=2 && seg [seg .size ()- 1 ].value == left_trimmed && seg [seg .size ()- 2 ].type == segment_type::MARKER ) { 567start = seg [seg .size ()- 2 ].value ; 568 } 569end = diff .left ; 570mode = reasoning_mode::TAG_BASED ; 571 } 572 } 573 }else if (!left_trimmed .empty ()&& !right_trimmed .empty ()) { 574// Full-output diff is noisy (e.g., SmolLM3 changes the system message when enable_thinking flips). 575// Try to find reasoning markers by tail-anchoring: 576// one output's generation prompt tail may appear in the other with extra reasoning markers appended. 577const auto & output_A = comparison -> output_A ; 578const auto & output_B = comparison -> output_B ; 579const size_t anchor_len = 64 ; 580 581for (int dir = 0 ;dir < 2 ;dir ++ ) { 582const auto & base = dir == 0 ?output_B :output_A ; 583const auto & extended = dir == 0 ?output_A :output_B ; 584 585size_t len = std::min (base .size (),anchor_len ); 586 std::string anchor = base .substr (base .size ()- len ); 587auto pos = extended .rfind (anchor ); 588if (pos == std::string::npos || pos + len >=extended .size ()) { 589continue ; 590 } 591 592 std::string extra = trim_whitespace (extended .substr (pos + len )); 593if (extra .empty ()) { 594continue ; 595 } 596 597auto seg = prune_whitespace_segments (segmentize_markers (extra )); 598if (seg .size ()== 2 && seg [0 ].type == segment_type::MARKER && seg [1 ].type == segment_type::MARKER ) { 599if (start .empty ()) { 600start = seg [0 ].value ; 601 } 602if (end .empty ()) { 603end = seg [1 ].value ; 604 } 605mode = reasoning_mode::TAG_BASED ; 606break ; 607 } 608 } 609 } 610 611if (mode == reasoning_mode::NONE && start .empty ()&& !end .empty ()) { 612mode = reasoning_mode::TAG_BASED ; 613 } 614} 615 616void analyze_reasoning::compare_reasoning_scope () { 617json assistant_reasoning_content = json { 618 {"role" ,"assistant" }, 619 {"content" ,ASSISTANT_MSG }, 620 {"reasoning_content" ,THINKING_CONTENT } 621 }; 622 623json assistant_reasoning_tools = json { 624 {"role" ,"assistant" }, 625 {"content" ,nullptr }, 626 {"reasoning_content" ,THINKING_CONTENT }, 627 {"tool_calls" , 628 json::array ({build_tool_call (FUN_FIRST ,json { {ARG_FIRST ,"VVVV" }, {ARG_SECOND ,"XXXX" } }) }) } 629 }; 630 631template_params params ; 632params .messages = json::array ({user_msg ,assistant_reasoning_content }); 633params .tools = tools ; 634params .add_generation_prompt = false; 635params .enable_thinking = true; 636 637auto comparison = compare_variants ( 638* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_reasoning_tools }); }); 639 640if (!comparison ) { 641LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 642return ; 643 } 644 645 std::string reasoning_content = THINKING_CONTENT ; 646 647// Check if reasoning only appears in variant B (with tools) 648bool reasoning_in_A = comparison -> output_A .find (reasoning_content )!= std::string::npos ; 649bool reasoning_in_B = comparison -> output_B .find (reasoning_content )!= std::string::npos ; 650 651if (!reasoning_in_A && reasoning_in_B ) { 652mode = reasoning_mode::TOOLS_ONLY ; 653LOG_DBG (ANSI_ORANGE "%s: Detected TOOLS_ONLY reasoning mode\n" ANSI_RESET ,__func__ ); 654 655auto parser_wrapped = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 656return p .tag ("pre" ,p .marker ()+ p .space ())+ p .literal (reasoning_content )+ p .space ()+ p .tag ("post" , (p .marker ()+ p .space ())); 657 }); 658auto result = parser_wrapped .parse_anywhere_and_extract (comparison -> output_B ); 659if (result .result .success ()) { 660start = result .tags ["pre" ]; 661end = result .tags ["post" ]; 662 }else { 663auto parser_delimiter = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 664return p .literal (reasoning_content )+ p .space ()+ p .optional (p .tag ("post" , (p .marker ()+ p .space ()))); 665 }); 666result = parser_delimiter .parse_anywhere_and_extract (comparison -> output_B ); 667if (result .result .success ()) { 668end = result .tags ["post" ]; 669 }else { 670LOG_DBG (ANSI_ORANGE "%s: Unable to extract reasoning markers, falling back to reasoning = NONE\n" ANSI_RESET ,__func__ ); 671mode = reasoning_mode::NONE ; 672 } 673 } 674 } 675} 676 677analyze_content::analyze_content (const common_chat_template & tmpl ,const analyze_reasoning & reasoning ) 678 :analyze_base (tmpl ) { 679LOG_DBG (ANSI_ORANGE "Phase 2: Content analysis\n" ANSI_RESET ); 680 681json assistant_content_only = json { 682 {"role" ,"assistant" }, 683 {"content" ,ASSISTANT_MSG } 684 }; 685 686json assistant_with_tools = json { 687 {"role" ,"assistant" }, 688 {"content" ,"" }, 689 {"tool_calls" , json::array ({build_tool_call ("test_func" ,json { {"arg1" ,"value1" } }) }) } 690 }; 691 692json assistant_with_reasoning = json { 693 {"role" ,"assistant" }, 694 {"content" ,"" }, 695 {"reasoning_content" ,THINKING_CONTENT } 696 }; 697 698template_params params_content_only ; 699params_content_only .messages = json::array ({user_msg ,assistant_content_only }); 700params_content_only .add_generation_prompt = false; 701params_content_only .enable_thinking = true; 702params_content_only .tools = tools ; 703 704auto comparison_with_tools = compare_variants (tmpl ,params_content_only , [& ](template_params & p ) { 705p .messages = json::array ({user_msg ,assistant_with_tools }); 706 }); 707 708auto comparison_with_reasoning = compare_variants (tmpl ,params_content_only , [& ](template_params & p ) { 709p .messages = json::array ({user_msg ,assistant_with_reasoning }); 710 }); 711 712if (!comparison_with_tools || !comparison_with_reasoning ) { 713LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 714return ; 715 } 716 717const auto & diff_tools = comparison_with_tools -> diff ; 718const auto & diff_reasoning = comparison_with_reasoning -> diff ; 719 720 std::string response = ASSISTANT_MSG ; 721 722bool found_plain_content = false; 723if (trim_whitespace (diff_tools .left )== response ) { 724auto parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 725return p .space ()+ diff_reasoning .left + p .space ()+ p .optional (p .marker ())+ p .space ()+ p .end (); 726 }); 727if (parser .parse_and_extract (diff_reasoning .left ).result .success ()) { 728// We only have the content text in the diff (possibly with a stray EOG marker), so no markers 729mode = content_mode::PLAIN ; 730found_plain_content = true; 731 }else if (reasoning .mode != reasoning_mode::NONE && !reasoning .end .empty ()) { 732auto post_reasoning_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 733return p .literal (reasoning .end )+ p .space ()+ p .literal (response ); 734 }); 735if (post_reasoning_parser .parse_anywhere_and_extract (diff_reasoning .left ).result .success ()) { 736mode = content_mode::PLAIN ; 737found_plain_content = true; 738 } 739 } 740 } 741if (!found_plain_content ) { 742 std::string rdiff = diff_reasoning .left ; 743if (!reasoning .end .empty ()&& rdiff .find (reasoning .end )!= std::string::npos ) { 744rdiff = rdiff .substr (rdiff .find (reasoning .end )+ reasoning .end .length ()); 745 } 746// Take the more promising diff 747 std::string pure_content = rdiff .length ()> diff_tools .left .length () ?rdiff :diff_tools .left ; 748auto parser_wrapped = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 749return p .tag ("pre" ,p .marker ()+ p .space ())+ p .literal (response )+ p .space ()+ p .tag ("post" , (p .marker ()+ p .space ()))+ p .rest (); 750 }); 751auto result = parser_wrapped .parse_anywhere_and_extract (pure_content ); 752start = result .tags ["pre" ]; 753end = result .tags ["post" ]; 754// TODO: WRAPPED_WITH_REASONING 755 } 756 757// Determine content mode 758if (!start .empty ()|| !end .empty ()) { 759mode = content_mode::ALWAYS_WRAPPED ; 760// TODO: END_DELIMITED content mode - delimited at end but not at start? 761 } 762} 763 764bool analyze_content::is_always_wrapped ()const { 765return mode == content_mode::ALWAYS_WRAPPED && !start .empty ()&& !end .empty (); 766} 767 768analyze_tools::analyze_tools (const common_chat_template & tmpl , 769const jinja::caps & caps , 770const analyze_reasoning & reasoning ) 771 :analyze_base (tmpl ) { 772LOG_DBG (ANSI_ORANGE "Phase 3: Tool call analysis\n" ANSI_RESET ); 773 774analyze_tool_calls (reasoning ,caps .supports_parallel_tool_calls ); 775 776if (format .mode != tool_format::NONE && format .mode != tool_format::JSON_NATIVE ) { 777if (caps .supports_parallel_tool_calls ) { 778check_per_call_markers (); 779 } 780LOG_DBG (ANSI_ORANGE "Phase 3a: Function call analysis\n" ANSI_RESET ); 781extract_function_markers (); 782LOG_DBG (ANSI_ORANGE "Phase 3b: Argument analysis\n" ANSI_RESET ); 783if (format .mode == tool_format::TAG_WITH_TAGGED ) { 784analyze_arguments (); 785 } 786extract_argument_separator (); 787extract_args_markers (); 788LOG_DBG (ANSI_ORANGE "Phase 3c: Call id analysis\n" ANSI_RESET ); 789extract_call_id_markers (); 790 } 791} 792 793void analyze_tools::analyze_tool_calls (const analyze_reasoning & reasoning ,bool supports_parallel_tool_calls ) { 794json assistant_no_tools = json { 795 {"role" ,"assistant" }, 796 {"content" ,ASSISTANT_MSG } 797 }; 798 799json assistant_with_tools = json { 800 {"role" ,"assistant" }, 801 {"content" ,"" }, 802 {"tool_calls" , json::array ({first_tool_call }) } 803 }; 804 805template_params params ; 806params .messages = json::array ({user_msg ,assistant_no_tools }); 807params .tools = tools ; 808params .add_generation_prompt = false; 809params .enable_thinking = true; 810 811auto comparison = compare_variants ( 812* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_with_tools }); }); 813 814if (!comparison ) { 815LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 816return ; 817 } 818 819const auto & diff = comparison -> diff ; 820 821 std::string tool_section = diff .right ; 822 823if (tool_section .empty ()) { 824return ; 825 } 826 827analyze_tool_call_format (tool_section ,FUN_FIRST ,ARG_FIRST ,reasoning ,supports_parallel_tool_calls ); 828} 829 830void analyze_tools::analyze_tool_call_format (const std::string & haystack , 831const std::string & fun_name_needle , 832const std::string & arg_name_needle , 833const analyze_reasoning & reasoning , 834bool supports_parallel_tool_calls ) { 835if (fun_name_needle .empty ()|| arg_name_needle .empty ()|| haystack .empty ()) { 836return ; 837 } 838 839auto in_json_haystack = [& haystack ](const std::string & needle )-> bool { 840auto parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 841return p .choice ({p .literal ("{" ),p .literal (":" ) }) <<p .choice ({ 842p .tag ("dq" ,p .literal ("\"" )+ p .literal (needle )+ p .literal ("\"" )) }); 843 }); 844auto result = parser .parse_anywhere_and_extract (haystack ); 845return result .result .success (); 846 }; 847 848auto fun_quote = in_json_haystack (fun_name_needle ); 849auto arg_quote = in_json_haystack (arg_name_needle ); 850 851if (fun_quote ) { 852// no need to check further, we're in JSON land 853format .mode = tool_format::JSON_NATIVE ; 854 }else if (arg_quote ) { 855format .mode = tool_format::TAG_WITH_JSON ; 856 }else { 857format .mode = tool_format::TAG_WITH_TAGGED ; 858 } 859 860// first, remove any reasoning markers 861 std::string clean_haystack = haystack ; 862if (!reasoning .start .empty ()) { 863auto pos = haystack .find (reasoning .start ); 864if (pos != std::string::npos ) { 865clean_haystack = haystack .substr (0 ,pos )+ haystack .substr (pos + reasoning .start .length ()); 866 } 867 } 868if (!reasoning .end .empty ()) { 869auto pos = clean_haystack .find (reasoning .end ); 870if (pos != std::string::npos ) { 871clean_haystack = clean_haystack .substr (0 ,pos )+ clean_haystack .substr (pos + reasoning .end .length ()); 872 } 873 } 874 875if (format .mode == tool_format::JSON_NATIVE ) { 876analyze_tool_call_format_json_native (clean_haystack ,fun_name_needle ,arg_name_needle ); 877if (supports_parallel_tool_calls ) { 878analyze_json_native_parallel_calls (); 879 } 880 }else { 881analyze_tool_call_format_non_json (clean_haystack ,fun_name_needle ); 882 } 883// always relax whitespace requirements on ending markers since they don't influence content 884format .section_end = trim_whitespace (format .section_end ); 885format .per_call_end = trim_whitespace (format .per_call_end ); 886} 887 888void analyze_tools::analyze_json_native_parallel_calls () { 889json assistant_one_tool = json { 890 {"role" ,"assistant" }, 891 {"content" ,"" }, 892 {"tool_calls" , json::array ({first_tool_call }) } 893 }; 894 895json assistant_two_tools = json { 896 {"role" ,"assistant" }, 897 {"content" ,"" }, 898 {"tool_calls" , json::array ({first_tool_call ,second_tool_call }) } 899 }; 900 901template_params params ; 902params .messages = json::array ({user_msg ,assistant_one_tool }); 903params .tools = tools ; 904params .add_generation_prompt = false; 905params .enable_thinking = true; 906 907auto comparison = compare_variants ( 908* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_two_tools }); }); 909 910if (!comparison ) { 911LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 912return ; 913 } 914 915 std::string & second_call = comparison -> diff .right ; 916if (!format .section_start .empty ()&& second_call .find (format .section_start )!= std::string::npos ) { 917format .per_call_start = format .section_start ; 918format .per_call_end = format .section_end ; 919format .section_start .clear (); 920format .section_end .clear (); 921 } 922} 923 924void analyze_tools::analyze_tool_call_format_json_native (const std::string & clean_haystack , 925const std::string & fun_name_needle , 926const std::string & arg_name_needle ) { 927// we might not have the typical OpenAI tool calling structure 928int json_start = clean_haystack .find_first_of ('{' ); 929int json_end = clean_haystack .find_last_of ('}' ); 930 std::string cut = clean_haystack .substr (json_start ,json_end - json_start + 1 ); 931json call_struct = json::parse (cut ); 932auto register_field = [& ](const std::string & prefix ,const common_json_entry & subel ) { 933if (subel .value ().is_string ()&& std::string (subel .value ()).find ("call0000" )!= std::string::npos ) { 934format .id_field = !prefix .empty () ?prefix + "." + subel .key () :subel .key (); 935 }else if (subel .value ().is_string ()&& std::string (subel .value ())== fun_name_needle ) { 936format .name_field = !prefix .empty () ?prefix + "." + subel .key () :subel .key (); 937 }else if (subel .value ().dump ().find (arg_name_needle )!= 938 std::string::npos ) {// handle both string and JSON obj variants 939format .args_field = !prefix .empty () ?prefix + "." + subel .key () :subel .key (); 940 }else if (subel .key ().find ("id" )!= std::string::npos ) { 941// heuristics for generated id field 942format .gen_id_field = !prefix .empty () ?prefix + "." + subel .key () :subel .key (); 943 } 944 }; 945for (const auto & el :call_struct .items ()) { 946if (el .key ()== fun_name_needle ) { 947format .fun_name_is_key = true; 948// When function name is the key, there's no name field and args are direct 949format .name_field .clear (); 950format .args_field .clear (); 951// Don't register this element - the function name IS the key, not a field 952 }else { 953if (el .value ().is_object ()&& 954el .value ().dump ().find (arg_name_needle )== std::string::npos ) {// not the args object 955format .function_field = el .key (); 956for (const auto & subel :el .value ().items ()) { 957register_field (el .key (),subel ); 958 } 959 } 960// Register this element as a potential field 961register_field ("" ,el ); 962 } 963 } 964auto array_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 965return p .tag ("pre" ,p .literal ("[" )+ p .space ())+ p .literal (cut )+ p .tag ("post" ,p .space ()+ p .literal ("]" )); 966 }); 967 968auto ar_parse_res = array_parser .parse_anywhere_and_extract (clean_haystack ); 969if (ar_parse_res .result .success ()) { 970format .tools_array_wrapped = true; 971json_start -= ar_parse_res .tags ["pre" ].length (); 972json_end += ar_parse_res .tags ["post" ].length (); 973 } 974json_end ++ ;// we want to move past the closing char for end marker extraction 975 976 std::vector < std::pair < size_t , std::string >> located_params ; 977if (!format .name_field .empty ()) { 978located_params .push_back ({clean_haystack .find (format .name_field ),format .name_field }); 979 } 980if (!format .args_field .empty ()) { 981located_params .push_back ({clean_haystack .find (format .args_field ),format .args_field }); 982 } 983if (!format .id_field .empty ()) { 984located_params .push_back ({clean_haystack .find (format .id_field ),format .id_field }); 985 } 986if (!format .gen_id_field .empty ()) { 987located_params .push_back ({clean_haystack .find (format .gen_id_field ),format .gen_id_field }); 988 } 989 std::sort (located_params .begin (),located_params .end ()); 990for (auto & pair :located_params ) { 991format .parameter_order .push_back (pair .second ); 992 } 993// we can immediately extract tool calling markers too 994format .section_start = trim_leading_whitespace (clean_haystack .substr (0 ,json_start )); 995format .section_end = trim_whitespace (clean_haystack .substr (json_end )); 996// When tools_array_wrapped is true, the closing bracket is part of the array structure, 997// not a separate section end marker. Clear tool_section_end to avoid duplicate brackets. 998if (format .tools_array_wrapped && format .section_end == "]" ) { 999format .section_end .clear (); 1000 } 1001} 1002 1003void analyze_tools::analyze_tool_call_format_non_json (const std::string & clean_haystack , 1004const std::string & fun_name_needle ) { 1005// first, let's find out if the function is inside a tag or standalone 1006auto fun_marker_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1007return p .tag ("fun_marker" ,p .choice ({ 1008p .tag ("fun_pre" ,p .literal ("<" )+ p .until_one_of ({">" ,fun_name_needle }))+ p .literal (fun_name_needle )+ 1009p .tag ("fun_post" ,p .negate (p .space ()+ p .literal ("<" ))+ p .until (">" )+ p .literal (">" ))+ p .space (), 1010p .tag ("fun_pre" ,p .literal ("[" )+ p .until_one_of ({"]" ,fun_name_needle }))+ p .literal (fun_name_needle )+ 1011p .tag ("fun_post" ,p .negate (p .space ()+ p .literal ("[" )+ p .until ("]" )+ p .literal ("]" ))+ p .space ()) })); 1012 }); 1013auto fun_res = fun_marker_parser .parse_anywhere_and_extract (clean_haystack ); 1014 std::string fun_marker = fun_name_needle ; 1015if (fun_res .result .success ()) { 1016fun_marker = fun_res .tags ["fun_marker" ]; 1017 } 1018// now, consume up to two markers, then treat everything up to the function marker as function name prefix 1019auto per_tool_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1020return p .tag ("sec_start" ,p .marker ()+ p .space ())+ p .tag ("call_start" ,p .marker ()+ p .space ())+ 1021p .tag ("fun_pre" ,p .until (fun_marker ))+ fun_marker + p .tag ("rest" ,p .rest ()); 1022 }); 1023auto section_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1024return p .tag ("sec_start" ,p .marker ()+ p .space ())+ fun_marker + p .tag ("rest" ,p .rest ()); 1025 }); 1026auto result = per_tool_parser .parse_anywhere_and_extract (clean_haystack ); 1027tagged_parse_result result_end ; 1028if (result .result .success ()) { 1029auto double_closer_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1030return p .tag ("call_end" ,p .marker ()+ p .space ())+ p .tag ("sec_end" ,p .marker ()+ p .space ())+ p .end (); 1031 }); 1032result_end = double_closer_parser .parse_anywhere_and_extract (result .tags ["rest" ]); 1033function .name_prefix = fun_res .tags ["fun_pre" ]+ function .name_prefix ; 1034 }else { 1035result = section_parser .parse_anywhere_and_extract (clean_haystack ); 1036auto single_closer_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1037return p .tag ("sec_end" ,p .marker ()+ p .space ())+ p .end (); 1038 }); 1039result_end = single_closer_parser .parse_anywhere_and_extract (result .tags ["rest" ]); 1040 } 1041format .per_call_start = result .tags ["call_start" ]; 1042format .per_call_end = result_end .tags ["call_end" ]; 1043format .section_start = result .tags ["sec_start" ]; 1044format .section_end = result_end .tags ["sec_end" ]; 1045} 1046 1047void analyze_tools::check_per_call_markers () { 1048json assistant_one_tool = json { 1049 {"role" ,"assistant" }, 1050 {"content" ,"" }, 1051 {"tool_calls" , json::array ({first_tool_call }) } 1052 }; 1053 1054json assistant_two_tools = json { 1055 {"role" ,"assistant" }, 1056 {"content" ,"" }, 1057 {"tool_calls" , json::array ({first_tool_call ,second_tool_call }) } 1058 }; 1059 1060template_params params ; 1061params .messages = json::array ({user_msg ,assistant_one_tool }); 1062params .tools = tools ; 1063params .add_generation_prompt = false; 1064params .enable_thinking = true; 1065 1066auto one_vs_two = compare_variants ( 1067* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_two_tools }); }); 1068 1069if (!one_vs_two ) { 1070LOG_DBG (ANSI_ORANGE "%s: Generating double tool call comparison failed\n" ANSI_RESET ,__func__ ); 1071return ; 1072 } 1073 1074diff_split filter_common_call_part = calculate_diff_split (one_vs_two -> diff .suffix ,one_vs_two -> diff .right ); 1075 1076 std::string second_tool_content = trim_leading_whitespace (filter_common_call_part .right ); 1077if (!format .section_start .empty ()&& 1078second_tool_content .find (format .section_start )== 0 ) { 1079format .per_call_start = format .section_start ; 1080format .per_call_end = format .section_end ; 1081format .section_start .clear (); 1082format .section_end .clear (); 1083 } 1084 1085if (!format .per_call_end .empty ()) { 1086auto count_occurrences = [](const std::string & haystack ,const std::string & needle ) { 1087size_t count = 0 ; 1088for (size_t pos = haystack .find (needle );pos != std::string::npos ; 1089pos = haystack .find (needle ,pos + needle .size ())) { 1090count ++ ; 1091 } 1092return count ; 1093 }; 1094size_t calls_one = count_occurrences (one_vs_two -> output_A ,format .per_call_end ); 1095size_t calls_two = count_occurrences (one_vs_two -> output_B ,format .per_call_end ); 1096if (calls_one > 0 && calls_one == calls_two ) { 1097format .section_end = format .per_call_end ; 1098format .per_call_end .clear (); 1099 } 1100 } 1101} 1102 1103void analyze_tools::extract_function_markers () { 1104json assistant_nocall = json { 1105 {"role" ,"assistant" }, 1106 {"content" ,ASSISTANT_MSG }, 1107 }; 1108 1109json assistant_foofoo = json { 1110 {"role" ,"assistant" }, 1111 {"content" ,"" }, 1112 {"tool_calls" , json::array ({first_tool_call }) } 1113 }; 1114 1115json assistant_barbar = json { 1116 {"role" ,"assistant" }, 1117 {"content" ,"" }, 1118 {"tool_calls" , json::array ({second_tool_call }) } 1119 }; 1120 1121template_params params ; 1122params .messages = json::array ({user_msg ,assistant_foofoo }); 1123params .tools = tools ; 1124params .add_generation_prompt = false; 1125params .enable_thinking = true; 1126 1127auto comparison = compare_variants ( 1128* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_barbar }); }); 1129 1130if (!comparison ) { 1131LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 1132return ; 1133 } 1134 1135const auto & diff = comparison -> diff ; 1136 1137if (diff .left .find (FUN_FIRST )!= std::string::npos && diff .right .find (FUN_SECOND )!= std::string::npos ) { 1138 std::string prefix_marker ; 1139if (!format .per_call_start .empty ()) { 1140prefix_marker = format .per_call_start ; 1141 }else { 1142prefix_marker = format .section_start ; 1143 } 1144if (!prefix_marker .empty ()&& diff .prefix .rfind (prefix_marker )!= std::string::npos ) { 1145function .name_prefix = 1146diff .prefix .substr (diff .prefix .rfind (prefix_marker )+ prefix_marker .size ()); 1147 } 1148 1149// Extract name prefix/suffix from diff.left (stop at the next marker boundary) 1150auto name_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1151return p .tag ("pre" ,p .until (FUN_FIRST ))+ p .literal (FUN_FIRST )+ 1152p .tag ("post" ,p .zero_or_more (p .negate (p .marker ())+ p .any ())); 1153 }); 1154auto name_result = name_parser .parse_and_extract (diff .left ); 1155if (name_result .result .success ()) { 1156function .name_prefix += name_result .tags ["pre" ]; 1157function .name_suffix = name_result .tags ["post" ]; 1158 } 1159 1160// Extend name_suffix with content from diff.suffix before args begin 1161if (format .mode == tool_format::TAG_WITH_JSON ) { 1162// For JSON: name_suffix extends to the first non-marker { or [, including any 1163// markers along the way. Only applies if there's at least one marker after 1164// the JSON content (matching the original "stop < seg_suf.size() - 1" guard). 1165auto suffix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1166auto non_json = p .marker () | (p .negate (p .literal ("{" ))+ p .negate (p .literal ("[" ))+ p .any ()); 1167auto after_json = p .zero_or_more (p .negate (p .marker ())+ p .any ())+ p .marker (); 1168return p .tag ("ext" ,p .zero_or_more (non_json ))+ after_json ; 1169 }); 1170auto suf_result = suffix_parser .parse_and_extract (diff .suffix ); 1171if (suf_result .result .success ()) { 1172function .name_suffix += suf_result .tags ["ext" ]; 1173 } 1174 }else { 1175// For tagged: name_suffix extends to the first marker (arg marker) 1176auto suffix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1177return p .tag ("ext" ,p .zero_or_more (p .negate (p .marker ())+ p .any ())); 1178 }); 1179auto suf_result = suffix_parser .parse_and_extract (diff .suffix ); 1180if (suf_result .result .success ()) { 1181function .name_suffix += suf_result .tags ["ext" ]; 1182 1183auto arg_start = [& ](common_peg_parser_builder & p ) { 1184return p .marker ()+ p .space ()+ p .choice ({p .literal (ARG_FIRST ),p .literal (ARG_SECOND ) }); 1185 }; 1186auto sep_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1187return p .tag ("sep" ,p .zero_or_more (p .negate (arg_start (p ))+ p .any ()))+ arg_start (p ); 1188 }); 1189auto sep_result = sep_parser .parse_and_extract (diff .suffix .substr (suf_result .tags ["ext" ].size ())); 1190if (sep_result .result .success ()) { 1191function .args_separator = trim_whitespace (sep_result .tags ["sep" ]); 1192 } 1193 } 1194 } 1195 1196// Extract the closer (between last arg and call/section end marker) 1197 std::string suffix_marker ; 1198if (!format .per_call_end .empty ()) { 1199suffix_marker = format .per_call_end ; 1200 }else { 1201suffix_marker = format .section_end ; 1202 } 1203 std::string closer_suffix ; 1204if (suffix_marker .empty ()) { 1205// we'll have to rely on an extra diff with no-calls version 1206auto notool_comp = compare_variants ( 1207* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_nocall }); }); 1208if (notool_comp ) { 1209auto nt_diff = notool_comp -> diff ; 1210closer_suffix = nt_diff .left .substr (nt_diff .left .find ("YYYY" )+ 4 ); 1211 } 1212 }else { 1213closer_suffix = diff .suffix .substr (0 ,diff .suffix .find (suffix_marker )); 1214 } 1215if (!closer_suffix .empty ()) { 1216if (format .mode == tool_format::TAG_WITH_TAGGED ) { 1217// After last arg value, skip the closing arg marker, rest is closer 1218auto closer_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1219return p .until ("YYYY" )+ p .literal ("YYYY" )+ p .space ()+ 1220p .marker ()+ p .space ()+ 1221p .tag ("close" ,p .rest ()); 1222 }); 1223auto close_result = closer_parser .parse_and_extract (closer_suffix ); 1224if (close_result .result .success ()) { 1225function .close = close_result .tags ["close" ]; 1226 } 1227 }else if (format .mode == tool_format::TAG_WITH_JSON ) { 1228// After last arg value, find end of JSON args, rest is closer 1229auto closer_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1230return p .until ("YYYY" )+ p .literal ("YYYY" )+ p .tag ("post_val" ,p .rest ()); 1231 }); 1232auto close_result = closer_parser .parse_and_extract (closer_suffix ); 1233if (close_result .result .success ()) { 1234const auto & post = close_result .tags ["post_val" ]; 1235size_t pos = post .find_last_of ("}]" ); 1236if (pos != std::string::npos && pos < post .size ()- 1 ) { 1237function .close = trim_leading_whitespace (post .substr (pos + 1 )); 1238 } 1239 } 1240 } 1241 } 1242function .close = trim_leading_whitespace (function .close ); 1243 } 1244} 1245 1246void analyze_tools::analyze_arguments () { 1247extract_argument_name_markers (); 1248extract_argument_value_markers (); 1249} 1250 1251void analyze_tools::extract_argument_name_markers () { 1252json assistant_first_arg = json { 1253 {"role" ,"assistant" }, 1254 {"content" ,"" }, 1255 {"tool_calls" , json::array ({first_tool_call_one_arg }) } 1256 }; 1257 1258json assistant_second_arg = json { 1259 {"role" ,"assistant" }, 1260 {"content" ,"" }, 1261 {"tool_calls" , json::array ({first_tool_call_other_arg }) } 1262 }; 1263 1264template_params params ; 1265params .messages = json::array ({user_msg ,assistant_first_arg }); 1266params .tools = tools ; 1267params .add_generation_prompt = false; 1268params .enable_thinking = true; 1269 1270auto comparison = compare_variants ( 1271* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_second_arg }); }); 1272 1273if (!comparison ) { 1274LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 1275return ; 1276 } 1277 1278const auto & diff = comparison -> diff ; 1279 1280if (!diff .left .empty ()&& !diff .right .empty ()) { 1281// Parse both sides to find ARG_FIRST/ARG_SECOND and extract the surrounding structure 1282auto left_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1283return p .tag ("pre" ,p .until (ARG_FIRST ))+ p .literal (ARG_FIRST )+ 1284p .tag ("suffix" ,p .until_one_of ({"\"" ,"X" })); 1285 }); 1286auto right_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1287return p .tag ("pre" ,p .until (ARG_SECOND ))+ p .literal (ARG_SECOND )+ 1288p .tag ("suffix" ,p .until_one_of ({"\"" ,"Y" })); 1289 }); 1290auto left_result = left_parser .parse_anywhere_and_extract (diff .left ); 1291auto right_result = right_parser .parse_anywhere_and_extract (diff .right ); 1292 1293if (left_result .result .success ()&& right_result .result .success ()&& 1294 !left_result .tags ["pre" ].empty ()&& 1295left_result .tags ["pre" ]== right_result .tags ["pre" ]&& 1296left_result .tags ["suffix" ]== right_result .tags ["suffix" ]) { 1297// Name is inside a structure (e.g., JSON key): prefix is the shared wrapper 1298arguments .name_prefix = left_result .tags ["pre" ]; 1299arguments .name_suffix = left_result .tags ["suffix" ]; 1300 }else if (diff .left .substr (0 ,ARG_FIRST .length ())== ARG_FIRST && diff .right .substr (0 ,ARG_SECOND .length ())== ARG_SECOND ) { 1301// Name is directly in the diff: prefix comes from last marker in diff.prefix 1302auto pre_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1303auto last_marker = p .marker ()+ p .zero_or_more (p .negate (p .marker ())+ p .any ())+ p .end (); 1304return p .zero_or_more (p .negate (last_marker )+ p .any ())+ p .tag ("name_prefix" ,last_marker ); 1305 }); 1306auto pre_result = pre_parser .parse_and_extract (diff .prefix ); 1307arguments .name_prefix = pre_result .result .success () 1308 ?pre_result .tags ["name_prefix" ] :diff .prefix ; 1309 1310// Suffix extends from after ARG_FIRST to the first marker (+ optional whitespace). 1311// The marker could be in diff.left itself or in diff.suffix, so we concatenate. 1312 std::string after_first = diff .left .substr (ARG_FIRST .length ())+ diff .suffix ; 1313auto suffix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1314return p .tag ("suffix" ,p .zero_or_more (p .negate (p .marker ())+ p .any ())+ 1315p .marker ()+ p .space ()); 1316 }); 1317auto suf_result = suffix_parser .parse_anywhere_and_extract (after_first ); 1318if (suf_result .result .success ()) { 1319arguments .name_suffix = suf_result .tags ["suffix" ]; 1320 } 1321 } 1322 } 1323} 1324 1325void analyze_tools::extract_argument_value_markers () { 1326json assistant_val_X = json { 1327 {"role" ,"assistant" }, 1328 {"content" ,"" }, 1329 {"tool_calls" , json::array ({first_tool_call_one_arg }) } 1330 }; 1331 1332json assistant_val_Y = json { 1333 {"role" ,"assistant" }, 1334 {"content" ,"" }, 1335 {"tool_calls" , json::array ({first_tool_call_one_arg_other_val }) } 1336 }; 1337 1338template_params params ; 1339params .messages = json::array ({user_msg ,assistant_val_X }); 1340params .tools = tools ; 1341params .add_generation_prompt = false; 1342params .enable_thinking = true; 1343 1344auto comparison = compare_variants ( 1345* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_val_Y }); }); 1346 1347if (!comparison ) { 1348LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 1349return ; 1350 } 1351 1352const auto & diff = comparison -> diff ; 1353 1354if (diff .left == "XXXX" && diff .right == "YYYY" ) { 1355 std::string arg_name_ending = ARG_FIRST + arguments .name_suffix ; 1356 std::string prefix = diff .prefix ; 1357if (prefix .rfind (arg_name_ending )!= std::string::npos ) { 1358prefix = prefix .substr (prefix .rfind (arg_name_ending )+ arg_name_ending .size ()); 1359 } 1360if (!prefix .empty ()) { 1361// Find the last marker + any trailing non-marker text to end 1362auto prefix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1363auto last_marker = p .marker ()+ p .zero_or_more (p .negate (p .marker ())+ p .any ())+ p .end (); 1364return p .zero_or_more (p .negate (last_marker )+ p .any ())+ p .tag ("val_prefix" ,last_marker ); 1365 }); 1366auto pre_result = prefix_parser .parse_and_extract (prefix ); 1367arguments .value_prefix = pre_result .result .success () ?pre_result .tags ["val_prefix" ] :prefix ; 1368 } 1369 1370 std::string value_suffix = diff .suffix ; 1371if (!function .close .empty ()) { 1372size_t func_close_pos = value_suffix .find (function .close ); 1373if (func_close_pos != std::string::npos ) { 1374value_suffix = value_suffix .substr (0 ,func_close_pos ); 1375 } 1376 }else if (!format .per_call_end .empty ()|| !format .section_end .empty ()) { 1377 std::string end_marker = 1378 !format .per_call_end .empty () ?format .per_call_end :format .section_end ; 1379size_t end_marker_pos = value_suffix .find (end_marker ); 1380if (end_marker_pos != std::string::npos ) { 1381value_suffix = value_suffix .substr (0 ,end_marker_pos ); 1382 } 1383 } 1384if (!trim_whitespace (value_suffix ).empty ()) { 1385arguments .value_suffix = value_suffix ; 1386 } 1387 } 1388} 1389 1390void analyze_tools::extract_argument_separator () { 1391json assistant_one_arg = json { 1392 {"role" ,"assistant" }, 1393 {"content" ,"" }, 1394 {"tool_calls" , json::array ({first_tool_call_one_arg }) } 1395 }; 1396 1397json assistant_two_args = json { 1398 {"role" ,"assistant" }, 1399 {"content" ,"" }, 1400 {"tool_calls" , json::array ({first_tool_call }) } 1401 }; 1402 1403template_params params ; 1404params .messages = json::array ({user_msg ,assistant_one_arg }); 1405params .tools = tools ; 1406params .add_generation_prompt = false; 1407params .enable_thinking = true; 1408 1409auto comparison = compare_variants ( 1410* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_two_args }); }); 1411 1412if (!comparison ) { 1413LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 1414return ; 1415 } 1416 1417const auto & diff = comparison -> diff ; 1418 1419if (!diff .right .empty ()) { 1420 std::string separator = until_common_prefix (diff .right ,ARG_FIRST ,ARG_SECOND ); 1421arguments .separator = separator ; 1422 } 1423} 1424 1425void analyze_tools::extract_args_markers () { 1426json assistant_no_args = json { 1427 {"role" ,"assistant" }, 1428 {"content" ,"" }, 1429 {"tool_calls" , json::array ({first_tool_call_zero_args }) } 1430 }; 1431 1432json assistant_with_args = json { 1433 {"role" ,"assistant" }, 1434 {"content" ,"" }, 1435 {"tool_calls" , json::array ({first_tool_call_one_arg }) } 1436 }; 1437 1438template_params params ; 1439params .messages = json::array ({user_msg ,assistant_no_args }); 1440params .tools = tools ; 1441params .add_generation_prompt = false; 1442params .enable_thinking = true; 1443 1444auto comparison = compare_variants ( 1445* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_with_args }); }); 1446 1447if (!comparison ) { 1448LOG_DBG (ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET ,__func__ ); 1449return ; 1450 } 1451 1452const auto & diff = comparison -> diff ; 1453 1454if (format .mode == tool_format::JSON_NATIVE ) { 1455 std::string prefix_marker = !format .section_start .empty () ?format .section_start :format .per_call_start ; 1456 std::string suffix_marker = !format .section_end .empty () ?format .section_end :format .per_call_end ; 1457// these might happen earlier in the tools section as an example or somewhere else, so we need to find the closest ones 1458size_t prefix_pos = prefix_marker .empty () ?0 :diff .prefix .rfind (prefix_marker ); 1459size_t suffix_pos = suffix_marker .empty () ?diff .suffix .size () :diff .suffix .find (suffix_marker ); 1460if (prefix_pos == std::string::npos ) { 1461prefix_pos = 0 ; 1462 } 1463if (suffix_pos == std::string::npos ) { 1464suffix_pos = diff .suffix .size (); 1465 } 1466 std::string prefix_cut = diff .prefix .substr (prefix_pos + prefix_marker .size ()); 1467 std::string suffix_cut = diff .suffix .substr (0 ,suffix_pos ); 1468 std::string args_start = until_common_prefix (prefix_cut ,"{}" ,"{\"first\":" ); 1469 std::string args_end = after_common_suffix (suffix_cut ,"{}" ,"\"XXXX\"}" ); 1470 1471if (!args_start .empty ()|| !args_end .empty ()) { 1472size_t find_fun = args_start .find (FUN_FIRST ); 1473if (find_fun != std::string::npos ) { 1474args_start = args_start .substr (find_fun + FUN_FIRST .size (),args_start .size ()- find_fun - FUN_FIRST .size ()); 1475 } 1476size_t find_call_id = args_start .find (CALL_ID_001 ); 1477if (find_call_id != std::string::npos ) { 1478args_start = args_start .substr (find_call_id + CALL_ID_001 .size (),args_start .size ()- find_call_id - CALL_ID_001 .size ()); 1479 } 1480arguments .start = args_start ; 1481arguments .end = args_end ; 1482 } 1483 } 1484} 1485 1486void analyze_tools::extract_call_id_markers () { 1487json assistant_id1 = json { 1488 {"role" ,"assistant" }, 1489 {"content" ,"" }, 1490 {"tool_calls" , json::array ({first_tool_call }) } 1491 }; 1492 1493json assistant_id2 = json { 1494 {"role" ,"assistant" }, 1495 {"content" ,"" }, 1496 {"tool_calls" , json::array ({first_tool_call_alt_id }) } 1497 }; 1498 1499template_params params ; 1500params .messages = json::array ({user_msg ,assistant_id1 }); 1501params .tools = tools ; 1502params .add_generation_prompt = false; 1503params .enable_thinking = true; 1504 1505auto comparison = compare_variants ( 1506* tmpl ,params , [& ](template_params & p ) {p .messages = json::array ({user_msg ,assistant_id2 }); }); 1507 1508if (!comparison ) { 1509LOG_DBG (ANSI_ORANGE "%s: Template application failed for call_id detection\n" ANSI_RESET ,__func__ ); 1510return ; 1511 } 1512 1513const auto & diff = comparison -> diff ; 1514 1515if (diff .left .empty ()&& diff .right .empty ()) { 1516return ; 1517 } 1518 1519 std::string id_value_1 = CALL_ID_001 ; 1520 std::string id_value_2 = CALL_ID_999 ; 1521 1522size_t common_id_prefix_len = 0 ; 1523for (size_t i = 0 ;i < std::min (id_value_1 .length (),id_value_2 .length ());i ++ ) { 1524if (id_value_1 [i ]== id_value_2 [i ]) { 1525common_id_prefix_len ++ ; 1526 }else { 1527break ; 1528 } 1529 } 1530 std::string common_id_part = id_value_1 .substr (0 ,common_id_prefix_len ); 1531 1532// Check if the function name is in the prefix (normal case: BETWEEN_FUNC_AND_ARGS or POST_ARGS) 1533// or in the suffix (call_id is PRE_FUNC_NAME) 1534 std::string func_name = FUN_FIRST ; 1535size_t func_name_in_prefix = diff .prefix .rfind (func_name ); 1536size_t func_name_in_suffix = diff .suffix .find (func_name ); 1537 1538// Helper: find the last marker in a string (returns just the marker, not trailing text) 1539auto find_last_marker = [](const std::string & str )-> std::string { 1540auto parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1541auto last = p .marker ()+ p .zero_or_more (p .negate (p .marker ())+ p .any ())+ p .end (); 1542return p .zero_or_more (p .negate (last )+ p .any ())+ p .tag ("m" ,p .marker ()); 1543 }); 1544auto res = parser .parse_anywhere_and_extract (str ); 1545return res .result .success () ?res .tags ["m" ] :"" ; 1546 }; 1547 1548// Helper: find the first marker in a string 1549auto find_first_marker = [](const std::string & str )-> std::string { 1550auto parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1551return p .tag ("m" ,p .marker ()); 1552 }); 1553auto res = parser .parse_anywhere_and_extract (str ); 1554return res .result .success () ?res .tags ["m" ] :"" ; 1555 }; 1556 1557if (func_name_in_prefix != std::string::npos && func_name_in_suffix == std::string::npos ) { 1558// Function name is only in prefix - call_id is BETWEEN_FUNC_AND_ARGS or POST_ARGS 1559// Check if args indicator "{" is in prefix or suffix 1560size_t args_in_prefix = diff .prefix .find ('{' ,func_name_in_prefix ); 1561size_t args_in_suffix = diff .suffix .find ('{' ); 1562 1563if (args_in_suffix != std::string::npos && 1564 (args_in_prefix == std::string::npos || args_in_prefix > diff .prefix .length ())) { 1565// Args are in suffix, so call_id is BETWEEN_FUNC_AND_ARGS 1566call_id .pos = call_id_position::BETWEEN_FUNC_AND_ARGS ; 1567 1568// Find call_id_prefix: marker immediately preceding common_id_part (no intervening markers) 1569 std::string after_func = diff .prefix .substr (func_name_in_prefix + func_name .length ()); 1570auto id_prefix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1571return p .tag ("prefix" ,p .marker ())+ 1572p .zero_or_more (p .negate (p .marker ())+ p .negate (p .literal (common_id_part ))+ p .any ())+ 1573p .literal (common_id_part ); 1574 }); 1575auto id_res = id_prefix_parser .parse_anywhere_and_extract (after_func ); 1576if (id_res .result .success ()) { 1577call_id .prefix = id_res .tags ["prefix" ]; 1578 }else { 1579// Fallback: use the last marker in after_func 1580call_id .prefix = find_last_marker (after_func ); 1581 } 1582 1583// Extract call_id_suffix: the first marker in the suffix before args "{" 1584auto suffix_parser = build_tagged_peg_parser ([& ](common_peg_parser_builder & p ) { 1585return p .zero_or_more (p .negate (p .marker ())+ p .negate (p .literal ("{" ))+ p .any ())+ 1586p .tag ("suffix" ,p .marker ()); 1587 }); 1588auto suf_res = suffix_parser .parse_anywhere_and_extract (diff .suffix ); 1589if (suf_res .result .success ()) { 1590call_id .suffix = suf_res .tags ["suffix" ]; 1591 } 1592 }else if (args_in_prefix != std::string::npos ) { 1593// Args are in prefix, so call_id is POST_ARGS 1594call_id .pos = call_id_position::POST_ARGS ; 1595 1596// Extract last marker between args closing brace and the ID 1597 std::string after_args = diff .prefix .substr (args_in_prefix ); 1598size_t closing_brace = after_args .rfind ('}' ); 1599if (closing_brace != std::string::npos ) { 1600 std::string between_args_and_id = after_args .substr (closing_brace + 1 ); 1601call_id .prefix = find_last_marker (between_args_and_id ); 1602 } 1603 1604// call_id_suffix: first marker in diff.suffix 1605call_id .suffix = find_first_marker (diff .suffix ); 1606 } 1607 }else if (func_name_in_suffix != std::string::npos && func_name_in_prefix == std::string::npos ) { 1608// Function name is only in suffix - call_id is PRE_FUNC_NAME 1609call_id .pos = call_id_position::PRE_FUNC_NAME ; 1610 1611// call_id_prefix: last marker in diff.prefix 1612call_id .prefix = find_last_marker (diff .prefix ); 1613 1614// call_id_suffix: first marker in the portion of diff.suffix before func_name 1615 std::string before_func = diff .suffix .substr (0 ,func_name_in_suffix ); 1616call_id .suffix = find_first_marker (before_func ); 1617 } 1618 1619if (call_id .prefix == arguments .end ) { 1620call_id .prefix = "" ; 1621 } 1622 1623if (call_id .suffix == arguments .start ) { 1624call_id .suffix = "" ; 1625 } 1626 1627// When call_id is detected, per_call_end may have been incorrectly set to include 1628// the call_id_suffix and sample args. Clear it if it starts with call_id_suffix. 1629if (call_id .pos != call_id_position::NONE && !call_id .suffix .empty ()&& 1630format .per_call_end .find (call_id .suffix )== 0 ) { 1631format .per_call_end .clear (); 1632 } 1633} 1634 1635}// namespace autoparser