char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Piotr Wilkin (ilintar)autoparser: support case of JSON_NATIVE with per-call markers (test case: Reka-Edge) (#21892)e1a9a6dcb

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.0 KiB161 linesraw
1{%- macro render_content(content, num_img_tokens, num_video_frames) -%}
2    {%- if content is string -%}
3        {{- content -}}
4    {%- elif content is sequence -%}
5        {%- set ns = namespace(out="", prev_was_text=false) -%}
6        {%- for item in content -%}
7            {%- set item_type = item.get("type") -%}
8            {%- if item_type == "text" or item.get("text") is not none -%}
9                {%- set text = item.get("text", "") -%}
10                {%- if text -%}
11                    {%- if ns.prev_was_text -%}
12                        {%- set ns.out = ns.out ~ " " -%}
13                    {%- endif -%}
14                    {%- set ns.out = ns.out ~ text -%}
15                {%- endif -%}
16                {%- set ns.prev_was_text = text != "" -%}
17            {%- elif item_type in ["image", "image_url"] or item.get("image") is not none or item.get("image_url") is not none -%}
18                {%- set ns.out = ns.out ~ "<image>" ~ ("<REKA_IMG_TOKEN>" * num_img_tokens) ~ "</image>" -%}
19                {%- set ns.prev_was_text = false -%}
20            {%- elif item_type in ["video", "video_url"] or item.get("video") is not none or item.get("video_url") is not none -%}
21                {%- set repeat_tokens = num_img_tokens * num_video_frames -%}
22                {%- set ns.out = ns.out ~ "<video>" ~ ("<REKA_IMG_TOKEN>" * repeat_tokens) ~ "</video>" -%}
23                {%- set ns.prev_was_text = false -%}
24            {%- endif -%}
25        {%- endfor -%}
26        {{- ns.out -}}
27    {%- endif -%}
28{%- endmacro -%}
29{%- set ns = namespace(out="", last_query_index=messages|length - 1) -%}
30{%- for msg in messages[::-1] -%}
31    {%- set idx = messages|length - 1 - loop.index0 -%}
32    {%- if msg.get("role") == "user" -%}
33        {%- set content = msg.get("content", "") -%}
34        {%- if not (content is string and content.startswith("<tool_response>") and content.endswith("</tool_response>")) -%}
35            {%- set ns.last_query_index = idx -%}
36            {%- break -%}
37        {%- endif -%}
38    {%- endif -%}
39{%- endfor -%}
40{%- set last_query_index = ns.last_query_index -%}
41{%- set num_img_tokens = num_img_tokens | default(64, true) | int -%}
42{%- set num_video_frames = num_video_frames | default(6, true) | int -%}
43{%- set start_idx = 0 -%}
44{%- set system_text = "" -%}
45{%- if messages|length > 0 and messages[0].get("role") in ["system", "developer"] -%}
46    {%- set system_text = render_content(messages[0].get("content", ""), num_img_tokens, num_video_frames) -%}
47    {%- set start_idx = 1 -%}
48{%- endif -%}
49{%- if tools or system_text -%}
50    {%- set preamble_ns = namespace(text="") -%}
51    {%- if system_text -%}
52        {%- set preamble_ns.text = "system: " ~ system_text -%}
53    {%- endif -%}
54    {%- if tools -%}
55        {%- if preamble_ns.text -%}
56            {%- set preamble_ns.text = preamble_ns.text ~ "\n\n" -%}
57        {%- else -%}
58            {%- set preamble_ns.text = "system: " -%}
59        {%- endif -%}
60        {%- set preamble_ns.text = preamble_ns.text
61            ~ "# Tools\n\n"
62            ~ "You may call one or more functions to assist with the user query.\n\n"
63            ~ "You are provided with function signatures within <tools></tools> XML tags:\n"
64            ~ "<tools>" -%}
65        {%- for tool in tools -%}
66            {%- set preamble_ns.text = preamble_ns.text ~ "\n" ~ (tool | tojson(ensure_ascii=True)) -%}
67        {%- endfor -%}
68        {%- set preamble_ns.text = preamble_ns.text
69            ~ "\n</tools>\n\n"
70            ~ "For each function call, return a json object with function name and arguments "
71            ~ "within <tool_call></tool_call> XML tags:\n"
72            ~ "<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>" -%}
73    {%- endif -%}
74    {%- set ns.out = ns.out ~ preamble_ns.text ~ "\n\n<sep>" -%}
75{%- endif -%}
76{%- for idx in range(start_idx, messages|length) -%}
77    {%- set message = messages[idx] -%}
78    {%- set role = message.get("role") -%}
79    {%- set content = message.get("content") -%}
80    {%- if role == "user" -%}
81        {%- set prefix_ns = namespace(value="human: ") -%}
82        {%- if content is sequence and content is not string -%}
83            {%- for item in content -%}
84                {%- if item.get("type") == "text" or item.get("text") is not none -%}
85                    {%- set text = item.get("text", "") -%}
86                    {%- if text -%}
87                        {%- break -%}
88                    {%- endif -%}
89                {%- elif item.get("type") in ["image", "image_url", "video", "video_url"] -%}
90                    {%- set prefix_ns.value = "human:" -%}
91                    {%- break -%}
92                {%- endif -%}
93            {%- endfor -%}
94        {%- endif -%}
95        {%- set ns.out = ns.out ~ prefix_ns.value ~ render_content(content, num_img_tokens, num_video_frames) ~ "<sep>" -%}
96    {%- elif role == "assistant" -%}
97        {%- set tool_calls = message.get("tool_calls") -%}
98        {%- set content_text = render_content(content, num_img_tokens, num_video_frames) -%}
99        {%- set reasoning_text = "" -%}
100        {%- if message.get("reasoning_content") is string -%}
101            {%- set reasoning_text = message.get("reasoning_content") -%}
102        {%- elif "</think>" in content_text -%}
103            {%- set reasoning_text = content_text.split("</think>", 1)[0].rstrip("\n").split("<think>")[-1].lstrip("\n") -%}
104            {%- set content_text = content_text.split("</think>", 1)[1].lstrip("\n") -%}
105        {%- endif -%}
106        {%- set ns.out = ns.out ~ "assistant: " -%}
107        {%- set include_thinking = enable_thinking is true
108            and idx > last_query_index
109            and (idx == messages|length - 1 or reasoning_text)
110        -%}
111        {%- if include_thinking -%}
112            {%- set ns.out = ns.out ~ "<think>\n" ~ (reasoning_text.strip() ) ~ "\n</think>\n\n" -%}
113        {%- endif -%}
114        {%- set ns.out = ns.out ~ content_text -%}
115        {%- if tool_calls -%}
116            {%- if content_text and not ns.out.endswith("\n") -%}
117                {%- set ns.out = ns.out ~ "\n" -%}
118            {%- endif -%}
119            {%- for tool_call in tool_calls -%}
120                {%- if tool_call.get("function") is not none -%}
121                    {%- set tool_call = tool_call.get("function") -%}
122                {%- endif -%}
123                {%- set arguments = tool_call.get("arguments", {}) -%}
124                {%- if arguments is string -%}
125                    {%- set arguments_json = arguments -%}
126                {%- elif arguments is mapping -%}
127                    {%- set arguments_json = arguments | tojson(ensure_ascii=True) -%}
128                {%- else -%}
129                    {%- set arguments_json = arguments | tojson(ensure_ascii=True) -%}
130                {%- endif -%}
131                {%- set ns.out = ns.out
132                    ~ "<tool_call>\n"
133                    ~ "{\"name\": \"" ~ tool_call.get("name", "") ~ "\", \"arguments\": "
134                    ~ arguments_json
135                    ~ "}\n</tool_call>" -%}
136            {%- endfor -%}
137        {%- endif -%}
138        {%- if not (continue_final_message and idx == messages|length - 1) -%}
139            {%- set ns.out = ns.out ~ "\n\n<sep>" -%}
140        {%- endif -%}
141    {%- elif role == "tool" -%}
142        {%- if idx == start_idx or messages[idx - 1].get("role") != "tool" -%}
143            {%- set ns.out = ns.out ~ "human: " -%}
144        {%- endif -%}
145        {%- set response_text = render_content(content, num_img_tokens, num_video_frames) -%}
146        {%- set ns.out = ns.out ~ "<tool_response>\n" ~ response_text ~ "\n</tool_response>" -%}
147        {%- if idx == messages|length - 1 or messages[idx + 1].get("role") != "tool" -%}
148            {%- set ns.out = ns.out ~ "<sep>" -%}
149        {%- endif -%}
150    {%- endif -%}
151{%- endfor -%}
152{%- if add_generation_prompt
153    and (messages|length == 0 or messages[-1].get("role") != "assistant")
154-%}
155    {%- if enable_thinking is true -%}
156        {%- set ns.out = ns.out ~ "assistant: <think>\n" -%}
157    {%- else -%}
158        {%- set ns.out = ns.out ~ "assistant:" -%}
159    {%- endif -%}
160{%- endif -%}
161{{- ns.out -}}