char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Olivier ChafikTool call support (generic + native for Llama, Functionary, Hermes, Mistral, Firefunction, DeepSeek) w/ lazy grammars (#9639)8b576b6c5

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.9 KiB152 linesraw
1{%- macro json_to_python_type(json_spec) %}
2{%- set basic_type_map = {
3    "string": "str",
4    "number": "float",
5    "integer": "int",
6    "boolean": "bool"
7} %}
8
9{%- if basic_type_map[json_spec.type] is defined %}
10    {{- basic_type_map[json_spec.type] }}
11{%- elif json_spec.type == "array" %}
12    {{- "list[" +  json_to_python_type(json_spec|items) + "]"}}
13{%- elif json_spec.type == "object" %}
14    {%- if json_spec.additionalProperties is defined %}
15        {{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
16    {%- else %}
17        {{- "dict" }}
18    {%- endif %}
19{%- elif json_spec.type is iterable %}
20    {{- "Union[" }}
21    {%- for t in json_spec.type %}
22      {{- json_to_python_type({"type": t}) }}
23      {%- if not loop.last %}
24        {{- "," }} 
25    {%- endif %}
26    {%- endfor %}
27    {{- "]" }}
28{%- else %}
29    {{- "Any" }}
30{%- endif %}
31{%- endmacro %}
32
33
34{{- bos_token }}
35{{- '<|im_start|>system
36' }}
37{{- "You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
38{%- for tool in tools %}
39    {%- if tool.function is defined %}
40        {%- set tool = tool.function %}
41    {%- endif %}
42    {{- '{"type": "function", "function": ' }}
43    {{- '{"name": "' + tool.name + '", ' }}
44    {{- '"description": "' + tool.name + '(' }}
45    {%- for param_name, param_fields in tool.parameters.properties|items %}
46        {{- param_name + ": " + json_to_python_type(param_fields) }}
47        {%- if not loop.last %}
48            {{- ", " }}
49        {%- endif %}
50    {%- endfor %}
51    {{- ")" }}
52    {%- if tool.return is defined %}
53        {{- " -> " + json_to_python_type(tool.return) }}
54    {%- endif %}
55    {{- " - " + tool.description + "
56
57" }}
58    {%- for param_name, param_fields in tool.parameters.properties|items %}
59        {%- if loop.first %}
60            {{- "    Args:
61" }}
62        {%- endif %}
63        {{- "        " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
64    {%- endfor %}
65    {%- if tool.return is defined and tool.return.description is defined %}
66        {{- "
67    Returns:
68        " + tool.return.description }}
69    {%- endif %}
70    {{- '"' }}
71    {{- ', "parameters": ' }}
72    {%- if tool.parameters.properties | length == 0 %}
73        {{- "{}" }}
74    {%- else %}
75        {{- tool.parameters|tojson }}
76    {%- endif %}
77    {{- "}" }}
78    {%- if not loop.last %}
79        {{- "
80" }}
81    {%- endif %}
82{%- endfor %}
83{{- " </tools>" }}
84{{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
85' }}
86{{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
87" }}
88{{- "<tool_call>
89" }}
90{{- '{"name": <function-name>, "arguments": <args-dict>}
91' }}
92{{- '</tool_call><|im_end|>
93' }}
94{%- for message in messages %}
95    {%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
96        {{- '<|im_start|>' + message.role + '
97' + message.content + '<|im_end|>' + '
98' }}
99    {%- elif message.role == "assistant" %}
100        {{- '<|im_start|>' + message.role }}
101    {%- for tool_call in message.tool_calls %}
102       {{- '
103<tool_call>
104' }}           {%- if tool_call.function is defined %}
105                {%- set tool_call = tool_call.function %}
106            {%- endif %}
107            {{- '{' }}
108            {{- '"name": "' }}
109            {{- tool_call.name }}
110            {{- '"' }}
111            {{- ', '}}
112            {%- if tool_call.arguments is defined %}
113                {{- '"arguments": ' }}
114                {%- if tool_call.arguments is string %}
115                    {{- tool_call.arguments }}
116                {%- else %}
117                    {{- tool_call.arguments|tojson }}
118                {%- endif %}
119            {%- endif %}
120             {{- '}' }}
121            {{- '
122</tool_call>' }}
123    {%- endfor %}
124        {{- '<|im_end|>
125' }}
126    {%- elif message.role == "tool" %}
127        {%- if loop.previtem and loop.previtem.role != "tool" %}
128            {{- '<|im_start|>tool
129' }}
130        {%- endif %}
131        {{- '<tool_response>
132' }}
133        {{- message.content }}
134        {%- if not loop.last %}
135            {{- '
136</tool_response>
137' }}
138        {%- else %}
139            {{- '
140</tool_response>' }}
141        {%- endif %}
142        {%- if not loop.last and loop.nextitem.role != "tool" %}
143            {{- '<|im_end|>' }}
144        {%- elif loop.last %}
145            {{- '<|im_end|>' }}
146        {%- endif %}
147    {%- endif %}
148{%- endfor %}
149{%- if add_generation_prompt %}
150    {{- '<|im_start|>assistant
151' }}
152{%- endif %}