agent-tool-parser 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_tool_parser/__init__.py +70 -0
- agent_tool_parser/cleaners.py +283 -0
- agent_tool_parser/models.py +36 -0
- agent_tool_parser/parser.py +1219 -0
- agent_tool_parser/py.typed +1 -0
- agent_tool_parser-0.1.0.dist-info/METADATA +432 -0
- agent_tool_parser-0.1.0.dist-info/RECORD +9 -0
- agent_tool_parser-0.1.0.dist-info/WHEEL +4 -0
- agent_tool_parser-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1219 @@
|
|
|
1
|
+
# Copyright (c) 2026 InBoost Team
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import ast
|
|
7
|
+
import re
|
|
8
|
+
import warnings
|
|
9
|
+
from collections.abc import Collection
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from agent_tool_parser.cleaners import (
|
|
13
|
+
_CODE_PARAM_KEYS_DEFAULT,
|
|
14
|
+
clean_param_val,
|
|
15
|
+
extract_json_objects,
|
|
16
|
+
safe_json_loads,
|
|
17
|
+
strip_thinking,
|
|
18
|
+
)
|
|
19
|
+
from agent_tool_parser.models import ToolCall, ToolError
|
|
20
|
+
|
|
21
|
+
_DEFAULT_ALIASES: dict[str, str] = {
|
|
22
|
+
"readfile": "read_file",
|
|
23
|
+
"writefile": "write_file",
|
|
24
|
+
"strreplace": "str_replace",
|
|
25
|
+
"tool_search": "search",
|
|
26
|
+
"tool_bash": "bash",
|
|
27
|
+
"tool_read_file": "read_file",
|
|
28
|
+
"tool_write_file": "write_file",
|
|
29
|
+
"tool_str_replace": "str_replace",
|
|
30
|
+
"grep": "search",
|
|
31
|
+
"grep_search": "search",
|
|
32
|
+
"view": "read_file",
|
|
33
|
+
"view_file": "read_file",
|
|
34
|
+
"edit": "str_replace",
|
|
35
|
+
"edit_file": "str_replace",
|
|
36
|
+
"sh": "bash",
|
|
37
|
+
"shell": "bash",
|
|
38
|
+
"terminal": "bash",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
_DEFAULT_PARAM_ALIASES: dict[str, str] = {
|
|
42
|
+
"cmd": "command",
|
|
43
|
+
"file": "path",
|
|
44
|
+
"filepath": "path",
|
|
45
|
+
"file_path": "path",
|
|
46
|
+
"target_file": "path",
|
|
47
|
+
"filename": "path",
|
|
48
|
+
"query": "pattern",
|
|
49
|
+
"old": "old_str",
|
|
50
|
+
"new": "new_str",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
_JSON_BLOCK_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
|
|
54
|
+
_JSON_ARRAY_BLOCK_RE = re.compile(r"```(?:json)?\s*(\[.*?\])\s*```", re.DOTALL)
|
|
55
|
+
|
|
56
|
+
_INVOKE_TAGS = (
|
|
57
|
+
r"tool_invoke|invoke|tool_call|call|tool|invocation|"
|
|
58
|
+
r"function_call|function|action|tool_use|ant_tool_use|function_use"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
_INVOKE_RE = re.compile(
|
|
62
|
+
r"""<[||]*(?:dsml[||]*)?(?P<tag>""" + _INVOKE_TAGS + r""")"""
|
|
63
|
+
r"""(?::(?P<colon_tool>[\w-]+))?\b(?P<attrs>[^>]*)>(?P<body>.*?)"""
|
|
64
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=tag)(?::[\w-]+)?\s*>|"""
|
|
65
|
+
r"""(?=<[||]*(?:dsml[||]*)?(?:""" + _INVOKE_TAGS + r""")\b)|"""
|
|
66
|
+
r"""(?=</[||]*(?:dsml[||]*)?(?:tool_calls|function_calls|calls|tools)\s*>)|$)""",
|
|
67
|
+
re.DOTALL | re.IGNORECASE,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
_PARAM_RE = re.compile(
|
|
71
|
+
r"""<[||]*(?:dsml[||]*)?(?P<ptag>parameter|param|arg|argument)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w-]+)['"]?[^>]*>(?P<pval>.*?)"""
|
|
72
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=ptag)\s*>|"""
|
|
73
|
+
r"""(?=<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument)\b)|"""
|
|
74
|
+
r"""(?=</?[||]*(?:dsml[||]*)?(?:"""
|
|
75
|
+
+ _INVOKE_TAGS
|
|
76
|
+
+ r"""|tool_calls|function_calls|calls|tools)\b)|$)""",
|
|
77
|
+
re.DOTALL | re.IGNORECASE,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
_PARAM_START_RE = re.compile(
|
|
81
|
+
r"""<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w-]+)['"]?[^>]*>""",
|
|
82
|
+
re.IGNORECASE,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
_PARAM_END_RE = re.compile(
|
|
86
|
+
r"""</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument)\s*>""",
|
|
87
|
+
re.IGNORECASE,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
_INVOKE_END_RE = re.compile(
|
|
91
|
+
r"""</[||]*(?:dsml[||]*)?(?:"""
|
|
92
|
+
+ _INVOKE_TAGS
|
|
93
|
+
+ r""")(?:(?::[\w-]+)?\s*>)|</[||]*(?:dsml[||]*)?(?:tool_calls|function_calls|calls|tools)\s*>""",
|
|
94
|
+
re.IGNORECASE,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _extract_xml_parameters(body: str) -> list[tuple[str, str]]:
|
|
99
|
+
cdata_spans: list[tuple[int, int]] = []
|
|
100
|
+
cur = 0
|
|
101
|
+
while True:
|
|
102
|
+
idx = body.find("<![CDATA[", cur)
|
|
103
|
+
if idx == -1:
|
|
104
|
+
break
|
|
105
|
+
end_idx = body.find("]]>", idx)
|
|
106
|
+
if end_idx != -1:
|
|
107
|
+
cdata_spans.append((idx, end_idx + 3))
|
|
108
|
+
cur = end_idx + 3
|
|
109
|
+
else:
|
|
110
|
+
cdata_spans.append((idx, len(body)))
|
|
111
|
+
break
|
|
112
|
+
|
|
113
|
+
starts: list[tuple[int, int, str]] = []
|
|
114
|
+
for m in _PARAM_START_RE.finditer(body):
|
|
115
|
+
s = m.start()
|
|
116
|
+
if any(cs <= s < ce for cs, ce in cdata_spans):
|
|
117
|
+
continue
|
|
118
|
+
starts.append((s, m.end(), m.group("pname").strip().lower()))
|
|
119
|
+
|
|
120
|
+
results: list[tuple[str, str]] = []
|
|
121
|
+
for i in range(len(starts)):
|
|
122
|
+
_, val_start, pname = starts[i]
|
|
123
|
+
upper_bound = starts[i + 1][0] if i + 1 < len(starts) else len(body)
|
|
124
|
+
slice_str = body[val_start:upper_bound]
|
|
125
|
+
cdata_start = slice_str.find("<![CDATA[")
|
|
126
|
+
if cdata_start != -1:
|
|
127
|
+
cdata_end = slice_str.find("]]>", cdata_start)
|
|
128
|
+
search_from = (cdata_end + 3) if cdata_end != -1 else len(slice_str)
|
|
129
|
+
else:
|
|
130
|
+
search_from = 0
|
|
131
|
+
|
|
132
|
+
m_end = _PARAM_END_RE.search(slice_str, search_from)
|
|
133
|
+
if m_end:
|
|
134
|
+
val = slice_str[: m_end.start()]
|
|
135
|
+
else:
|
|
136
|
+
m_inv_end = _INVOKE_END_RE.search(slice_str, search_from)
|
|
137
|
+
if m_inv_end:
|
|
138
|
+
val = slice_str[: m_inv_end.start()]
|
|
139
|
+
else:
|
|
140
|
+
val = slice_str
|
|
141
|
+
results.append((pname, val))
|
|
142
|
+
return results
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
_QWEN_AGENT_RE = re.compile(
|
|
146
|
+
r"""(?:<[||]*action_start[||]*>\s*)?<[||]*action_name[||]*>\s*(?P<name>[\w-]+)\s*"""
|
|
147
|
+
r"""<[||]*action_args[||]*>\s*(?P<args>.*?)(?:<[||]*action_end[||]*>|$)""",
|
|
148
|
+
re.DOTALL | re.IGNORECASE,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
_CHATGLM_RE = re.compile(
|
|
152
|
+
r"""(?:✿|\b)FUNCTION(?:✿|\b)\s*:?\s*(?P<name>[\w-]+)\s*"""
|
|
153
|
+
r"""(?:✿|\b)ARGS(?:✿|\b)\s*:?\s*(?P<args>.*)""",
|
|
154
|
+
re.DOTALL | re.IGNORECASE,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
_LLAMA_PYTHON_TAG_RE = re.compile(
|
|
158
|
+
r"""<[||]*python_tag[||]*>(?P<body>.*?)(?:<[||]*python_tag[||]*>|<[||]*eom_id[||]*>|<[||]*eot_id[||]*>|$)""",
|
|
159
|
+
re.DOTALL | re.IGNORECASE,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
_MISTRAL_TOOL_CALL_RE = re.compile(
|
|
163
|
+
r"""\[TOOL_CALLS?\]\s*(?P<body>\[.*?\]|\{.*?\})""",
|
|
164
|
+
re.DOTALL | re.IGNORECASE,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
_REACT_RE = re.compile(
|
|
168
|
+
r"""Action:\s*(?P<name>[\w-]+)\s*\n\s*Action\s+Input:\s*(?P<args>.*?)(?=(?:\n\s*Action\s*:|\n\s*Thought\s*:|$))""",
|
|
169
|
+
re.DOTALL | re.IGNORECASE,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
_DIRECT_TAG_TNAME_RE = re.compile(
|
|
173
|
+
r"<(?:tool_name|function_name|tool)>([\w-]+)</(?:tool_name|function_name|tool)>",
|
|
174
|
+
re.IGNORECASE,
|
|
175
|
+
)
|
|
176
|
+
_TAG_PAIRS_RE = re.compile(r"<([\w-]+)>(.*?)</\1>", re.DOTALL)
|
|
177
|
+
_ATTR_TOOL_NAME_RE = re.compile(
|
|
178
|
+
r"""\b(?:name|function|tool|action)\s*=\s*['"]?([\w-]+)['"]?""",
|
|
179
|
+
re.IGNORECASE,
|
|
180
|
+
)
|
|
181
|
+
_CHILD_TOOL_NAME_RE = re.compile(
|
|
182
|
+
r"<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)",
|
|
183
|
+
re.IGNORECASE,
|
|
184
|
+
)
|
|
185
|
+
_ARGUMENTS_BLOCK_RE = re.compile(
|
|
186
|
+
r"<(?:arguments|parameters)>(.*?)(?:</(?:arguments|parameters)>|$)",
|
|
187
|
+
re.DOTALL | re.IGNORECASE,
|
|
188
|
+
)
|
|
189
|
+
_WRAP_TAG_MATCHES_RE = re.compile(
|
|
190
|
+
r"<([\w-]+)>(.*?)(?:</\1>|(?=<[\w-]+>)|$)",
|
|
191
|
+
re.DOTALL,
|
|
192
|
+
)
|
|
193
|
+
_PYTHON_CODEBLOCK_RE = re.compile(r"```(?:python|py)?\s*(.*?)\s*```", re.DOTALL)
|
|
194
|
+
_FENCE_CODEBLOCK_RE = re.compile(r"```(?:[\w-]+)?\s*(.*?)\s*```", re.DOTALL)
|
|
195
|
+
_DEEPSEEK_SEP_RE = re.compile(r"<[||]*tool\s*sep[||]*>", re.IGNORECASE)
|
|
196
|
+
_DEEPSEEK_FN_RE = re.compile(
|
|
197
|
+
r"""(?:function|name|tool|action)\s*[:=]\s*['"]?([\w-]+)['"]?""",
|
|
198
|
+
re.IGNORECASE,
|
|
199
|
+
)
|
|
200
|
+
_DEEPSEEK_BLOCK_RE = re.compile(
|
|
201
|
+
r"<[||]*tool\s*call\s*begin[||]*>(.*?)(?:<[||]*tool\s*call\s*end[||]*>|<[||]*tool\s*calls?\s*end[||]*>|$)",
|
|
202
|
+
re.DOTALL | re.IGNORECASE,
|
|
203
|
+
)
|
|
204
|
+
_DEEPSEEK_TRIGGER_RE = re.compile(r"tool\s*calls?\s*begin|tool\s*sep", re.IGNORECASE)
|
|
205
|
+
_STANDALONE_TRIGGER_RE = re.compile(r"dsml|tool_calls|function_calls", re.IGNORECASE)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
_PYTHON_BUILTINS_IGNORE = {
|
|
209
|
+
"print",
|
|
210
|
+
"len",
|
|
211
|
+
"range",
|
|
212
|
+
"str",
|
|
213
|
+
"int",
|
|
214
|
+
"float",
|
|
215
|
+
"list",
|
|
216
|
+
"dict",
|
|
217
|
+
"set",
|
|
218
|
+
"tuple",
|
|
219
|
+
"isinstance",
|
|
220
|
+
"type",
|
|
221
|
+
"enumerate",
|
|
222
|
+
"zip",
|
|
223
|
+
"sum",
|
|
224
|
+
"min",
|
|
225
|
+
"max",
|
|
226
|
+
"open",
|
|
227
|
+
"help",
|
|
228
|
+
"id",
|
|
229
|
+
"input",
|
|
230
|
+
"eval",
|
|
231
|
+
"exec",
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class ToolParser:
|
|
236
|
+
"""Configurable, robust multi-format LLM tool-call parser."""
|
|
237
|
+
|
|
238
|
+
def __init__(
|
|
239
|
+
self,
|
|
240
|
+
allowed_tools: Collection[str] | None = None,
|
|
241
|
+
tool_aliases: dict[str, str] | None = None,
|
|
242
|
+
param_aliases: dict[str, str] | None = None,
|
|
243
|
+
code_param_keys: Collection[str] | None = None,
|
|
244
|
+
strip_thinking: bool = True,
|
|
245
|
+
allow_shell_fallback: bool = False,
|
|
246
|
+
) -> None:
|
|
247
|
+
self.allowed_tools: set[str] | None = (
|
|
248
|
+
set(allowed_tools) if allowed_tools is not None else None
|
|
249
|
+
)
|
|
250
|
+
self.tool_aliases: dict[str, str] = dict(_DEFAULT_ALIASES)
|
|
251
|
+
if tool_aliases:
|
|
252
|
+
self.tool_aliases.update({k.lower(): v.lower() for k, v in tool_aliases.items()})
|
|
253
|
+
|
|
254
|
+
self.param_aliases: dict[str, str] = dict(_DEFAULT_PARAM_ALIASES)
|
|
255
|
+
if param_aliases:
|
|
256
|
+
self.param_aliases.update({k.lower(): v for k, v in param_aliases.items()})
|
|
257
|
+
|
|
258
|
+
self.code_param_keys: set[str] = (
|
|
259
|
+
set(code_param_keys) if code_param_keys is not None else set(_CODE_PARAM_KEYS_DEFAULT)
|
|
260
|
+
)
|
|
261
|
+
self.strip_thinking: bool = strip_thinking
|
|
262
|
+
self.allow_shell_fallback: bool = allow_shell_fallback
|
|
263
|
+
|
|
264
|
+
def normalize_name(self, name: str) -> str:
|
|
265
|
+
"""Normalizes tool name to lowercase and applies registered aliases."""
|
|
266
|
+
n = (name or "").strip().lower()
|
|
267
|
+
return self.tool_aliases.get(n, n)
|
|
268
|
+
|
|
269
|
+
def normalize_args(self, name: str, args: Any) -> dict[str, Any]:
|
|
270
|
+
"""Unpacks nested wrappers and normalizes parameter names & types."""
|
|
271
|
+
if isinstance(args, str):
|
|
272
|
+
args = safe_json_loads(args, default={})
|
|
273
|
+
|
|
274
|
+
if not isinstance(args, dict):
|
|
275
|
+
return {}
|
|
276
|
+
|
|
277
|
+
norm_args = dict(args)
|
|
278
|
+
|
|
279
|
+
# Unpack inner wrapper dictionaries (e.g. {"arguments": {...}})
|
|
280
|
+
for wrap_key in (
|
|
281
|
+
"arguments",
|
|
282
|
+
"parameters",
|
|
283
|
+
"params",
|
|
284
|
+
"args",
|
|
285
|
+
"action_input",
|
|
286
|
+
"tool_input",
|
|
287
|
+
"input",
|
|
288
|
+
):
|
|
289
|
+
if wrap_key in norm_args:
|
|
290
|
+
sub = norm_args.pop(wrap_key)
|
|
291
|
+
if isinstance(sub, str):
|
|
292
|
+
sub = safe_json_loads(sub, default={})
|
|
293
|
+
if isinstance(sub, dict):
|
|
294
|
+
for k, v in sub.items():
|
|
295
|
+
norm_args.setdefault(k, v)
|
|
296
|
+
|
|
297
|
+
# Map positional argument keys if present
|
|
298
|
+
if name in ("bash", "sh", "shell", "terminal") and "arg0" in norm_args:
|
|
299
|
+
norm_args.setdefault("command", norm_args.pop("arg0"))
|
|
300
|
+
elif name in ("search", "grep") and "arg0" in norm_args:
|
|
301
|
+
norm_args.setdefault("pattern", norm_args.pop("arg0"))
|
|
302
|
+
elif name in ("read_file", "view") and "arg0" in norm_args:
|
|
303
|
+
norm_args.setdefault("path", norm_args.pop("arg0"))
|
|
304
|
+
elif name in ("write_file",) and "arg0" in norm_args:
|
|
305
|
+
norm_args.setdefault("path", norm_args.pop("arg0"))
|
|
306
|
+
if "arg1" in norm_args:
|
|
307
|
+
norm_args.setdefault("content", norm_args.pop("arg1"))
|
|
308
|
+
|
|
309
|
+
# Apply parameter name aliases
|
|
310
|
+
final_args: dict[str, Any] = {}
|
|
311
|
+
for k, v in norm_args.items():
|
|
312
|
+
key_str = str(k)
|
|
313
|
+
norm_k = self.param_aliases.get(key_str.lower(), key_str)
|
|
314
|
+
final_args[norm_k] = v
|
|
315
|
+
|
|
316
|
+
# Convert numeric string parameters to int where expected
|
|
317
|
+
for int_key in ("offset", "limit", "line_number", "line", "timeout"):
|
|
318
|
+
if int_key in final_args:
|
|
319
|
+
val = final_args[int_key]
|
|
320
|
+
if isinstance(val, str) and val.strip().lstrip("-").isdigit():
|
|
321
|
+
try:
|
|
322
|
+
final_args[int_key] = int(val.strip())
|
|
323
|
+
except (ValueError, TypeError):
|
|
324
|
+
pass
|
|
325
|
+
|
|
326
|
+
return final_args
|
|
327
|
+
|
|
328
|
+
def _is_tool_allowed(self, name: str) -> bool:
|
|
329
|
+
if not name or not name.strip():
|
|
330
|
+
return False
|
|
331
|
+
if self.allowed_tools is None:
|
|
332
|
+
return True
|
|
333
|
+
return name in self.allowed_tools
|
|
334
|
+
|
|
335
|
+
def _extract_name_and_args_from_dict(self, data: Any) -> tuple[str | None, dict[str, Any]]:
|
|
336
|
+
"""Extracts tool name and arguments dictionary from diverse JSON structures
|
|
337
|
+
(OpenAI, Anthropic, Command-R, standard tool dicts)."""
|
|
338
|
+
if isinstance(data, list):
|
|
339
|
+
if not data:
|
|
340
|
+
return None, {}
|
|
341
|
+
return self._extract_name_and_args_from_dict(data[0])
|
|
342
|
+
|
|
343
|
+
if not isinstance(data, dict):
|
|
344
|
+
return None, {}
|
|
345
|
+
|
|
346
|
+
# 1. OpenAI tool_calls wrapper: {"tool_calls": [{"type": "function", "function": ...}]}
|
|
347
|
+
if "tool_calls" in data and isinstance(data["tool_calls"], list) and data["tool_calls"]:
|
|
348
|
+
return self._extract_name_and_args_from_dict(data["tool_calls"][0])
|
|
349
|
+
|
|
350
|
+
# 2. OpenAI function_call wrapper: {"function_call": {"name": ..., "arguments": ...}}
|
|
351
|
+
if "function_call" in data and isinstance(data["function_call"], dict):
|
|
352
|
+
return self._extract_name_and_args_from_dict(data["function_call"])
|
|
353
|
+
|
|
354
|
+
# 3. OpenAI function object: {"type": "function", "function": {"name": ..., "arguments": ...}}
|
|
355
|
+
if "function" in data and isinstance(data["function"], dict):
|
|
356
|
+
return self._extract_name_and_args_from_dict(data["function"])
|
|
357
|
+
|
|
358
|
+
raw_name = None
|
|
359
|
+
for k in ("name", "tool", "tool_name", "action", "function"):
|
|
360
|
+
v = data.get(k)
|
|
361
|
+
if isinstance(v, str) and v.strip():
|
|
362
|
+
raw_name = v.strip()
|
|
363
|
+
break
|
|
364
|
+
|
|
365
|
+
args_val = None
|
|
366
|
+
for k in (
|
|
367
|
+
"arguments",
|
|
368
|
+
"parameters",
|
|
369
|
+
"params",
|
|
370
|
+
"args",
|
|
371
|
+
"action_input",
|
|
372
|
+
"tool_input",
|
|
373
|
+
):
|
|
374
|
+
if k in data:
|
|
375
|
+
args_val = data[k]
|
|
376
|
+
break
|
|
377
|
+
|
|
378
|
+
args: dict[str, Any] = {}
|
|
379
|
+
if args_val is not None:
|
|
380
|
+
if isinstance(args_val, str):
|
|
381
|
+
parsed = safe_json_loads(args_val, default=None)
|
|
382
|
+
if isinstance(parsed, dict):
|
|
383
|
+
args = parsed
|
|
384
|
+
elif parsed is not None:
|
|
385
|
+
args = {"input": parsed}
|
|
386
|
+
else:
|
|
387
|
+
args = {"input": args_val}
|
|
388
|
+
elif isinstance(args_val, dict):
|
|
389
|
+
args = dict(args_val)
|
|
390
|
+
else:
|
|
391
|
+
args = {"input": args_val}
|
|
392
|
+
|
|
393
|
+
# Collect any remaining root arguments
|
|
394
|
+
extra_keys = {
|
|
395
|
+
k: v
|
|
396
|
+
for k, v in data.items()
|
|
397
|
+
if k
|
|
398
|
+
not in {
|
|
399
|
+
"name",
|
|
400
|
+
"tool",
|
|
401
|
+
"tool_name",
|
|
402
|
+
"action",
|
|
403
|
+
"function",
|
|
404
|
+
"type",
|
|
405
|
+
"id",
|
|
406
|
+
"arguments",
|
|
407
|
+
"parameters",
|
|
408
|
+
"params",
|
|
409
|
+
"args",
|
|
410
|
+
"action_input",
|
|
411
|
+
"tool_input",
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
for k, v in extra_keys.items():
|
|
415
|
+
args.setdefault(k, v)
|
|
416
|
+
|
|
417
|
+
return (str(raw_name).strip() if raw_name else None), args
|
|
418
|
+
|
|
419
|
+
def _extract_calls_from_dict(self, data: Any, source: str = "") -> list[ToolCall]:
|
|
420
|
+
"""Extracts one or more tool calls from arbitrary JSON structures."""
|
|
421
|
+
calls: list[ToolCall] = []
|
|
422
|
+
|
|
423
|
+
if isinstance(data, list):
|
|
424
|
+
for item in data:
|
|
425
|
+
calls.extend(self._extract_calls_from_dict(item, source=source))
|
|
426
|
+
return calls
|
|
427
|
+
|
|
428
|
+
if not isinstance(data, dict):
|
|
429
|
+
return calls
|
|
430
|
+
|
|
431
|
+
# 1. OpenAI tool_calls wrapper: {"tool_calls": [...]}
|
|
432
|
+
if "tool_calls" in data and isinstance(data["tool_calls"], list) and data["tool_calls"]:
|
|
433
|
+
for item in data["tool_calls"]:
|
|
434
|
+
calls.extend(self._extract_calls_from_dict(item, source=source))
|
|
435
|
+
return calls
|
|
436
|
+
|
|
437
|
+
# 2. OpenAI function_call wrapper: {"function_call": {...}}
|
|
438
|
+
if "function_call" in data and isinstance(data["function_call"], dict):
|
|
439
|
+
return self._extract_calls_from_dict(data["function_call"], source=source)
|
|
440
|
+
|
|
441
|
+
# 3. Single tool call
|
|
442
|
+
raw_name, args = self._extract_name_and_args_from_dict(data)
|
|
443
|
+
if raw_name:
|
|
444
|
+
name = self.normalize_name(raw_name)
|
|
445
|
+
if self._is_tool_allowed(name):
|
|
446
|
+
calls.append(
|
|
447
|
+
ToolCall(
|
|
448
|
+
name=name,
|
|
449
|
+
args=self.normalize_args(name, args),
|
|
450
|
+
raw_source=source,
|
|
451
|
+
)
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
return calls
|
|
455
|
+
|
|
456
|
+
def _try_parse_deepseek_native(self, text: str) -> list[ToolCall]:
|
|
457
|
+
"""Parses DeepSeek V3 / R1 native format with special tokens:
|
|
458
|
+
<|tool calls begin|><|tool call begin|>function=...<|tool sep|>{...}<|tool call end|>
|
|
459
|
+
"""
|
|
460
|
+
if "tool" not in text.lower():
|
|
461
|
+
return []
|
|
462
|
+
if not _DEEPSEEK_TRIGGER_RE.search(text):
|
|
463
|
+
return []
|
|
464
|
+
|
|
465
|
+
calls: list[ToolCall] = []
|
|
466
|
+
matches = list(_DEEPSEEK_BLOCK_RE.finditer(text))
|
|
467
|
+
|
|
468
|
+
bodies = [m.group(1).strip() for m in matches if m.group(1).strip()]
|
|
469
|
+
if not bodies and _DEEPSEEK_SEP_RE.search(text):
|
|
470
|
+
bodies = [text.strip()]
|
|
471
|
+
|
|
472
|
+
for body in bodies:
|
|
473
|
+
tool_name = None
|
|
474
|
+
json_str = body
|
|
475
|
+
|
|
476
|
+
if _DEEPSEEK_SEP_RE.search(body):
|
|
477
|
+
parts = _DEEPSEEK_SEP_RE.split(body, maxsplit=1)
|
|
478
|
+
header = parts[0].strip()
|
|
479
|
+
json_str = parts[1].strip()
|
|
480
|
+
m_fn = _DEEPSEEK_FN_RE.search(header)
|
|
481
|
+
if m_fn:
|
|
482
|
+
tool_name = m_fn.group(1)
|
|
483
|
+
elif header and header != "type=function":
|
|
484
|
+
tool_name = header
|
|
485
|
+
|
|
486
|
+
# Extract JSON from json_str
|
|
487
|
+
m_code = _JSON_BLOCK_RE.search(json_str)
|
|
488
|
+
if m_code:
|
|
489
|
+
json_str = m_code.group(1)
|
|
490
|
+
else:
|
|
491
|
+
candidates = extract_json_objects(json_str)
|
|
492
|
+
if candidates:
|
|
493
|
+
json_str = candidates[0]
|
|
494
|
+
|
|
495
|
+
data = safe_json_loads(json_str, default={})
|
|
496
|
+
if isinstance(data, dict):
|
|
497
|
+
d_name, d_args = self._extract_name_and_args_from_dict(data)
|
|
498
|
+
if not tool_name and d_name:
|
|
499
|
+
tool_name = d_name
|
|
500
|
+
args = d_args
|
|
501
|
+
else:
|
|
502
|
+
args = {}
|
|
503
|
+
|
|
504
|
+
if tool_name:
|
|
505
|
+
norm_name = self.normalize_name(tool_name)
|
|
506
|
+
if self._is_tool_allowed(norm_name):
|
|
507
|
+
calls.append(
|
|
508
|
+
ToolCall(
|
|
509
|
+
name=norm_name,
|
|
510
|
+
args=self.normalize_args(norm_name, args),
|
|
511
|
+
raw_source=body,
|
|
512
|
+
)
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
return calls
|
|
516
|
+
|
|
517
|
+
def _try_parse_xml_or_dsml(self, text: str) -> list[ToolCall]:
|
|
518
|
+
"""Parses DSML (<|DSML|invoke...>), Anthropic (<tool_use>...), and XML formats (Hermes, Claude, Kimi)."""
|
|
519
|
+
if "<" not in text:
|
|
520
|
+
return []
|
|
521
|
+
|
|
522
|
+
calls: list[ToolCall] = []
|
|
523
|
+
|
|
524
|
+
# 1. Hermes / Kimi direct tags format: <tool_name>search</tool_name><pattern>...</pattern>
|
|
525
|
+
# Only check direct tags if there are no outer invoke/tool_use wrapper tags
|
|
526
|
+
has_invoke_wrapper = bool(_INVOKE_RE.search(text))
|
|
527
|
+
if not has_invoke_wrapper and any(
|
|
528
|
+
f"<{t}>" in text.lower() for t in ("tool_name", "function_name", "tool")
|
|
529
|
+
):
|
|
530
|
+
m_tname = _DIRECT_TAG_TNAME_RE.search(text)
|
|
531
|
+
if m_tname:
|
|
532
|
+
tname = self.normalize_name(m_tname.group(1))
|
|
533
|
+
params: dict[str, Any] = {}
|
|
534
|
+
tag_matches = _TAG_PAIRS_RE.finditer(text)
|
|
535
|
+
for tm in tag_matches:
|
|
536
|
+
ptag = tm.group(1).lower()
|
|
537
|
+
if ptag not in ("tool_name", "function_name", "tool"):
|
|
538
|
+
val = clean_param_val(
|
|
539
|
+
tm.group(2), is_code_param=(ptag in self.code_param_keys)
|
|
540
|
+
)
|
|
541
|
+
params[ptag] = val
|
|
542
|
+
|
|
543
|
+
if self._is_tool_allowed(tname):
|
|
544
|
+
calls.append(
|
|
545
|
+
ToolCall(
|
|
546
|
+
name=tname,
|
|
547
|
+
args=self.normalize_args(tname, params),
|
|
548
|
+
raw_source=text,
|
|
549
|
+
)
|
|
550
|
+
)
|
|
551
|
+
return calls
|
|
552
|
+
|
|
553
|
+
# 2. Match invoke/tool/call/tool_use tags
|
|
554
|
+
inv_matches = list(_INVOKE_RE.finditer(text))
|
|
555
|
+
for inv in inv_matches:
|
|
556
|
+
colon_tool = inv.group("colon_tool")
|
|
557
|
+
attrs = inv.group("attrs") or ""
|
|
558
|
+
body = inv.group("body") or ""
|
|
559
|
+
|
|
560
|
+
tool_name = colon_tool
|
|
561
|
+
if not tool_name:
|
|
562
|
+
m_fn = _ATTR_TOOL_NAME_RE.search(attrs)
|
|
563
|
+
if m_fn:
|
|
564
|
+
tool_name = m_fn.group(1)
|
|
565
|
+
|
|
566
|
+
# Check child tags for tool name (Anthropic format: <name>read_file</name>)
|
|
567
|
+
if not tool_name:
|
|
568
|
+
m_child = _CHILD_TOOL_NAME_RE.search(body)
|
|
569
|
+
if m_child:
|
|
570
|
+
tool_name = m_child.group(1)
|
|
571
|
+
|
|
572
|
+
params = {}
|
|
573
|
+
|
|
574
|
+
# Check DSML / standard parameter tags: <parameter name="...">
|
|
575
|
+
for pname, pval in _extract_xml_parameters(body):
|
|
576
|
+
is_code = pname in self.code_param_keys
|
|
577
|
+
params[pname] = clean_param_val(pval, is_code_param=is_code)
|
|
578
|
+
|
|
579
|
+
# Check for <arguments>...</arguments> or <parameters>...</parameters> block (Anthropic)
|
|
580
|
+
m_wrap = _ARGUMENTS_BLOCK_RE.search(body)
|
|
581
|
+
if m_wrap:
|
|
582
|
+
wrap_body = m_wrap.group(1).strip()
|
|
583
|
+
cand = extract_json_objects(wrap_body)
|
|
584
|
+
if cand:
|
|
585
|
+
d = safe_json_loads(cand[0], default=None)
|
|
586
|
+
if isinstance(d, dict):
|
|
587
|
+
params.update(d)
|
|
588
|
+
if not params:
|
|
589
|
+
tag_matches = _WRAP_TAG_MATCHES_RE.finditer(wrap_body)
|
|
590
|
+
for tm in tag_matches:
|
|
591
|
+
ptag = tm.group(1).lower()
|
|
592
|
+
if ptag not in ("arguments", "parameters"):
|
|
593
|
+
val = clean_param_val(
|
|
594
|
+
tm.group(2),
|
|
595
|
+
is_code_param=(ptag in self.code_param_keys),
|
|
596
|
+
)
|
|
597
|
+
params[ptag] = val
|
|
598
|
+
|
|
599
|
+
# Check for direct child XML tags in body if still no params
|
|
600
|
+
if not params:
|
|
601
|
+
tag_matches = _TAG_PAIRS_RE.finditer(body)
|
|
602
|
+
for tm in tag_matches:
|
|
603
|
+
ptag = tm.group(1).lower()
|
|
604
|
+
if ptag not in (
|
|
605
|
+
"name",
|
|
606
|
+
"tool_name",
|
|
607
|
+
"function_name",
|
|
608
|
+
"tool",
|
|
609
|
+
"function",
|
|
610
|
+
"arguments",
|
|
611
|
+
"parameters",
|
|
612
|
+
):
|
|
613
|
+
val = clean_param_val(
|
|
614
|
+
tm.group(2), is_code_param=(ptag in self.code_param_keys)
|
|
615
|
+
)
|
|
616
|
+
params[ptag] = val
|
|
617
|
+
|
|
618
|
+
if not tool_name:
|
|
619
|
+
tool_name = (
|
|
620
|
+
params.pop("tool", None)
|
|
621
|
+
or params.pop("name", None)
|
|
622
|
+
or params.pop("action", None)
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
# If still no params, check for embedded JSON in body
|
|
626
|
+
if not params:
|
|
627
|
+
m_code = _JSON_BLOCK_RE.search(body)
|
|
628
|
+
json_str = m_code.group(1) if m_code else None
|
|
629
|
+
if not json_str:
|
|
630
|
+
candidates = extract_json_objects(body)
|
|
631
|
+
if candidates:
|
|
632
|
+
json_str = candidates[0]
|
|
633
|
+
if json_str:
|
|
634
|
+
d = safe_json_loads(json_str, default={})
|
|
635
|
+
if isinstance(d, dict):
|
|
636
|
+
d_name, d_args = self._extract_name_and_args_from_dict(d)
|
|
637
|
+
if not tool_name and d_name:
|
|
638
|
+
tool_name = d_name
|
|
639
|
+
params.update(d_args)
|
|
640
|
+
|
|
641
|
+
if tool_name:
|
|
642
|
+
norm_name = self.normalize_name(tool_name)
|
|
643
|
+
if self._is_tool_allowed(norm_name):
|
|
644
|
+
calls.append(
|
|
645
|
+
ToolCall(
|
|
646
|
+
name=norm_name,
|
|
647
|
+
args=self.normalize_args(norm_name, params),
|
|
648
|
+
raw_source=inv.group(0),
|
|
649
|
+
)
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
if calls:
|
|
653
|
+
return calls
|
|
654
|
+
|
|
655
|
+
# 3. Standalone parameters directly inside <tool_calls> without invoke wrapper
|
|
656
|
+
if _STANDALONE_TRIGGER_RE.search(text):
|
|
657
|
+
standalone_params: dict[str, Any] = {}
|
|
658
|
+
for pname, pval in _extract_xml_parameters(text):
|
|
659
|
+
is_code = pname in self.code_param_keys
|
|
660
|
+
standalone_params[pname] = clean_param_val(pval, is_code_param=is_code)
|
|
661
|
+
|
|
662
|
+
tool_name = (
|
|
663
|
+
standalone_params.pop("tool", None)
|
|
664
|
+
or standalone_params.pop("name", None)
|
|
665
|
+
or standalone_params.pop("action", None)
|
|
666
|
+
)
|
|
667
|
+
if not tool_name and self.allowed_tools:
|
|
668
|
+
lower_text = text.lower()
|
|
669
|
+
for t_tag in self.allowed_tools:
|
|
670
|
+
if t_tag.lower() not in lower_text:
|
|
671
|
+
continue
|
|
672
|
+
if re.search(rf"<[||]*(?:dsml[||]*)?{t_tag} [^>]*>", text, re.IGNORECASE):
|
|
673
|
+
tool_name = t_tag
|
|
674
|
+
break
|
|
675
|
+
|
|
676
|
+
if tool_name:
|
|
677
|
+
norm_name = self.normalize_name(tool_name)
|
|
678
|
+
if self._is_tool_allowed(norm_name):
|
|
679
|
+
calls.append(
|
|
680
|
+
ToolCall(
|
|
681
|
+
name=norm_name,
|
|
682
|
+
args=self.normalize_args(norm_name, standalone_params),
|
|
683
|
+
raw_source=text,
|
|
684
|
+
)
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
return calls
|
|
688
|
+
|
|
689
|
+
def _try_parse_qwen_agent(self, text: str) -> list[ToolCall]:
|
|
690
|
+
"""Parses Qwen-Agent format:
|
|
691
|
+
<|action_start|><|action_name|>tool_name<|action_args|>args<|action_end|>
|
|
692
|
+
"""
|
|
693
|
+
if "action_name" not in text:
|
|
694
|
+
return []
|
|
695
|
+
|
|
696
|
+
calls: list[ToolCall] = []
|
|
697
|
+
for m in _QWEN_AGENT_RE.finditer(text):
|
|
698
|
+
raw_name = m.group("name").strip()
|
|
699
|
+
name = self.normalize_name(raw_name)
|
|
700
|
+
if not self._is_tool_allowed(name):
|
|
701
|
+
continue
|
|
702
|
+
|
|
703
|
+
args_str = m.group("args").strip()
|
|
704
|
+
args: dict[str, Any] = {}
|
|
705
|
+
if args_str:
|
|
706
|
+
d = safe_json_loads(args_str, default=None)
|
|
707
|
+
if isinstance(d, dict):
|
|
708
|
+
args = d
|
|
709
|
+
elif d is not None:
|
|
710
|
+
args = {"input": d}
|
|
711
|
+
else:
|
|
712
|
+
args = {"input": args_str}
|
|
713
|
+
|
|
714
|
+
calls.append(
|
|
715
|
+
ToolCall(
|
|
716
|
+
name=name,
|
|
717
|
+
args=self.normalize_args(name, args),
|
|
718
|
+
raw_source=m.group(0),
|
|
719
|
+
)
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
return calls
|
|
723
|
+
|
|
724
|
+
def _try_parse_chatglm(self, text: str) -> list[ToolCall]:
|
|
725
|
+
"""Parses ChatGLM / Legacy Qwen format:
|
|
726
|
+
✿FUNCTION✿: tool_name
|
|
727
|
+
✿ARGS✿: {"path": "foo.py"}
|
|
728
|
+
"""
|
|
729
|
+
if "FUNCTION" not in text:
|
|
730
|
+
return []
|
|
731
|
+
|
|
732
|
+
calls: list[ToolCall] = []
|
|
733
|
+
for m in _CHATGLM_RE.finditer(text):
|
|
734
|
+
raw_name = m.group("name").strip()
|
|
735
|
+
name = self.normalize_name(raw_name)
|
|
736
|
+
if not self._is_tool_allowed(name):
|
|
737
|
+
continue
|
|
738
|
+
|
|
739
|
+
args_str = m.group("args").strip()
|
|
740
|
+
args: dict[str, Any] = {}
|
|
741
|
+
if args_str:
|
|
742
|
+
d = safe_json_loads(args_str, default=None)
|
|
743
|
+
if isinstance(d, dict):
|
|
744
|
+
args = d
|
|
745
|
+
elif d is not None:
|
|
746
|
+
args = {"input": d}
|
|
747
|
+
else:
|
|
748
|
+
args = {"input": args_str}
|
|
749
|
+
|
|
750
|
+
calls.append(
|
|
751
|
+
ToolCall(
|
|
752
|
+
name=name,
|
|
753
|
+
args=self.normalize_args(name, args),
|
|
754
|
+
raw_source=m.group(0),
|
|
755
|
+
)
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
return calls
|
|
759
|
+
|
|
760
|
+
def _try_parse_mistral(self, text: str) -> list[ToolCall]:
|
|
761
|
+
"""Parses Mistral [TOOL_CALLS] format:
|
|
762
|
+
[TOOL_CALLS] [{"name": "search", "arguments": {"query": "foo"}}]
|
|
763
|
+
"""
|
|
764
|
+
if "[TOOL_CALL" not in text:
|
|
765
|
+
return []
|
|
766
|
+
|
|
767
|
+
calls: list[ToolCall] = []
|
|
768
|
+
for m in _MISTRAL_TOOL_CALL_RE.finditer(text):
|
|
769
|
+
body = m.group("body").strip()
|
|
770
|
+
data = safe_json_loads(body, default=None)
|
|
771
|
+
if data is not None:
|
|
772
|
+
extracted = self._extract_calls_from_dict(data, source=m.group(0))
|
|
773
|
+
calls.extend(extracted)
|
|
774
|
+
|
|
775
|
+
return calls
|
|
776
|
+
|
|
777
|
+
def _try_parse_llama_python_tag(self, text: str) -> list[ToolCall]:
|
|
778
|
+
"""Parses Llama 3.1+ <|python_tag|> format:
|
|
779
|
+
<|python_tag|>bash.call(command="pytest")
|
|
780
|
+
or
|
|
781
|
+
<|python_tag|>{"name": "bash", "parameters": {"command": "pytest"}}
|
|
782
|
+
"""
|
|
783
|
+
if "python_tag" not in text:
|
|
784
|
+
return []
|
|
785
|
+
|
|
786
|
+
calls: list[ToolCall] = []
|
|
787
|
+
for m in _LLAMA_PYTHON_TAG_RE.finditer(text):
|
|
788
|
+
body = m.group("body").strip()
|
|
789
|
+
# 1. Try parsing body as JSON
|
|
790
|
+
sub_calls = self._try_parse_json(body)
|
|
791
|
+
if sub_calls:
|
|
792
|
+
for c in sub_calls:
|
|
793
|
+
c.raw_source = m.group(0)
|
|
794
|
+
calls.extend(sub_calls)
|
|
795
|
+
continue
|
|
796
|
+
|
|
797
|
+
# 2. Try parsing body as Python call
|
|
798
|
+
sub_calls = self._try_parse_python_expr(body)
|
|
799
|
+
if sub_calls:
|
|
800
|
+
for c in sub_calls:
|
|
801
|
+
c.raw_source = m.group(0)
|
|
802
|
+
calls.extend(sub_calls)
|
|
803
|
+
|
|
804
|
+
return calls
|
|
805
|
+
|
|
806
|
+
def _try_parse_python_expr(self, text: str) -> list[ToolCall]:
|
|
807
|
+
"""Parses Python function call expressions like read_file(path="foo.py", offset=10)
|
|
808
|
+
or bash.call(command="pytest") using standard library ast.
|
|
809
|
+
"""
|
|
810
|
+
if "(" not in text:
|
|
811
|
+
return []
|
|
812
|
+
|
|
813
|
+
candidate = text.strip()
|
|
814
|
+
# Strip markdown ```python ... ``` if wrapped
|
|
815
|
+
if candidate.startswith("```"):
|
|
816
|
+
m = _PYTHON_CODEBLOCK_RE.search(candidate)
|
|
817
|
+
if m:
|
|
818
|
+
candidate = m.group(1).strip()
|
|
819
|
+
|
|
820
|
+
calls: list[ToolCall] = []
|
|
821
|
+
|
|
822
|
+
# Try parsing entire candidate as AST module
|
|
823
|
+
cand_clean = candidate
|
|
824
|
+
if cand_clean.startswith("call:"):
|
|
825
|
+
cand_clean = cand_clean[5:].strip()
|
|
826
|
+
|
|
827
|
+
try:
|
|
828
|
+
with warnings.catch_warnings():
|
|
829
|
+
warnings.simplefilter("ignore")
|
|
830
|
+
tree = ast.parse(cand_clean)
|
|
831
|
+
for node in tree.body:
|
|
832
|
+
if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
|
|
833
|
+
call = self._ast_call_to_tool_call(node.value, source=candidate)
|
|
834
|
+
if call:
|
|
835
|
+
calls.append(call)
|
|
836
|
+
if calls:
|
|
837
|
+
return calls
|
|
838
|
+
except (SyntaxError, ValueError, TypeError):
|
|
839
|
+
pass
|
|
840
|
+
|
|
841
|
+
# If full parse failed (e.g. conversational text with python calls), parse line by line
|
|
842
|
+
lines = [candidate] if "\n" not in candidate else candidate.splitlines()
|
|
843
|
+
for line in lines:
|
|
844
|
+
orig_line = line.strip()
|
|
845
|
+
line_clean = orig_line
|
|
846
|
+
if line_clean.startswith("call:"):
|
|
847
|
+
line_clean = line_clean[5:].strip()
|
|
848
|
+
if not line_clean or line_clean.startswith("#") or "(" not in line_clean:
|
|
849
|
+
continue
|
|
850
|
+
|
|
851
|
+
call_node = None
|
|
852
|
+
for suffix in ("", ")", '")', '"))', "}"):
|
|
853
|
+
try:
|
|
854
|
+
with warnings.catch_warnings():
|
|
855
|
+
warnings.simplefilter("ignore")
|
|
856
|
+
tree = ast.parse(line_clean + suffix)
|
|
857
|
+
if (
|
|
858
|
+
tree.body
|
|
859
|
+
and isinstance(tree.body[0], ast.Expr)
|
|
860
|
+
and isinstance(tree.body[0].value, ast.Call)
|
|
861
|
+
):
|
|
862
|
+
call_node = tree.body[0].value
|
|
863
|
+
break
|
|
864
|
+
except (SyntaxError, ValueError, TypeError):
|
|
865
|
+
continue
|
|
866
|
+
|
|
867
|
+
if call_node is None:
|
|
868
|
+
continue
|
|
869
|
+
|
|
870
|
+
call = self._ast_call_to_tool_call(call_node, source=orig_line)
|
|
871
|
+
if call:
|
|
872
|
+
calls.append(call)
|
|
873
|
+
|
|
874
|
+
return calls
|
|
875
|
+
|
|
876
|
+
def _ast_call_to_tool_call(self, call_node: ast.Call, source: str = "") -> ToolCall | None:
|
|
877
|
+
raw_name = self._extract_ast_func_name(call_node.func)
|
|
878
|
+
if not raw_name:
|
|
879
|
+
return None
|
|
880
|
+
|
|
881
|
+
if self.allowed_tools is None and raw_name.lower() in _PYTHON_BUILTINS_IGNORE:
|
|
882
|
+
return None
|
|
883
|
+
|
|
884
|
+
name = self.normalize_name(raw_name)
|
|
885
|
+
if not self._is_tool_allowed(name):
|
|
886
|
+
return None
|
|
887
|
+
|
|
888
|
+
args: dict[str, Any] = {}
|
|
889
|
+
for kw in call_node.keywords:
|
|
890
|
+
if kw.arg:
|
|
891
|
+
try:
|
|
892
|
+
val = ast.literal_eval(kw.value)
|
|
893
|
+
except (
|
|
894
|
+
ValueError,
|
|
895
|
+
SyntaxError,
|
|
896
|
+
TypeError,
|
|
897
|
+
MemoryError,
|
|
898
|
+
RecursionError,
|
|
899
|
+
):
|
|
900
|
+
try:
|
|
901
|
+
val = ast.unparse(kw.value)
|
|
902
|
+
except (ValueError, TypeError, AttributeError):
|
|
903
|
+
val = ""
|
|
904
|
+
args[kw.arg] = val
|
|
905
|
+
|
|
906
|
+
for idx, p_arg in enumerate(call_node.args):
|
|
907
|
+
try:
|
|
908
|
+
val = ast.literal_eval(p_arg)
|
|
909
|
+
except (ValueError, SyntaxError, TypeError, MemoryError, RecursionError):
|
|
910
|
+
try:
|
|
911
|
+
val = ast.unparse(p_arg)
|
|
912
|
+
except (ValueError, TypeError, AttributeError):
|
|
913
|
+
val = ""
|
|
914
|
+
args[f"arg{idx}"] = val
|
|
915
|
+
|
|
916
|
+
return ToolCall(
|
|
917
|
+
name=name,
|
|
918
|
+
args=self.normalize_args(name, args),
|
|
919
|
+
raw_source=source,
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
def _extract_ast_func_name(self, func_node: ast.AST) -> str | None:
|
|
923
|
+
if isinstance(func_node, ast.Name):
|
|
924
|
+
return func_node.id
|
|
925
|
+
elif isinstance(func_node, ast.Attribute):
|
|
926
|
+
if func_node.attr in ("call", "run", "execute", "invoke") and isinstance(
|
|
927
|
+
func_node.value, ast.Name
|
|
928
|
+
):
|
|
929
|
+
return func_node.value.id
|
|
930
|
+
elif isinstance(func_node.value, ast.Name) and func_node.value.id in (
|
|
931
|
+
"tool",
|
|
932
|
+
"tools",
|
|
933
|
+
"action",
|
|
934
|
+
"functions",
|
|
935
|
+
):
|
|
936
|
+
return func_node.attr
|
|
937
|
+
else:
|
|
938
|
+
return func_node.attr
|
|
939
|
+
return None
|
|
940
|
+
|
|
941
|
+
def _try_parse_react(self, text: str) -> list[ToolCall]:
|
|
942
|
+
"""Parses ReAct format:
|
|
943
|
+
Action: bash
|
|
944
|
+
Action Input: pytest -v
|
|
945
|
+
"""
|
|
946
|
+
if "Action:" not in text and "action:" not in text:
|
|
947
|
+
return []
|
|
948
|
+
|
|
949
|
+
calls: list[ToolCall] = []
|
|
950
|
+
for m in _REACT_RE.finditer(text):
|
|
951
|
+
raw_name = m.group("name").strip()
|
|
952
|
+
name = self.normalize_name(raw_name)
|
|
953
|
+
if not self._is_tool_allowed(name):
|
|
954
|
+
continue
|
|
955
|
+
|
|
956
|
+
action_input = m.group("args").strip()
|
|
957
|
+
args: dict[str, Any] = {}
|
|
958
|
+
|
|
959
|
+
# Strip markdown fence if present
|
|
960
|
+
m_fence = _FENCE_CODEBLOCK_RE.search(action_input)
|
|
961
|
+
if m_fence:
|
|
962
|
+
action_input = m_fence.group(1).strip()
|
|
963
|
+
|
|
964
|
+
# Try parsing action_input as JSON
|
|
965
|
+
cand = extract_json_objects(action_input)
|
|
966
|
+
if cand:
|
|
967
|
+
d = safe_json_loads(cand[0], default=None)
|
|
968
|
+
if isinstance(d, dict):
|
|
969
|
+
args = d
|
|
970
|
+
|
|
971
|
+
if not args:
|
|
972
|
+
if name == "bash":
|
|
973
|
+
args = {"command": action_input}
|
|
974
|
+
elif name in ("read_file", "view_file", "cat"):
|
|
975
|
+
args = {"path": action_input}
|
|
976
|
+
else:
|
|
977
|
+
args = {"input": action_input}
|
|
978
|
+
|
|
979
|
+
calls.append(
|
|
980
|
+
ToolCall(
|
|
981
|
+
name=name,
|
|
982
|
+
args=self.normalize_args(name, args),
|
|
983
|
+
raw_source=m.group(0),
|
|
984
|
+
)
|
|
985
|
+
)
|
|
986
|
+
|
|
987
|
+
return calls
|
|
988
|
+
|
|
989
|
+
def _try_parse_json(self, raw: str) -> list[ToolCall]:
|
|
990
|
+
"""Extracts tool calls formatted as JSON (markdown blocks, OpenAI formats, conversational JSON)."""
|
|
991
|
+
if "{" not in raw and "[" not in raw and "```" not in raw:
|
|
992
|
+
return []
|
|
993
|
+
|
|
994
|
+
calls: list[ToolCall] = []
|
|
995
|
+
|
|
996
|
+
# 1. Check all markdown code blocks (```json ... ```)
|
|
997
|
+
m_blocks = list(_JSON_BLOCK_RE.finditer(raw)) + list(_JSON_ARRAY_BLOCK_RE.finditer(raw))
|
|
998
|
+
if m_blocks:
|
|
999
|
+
for mb in m_blocks:
|
|
1000
|
+
blob = mb.group(1)
|
|
1001
|
+
data = safe_json_loads(blob, default=None)
|
|
1002
|
+
if data is not None:
|
|
1003
|
+
extracted = self._extract_calls_from_dict(data, source=blob)
|
|
1004
|
+
calls.extend(extracted)
|
|
1005
|
+
if calls:
|
|
1006
|
+
return calls
|
|
1007
|
+
|
|
1008
|
+
# 2. Check if raw text is a top-level JSON array
|
|
1009
|
+
trimmed = raw.strip()
|
|
1010
|
+
if trimmed.startswith("[") and trimmed.endswith("]"):
|
|
1011
|
+
data = safe_json_loads(trimmed, default=None)
|
|
1012
|
+
if data is not None:
|
|
1013
|
+
extracted = self._extract_calls_from_dict(data, source=trimmed)
|
|
1014
|
+
if extracted:
|
|
1015
|
+
return extracted
|
|
1016
|
+
|
|
1017
|
+
# 3. Check JSON candidates via extract_json_objects
|
|
1018
|
+
candidates = extract_json_objects(raw)
|
|
1019
|
+
if candidates:
|
|
1020
|
+
for cand in candidates:
|
|
1021
|
+
data = safe_json_loads(cand, default=None)
|
|
1022
|
+
if data is not None:
|
|
1023
|
+
extracted = self._extract_calls_from_dict(data, source=cand)
|
|
1024
|
+
calls.extend(extracted)
|
|
1025
|
+
|
|
1026
|
+
return calls
|
|
1027
|
+
|
|
1028
|
+
def _try_parse_shell(self, raw: str) -> list[ToolCall]:
|
|
1029
|
+
"""Fallback for command lines starting with $, #, python, pytest, git."""
|
|
1030
|
+
if not self.allow_shell_fallback:
|
|
1031
|
+
return []
|
|
1032
|
+
|
|
1033
|
+
calls: list[ToolCall] = []
|
|
1034
|
+
lines = [ln.strip() for ln in raw.splitlines() if ln.strip()]
|
|
1035
|
+
for line in lines:
|
|
1036
|
+
if line.startswith(("$ ", "# ")):
|
|
1037
|
+
cmd = line[2:].strip()
|
|
1038
|
+
calls.append(ToolCall(name="bash", args={"command": cmd}, raw_source=line))
|
|
1039
|
+
elif line.startswith(("python ", "git ", "pytest ")):
|
|
1040
|
+
calls.append(ToolCall(name="bash", args={"command": line}, raw_source=line))
|
|
1041
|
+
elif line.lower() == "submit":
|
|
1042
|
+
calls.append(ToolCall(name="submit", args={}, raw_source=line))
|
|
1043
|
+
|
|
1044
|
+
return calls
|
|
1045
|
+
|
|
1046
|
+
def parse_all(self, text: str) -> list[ToolCall]:
|
|
1047
|
+
"""Parses an LLM response string and returns all structured ToolCalls found.
|
|
1048
|
+
|
|
1049
|
+
Raises:
|
|
1050
|
+
ToolError: If no valid tool call could be extracted or tool is not allowed.
|
|
1051
|
+
"""
|
|
1052
|
+
raw = (text or "").strip()
|
|
1053
|
+
target = strip_thinking(raw) if self.strip_thinking else raw
|
|
1054
|
+
|
|
1055
|
+
# 1. Native DeepSeek special tokens format
|
|
1056
|
+
calls = self._try_parse_deepseek_native(target)
|
|
1057
|
+
if calls:
|
|
1058
|
+
return calls
|
|
1059
|
+
|
|
1060
|
+
# 2. Qwen-Agent tokens
|
|
1061
|
+
calls = self._try_parse_qwen_agent(target)
|
|
1062
|
+
if calls:
|
|
1063
|
+
return calls
|
|
1064
|
+
|
|
1065
|
+
# 3. Mistral [TOOL_CALLS] tokens
|
|
1066
|
+
calls = self._try_parse_mistral(target)
|
|
1067
|
+
if calls:
|
|
1068
|
+
return calls
|
|
1069
|
+
|
|
1070
|
+
# 4. Llama 3.1+ <|python_tag|>
|
|
1071
|
+
calls = self._try_parse_llama_python_tag(target)
|
|
1072
|
+
if calls:
|
|
1073
|
+
return calls
|
|
1074
|
+
|
|
1075
|
+
# 5. XML / DSML / Anthropic <tool_use> / Hermes <tool_call>
|
|
1076
|
+
calls = self._try_parse_xml_or_dsml(target)
|
|
1077
|
+
if calls:
|
|
1078
|
+
return calls
|
|
1079
|
+
|
|
1080
|
+
# 6. ChatGLM / Legacy Qwen ✿FUNCTION✿
|
|
1081
|
+
calls = self._try_parse_chatglm(target)
|
|
1082
|
+
if calls:
|
|
1083
|
+
return calls
|
|
1084
|
+
|
|
1085
|
+
# 7. ReAct Action: / Action Input:
|
|
1086
|
+
calls = self._try_parse_react(target)
|
|
1087
|
+
if calls:
|
|
1088
|
+
return calls
|
|
1089
|
+
|
|
1090
|
+
# 8. JSON (OpenAI formats, markdown blocks, conversational inline JSON)
|
|
1091
|
+
calls = self._try_parse_json(target)
|
|
1092
|
+
if calls:
|
|
1093
|
+
return calls
|
|
1094
|
+
|
|
1095
|
+
# 9. Python function call syntax (read_file(...) or bash.call(...))
|
|
1096
|
+
calls = self._try_parse_python_expr(target)
|
|
1097
|
+
if calls:
|
|
1098
|
+
return calls
|
|
1099
|
+
|
|
1100
|
+
# 10. Optional shell fallback
|
|
1101
|
+
calls = self._try_parse_shell(raw)
|
|
1102
|
+
if calls:
|
|
1103
|
+
return calls
|
|
1104
|
+
|
|
1105
|
+
preview = raw[:200] + ("..." if len(raw) > 200 else "")
|
|
1106
|
+
raise ToolError(f"No recognized tool call found in LLM response: {preview!r}")
|
|
1107
|
+
|
|
1108
|
+
parse_many = parse_all
|
|
1109
|
+
parse_calls = parse_all
|
|
1110
|
+
|
|
1111
|
+
def parse(self, text: str) -> ToolCall:
|
|
1112
|
+
"""Parses an LLM response string and returns the first structured ToolCall.
|
|
1113
|
+
|
|
1114
|
+
Raises:
|
|
1115
|
+
ToolError: If no valid tool call could be extracted or tool is not allowed.
|
|
1116
|
+
"""
|
|
1117
|
+
calls = self.parse_all(text)
|
|
1118
|
+
return calls[0]
|
|
1119
|
+
|
|
1120
|
+
def try_parse_all(self, text: str) -> list[ToolCall]:
|
|
1121
|
+
"""Attempts to parse all tool calls, returning empty list on failure instead of raising."""
|
|
1122
|
+
try:
|
|
1123
|
+
return self.parse_all(text)
|
|
1124
|
+
except ToolError:
|
|
1125
|
+
return []
|
|
1126
|
+
|
|
1127
|
+
try_parse_many = try_parse_all
|
|
1128
|
+
try_parse_calls = try_parse_all
|
|
1129
|
+
|
|
1130
|
+
def try_parse(self, text: str) -> ToolCall | None:
|
|
1131
|
+
"""Attempts to parse tool call, returning None on failure instead of raising."""
|
|
1132
|
+
try:
|
|
1133
|
+
return self.parse(text)
|
|
1134
|
+
except ToolError:
|
|
1135
|
+
return None
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def parse_tool_call(
|
|
1139
|
+
text: str,
|
|
1140
|
+
allowed_tools: Collection[str] | None = None,
|
|
1141
|
+
tool_aliases: dict[str, str] | None = None,
|
|
1142
|
+
param_aliases: dict[str, str] | None = None,
|
|
1143
|
+
code_param_keys: Collection[str] | None = None,
|
|
1144
|
+
strip_thinking: bool = True,
|
|
1145
|
+
allow_shell_fallback: bool = False,
|
|
1146
|
+
) -> ToolCall:
|
|
1147
|
+
"""Convenience function to parse an LLM response with custom or default settings."""
|
|
1148
|
+
parser = ToolParser(
|
|
1149
|
+
allowed_tools=allowed_tools,
|
|
1150
|
+
tool_aliases=tool_aliases,
|
|
1151
|
+
param_aliases=param_aliases,
|
|
1152
|
+
code_param_keys=code_param_keys,
|
|
1153
|
+
strip_thinking=strip_thinking,
|
|
1154
|
+
allow_shell_fallback=allow_shell_fallback,
|
|
1155
|
+
)
|
|
1156
|
+
return parser.parse(text)
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
def try_parse_tool_call(
|
|
1160
|
+
text: str,
|
|
1161
|
+
allowed_tools: Collection[str] | None = None,
|
|
1162
|
+
tool_aliases: dict[str, str] | None = None,
|
|
1163
|
+
param_aliases: dict[str, str] | None = None,
|
|
1164
|
+
code_param_keys: Collection[str] | None = None,
|
|
1165
|
+
strip_thinking: bool = True,
|
|
1166
|
+
allow_shell_fallback: bool = False,
|
|
1167
|
+
) -> ToolCall | None:
|
|
1168
|
+
"""Convenience function to parse an LLM response, returning None instead of raising."""
|
|
1169
|
+
parser = ToolParser(
|
|
1170
|
+
allowed_tools=allowed_tools,
|
|
1171
|
+
tool_aliases=tool_aliases,
|
|
1172
|
+
param_aliases=param_aliases,
|
|
1173
|
+
code_param_keys=code_param_keys,
|
|
1174
|
+
strip_thinking=strip_thinking,
|
|
1175
|
+
allow_shell_fallback=allow_shell_fallback,
|
|
1176
|
+
)
|
|
1177
|
+
return parser.try_parse(text)
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def parse_tool_calls(
|
|
1181
|
+
text: str,
|
|
1182
|
+
allowed_tools: Collection[str] | None = None,
|
|
1183
|
+
tool_aliases: dict[str, str] | None = None,
|
|
1184
|
+
param_aliases: dict[str, str] | None = None,
|
|
1185
|
+
code_param_keys: Collection[str] | None = None,
|
|
1186
|
+
strip_thinking: bool = True,
|
|
1187
|
+
allow_shell_fallback: bool = False,
|
|
1188
|
+
) -> list[ToolCall]:
|
|
1189
|
+
"""Parses an LLM response string and returns all structured ToolCalls found."""
|
|
1190
|
+
parser = ToolParser(
|
|
1191
|
+
allowed_tools=allowed_tools,
|
|
1192
|
+
tool_aliases=tool_aliases,
|
|
1193
|
+
param_aliases=param_aliases,
|
|
1194
|
+
code_param_keys=code_param_keys,
|
|
1195
|
+
strip_thinking=strip_thinking,
|
|
1196
|
+
allow_shell_fallback=allow_shell_fallback,
|
|
1197
|
+
)
|
|
1198
|
+
return parser.parse_all(text)
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
def try_parse_tool_calls(
|
|
1202
|
+
text: str,
|
|
1203
|
+
allowed_tools: Collection[str] | None = None,
|
|
1204
|
+
tool_aliases: dict[str, str] | None = None,
|
|
1205
|
+
param_aliases: dict[str, str] | None = None,
|
|
1206
|
+
code_param_keys: Collection[str] | None = None,
|
|
1207
|
+
strip_thinking: bool = True,
|
|
1208
|
+
allow_shell_fallback: bool = False,
|
|
1209
|
+
) -> list[ToolCall]:
|
|
1210
|
+
"""Attempts to parse all tool calls, returning empty list on failure instead of raising."""
|
|
1211
|
+
parser = ToolParser(
|
|
1212
|
+
allowed_tools=allowed_tools,
|
|
1213
|
+
tool_aliases=tool_aliases,
|
|
1214
|
+
param_aliases=param_aliases,
|
|
1215
|
+
code_param_keys=code_param_keys,
|
|
1216
|
+
strip_thinking=strip_thinking,
|
|
1217
|
+
allow_shell_fallback=allow_shell_fallback,
|
|
1218
|
+
)
|
|
1219
|
+
return parser.try_parse_all(text)
|