agent-tool-parser 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/Cargo.lock +4 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/Cargo.toml +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/PKG-INFO +5 -3
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/README.md +4 -2
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/README.md +4 -2
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/parser.rs +64 -9
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/raw.rs +35 -12
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/streaming.rs +63 -11
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/tests/benchmark_tests.rs +33 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/tests/degenerate_tests.rs +13 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/pyproject.toml +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/__init__.py +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/parser.py +36 -14
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/raw.py +29 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/streaming.py +15 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/LICENSE +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/Cargo.toml +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/canonical.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/cleaners.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/lib.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/models.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/python_calls.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/semantics.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/vector.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-py/Cargo.toml +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-py/src/lib.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/canonical.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/cleaners.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/models.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/py.typed +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/python_calls.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/semantics.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/src/agent_tool_parser/vector.py +0 -0
|
@@ -4,7 +4,7 @@ version = 4
|
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "agent-tool-parser-c"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.6"
|
|
8
8
|
dependencies = [
|
|
9
9
|
"agent-tool-parser-core",
|
|
10
10
|
"serde_json",
|
|
@@ -12,7 +12,7 @@ dependencies = [
|
|
|
12
12
|
|
|
13
13
|
[[package]]
|
|
14
14
|
name = "agent-tool-parser-core"
|
|
15
|
-
version = "0.1.
|
|
15
|
+
version = "0.1.6"
|
|
16
16
|
dependencies = [
|
|
17
17
|
"html-escape",
|
|
18
18
|
"memchr",
|
|
@@ -24,7 +24,7 @@ dependencies = [
|
|
|
24
24
|
|
|
25
25
|
[[package]]
|
|
26
26
|
name = "agent-tool-parser-py"
|
|
27
|
-
version = "0.1.
|
|
27
|
+
version = "0.1.6"
|
|
28
28
|
dependencies = [
|
|
29
29
|
"agent-tool-parser-core",
|
|
30
30
|
"pyo3",
|
|
@@ -34,7 +34,7 @@ dependencies = [
|
|
|
34
34
|
|
|
35
35
|
[[package]]
|
|
36
36
|
name = "agent-tool-parser-wasm"
|
|
37
|
-
version = "0.1.
|
|
37
|
+
version = "0.1.6"
|
|
38
38
|
dependencies = [
|
|
39
39
|
"agent-tool-parser-core",
|
|
40
40
|
"serde-wasm-bindgen",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agent-tool-parser
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -41,9 +41,9 @@ Project-URL: Repository, https://github.com/inboost-dev/agent-tool-parser.git
|
|
|
41
41
|
# agent-tool-parser ⚡
|
|
42
42
|
|
|
43
43
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
44
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
45
45
|
[](https://pypi.org/)
|
|
46
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
47
47
|
[](https://www.rust-lang.org/)
|
|
48
48
|
[](https://opensource.org/licenses/MIT)
|
|
49
49
|
[]()
|
|
@@ -421,6 +421,8 @@ agent-tool-parser/
|
|
|
421
421
|
| :--- | :--- | :---: |
|
|
422
422
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
423
423
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
424
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
425
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
424
426
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
425
427
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
426
428
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# agent-tool-parser ⚡
|
|
2
2
|
|
|
3
3
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
4
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
5
5
|
[](https://pypi.org/)
|
|
6
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
7
7
|
[](https://www.rust-lang.org/)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
9
|
[]()
|
|
@@ -381,6 +381,8 @@ agent-tool-parser/
|
|
|
381
381
|
| :--- | :--- | :---: |
|
|
382
382
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
383
383
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
384
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
385
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
384
386
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
385
387
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
386
388
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# agent-tool-parser ⚡
|
|
2
2
|
|
|
3
3
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
4
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
5
5
|
[](https://pypi.org/)
|
|
6
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
7
7
|
[](https://www.rust-lang.org/)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
9
|
[]()
|
|
@@ -381,6 +381,8 @@ agent-tool-parser/
|
|
|
381
381
|
| :--- | :--- | :---: |
|
|
382
382
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
383
383
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
384
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
385
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
384
386
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
385
387
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
386
388
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/parser.rs
RENAMED
|
@@ -13,28 +13,40 @@ static JSON_BLOCK_RE: Lazy<Regex> =
|
|
|
13
13
|
static JSON_ARRAY_BLOCK_RE: Lazy<Regex> =
|
|
14
14
|
Lazy::new(|| Regex::new(r"(?s)```(?:json)?\s*(\[.*?\])\s*```").unwrap());
|
|
15
15
|
|
|
16
|
+
/// Safely find the closest char boundary at or below `index` (MSRV 1.75+ compatible).
|
|
17
|
+
#[inline]
|
|
18
|
+
pub(crate) fn floor_char_boundary(s: &str, mut index: usize) -> usize {
|
|
19
|
+
if index >= s.len() {
|
|
20
|
+
return s.len();
|
|
21
|
+
}
|
|
22
|
+
while !s.is_char_boundary(index) {
|
|
23
|
+
index -= 1;
|
|
24
|
+
}
|
|
25
|
+
index
|
|
26
|
+
}
|
|
27
|
+
|
|
16
28
|
pub(crate) static INVOKE_RE: Lazy<Regex> = Lazy::new(|| {
|
|
17
29
|
Regex::new(
|
|
18
|
-
r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
30
|
+
r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)(?:</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>|<[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)\b|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>|$)"#
|
|
19
31
|
).unwrap()
|
|
20
32
|
});
|
|
21
33
|
|
|
22
34
|
static PARAM_START_RE: Lazy<Regex> = Lazy::new(|| {
|
|
23
35
|
Regex::new(
|
|
24
|
-
r#"(?is)<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
36
|
+
r#"(?is)<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>"#
|
|
25
37
|
).unwrap()
|
|
26
38
|
});
|
|
27
39
|
|
|
28
40
|
static PARAM_END_RE: Lazy<Regex> = Lazy::new(|| {
|
|
29
41
|
Regex::new(
|
|
30
|
-
r#"(?is)</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>"#,
|
|
42
|
+
r#"(?is)</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>"#,
|
|
31
43
|
)
|
|
32
44
|
.unwrap()
|
|
33
45
|
});
|
|
34
46
|
|
|
35
47
|
static INVOKE_END_RE: Lazy<Regex> = Lazy::new(|| {
|
|
36
48
|
Regex::new(
|
|
37
|
-
r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
49
|
+
r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[a-zA-Z0-9_.:-]+)?\s*>|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>"#
|
|
38
50
|
).unwrap()
|
|
39
51
|
});
|
|
40
52
|
|
|
@@ -71,7 +83,16 @@ pub(crate) fn extract_xml_parameters(body: &str) -> Vec<(String, String)> {
|
|
|
71
83
|
{
|
|
72
84
|
return None;
|
|
73
85
|
}
|
|
74
|
-
let pname = caps.name("pname")
|
|
86
|
+
let pname = if let Some(m) = caps.name("pname") {
|
|
87
|
+
m.as_str().trim().to_lowercase()
|
|
88
|
+
} else if let Some(m) = caps.name("pname_eq") {
|
|
89
|
+
m.as_str().trim().to_lowercase()
|
|
90
|
+
} else {
|
|
91
|
+
String::new()
|
|
92
|
+
};
|
|
93
|
+
if pname.is_empty() {
|
|
94
|
+
return None;
|
|
95
|
+
}
|
|
75
96
|
Some((start, m.end(), pname))
|
|
76
97
|
})
|
|
77
98
|
.collect();
|
|
@@ -150,10 +171,17 @@ static ATTRS_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
|
150
171
|
Regex::new(r#"(?i)\b(?:name|function|tool|action)\s*=\s*['"]?([\w-]+)['"]?"#).unwrap()
|
|
151
172
|
});
|
|
152
173
|
|
|
153
|
-
static CHILD_TAG_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
174
|
+
pub(crate) static CHILD_TAG_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
154
175
|
Regex::new(r#"(?i)<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)"#).unwrap()
|
|
155
176
|
});
|
|
156
177
|
|
|
178
|
+
pub(crate) static CHILD_TAG_FN_EQ_RE: Lazy<Regex> = Lazy::new(|| {
|
|
179
|
+
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?(?P<name>[\w.:-]+)"#).unwrap()
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
pub(crate) static TOOL_CALL_CONTAINER_RE: Lazy<Regex> =
|
|
183
|
+
Lazy::new(|| Regex::new(r#"(?is)</?[||]*(?:dsml[||]*)?tool_call\s*>"#).unwrap());
|
|
184
|
+
|
|
157
185
|
static ARGUMENTS_BLOCK_RE: Lazy<Regex> = Lazy::new(|| {
|
|
158
186
|
Regex::new(r#"(?is)<(?:arguments|parameters)>(.*?)(?:</(?:arguments|parameters)>|$)"#).unwrap()
|
|
159
187
|
});
|
|
@@ -537,7 +565,8 @@ impl ToolParser {
|
|
|
537
565
|
Err(ToolError::new(format!(
|
|
538
566
|
"No valid tool call found in response: {:?}",
|
|
539
567
|
if text.len() > 100 {
|
|
540
|
-
&text
|
|
568
|
+
let end = floor_char_boundary(&text, 100);
|
|
569
|
+
&text[..end]
|
|
541
570
|
} else {
|
|
542
571
|
&text
|
|
543
572
|
}
|
|
@@ -642,6 +671,19 @@ impl ToolParser {
|
|
|
642
671
|
return Vec::new();
|
|
643
672
|
}
|
|
644
673
|
|
|
674
|
+
let lower = text.to_lowercase();
|
|
675
|
+
let cleaned_buf;
|
|
676
|
+
let text = if (lower.contains("<tool_call>") || lower.contains("</tool_call>"))
|
|
677
|
+
&& (lower.contains("<function")
|
|
678
|
+
|| lower.contains("<invoke")
|
|
679
|
+
|| lower.contains("<action"))
|
|
680
|
+
{
|
|
681
|
+
cleaned_buf = TOOL_CALL_CONTAINER_RE.replace_all(text, "").into_owned();
|
|
682
|
+
&cleaned_buf
|
|
683
|
+
} else {
|
|
684
|
+
text
|
|
685
|
+
};
|
|
686
|
+
|
|
645
687
|
let mut calls = Vec::new();
|
|
646
688
|
|
|
647
689
|
// 1. Direct tag format: <tool_name>search</tool_name><pattern>...</pattern>
|
|
@@ -690,15 +732,28 @@ impl ToolParser {
|
|
|
690
732
|
if tool_name.is_none() {
|
|
691
733
|
if let Some(caps) = CHILD_TAG_NAME_RE.captures(body) {
|
|
692
734
|
tool_name = caps.get(1).map(|m| m.as_str().to_string());
|
|
735
|
+
} else if let Some(caps) = CHILD_TAG_FN_EQ_RE.captures(body) {
|
|
736
|
+
tool_name = caps.name("name").map(|m| m.as_str().to_string());
|
|
693
737
|
}
|
|
694
738
|
}
|
|
695
739
|
|
|
696
740
|
let mut params = Map::new();
|
|
697
741
|
|
|
698
|
-
// Extract <parameter name="...">
|
|
742
|
+
// Extract <parameter name="..."> or <parameter=...>
|
|
699
743
|
for (pname, pval) in extract_xml_parameters(body) {
|
|
700
744
|
let is_code = self.config.code_param_keys.contains(&pname);
|
|
701
|
-
|
|
745
|
+
let cleaned = clean_param_val(&pval, is_code);
|
|
746
|
+
let val_json = if !is_code && (cleaned.starts_with('{') || cleaned.starts_with('['))
|
|
747
|
+
{
|
|
748
|
+
if let Some(j) = safe_json_loads(&cleaned) {
|
|
749
|
+
j
|
|
750
|
+
} else {
|
|
751
|
+
Value::String(cleaned)
|
|
752
|
+
}
|
|
753
|
+
} else {
|
|
754
|
+
Value::String(cleaned)
|
|
755
|
+
};
|
|
756
|
+
params.insert(pname, val_json);
|
|
702
757
|
}
|
|
703
758
|
|
|
704
759
|
// Extract <arguments>...</arguments>
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/raw.rs
RENAMED
|
@@ -132,10 +132,22 @@ impl RawToolCall {
|
|
|
132
132
|
/// Extract raw tool calls from text without full JSON deserialization overhead.
|
|
133
133
|
pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
134
134
|
let clean = strip_thinking(text);
|
|
135
|
+
let lower = clean.to_lowercase();
|
|
136
|
+
let cleaned_buf;
|
|
137
|
+
let clean = if (lower.contains("<tool_call>") || lower.contains("</tool_call>"))
|
|
138
|
+
&& (lower.contains("<function") || lower.contains("<invoke") || lower.contains("<action"))
|
|
139
|
+
{
|
|
140
|
+
cleaned_buf = crate::parser::TOOL_CALL_CONTAINER_RE
|
|
141
|
+
.replace_all(&clean, "")
|
|
142
|
+
.into_owned();
|
|
143
|
+
&cleaned_buf
|
|
144
|
+
} else {
|
|
145
|
+
&clean
|
|
146
|
+
};
|
|
135
147
|
let mut results = Vec::new();
|
|
136
148
|
|
|
137
149
|
// 1. DSML and XML invocation blocks
|
|
138
|
-
for caps in INVOKE_RE.captures_iter(
|
|
150
|
+
for caps in INVOKE_RE.captures_iter(clean) {
|
|
139
151
|
let full = caps.get(0).unwrap().as_str();
|
|
140
152
|
let colon_tool = caps.name("colon_tool").map(|m| m.as_str().trim());
|
|
141
153
|
let attrs = caps.name("attrs").map(|m| m.as_str()).unwrap_or("");
|
|
@@ -151,11 +163,22 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
151
163
|
None
|
|
152
164
|
};
|
|
153
165
|
|
|
154
|
-
let name = name_opt
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
166
|
+
let name = name_opt
|
|
167
|
+
.or_else(|| {
|
|
168
|
+
ATTR_NAME_RE
|
|
169
|
+
.captures(attrs)
|
|
170
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
171
|
+
})
|
|
172
|
+
.or_else(|| {
|
|
173
|
+
crate::parser::CHILD_TAG_NAME_RE
|
|
174
|
+
.captures(body)
|
|
175
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
176
|
+
})
|
|
177
|
+
.or_else(|| {
|
|
178
|
+
crate::parser::CHILD_TAG_FN_EQ_RE
|
|
179
|
+
.captures(body)
|
|
180
|
+
.map(|c| c.name("name").unwrap().as_str().trim().to_string())
|
|
181
|
+
});
|
|
159
182
|
|
|
160
183
|
if let Some(tname) = name {
|
|
161
184
|
if !tname.is_empty() {
|
|
@@ -169,7 +192,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
169
192
|
}
|
|
170
193
|
|
|
171
194
|
// 2. Native DeepSeek tokens
|
|
172
|
-
for caps in NATIVE_DEEPSEEK_RE.captures_iter(
|
|
195
|
+
for caps in NATIVE_DEEPSEEK_RE.captures_iter(clean) {
|
|
173
196
|
let full = caps.get(0).unwrap().as_str();
|
|
174
197
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
175
198
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -183,7 +206,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
183
206
|
}
|
|
184
207
|
|
|
185
208
|
// 3. Qwen action tokens
|
|
186
|
-
for caps in QWEN_ACTION_RE.captures_iter(
|
|
209
|
+
for caps in QWEN_ACTION_RE.captures_iter(clean) {
|
|
187
210
|
let full = caps.get(0).unwrap().as_str();
|
|
188
211
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
189
212
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -197,7 +220,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
197
220
|
}
|
|
198
221
|
|
|
199
222
|
// 4. Mistral [TOOL_CALLS]
|
|
200
|
-
if let Some(caps) = MISTRAL_RE.captures(
|
|
223
|
+
if let Some(caps) = MISTRAL_RE.captures(clean) {
|
|
201
224
|
let full = caps.get(0).unwrap().as_str();
|
|
202
225
|
let arr_str = caps.get(1).unwrap().as_str();
|
|
203
226
|
if let Some(Value::Array(items)) = safe_json_loads(arr_str) {
|
|
@@ -225,7 +248,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
225
248
|
}
|
|
226
249
|
|
|
227
250
|
// 5. ReAct pattern
|
|
228
|
-
if let Some(caps) = REACT_ACTION_RE.captures(
|
|
251
|
+
if let Some(caps) = REACT_ACTION_RE.captures(clean) {
|
|
229
252
|
let full = caps.get(0).unwrap().as_str();
|
|
230
253
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
231
254
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -239,7 +262,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
239
262
|
}
|
|
240
263
|
|
|
241
264
|
// 6. JSON objects in Markdown or raw text
|
|
242
|
-
let json_objs = extract_json_objects(
|
|
265
|
+
let json_objs = extract_json_objects(clean);
|
|
243
266
|
for raw_json in json_objs {
|
|
244
267
|
if let Some(Value::Object(map)) = safe_json_loads(&raw_json) {
|
|
245
268
|
// Check standard OpenAI tool_calls structure
|
|
@@ -307,7 +330,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
307
330
|
|
|
308
331
|
// 7. Python function calls in Markdown or raw text
|
|
309
332
|
let py_calls = crate::python_calls::extract_python_function_calls(
|
|
310
|
-
|
|
333
|
+
clean,
|
|
311
334
|
&crate::models::ToolParserConfig::default(),
|
|
312
335
|
);
|
|
313
336
|
for pc in py_calls {
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/streaming.rs
RENAMED
|
@@ -13,7 +13,7 @@ static ATTR_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
|
13
13
|
});
|
|
14
14
|
|
|
15
15
|
static TAG_CLOSE_RE: Lazy<Regex> = Lazy::new(|| {
|
|
16
|
-
Regex::new(r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
16
|
+
Regex::new(r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>"#).unwrap()
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
static NATIVE_TOKEN_START_RE: Lazy<Regex> = Lazy::new(|| {
|
|
@@ -146,7 +146,7 @@ impl StreamingToolParser {
|
|
|
146
146
|
|
|
147
147
|
// 2. Check for XML / DSML invoke tag
|
|
148
148
|
static INVOKE_OPEN_RE: Lazy<Regex> = Lazy::new(|| {
|
|
149
|
-
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
149
|
+
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>"#).unwrap()
|
|
150
150
|
});
|
|
151
151
|
|
|
152
152
|
if let Some(caps) = INVOKE_OPEN_RE.captures(&self.buffer) {
|
|
@@ -166,11 +166,28 @@ impl StreamingToolParser {
|
|
|
166
166
|
None
|
|
167
167
|
};
|
|
168
168
|
|
|
169
|
-
let name_opt = raw_name
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
169
|
+
let name_opt = raw_name
|
|
170
|
+
.or_else(|| {
|
|
171
|
+
ATTR_NAME_RE
|
|
172
|
+
.captures(attrs)
|
|
173
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
174
|
+
})
|
|
175
|
+
.or_else(|| {
|
|
176
|
+
let tag_lower = caps
|
|
177
|
+
.name("tag")
|
|
178
|
+
.map(|t| t.as_str().to_lowercase())
|
|
179
|
+
.unwrap_or_default();
|
|
180
|
+
if tag_lower == "tool_call" || tag_lower == "tool_calls" {
|
|
181
|
+
let after_tag = &self.buffer[end..];
|
|
182
|
+
let check_len =
|
|
183
|
+
crate::parser::floor_char_boundary(after_tag, 300);
|
|
184
|
+
JSON_NAME_RE.captures(&after_tag[..check_len]).map(|c| {
|
|
185
|
+
c.name("name").unwrap().as_str().trim().to_string()
|
|
186
|
+
})
|
|
187
|
+
} else {
|
|
188
|
+
None
|
|
189
|
+
}
|
|
190
|
+
});
|
|
174
191
|
|
|
175
192
|
if let Some(name) = name_opt {
|
|
176
193
|
let norm_name = self.parser.normalize_name(&name);
|
|
@@ -327,8 +344,9 @@ impl StreamingToolParser {
|
|
|
327
344
|
}
|
|
328
345
|
|
|
329
346
|
if self.buffer.len() > 500 {
|
|
330
|
-
|
|
331
|
-
|
|
347
|
+
let check_start = crate::parser::floor_char_boundary(&self.buffer, 100);
|
|
348
|
+
if let Some(nl) = self.buffer[check_start..].find('\n') {
|
|
349
|
+
let cut = check_start + nl + 1;
|
|
332
350
|
events.push(StreamEvent::Text(self.buffer[..cut].to_string()));
|
|
333
351
|
self.buffer = self.buffer[cut..].to_string();
|
|
334
352
|
continue;
|
|
@@ -421,7 +439,12 @@ impl StreamingToolParser {
|
|
|
421
439
|
// Construct synthetic full string for parsing if needed
|
|
422
440
|
let synthetic = match format {
|
|
423
441
|
ToolStreamFormat::XmlDsml => {
|
|
424
|
-
|
|
442
|
+
let trimmed = call_str.trim();
|
|
443
|
+
if trimmed.starts_with('{') {
|
|
444
|
+
trimmed.to_string()
|
|
445
|
+
} else {
|
|
446
|
+
format!(r#"<invoke name="{}">{}</invoke>"#, name, call_str)
|
|
447
|
+
}
|
|
425
448
|
}
|
|
426
449
|
ToolStreamFormat::NativeToken => {
|
|
427
450
|
format!(
|
|
@@ -512,7 +535,12 @@ impl StreamingToolParser {
|
|
|
512
535
|
if !self.buffer.is_empty() {
|
|
513
536
|
let synthetic = match format {
|
|
514
537
|
ToolStreamFormat::XmlDsml => {
|
|
515
|
-
|
|
538
|
+
let trimmed = self.buffer.trim();
|
|
539
|
+
if trimmed.starts_with('{') {
|
|
540
|
+
trimmed.to_string()
|
|
541
|
+
} else {
|
|
542
|
+
format!(r#"<invoke name="{}">{}</invoke>"#, name, self.buffer)
|
|
543
|
+
}
|
|
516
544
|
}
|
|
517
545
|
ToolStreamFormat::NativeToken => {
|
|
518
546
|
format!(
|
|
@@ -658,4 +686,28 @@ mod tests {
|
|
|
658
686
|
assert!(completed.is_some());
|
|
659
687
|
assert_eq!(completed.unwrap().name, "search");
|
|
660
688
|
}
|
|
689
|
+
|
|
690
|
+
#[test]
|
|
691
|
+
fn test_streaming_qwen_xml() {
|
|
692
|
+
let mut parser = StreamingToolParser::new();
|
|
693
|
+
let chunk1 = "Let me check:\n<function=calculator>\n<parameter=expression>";
|
|
694
|
+
let events1 = parser.feed(chunk1);
|
|
695
|
+
assert!(events1.iter().any(
|
|
696
|
+
|e| matches!(e, StreamEvent::ToolCallStarted { name, .. } if name == "calculator")
|
|
697
|
+
));
|
|
698
|
+
|
|
699
|
+
let chunk2 = "2 + 2</parameter>\n</function>\nResult ready.";
|
|
700
|
+
let events2 = parser.feed(chunk2);
|
|
701
|
+
let completed = events2.iter().find_map(|e| match e {
|
|
702
|
+
StreamEvent::ToolCallCompleted(tc) => Some(tc),
|
|
703
|
+
_ => None,
|
|
704
|
+
});
|
|
705
|
+
assert!(completed.is_some());
|
|
706
|
+
let tc = completed.unwrap();
|
|
707
|
+
assert_eq!(tc.name, "calculator");
|
|
708
|
+
assert_eq!(tc.args["expression"], "2 + 2");
|
|
709
|
+
assert!(events2
|
|
710
|
+
.iter()
|
|
711
|
+
.any(|e| matches!(e, StreamEvent::Text(t) if t.contains("Result ready"))));
|
|
712
|
+
}
|
|
661
713
|
}
|
|
@@ -280,3 +280,36 @@ fn test_large_payload_stability_and_performance() {
|
|
|
280
280
|
elapsed_ms
|
|
281
281
|
);
|
|
282
282
|
}
|
|
283
|
+
|
|
284
|
+
#[test]
|
|
285
|
+
fn test_qwen_xml_function_parameter_format() {
|
|
286
|
+
let text = "<function=get_current_weather>\n <parameter=location>San Francisco, CA</parameter>\n <parameter=format>celsius</parameter>\n </function>";
|
|
287
|
+
let call = parse_tool_call(text).unwrap();
|
|
288
|
+
assert_eq!(call.name, "get_current_weather");
|
|
289
|
+
assert_eq!(call.args["location"], "San Francisco, CA");
|
|
290
|
+
assert_eq!(call.args["format"], "celsius");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
#[test]
|
|
294
|
+
fn test_qwen_xml_wrapped_in_tool_call() {
|
|
295
|
+
let text = r#"<tool_call>
|
|
296
|
+
<function=get_current_weather>
|
|
297
|
+
<parameter=location>Tokyo</parameter>
|
|
298
|
+
<parameter=unit>celsius</parameter>
|
|
299
|
+
</function>
|
|
300
|
+
</tool_call>"#;
|
|
301
|
+
let call = parse_tool_call(text).unwrap();
|
|
302
|
+
assert_eq!(call.name, "get_current_weather");
|
|
303
|
+
assert_eq!(call.args["location"], "Tokyo");
|
|
304
|
+
assert_eq!(call.args["unit"], "celsius");
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#[test]
|
|
308
|
+
fn test_tool_call_with_raw_json() {
|
|
309
|
+
let text = r#"<tool_call>
|
|
310
|
+
{"name": "fetch", "arguments": {"url": "https://api.inboost.pro"}}
|
|
311
|
+
</tool_call>"#;
|
|
312
|
+
let call = parse_tool_call(text).unwrap();
|
|
313
|
+
assert_eq!(call.name, "fetch");
|
|
314
|
+
assert_eq!(call.args["url"], "https://api.inboost.pro");
|
|
315
|
+
}
|
|
@@ -442,3 +442,16 @@ fn test_degenerate_massive_string_without_catastrophic_backtracking() {
|
|
|
442
442
|
elapsed
|
|
443
443
|
);
|
|
444
444
|
}
|
|
445
|
+
|
|
446
|
+
#[test]
|
|
447
|
+
fn test_degenerate_utf8_char_boundary_error_preview() {
|
|
448
|
+
// 98 ASCII bytes + 3-byte CJK character '钮' (bytes 98..101).
|
|
449
|
+
// Slicing at byte 100 without boundary check panics with non-char-boundary.
|
|
450
|
+
let input = format!("{}钮", "a".repeat(98));
|
|
451
|
+
let err = parse_tool_call(&input).unwrap_err();
|
|
452
|
+
assert!(err.to_string().contains("No valid tool call found"));
|
|
453
|
+
|
|
454
|
+
// Also verify non-raising try_parse variants return empty / None without panic
|
|
455
|
+
assert!(try_parse_tool_call(&input).is_none());
|
|
456
|
+
assert!(try_parse_tool_calls(&input).is_empty());
|
|
457
|
+
}
|
|
@@ -11,7 +11,7 @@ include = ["LICENSE"]
|
|
|
11
11
|
|
|
12
12
|
[project]
|
|
13
13
|
name = "agent-tool-parser"
|
|
14
|
-
version = "0.1.
|
|
14
|
+
version = "0.1.6"
|
|
15
15
|
description = "High-performance dual-engine (Rust + Python) tool call parser for DeepSeek R1/V3, Claude 3.7, Qwen 2.5, Llama 3, OpenAI & ReAct formats with auto-repair and zero dependencies."
|
|
16
16
|
readme = "README.md"
|
|
17
17
|
requires-python = ">=3.9"
|
|
@@ -149,7 +149,7 @@ if not hasattr(ToolCall, "is_search"):
|
|
|
149
149
|
if not hasattr(ToolCall, "is_submit"):
|
|
150
150
|
ToolCall.is_submit = property(lambda self: is_submit_tool(self.name))
|
|
151
151
|
|
|
152
|
-
__version__ = "0.1.
|
|
152
|
+
__version__ = "0.1.6"
|
|
153
153
|
__all__ = [
|
|
154
154
|
"ACCELERATED",
|
|
155
155
|
"ARG_ALIASES",
|
|
@@ -55,43 +55,43 @@ _JSON_ARRAY_BLOCK_RE = re.compile(r"```(?:json)?\s*(\[.*?\])\s*```", re.DOTALL)
|
|
|
55
55
|
|
|
56
56
|
_INVOKE_TAGS = (
|
|
57
57
|
r"tool_invoke|invoke|tool_call|call|tool|invocation|"
|
|
58
|
-
r"function_call|function|action|tool_use|ant_tool_use|function_use|"
|
|
58
|
+
r"function_call|function|func|action|tool_use|ant_tool_use|function_use|"
|
|
59
59
|
r"действие|вызов_функции|функция|инструмент"
|
|
60
60
|
)
|
|
61
61
|
|
|
62
62
|
_INVOKE_RE = re.compile(
|
|
63
63
|
r"""<[||]*(?:dsml[||]*)?(?P<tag>""" + _INVOKE_TAGS + r""")"""
|
|
64
|
-
r"""(
|
|
65
|
-
r"""(?:</[||]*(?:dsml[||]*)?(?P=tag)(
|
|
64
|
+
r"""(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)"""
|
|
65
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=tag)(?:(?::|=)\s*[\w.:-]+)?\s*>|"""
|
|
66
66
|
r"""(?=<[||]*(?:dsml[||]*)?(?:""" + _INVOKE_TAGS + r""")\b)|"""
|
|
67
|
-
r"""(?=</[||]*(?:dsml[||]*)?(?:tool_calls|function_calls|calls|tools)\s*>)|$)""",
|
|
67
|
+
r"""(?=</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>)|$)""",
|
|
68
68
|
re.DOTALL | re.IGNORECASE,
|
|
69
69
|
)
|
|
70
70
|
|
|
71
71
|
_PARAM_RE = re.compile(
|
|
72
|
-
r"""<[||]*(?:dsml[||]*)?(?P<ptag>parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
73
|
-
r"""(?:</[||]*(?:dsml[||]*)?(?P=ptag)\s*>|"""
|
|
72
|
+
r"""<[||]*(?:dsml[||]*)?(?P<ptag>parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>(?P<pval>.*?)"""
|
|
73
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=ptag)(?:(?::|=)\s*[\w.:-]+)?\s*>|"""
|
|
74
74
|
r"""(?=<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b)|"""
|
|
75
75
|
r"""(?=</?[||]*(?:dsml[||]*)?(?:"""
|
|
76
76
|
+ _INVOKE_TAGS
|
|
77
|
-
+ r"""|tool_calls|function_calls|calls|tools)\b)|$)""",
|
|
77
|
+
+ r"""|tool_call|tool_calls|function_calls|calls|tools)\b)|$)""",
|
|
78
78
|
re.DOTALL | re.IGNORECASE,
|
|
79
79
|
)
|
|
80
80
|
|
|
81
81
|
_PARAM_START_RE = re.compile(
|
|
82
|
-
r"""<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
82
|
+
r"""<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>""",
|
|
83
83
|
re.IGNORECASE,
|
|
84
84
|
)
|
|
85
85
|
|
|
86
86
|
_PARAM_END_RE = re.compile(
|
|
87
|
-
r"""</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>""",
|
|
87
|
+
r"""</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>""",
|
|
88
88
|
re.IGNORECASE,
|
|
89
89
|
)
|
|
90
90
|
|
|
91
91
|
_INVOKE_END_RE = re.compile(
|
|
92
92
|
r"""</[||]*(?:dsml[||]*)?(?:"""
|
|
93
93
|
+ _INVOKE_TAGS
|
|
94
|
-
+ r""")(?:(
|
|
94
|
+
+ r""")(?:(?::|=)\s*[\w.:-]+)?\s*>|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>""",
|
|
95
95
|
re.IGNORECASE,
|
|
96
96
|
)
|
|
97
97
|
|
|
@@ -132,7 +132,10 @@ def _extract_xml_parameters(body: str) -> list[tuple[str, str]]:
|
|
|
132
132
|
s = m.start()
|
|
133
133
|
if any(cs <= s < ce for cs, ce in cdata_spans):
|
|
134
134
|
continue
|
|
135
|
-
|
|
135
|
+
pname = (m.group("pname") or m.group("pname_eq") or "").strip().lower()
|
|
136
|
+
if not pname:
|
|
137
|
+
continue
|
|
138
|
+
starts.append((s, m.end(), pname))
|
|
136
139
|
|
|
137
140
|
results: list[tuple[str, str]] = []
|
|
138
141
|
for i in range(len(starts)):
|
|
@@ -203,6 +206,10 @@ _CHILD_TOOL_NAME_RE = re.compile(
|
|
|
203
206
|
r"<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)",
|
|
204
207
|
re.IGNORECASE,
|
|
205
208
|
)
|
|
209
|
+
_CHILD_TOOL_FN_EQ_RE = re.compile(
|
|
210
|
+
r"""<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?([\w.:-]+)""",
|
|
211
|
+
re.IGNORECASE,
|
|
212
|
+
)
|
|
206
213
|
_ARGUMENTS_BLOCK_RE = re.compile(
|
|
207
214
|
r"<(?:arguments|parameters)>(.*?)(?:</(?:arguments|parameters)>|$)",
|
|
208
215
|
re.DOTALL | re.IGNORECASE,
|
|
@@ -540,6 +547,12 @@ class ToolParser:
|
|
|
540
547
|
if "<" not in text:
|
|
541
548
|
return []
|
|
542
549
|
|
|
550
|
+
lower = text.lower()
|
|
551
|
+
if ("<tool_call>" in lower or "</tool_call>" in lower) and any(
|
|
552
|
+
t in lower for t in ("<function", "<invoke", "<action")
|
|
553
|
+
):
|
|
554
|
+
text = re.sub(r"</?[||]*(?:dsml[||]*)?tool_call\s*>", "", text)
|
|
555
|
+
|
|
543
556
|
calls: list[ToolCall] = []
|
|
544
557
|
|
|
545
558
|
# 1. Hermes / Kimi direct tags format: <tool_name>search</tool_name><pattern>...</pattern>
|
|
@@ -584,18 +597,27 @@ class ToolParser:
|
|
|
584
597
|
if m_fn:
|
|
585
598
|
tool_name = m_fn.group(1)
|
|
586
599
|
|
|
587
|
-
# Check child tags for tool name (Anthropic format: <name>read_file</name>)
|
|
600
|
+
# Check child tags for tool name (Anthropic format: <name>read_file</name> or <function=search>)
|
|
588
601
|
if not tool_name:
|
|
589
602
|
m_child = _CHILD_TOOL_NAME_RE.search(body)
|
|
590
603
|
if m_child:
|
|
591
604
|
tool_name = m_child.group(1)
|
|
605
|
+
else:
|
|
606
|
+
m_child_eq = _CHILD_TOOL_FN_EQ_RE.search(body)
|
|
607
|
+
if m_child_eq:
|
|
608
|
+
tool_name = m_child_eq.group(1)
|
|
592
609
|
|
|
593
610
|
params = {}
|
|
594
611
|
|
|
595
|
-
# Check DSML / standard parameter tags: <parameter name="...">
|
|
612
|
+
# Check DSML / standard parameter tags: <parameter name="..."> or <parameter=...>
|
|
596
613
|
for pname, pval in _extract_xml_parameters(body):
|
|
597
614
|
is_code = pname in self.code_param_keys
|
|
598
|
-
|
|
615
|
+
cleaned = clean_param_val(pval, is_code_param=is_code)
|
|
616
|
+
if not is_code and (cleaned.startswith("{") or cleaned.startswith("[")):
|
|
617
|
+
parsed_val = safe_json_loads(cleaned, default=None)
|
|
618
|
+
params[pname] = parsed_val if parsed_val is not None else cleaned
|
|
619
|
+
else:
|
|
620
|
+
params[pname] = cleaned
|
|
599
621
|
|
|
600
622
|
# Check for <arguments>...</arguments> or <parameters>...</parameters> block (Anthropic)
|
|
601
623
|
m_wrap = _ARGUMENTS_BLOCK_RE.search(body)
|
|
@@ -20,8 +20,18 @@ _ATTR_NAME_RE = re.compile(
|
|
|
20
20
|
re.IGNORECASE,
|
|
21
21
|
)
|
|
22
22
|
|
|
23
|
+
_CHILD_TOOL_NAME_RE = re.compile(
|
|
24
|
+
r"<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)",
|
|
25
|
+
re.IGNORECASE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_CHILD_TOOL_FN_EQ_RE = re.compile(
|
|
29
|
+
r"""<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?([\w.:-]+)""",
|
|
30
|
+
re.IGNORECASE,
|
|
31
|
+
)
|
|
32
|
+
|
|
23
33
|
_INVOKE_RE = re.compile(
|
|
24
|
-
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
34
|
+
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['\"]?(?P<colon_tool>[\w.:-]+)['\"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)(?:</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>|<[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)\b|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>|$)",
|
|
25
35
|
re.IGNORECASE | re.DOTALL,
|
|
26
36
|
)
|
|
27
37
|
|
|
@@ -43,12 +53,12 @@ _REACT_ACTION_RE = re.compile(
|
|
|
43
53
|
_MISTRAL_RE = re.compile(r"\[TOOL_CALLS\]\s*(\[\s*\{.*?\}\s*\])", re.IGNORECASE | re.DOTALL)
|
|
44
54
|
|
|
45
55
|
_PARAM_START_RE = re.compile(
|
|
46
|
-
r"<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['\"]?(?P<pname>[\w
|
|
56
|
+
r"<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['\"]?(?P<pname_eq>[\w.:-]+)['\"]?|\b[^>]*?\bname\s*=\s*['\"]?(?P<pname>[\w.:-]+)['\"]?)[^>]*>",
|
|
47
57
|
re.IGNORECASE | re.DOTALL,
|
|
48
58
|
)
|
|
49
59
|
|
|
50
60
|
_PARAM_END_RE = re.compile(
|
|
51
|
-
r"</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>",
|
|
61
|
+
r"</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>",
|
|
52
62
|
re.IGNORECASE | re.DOTALL,
|
|
53
63
|
)
|
|
54
64
|
|
|
@@ -98,7 +108,9 @@ class RawToolCall:
|
|
|
98
108
|
start = m.start()
|
|
99
109
|
if any(cs <= start < ce for cs, ce in cdata_spans):
|
|
100
110
|
continue
|
|
101
|
-
pname = m.group("pname").strip().lower()
|
|
111
|
+
pname = (m.group("pname") or m.group("pname_eq") or "").strip().lower()
|
|
112
|
+
if not pname:
|
|
113
|
+
continue
|
|
102
114
|
valid_starts.append((start, m.end(), pname))
|
|
103
115
|
|
|
104
116
|
for i, (_, val_start, pname) in enumerate(valid_starts):
|
|
@@ -163,6 +175,11 @@ class RawToolCall:
|
|
|
163
175
|
def extract_raw_tool_calls(text: str) -> list[RawToolCall]:
|
|
164
176
|
"""Extract raw tool calls from text without full JSON parsing overhead."""
|
|
165
177
|
clean = strip_thinking(text)
|
|
178
|
+
lower = clean.lower()
|
|
179
|
+
if ("<tool_call>" in lower or "</tool_call>" in lower) and any(
|
|
180
|
+
t in lower for t in ("<function", "<invoke", "<action")
|
|
181
|
+
):
|
|
182
|
+
clean = re.sub(r"</?[||]*(?:dsml[||]*)?tool_call\s*>", "", clean)
|
|
166
183
|
results: list[RawToolCall] = []
|
|
167
184
|
|
|
168
185
|
# 1. DSML and XML invocation blocks
|
|
@@ -179,6 +196,14 @@ def extract_raw_tool_calls(text: str) -> list[RawToolCall]:
|
|
|
179
196
|
attr_m = _ATTR_NAME_RE.search(attrs)
|
|
180
197
|
if attr_m:
|
|
181
198
|
name = attr_m.group(1).strip()
|
|
199
|
+
else:
|
|
200
|
+
m_child = _CHILD_TOOL_NAME_RE.search(body)
|
|
201
|
+
if m_child:
|
|
202
|
+
name = m_child.group(1).strip()
|
|
203
|
+
else:
|
|
204
|
+
m_child_eq = _CHILD_TOOL_FN_EQ_RE.search(body)
|
|
205
|
+
if m_child_eq:
|
|
206
|
+
name = m_child_eq.group(1).strip()
|
|
182
207
|
|
|
183
208
|
if name:
|
|
184
209
|
results.append(RawToolCall(name=name, raw_args=body.strip(), raw_source=full))
|
|
@@ -15,12 +15,12 @@ _ATTR_NAME_RE = re.compile(
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
_INVOKE_OPEN_RE = re.compile(
|
|
18
|
-
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
18
|
+
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['\"]?(?P<colon_tool>[\w.:-]+)['\"]?)?\b(?P<attrs>[^>]*)>",
|
|
19
19
|
re.IGNORECASE,
|
|
20
20
|
)
|
|
21
21
|
|
|
22
22
|
_TAG_CLOSE_RE = re.compile(
|
|
23
|
-
r"</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
23
|
+
r"</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>",
|
|
24
24
|
re.IGNORECASE,
|
|
25
25
|
)
|
|
26
26
|
|
|
@@ -198,6 +198,11 @@ class StreamingToolParser:
|
|
|
198
198
|
attr_m = _ATTR_NAME_RE.search(attrs)
|
|
199
199
|
if attr_m:
|
|
200
200
|
raw_name = attr_m.group(1).strip()
|
|
201
|
+
elif m.group("tag").lower() in ("tool_call", "tool_calls"):
|
|
202
|
+
after_tag = self.buffer[end:]
|
|
203
|
+
json_m = _JSON_NAME_RE.search(after_tag[:300])
|
|
204
|
+
if json_m:
|
|
205
|
+
raw_name = json_m.group("name").strip()
|
|
201
206
|
|
|
202
207
|
if raw_name:
|
|
203
208
|
norm_name = self.parser.normalize_name(raw_name)
|
|
@@ -433,7 +438,10 @@ class StreamingToolParser:
|
|
|
433
438
|
call_str = self.buffer[:body_end]
|
|
434
439
|
name = self._current_name or "unknown"
|
|
435
440
|
if self._current_format == "xml_dsml":
|
|
436
|
-
|
|
441
|
+
if call_str.strip().startswith("{"):
|
|
442
|
+
synthetic = call_str.strip()
|
|
443
|
+
else:
|
|
444
|
+
synthetic = f'<invoke name="{name}">{call_str}</invoke>'
|
|
437
445
|
elif self._current_format == "native_token":
|
|
438
446
|
synthetic = f"<|tool call begin|>function={name}<|tool sep|>{call_str}<|tool call end|>"
|
|
439
447
|
elif self._current_format == "qwen":
|
|
@@ -495,7 +503,10 @@ class StreamingToolParser:
|
|
|
495
503
|
if self.buffer:
|
|
496
504
|
name = self._current_name or "unknown"
|
|
497
505
|
if self._current_format == "xml_dsml":
|
|
498
|
-
|
|
506
|
+
if self.buffer.strip().startswith("{"):
|
|
507
|
+
synthetic = self.buffer.strip()
|
|
508
|
+
else:
|
|
509
|
+
synthetic = f'<invoke name="{name}">{self.buffer}</invoke>'
|
|
499
510
|
elif self._current_format == "native_token":
|
|
500
511
|
synthetic = f"<|tool call begin|>function={name}<|tool sep|>{self.buffer}<|tool call end|>"
|
|
501
512
|
elif self._current_format == "qwen":
|
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/Cargo.toml
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/canonical.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/cleaners.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/lib.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/models.rs
RENAMED
|
File without changes
|
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/semantics.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.6}/crates/agent-tool-parser-core/src/vector.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|