agent-tool-parser 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/Cargo.lock +4 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/Cargo.toml +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/PKG-INFO +5 -3
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/README.md +4 -2
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/README.md +4 -2
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/parser.rs +50 -8
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/raw.rs +35 -12
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/streaming.rs +59 -9
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/tests/benchmark_tests.rs +33 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/pyproject.toml +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/__init__.py +1 -1
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/parser.py +36 -14
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/raw.py +29 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/streaming.py +15 -4
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/LICENSE +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/Cargo.toml +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/canonical.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/cleaners.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/lib.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/models.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/python_calls.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/semantics.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/vector.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/tests/degenerate_tests.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-py/Cargo.toml +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-py/src/lib.rs +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/canonical.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/cleaners.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/models.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/py.typed +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/python_calls.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/semantics.py +0 -0
- {agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/src/agent_tool_parser/vector.py +0 -0
|
@@ -4,7 +4,7 @@ version = 4
|
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "agent-tool-parser-c"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.5"
|
|
8
8
|
dependencies = [
|
|
9
9
|
"agent-tool-parser-core",
|
|
10
10
|
"serde_json",
|
|
@@ -12,7 +12,7 @@ dependencies = [
|
|
|
12
12
|
|
|
13
13
|
[[package]]
|
|
14
14
|
name = "agent-tool-parser-core"
|
|
15
|
-
version = "0.1.
|
|
15
|
+
version = "0.1.5"
|
|
16
16
|
dependencies = [
|
|
17
17
|
"html-escape",
|
|
18
18
|
"memchr",
|
|
@@ -24,7 +24,7 @@ dependencies = [
|
|
|
24
24
|
|
|
25
25
|
[[package]]
|
|
26
26
|
name = "agent-tool-parser-py"
|
|
27
|
-
version = "0.1.
|
|
27
|
+
version = "0.1.5"
|
|
28
28
|
dependencies = [
|
|
29
29
|
"agent-tool-parser-core",
|
|
30
30
|
"pyo3",
|
|
@@ -34,7 +34,7 @@ dependencies = [
|
|
|
34
34
|
|
|
35
35
|
[[package]]
|
|
36
36
|
name = "agent-tool-parser-wasm"
|
|
37
|
-
version = "0.1.
|
|
37
|
+
version = "0.1.5"
|
|
38
38
|
dependencies = [
|
|
39
39
|
"agent-tool-parser-core",
|
|
40
40
|
"serde-wasm-bindgen",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agent-tool-parser
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -41,9 +41,9 @@ Project-URL: Repository, https://github.com/inboost-dev/agent-tool-parser.git
|
|
|
41
41
|
# agent-tool-parser ⚡
|
|
42
42
|
|
|
43
43
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
44
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
45
45
|
[](https://pypi.org/)
|
|
46
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
47
47
|
[](https://www.rust-lang.org/)
|
|
48
48
|
[](https://opensource.org/licenses/MIT)
|
|
49
49
|
[]()
|
|
@@ -421,6 +421,8 @@ agent-tool-parser/
|
|
|
421
421
|
| :--- | :--- | :---: |
|
|
422
422
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
423
423
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
424
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
425
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
424
426
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
425
427
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
426
428
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# agent-tool-parser ⚡
|
|
2
2
|
|
|
3
3
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
4
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
5
5
|
[](https://pypi.org/)
|
|
6
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
7
7
|
[](https://www.rust-lang.org/)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
9
|
[]()
|
|
@@ -381,6 +381,8 @@ agent-tool-parser/
|
|
|
381
381
|
| :--- | :--- | :---: |
|
|
382
382
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
383
383
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
384
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
385
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
384
386
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
385
387
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
386
388
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# agent-tool-parser ⚡
|
|
2
2
|
|
|
3
3
|
[](https://inboost-dev.github.io/agent-tool-parser/)
|
|
4
|
-
[](https://pypi.org/project/agent-tool-parser/)
|
|
5
5
|
[](https://pypi.org/)
|
|
6
|
-
[](https://crates.io/crates/agent-tool-parser-core)
|
|
7
7
|
[](https://www.rust-lang.org/)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
9
|
[]()
|
|
@@ -381,6 +381,8 @@ agent-tool-parser/
|
|
|
381
381
|
| :--- | :--- | :---: |
|
|
382
382
|
| **Anthropic Claude XML** | `<tool_use><name>read_file</name><arguments>{"path": "src/core.py"}</arguments></tool_use>` | Supported |
|
|
383
383
|
| **OpenAI Function Format** | `{"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": "{...}"}}]}` | Supported |
|
|
384
|
+
| **Qwen / Open-Weights XML** | `<function=search>\n<parameter=query>vector</parameter>\n</function>` | Supported |
|
|
385
|
+
| **Container Tag Wrappers** | `<tool_call><function=search>...</function></tool_call>` | Supported |
|
|
384
386
|
| **DeepSeek Native Tokens** | `<|tool call begin|>function=bash<|tool sep|>{"command": "pytest"}<|tool call end|>` | Supported |
|
|
385
387
|
| **DeepSeek DSML** | `<|DSML|tool name=search><|DSML|parameter name=query>vector</|DSML|parameter>` | Supported |
|
|
386
388
|
| **Qwen-Agent Tokens** | `<\|action_start\|><\|action_name\|>run<\|action_args\|>{"script": "main.py"}<\|action_end\|>` | Supported |
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/parser.rs
RENAMED
|
@@ -15,26 +15,26 @@ static JSON_ARRAY_BLOCK_RE: Lazy<Regex> =
|
|
|
15
15
|
|
|
16
16
|
pub(crate) static INVOKE_RE: Lazy<Regex> = Lazy::new(|| {
|
|
17
17
|
Regex::new(
|
|
18
|
-
r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
18
|
+
r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)(?:</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>|<[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)\b|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>|$)"#
|
|
19
19
|
).unwrap()
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
static PARAM_START_RE: Lazy<Regex> = Lazy::new(|| {
|
|
23
23
|
Regex::new(
|
|
24
|
-
r#"(?is)<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
24
|
+
r#"(?is)<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>"#
|
|
25
25
|
).unwrap()
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
static PARAM_END_RE: Lazy<Regex> = Lazy::new(|| {
|
|
29
29
|
Regex::new(
|
|
30
|
-
r#"(?is)</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>"#,
|
|
30
|
+
r#"(?is)</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>"#,
|
|
31
31
|
)
|
|
32
32
|
.unwrap()
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
static INVOKE_END_RE: Lazy<Regex> = Lazy::new(|| {
|
|
36
36
|
Regex::new(
|
|
37
|
-
r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
37
|
+
r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[a-zA-Z0-9_.:-]+)?\s*>|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>"#
|
|
38
38
|
).unwrap()
|
|
39
39
|
});
|
|
40
40
|
|
|
@@ -71,7 +71,16 @@ pub(crate) fn extract_xml_parameters(body: &str) -> Vec<(String, String)> {
|
|
|
71
71
|
{
|
|
72
72
|
return None;
|
|
73
73
|
}
|
|
74
|
-
let pname = caps.name("pname")
|
|
74
|
+
let pname = if let Some(m) = caps.name("pname") {
|
|
75
|
+
m.as_str().trim().to_lowercase()
|
|
76
|
+
} else if let Some(m) = caps.name("pname_eq") {
|
|
77
|
+
m.as_str().trim().to_lowercase()
|
|
78
|
+
} else {
|
|
79
|
+
String::new()
|
|
80
|
+
};
|
|
81
|
+
if pname.is_empty() {
|
|
82
|
+
return None;
|
|
83
|
+
}
|
|
75
84
|
Some((start, m.end(), pname))
|
|
76
85
|
})
|
|
77
86
|
.collect();
|
|
@@ -150,10 +159,17 @@ static ATTRS_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
|
150
159
|
Regex::new(r#"(?i)\b(?:name|function|tool|action)\s*=\s*['"]?([\w-]+)['"]?"#).unwrap()
|
|
151
160
|
});
|
|
152
161
|
|
|
153
|
-
static CHILD_TAG_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
162
|
+
pub(crate) static CHILD_TAG_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
154
163
|
Regex::new(r#"(?i)<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)"#).unwrap()
|
|
155
164
|
});
|
|
156
165
|
|
|
166
|
+
pub(crate) static CHILD_TAG_FN_EQ_RE: Lazy<Regex> = Lazy::new(|| {
|
|
167
|
+
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?(?P<name>[\w.:-]+)"#).unwrap()
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
pub(crate) static TOOL_CALL_CONTAINER_RE: Lazy<Regex> =
|
|
171
|
+
Lazy::new(|| Regex::new(r#"(?is)</?[||]*(?:dsml[||]*)?tool_call\s*>"#).unwrap());
|
|
172
|
+
|
|
157
173
|
static ARGUMENTS_BLOCK_RE: Lazy<Regex> = Lazy::new(|| {
|
|
158
174
|
Regex::new(r#"(?is)<(?:arguments|parameters)>(.*?)(?:</(?:arguments|parameters)>|$)"#).unwrap()
|
|
159
175
|
});
|
|
@@ -642,6 +658,19 @@ impl ToolParser {
|
|
|
642
658
|
return Vec::new();
|
|
643
659
|
}
|
|
644
660
|
|
|
661
|
+
let lower = text.to_lowercase();
|
|
662
|
+
let cleaned_buf;
|
|
663
|
+
let text = if (lower.contains("<tool_call>") || lower.contains("</tool_call>"))
|
|
664
|
+
&& (lower.contains("<function")
|
|
665
|
+
|| lower.contains("<invoke")
|
|
666
|
+
|| lower.contains("<action"))
|
|
667
|
+
{
|
|
668
|
+
cleaned_buf = TOOL_CALL_CONTAINER_RE.replace_all(text, "").into_owned();
|
|
669
|
+
&cleaned_buf
|
|
670
|
+
} else {
|
|
671
|
+
text
|
|
672
|
+
};
|
|
673
|
+
|
|
645
674
|
let mut calls = Vec::new();
|
|
646
675
|
|
|
647
676
|
// 1. Direct tag format: <tool_name>search</tool_name><pattern>...</pattern>
|
|
@@ -690,15 +719,28 @@ impl ToolParser {
|
|
|
690
719
|
if tool_name.is_none() {
|
|
691
720
|
if let Some(caps) = CHILD_TAG_NAME_RE.captures(body) {
|
|
692
721
|
tool_name = caps.get(1).map(|m| m.as_str().to_string());
|
|
722
|
+
} else if let Some(caps) = CHILD_TAG_FN_EQ_RE.captures(body) {
|
|
723
|
+
tool_name = caps.name("name").map(|m| m.as_str().to_string());
|
|
693
724
|
}
|
|
694
725
|
}
|
|
695
726
|
|
|
696
727
|
let mut params = Map::new();
|
|
697
728
|
|
|
698
|
-
// Extract <parameter name="...">
|
|
729
|
+
// Extract <parameter name="..."> or <parameter=...>
|
|
699
730
|
for (pname, pval) in extract_xml_parameters(body) {
|
|
700
731
|
let is_code = self.config.code_param_keys.contains(&pname);
|
|
701
|
-
|
|
732
|
+
let cleaned = clean_param_val(&pval, is_code);
|
|
733
|
+
let val_json = if !is_code && (cleaned.starts_with('{') || cleaned.starts_with('['))
|
|
734
|
+
{
|
|
735
|
+
if let Some(j) = safe_json_loads(&cleaned) {
|
|
736
|
+
j
|
|
737
|
+
} else {
|
|
738
|
+
Value::String(cleaned)
|
|
739
|
+
}
|
|
740
|
+
} else {
|
|
741
|
+
Value::String(cleaned)
|
|
742
|
+
};
|
|
743
|
+
params.insert(pname, val_json);
|
|
702
744
|
}
|
|
703
745
|
|
|
704
746
|
// Extract <arguments>...</arguments>
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/raw.rs
RENAMED
|
@@ -132,10 +132,22 @@ impl RawToolCall {
|
|
|
132
132
|
/// Extract raw tool calls from text without full JSON deserialization overhead.
|
|
133
133
|
pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
134
134
|
let clean = strip_thinking(text);
|
|
135
|
+
let lower = clean.to_lowercase();
|
|
136
|
+
let cleaned_buf;
|
|
137
|
+
let clean = if (lower.contains("<tool_call>") || lower.contains("</tool_call>"))
|
|
138
|
+
&& (lower.contains("<function") || lower.contains("<invoke") || lower.contains("<action"))
|
|
139
|
+
{
|
|
140
|
+
cleaned_buf = crate::parser::TOOL_CALL_CONTAINER_RE
|
|
141
|
+
.replace_all(&clean, "")
|
|
142
|
+
.into_owned();
|
|
143
|
+
&cleaned_buf
|
|
144
|
+
} else {
|
|
145
|
+
&clean
|
|
146
|
+
};
|
|
135
147
|
let mut results = Vec::new();
|
|
136
148
|
|
|
137
149
|
// 1. DSML and XML invocation blocks
|
|
138
|
-
for caps in INVOKE_RE.captures_iter(
|
|
150
|
+
for caps in INVOKE_RE.captures_iter(clean) {
|
|
139
151
|
let full = caps.get(0).unwrap().as_str();
|
|
140
152
|
let colon_tool = caps.name("colon_tool").map(|m| m.as_str().trim());
|
|
141
153
|
let attrs = caps.name("attrs").map(|m| m.as_str()).unwrap_or("");
|
|
@@ -151,11 +163,22 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
151
163
|
None
|
|
152
164
|
};
|
|
153
165
|
|
|
154
|
-
let name = name_opt
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
166
|
+
let name = name_opt
|
|
167
|
+
.or_else(|| {
|
|
168
|
+
ATTR_NAME_RE
|
|
169
|
+
.captures(attrs)
|
|
170
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
171
|
+
})
|
|
172
|
+
.or_else(|| {
|
|
173
|
+
crate::parser::CHILD_TAG_NAME_RE
|
|
174
|
+
.captures(body)
|
|
175
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
176
|
+
})
|
|
177
|
+
.or_else(|| {
|
|
178
|
+
crate::parser::CHILD_TAG_FN_EQ_RE
|
|
179
|
+
.captures(body)
|
|
180
|
+
.map(|c| c.name("name").unwrap().as_str().trim().to_string())
|
|
181
|
+
});
|
|
159
182
|
|
|
160
183
|
if let Some(tname) = name {
|
|
161
184
|
if !tname.is_empty() {
|
|
@@ -169,7 +192,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
169
192
|
}
|
|
170
193
|
|
|
171
194
|
// 2. Native DeepSeek tokens
|
|
172
|
-
for caps in NATIVE_DEEPSEEK_RE.captures_iter(
|
|
195
|
+
for caps in NATIVE_DEEPSEEK_RE.captures_iter(clean) {
|
|
173
196
|
let full = caps.get(0).unwrap().as_str();
|
|
174
197
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
175
198
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -183,7 +206,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
183
206
|
}
|
|
184
207
|
|
|
185
208
|
// 3. Qwen action tokens
|
|
186
|
-
for caps in QWEN_ACTION_RE.captures_iter(
|
|
209
|
+
for caps in QWEN_ACTION_RE.captures_iter(clean) {
|
|
187
210
|
let full = caps.get(0).unwrap().as_str();
|
|
188
211
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
189
212
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -197,7 +220,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
197
220
|
}
|
|
198
221
|
|
|
199
222
|
// 4. Mistral [TOOL_CALLS]
|
|
200
|
-
if let Some(caps) = MISTRAL_RE.captures(
|
|
223
|
+
if let Some(caps) = MISTRAL_RE.captures(clean) {
|
|
201
224
|
let full = caps.get(0).unwrap().as_str();
|
|
202
225
|
let arr_str = caps.get(1).unwrap().as_str();
|
|
203
226
|
if let Some(Value::Array(items)) = safe_json_loads(arr_str) {
|
|
@@ -225,7 +248,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
225
248
|
}
|
|
226
249
|
|
|
227
250
|
// 5. ReAct pattern
|
|
228
|
-
if let Some(caps) = REACT_ACTION_RE.captures(
|
|
251
|
+
if let Some(caps) = REACT_ACTION_RE.captures(clean) {
|
|
229
252
|
let full = caps.get(0).unwrap().as_str();
|
|
230
253
|
let name = caps.name("name").unwrap().as_str().trim();
|
|
231
254
|
let args = caps.name("args").unwrap().as_str().trim();
|
|
@@ -239,7 +262,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
239
262
|
}
|
|
240
263
|
|
|
241
264
|
// 6. JSON objects in Markdown or raw text
|
|
242
|
-
let json_objs = extract_json_objects(
|
|
265
|
+
let json_objs = extract_json_objects(clean);
|
|
243
266
|
for raw_json in json_objs {
|
|
244
267
|
if let Some(Value::Object(map)) = safe_json_loads(&raw_json) {
|
|
245
268
|
// Check standard OpenAI tool_calls structure
|
|
@@ -307,7 +330,7 @@ pub fn extract_raw_tool_calls(text: &str) -> Vec<RawToolCall> {
|
|
|
307
330
|
|
|
308
331
|
// 7. Python function calls in Markdown or raw text
|
|
309
332
|
let py_calls = crate::python_calls::extract_python_function_calls(
|
|
310
|
-
|
|
333
|
+
clean,
|
|
311
334
|
&crate::models::ToolParserConfig::default(),
|
|
312
335
|
);
|
|
313
336
|
for pc in py_calls {
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/streaming.rs
RENAMED
|
@@ -13,7 +13,7 @@ static ATTR_NAME_RE: Lazy<Regex> = Lazy::new(|| {
|
|
|
13
13
|
});
|
|
14
14
|
|
|
15
15
|
static TAG_CLOSE_RE: Lazy<Regex> = Lazy::new(|| {
|
|
16
|
-
Regex::new(r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
16
|
+
Regex::new(r#"(?is)</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>"#).unwrap()
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
static NATIVE_TOKEN_START_RE: Lazy<Regex> = Lazy::new(|| {
|
|
@@ -146,7 +146,7 @@ impl StreamingToolParser {
|
|
|
146
146
|
|
|
147
147
|
// 2. Check for XML / DSML invoke tag
|
|
148
148
|
static INVOKE_OPEN_RE: Lazy<Regex> = Lazy::new(|| {
|
|
149
|
-
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
149
|
+
Regex::new(r#"(?is)<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>"#).unwrap()
|
|
150
150
|
});
|
|
151
151
|
|
|
152
152
|
if let Some(caps) = INVOKE_OPEN_RE.captures(&self.buffer) {
|
|
@@ -166,11 +166,27 @@ impl StreamingToolParser {
|
|
|
166
166
|
None
|
|
167
167
|
};
|
|
168
168
|
|
|
169
|
-
let name_opt = raw_name
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
169
|
+
let name_opt = raw_name
|
|
170
|
+
.or_else(|| {
|
|
171
|
+
ATTR_NAME_RE
|
|
172
|
+
.captures(attrs)
|
|
173
|
+
.map(|c| c.get(1).unwrap().as_str().trim().to_string())
|
|
174
|
+
})
|
|
175
|
+
.or_else(|| {
|
|
176
|
+
let tag_lower = caps
|
|
177
|
+
.name("tag")
|
|
178
|
+
.map(|t| t.as_str().to_lowercase())
|
|
179
|
+
.unwrap_or_default();
|
|
180
|
+
if tag_lower == "tool_call" || tag_lower == "tool_calls" {
|
|
181
|
+
let after_tag = &self.buffer[end..];
|
|
182
|
+
let check_len = after_tag.len().min(300);
|
|
183
|
+
JSON_NAME_RE.captures(&after_tag[..check_len]).map(|c| {
|
|
184
|
+
c.name("name").unwrap().as_str().trim().to_string()
|
|
185
|
+
})
|
|
186
|
+
} else {
|
|
187
|
+
None
|
|
188
|
+
}
|
|
189
|
+
});
|
|
174
190
|
|
|
175
191
|
if let Some(name) = name_opt {
|
|
176
192
|
let norm_name = self.parser.normalize_name(&name);
|
|
@@ -421,7 +437,12 @@ impl StreamingToolParser {
|
|
|
421
437
|
// Construct synthetic full string for parsing if needed
|
|
422
438
|
let synthetic = match format {
|
|
423
439
|
ToolStreamFormat::XmlDsml => {
|
|
424
|
-
|
|
440
|
+
let trimmed = call_str.trim();
|
|
441
|
+
if trimmed.starts_with('{') {
|
|
442
|
+
trimmed.to_string()
|
|
443
|
+
} else {
|
|
444
|
+
format!(r#"<invoke name="{}">{}</invoke>"#, name, call_str)
|
|
445
|
+
}
|
|
425
446
|
}
|
|
426
447
|
ToolStreamFormat::NativeToken => {
|
|
427
448
|
format!(
|
|
@@ -512,7 +533,12 @@ impl StreamingToolParser {
|
|
|
512
533
|
if !self.buffer.is_empty() {
|
|
513
534
|
let synthetic = match format {
|
|
514
535
|
ToolStreamFormat::XmlDsml => {
|
|
515
|
-
|
|
536
|
+
let trimmed = self.buffer.trim();
|
|
537
|
+
if trimmed.starts_with('{') {
|
|
538
|
+
trimmed.to_string()
|
|
539
|
+
} else {
|
|
540
|
+
format!(r#"<invoke name="{}">{}</invoke>"#, name, self.buffer)
|
|
541
|
+
}
|
|
516
542
|
}
|
|
517
543
|
ToolStreamFormat::NativeToken => {
|
|
518
544
|
format!(
|
|
@@ -658,4 +684,28 @@ mod tests {
|
|
|
658
684
|
assert!(completed.is_some());
|
|
659
685
|
assert_eq!(completed.unwrap().name, "search");
|
|
660
686
|
}
|
|
687
|
+
|
|
688
|
+
#[test]
|
|
689
|
+
fn test_streaming_qwen_xml() {
|
|
690
|
+
let mut parser = StreamingToolParser::new();
|
|
691
|
+
let chunk1 = "Let me check:\n<function=calculator>\n<parameter=expression>";
|
|
692
|
+
let events1 = parser.feed(chunk1);
|
|
693
|
+
assert!(events1.iter().any(
|
|
694
|
+
|e| matches!(e, StreamEvent::ToolCallStarted { name, .. } if name == "calculator")
|
|
695
|
+
));
|
|
696
|
+
|
|
697
|
+
let chunk2 = "2 + 2</parameter>\n</function>\nResult ready.";
|
|
698
|
+
let events2 = parser.feed(chunk2);
|
|
699
|
+
let completed = events2.iter().find_map(|e| match e {
|
|
700
|
+
StreamEvent::ToolCallCompleted(tc) => Some(tc),
|
|
701
|
+
_ => None,
|
|
702
|
+
});
|
|
703
|
+
assert!(completed.is_some());
|
|
704
|
+
let tc = completed.unwrap();
|
|
705
|
+
assert_eq!(tc.name, "calculator");
|
|
706
|
+
assert_eq!(tc.args["expression"], "2 + 2");
|
|
707
|
+
assert!(events2
|
|
708
|
+
.iter()
|
|
709
|
+
.any(|e| matches!(e, StreamEvent::Text(t) if t.contains("Result ready"))));
|
|
710
|
+
}
|
|
661
711
|
}
|
|
@@ -280,3 +280,36 @@ fn test_large_payload_stability_and_performance() {
|
|
|
280
280
|
elapsed_ms
|
|
281
281
|
);
|
|
282
282
|
}
|
|
283
|
+
|
|
284
|
+
#[test]
|
|
285
|
+
fn test_qwen_xml_function_parameter_format() {
|
|
286
|
+
let text = "<function=get_current_weather>\n <parameter=location>San Francisco, CA</parameter>\n <parameter=format>celsius</parameter>\n </function>";
|
|
287
|
+
let call = parse_tool_call(text).unwrap();
|
|
288
|
+
assert_eq!(call.name, "get_current_weather");
|
|
289
|
+
assert_eq!(call.args["location"], "San Francisco, CA");
|
|
290
|
+
assert_eq!(call.args["format"], "celsius");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
#[test]
|
|
294
|
+
fn test_qwen_xml_wrapped_in_tool_call() {
|
|
295
|
+
let text = r#"<tool_call>
|
|
296
|
+
<function=get_current_weather>
|
|
297
|
+
<parameter=location>Tokyo</parameter>
|
|
298
|
+
<parameter=unit>celsius</parameter>
|
|
299
|
+
</function>
|
|
300
|
+
</tool_call>"#;
|
|
301
|
+
let call = parse_tool_call(text).unwrap();
|
|
302
|
+
assert_eq!(call.name, "get_current_weather");
|
|
303
|
+
assert_eq!(call.args["location"], "Tokyo");
|
|
304
|
+
assert_eq!(call.args["unit"], "celsius");
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#[test]
|
|
308
|
+
fn test_tool_call_with_raw_json() {
|
|
309
|
+
let text = r#"<tool_call>
|
|
310
|
+
{"name": "fetch", "arguments": {"url": "https://api.inboost.pro"}}
|
|
311
|
+
</tool_call>"#;
|
|
312
|
+
let call = parse_tool_call(text).unwrap();
|
|
313
|
+
assert_eq!(call.name, "fetch");
|
|
314
|
+
assert_eq!(call.args["url"], "https://api.inboost.pro");
|
|
315
|
+
}
|
|
@@ -11,7 +11,7 @@ include = ["LICENSE"]
|
|
|
11
11
|
|
|
12
12
|
[project]
|
|
13
13
|
name = "agent-tool-parser"
|
|
14
|
-
version = "0.1.
|
|
14
|
+
version = "0.1.5"
|
|
15
15
|
description = "High-performance dual-engine (Rust + Python) tool call parser for DeepSeek R1/V3, Claude 3.7, Qwen 2.5, Llama 3, OpenAI & ReAct formats with auto-repair and zero dependencies."
|
|
16
16
|
readme = "README.md"
|
|
17
17
|
requires-python = ">=3.9"
|
|
@@ -149,7 +149,7 @@ if not hasattr(ToolCall, "is_search"):
|
|
|
149
149
|
if not hasattr(ToolCall, "is_submit"):
|
|
150
150
|
ToolCall.is_submit = property(lambda self: is_submit_tool(self.name))
|
|
151
151
|
|
|
152
|
-
__version__ = "0.1.
|
|
152
|
+
__version__ = "0.1.5"
|
|
153
153
|
__all__ = [
|
|
154
154
|
"ACCELERATED",
|
|
155
155
|
"ARG_ALIASES",
|
|
@@ -55,43 +55,43 @@ _JSON_ARRAY_BLOCK_RE = re.compile(r"```(?:json)?\s*(\[.*?\])\s*```", re.DOTALL)
|
|
|
55
55
|
|
|
56
56
|
_INVOKE_TAGS = (
|
|
57
57
|
r"tool_invoke|invoke|tool_call|call|tool|invocation|"
|
|
58
|
-
r"function_call|function|action|tool_use|ant_tool_use|function_use|"
|
|
58
|
+
r"function_call|function|func|action|tool_use|ant_tool_use|function_use|"
|
|
59
59
|
r"действие|вызов_функции|функция|инструмент"
|
|
60
60
|
)
|
|
61
61
|
|
|
62
62
|
_INVOKE_RE = re.compile(
|
|
63
63
|
r"""<[||]*(?:dsml[||]*)?(?P<tag>""" + _INVOKE_TAGS + r""")"""
|
|
64
|
-
r"""(
|
|
65
|
-
r"""(?:</[||]*(?:dsml[||]*)?(?P=tag)(
|
|
64
|
+
r"""(?:(?::|=)\s*['"]?(?P<colon_tool>[\w.:-]+)['"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)"""
|
|
65
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=tag)(?:(?::|=)\s*[\w.:-]+)?\s*>|"""
|
|
66
66
|
r"""(?=<[||]*(?:dsml[||]*)?(?:""" + _INVOKE_TAGS + r""")\b)|"""
|
|
67
|
-
r"""(?=</[||]*(?:dsml[||]*)?(?:tool_calls|function_calls|calls|tools)\s*>)|$)""",
|
|
67
|
+
r"""(?=</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>)|$)""",
|
|
68
68
|
re.DOTALL | re.IGNORECASE,
|
|
69
69
|
)
|
|
70
70
|
|
|
71
71
|
_PARAM_RE = re.compile(
|
|
72
|
-
r"""<[||]*(?:dsml[||]*)?(?P<ptag>parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
73
|
-
r"""(?:</[||]*(?:dsml[||]*)?(?P=ptag)\s*>|"""
|
|
72
|
+
r"""<[||]*(?:dsml[||]*)?(?P<ptag>parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>(?P<pval>.*?)"""
|
|
73
|
+
r"""(?:</[||]*(?:dsml[||]*)?(?P=ptag)(?:(?::|=)\s*[\w.:-]+)?\s*>|"""
|
|
74
74
|
r"""(?=<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b)|"""
|
|
75
75
|
r"""(?=</?[||]*(?:dsml[||]*)?(?:"""
|
|
76
76
|
+ _INVOKE_TAGS
|
|
77
|
-
+ r"""|tool_calls|function_calls|calls|tools)\b)|$)""",
|
|
77
|
+
+ r"""|tool_call|tool_calls|function_calls|calls|tools)\b)|$)""",
|
|
78
78
|
re.DOTALL | re.IGNORECASE,
|
|
79
79
|
)
|
|
80
80
|
|
|
81
81
|
_PARAM_START_RE = re.compile(
|
|
82
|
-
r"""<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w
|
|
82
|
+
r"""<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['"]?(?P<pname_eq>[\w.:-]+)['"]?|\b[^>]*?\bname\s*=\s*['"]?(?P<pname>[\w.:-]+)['"]?)[^>]*>""",
|
|
83
83
|
re.IGNORECASE,
|
|
84
84
|
)
|
|
85
85
|
|
|
86
86
|
_PARAM_END_RE = re.compile(
|
|
87
|
-
r"""</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>""",
|
|
87
|
+
r"""</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>""",
|
|
88
88
|
re.IGNORECASE,
|
|
89
89
|
)
|
|
90
90
|
|
|
91
91
|
_INVOKE_END_RE = re.compile(
|
|
92
92
|
r"""</[||]*(?:dsml[||]*)?(?:"""
|
|
93
93
|
+ _INVOKE_TAGS
|
|
94
|
-
+ r""")(?:(
|
|
94
|
+
+ r""")(?:(?::|=)\s*[\w.:-]+)?\s*>|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>""",
|
|
95
95
|
re.IGNORECASE,
|
|
96
96
|
)
|
|
97
97
|
|
|
@@ -132,7 +132,10 @@ def _extract_xml_parameters(body: str) -> list[tuple[str, str]]:
|
|
|
132
132
|
s = m.start()
|
|
133
133
|
if any(cs <= s < ce for cs, ce in cdata_spans):
|
|
134
134
|
continue
|
|
135
|
-
|
|
135
|
+
pname = (m.group("pname") or m.group("pname_eq") or "").strip().lower()
|
|
136
|
+
if not pname:
|
|
137
|
+
continue
|
|
138
|
+
starts.append((s, m.end(), pname))
|
|
136
139
|
|
|
137
140
|
results: list[tuple[str, str]] = []
|
|
138
141
|
for i in range(len(starts)):
|
|
@@ -203,6 +206,10 @@ _CHILD_TOOL_NAME_RE = re.compile(
|
|
|
203
206
|
r"<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)",
|
|
204
207
|
re.IGNORECASE,
|
|
205
208
|
)
|
|
209
|
+
_CHILD_TOOL_FN_EQ_RE = re.compile(
|
|
210
|
+
r"""<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?([\w.:-]+)""",
|
|
211
|
+
re.IGNORECASE,
|
|
212
|
+
)
|
|
206
213
|
_ARGUMENTS_BLOCK_RE = re.compile(
|
|
207
214
|
r"<(?:arguments|parameters)>(.*?)(?:</(?:arguments|parameters)>|$)",
|
|
208
215
|
re.DOTALL | re.IGNORECASE,
|
|
@@ -540,6 +547,12 @@ class ToolParser:
|
|
|
540
547
|
if "<" not in text:
|
|
541
548
|
return []
|
|
542
549
|
|
|
550
|
+
lower = text.lower()
|
|
551
|
+
if ("<tool_call>" in lower or "</tool_call>" in lower) and any(
|
|
552
|
+
t in lower for t in ("<function", "<invoke", "<action")
|
|
553
|
+
):
|
|
554
|
+
text = re.sub(r"</?[||]*(?:dsml[||]*)?tool_call\s*>", "", text)
|
|
555
|
+
|
|
543
556
|
calls: list[ToolCall] = []
|
|
544
557
|
|
|
545
558
|
# 1. Hermes / Kimi direct tags format: <tool_name>search</tool_name><pattern>...</pattern>
|
|
@@ -584,18 +597,27 @@ class ToolParser:
|
|
|
584
597
|
if m_fn:
|
|
585
598
|
tool_name = m_fn.group(1)
|
|
586
599
|
|
|
587
|
-
# Check child tags for tool name (Anthropic format: <name>read_file</name>)
|
|
600
|
+
# Check child tags for tool name (Anthropic format: <name>read_file</name> or <function=search>)
|
|
588
601
|
if not tool_name:
|
|
589
602
|
m_child = _CHILD_TOOL_NAME_RE.search(body)
|
|
590
603
|
if m_child:
|
|
591
604
|
tool_name = m_child.group(1)
|
|
605
|
+
else:
|
|
606
|
+
m_child_eq = _CHILD_TOOL_FN_EQ_RE.search(body)
|
|
607
|
+
if m_child_eq:
|
|
608
|
+
tool_name = m_child_eq.group(1)
|
|
592
609
|
|
|
593
610
|
params = {}
|
|
594
611
|
|
|
595
|
-
# Check DSML / standard parameter tags: <parameter name="...">
|
|
612
|
+
# Check DSML / standard parameter tags: <parameter name="..."> or <parameter=...>
|
|
596
613
|
for pname, pval in _extract_xml_parameters(body):
|
|
597
614
|
is_code = pname in self.code_param_keys
|
|
598
|
-
|
|
615
|
+
cleaned = clean_param_val(pval, is_code_param=is_code)
|
|
616
|
+
if not is_code and (cleaned.startswith("{") or cleaned.startswith("[")):
|
|
617
|
+
parsed_val = safe_json_loads(cleaned, default=None)
|
|
618
|
+
params[pname] = parsed_val if parsed_val is not None else cleaned
|
|
619
|
+
else:
|
|
620
|
+
params[pname] = cleaned
|
|
599
621
|
|
|
600
622
|
# Check for <arguments>...</arguments> or <parameters>...</parameters> block (Anthropic)
|
|
601
623
|
m_wrap = _ARGUMENTS_BLOCK_RE.search(body)
|
|
@@ -20,8 +20,18 @@ _ATTR_NAME_RE = re.compile(
|
|
|
20
20
|
re.IGNORECASE,
|
|
21
21
|
)
|
|
22
22
|
|
|
23
|
+
_CHILD_TOOL_NAME_RE = re.compile(
|
|
24
|
+
r"<(?:name|tool_name|function_name|tool|function)>\s*([\w-]+)(?:</(?:name|tool_name|function_name|tool|function)>|\s|$)",
|
|
25
|
+
re.IGNORECASE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_CHILD_TOOL_FN_EQ_RE = re.compile(
|
|
29
|
+
r"""<[||]*(?:dsml[||]*)?(?:function|tool|invoke|action|func)\s*=\s*['"]?([\w.:-]+)""",
|
|
30
|
+
re.IGNORECASE,
|
|
31
|
+
)
|
|
32
|
+
|
|
23
33
|
_INVOKE_RE = re.compile(
|
|
24
|
-
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
34
|
+
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['\"]?(?P<colon_tool>[\w.:-]+)['\"]?)?\b(?P<attrs>[^>]*)>(?P<body>.*?)(?:</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>|<[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)\b|</[||]*(?:dsml[||]*)?(?:tool_call|tool_calls|function_calls|calls|tools)\s*>|$)",
|
|
25
35
|
re.IGNORECASE | re.DOTALL,
|
|
26
36
|
)
|
|
27
37
|
|
|
@@ -43,12 +53,12 @@ _REACT_ACTION_RE = re.compile(
|
|
|
43
53
|
_MISTRAL_RE = re.compile(r"\[TOOL_CALLS\]\s*(\[\s*\{.*?\}\s*\])", re.IGNORECASE | re.DOTALL)
|
|
44
54
|
|
|
45
55
|
_PARAM_START_RE = re.compile(
|
|
46
|
-
r"<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\b[^>]*?\bname\s*=\s*['\"]?(?P<pname>[\w
|
|
56
|
+
r"<[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:=\s*['\"]?(?P<pname_eq>[\w.:-]+)['\"]?|\b[^>]*?\bname\s*=\s*['\"]?(?P<pname>[\w.:-]+)['\"]?)[^>]*>",
|
|
47
57
|
re.IGNORECASE | re.DOTALL,
|
|
48
58
|
)
|
|
49
59
|
|
|
50
60
|
_PARAM_END_RE = re.compile(
|
|
51
|
-
r"</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)\s*>",
|
|
61
|
+
r"</[||]*(?:dsml[||]*)?(?:parameter|param|arg|argument|параметр|аргумент)(?:(?::|=)\s*[\w.:-]+)?\s*>",
|
|
52
62
|
re.IGNORECASE | re.DOTALL,
|
|
53
63
|
)
|
|
54
64
|
|
|
@@ -98,7 +108,9 @@ class RawToolCall:
|
|
|
98
108
|
start = m.start()
|
|
99
109
|
if any(cs <= start < ce for cs, ce in cdata_spans):
|
|
100
110
|
continue
|
|
101
|
-
pname = m.group("pname").strip().lower()
|
|
111
|
+
pname = (m.group("pname") or m.group("pname_eq") or "").strip().lower()
|
|
112
|
+
if not pname:
|
|
113
|
+
continue
|
|
102
114
|
valid_starts.append((start, m.end(), pname))
|
|
103
115
|
|
|
104
116
|
for i, (_, val_start, pname) in enumerate(valid_starts):
|
|
@@ -163,6 +175,11 @@ class RawToolCall:
|
|
|
163
175
|
def extract_raw_tool_calls(text: str) -> list[RawToolCall]:
|
|
164
176
|
"""Extract raw tool calls from text without full JSON parsing overhead."""
|
|
165
177
|
clean = strip_thinking(text)
|
|
178
|
+
lower = clean.lower()
|
|
179
|
+
if ("<tool_call>" in lower or "</tool_call>" in lower) and any(
|
|
180
|
+
t in lower for t in ("<function", "<invoke", "<action")
|
|
181
|
+
):
|
|
182
|
+
clean = re.sub(r"</?[||]*(?:dsml[||]*)?tool_call\s*>", "", clean)
|
|
166
183
|
results: list[RawToolCall] = []
|
|
167
184
|
|
|
168
185
|
# 1. DSML and XML invocation blocks
|
|
@@ -179,6 +196,14 @@ def extract_raw_tool_calls(text: str) -> list[RawToolCall]:
|
|
|
179
196
|
attr_m = _ATTR_NAME_RE.search(attrs)
|
|
180
197
|
if attr_m:
|
|
181
198
|
name = attr_m.group(1).strip()
|
|
199
|
+
else:
|
|
200
|
+
m_child = _CHILD_TOOL_NAME_RE.search(body)
|
|
201
|
+
if m_child:
|
|
202
|
+
name = m_child.group(1).strip()
|
|
203
|
+
else:
|
|
204
|
+
m_child_eq = _CHILD_TOOL_FN_EQ_RE.search(body)
|
|
205
|
+
if m_child_eq:
|
|
206
|
+
name = m_child_eq.group(1).strip()
|
|
182
207
|
|
|
183
208
|
if name:
|
|
184
209
|
results.append(RawToolCall(name=name, raw_args=body.strip(), raw_source=full))
|
|
@@ -15,12 +15,12 @@ _ATTR_NAME_RE = re.compile(
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
_INVOKE_OPEN_RE = re.compile(
|
|
18
|
-
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
18
|
+
r"<[||]*(?:dsml[||]*)?(?P<tag>tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*['\"]?(?P<colon_tool>[\w.:-]+)['\"]?)?\b(?P<attrs>[^>]*)>",
|
|
19
19
|
re.IGNORECASE,
|
|
20
20
|
)
|
|
21
21
|
|
|
22
22
|
_TAG_CLOSE_RE = re.compile(
|
|
23
|
-
r"</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(
|
|
23
|
+
r"</[||]*(?:dsml[||]*)?(?:tool_invoke|invoke|tool_call|call|tool|invocation|function_call|function|func|action|tool_use|ant_tool_use|function_use|действие|вызов_функции|функция|инструмент)(?:(?::|=)\s*[\w.:-]+)?\s*>",
|
|
24
24
|
re.IGNORECASE,
|
|
25
25
|
)
|
|
26
26
|
|
|
@@ -198,6 +198,11 @@ class StreamingToolParser:
|
|
|
198
198
|
attr_m = _ATTR_NAME_RE.search(attrs)
|
|
199
199
|
if attr_m:
|
|
200
200
|
raw_name = attr_m.group(1).strip()
|
|
201
|
+
elif m.group("tag").lower() in ("tool_call", "tool_calls"):
|
|
202
|
+
after_tag = self.buffer[end:]
|
|
203
|
+
json_m = _JSON_NAME_RE.search(after_tag[:300])
|
|
204
|
+
if json_m:
|
|
205
|
+
raw_name = json_m.group("name").strip()
|
|
201
206
|
|
|
202
207
|
if raw_name:
|
|
203
208
|
norm_name = self.parser.normalize_name(raw_name)
|
|
@@ -433,7 +438,10 @@ class StreamingToolParser:
|
|
|
433
438
|
call_str = self.buffer[:body_end]
|
|
434
439
|
name = self._current_name or "unknown"
|
|
435
440
|
if self._current_format == "xml_dsml":
|
|
436
|
-
|
|
441
|
+
if call_str.strip().startswith("{"):
|
|
442
|
+
synthetic = call_str.strip()
|
|
443
|
+
else:
|
|
444
|
+
synthetic = f'<invoke name="{name}">{call_str}</invoke>'
|
|
437
445
|
elif self._current_format == "native_token":
|
|
438
446
|
synthetic = f"<|tool call begin|>function={name}<|tool sep|>{call_str}<|tool call end|>"
|
|
439
447
|
elif self._current_format == "qwen":
|
|
@@ -495,7 +503,10 @@ class StreamingToolParser:
|
|
|
495
503
|
if self.buffer:
|
|
496
504
|
name = self._current_name or "unknown"
|
|
497
505
|
if self._current_format == "xml_dsml":
|
|
498
|
-
|
|
506
|
+
if self.buffer.strip().startswith("{"):
|
|
507
|
+
synthetic = self.buffer.strip()
|
|
508
|
+
else:
|
|
509
|
+
synthetic = f'<invoke name="{name}">{self.buffer}</invoke>'
|
|
499
510
|
elif self._current_format == "native_token":
|
|
500
511
|
synthetic = f"<|tool call begin|>function={name}<|tool sep|>{self.buffer}<|tool call end|>"
|
|
501
512
|
elif self._current_format == "qwen":
|
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/Cargo.toml
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/canonical.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/cleaners.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/lib.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/models.rs
RENAMED
|
File without changes
|
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/semantics.rs
RENAMED
|
File without changes
|
{agent_tool_parser-0.1.4 → agent_tool_parser-0.1.5}/crates/agent-tool-parser-core/src/vector.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|