agent-tool-parser 0.1.0__py3-none-any.whl

agent_tool_parser/__init__.py

@@ -0,0 +1,70 @@
+"""agent-tool-parser: Fast, multi-format parser for LLM agent tool calls.
+
+Copyright (c) 2026 InBoost Team.
+Licensed under the MIT License. Provided "AS IS" without warranty of any kind.
+See LICENSE and README.md for full terms and execution safety disclaimers.
+"""
+
+from __future__ import annotations
+
+import os
+
+# Allow forcing pure-Python mode via environment variable for benchmarking or debugging
+_DISABLE_ACCEL = os.getenv("AGENT_TOOL_PARSER_NO_EXT", "0") in ("1", "true", "True")
+
+ACCELERATED: bool = False
+
+if not _DISABLE_ACCEL:
+    try:
+        from agent_tool_parser._accelerated import (  # type: ignore[import-untyped,import-not-found]
+            ToolCall,
+            ToolError,
+            ToolParser,
+            clean_json_str,
+            parse_tool_call,
+            parse_tool_calls,
+            safe_json_loads,
+            strip_thinking,
+            try_parse_tool_call,
+            try_parse_tool_calls,
+        )
+
+        ACCELERATED = True
+    except ImportError:
+        pass
+
+if not ACCELERATED:
+    from agent_tool_parser.cleaners import (
+        clean_json_str,
+        clean_param_val,
+        extract_json_objects,
+        safe_json_loads,
+        strip_thinking,
+    )
+    from agent_tool_parser.models import ToolCall, ToolError
+    from agent_tool_parser.parser import (
+        ToolParser,
+        parse_tool_call,
+        parse_tool_calls,
+        try_parse_tool_call,
+        try_parse_tool_calls,
+    )
+else:
+    from agent_tool_parser.cleaners import clean_param_val, extract_json_objects
+
+__version__ = "0.1.0"
+__all__ = [
+    "ACCELERATED",
+    "ToolCall",
+    "ToolError",
+    "ToolParser",
+    "clean_json_str",
+    "clean_param_val",
+    "extract_json_objects",
+    "parse_tool_call",
+    "parse_tool_calls",
+    "safe_json_loads",
+    "strip_thinking",
+    "try_parse_tool_call",
+    "try_parse_tool_calls",
+]

agent_tool_parser/cleaners.py

@@ -0,0 +1,283 @@
+# Copyright (c) 2026 InBoost Team
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import html
+import json
+import re
+from typing import Any
+
+_CODE_PARAM_KEYS_DEFAULT: set[str] = {
+    "old_str",
+    "new_str",
+    "content",
+    "old",
+    "new",
+    "code",
+    "text",
+    "replacement",
+    "patch",
+    "diff",
+}
+
+_THINK_OPEN_RE = re.compile(r"<[｜|]?(?:think|thought|reasoning)[｜|]?>", re.IGNORECASE)
+_THINK_CLOSE_RE = re.compile(r"</[｜|]?(?:think|thought|reasoning)[｜|]?>", re.IGNORECASE)
+
+_TOOL_START_CANDIDATES = (
+    "<tool_call",
+    "<tool_calls",
+    "<tools",
+    "<tool",
+    "<invoke",
+    "<dsml",
+    "<call",
+    "<tool_use",
+    "<ant_tool_use",
+    "<action_start",
+    "<action_name",
+    "<python_tag",
+    "```",
+    "action:",
+    "✿function✿",
+)
+
+_CDATA_RE = re.compile(r"<!\[CDATA\[(.*?)(?:\]\]>|$)", re.DOTALL)
+_SINGLE_QUOTE_KEYS_RE = re.compile(r"([{,]\s*)'([^']+)'(\s*:)")
+_SINGLE_QUOTE_VALS_RE = re.compile(r"(:\s*)'([^']*)'(\s*[,}])")
+_TRAILING_COMMA_RE = re.compile(r",\s*([}\]])")
+_PY_TRUE_RE = re.compile(r"\bTrue\b")
+_PY_FALSE_RE = re.compile(r"\bFalse\b")
+_PY_NONE_RE = re.compile(r"\bNone\b")
+
+
+def strip_thinking(text: str) -> str:
+    """Strips internal reasoning / thinking tags (<think>...</think>)."""
+    if "<" not in text:
+        return text
+
+    open_m = _THINK_OPEN_RE.search(text)
+    if open_m:
+        after_open = text[open_m.end() :]
+        # 1. Look for explicit closing tag
+        close_m = _THINK_CLOSE_RE.search(after_open)
+        if close_m:
+            before = text[: open_m.start()]
+            after = after_open[close_m.end() :]
+            result = before + after
+            if _THINK_OPEN_RE.search(result):
+                return strip_thinking(result)
+            return result.strip()
+
+        # 2. No closing tag: find first tool candidate
+        lower = after_open.lower()
+        min_pos = None
+        for cand in _TOOL_START_CANDIDATES:
+            pos = lower.find(cand)
+            if pos != -1:
+                if min_pos is None or pos < min_pos:
+                    min_pos = pos
+        if min_pos is not None:
+            before = text[: open_m.start()]
+            after = after_open[min_pos:]
+            return (before + after).strip()
+
+        before = text[: open_m.start()].strip()
+        if before:
+            return before
+
+    return text
+
+
+def clean_param_val(val: str, is_code_param: bool = False) -> str:
+    """Cleans an extracted XML/DSML parameter value.
+
+    1. Unwraps <![CDATA[...]]> blocks if present (preserving raw code).
+    2. Unescapes HTML/XML entities (&amp;, &lt;, &gt;, &quot;) only if not in CDATA.
+    3. Trims whitespace appropriately: preserves indentation for code/diffs,
+       stripping only leading/trailing bounding newlines.
+    """
+    has_cdata = False
+    if "<![CDATA[" in val:
+        m_cdata = _CDATA_RE.search(val)
+        if m_cdata:
+            val = m_cdata.group(1)
+            has_cdata = True
+
+    if not has_cdata and "&" in val:
+        val = html.unescape(val)
+
+    if is_code_param:
+        val = val.removeprefix("\r\n").removeprefix("\n").removesuffix("\r\n").removesuffix("\n")
+    else:
+        val = val.strip()
+
+    return val
+
+
+def clean_json_str(s: str) -> str:
+    """Applies heuristic repairs to malformed JSON strings.
+
+    - Replaces single-quoted keys and string values with double quotes.
+    - Strips trailing commas before closing braces/brackets.
+    - Normalizes Python literals (True, False, None) to JSON (true, false, null).
+    """
+    if "'" in s:
+        if '"' not in s:
+            s = s.replace("'", '"')
+        else:
+            # Single-quoted keys: {'key': ...} -> {"key": ...}
+            s = _SINGLE_QUOTE_KEYS_RE.sub(r'\g<1>"\g<2>"\g<3>', s)
+
+            # Single-quoted values: : 'val' -> : "val" (escaping any inner double quotes)
+            def _replace_sq_val(m: re.Match[str]) -> str:
+                prefix, val, suffix = m.group(1), m.group(2), m.group(3)
+                val_escaped = val.replace('\\"', '"').replace('"', '\\"')
+                return f'{prefix}"{val_escaped}"{suffix}'
+
+            s = _SINGLE_QUOTE_VALS_RE.sub(_replace_sq_val, s)
+
+    # Fix trailing commas: {"a": 1,} -> {"a": 1}
+    if "," in s:
+        s = _TRAILING_COMMA_RE.sub(r"\g<1>", s)
+
+    # Replace Python literals
+    if "True" in s:
+        s = _PY_TRUE_RE.sub("true", s)
+    if "False" in s:
+        s = _PY_FALSE_RE.sub("false", s)
+    if "None" in s:
+        s = _PY_NONE_RE.sub("null", s)
+    return s
+
+
+try:
+    import orjson  # type: ignore[import-not-found,import-untyped]
+
+    def _fast_json_loads(s: str) -> Any:
+        return orjson.loads(s)
+except ImportError:
+    import json
+
+    def _fast_json_loads(s: str) -> Any:
+        return json.loads(s, strict=False)
+
+
+try:
+    import json_repair  # type: ignore[import-not-found,import-untyped]
+
+    def _repair_json_ext(s: str) -> Any:
+        return json_repair.repair_json(s, return_objects=True)
+except ImportError:
+    _repair_json_ext = None  # type: ignore[assignment]
+
+
+def safe_json_loads(s: str, default: Any = None) -> Any:
+    """Safely parses JSON with fallback to heuristic repair, returning default on failure.
+
+    Args:
+        s: Raw JSON string or malformed candidate.
+        default: Fallback value if parsing fails completely.
+
+    Returns:
+        Parsed JSON object or default value.
+    """
+    if not s or not isinstance(s, str):
+        return default
+    try:
+        return _fast_json_loads(s)
+    except Exception:
+        pass
+
+    try:
+        return _fast_json_loads(clean_json_str(s))
+    except Exception:
+        pass
+
+    if _repair_json_ext is not None:
+        try:
+            res = _repair_json_ext(s)
+            if isinstance(res, (dict, list)):
+                return res
+        except Exception:
+            pass
+
+    return default
+
+
+def extract_json_objects(text: str) -> list[str]:
+    """Extracts all balanced JSON object candidates from arbitrary text using bracket-depth tracking.
+
+    Handles double and single quoted strings, escape characters, and auto-completes
+    truncated objects (including unclosed arrays and nested objects) if the LLM stream
+    was cut off mid-generation.
+    """
+    if "{" not in text:
+        return []
+
+    results: list[str] = []
+    stack: list[str] = []
+    start = -1
+    in_string = False
+    quote_char: str | None = None
+    escape = False
+
+    for i, char in enumerate(text):
+        if char in ('"', "'") and not escape:
+            if not in_string:
+                in_string = True
+                quote_char = char
+            elif quote_char == char:
+                in_string = False
+                quote_char = None
+        elif char == "\\" and in_string:
+            escape = not escape
+            continue
+        elif not in_string:
+            if char == "{":
+                if not stack:
+                    start = i
+                stack.append("{")
+            elif char == "[":
+                if stack:
+                    stack.append("[")
+            elif char == "}":
+                if stack and stack[-1] == "{":
+                    stack.pop()
+                    if not stack and start != -1:
+                        results.append(text[start : i + 1])
+                        start = -1
+                elif stack and "{" in stack:
+                    while stack and stack[-1] != "{":
+                        stack.pop()
+                    if stack:
+                        stack.pop()
+                        if not stack and start != -1:
+                            results.append(text[start : i + 1])
+                            start = -1
+            elif char == "]":
+                if stack and stack[-1] == "[":
+                    stack.pop()
+        if escape:
+            escape = False
+
+    # Auto-repair truncated tail if JSON generation was cut off mid-stream
+    if stack and start != -1:
+        tail = text[start:]
+        if in_string and quote_char:
+            if escape or tail.endswith("\\"):
+                tail = tail.rstrip("\\")
+            tail += quote_char
+        trimmed_tail = tail.rstrip()
+        if trimmed_tail.endswith(":"):
+            tail = trimmed_tail + '""'
+        elif trimmed_tail.endswith(","):
+            tail = trimmed_tail[:-1]
+        for b in reversed(stack):
+            if b == "[":
+                tail += "]"
+            elif b == "{":
+                tail += "}"
+        results.append(tail)
+
+    return results

agent_tool_parser/models.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2026 InBoost Team
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ToolCall:
+    """Represents a structured tool invocation extracted from LLM output.
+
+    Attributes:
+        name: Normalized name of the tool to execute.
+        args: Dictionary of parsed arguments/parameters for the tool.
+        raw_source: Snippet of the original response that matched.
+    """
+
+    name: str
+    args: dict[str, Any] = field(default_factory=dict)
+    raw_source: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "args": self.args,
+            "raw_source": self.raw_source,
+        }
+
+    def __repr__(self) -> str:
+        return f"ToolCall(name={self.name!r}, args={self.args!r})"
+
+
+class ToolError(ValueError):
+    """Raised when no valid tool call could be extracted or validation failed."""