abstractcore 2.6.8__py3-none-any.whl → 2.9.0__py3-none-any.whl

abstractcore/tools/registry.py

@@ -17,6 +17,35 @@ from ..utils.structured_logging import get_logger
 logger = get_logger(__name__)
 
 
+def _error_from_output(value: Any) -> Optional[str]:
+    """Detect tool failures reported as string outputs (instead of exceptions)."""
+    # Allow tools to return structured outputs while still communicating failure
+    # without raising exceptions. We only treat this as an error when the tool
+    # explicitly marks itself as unsuccessful.
+    if isinstance(value, dict):
+        success = value.get("success")
+        ok = value.get("ok")
+        if success is False or ok is False:
+            err = value.get("error") or value.get("message") or "Tool reported failure"
+            text = str(err).strip()
+            return text or "Tool reported failure"
+        return None
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text:
+        return None
+    if text.startswith("Error:"):
+        cleaned = text[len("Error:") :].strip()
+        return cleaned or text
+    if text.startswith(("❌", "🚫", "⏰")):
+        cleaned = text.lstrip("❌🚫⏰").strip()
+        if cleaned.startswith("Error:"):
+            cleaned = cleaned[len("Error:") :].strip()
+        return cleaned or text
+    return None
+
+
 class ToolRegistry:
     """Registry for managing available tools."""
 
@@ -149,10 +178,41 @@ class ToolRegistry:
             return error_result
 
         try:
+            from .arg_canonicalizer import canonicalize_tool_arguments
+
+            arguments = canonicalize_tool_arguments(tool_call.name, tool_call.arguments)
+
             # Execute the function with the provided arguments
-            result = tool_def.function(**tool_call.arguments)
+            result = tool_def.function(**arguments)
             duration_ms = (time.time() - start_time) * 1000
 
+            implied_error = _error_from_output(result)
+            if implied_error is not None:
+                error_result = ToolResult(
+                    call_id=tool_call.call_id or "",
+                    # Preserve structured outputs for post-mortem evidence/provenance.
+                    # For string-only error outputs, store the message in `error` and keep output empty.
+                    output=result if not isinstance(result, str) else "",
+                    error=implied_error,
+                    success=False,
+                )
+
+                # Emit tool error event
+                result_data = create_tool_event(
+                    tool_name=tool_call.name,
+                    arguments=arguments,
+                    success=False,
+                    error=implied_error,
+                )
+                emit_global(
+                    EventType.TOOL_COMPLETED,
+                    result_data,
+                    source="ToolRegistry",
+                    duration_ms=duration_ms,
+                )
+
+                return error_result
+
             success_result = ToolResult(
                 call_id=tool_call.call_id or "",
                 output=result,
@@ -162,7 +222,7 @@ class ToolRegistry:
             # Emit successful tool result event
             result_data = create_tool_event(
                 tool_name=tool_call.name,
-                arguments=tool_call.arguments,
+                arguments=arguments,
                 result=result,
                 success=True
             )
@@ -172,6 +232,51 @@ class ToolRegistry:
             return success_result
 
         except TypeError as e:
+            # Some models include wrapper/meta keys ("name", nested "arguments") or
+            # stray extras in tool kwargs. Retry once with a sanitized argument dict.
+            try:
+                wrapper_keys = {"name", "arguments", "call_id", "id"}
+                from .arg_canonicalizer import canonicalize_tool_arguments
+
+                args = canonicalize_tool_arguments(tool_call.name, tool_call.arguments)
+                for _ in range(4):
+                    inner = args.get("arguments")
+                    if not isinstance(inner, dict):
+                        break
+                    extras = {k: v for k, v in args.items() if k not in wrapper_keys}
+                    merged = dict(inner)
+                    for k, v in extras.items():
+                        merged.setdefault(k, v)
+                    args = merged
+
+                allowed = set(tool_def.parameters.keys()) if isinstance(tool_def.parameters, dict) else set()
+                if allowed:
+                    args = {k: v for k, v in args.items() if k in allowed}
+
+                if args != dict(tool_call.arguments or {}):
+                    result = tool_def.function(**args)
+                    duration_ms = (time.time() - start_time) * 1000
+                    success_result = ToolResult(
+                        call_id=tool_call.call_id or "",
+                        output=result,
+                        success=True,
+                    )
+                    result_data = create_tool_event(
+                        tool_name=tool_call.name,
+                        arguments=args,
+                        result=result,
+                        success=True,
+                    )
+                    emit_global(
+                        EventType.TOOL_COMPLETED,
+                        result_data,
+                        source="ToolRegistry",
+                        duration_ms=duration_ms,
+                    )
+                    return success_result
+            except Exception:
+                pass
+
             duration_ms = (time.time() - start_time) * 1000
             error_msg = f"Invalid arguments for tool '{tool_call.name}': {e}"
             logger.warning(error_msg)

abstractcore/tools/syntax_rewriter.py

@@ -8,7 +8,7 @@ Supports multiple target formats including OpenAI, Codex, and custom agent forma
 import re
 import json
 import uuid
-from typing import List, Dict, Any, Optional, Union
+from typing import List, Dict, Any, Optional, Union, Iterable
 from dataclasses import dataclass
 from enum import Enum
 
@@ -87,7 +87,7 @@ class ToolCallSyntaxRewriter:
     def rewrite_content(
         self,
         content: str,
-        detected_tool_calls: Optional[List[ToolCall]] = None
+        detected_tool_calls: Optional[List[Any]] = None
     ) -> str:
         """
         Rewrite tool call syntax in content.
@@ -99,7 +99,9 @@ class ToolCallSyntaxRewriter:
         Returns:
             Content with rewritten tool call syntax
         """
-        if not content or not content.strip():
+        # Allow formatting tool calls even when there's no surrounding assistant text.
+        # This is useful for streaming: a chunk may contain only tool calls.
+        if (not isinstance(content, str) or not content.strip()) and not detected_tool_calls:
             return content
 
         # Passthrough mode - return unchanged
@@ -110,6 +112,8 @@ class ToolCallSyntaxRewriter:
         if detected_tool_calls is None:
             detected_tool_calls = parse_tool_calls(content, self.model_name)
             logger.debug(f"Detected {len(detected_tool_calls)} tool calls in content")
+        else:
+            detected_tool_calls = list(self._coerce_tool_calls(detected_tool_calls))
 
         # No tool calls found
         if not detected_tool_calls:
@@ -118,7 +122,7 @@ class ToolCallSyntaxRewriter:
         # Apply format-specific rewriting
         return self._apply_format_conversion(content, detected_tool_calls)
 
-    def convert_to_openai_format(self, tool_calls: List[ToolCall]) -> List[Dict[str, Any]]:
+    def convert_to_openai_format(self, tool_calls: List[Any]) -> List[Dict[str, Any]]:
         """
         Convert tool calls to OpenAI API format.
 
@@ -130,7 +134,7 @@ class ToolCallSyntaxRewriter:
         """
         openai_tools = []
 
-        for tool_call in tool_calls:
+        for tool_call in self._coerce_tool_calls(tool_calls):
             # Ensure we have a call ID
             call_id = tool_call.call_id or f"call_{uuid.uuid4().hex[:8]}"
 
@@ -161,6 +165,54 @@ class ToolCallSyntaxRewriter:
 
         return openai_tools
 
+    def _coerce_tool_calls(self, tool_calls: Iterable[Any]) -> Iterable[ToolCall]:
+        """Coerce ToolCall-like inputs (dicts, dataclasses) into ToolCall objects."""
+        for tc in tool_calls or []:
+            if isinstance(tc, ToolCall):
+                yield tc
+                continue
+
+            if isinstance(tc, dict):
+                raw_id = tc.get("call_id")
+                if raw_id is None:
+                    raw_id = tc.get("id")
+
+                raw_name = tc.get("name")
+                raw_args = tc.get("arguments")
+                func = tc.get("function") if isinstance(tc.get("function"), dict) else None
+                if func and (not isinstance(raw_name, str) or not raw_name.strip()):
+                    raw_name = func.get("name")
+                if func and raw_args is None:
+                    raw_args = func.get("arguments")
+
+                if not isinstance(raw_name, str) or not raw_name.strip():
+                    continue
+                name = raw_name.strip()
+
+                arguments: Any = raw_args if raw_args is not None else {}
+                if isinstance(arguments, str):
+                    # Try to parse JSON-ish args; fall back to string.
+                    try:
+                        parsed = json.loads(arguments)
+                        arguments = parsed if isinstance(parsed, dict) else {}
+                    except Exception:
+                        arguments = {}
+                if not isinstance(arguments, dict):
+                    arguments = {}
+
+                yield ToolCall(name=name, arguments=arguments, call_id=str(raw_id) if raw_id is not None else None)
+                continue
+
+            raw_name = getattr(tc, "name", None)
+            raw_args = getattr(tc, "arguments", None)
+            raw_id = getattr(tc, "call_id", None)
+            if not isinstance(raw_name, str) or not raw_name.strip():
+                continue
+            arguments = raw_args if raw_args is not None else {}
+            if not isinstance(arguments, (dict, str)):
+                arguments = {}
+            yield ToolCall(name=raw_name.strip(), arguments=arguments, call_id=str(raw_id) if raw_id is not None else None)
+
     def _apply_format_conversion(self, content: str, tool_calls: List[ToolCall]) -> str:
         """Apply format-specific conversion."""
 
@@ -348,10 +400,17 @@ class ToolCallSyntaxRewriter:
             r'<function_call>.*?</function_call>',
             r'<tool_call>.*?</tool_call>',
             r'```tool_code.*?```',
+            # Harmony/ChatML tool transcript: <|channel|>... to=tool ... <|message|>{...}
+            r'<\|channel\|>\s*[a-zA-Z0-9_\-]+\s+to=[a-zA-Z0-9_\-\.]+\b.*?<\|message\|>\s*\{.*?\}',
         ]
         
         for pattern in complete_patterns:
             cleaned = re.sub(pattern, '', cleaned, flags=re.DOTALL | re.IGNORECASE)
+
+        # Remove any remaining Harmony conversation tags that shouldn't appear.
+        cleaned = re.sub(r'<\|channel\|>', '', cleaned)
+        cleaned = re.sub(r'<\|message\|>', '', cleaned)
+        cleaned = re.sub(r'<\|constrain\|>', '', cleaned)
         
         # Second pass: remove orphaned tags (from malformed tool calls)
         orphaned_patterns = [
@@ -362,6 +421,9 @@ class ToolCallSyntaxRewriter:
             r'<tool_call>',
             r'</tool_call>',
             r'```tool_code',
+            r'<\|channel\|>',
+            r'<\|message\|>',
+            r'<\|constrain\|>',
         ]
         
         for pattern in orphaned_patterns:
@@ -468,4 +530,4 @@ def auto_detect_format(
         return SyntaxFormat.XML
 
     # Default to OpenAI format for maximum compatibility
-    return SyntaxFormat.OPENAI
+    return SyntaxFormat.OPENAI

abstractcore/tools/tag_rewriter.py

@@ -10,6 +10,7 @@ import json
 from typing import Dict, Any, Optional, Tuple, List
 from dataclasses import dataclass
 from ..utils.structured_logging import get_logger
+from ..utils.jsonish import loads_dict_like as _loads_dict_like
 
 logger = get_logger(__name__)
 
@@ -171,6 +172,12 @@ class ToolCallTagRewriter:
             logger.debug("Early return: text is empty or preserve_json is False")
             return text
 
+        # Pre-pass: convert CLI-like `tool: [name]: {...}` calls into canonical tagged JSON.
+        # Some OSS models emit this format even when prompted for other tags.
+        text = self._rewrite_bracket_prefix_calls(text)
+        # Pre-pass: convert Harmony/ChatML tool transcript format into canonical tagged JSON.
+        text = self._rewrite_harmony_prefix_calls(text)
+
         # Check if we already have the target format (avoid double-tagging)
         # Check using output tags (with angle brackets)
         if (self._output_start_tag in text and
@@ -216,6 +223,184 @@ class ToolCallTagRewriter:
 
         logger.debug(f"Final rewritten text: {rewritten[:200] if rewritten else None}")
         return rewritten
+
+    def _rewrite_bracket_prefix_calls(self, text: str) -> str:
+        """Rewrite `tool: [name]: {args}` lines into the configured tag format."""
+        if not text:
+            return text
+
+        pattern = re.compile(r"(?im)^\s*tool\s*:\s*\[([a-zA-Z0-9_\-]+)\]\s*:\s*")
+
+        def _find_matching_brace(s: str, start: int) -> int:
+            depth = 0
+            in_string = False
+            quote = ""
+            escaped = False
+            for i in range(start, len(s)):
+                ch = s[i]
+                if in_string:
+                    if escaped:
+                        escaped = False
+                        continue
+                    if ch == "\\":
+                        escaped = True
+                        continue
+                    if ch == quote:
+                        in_string = False
+                        quote = ""
+                    continue
+                if ch in ("'", '"'):
+                    in_string = True
+                    quote = ch
+                    continue
+                if ch == "{":
+                    depth += 1
+                    continue
+                if ch == "}":
+                    depth -= 1
+                    if depth == 0:
+                        return i
+            return -1
+
+        rewritten = text
+        matches = list(pattern.finditer(text))
+        if not matches:
+            return text
+
+        # Replace from end to start to preserve match indices while mutating the string.
+        for match in reversed(matches):
+            name = str(match.group(1) or "").strip()
+            if not name:
+                continue
+
+            brace_start = rewritten.find("{", match.end())
+            if brace_start == -1:
+                continue
+            between = rewritten[match.end() : brace_start]
+            if between and any(not c.isspace() for c in between):
+                continue
+
+            brace_end = _find_matching_brace(rewritten, brace_start)
+            if brace_end == -1:
+                continue
+
+            raw_args = rewritten[brace_start : brace_end + 1]
+            args = _loads_dict_like(raw_args)
+            if not isinstance(args, dict):
+                continue
+
+            # Some models emit a wrapper payload even when a syntax expects arguments-only JSON:
+            #   {"name":"tool","arguments":{...}}
+            # Unwrap so we rewrite into canonical {"name": ..., "arguments": {...}} once.
+            inner_args = args.get("arguments")
+            if isinstance(inner_args, dict):
+                args = inner_args
+            elif isinstance(inner_args, str):
+                parsed = _loads_dict_like(inner_args)
+                if isinstance(parsed, dict):
+                    args = parsed
+
+            payload = json.dumps({"name": name, "arguments": args}, ensure_ascii=False)
+            replacement = f"{self._output_start_tag}{payload}{self._output_end_tag}"
+            rewritten = rewritten[: match.start()] + replacement + rewritten[brace_end + 1 :]
+
+        return rewritten
+
+    def _rewrite_harmony_prefix_calls(self, text: str) -> str:
+        """Rewrite Harmony/ChatML tool transcript blocks into the configured tag format.
+
+        Example:
+            <|channel|>commentary to=list_files <|constrain|>json<|message|>{"directory_path":"."}
+        """
+        if not text:
+            return text
+        if "<|channel|>" not in text or "<|message|>" not in text or "to=" not in text:
+            return text
+
+        header_re = re.compile(
+            r"(?i)<\|channel\|>\s*[a-zA-Z0-9_\-]+\s+to=([a-zA-Z0-9_\-\.]+)\b"
+        )
+
+        def _find_matching_brace(s: str, start: int) -> int:
+            depth = 0
+            in_string = False
+            quote = ""
+            escaped = False
+            for i in range(start, len(s)):
+                ch = s[i]
+                if in_string:
+                    if escaped:
+                        escaped = False
+                        continue
+                    if ch == "\\":
+                        escaped = True
+                        continue
+                    if ch == quote:
+                        in_string = False
+                        quote = ""
+                    continue
+                if ch in ("'", '"'):
+                    in_string = True
+                    quote = ch
+                    continue
+                if ch == "{":
+                    depth += 1
+                    continue
+                if ch == "}":
+                    depth -= 1
+                    if depth == 0:
+                        return i
+            return -1
+
+        rewritten = text
+        matches = list(header_re.finditer(text))
+        if not matches:
+            return text
+
+        msg_tag = "<|message|>"
+        for match in reversed(matches):
+            raw_name = str(match.group(1) or "").strip()
+            if not raw_name:
+                continue
+            name = raw_name
+            if name.startswith("functions."):
+                name = name.split(".", 1)[1].strip()
+            if not name:
+                continue
+
+            msg_start = rewritten.find(msg_tag, match.end())
+            if msg_start == -1:
+                continue
+            brace_start = rewritten.find("{", msg_start + len(msg_tag))
+            if brace_start == -1:
+                continue
+            between = rewritten[msg_start + len(msg_tag) : brace_start]
+            if between and any(not c.isspace() for c in between):
+                continue
+            brace_end = _find_matching_brace(rewritten, brace_start)
+            if brace_end == -1:
+                continue
+
+            raw_args = rewritten[brace_start : brace_end + 1]
+            args = _loads_dict_like(raw_args)
+            if not isinstance(args, dict):
+                continue
+
+            # Some models emit a wrapper payload in the Harmony message JSON:
+            #   {"name":"tool","arguments":{...},"call_id": "..."}
+            inner_args = args.get("arguments")
+            if isinstance(inner_args, dict):
+                args = inner_args
+            elif isinstance(inner_args, str):
+                parsed = _loads_dict_like(inner_args)
+                if isinstance(parsed, dict):
+                    args = parsed
+
+            payload = json.dumps({"name": name, "arguments": args}, ensure_ascii=False)
+            replacement = f"{self._output_start_tag}{payload}{self._output_end_tag}"
+            rewritten = rewritten[: match.start()] + replacement + rewritten[brace_end + 1 :]
+
+        return rewritten
     
     def rewrite_streaming_chunk(self, chunk: str, buffer: str = "") -> Tuple[str, str]:
         """
@@ -503,4 +688,4 @@ def create_tag_rewriter(cli_name: str = "qwen3") -> ToolCallTagRewriter:
         ToolCallTagRewriter instance
     """
     tags = get_predefined_tags(cli_name)
-    return ToolCallTagRewriter(tags)
+    return ToolCallTagRewriter(tags)

abstractcore/utils/jsonish.py

@@ -0,0 +1,111 @@
+"""Relaxed JSON/Python-literal parsing helpers.
+
+Some models emit tool-call JSON that is "almost JSON" (e.g., Python booleans,
+single quotes) or includes unescaped control characters (notably literal
+newlines inside string values). These helpers provide a single, robust way to
+parse dict-like payloads across AbstractCore.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+from typing import Any, Dict, Optional
+
+
+def _escape_control_chars_in_strings(text: str) -> str:
+    """Escape literal control chars that appear inside quoted strings.
+
+    This turns invalid JSON like:
+        {"content":"line1
+        line2"}
+    into valid JSON:
+        {"content":"line1\\nline2"}
+
+    Works for both single- and double-quoted strings (for Python-literal fallbacks).
+    """
+    if not text:
+        return text
+
+    out: list[str] = []
+    in_string = False
+    quote = ""
+    escaped = False
+
+    for ch in text:
+        if in_string:
+            if escaped:
+                out.append(ch)
+                escaped = False
+                continue
+            if ch == "\\":
+                out.append(ch)
+                escaped = True
+                continue
+            if ch == quote:
+                out.append(ch)
+                in_string = False
+                quote = ""
+                continue
+            if ch == "\n":
+                out.append("\\n")
+                continue
+            if ch == "\r":
+                out.append("\\r")
+                continue
+            if ch == "\t":
+                out.append("\\t")
+                continue
+            out.append(ch)
+            continue
+
+        if ch in ("'", '"'):
+            in_string = True
+            quote = ch
+            out.append(ch)
+            continue
+
+        out.append(ch)
+
+    return "".join(out)
+
+
+def loads_dict_like(raw: Any) -> Optional[Dict[str, Any]]:
+    """Parse a JSON-ish or Python-literal dict safely."""
+    if raw is None:
+        return None
+
+    text = str(raw).strip()
+    if not text:
+        return None
+
+    try:
+        value = json.loads(text)
+        if isinstance(value, dict):
+            return value
+    except json.JSONDecodeError:
+        fixed = _escape_control_chars_in_strings(text)
+        if fixed != text:
+            try:
+                value = json.loads(fixed)
+                if isinstance(value, dict):
+                    return value
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+    candidate = re.sub(r"\btrue\b", "True", text, flags=re.IGNORECASE)
+    candidate = re.sub(r"\bfalse\b", "False", candidate, flags=re.IGNORECASE)
+    candidate = re.sub(r"\bnull\b", "None", candidate, flags=re.IGNORECASE)
+    candidate = _escape_control_chars_in_strings(candidate)
+    try:
+        value = ast.literal_eval(candidate)
+    except Exception:
+        return None
+
+    if not isinstance(value, dict):
+        return None
+    return {str(k): v for k, v in value.items()}
+

abstractcore/utils/version.py

@@ -11,4 +11,4 @@ including when the package is installed from PyPI where pyproject.toml is not av
 
 # Package version - update this when releasing new versions
 # This must be manually synchronized with the version in pyproject.toml
-__version__ = "2.6.8"
+__version__ = "2.9.0"

{abstractcore-2.6.8.dist-info → abstractcore-2.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstractcore
-Version: 2.6.8
+Version: 2.9.0
 Summary: Unified interface to all LLM providers with essential infrastructure for tool calling, streaming, and model management
 Author-email: Laurent-Philippe Albou <contact@abstractcore.ai>
 Maintainer-email: Laurent-Philippe Albou <contact@abstractcore.ai>
@@ -57,8 +57,15 @@ Provides-Extra: processing
 Provides-Extra: tools
 Requires-Dist: beautifulsoup4<5.0.0,>=4.12.0; extra == "tools"
 Requires-Dist: lxml<6.0.0,>=4.9.0; extra == "tools"
-Requires-Dist: duckduckgo-search<4.0.0,>=3.8.0; extra == "tools"
+Requires-Dist: ddgs<10.0.0,>=9.10.0; python_version >= "3.10" and extra == "tools"
+Requires-Dist: duckduckgo-search<4.0.0,>=3.8.0; python_version < "3.10" and extra == "tools"
 Requires-Dist: psutil<6.0.0,>=5.9.0; extra == "tools"
+Provides-Extra: tool
+Requires-Dist: beautifulsoup4<5.0.0,>=4.12.0; extra == "tool"
+Requires-Dist: lxml<6.0.0,>=4.9.0; extra == "tool"
+Requires-Dist: ddgs<10.0.0,>=9.10.0; python_version >= "3.10" and extra == "tool"
+Requires-Dist: duckduckgo-search<4.0.0,>=3.8.0; python_version < "3.10" and extra == "tool"
+Requires-Dist: psutil<6.0.0,>=5.9.0; extra == "tool"
 Provides-Extra: media
 Requires-Dist: Pillow<12.0.0,>=10.0.0; extra == "media"
 Requires-Dist: pymupdf4llm<1.0.0,>=0.0.20; extra == "media"
@@ -282,6 +289,8 @@ print(f"Summary: {response.get_summary()}")         # "Model: gpt-4o-mini | Toke
 
 AbstractCore includes a comprehensive set of ready-to-use tools for common tasks:
 
+> Note: `abstractcore.tools.common_tools` requires `abstractcore[tools]` (BeautifulSoup, lxml, web search backends, etc.).
+
 ```python
 from abstractcore.tools.common_tools import fetch_url, search_files, read_file