chuk-tool-processor 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

chuk_tool_processor/plugins/discovery.py

@@ -1,121 +1,164 @@
 # chuk_tool_processor/plugins/discovery.py
-"""Plugin discovery & registry utilities for chuk_tool_processor"""
+"""Async-friendly plugin discovery & registry utilities for chuk_tool_processor."""
+
 from __future__ import annotations
 
 import importlib
 import inspect
 import logging
 import pkgutil
+from types import ModuleType
 from typing import Any, Dict, List, Optional, Set, Type
 
+from chuk_tool_processor.plugins.parsers.base import ParserPlugin
+from chuk_tool_processor.models.execution_strategy import ExecutionStrategy
+
+__all__ = [
+    "plugin_registry",
+    "PluginRegistry",
+    "PluginDiscovery",
+    "discover_default_plugins",
+    "discover_plugins",
+    "plugin",
+]
+
 logger = logging.getLogger(__name__)
 
 
+# -----------------------------------------------------------------------------
+# In-memory registry
+# -----------------------------------------------------------------------------
 class PluginRegistry:
-    """In‑memory registry keyed by *category → name*."""
+    """Thread-safe (GIL) in-memory registry keyed by *category → name*."""
 
-    def __init__(self) -> None:  # no side‑effects in import time
+    def __init__(self) -> None:
+        # category → {name → object}
         self._plugins: Dict[str, Dict[str, Any]] = {}
 
-    # ------------------------------------------------------------------
+    # --------------------------------------------------------------------- #
+    # Public API
+    # --------------------------------------------------------------------- #
     def register_plugin(self, category: str, name: str, plugin: Any) -> None:
         self._plugins.setdefault(category, {})[name] = plugin
         logger.debug("Registered plugin %s.%s", category, name)
 
-    def get_plugin(self, category: str, name: str) -> Optional[Any]:
+    def get_plugin(self, category: str, name: str) -> Optional[Any]:  # noqa: D401
         return self._plugins.get(category, {}).get(name)
 
     def list_plugins(self, category: str | None = None) -> Dict[str, List[str]]:
-        if category:
-            return {category: list(self._plugins.get(category, {}))}
-        return {cat: list(names) for cat, names in self._plugins.items()}
+        if category is not None:
+            return {category: sorted(self._plugins.get(category, {}))}
+        return {cat: sorted(names) for cat, names in self._plugins.items()}
 
 
+# -----------------------------------------------------------------------------
+# Discovery
+# -----------------------------------------------------------------------------
 class PluginDiscovery:
-    """Recursively scans packages for plugin classes and registers them."""
+    """
+    Recursively scans *package_paths* for plugin classes and registers them.
 
-    def __init__(self, registry: PluginRegistry) -> None:
-        self.registry = registry
-        self._seen: Set[str] = set()
+    * Parser plugins – concrete subclasses of :class:`ParserPlugin`
+      with an **async** ``try_parse`` coroutine.
 
-        # optional parser subsystem
-        try:
-            from chuk_tool_processor.parsers.base import ParserPlugin as _PP  # noqa: WPS433
-        except ModuleNotFoundError:
-            _PP = None
-        self.ParserPlugin = _PP
+    * Execution strategies – concrete subclasses of
+      :class:`ExecutionStrategy`.
 
-        # ExecutionStrategy always present inside core models
-        from chuk_tool_processor.models.execution_strategy import ExecutionStrategy  # noqa: WPS433
+    * Explicitly-decorated plugins – classes tagged with ``@plugin(...)``.
+    """
 
-        self.ExecutionStrategy = ExecutionStrategy
+    # ------------------------------------------------------------------ #
+    def __init__(self, registry: PluginRegistry) -> None:
+        self._registry = registry
+        self._seen_modules: Set[str] = set()
 
-    # ------------------------------------------------------------------
+    # ------------------------------------------------------------------ #
     def discover_plugins(self, package_paths: List[str]) -> None:
-        for pkg in package_paths:
-            self._walk(pkg)
+        """Import every package in *package_paths* and walk its subtree."""
+        for pkg_path in package_paths:
+            self._walk(pkg_path)
 
-    # ------------------------------------------------------------------
+    # ------------------------------------------------------------------ #
+    # Internal helpers
+    # ------------------------------------------------------------------ #
     def _walk(self, pkg_path: str) -> None:
         try:
-            pkg = importlib.import_module(pkg_path)
+            root_pkg = importlib.import_module(pkg_path)
         except ImportError as exc:  # pragma: no cover
             logger.warning("Cannot import package %s: %s", pkg_path, exc)
             return
 
-        for _, mod_name, is_pkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + "."):
-            if mod_name in self._seen:
+        self._inspect_module(root_pkg)
+
+        for _, mod_name, is_pkg in pkgutil.iter_modules(root_pkg.__path__, root_pkg.__name__ + "."):
+            if mod_name in self._seen_modules:
+                continue
+            self._seen_modules.add(mod_name)
+
+            try:
+                mod = importlib.import_module(mod_name)
+            except ImportError as exc:  # pragma: no cover
+                logger.debug("Cannot import module %s: %s", mod_name, exc)
                 continue
-            self._seen.add(mod_name)
-            self._inspect_module(mod_name)
+
+            self._inspect_module(mod)
+
             if is_pkg:
                 self._walk(mod_name)
 
-    # ------------------------------------------------------------------
-    def _inspect_module(self, mod_name: str) -> None:
-        try:
-            module = importlib.import_module(mod_name)
-        except ImportError as exc:  # pragma: no cover
-            logger.warning("Cannot import module %s: %s", mod_name, exc)
-            return
-
+    # ------------------------------------------------------------------ #
+    def _inspect_module(self, module: ModuleType) -> None:
         for attr in module.__dict__.values():
             if inspect.isclass(attr):
                 self._maybe_register(attr)
 
-    # ------------------------------------------------------------------
+    # ------------------------------------------------------------------ #
     def _maybe_register(self, cls: Type) -> None:
-        """Register *cls* in all relevant plugin categories."""
-
-        # ---------------- parser plugins ------------------------------
-        looks_like_parser = callable(getattr(cls, "try_parse", None))
-        if looks_like_parser and not inspect.isabstract(cls):
-            # skip ABC base itself if available
-            if self.ParserPlugin and cls is self.ParserPlugin:
-                pass
-            else:
-                self.registry.register_plugin("parser", cls.__name__, cls())
-
-        # --------------- execution strategies -------------------------
-        if (
-            issubclass(cls, self.ExecutionStrategy)
-            and cls is not self.ExecutionStrategy
-            and not inspect.isabstract(cls)
-        ):
-            self.registry.register_plugin("execution_strategy", cls.__name__, cls)
-
-        # --------------- explicit @plugin decorator -------------------
-        meta = getattr(cls, "_plugin_meta", None)
-        if meta and not inspect.isabstract(cls):
-            self.registry.register_plugin(meta.get("category", "unknown"), meta.get("name", cls.__name__), cls())
-
-
-# ----------------------------------------------------------------------
-# public decorator helper
-# ----------------------------------------------------------------------
+        """Register *cls* in all matching plugin categories."""
+        if inspect.isabstract(cls):
+            return
 
+        # ------------------- Parser plugins -------------------------
+        if issubclass(cls, ParserPlugin) and cls is not ParserPlugin:
+            if not inspect.iscoroutinefunction(getattr(cls, "try_parse", None)):
+                logger.warning("Skipping parser plugin %s: try_parse is not async", cls.__qualname__)
+            else:
+                try:
+                    self._registry.register_plugin("parser", cls.__name__, cls())
+                except Exception as exc:  # pragma: no cover
+                    logger.warning("Cannot instantiate parser plugin %s: %s", cls.__qualname__, exc)
+
+        # ---------------- Execution strategies ---------------------
+        if issubclass(cls, ExecutionStrategy) and cls is not ExecutionStrategy:
+            self._registry.register_plugin("execution_strategy", cls.__name__, cls)
+
+        # ------------- Explicit @plugin decorator ------------------
+        meta: Optional[dict] = getattr(cls, "_plugin_meta", None)
+        if meta:
+            category = meta.get("category", "unknown")
+            name = meta.get("name", cls.__name__)
+            try:
+                plugin_obj: Any = cls() if callable(getattr(cls, "__init__", None)) else cls
+                self._registry.register_plugin(category, name, plugin_obj)
+            except Exception as exc:  # pragma: no cover
+                logger.warning("Cannot instantiate decorated plugin %s: %s", cls.__qualname__, exc)
+
+
+# -----------------------------------------------------------------------------
+# Decorator helper
+# -----------------------------------------------------------------------------
 def plugin(category: str, name: str | None = None):
-    """Decorator to mark a class as a plugin for explicit registration."""
+    """
+    Decorator that marks a concrete class as a plugin for *category*.
+
+    Example
+    -------
+    ```python
+    @plugin("transport", name="sse")
+    class MySSETransport:
+        ...
+    ```
+    """
 
     def decorator(cls):
         cls._plugin_meta = {"category": category, "name": name or cls.__name__}
@@ -124,15 +167,17 @@ def plugin(category: str, name: str | None = None):
     return decorator
 
 
-# ----------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Singletons & convenience wrappers
-# ----------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 plugin_registry = PluginRegistry()
 
 
 def discover_default_plugins() -> None:
+    """Discover plugins shipped inside *chuk_tool_processor.plugins*."""
     PluginDiscovery(plugin_registry).discover_plugins(["chuk_tool_processor.plugins"])
 
 
 def discover_plugins(package_paths: List[str]) -> None:
+    """Discover plugins from arbitrary external *package_paths*."""
     PluginDiscovery(plugin_registry).discover_plugins(package_paths)

chuk_tool_processor/plugins/parsers/base.py

@@ -1,18 +1,26 @@
 # chuk_tool_processor/parsers/base.py
+"""Async-native parser-plugin base interface."""
+
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from typing import List
+
 from chuk_tool_processor.models.tool_call import ToolCall
 
+__all__ = ["ParserPlugin"]
+
 
 class ParserPlugin(ABC):
     """
-    Minimal interface every parser plug-in must implement.
+    Every parser plugin **must** implement the async ``try_parse`` coroutine.
 
-    The processor will feed the *raw text* (or dict) it receives from upstream
-    into `try_parse`.  If the plugin recognises the format it should return a
-    list of ToolCall objects; otherwise return an empty list.
+    The processor awaits it and expects *a list* of :class:`ToolCall`
+    objects. If the plugin doesn’t recognise the input it should return an
+    empty list.
     """
 
     @abstractmethod
-    def try_parse(self, raw: str) -> List[ToolCall]:
+    async def try_parse(self, raw: str | object) -> List[ToolCall]:  # noqa: D401
+        """Attempt to parse *raw* into one or more :class:`ToolCall` objects."""
         ...

chuk_tool_processor/plugins/parsers/{function_call_tool_plugin.py → function_call_tool.py}

@@ -1,33 +1,49 @@
-# chuk_tool_processor/plugins/function_call_tool_plugin.py
-"""Function-call parser plugin.
-* Accepts dict **or** string input.
-* Coerces non-dict `arguments` to `{}` instead of rejecting.
-* Inherits from ``ParserPlugin`` so discovery categorises it as a *parser*.
-"""
+# chuk_tool_processor/plugins/parsers/function_call_tool.py
+"""Async parser for OpenAI-style single `function_call` objects."""
+
 from __future__ import annotations
 
 import json
 import re
 from typing import Any, Dict, List
+
 from pydantic import ValidationError
 
-# imports
-from .base import ParserPlugin
-from chuk_tool_processor.models.tool_call import ToolCall
 from chuk_tool_processor.logging import get_logger
+from chuk_tool_processor.models.tool_call import ToolCall
+from chuk_tool_processor.plugins.parsers.base import ParserPlugin
+
+__all__ = ["FunctionCallPlugin"]
 
-# logger
-logger = get_logger("chuk_tool_processor.plugins.function_call_tool")
+logger = get_logger(__name__)
 
-# balanced‐brace JSON object regex – one level only (good enough for payloads)
+# One-level balanced JSON object (good enough for embedded argument blocks)
 _JSON_OBJECT = re.compile(r"\{(?:[^{}]|(?:\{[^{}]*\}))*\}")
 
 
+class PluginMeta:
+    """Optional self-description used by the plugin-discovery system (if present)."""
+
+    name: str = "function_call"
+    description: str = (
+        "Parses a single OpenAI-style `function_call` JSON object (including "
+        "strings that embed such an object)."
+    )
+    version: str = "1.0.0"
+    author: str = "chuk_tool_processor"
+
+
 class FunctionCallPlugin(ParserPlugin):
-    """Parse OpenAI-style **single** ``function_call`` objects."""
+    """Parse messages containing a *single* `function_call` entry."""
+
+    # --------------------------------------------------------------------- #
+    # Public API
+    # --------------------------------------------------------------------- #
 
-    def try_parse(self, raw: str | Dict[str, Any]) -> List[ToolCall]:
+    async def try_parse(self, raw: str | Dict[str, Any]) -> List[ToolCall]:
         payload: Dict[str, Any] | None
+
+        # 1️⃣  Primary path ─ whole payload is JSON
         if isinstance(raw, dict):
             payload = raw
         else:
@@ -38,22 +54,24 @@ class FunctionCallPlugin(ParserPlugin):
 
         calls: List[ToolCall] = []
 
-        # primary path -----------------------------------------------------
         if isinstance(payload, dict):
             calls.extend(self._extract_from_payload(payload))
 
-        # fallback – scan raw text for nested JSON blocks ------------------
+        # 2️⃣  Fallback path ─ scan for *nested* JSON objects inside a string
         if not calls and isinstance(raw, str):
-            for m in _JSON_OBJECT.finditer(raw):
+            for match in _JSON_OBJECT.finditer(raw):
                 try:
-                    sub = json.loads(m.group(0))
+                    sub = json.loads(match.group(0))
                 except json.JSONDecodeError:
                     continue
                 calls.extend(self._extract_from_payload(sub))
 
         return calls
 
-    # ------------------------------------------------------------------
+    # ------------------------------------------------------------------ #
+    # Helpers
+    # ------------------------------------------------------------------ #
+
     def _extract_from_payload(self, payload: Dict[str, Any]) -> List[ToolCall]:
         fc = payload.get("function_call")
         if not isinstance(fc, dict):
@@ -62,12 +80,13 @@ class FunctionCallPlugin(ParserPlugin):
         name = fc.get("name")
         args = fc.get("arguments", {})
 
-        # arguments may be JSON‑encoded string or anything else
+        # Arguments may themselves be JSON in *string* form
         if isinstance(args, str):
             try:
                 args = json.loads(args)
             except json.JSONDecodeError:
                 args = {}
+
         if not isinstance(args, dict):
             args = {}
 

chuk_tool_processor/plugins/parsers/json_tool.py

@@ -0,0 +1,50 @@
+# chuk_tool_processor/plugins/parsers/json_tool.py
+"""Async JSON `tool_calls` parser plugin."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, List
+
+from pydantic import ValidationError
+
+from chuk_tool_processor.logging import get_logger
+from chuk_tool_processor.models.tool_call import ToolCall
+from chuk_tool_processor.plugins.parsers.base import ParserPlugin
+
+__all__ = ["JsonToolPlugin"]
+
+logger = get_logger(__name__)
+
+
+class PluginMeta:
+    """Optional self-description consumed by the plugin-discovery subsystem."""
+    name: str = "json_tool_calls"
+    description: str = "Parses a JSON object containing a `tool_calls` array."
+    version: str = "1.0.0"
+    author: str = "chuk_tool_processor"
+
+
+class JsonToolPlugin(ParserPlugin):
+    """Extracts a *list* of :class:`ToolCall` objects from a `tool_calls` array."""
+
+    async def try_parse(self, raw: str | Any) -> List[ToolCall]:  # noqa: D401
+        # Decode JSON if we were given a string
+        try:
+            data = json.loads(raw) if isinstance(raw, str) else raw
+        except json.JSONDecodeError:
+            logger.debug("json_tool: input is not valid JSON")
+            return []
+
+        if not isinstance(data, dict):
+            return []
+
+        calls: List[ToolCall] = []
+        for entry in data.get("tool_calls", []):
+            try:
+                calls.append(ToolCall(**entry))
+            except ValidationError:
+                logger.debug("json_tool: validation error on entry %s", entry)
+                continue
+
+        return calls

chuk_tool_processor/plugins/parsers/openai_tool.py

@@ -0,0 +1,88 @@
+# chuk_tool_processor/plugins/parsers/openai_tool.py
+"""Async parser for OpenAI-style `tool_calls` arrays."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, List
+
+from pydantic import ValidationError
+
+from chuk_tool_processor.logging import get_logger
+from chuk_tool_processor.models.tool_call import ToolCall
+from chuk_tool_processor.plugins.parsers.base import ParserPlugin
+
+__all__ = ["OpenAIToolPlugin"]
+
+logger = get_logger(__name__)
+
+
+class PluginMeta:
+    """Optional descriptor consumed by the plugin-discovery system."""
+    name: str = "openai_tool_calls"
+    description: str = "Parses Chat-Completions responses containing `tool_calls`."
+    version: str = "1.0.0"
+    author: str = "chuk_tool_processor"
+
+
+class OpenAIToolPlugin(ParserPlugin):
+    """
+    Understands structures like::
+
+        {
+            "tool_calls": [
+                {
+                    "id": "call_abc",
+                    "type": "function",
+                    "function": {
+                        "name": "weather",
+                        "arguments": "{\"location\": \"New York\"}"
+                    }
+                }
+            ]
+        }
+    """
+
+    async def try_parse(self, raw: str | Any) -> List[ToolCall]:  # noqa: D401
+        # ------------------------------------------------------------------ #
+        # 1. Decode JSON when the input is a string
+        # ------------------------------------------------------------------ #
+        try:
+            data = json.loads(raw) if isinstance(raw, str) else raw
+        except json.JSONDecodeError:
+            logger.debug("openai_tool_plugin: input is not valid JSON")
+            return []
+
+        if not isinstance(data, dict) or "tool_calls" not in data:
+            return []
+
+        # ------------------------------------------------------------------ #
+        # 2. Build ToolCall objects
+        # ------------------------------------------------------------------ #
+        calls: List[ToolCall] = []
+        for entry in data["tool_calls"]:
+            fn = entry.get("function", {})
+            name = fn.get("name")
+            args = fn.get("arguments", {})
+
+            # Arguments may be double-encoded JSON
+            if isinstance(args, str):
+                try:
+                    args = json.loads(args)
+                except json.JSONDecodeError:
+                    args = {}
+
+            if not isinstance(name, str) or not name:
+                continue
+
+            try:
+                calls.append(
+                    ToolCall(tool=name, arguments=args if isinstance(args, dict) else {})
+                )
+            except ValidationError:
+                logger.debug(
+                    "openai_tool_plugin: validation error while building ToolCall for %s",
+                    name,
+                )
+
+        return calls

chuk_tool_processor/plugins/parsers/xml_tool.py

@@ -1,45 +1,99 @@
-# chuk_tool_processor/plugins/xml_tool.py
-"""XML tool-call parser plugin.
-Understands `<tool name="..." args='{"x":1}'/>` single-line constructs –
-format used by many examples and test-fixtures.
+# chuk_tool_processor/plugins/parsers/xml_tool.py
 """
+Async-native parser for single-line XML-style tool-call tags, e.g.
+
+    <tool name="translate" args="{\"text\": \"Hello\", \"target\": \"es\"}"/>
+
+The *args* attribute may be
+
+1. A proper JSON object:                   args="{"x": 1}"
+2. A JSON-encoded string (most common):    args="{\"x\": 1}"
+3. The empty string:                       args=""
+
+All variants are normalised to a **dict** of arguments.
+"""
+
 from __future__ import annotations
 
 import json
 import re
 from typing import List
+
 from pydantic import ValidationError
 
-# imports
-from .base import ParserPlugin
+from chuk_tool_processor.logging import get_logger
 from chuk_tool_processor.models.tool_call import ToolCall
+from chuk_tool_processor.plugins.parsers.base import ParserPlugin
+
+__all__: list[str] = ["XmlToolPlugin"]
+
+logger = get_logger(__name__)
 
 
+class PluginMeta:
+    """Optional descriptor that can be used by the plugin-discovery mechanism."""
+    name: str = "xml_tool_tag"
+    description: str = "Parses <tool …/> XML tags into ToolCall objects."
+    version: str = "1.0.0"
+    author: str = "chuk_tool_processor"
+
 
 class XmlToolPlugin(ParserPlugin):
-    """Parse XML-like tool-call tags."""
+    """Convert `<tool …/>` tags into :class:`ToolCall` objects."""
 
     _TAG = re.compile(
         r"<tool\s+"
-        r"name=(?P<q1>[\"\'])(?P<tool>.+?)(?P=q1)\s+"
-        r"args=(?P<q2>[\"\'])(?P<args>.*?)(?P=q2)\s*/>"
+        r"name=(?P<q1>[\"'])(?P<tool>.+?)(?P=q1)\s+"
+        r"args=(?P<q2>[\"'])(?P<args>.*?)(?P=q2)\s*/>",
+        flags=re.IGNORECASE | re.DOTALL,
     )
 
-    def try_parse(self, raw):  # type: ignore[override]
-        if not isinstance(raw, str):  # XML form only exists in strings
+    # ------------------------------------------------------------------ #
+    async def try_parse(self, raw: str | object) -> List[ToolCall]:  # noqa: D401
+        if not isinstance(raw, str):
             return []
 
         calls: List[ToolCall] = []
-        for m in self._TAG.finditer(raw):
-            tool_name = m.group("tool")
-            raw_args = m.group("args")
-            try:
-                args = json.loads(raw_args) if raw_args else {}
-            except json.JSONDecodeError:
-                args = {}
+
+        for match in self._TAG.finditer(raw):
+            name = match.group("tool")
+            raw_args = match.group("args") or ""
+            args = self._decode_args(raw_args)
 
             try:
-                calls.append(ToolCall(tool=tool_name, arguments=args))
+                calls.append(ToolCall(tool=name, arguments=args))
             except ValidationError:
-                continue  # skip malformed
+                logger.debug("xml_tool_plugin: validation error for <%s>", name)
+
         return calls
+
+    # ------------------------------------------------------------------ #
+    # Helper – robust JSON decode for the args attribute
+    # ------------------------------------------------------------------ #
+    @staticmethod
+    def _decode_args(raw_args: str) -> dict:
+        """Best-effort decoding of the *args* attribute to a dict."""
+        if not raw_args:
+            return {}
+
+        # 1️⃣ Try direct JSON
+        try:
+            parsed = json.loads(raw_args)
+        except json.JSONDecodeError:
+            parsed = None
+
+        # 2️⃣ If still None, the value might be a JSON-encoded string
+        if parsed is None:
+            try:
+                parsed = json.loads(raw_args.encode().decode("unicode_escape"))
+            except json.JSONDecodeError:
+                parsed = None
+
+        # 3️⃣ Last resort – naive unescaping of \" → "
+        if parsed is None:
+            try:
+                parsed = json.loads(raw_args.replace(r"\"", "\""))
+            except json.JSONDecodeError:
+                parsed = {}
+
+        return parsed if isinstance(parsed, dict) else {}