chuk-tool-processor 0.1.0__py3-none-any.whl

chuk_tool_processor/registry/providers/memory.py

@@ -0,0 +1,165 @@
+# chuk_tool_processor/registry/providers/memory.py
+"""
+In-memory implementation of the tool registry.
+"""
+
+import inspect
+from typing import Any, Dict, List, Optional, Tuple
+
+from chuk_tool_processor.core.exceptions import ToolNotFoundError
+from chuk_tool_processor.registry.interface import ToolRegistryInterface
+from chuk_tool_processor.registry.metadata import ToolMetadata
+
+
+class InMemoryToolRegistry(ToolRegistryInterface):
+    """
+    In-memory implementation of ToolRegistryInterface with namespace support.
+    
+    This implementation stores tools and their metadata in memory,
+    organized by namespace. It's suitable for single-process applications
+    or for testing, but doesn't provide persistence or sharing across
+    multiple processes.
+    """
+    def __init__(self):
+        """Initialize the in-memory registry."""
+        # Store tools as {namespace: {name: tool}}
+        self._tools: Dict[str, Dict[str, Any]] = {}
+        # Store metadata as {namespace: {name: metadata}}
+        self._metadata: Dict[str, Dict[str, ToolMetadata]] = {}
+
+    def register_tool(
+        self, 
+        tool: Any, 
+        name: Optional[str] = None,
+        namespace: str = "default",
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """
+        Register a tool implementation.
+
+        Args:
+            tool: The tool class or instance with an `execute` method.
+            name: Optional explicit name; if omitted, uses tool.__name__.
+            namespace: Namespace for the tool (default: "default").
+            metadata: Optional additional metadata for the tool.
+        """
+        # Ensure the namespace exists
+        if namespace not in self._tools:
+            self._tools[namespace] = {}
+            self._metadata[namespace] = {}
+            
+        # Determine tool name
+        key = name or getattr(tool, "__name__", None) or repr(tool)
+        
+        # Register the tool
+        self._tools[namespace][key] = tool
+        
+        # Create and store metadata
+        is_async = inspect.iscoroutinefunction(getattr(tool, "execute", None))
+        
+        # Get description from docstring if available
+        description = None
+        if hasattr(tool, "__doc__") and tool.__doc__:
+            description = inspect.getdoc(tool)
+        
+        # Create metadata object
+        meta_dict = {
+            "name": key,
+            "namespace": namespace,
+            "is_async": is_async
+        }
+        
+        # Add description if available (but don't override metadata if provided)
+        if description and not (metadata and "description" in metadata):
+            meta_dict["description"] = description
+            
+        # Add any additional metadata
+        if metadata:
+            meta_dict.update(metadata)
+            
+        tool_metadata = ToolMetadata(**meta_dict)
+        
+        self._metadata[namespace][key] = tool_metadata
+
+    def get_tool(self, name: str, namespace: str = "default") -> Optional[Any]:
+        """
+        Retrieve a registered tool by name and namespace.
+        
+        Args:
+            name: The name of the tool.
+            namespace: The namespace of the tool (default: "default").
+            
+        Returns:
+            The tool implementation or None if not found.
+        """
+        if namespace not in self._tools:
+            return None
+        return self._tools[namespace].get(name)
+
+    def get_tool_strict(self, name: str, namespace: str = "default") -> Any:
+        """
+        Retrieve a registered tool by name and namespace, raising an exception if not found.
+        
+        Args:
+            name: The name of the tool.
+            namespace: The namespace of the tool (default: "default").
+            
+        Returns:
+            The tool implementation.
+            
+        Raises:
+            ToolNotFoundError: If the tool is not found.
+        """
+        tool = self.get_tool(name, namespace)
+        if tool is None:
+            raise ToolNotFoundError(f"{namespace}.{name}")
+        return tool
+
+    def get_metadata(self, name: str, namespace: str = "default") -> Optional[ToolMetadata]:
+        """
+        Retrieve metadata for a registered tool.
+        
+        Args:
+            name: The name of the tool.
+            namespace: The namespace of the tool (default: "default").
+            
+        Returns:
+            ToolMetadata if found, None otherwise.
+        """
+        if namespace not in self._metadata:
+            return None
+        return self._metadata[namespace].get(name)
+
+    def list_tools(self, namespace: Optional[str] = None) -> List[Tuple[str, str]]:
+        """
+        List all registered tool names, optionally filtered by namespace.
+        
+        Args:
+            namespace: Optional namespace filter.
+            
+        Returns:
+            List of (namespace, name) tuples.
+        """
+        result = []
+        
+        if namespace:
+            # List tools in specific namespace
+            if namespace in self._tools:
+                for name in self._tools[namespace].keys():
+                    result.append((namespace, name))
+        else:
+            # List all tools
+            for ns, tools in self._tools.items():
+                for name in tools.keys():
+                    result.append((ns, name))
+                    
+        return result
+
+    def list_namespaces(self) -> List[str]:
+        """
+        List all registered namespaces.
+        
+        Returns:
+            List of namespace names.
+        """
+        return list(self._tools.keys())

chuk_tool_processor/utils/logging.py

@@ -0,0 +1,260 @@
+# chuk_tool_processor/logging.py
+import json
+import logging
+import sys
+import time
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, Union
+
+# Configure the root logger
+root_logger = logging.getLogger("chuk_tool_processor")
+root_logger.setLevel(logging.INFO)
+
+# Create a handler for stderr
+handler = logging.StreamHandler(sys.stderr)
+handler.setLevel(logging.INFO)
+
+# Create a formatter for structured logging
+class StructuredFormatter(logging.Formatter):
+    """
+    Custom formatter for structured JSON logging.
+    """
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        Format log record as JSON.
+        """
+        # Basic log data
+        log_data = {
+            "timestamp": datetime.fromtimestamp(record.created, timezone.utc)
+                              .isoformat().replace("+00:00", "Z"),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "logger": record.name,
+            "pid": record.process,
+            "thread": record.thread,
+            "file": record.filename,
+            "line": record.lineno,
+            "function": record.funcName,
+        }
+        
+        # Add traceback if present
+        if record.exc_info:
+            log_data["traceback"] = self.formatException(record.exc_info)
+        
+        # Add extra fields
+        if hasattr(record, "extra"):
+            log_data.update(record.extra)
+        
+        # Add structured logging context if present
+        if hasattr(record, "context"):
+            log_data["context"] = record.context
+        
+        return json.dumps(log_data)
+
+
+# Configure the formatter
+formatter = StructuredFormatter()
+handler.setFormatter(formatter)
+
+# Add the handler to the root logger
+root_logger.addHandler(handler)
+
+# Thread-local context storage
+class LogContext:
+    """
+    Thread-local storage for log context.
+    """
+    def __init__(self):
+        self.context = {}
+        self.request_id = None
+    
+    def set(self, key: str, value: Any) -> None:
+        self.context[key] = value
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        return self.context.get(key, default)
+    
+    def update(self, values: Dict[str, Any]) -> None:
+        self.context.update(values)
+    
+    def clear(self) -> None:
+        self.context = {}
+        self.request_id = None
+    
+    def start_request(self, request_id: Optional[str] = None) -> str:
+        self.request_id = request_id or str(uuid.uuid4())
+        self.context["request_id"] = self.request_id
+        return self.request_id
+    
+    def end_request(self) -> None:
+        self.clear()
+
+
+# Create global log context
+log_context = LogContext()
+
+
+class StructuredAdapter(logging.LoggerAdapter):
+    """
+    Adapter to add structured context to log messages.
+    """
+    def process(self, msg: str, kwargs: Dict[str, Any]) -> tuple:
+        kwargs = kwargs.copy() if kwargs else {}
+        extra = kwargs.get("extra", {})
+        if log_context.context:
+            context_copy = log_context.context.copy()
+            if "context" in extra:
+                extra["context"].update(context_copy)
+            else:
+                extra["context"] = context_copy
+        kwargs["extra"] = extra
+        return msg, kwargs
+
+
+def get_logger(name: str) -> StructuredAdapter:
+    logger = logging.getLogger(name)
+    return StructuredAdapter(logger, {})
+
+
+@contextmanager
+def log_context_span(
+    operation: str,
+    extra: Optional[Dict[str, Any]] = None,
+    log_duration: bool = True,
+    level: int = logging.INFO
+):
+    logger = get_logger(f"chuk_tool_processor.span.{operation}")
+    start_time = time.time()
+    span_id = str(uuid.uuid4())
+    span_context = {
+        "span_id": span_id,
+        "operation": operation,
+        "start_time": datetime.fromtimestamp(start_time, timezone.utc)
+                            .isoformat().replace("+00:00", "Z"),
+    }
+    if extra:
+        span_context.update(extra)
+    previous_context = log_context.context.copy() if log_context.context else {}
+    log_context.update(span_context)
+    logger.log(level, f"Starting {operation}")
+    try:
+        yield
+        if log_duration:
+            duration = time.time() - start_time
+            logger.log(level, f"Completed {operation}", extra={"context": {"duration": duration}})
+        else:
+            logger.log(level, f"Completed {operation}")
+    except Exception as e:
+        duration = time.time() - start_time
+        logger.exception(
+            f"Error in {operation}: {str(e)}",
+            extra={"context": {"duration": duration, "error": str(e)}}
+        )
+        raise
+    finally:
+        log_context.clear()
+        if previous_context:
+            log_context.update(previous_context)
+
+
+@contextmanager
+def request_logging(request_id: Optional[str] = None):
+    logger = get_logger("chuk_tool_processor.request")
+    request_id = log_context.start_request(request_id)
+    start_time = time.time()
+    logger.info(f"Starting request {request_id}")
+    try:
+        yield request_id
+        duration = time.time() - start_time
+        logger.info(
+            f"Completed request {request_id}",
+            extra={"context": {"duration": duration}}
+        )
+    except Exception as e:
+        duration = time.time() - start_time
+        logger.exception(
+            f"Error in request {request_id}: {str(e)}",
+            extra={"context": {"duration": duration, "error": str(e)}}
+        )
+        raise
+    finally:
+        log_context.end_request()
+
+
+def log_tool_call(tool_call, tool_result):
+    logger = get_logger("chuk_tool_processor.tool_call")
+    duration = (tool_result.end_time - tool_result.start_time).total_seconds()
+    context = {
+        "tool": tool_call.tool,
+        "arguments": tool_call.arguments,
+        "result": tool_result.result,
+        "error": tool_result.error,
+        "duration": duration,
+        "machine": tool_result.machine,
+        "pid": tool_result.pid,
+    }
+    if hasattr(tool_result, "cached") and tool_result.cached:
+        context["cached"] = True
+    if hasattr(tool_result, "attempts") and tool_result.attempts:
+        context["attempts"] = tool_result.attempts
+    if tool_result.error:
+        logger.error(
+            f"Tool {tool_call.tool} failed: {tool_result.error}",
+            extra={"context": context}
+        )
+    else:
+        logger.info(
+            f"Tool {tool_call.tool} succeeded in {duration:.3f}s",
+            extra={"context": context}
+        )
+
+
+class MetricsLogger:
+    def __init__(self):
+        self.logger = get_logger("chuk_tool_processor.metrics")
+    def log_tool_execution(
+        self,
+        tool: str,
+        success: bool,
+        duration: float,
+        error: Optional[str] = None,
+        cached: bool = False,
+        attempts: int = 1
+    ):
+        self.logger.info(
+            f"Tool execution metric: {tool}",
+            extra={
+                "context": {
+                    "metric_type": "tool_execution",
+                    "tool": tool,
+                    "success": success,
+                    "duration": duration,
+                    "error": error,
+                    "cached": cached,
+                    "attempts": attempts,
+                }
+            }
+        )
+    def log_parser_metric(
+        self,
+        parser: str,
+        success: bool,
+        duration: float,
+        num_calls: int
+    ):
+        self.logger.info(
+            f"Parser metric: {parser}",
+            extra={
+                "context": {
+                    "metric_type": "parser",
+                    "parser": parser,
+                    "success": success,
+                    "duration": duration,
+                    "num_calls": num_calls,
+                }
+            }
+        )
+
+metrics = MetricsLogger()

chuk_tool_processor/utils/validation.py

@@ -0,0 +1,192 @@
+# chuk_tool_processor/utils/validation.py
+from typing import Any, Dict, Optional, Type, get_type_hints, Union, List, Callable
+from pydantic import BaseModel, ValidationError, create_model
+import inspect
+from functools import wraps
+
+from chuk_tool_processor.core.exceptions import ToolValidationError
+
+
+def validate_arguments(tool_name: str, tool_func: Callable, args: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Validate tool arguments against function signature.
+    
+    Args:
+        tool_name: Name of the tool for error reporting.
+        tool_func: Tool function to validate against.
+        args: Arguments to validate.
+        
+    Returns:
+        Validated arguments dict.
+        
+    Raises:
+        ToolValidationError: If validation fails.
+    """
+    try:
+        # Get type hints from function
+        type_hints = get_type_hints(tool_func)
+        
+        # Remove return type hint if present
+        if 'return' in type_hints:
+            type_hints.pop('return')
+        
+        # Create dynamic Pydantic model for validation
+        field_definitions = {
+            name: (type_hint, ...) for name, type_hint in type_hints.items()
+        }
+        
+        # Add optional fields based on default values
+        sig = inspect.signature(tool_func)
+        for param_name, param in sig.parameters.items():
+            if param.default is not inspect.Parameter.empty:
+                if param_name in field_definitions:
+                    field_type, _ = field_definitions[param_name]
+                    field_definitions[param_name] = (field_type, param.default)
+        
+        # Create model
+        model = create_model(f"{tool_name}Args", **field_definitions)
+        
+        # Validate args
+        validated = model(**args)
+        return validated.dict()
+    
+    except ValidationError as e:
+        raise ToolValidationError(tool_name, e.errors())
+    except Exception as e:
+        raise ToolValidationError(tool_name, {"general": str(e)})
+
+
+def validate_result(tool_name: str, tool_func: Callable, result: Any) -> Any:
+    """
+    Validate tool result against function return type.
+    
+    Args:
+        tool_name: Name of the tool for error reporting.
+        tool_func: Tool function to validate against.
+        result: Result to validate.
+        
+    Returns:
+        Validated result.
+        
+    Raises:
+        ToolValidationError: If validation fails.
+    """
+    try:
+        # Get return type hint
+        type_hints = get_type_hints(tool_func)
+        return_type = type_hints.get('return')
+        
+        if return_type is None:
+            # No return type to validate against
+            return result
+        
+        # Create dynamic Pydantic model for validation
+        model = create_model(
+            f"{tool_name}Result",
+            result=(return_type, ...)
+        )
+        
+        # Validate result
+        validated = model(result=result)
+        return validated.result
+    
+    except ValidationError as e:
+        raise ToolValidationError(tool_name, e.errors())
+    except Exception as e:
+        raise ToolValidationError(tool_name, {"general": str(e)})
+
+
+def with_validation(cls):
+    """
+    Class decorator to add type validation to tool classes.
+    
+    Example:
+        @with_validation
+        class MyTool:
+            def execute(self, x: int, y: str) -> float:
+                return float(x) + float(y)
+    """
+    original_execute = cls.execute
+    
+    @wraps(original_execute)
+    def execute_with_validation(self, **kwargs):
+        # Get tool name
+        tool_name = getattr(cls, "__name__", repr(cls))
+        
+        # Validate arguments
+        validated_args = validate_arguments(tool_name, original_execute, kwargs)
+        
+        # Execute the tool
+        result = original_execute(self, **validated_args)
+        
+        # Validate result
+        return validate_result(tool_name, original_execute, result)
+    
+    cls.execute = execute_with_validation
+    return cls
+
+
+class ValidatedTool(BaseModel):
+    """
+    Base class for tools with built-in validation.
+    
+    Example:
+        class AddTool(ValidatedTool):
+            class Arguments(BaseModel):
+                x: int
+                y: int
+                
+            class Result(BaseModel):
+                sum: int
+                
+            def execute(self, x: int, y: int) -> Result:
+                return self.Result(sum=x + y)
+    """
+    class Arguments(BaseModel):
+        """Base arguments model to be overridden by subclasses."""
+        pass
+    
+    class Result(BaseModel):
+        """Base result model to be overridden by subclasses."""
+        pass
+    
+    def execute(self, **kwargs) -> Any:
+        """
+        Execute the tool with validated arguments.
+        
+        Args:
+            **kwargs: Arguments to validate against Arguments model.
+            
+        Returns:
+            Validated result according to Result model.
+            
+        Raises:
+            ToolValidationError: If validation fails.
+        """
+        try:
+            # Validate arguments
+            validated_args = self.Arguments(**kwargs)
+            
+            # Execute implementation
+            result = self._execute(**validated_args.dict())
+            
+            # Validate result if it's not already a Result instance
+            if not isinstance(result, self.Result):
+                result = self.Result(**result if isinstance(result, dict) else {"value": result})
+                
+            return result
+            
+        except ValidationError as e:
+            raise ToolValidationError(self.__class__.__name__, e.errors())
+    
+    def _execute(self, **kwargs) -> Any:
+        """
+        Implementation method to be overridden by subclasses.
+        
+        Args:
+            **kwargs: Validated arguments.
+            
+        Returns:
+            Result that will be validated against Result model.
+        """
+        raise NotImplementedError("Subclasses must implement _execute")