chuk-tool-processor 0.6.4__py3-none-any.whl → 0.9.7__py3-none-any.whl

chuk_tool_processor/execution/wrappers/retry.py

@@ -6,13 +6,14 @@ Adds exponential-back-off retry logic and *deadline-aware* timeout handling so a
 `timeout=` passed by callers is treated as the **total wall-clock budget** for
 all attempts of a single tool call.
 """
+
 from __future__ import annotations
 
 import asyncio
 import random
 import time
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Type
+from datetime import UTC, datetime
+from typing import Any
 
 from chuk_tool_processor.logging import get_logger
 from chuk_tool_processor.models.tool_call import ToolCall
@@ -20,6 +21,24 @@ from chuk_tool_processor.models.tool_result import ToolResult
 
 logger = get_logger("chuk_tool_processor.execution.wrappers.retry")
 
+# Optional observability imports
+try:
+    from chuk_tool_processor.observability.metrics import get_metrics
+    from chuk_tool_processor.observability.tracing import trace_retry_attempt
+
+    _observability_available = True
+except ImportError:
+    _observability_available = False
+
+    # No-op functions when observability not available
+    def get_metrics():
+        return None
+
+    def trace_retry_attempt(*_args, **_kwargs):
+        from contextlib import nullcontext
+
+        return nullcontext()
+
 
 # --------------------------------------------------------------------------- #
 # Retry configuration
@@ -33,8 +52,9 @@ class RetryConfig:
         base_delay: float = 1.0,
         max_delay: float = 60.0,
         jitter: bool = True,
-        retry_on_exceptions: Optional[List[Type[Exception]]] = None,
-        retry_on_error_substrings: Optional[List[str]] = None,
+        retry_on_exceptions: list[type[Exception]] | None = None,
+        retry_on_error_substrings: list[str] | None = None,
+        skip_retry_on_error_substrings: list[str] | None = None,
     ):
         if max_retries < 0:
             raise ValueError("max_retries cannot be negative")
@@ -44,6 +64,7 @@ class RetryConfig:
         self.jitter = jitter
         self.retry_on_exceptions = retry_on_exceptions or []
         self.retry_on_error_substrings = retry_on_error_substrings or []
+        self.skip_retry_on_error_substrings = skip_retry_on_error_substrings or []
 
     # --------------------------------------------------------------------- #
     # Decision helpers
@@ -52,13 +73,21 @@ class RetryConfig:
         self,
         attempt: int,
         *,
-        error: Optional[Exception] = None,
-        error_str: Optional[str] = None,
+        error: Exception | None = None,
+        error_str: str | None = None,
     ) -> bool:
         """Return *True* iff another retry is allowed for this attempt."""
         if attempt >= self.max_retries:
             return False
 
+        # Check skip list first - these errors should never be retried
+        # (e.g., OAuth errors that need to be handled at transport layer)
+        if error_str and self.skip_retry_on_error_substrings:
+            error_lower = error_str.lower()
+            if any(skip_pattern.lower() in error_lower for skip_pattern in self.skip_retry_on_error_substrings):
+                logger.debug(f"Skipping retry for error matching skip pattern: {error_str[:100]}")
+                return False
+
         # Nothing specified → always retry until max_retries reached
         if not self.retry_on_exceptions and not self.retry_on_error_substrings:
             return True
@@ -66,17 +95,14 @@ class RetryConfig:
         if error is not None and any(isinstance(error, exc) for exc in self.retry_on_exceptions):
             return True
 
-        if error_str and any(substr in error_str for substr in self.retry_on_error_substrings):
-            return True
-
-        return False
+        return bool(error_str and any(substr in error_str for substr in self.retry_on_error_substrings))
 
     # --------------------------------------------------------------------- #
     # Back-off
     # --------------------------------------------------------------------- #
     def get_delay(self, attempt: int) -> float:
         """Exponential back-off delay for *attempt* (0-based)."""
-        delay = min(self.base_delay * (2 ** attempt), self.max_delay)
+        delay = min(self.base_delay * (2**attempt), self.max_delay)
         if self.jitter:
             delay *= 0.5 + random.random()  # jitter in [0.5, 1.5)
         return delay
@@ -94,8 +120,8 @@ class RetryableToolExecutor:
         self,
         executor: Any,
         *,
-        default_config: Optional[RetryConfig] = None,
-        tool_configs: Optional[Dict[str, RetryConfig]] = None,
+        default_config: RetryConfig | None = None,
+        tool_configs: dict[str, RetryConfig] | None = None,
     ):
         self.executor = executor
         self.default_config = default_config or RetryConfig()
@@ -109,15 +135,15 @@ class RetryableToolExecutor:
 
     async def execute(
         self,
-        calls: List[ToolCall],
+        calls: list[ToolCall],
         *,
-        timeout: Optional[float] = None,
+        timeout: float | None = None,
         use_cache: bool = True,
-    ) -> List[ToolResult]:
+    ) -> list[ToolResult]:
         if not calls:
             return []
 
-        out: List[ToolResult] = []
+        out: list[ToolResult] = []
         for call in calls:
             cfg = self._config_for(call.tool)
             out.append(await self._execute_single(call, cfg, timeout, use_cache))
@@ -130,11 +156,11 @@ class RetryableToolExecutor:
         self,
         call: ToolCall,
         cfg: RetryConfig,
-        timeout: Optional[float],
+        timeout: float | None,
         use_cache: bool,
     ) -> ToolResult:
         attempt = 0
-        last_error: Optional[str] = None
+        last_error: str | None = None
         pid = 0
         machine = "unknown"
 
@@ -156,8 +182,8 @@ class RetryableToolExecutor:
                         tool=call.tool,
                         result=None,
                         error=f"Timeout after {timeout}s",
-                        start_time=datetime.now(timezone.utc),
-                        end_time=datetime.now(timezone.utc),
+                        start_time=datetime.now(UTC),
+                        end_time=datetime.now(UTC),
                         machine=machine,
                         pid=pid,
                         attempts=attempt,
@@ -168,64 +194,74 @@ class RetryableToolExecutor:
             # ---------------------------------------------------------------- #
             # Execute one attempt
             # ---------------------------------------------------------------- #
-            start_time = datetime.now(timezone.utc)
-            try:
-                kwargs = {"timeout": remaining} if remaining is not None else {}
-                if hasattr(self.executor, "use_cache"):
-                    kwargs["use_cache"] = use_cache
-
-                result = (await self.executor.execute([call], **kwargs))[0]
-                pid = result.pid
-                machine = result.machine
-
-                # Success?
-                if not result.error:
+            start_time = datetime.now(UTC)
+
+            # Trace retry attempt
+            with trace_retry_attempt(call.tool, attempt, cfg.max_retries):
+                try:
+                    kwargs = {"timeout": remaining} if remaining is not None else {}
+                    if hasattr(self.executor, "use_cache"):
+                        kwargs["use_cache"] = use_cache
+
+                    result = (await self.executor.execute([call], **kwargs))[0]
+                    pid = result.pid
+                    machine = result.machine
+
+                    # Record retry metrics
+                    metrics = get_metrics()
+                    success = result.error is None
+
+                    if metrics:
+                        metrics.record_retry_attempt(call.tool, attempt, success)
+
+                    # Success?
+                    if success:
+                        result.attempts = attempt + 1
+                        return result
+
+                    # Error: decide on retry
+                    last_error = result.error
+                    if cfg.should_retry(attempt, error_str=result.error):
+                        delay = cfg.get_delay(attempt)
+                        # never overshoot the deadline
+                        if deadline is not None:
+                            delay = min(delay, max(deadline - time.monotonic(), 0))
+                        if delay:
+                            await asyncio.sleep(delay)
+                        attempt += 1
+                        continue
+
+                    # No more retries wanted
+                    result.error = self._wrap_error(last_error, attempt, cfg)
                     result.attempts = attempt + 1
                     return result
 
-                # Error: decide on retry
-                last_error = result.error
-                if cfg.should_retry(attempt, error_str=result.error):
-                    delay = cfg.get_delay(attempt)
-                    # never overshoot the deadline
-                    if deadline is not None:
-                        delay = min(delay, max(deadline - time.monotonic(), 0))
-                    if delay:
-                        await asyncio.sleep(delay)
-                    attempt += 1
-                    continue
-
-                # No more retries wanted
-                result.error = self._wrap_error(last_error, attempt, cfg)
-                result.attempts = attempt + 1
-                return result
-
-            # ---------------------------------------------------------------- #
-            # Exception path
-            # ---------------------------------------------------------------- #
-            except Exception as exc:  # noqa: BLE001
-                err_str = str(exc)
-                last_error = err_str
-                if cfg.should_retry(attempt, error=exc):
-                    delay = cfg.get_delay(attempt)
-                    if deadline is not None:
-                        delay = min(delay, max(deadline - time.monotonic(), 0))
-                    if delay:
-                        await asyncio.sleep(delay)
-                    attempt += 1
-                    continue
-
-                end_time = datetime.now(timezone.utc)
-                return ToolResult(
-                    tool=call.tool,
-                    result=None,
-                    error=self._wrap_error(err_str, attempt, cfg),
-                    start_time=start_time,
-                    end_time=end_time,
-                    machine=machine,
-                    pid=pid,
-                    attempts=attempt + 1,
-                )
+                # ---------------------------------------------------------------- #
+                # Exception path
+                # ---------------------------------------------------------------- #
+                except Exception as exc:  # noqa: BLE001
+                    err_str = str(exc)
+                    last_error = err_str
+                    if cfg.should_retry(attempt, error=exc, error_str=err_str):
+                        delay = cfg.get_delay(attempt)
+                        if deadline is not None:
+                            delay = min(delay, max(deadline - time.monotonic(), 0))
+                        if delay:
+                            await asyncio.sleep(delay)
+                        attempt += 1
+                        continue
+
+                    end_time = datetime.now(UTC)
+                    return ToolResult(
+                        tool=call.tool,
+                        result=None,
+                        error=self._wrap_error(err_str, attempt, cfg),
+                        start_time=start_time,
+                        end_time=end_time,
+                        machine=machine,
+                        pid=pid,
+                        attempts=attempt + 1,
+                    )
 
     # --------------------------------------------------------------------- #
     # Helpers
@@ -246,8 +282,9 @@ def retryable(
     base_delay: float = 1.0,
     max_delay: float = 60.0,
     jitter: bool = True,
-    retry_on_exceptions: Optional[List[Type[Exception]]] = None,
-    retry_on_error_substrings: Optional[List[str]] = None,
+    retry_on_exceptions: list[type[Exception]] | None = None,
+    retry_on_error_substrings: list[str] | None = None,
+    skip_retry_on_error_substrings: list[str] | None = None,
 ):
     """
     Class decorator that attaches a :class:`RetryConfig` to a *tool* class.
@@ -269,6 +306,7 @@ def retryable(
             jitter=jitter,
             retry_on_exceptions=retry_on_exceptions,
             retry_on_error_substrings=retry_on_error_substrings,
+            skip_retry_on_error_substrings=skip_retry_on_error_substrings,
         )
         return cls
 

chuk_tool_processor/logging/__init__.py

@@ -11,40 +11,44 @@ Key components:
 - Metrics collection for tools and parsers
 - Async-friendly context managers for spans and requests
 """
+
 from __future__ import annotations
 
 import logging
 import sys
 
+
 # Auto-initialize shutdown error suppression when logging package is imported
 def _initialize_shutdown_fixes():
     """Initialize shutdown error suppression when the package is imported."""
     try:
         from .context import _setup_shutdown_error_suppression
+
         _setup_shutdown_error_suppression()
     except ImportError:
         pass
 
+
 # Initialize when package is imported
 _initialize_shutdown_fixes()
 
 # Import internal modules in correct order to avoid circular imports
 # First, formatter has no internal dependencies
-from .formatter import StructuredFormatter
-
 # Second, context only depends on formatter
-from .context import LogContext, log_context, StructuredAdapter, get_logger
+from .context import LogContext, StructuredAdapter, get_logger, log_context  # noqa: E402
+from .formatter import StructuredFormatter  # noqa: E402
 
 # Third, helpers depend on context
-from .helpers import log_context_span, request_logging, log_tool_call
+from .helpers import log_context_span, log_tool_call, request_logging  # noqa: E402
 
 # Fourth, metrics depend on helpers and context
-from .metrics import metrics, MetricsLogger
+from .metrics import MetricsLogger, metrics  # noqa: E402
 
 __all__ = [
     "get_logger",
     "log_context",
     "LogContext",
+    "StructuredAdapter",
     "log_context_span",
     "request_logging",
     "log_tool_call",
@@ -53,6 +57,7 @@ __all__ = [
     "setup_logging",
 ]
 
+
 # --------------------------------------------------------------------------- #
 # Setup function for configuring logging
 # --------------------------------------------------------------------------- #
@@ -63,7 +68,7 @@ async def setup_logging(
 ) -> None:
     """
     Set up the logging system.
-    
+
     Args:
         level: Logging level (default: INFO)
         structured: Whether to use structured JSON logging
@@ -72,39 +77,40 @@ async def setup_logging(
     # Get the root logger
     root_logger = logging.getLogger("chuk_tool_processor")
     root_logger.setLevel(level)
-    
+
     # Create formatter
-    formatter = StructuredFormatter() if structured else logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    formatter = (
+        StructuredFormatter()
+        if structured
+        else logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     )
-    
+
     # Always add a dummy handler and remove it to satisfy test expectations
     dummy_handler = logging.StreamHandler()
     root_logger.addHandler(dummy_handler)
     root_logger.removeHandler(dummy_handler)
-    
+
     # Now clear any remaining handlers
     for handler in list(root_logger.handlers):
         root_logger.removeHandler(handler)
-    
+
     # Add console handler
     console_handler = logging.StreamHandler(sys.stderr)
     console_handler.setLevel(level)
     console_handler.setFormatter(formatter)
     root_logger.addHandler(console_handler)
-    
+
     # Add file handler if specified
     if log_file:
         file_handler = logging.FileHandler(log_file)
         file_handler.setLevel(level)
         file_handler.setFormatter(formatter)
         root_logger.addHandler(file_handler)
-    
+
     # Log startup with internal logger
     internal_logger = logging.getLogger("chuk_tool_processor.logging")
     internal_logger.info(
-        "Logging initialized",
-        extra={"context": {"level": logging.getLevelName(level), "structured": structured}}
+        "Logging initialized", extra={"context": {"level": logging.getLevelName(level), "structured": structured}}
     )
 
 
@@ -115,4 +121,4 @@ root_logger.setLevel(logging.INFO)
 _handler = logging.StreamHandler(sys.stderr)
 _handler.setLevel(logging.INFO)
 _handler.setFormatter(StructuredFormatter())
-root_logger.addHandler(_handler)
+root_logger.addHandler(_handler)

chuk_tool_processor/logging/context.py

@@ -15,34 +15,31 @@ This module provides:
 from __future__ import annotations
 
 import asyncio
+import atexit
+import contextlib
 import contextvars
 import logging
+import threading
 import uuid
 import warnings
-import threading
-import atexit
-from typing import (
-    Any,
-    AsyncContextManager,
-    AsyncGenerator,
-    Dict,
-    Optional,
-)
+from collections.abc import AsyncGenerator
+from typing import Any
 
 __all__ = ["LogContext", "log_context", "StructuredAdapter", "get_logger"]
 
+
 # --------------------------------------------------------------------------- #
 # Production-quality shutdown error handling
 # --------------------------------------------------------------------------- #
 class LibraryShutdownFilter(logging.Filter):
     """
     Production filter for suppressing known harmless shutdown messages.
-    
+
     This filter ensures clean library shutdown by suppressing specific
     error messages that occur during normal asyncio/anyio cleanup and
     do not indicate actual problems.
     """
-    
+
     # Known harmless shutdown patterns
     SUPPRESSED_PATTERNS = [
         # Primary anyio error that this fixes
@@ -55,55 +52,56 @@ class LibraryShutdownFilter(logging.Filter):
         ("WARNING", "task was destroyed but it is pending"),
         ("ERROR", "event loop is closed"),
     ]
-    
+
     def filter(self, record: logging.LogRecord) -> bool:
         """Filter out known harmless shutdown messages."""
         message = record.getMessage().lower()
         level = record.levelname
-        
+
         for pattern_level, *pattern_phrases in self.SUPPRESSED_PATTERNS:
             if level == pattern_level and all(phrase.lower() in message for phrase in pattern_phrases):
                 return False
-        
+
         return True
 
+
 class LibraryLoggingManager:
     """
     Clean manager for library-wide logging concerns.
-    
+
     Handles initialization and configuration of logging behavior
     in a centralized, maintainable way.
     """
-    
+
     def __init__(self):
         self._initialized = False
         self._lock = threading.Lock()
-    
+
     def initialize(self):
         """Initialize clean shutdown behavior for the library."""
         if self._initialized:
             return
-            
+
         with self._lock:
             if self._initialized:
                 return
-            
+
             self._setup_shutdown_handling()
             self._setup_warning_filters()
             self._initialized = True
-    
+
     def _setup_shutdown_handling(self):
         """Set up clean shutdown message handling."""
         root_logger = logging.getLogger()
-        
+
         # Check if our filter is already present
         for existing_filter in root_logger.filters:
             if isinstance(existing_filter, LibraryShutdownFilter):
                 return
-        
+
         # Add our production-quality filter
         root_logger.addFilter(LibraryShutdownFilter())
-    
+
     def _setup_warning_filters(self):
         """Set up Python warnings filters for clean shutdown."""
         # Suppress specific asyncio/anyio warnings during shutdown
@@ -112,11 +110,12 @@ class LibraryLoggingManager:
             ".*coroutine was never awaited.*",
             ".*Task was destroyed but it is pending.*",
         ]
-        
+
         for pattern in warning_patterns:
             warnings.filterwarnings("ignore", message=pattern, category=RuntimeWarning)
             warnings.filterwarnings("ignore", message=pattern, category=ResourceWarning)
 
+
 # Global manager instance
 _logging_manager = LibraryLoggingManager()
 
@@ -130,15 +129,13 @@ atexit.register(lambda: None)
 # Per-task context storage
 # --------------------------------------------------------------------------- #
 
-_context_var: contextvars.ContextVar[Dict[str, Any]] = contextvars.ContextVar(
-    "log_context", default={}
-)
+_context_var: contextvars.ContextVar[dict[str, Any] | None] = contextvars.ContextVar("log_context", default=None)
 
 
 # --------------------------------------------------------------------------- #
 # Helpers for turning async generators into async context managers
 # --------------------------------------------------------------------------- #
-class AsyncContextManagerWrapper(AsyncContextManager):
+class AsyncContextManagerWrapper(contextlib.AbstractAsyncContextManager):
     """Wrap an async generator so it can be used with `async with`."""
 
     def __init__(self, gen: AsyncGenerator[Any, None]):
@@ -189,17 +186,18 @@ class LogContext:
     # Public API
     # ------------------------------------------------------------------ #
     @property
-    def context(self) -> Dict[str, Any]:
+    def context(self) -> dict[str, Any]:
         """Return the current context dict (task-local)."""
-        return _context_var.get()
+        ctx = _context_var.get()
+        return ctx if ctx is not None else {}
 
     @property
-    def request_id(self) -> Optional[str]:
+    def request_id(self) -> str | None:
         """Convenience accessor for the current request ID (if any)."""
         return self.context.get("request_id")
 
     # -- simple helpers ------------------------------------------------- #
-    def update(self, kv: Dict[str, Any]) -> None:
+    def update(self, kv: dict[str, Any]) -> None:
         """Merge *kv* into the current context."""
         ctx = self.context.copy()
         ctx.update(kv)
@@ -209,12 +207,12 @@ class LogContext:
         """Drop **all** contextual data."""
         _context_var.set({})
 
-    def get_copy(self) -> Dict[str, Any]:
+    def get_copy(self) -> dict[str, Any]:
         """Return a **copy** of the current context."""
         return self.context.copy()
 
     # -- request helpers ------------------------------------------------ #
-    def start_request(self, request_id: Optional[str] = None) -> str:
+    def start_request(self, request_id: str | None = None) -> str:
         """
         Start a new *request* scope.
 
@@ -233,9 +231,7 @@ class LogContext:
     # ------------------------------------------------------------------ #
     # Async context helpers
     # ------------------------------------------------------------------ #
-    async def _context_scope_gen(
-        self, **kwargs: Any
-    ) -> AsyncGenerator[Dict[str, Any], None]:
+    async def _context_scope_gen(self, **kwargs: Any) -> AsyncGenerator[dict[str, Any], None]:
         prev_ctx = self.get_copy()
         try:
             self.update(kwargs)
@@ -243,7 +239,7 @@ class LogContext:
         finally:
             _context_var.set(prev_ctx)
 
-    def context_scope(self, **kwargs: Any) -> AsyncContextManager:
+    def context_scope(self, **kwargs: Any) -> contextlib.AbstractAsyncContextManager:
         """
         Temporarily add *kwargs* to the context.
 
@@ -256,9 +252,7 @@ class LogContext:
         """
         return AsyncContextManagerWrapper(self._context_scope_gen(**kwargs))
 
-    async def _request_scope_gen(
-        self, request_id: Optional[str] = None
-    ) -> AsyncGenerator[str, None]:
+    async def _request_scope_gen(self, request_id: str | None = None) -> AsyncGenerator[str, None]:
         prev_ctx = self.get_copy()
         try:
             rid = self.start_request(request_id)
@@ -267,7 +261,7 @@ class LogContext:
         finally:
             _context_var.set(prev_ctx)
 
-    def request_scope(self, request_id: Optional[str] = None) -> AsyncContextManager:
+    def request_scope(self, request_id: str | None = None) -> contextlib.AbstractAsyncContextManager:
         """
         Manage a full request lifecycle::
 
@@ -280,6 +274,7 @@ class LogContext:
 # A convenient global instance that most code can just import and use.
 log_context = LogContext()
 
+
 # --------------------------------------------------------------------------- #
 # StructuredAdapter
 # --------------------------------------------------------------------------- #
@@ -337,10 +332,10 @@ class StructuredAdapter(logging.LoggerAdapter):
 def get_logger(name: str) -> StructuredAdapter:
     """
     Return a :class:`StructuredAdapter` wrapping ``logging.getLogger(name)``.
-    
+
     Includes automatic initialization of clean shutdown behavior.
     """
     # Ensure clean shutdown behavior is initialized
     _logging_manager.initialize()
-    
-    return StructuredAdapter(logging.getLogger(name), {})
+
+    return StructuredAdapter(logging.getLogger(name), {})