DeepFabric 4.4.0__py3-none-any.whl

deepfabric/llm/retry_handler.py

@@ -0,0 +1,270 @@
+"""Retry handler with intelligent backoff for LLM API calls."""
+
+import asyncio
+import logging
+import random
+import time
+
+from collections.abc import Callable, Coroutine
+from functools import wraps
+from typing import Any, TypeVar
+
+from .rate_limit_config import BackoffStrategy, RateLimitConfig
+from .rate_limit_detector import RateLimitDetector
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class RetryHandler:
+    """Intelligent retry handler for LLM API calls with provider-aware backoff."""
+
+    def __init__(self, config: RateLimitConfig, provider: str):
+        """Initialize retry handler.
+
+        Args:
+            config: Rate limit configuration
+            provider: Provider name (openai, anthropic, gemini, ollama)
+        """
+        self.config = config
+        self.provider = provider
+        self.detector = RateLimitDetector()
+
+    def should_retry(self, exception: Exception) -> bool:
+        """Determine if an exception should trigger a retry.
+
+        Args:
+            exception: The exception that occurred
+
+        Returns:
+            True if the error is retryable
+        """
+        # Check if it's a retryable error (rate limit, timeout, server error)
+        if not self.detector.is_retryable_error(exception, self.provider):
+            return False
+
+        # If it's a rate limit error, check if we should fail fast
+        if self.detector.is_rate_limit_error(exception, self.provider):
+            quota_info = self.detector.extract_quota_info(exception, self.provider)
+
+            # Don't retry if we should fail fast (e.g., daily quota exhausted)
+            if self.detector.should_fail_fast(quota_info):
+                logger.warning(
+                    "Failing fast for %s: %s (quota_info: %s)",
+                    self.provider,
+                    exception,
+                    quota_info,
+                )
+                return False
+
+        return True
+
+    def calculate_delay(
+        self,
+        attempt: int,
+        exception: Exception | None = None,
+    ) -> float:
+        """Calculate the delay before the next retry attempt.
+
+        Args:
+            attempt: Current attempt number (0-indexed)
+            exception: Optional exception to extract retry-after from
+
+        Returns:
+            Delay in seconds
+        """
+        # Extract retry-after from exception if available
+        retry_after = None
+        if exception and self.config.respect_retry_after:
+            quota_info = self.detector.extract_quota_info(exception, self.provider)
+            retry_after = quota_info.retry_after
+
+        # If provider specifies retry-after, use it (with max_delay cap)
+        if retry_after is not None:
+            delay = min(retry_after, self.config.max_delay)
+            logger.debug("Using retry-after header: %.2fs (capped at %.2fs)", retry_after, delay)
+            return delay
+
+        # Otherwise calculate delay based on backoff strategy
+        if self.config.backoff_strategy == BackoffStrategy.EXPONENTIAL:
+            delay = self.config.base_delay * (self.config.exponential_base**attempt)
+        elif self.config.backoff_strategy == BackoffStrategy.EXPONENTIAL_JITTER:
+            base_delay = self.config.base_delay * (self.config.exponential_base**attempt)
+            delay = base_delay
+        elif self.config.backoff_strategy == BackoffStrategy.LINEAR:
+            delay = self.config.base_delay * (attempt + 1)
+        elif self.config.backoff_strategy == BackoffStrategy.CONSTANT:
+            delay = self.config.base_delay
+        else:
+            # Default to exponential
+            delay = self.config.base_delay * (self.config.exponential_base**attempt)
+
+        # Apply jitter if enabled
+        if self.config.jitter:
+            # Add random jitter of ±25% to prevent thundering herd
+            jitter_range = delay * 0.25
+            delay = delay + random.uniform(-jitter_range, jitter_range)  # noqa: S311 # nosec
+
+        # Ensure delay is within bounds and return
+        return max(self.config.base_delay, min(delay, self.config.max_delay))
+
+    def on_backoff_handler(self, details: dict[str, Any]) -> None:
+        """Callback for backoff retry attempts.
+
+        Args:
+            details: Backoff details including attempt number, wait time, exception
+        """
+        exception = details.get("exception")
+        wait = details.get("wait", 0)
+        tries = details.get("tries", 0)
+
+        # Extract quota information if it's a rate limit error
+        quota_info_str = ""
+        if exception and self.detector.is_rate_limit_error(exception, self.provider):
+            quota_info = self.detector.extract_quota_info(exception, self.provider)
+            if quota_info.quota_type:
+                quota_info_str = f" (quota_type: {quota_info.quota_type})"
+
+        logger.warning(
+            "Rate limit/transient error for %s on attempt %d, backing off %.2fs%s: %s",
+            self.provider,
+            tries,
+            wait,
+            quota_info_str,
+            exception,
+        )
+
+    def on_giveup_handler(self, details: dict[str, Any]) -> None:
+        """Callback when giving up after max retries.
+
+        Args:
+            details: Backoff details including final exception
+        """
+        exception = details.get("exception")
+        tries = details.get("tries", 0)
+
+        logger.error(
+            "Giving up after %d attempts for %s: %s",
+            tries,
+            self.provider,
+            exception,
+        )
+
+
+def retry_with_backoff(func: Callable[..., T]) -> Callable[..., T]:
+    """Decorator to add retry logic with backoff to synchronous functions.
+
+    This decorator is applied to LLMClient methods to handle rate limits
+    and transient errors automatically.
+
+    Args:
+        func: Function to wrap with retry logic
+
+    Returns:
+        Wrapped function with retry capability
+    """
+
+    @wraps(func)
+    def wrapper(self, *args: Any, **kwargs: Any) -> T:
+        # Get retry handler from self (assumes LLMClient instance)
+        retry_handler = getattr(self, "retry_handler", None)
+        if not retry_handler or not isinstance(retry_handler, RetryHandler):
+            # No retry handler configured, call function directly
+            return func(self, *args, **kwargs)
+
+        config = retry_handler.config
+        attempt = 0
+
+        while attempt <= config.max_retries:
+            try:
+                return func(self, *args, **kwargs)
+            except Exception as e:
+                # Check if we should retry
+                if not retry_handler.should_retry(e):
+                    # Not retryable, raise immediately
+                    raise
+
+                # Check if we've exhausted retries
+                if attempt >= config.max_retries:
+                    retry_handler.on_giveup_handler({"exception": e, "tries": attempt + 1})
+                    raise
+
+                # Calculate delay and wait
+                delay = retry_handler.calculate_delay(attempt, e)
+                retry_handler.on_backoff_handler(
+                    {
+                        "exception": e,
+                        "wait": delay,
+                        "tries": attempt + 1,
+                    }
+                )
+
+                time.sleep(delay)
+                attempt += 1
+
+        # Should never reach here, but for type safety
+        msg = "Unexpected state in retry logic"
+        raise RuntimeError(msg)
+
+    return wrapper
+
+
+def retry_with_backoff_async(
+    func: Callable[..., Coroutine[Any, Any, T]],
+) -> Callable[..., Coroutine[Any, Any, T]]:
+    """Decorator to add retry logic with backoff to async functions.
+
+    This decorator is applied to async LLMClient methods to handle rate limits
+    and transient errors automatically.
+
+    Args:
+        func: Async function to wrap with retry logic
+
+    Returns:
+        Wrapped async function with retry capability
+    """
+
+    @wraps(func)
+    async def wrapper(self, *args: Any, **kwargs: Any) -> T:
+        # Get retry handler from self (assumes LLMClient instance)
+        retry_handler = getattr(self, "retry_handler", None)
+        if not retry_handler or not isinstance(retry_handler, RetryHandler):
+            # No retry handler configured, call function directly
+            return await func(self, *args, **kwargs)
+
+        config = retry_handler.config
+        attempt = 0
+
+        while attempt <= config.max_retries:
+            try:
+                return await func(self, *args, **kwargs)
+            except Exception as e:
+                # Check if we should retry
+                if not retry_handler.should_retry(e):
+                    # Not retryable, raise immediately
+                    raise
+
+                # Check if we've exhausted retries
+                if attempt >= config.max_retries:
+                    retry_handler.on_giveup_handler({"exception": e, "tries": attempt + 1})
+                    raise
+
+                # Calculate delay and wait
+                delay = retry_handler.calculate_delay(attempt, e)
+                retry_handler.on_backoff_handler(
+                    {
+                        "exception": e,
+                        "wait": delay,
+                        "tries": attempt + 1,
+                    }
+                )
+
+                await asyncio.sleep(delay)
+                attempt += 1
+
+        # Should never reach here, but for type safety
+        msg = "Unexpected state in retry logic"
+        raise RuntimeError(msg)
+
+    return wrapper

deepfabric/metrics.py

@@ -0,0 +1,212 @@
+import contextlib
+import logging
+import os
+import uuid
+
+from pathlib import Path
+
+from posthog import Posthog, identify_context, new_context
+
+from .tui import get_tui
+
+try:
+    import importlib.metadata
+
+    VERSION = importlib.metadata.version("deepfabric")
+except (ImportError, importlib.metadata.PackageNotFoundError):
+    VERSION = "development"
+
+
+# Initialize PostHog client
+posthog = Posthog(
+    project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
+    host="https://us.i.posthog.com",
+)
+
+logger = logging.getLogger(__name__)
+
+
+class _TelemetryState:
+    """Holds the state for the telemetry module."""
+
+    def __init__(self) -> None:
+        self.debug_trace: bool = False
+        self.user_id_announced: bool = False
+        self.user_id_cache: str | None = None
+        self.telemetry_failed_once: bool = False
+
+
+_state = _TelemetryState()
+
+
+APP_NAME = "DeepFabric"
+APP_AUTHOR = "DeepFabric"
+
+
+try:
+    from platformdirs import user_data_dir
+except ImportError:  # pragma: no cover - optional dependency
+
+    def user_data_dir(appname: str, appauthor: str | None = None) -> str:
+        if os.name == "nt":
+            base = os.environ.get("APPDATA") or os.path.expanduser(r"~\AppData\Roaming")
+        elif os.name == "posix":
+            base = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share")
+        else:
+            base = os.path.expanduser("~")
+        return str(Path(base) / (appauthor or appname) / appname)
+
+
+def _user_id_path() -> Path:
+    candidates: list[Path] = []
+    try:
+        candidates.append(Path(user_data_dir(APP_NAME, APP_AUTHOR)))
+    except Exception:
+        logger.debug("Failed to resolve platform data dir", exc_info=True)
+    candidates.append(Path.home() / f".{APP_NAME.lower()}")
+    candidates.append(Path.cwd())
+
+    for data_dir in candidates:
+        try:
+            data_dir.mkdir(parents=True, exist_ok=True)
+            return data_dir / "telemetry_id"
+        except Exception:
+            logger.debug("Failed to prepare telemetry directory %s", data_dir, exc_info=True)
+
+    return Path("telemetry_id")
+
+
+def _read_user_id(path: Path) -> str | None:
+    try:
+        if path.exists():
+            candidate = path.read_text(encoding="utf-8").strip()
+            if candidate:
+                uuid.UUID(candidate)
+                return candidate
+    except Exception:
+        logger.debug("Failed to read existing telemetry id", exc_info=True)
+    return None
+
+
+def _write_user_id(path: Path) -> str:
+    user_id = str(uuid.uuid4())
+    tmp_path = path.with_suffix(".tmp")
+    try:
+        tmp_path.write_text(user_id, encoding="utf-8")
+        if os.name == "posix":
+            os.chmod(tmp_path, 0o600)
+        tmp_path.replace(path)
+    except Exception:
+        logger.debug("Failed to persist telemetry id", exc_info=True)
+        return user_id
+    else:
+        return user_id
+    finally:
+        with contextlib.suppress(Exception):
+            if tmp_path.exists() and tmp_path != path:
+                tmp_path.unlink()
+
+
+def _get_user_id() -> str:
+    """Generate a stable, anonymous user ID persisted on disk."""
+    if _state.user_id_cache is not None:
+        return _state.user_id_cache
+
+    path = _user_id_path()
+    user_id = _read_user_id(path)
+    if user_id is None:
+        user_id = _write_user_id(path)
+
+    _state.user_id_cache = user_id
+    return user_id
+
+
+def _is_developer() -> bool:
+    """
+    Check if this session is marked as a developer session.
+
+    Returns:
+        bool: True if DEEPFABRIC_DEVELOPER environment variable is set to 'True'
+    """
+    return os.environ.get("DEEPFABRIC_DEVELOPER") == "True"
+
+
+def set_trace_debug(enabled: bool) -> None:
+    """Enable or disable debug output for telemetry events."""
+    _state.debug_trace = enabled
+    if not enabled:
+        _state.user_id_announced = False
+
+
+def _announce_user_id(user_id: str) -> None:
+    if _state.user_id_announced or not _state.debug_trace:
+        return
+    try:
+        get_tui().info(f"metrics user id: {user_id}")
+    except Exception:  # pragma: no cover - fallback to logging
+        logger.debug("metrics user id: %s", user_id)
+
+    _state.user_id_announced = True
+
+
+def trace(event_name, event_properties=None):
+    """
+    Send an analytics event if metrics is enabled.
+
+    Uses privacy-respecting identity tracking with a stable, anonymous user ID
+    stored on disk for reuse. Developer sessions are marked with
+    the is_developer flag when DEEPFABRIC_DEVELOPER=True.
+
+    Args:
+        event_name: Name of the event to track
+        event_properties: Optional dictionary of event properties
+    """
+    if not is_enabled():
+        return
+
+    try:
+        # Generate stable user ID
+        user_id = _get_user_id()
+        _announce_user_id(user_id)
+
+        # Add version and developer flag to all events
+        properties = event_properties or {}
+        properties["version"] = VERSION
+        properties["is_developer"] = _is_developer()
+
+        # Use identity context to associate events with the user
+        with new_context():
+            identify_context(user_id)
+            posthog.capture(
+                distinct_id=user_id,
+                event=event_name,
+                properties=properties,
+            )
+    except Exception:
+        if not _state.telemetry_failed_once:
+            _state.telemetry_failed_once = True
+            logger.warning(
+                "Failed to send telemetry event. Further failures will be logged at DEBUG level.",
+                exc_info=True,
+            )
+        else:
+            logger.debug("Failed to capture metrics event", exc_info=True)
+
+
+def is_enabled():
+    """Check if analytics is currently enabled."""
+    return (
+        os.environ.get("ANONYMIZED_TELEMETRY") != "False"
+        and os.environ.get("DEEPFABRIC_TESTING") != "True"
+    )
+
+
+def shutdown() -> None:
+    """
+    Shutdown the PostHog client, flushing any buffered events.
+
+    This should be called on application exit to ensure all metrics data is sent.
+    """
+    if is_enabled():
+        logger.debug("Shutting down metrics client.")
+        posthog.shutdown()