DeepFabric 4.4.0__py3-none-any.whl

deepfabric/llm/errors.py

@@ -0,0 +1,105 @@
+"""Error handling for LLM providers."""
+
+import anthropic
+import openai
+
+from ..exceptions import DataSetGeneratorError
+
+
+def handle_openai_error(e: Exception, provider: str, model_name: str) -> DataSetGeneratorError:  # noqa: PLR0911
+    """Handle OpenAI-specific errors with helpful messages."""
+    if isinstance(e, openai.AuthenticationError):
+        return DataSetGeneratorError(
+            f"Authentication failed for {provider}. Please check your API key."
+        )
+
+    if isinstance(e, openai.NotFoundError):
+        if provider == "ollama":
+            return DataSetGeneratorError(
+                f"Model '{model_name}' not found in Ollama. Please run: ollama pull {model_name}"
+            )
+        return DataSetGeneratorError(
+            f"Model '{model_name}' not found for {provider}. Please check the model name."
+        )
+
+    if isinstance(e, openai.APIConnectionError):
+        if provider == "ollama":
+            return DataSetGeneratorError(
+                "Cannot connect to Ollama server. Please ensure Ollama is running (try: ollama serve)"
+            )
+        return DataSetGeneratorError(
+            f"Network error connecting to {provider}. Please check your internet connection."
+        )
+
+    if isinstance(e, openai.RateLimitError):
+        return DataSetGeneratorError(
+            f"Rate limit exceeded for {provider}/{model_name}. Please wait and try again."
+        )
+
+    return DataSetGeneratorError(f"OpenAI API error for {provider}/{model_name}: {e}")
+
+
+def handle_anthropic_error(e: Exception, provider: str, model_name: str) -> DataSetGeneratorError:
+    """Handle Anthropic-specific errors with helpful messages."""
+    if isinstance(e, anthropic.AuthenticationError):
+        return DataSetGeneratorError(
+            f"Authentication failed for {provider}. Please check your ANTHROPIC_API_KEY."
+        )
+
+    if isinstance(e, anthropic.NotFoundError):
+        return DataSetGeneratorError(
+            f"Model '{model_name}' not found for {provider}. Please check the model name."
+        )
+
+    if isinstance(e, anthropic.APIConnectionError):
+        return DataSetGeneratorError(
+            f"Network error connecting to {provider}. Please check your internet connection."
+        )
+
+    if isinstance(e, anthropic.RateLimitError):
+        return DataSetGeneratorError(
+            f"Rate limit exceeded for {provider}/{model_name}. Please wait and try again."
+        )
+
+    return DataSetGeneratorError(f"Anthropic API error for {provider}/{model_name}: {e}")
+
+
+def handle_gemini_error(e: Exception, provider: str, model_name: str) -> DataSetGeneratorError:
+    """Handle Gemini-specific errors with helpful messages."""
+    error_str = str(e).lower()
+
+    if "invalid" in error_str and "model" in error_str:
+        return DataSetGeneratorError(
+            f"Model '{model_name}' not available for Gemini. Try: gemini-1.5-flash, gemini-1.5-pro"
+        )
+
+    if any(
+        keyword in error_str
+        for keyword in ["permission", "api_key", "authentication", "unauthorized"]
+    ):
+        return DataSetGeneratorError(
+            "Authentication failed for Gemini. Please check your GOOGLE_API_KEY or GEMINI_API_KEY."
+        )
+
+    if any(keyword in error_str for keyword in ["quota", "rate limit", "too many requests"]):
+        return DataSetGeneratorError(
+            f"Rate limit exceeded for Gemini/{model_name}. Please wait and try again."
+        )
+
+    if any(keyword in error_str for keyword in ["network", "connection", "timeout"]):
+        return DataSetGeneratorError(
+            "Network error connecting to Gemini. Please check your internet connection."
+        )
+
+    return DataSetGeneratorError(f"Gemini API error for {provider}/{model_name}: {e}")
+
+
+def handle_provider_error(e: Exception, provider: str, model_name: str) -> DataSetGeneratorError:
+    """Handle errors for any provider with appropriate error handler."""
+    if provider in ["openai", "ollama"]:
+        return handle_openai_error(e, provider, model_name)
+    if provider == "anthropic":
+        return handle_anthropic_error(e, provider, model_name)
+    if provider == "gemini":
+        return handle_gemini_error(e, provider, model_name)
+    return DataSetGeneratorError(f"Unknown provider error for {provider}/{model_name}: {e}")

deepfabric/llm/rate_limit_config.py

@@ -0,0 +1,262 @@
+"""Rate limiting configuration models for different LLM providers."""
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator
+
+
+class BackoffStrategy(str, Enum):
+    """Backoff strategy for retry attempts."""
+
+    EXPONENTIAL = "exponential"
+    EXPONENTIAL_JITTER = "exponential_jitter"
+    LINEAR = "linear"
+    CONSTANT = "constant"
+
+
+class RateLimitConfig(BaseModel):
+    """Base configuration for rate limiting and retry behavior.
+
+    This provides sensible defaults that work across all providers,
+    with provider-specific subclasses adding specialized behavior.
+    """
+
+    max_retries: int = Field(
+        default=5,
+        ge=0,
+        le=20,
+        description="Maximum number of retry attempts",
+    )
+    base_delay: float = Field(
+        default=1.0,
+        ge=0.1,
+        le=60.0,
+        description="Base delay in seconds before first retry",
+    )
+    max_delay: float = Field(
+        default=60.0,
+        ge=1.0,
+        le=300.0,
+        description="Maximum delay in seconds between retries",
+    )
+    backoff_strategy: BackoffStrategy = Field(
+        default=BackoffStrategy.EXPONENTIAL_JITTER,
+        description="Strategy for calculating retry delays",
+    )
+    exponential_base: float = Field(
+        default=2.0,
+        ge=1.1,
+        le=10.0,
+        description="Base multiplier for exponential backoff",
+    )
+    jitter: bool = Field(
+        default=True,
+        description="Add randomization to delays to prevent thundering herd",
+    )
+    respect_retry_after: bool = Field(
+        default=True,
+        description="Respect retry-after headers from provider responses",
+    )
+
+    # HTTP status codes that should trigger retry
+    retry_on_status_codes: set[int] = Field(
+        default_factory=lambda: {429, 500, 502, 503, 504},
+        description="HTTP status codes that trigger retry",
+    )
+
+    # Exception types/messages that should trigger retry
+    retry_on_exceptions: list[str] = Field(
+        default_factory=lambda: ["timeout", "connection", "network"],
+        description="Exception keywords that trigger retry (case-insensitive)",
+    )
+
+    @field_validator("max_delay")
+    @classmethod
+    def validate_max_delay(cls, v: float, info: "ValidationInfo") -> float:
+        """Ensure max_delay is greater than base_delay."""
+        if "base_delay" in info.data and v < info.data["base_delay"]:
+            msg = "max_delay must be greater than or equal to base_delay"
+            raise ValueError(msg)
+        return v
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert config to dictionary for serialization."""
+        return self.model_dump(mode="json")
+
+
+class OpenAIRateLimitConfig(RateLimitConfig):
+    """OpenAI-specific rate limit configuration.
+
+    OpenAI provides detailed rate limit headers (x-ratelimit-*) and
+    retry-after headers. This config enables monitoring and preemptive
+    backoff based on remaining capacity.
+    """
+
+    check_headers: bool = Field(
+        default=True,
+        description="Monitor x-ratelimit-* headers for capacity tracking",
+    )
+    preemptive_backoff: bool = Field(
+        default=False,
+        description="Back off preemptively when remaining capacity is low",
+    )
+    preemptive_threshold: float = Field(
+        default=0.1,
+        ge=0.0,
+        le=0.5,
+        description="Threshold (0-1) for preemptive backoff (e.g., 0.1 = 10% remaining)",
+    )
+
+
+class AnthropicRateLimitConfig(RateLimitConfig):
+    """Anthropic Claude-specific rate limit configuration.
+
+    Anthropic uses a token bucket algorithm with separate limits for
+    requests per minute (RPM), input tokens per minute (ITPM), and
+    output tokens per minute (OTPM). Rate limits vary by model and tier.
+    """
+
+    check_headers: bool = Field(
+        default=True,
+        description="Monitor anthropic-ratelimit-* headers",
+    )
+    token_bucket_aware: bool = Field(
+        default=True,
+        description="Account for token bucket continuous replenishment",
+    )
+    gradual_rampup: bool = Field(
+        default=True,
+        description="Enable gradual traffic ramp-up for new workloads",
+    )
+
+
+class GeminiRateLimitConfig(RateLimitConfig):
+    """Google Gemini-specific rate limit configuration.
+
+    Gemini has RPM, TPM, and RPD (requests per day) limits. Daily quotas
+    reset at midnight Pacific time. No explicit retry-after header, so
+    more conservative backoff is used. Rate limit errors include detailed
+    quota violation information.
+    """
+
+    base_delay: float = Field(
+        default=2.0,
+        ge=0.5,
+        le=60.0,
+        description="Higher default delay for Gemini (no retry-after header)",
+    )
+    max_delay: float = Field(
+        default=120.0,
+        ge=5.0,
+        le=600.0,
+        description="Longer max delay for daily quota exhaustion",
+    )
+    parse_quota_details: bool = Field(
+        default=True,
+        description="Extract quota metric details from RESOURCE_EXHAUSTED errors",
+    )
+    daily_quota_aware: bool = Field(
+        default=True,
+        description="Recognize daily quota exhaustion vs per-minute limits",
+    )
+
+
+class OllamaRateLimitConfig(RateLimitConfig):
+    """Ollama-specific rate limit configuration.
+
+    Ollama is typically run locally, so rate limiting is less common.
+    This config uses minimal retries, primarily for connection issues.
+    """
+
+    max_retries: int = Field(
+        default=2,
+        ge=0,
+        le=5,
+        description="Minimal retries for local Ollama server",
+    )
+    base_delay: float = Field(
+        default=0.5,
+        ge=0.1,
+        le=5.0,
+        description="Short delay for local server retry",
+    )
+    max_delay: float = Field(
+        default=5.0,
+        ge=1.0,
+        le=30.0,
+        description="Short max delay for local operations",
+    )
+    retry_on_status_codes: set[int] = Field(
+        default_factory=lambda: {500, 502, 503, 504},
+        description="Primarily retry server errors (429 unlikely for local)",
+    )
+
+
+class OpenRouterRateLimitConfig(RateLimitConfig):
+    """OpenRouter-specific rate limit configuration.
+
+    OpenRouter aggregates multiple LLM providers and uses OpenAI-compatible API.
+    It uses credit-based quotas with model-specific rate limits. Different models
+    have different RPM limits, and free model variants have daily limits.
+    Returns 402 Payment Required when account balance is negative.
+    """
+
+    retry_on_status_codes: set[int] = Field(
+        default_factory=lambda: {402, 429, 500, 502, 503, 504},
+        description="HTTP status codes that trigger retry (includes 402 for payment issues)",
+    )
+    check_credits: bool = Field(
+        default=False,
+        description="Monitor credit balance via /api/v1/key endpoint",
+    )
+
+
+def get_default_rate_limit_config(provider: str) -> RateLimitConfig:
+    """Get the default rate limit configuration for a provider.
+
+    Args:
+        provider: Provider name (openai, anthropic, gemini, ollama, openrouter)
+
+    Returns:
+        Provider-specific rate limit configuration with sensible defaults
+    """
+    configs = {
+        "openai": OpenAIRateLimitConfig(),
+        "anthropic": AnthropicRateLimitConfig(),
+        "gemini": GeminiRateLimitConfig(),
+        "ollama": OllamaRateLimitConfig(),
+        "openrouter": OpenRouterRateLimitConfig(),
+    }
+    return configs.get(provider, RateLimitConfig())
+
+
+def create_rate_limit_config(
+    provider: str,
+    config_dict: dict[str, Any] | None = None,
+) -> RateLimitConfig:
+    """Create a rate limit configuration from a dictionary.
+
+    Args:
+        provider: Provider name (openai, anthropic, gemini, ollama, openrouter)
+        config_dict: Configuration parameters as dictionary
+
+    Returns:
+        Provider-specific rate limit configuration
+
+    Raises:
+        ValueError: If configuration validation fails
+    """
+    if config_dict is None:
+        return get_default_rate_limit_config(provider)
+
+    config_classes = {
+        "openai": OpenAIRateLimitConfig,
+        "anthropic": AnthropicRateLimitConfig,
+        "gemini": GeminiRateLimitConfig,
+        "ollama": OllamaRateLimitConfig,
+        "openrouter": OpenRouterRateLimitConfig,
+    }
+
+    config_class = config_classes.get(provider, RateLimitConfig)
+    return config_class(**config_dict)

deepfabric/llm/rate_limit_detector.py

@@ -0,0 +1,278 @@
+"""Rate limit detection and error parsing for different LLM providers."""
+
+import logging
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import anthropic
+import openai
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class QuotaInfo:
+    """Information extracted from a rate limit error."""
+
+    is_rate_limit: bool = False
+    quota_type: str | None = None
+    limit_value: int | None = None
+    retry_after: float | None = None
+    daily_quota_exhausted: bool = False
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        parts = [f"QuotaInfo(is_rate_limit={self.is_rate_limit}"]
+        if self.quota_type:
+            parts.append(f"quota_type={self.quota_type}")
+        if self.retry_after:
+            parts.append(f"retry_after={self.retry_after}s")
+        if self.daily_quota_exhausted:
+            parts.append("daily_quota_exhausted=True")
+        return ", ".join(parts) + ")"
+
+
+class RateLimitDetector:
+    """Detect and parse rate limit errors from different LLM providers."""
+
+    @staticmethod
+    def is_rate_limit_error(exception: Exception, provider: str) -> bool:
+        """Check if an exception represents a rate limit error.
+
+        Args:
+            exception: The exception to check
+            provider: Provider name (openai, anthropic, gemini, ollama)
+
+        Returns:
+            True if the exception is a rate limit error
+        """
+        if provider == "openai":
+            return isinstance(exception, openai.RateLimitError)
+
+        if provider == "anthropic":
+            return isinstance(exception, anthropic.RateLimitError)
+
+        if provider == "gemini":
+            error_str = str(exception)
+            return "429" in error_str and "RESOURCE_EXHAUSTED" in error_str
+
+        if provider == "ollama":
+            # Ollama uses OpenAI-compatible API, but rate limits are unlikely
+            return isinstance(exception, openai.RateLimitError)
+
+        # Generic fallback: check for 429 in error message
+        error_str = str(exception).lower()
+        return "429" in error_str or "rate limit" in error_str
+
+    @staticmethod
+    def is_retryable_error(exception: Exception, provider: str) -> bool:
+        """Check if an exception should trigger a retry.
+
+        This includes rate limits, timeouts, and server errors.
+
+        Args:
+            exception: The exception to check
+            provider: Provider name
+
+        Returns:
+            True if the error is retryable
+        """
+        # Check if it's a rate limit error first
+        if RateLimitDetector.is_rate_limit_error(exception, provider):
+            return True
+
+        # Check for server errors and transient failures
+        error_str = str(exception).lower()
+        retryable_keywords = [
+            "timeout",
+            "connection",
+            "network",
+            "503",
+            "502",
+            "500",
+            "504",
+            "service unavailable",
+            "bad gateway",
+            "gateway timeout",
+        ]
+
+        return any(keyword in error_str for keyword in retryable_keywords)
+
+    @staticmethod
+    def extract_quota_info(exception: Exception, provider: str) -> QuotaInfo:
+        """Extract detailed quota information from a rate limit error.
+
+        Args:
+            exception: The rate limit exception
+            provider: Provider name
+
+        Returns:
+            QuotaInfo object with parsed details
+        """
+        if provider == "openai":
+            return RateLimitDetector._parse_openai_error(exception)
+        if provider == "anthropic":
+            return RateLimitDetector._parse_anthropic_error(exception)
+        if provider == "gemini":
+            return RateLimitDetector._parse_gemini_error(exception)
+        if provider == "ollama":
+            return RateLimitDetector._parse_openai_error(exception)
+
+        # Generic fallback
+        return QuotaInfo(is_rate_limit=RateLimitDetector.is_rate_limit_error(exception, provider))
+
+    @staticmethod
+    def _parse_openai_error(exception: Exception) -> QuotaInfo:
+        """Parse OpenAI rate limit error for quota details.
+
+        OpenAI provides detailed headers:
+        - x-ratelimit-limit-requests
+        - x-ratelimit-remaining-requests
+        - retry-after
+        """
+        quota_info = QuotaInfo()
+
+        if not isinstance(exception, openai.RateLimitError):
+            return quota_info
+
+        quota_info.is_rate_limit = True
+
+        # Try to extract retry-after from the exception
+        try:
+            if hasattr(exception, "response") and exception.response:
+                headers = exception.response.headers
+                if "retry-after" in headers:
+                    quota_info.retry_after = float(headers["retry-after"])
+
+                # Check for remaining capacity in headers
+                if "x-ratelimit-remaining-requests" in headers:
+                    remaining = int(headers["x-ratelimit-remaining-requests"])
+                    quota_info.details["remaining_requests"] = remaining
+
+                if "x-ratelimit-limit-requests" in headers:
+                    limit = int(headers["x-ratelimit-limit-requests"])
+                    quota_info.limit_value = limit
+                    quota_info.quota_type = "requests"
+
+        except (AttributeError, ValueError, KeyError) as e:
+            logger.debug("Could not parse OpenAI rate limit headers: %s", e)
+
+        # Parse error message for quota vs rate limit distinction
+        error_msg = str(exception).lower()
+        if "quota" in error_msg:
+            quota_info.daily_quota_exhausted = True
+            quota_info.quota_type = "quota"
+
+        return quota_info
+
+    @staticmethod
+    def _parse_anthropic_error(exception: Exception) -> QuotaInfo:
+        """Parse Anthropic rate limit error for quota details.
+
+        Anthropic provides:
+        - retry-after header
+        - anthropic-ratelimit-requests-remaining
+        - anthropic-ratelimit-tokens-remaining
+        """
+        quota_info = QuotaInfo()
+
+        if not isinstance(exception, anthropic.RateLimitError):
+            return quota_info
+
+        quota_info.is_rate_limit = True
+
+        try:
+            if hasattr(exception, "response") and exception.response:
+                headers = exception.response.headers
+                if "retry-after" in headers:
+                    quota_info.retry_after = float(headers["retry-after"])
+
+                # Extract remaining capacity
+                if "anthropic-ratelimit-requests-remaining" in headers:
+                    remaining = int(headers["anthropic-ratelimit-requests-remaining"])
+                    quota_info.details["remaining_requests"] = remaining
+
+                if "anthropic-ratelimit-tokens-remaining" in headers:
+                    remaining_tokens = int(headers["anthropic-ratelimit-tokens-remaining"])
+                    quota_info.details["remaining_tokens"] = remaining_tokens
+
+        except (AttributeError, ValueError, KeyError) as e:
+            logger.debug("Could not parse Anthropic rate limit headers: %s", e)
+
+        # Determine quota type from error message
+        error_msg = str(exception).lower()
+        if "request" in error_msg:
+            quota_info.quota_type = "requests_per_minute"
+        elif "token" in error_msg:
+            quota_info.quota_type = "tokens_per_minute"
+
+        return quota_info
+
+    @staticmethod
+    def _parse_gemini_error(exception: Exception) -> QuotaInfo:
+        """Parse Gemini RESOURCE_EXHAUSTED error for quota details.
+
+        Gemini errors include detailed quota violation information:
+        - quotaMetric (e.g., generate_requests_per_model_per_day)
+        - quotaId
+        - No explicit retry-after header
+        """
+        quota_info = QuotaInfo()
+
+        error_str = str(exception)
+        if "429" not in error_str or "RESOURCE_EXHAUSTED" not in error_str:
+            return quota_info
+
+        quota_info.is_rate_limit = True
+
+        # Try to parse the error response JSON if available
+        try:
+            # Look for quota metric in error string
+            if "quotaMetric" in error_str:
+                # Extract quota metric type
+                if "per_day" in error_str:
+                    quota_info.quota_type = "requests_per_day"
+                    quota_info.daily_quota_exhausted = True
+                elif "per_minute" in error_str:
+                    quota_info.quota_type = "requests_per_minute"
+
+            # Check for limit value in error message
+            if "limit: 0" in error_str:
+                quota_info.limit_value = 0
+                quota_info.details["limit_exceeded"] = True
+
+            # Extract specific quota details if present
+            if "generate_requests_per_model_per_day" in error_str:
+                quota_info.details["metric"] = "generate_requests_per_model_per_day"
+            elif "generate_requests_per_model_per_minute" in error_str:
+                quota_info.details["metric"] = "generate_requests_per_model_per_minute"
+
+        except Exception as e:  # noqa: BLE001
+            logger.debug("Could not parse Gemini quota details: %s", e)
+
+        # Gemini doesn't provide retry-after header, use None
+        quota_info.retry_after = None
+
+        return quota_info
+
+    @staticmethod
+    def should_fail_fast(quota_info: QuotaInfo) -> bool:
+        """Determine if we should fail fast instead of retrying.
+
+        Some quota errors are not worth retrying:
+        - Daily quota exhausted (won't reset for hours)
+        - Quota limit is 0 (account not set up properly)
+
+        Args:
+            quota_info: Parsed quota information
+
+        Returns:
+            True if we should fail fast and not retry
+        """
+        # Daily quota exhausted - won't reset until midnight
+        if quota_info.daily_quota_exhausted:
+            return True
+
+        # Quota limit is 0 - account issue, not transient
+        return quota_info.limit_value == 0