content-core 1.10.0__py3-none-any.whl

content_core/common/retry.py

@@ -0,0 +1,325 @@
+"""
+Retry decorators for handling transient failures in external operations.
+
+This module provides pre-configured retry decorators using tenacity for different
+operation types (YouTube, URL extraction, audio transcription, LLM calls, downloads).
+
+Each decorator uses exponential backoff with jitter to prevent thundering herd problems.
+
+Usage:
+    from content_core.common.retry import retry_youtube, retry_url_api
+
+    @retry_youtube()
+    async def get_video_title(video_id):
+        ...
+
+    @retry_url_api()
+    async def extract_url_jina(url):
+        ...
+"""
+
+from typing import Callable, Optional
+
+import aiohttp
+from tenacity import (
+    RetryError,
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from content_core.common.exceptions import NoTranscriptFound, NotFoundError
+from content_core.config import get_retry_config
+from content_core.logging import logger
+
+
+# Exceptions that should NOT be retried (permanent failures)
+NON_RETRYABLE_EXCEPTIONS = (
+    NoTranscriptFound,
+    NotFoundError,
+    ValueError,
+    TypeError,
+    KeyError,
+    AttributeError,
+)
+
+
+def is_retryable_exception(exception: BaseException) -> bool:
+    """
+    Determine if an exception should trigger a retry.
+
+    Returns True for transient/network errors, False for permanent failures.
+    """
+    # Never retry these - they indicate permanent failures
+    if isinstance(exception, NON_RETRYABLE_EXCEPTIONS):
+        return False
+
+    # Always retry network-related errors
+    if isinstance(exception, (aiohttp.ClientError, ConnectionError, TimeoutError, OSError)):
+        # But not if it's a client error (4xx) - those are usually permanent
+        if isinstance(exception, aiohttp.ClientResponseError):
+            status = exception.status
+            # Retry server errors (5xx) and rate limits (429)
+            return status >= 500 or status == 429
+        return True
+
+    # For generic exceptions, check if they look like transient errors
+    exc_msg = str(exception).lower()
+    transient_indicators = [
+        "timeout", "timed out", "connection", "network", "temporary",
+        "unavailable", "rate limit", "too many requests", "503", "502", "500"
+    ]
+    return any(indicator in exc_msg for indicator in transient_indicators)
+
+
+def log_retry_attempt(retry_state) -> None:
+    """
+    Log retry attempts with detailed information.
+
+    This is used as the before_sleep callback for tenacity decorators.
+
+    Args:
+        retry_state: Tenacity retry state containing attempt info and exception
+    """
+    func_name = retry_state.fn.__name__ if retry_state.fn else "unknown"
+    attempt_num = retry_state.attempt_number
+    exception = retry_state.outcome.exception() if retry_state.outcome else None
+
+    if exception:
+        exc_type = type(exception).__name__
+        exc_msg = str(exception)[:200]  # Truncate long messages
+        logger.warning(
+            f"Retry {attempt_num} for {func_name}: {exc_type}: {exc_msg}"
+        )
+    else:
+        logger.warning(f"Retry {attempt_num} for {func_name}: unknown error")
+
+
+def log_retry_exhausted(retry_state) -> None:
+    """
+    Log when all retry attempts have been exhausted.
+
+    Args:
+        retry_state: Tenacity retry state containing final attempt info
+    """
+    func_name = retry_state.fn.__name__ if retry_state.fn else "unknown"
+    attempt_num = retry_state.attempt_number
+    exception = retry_state.outcome.exception() if retry_state.outcome else None
+
+    if exception:
+        exc_type = type(exception).__name__
+        exc_msg = str(exception)[:500]
+        logger.error(
+            f"All {attempt_num} retries exhausted for {func_name}: {exc_type}: {exc_msg}"
+        )
+    else:
+        logger.error(f"All {attempt_num} retries exhausted for {func_name}")
+
+
+def retry_youtube(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for YouTube operations.
+
+    Uses longer delays due to YouTube's aggressive rate limiting.
+    Does NOT retry permanent failures like NoTranscriptFound.
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 5)
+        base_delay: Override base delay in seconds (default from config: 2)
+        max_delay: Override max delay in seconds (default from config: 60)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("youtube")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+def retry_url_api(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for API-based URL extraction (Jina, Firecrawl).
+
+    Retries on network errors and server errors (5xx, 429), but not client errors (4xx).
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 3)
+        base_delay: Override base delay in seconds (default from config: 1)
+        max_delay: Override max delay in seconds (default from config: 30)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("url_api")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+def retry_url_network(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for network-only URL operations (BeautifulSoup, HEAD requests).
+
+    Uses shorter delays as these are typically network-only issues.
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 3)
+        base_delay: Override base delay in seconds (default from config: 0.5)
+        max_delay: Override max delay in seconds (default from config: 10)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("url_network")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+def retry_audio_transcription(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for audio transcription (speech-to-text API calls).
+
+    Retries on transient errors, but not on permanent failures like invalid files.
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 3)
+        base_delay: Override base delay in seconds (default from config: 2)
+        max_delay: Override max delay in seconds (default from config: 30)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("audio")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+def retry_llm(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for LLM API calls (summary, cleanup).
+
+    Retries on transient errors like rate limits and timeouts, but not on
+    permanent failures like invalid API keys or malformed requests.
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 3)
+        base_delay: Override base delay in seconds (default from config: 1)
+        max_delay: Override max delay in seconds (default from config: 30)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("llm")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+def retry_download(
+    max_attempts: Optional[int] = None,
+    base_delay: Optional[float] = None,
+    max_delay: Optional[float] = None,
+) -> Callable:
+    """
+    Retry decorator for remote file downloads.
+
+    Retries on network errors and server errors (5xx, 429), but not client errors (4xx).
+
+    Args:
+        max_attempts: Override max retry attempts (default from config: 3)
+        base_delay: Override base delay in seconds (default from config: 1)
+        max_delay: Override max delay in seconds (default from config: 15)
+
+    Returns:
+        Configured tenacity retry decorator
+    """
+    config = get_retry_config("download")
+    attempts = max_attempts if max_attempts is not None else config["max_attempts"]
+    base = base_delay if base_delay is not None else config["base_delay"]
+    max_wait = max_delay if max_delay is not None else config["max_delay"]
+
+    return retry(
+        stop=stop_after_attempt(attempts),
+        wait=wait_random_exponential(multiplier=base, max=max_wait),
+        retry=retry_if_exception(is_retryable_exception),
+        before_sleep=log_retry_attempt,
+        reraise=True,
+    )
+
+
+# Export RetryError for use in exception handling
+__all__ = [
+    "retry_youtube",
+    "retry_url_api",
+    "retry_url_network",
+    "retry_audio_transcription",
+    "retry_llm",
+    "retry_download",
+    "log_retry_attempt",
+    "log_retry_exhausted",
+    "RetryError",
+]

content_core/common/state.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from content_core.common.types import DocumentEngine, UrlEngine
+
+
+class ProcessSourceState(BaseModel):
+    file_path: Optional[str] = ""
+    url: Optional[str] = ""
+    delete_source: bool = False
+    title: Optional[str] = ""
+    source_type: Optional[str] = ""
+    identified_type: Optional[str] = ""
+    identified_provider: Optional[str] = ""
+    metadata: Optional[dict] = Field(default_factory=lambda: {})
+    content: Optional[str] = ""
+    document_engine: Optional[DocumentEngine] = Field(
+        default=None,
+        description="Override document extraction engine: 'auto', 'simple', or 'docling'",
+    )
+    url_engine: Optional[UrlEngine] = Field(
+        default=None,
+        description="Override URL extraction engine: 'auto', 'simple', 'firecrawl', 'jina', 'crawl4ai', or 'docling'",
+    )
+    output_format: Optional[str] = Field(
+        default=None,
+        description="Override Docling output format: 'markdown', 'html', or 'json'",
+    )
+    audio_provider: Optional[str] = Field(
+        default=None,
+        description="Override speech-to-text provider (e.g., 'openai', 'google')",
+    )
+    audio_model: Optional[str] = Field(
+        default=None,
+        description="Override speech-to-text model name (e.g., 'whisper-1', 'chirp')",
+    )
+    proxy: Optional[str] = Field(
+        default=None,
+        description="Override proxy URL for this request (e.g., 'http://proxy:8080')",
+    )
+
+
+class ProcessSourceInput(BaseModel):
+    content: Optional[str] = ""
+    file_path: Optional[str] = ""
+    url: Optional[str] = ""
+    document_engine: Optional[str] = None
+    url_engine: Optional[str] = None
+    output_format: Optional[str] = None
+    audio_provider: Optional[str] = None
+    audio_model: Optional[str] = None
+    proxy: Optional[str] = None
+
+
+class ProcessSourceOutput(BaseModel):
+    title: Optional[str] = ""
+    file_path: Optional[str] = ""
+    url: Optional[str] = ""
+    source_type: Optional[str] = ""
+    identified_type: Optional[str] = ""
+    identified_provider: Optional[str] = ""
+    metadata: Optional[dict] = Field(default_factory=lambda: {})
+    content: Optional[str] = ""

content_core/common/types.py

@@ -0,0 +1,15 @@
+from typing import Literal
+
+DocumentEngine = Literal[
+    "auto",
+    "simple", 
+    "docling",
+]
+
+UrlEngine = Literal[
+    "auto",
+    "simple",
+    "firecrawl",
+    "jina",
+    "crawl4ai",
+]

content_core/common/utils.py

@@ -0,0 +1,31 @@
+import os
+import re
+import validators
+
+from .state import ProcessSourceInput
+
+
+async def process_input_content(content: str) -> str:
+    """
+    Process input content to handle URLs and file paths.
+    If the input is a URL or file path, extract the content from it.
+    """
+    # Check if content is a URL
+    if validators.url(content):
+        from content_core.extraction import extract_content
+        content_input = ProcessSourceInput(url=content)
+        extracted = await extract_content(content_input)
+        return extracted.content if extracted.content else str(extracted)
+
+    # Check if content is a file path (simplified check for demonstration)
+    if re.match(r"^[a-zA-Z0-9_/\-\.]+\.[a-zA-Z0-9]+$", content):
+        if os.path.exists(content):
+            from content_core.extraction import extract_content
+            content_input = ProcessSourceInput(file_path=content)
+            extracted = await extract_content(content_input)
+            return extracted.content if extracted.content else str(extracted)
+        else:
+            raise ValueError(f"File not found: {content}")
+
+    # If neither URL nor file path, return content as is
+    return content