svc-infra 0.1.595__py3-none-any.whl → 0.1.706__py3-none-any.whl

svc_infra/loaders/url.py

@@ -0,0 +1,235 @@
+"""URL content loader.
+
+Load content from URLs with automatic HTML text extraction.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+import httpx
+
+from .base import BaseLoader, ErrorStrategy
+from .models import LoadedContent
+
+logger = logging.getLogger(__name__)
+
+
+class URLLoader(BaseLoader):
+    """Load content from one or more URLs.
+
+    Fetches content from URLs and optionally extracts readable text from HTML.
+    Supports redirects, custom headers, and batch loading.
+
+    Args:
+        urls: Single URL or list of URLs to load.
+        headers: Optional HTTP headers to send with requests.
+        extract_text: If True (default), extract readable text from HTML pages.
+            Raw HTML is returned if False or if content is not HTML.
+        follow_redirects: Follow HTTP redirects (default: True).
+        timeout: Request timeout in seconds (default: 30).
+        extra_metadata: Additional metadata to attach to all loaded content.
+        on_error: How to handle errors ("skip" or "raise"). Default: "skip"
+
+    Example:
+        >>> # Load single URL
+        >>> loader = URLLoader("https://example.com/docs/guide.md")
+        >>> contents = await loader.load()
+        >>> print(contents[0].content[:100])
+        >>>
+        >>> # Load multiple URLs
+        >>> loader = URLLoader([
+        ...     "https://example.com/page1",
+        ...     "https://example.com/page2",
+        ... ])
+        >>> contents = await loader.load()
+        >>>
+        >>> # Disable HTML text extraction
+        >>> loader = URLLoader("https://example.com", extract_text=False)
+        >>> contents = await loader.load()  # Returns raw HTML
+        >>>
+        >>> # With custom headers (e.g., for APIs)
+        >>> loader = URLLoader(
+        ...     "https://api.example.com/docs",
+        ...     headers={"Authorization": "Bearer token123"},
+        ... )
+        >>> contents = await loader.load()
+
+    Note:
+        - HTML text extraction removes scripts, styles, nav, footer, etc.
+        - If BeautifulSoup is not installed, falls back to basic regex extraction
+        - Content type is detected from HTTP headers
+    """
+
+    def __init__(
+        self,
+        urls: str | list[str],
+        headers: dict[str, str] | None = None,
+        extract_text: bool = True,
+        follow_redirects: bool = True,
+        timeout: float = 30.0,
+        extra_metadata: dict[str, Any] | None = None,
+        on_error: ErrorStrategy = "skip",
+    ) -> None:
+        """Initialize the URL loader.
+
+        Args:
+            urls: Single URL or list of URLs
+            headers: HTTP headers to send
+            extract_text: Extract text from HTML (default: True)
+            follow_redirects: Follow redirects (default: True)
+            timeout: Request timeout in seconds
+            extra_metadata: Additional metadata for all content
+            on_error: Error handling strategy
+        """
+        super().__init__(on_error=on_error)
+
+        # Normalize urls to list
+        self.urls = [urls] if isinstance(urls, str) else list(urls)
+        self.headers = headers or {}
+        self.extract_text = extract_text
+        self.follow_redirects = follow_redirects
+        self.timeout = timeout
+        self.extra_metadata = extra_metadata or {}
+
+        # Validate URLs
+        for url in self.urls:
+            if not url.startswith(("http://", "https://")):
+                raise ValueError(
+                    f"Invalid URL: {url!r}. URLs must start with http:// or https://"
+                )
+
+    async def load(self) -> list[LoadedContent]:
+        """Load content from all URLs.
+
+        Returns:
+            List of LoadedContent objects for each successfully loaded URL.
+
+        Raises:
+            httpx.HTTPError: If request fails and on_error="raise".
+        """
+        contents: list[LoadedContent] = []
+
+        async with httpx.AsyncClient(
+            timeout=self.timeout,
+            follow_redirects=self.follow_redirects,
+        ) as client:
+            for url in self.urls:
+                try:
+                    logger.debug(f"Fetching: {url}")
+                    resp = await client.get(url, headers=self.headers)
+                    resp.raise_for_status()
+
+                    content_type = resp.headers.get("content-type", "")
+                    raw_content = resp.text
+
+                    # Extract text from HTML if requested
+                    if self.extract_text and "text/html" in content_type:
+                        content = self._extract_text_from_html(raw_content)
+                    else:
+                        content = raw_content
+
+                    # Parse content type (remove charset etc.)
+                    mime_type = (
+                        content_type.split(";")[0].strip() if content_type else None
+                    )
+
+                    loaded = LoadedContent(
+                        content=content,
+                        source=url,
+                        content_type=mime_type,
+                        metadata={
+                            "loader": "url",
+                            "url": url,
+                            "status_code": resp.status_code,
+                            "final_url": str(resp.url),  # After redirects
+                            **self.extra_metadata,
+                        },
+                    )
+                    contents.append(loaded)
+                    logger.debug(f"Loaded: {url} ({len(content)} chars)")
+
+                except httpx.HTTPStatusError as e:
+                    msg = f"HTTP {e.response.status_code} for {url}"
+                    if self.on_error == "raise":
+                        raise RuntimeError(msg) from e
+                    logger.warning(msg)
+
+                except httpx.RequestError as e:
+                    msg = f"Request failed for {url}: {e}"
+                    if self.on_error == "raise":
+                        raise RuntimeError(msg) from e
+                    logger.warning(msg)
+
+        return contents
+
+    @staticmethod
+    def _extract_text_from_html(html: str) -> str:
+        """Extract readable text from HTML content.
+
+        Tries to use BeautifulSoup if available, falls back to regex.
+
+        Args:
+            html: Raw HTML content
+
+        Returns:
+            Extracted text with scripts, styles, and navigation removed.
+        """
+        try:
+            from bs4 import BeautifulSoup
+
+            soup = BeautifulSoup(html, "html.parser")
+
+            # Remove non-content elements
+            for tag in soup(
+                ["script", "style", "nav", "footer", "header", "aside", "noscript"]
+            ):
+                tag.decompose()
+
+            # Get text with newlines preserved
+            text = soup.get_text(separator="\n", strip=True)
+
+            # Clean up excessive whitespace
+            text = re.sub(r"\n{3,}", "\n\n", text)
+            return text.strip()
+
+        except ImportError:
+            # Fallback: basic regex-based extraction
+            logger.debug("BeautifulSoup not installed, using regex fallback")
+
+            # Remove script and style blocks
+            text = re.sub(
+                r"<script[^>]*>.*?</script>",
+                "",
+                html,
+                flags=re.DOTALL | re.IGNORECASE,
+            )
+            text = re.sub(
+                r"<style[^>]*>.*?</style>",
+                "",
+                text,
+                flags=re.DOTALL | re.IGNORECASE,
+            )
+
+            # Remove all HTML tags
+            text = re.sub(r"<[^>]+>", " ", text)
+
+            # Decode common HTML entities
+            text = text.replace("&nbsp;", " ")
+            text = text.replace("&amp;", "&")
+            text = text.replace("&lt;", "<")
+            text = text.replace("&gt;", ">")
+            text = text.replace("&quot;", '"')
+            text = text.replace("&#39;", "'")
+
+            # Clean up whitespace
+            text = " ".join(text.split())
+            return text.strip()
+
+    def __repr__(self) -> str:
+        """Return string representation."""
+        if len(self.urls) == 1:
+            return f"URLLoader({self.urls[0]!r})"
+        return f"URLLoader([{len(self.urls)} URLs])"

svc_infra/logging/__init__.py

@@ -0,0 +1,374 @@
+"""Logging utilities for svc-infra applications.
+
+This module provides logging utilities optimized for containerized
+environments like Railway, Render, and Kubernetes, where log buffering
+can cause visibility issues.
+
+Features:
+- Force flush for immediate log visibility in containers
+- JSON-formatted structured logging
+- Context injection for request tracing
+- Pre-configured loggers with sensible defaults
+
+Example:
+    >>> from svc_infra.logging import flush, get_logger, configure_for_container
+    >>>
+    >>> # Configure logging at app startup
+    >>> configure_for_container()
+    >>>
+    >>> # Get a logger
+    >>> logger = get_logger(__name__)
+    >>> logger.info("Starting application", extra={"version": "1.0.0"})
+    >>>
+    >>> # Force flush after critical operations
+    >>> flush()
+"""
+
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import os
+import sys
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Iterator, Optional
+
+# Context variables for structured logging
+_log_context: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar(
+    "log_context", default={}
+)
+
+# Default log level from environment
+DEFAULT_LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO").upper()
+
+# Whether to use JSON format (default: True in containers)
+USE_JSON_FORMAT = os.environ.get("LOG_FORMAT", "").lower() != "text"
+
+
+def flush() -> None:
+    """
+    Force flush stdout and stderr for immediate log visibility.
+
+    In containerized environments (Docker, Railway, Kubernetes), Python's
+    output buffering can delay log visibility. Call this after critical
+    operations to ensure logs are immediately visible.
+
+    This is a no-op in terms of log content but ensures buffered output
+    is written to the underlying streams.
+
+    Example:
+        >>> import logging
+        >>> from svc_infra.logging import flush
+        >>>
+        >>> logging.info("Starting database migration...")
+        >>> # ... perform migration ...
+        >>> logging.info("Migration complete")
+        >>> flush()  # Ensure logs are visible in container logs
+    """
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+
+class JsonFormatter(logging.Formatter):
+    """
+    JSON log formatter for structured logging.
+
+    Produces JSON-formatted log lines suitable for log aggregation
+    systems like Datadog, Elastic, or CloudWatch.
+
+    Output format:
+        {"timestamp": "...", "level": "INFO", "logger": "...", "message": "...", ...}
+
+    Any extra fields passed to the logger are included in the output.
+    Context from `with_context()` is also merged in.
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format a log record as JSON."""
+        # Base log structure
+        log_dict: dict[str, Any] = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+
+        # Add exception info if present
+        if record.exc_info:
+            log_dict["exception"] = self.formatException(record.exc_info)
+
+        # Add context from context variable
+        context = _log_context.get()
+        if context:
+            log_dict.update(context)
+
+        # Add any extra fields from the log call
+        # Skip standard LogRecord attributes
+        standard_attrs = {
+            "name",
+            "msg",
+            "args",
+            "created",
+            "filename",
+            "funcName",
+            "levelname",
+            "levelno",
+            "lineno",
+            "module",
+            "msecs",
+            "pathname",
+            "process",
+            "processName",
+            "relativeCreated",
+            "stack_info",
+            "exc_info",
+            "exc_text",
+            "thread",
+            "threadName",
+            "taskName",
+            "message",
+        }
+        for key, value in record.__dict__.items():
+            if key not in standard_attrs and not key.startswith("_"):
+                log_dict[key] = value
+
+        return json.dumps(log_dict, default=str)
+
+
+class TextFormatter(logging.Formatter):
+    """
+    Human-readable text formatter with context support.
+
+    Suitable for local development where JSON is harder to read.
+
+    Output format:
+        2024-01-15 10:30:45 [INFO] logger.name: Message {context}
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format a log record as human-readable text."""
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+        base = f"{timestamp} [{record.levelname}] {record.name}: {record.getMessage()}"
+
+        # Add context if present
+        context = _log_context.get()
+        if context:
+            context_str = " ".join(f"{k}={v}" for k, v in context.items())
+            base = f"{base} [{context_str}]"
+
+        # Add exception if present
+        if record.exc_info:
+            base = f"{base}\n{self.formatException(record.exc_info)}"
+
+        return base
+
+
+def configure_for_container(
+    level: Optional[str] = None,
+    json_format: Optional[bool] = None,
+    stream: Any = None,
+) -> None:
+    """
+    Configure logging for containerized environments.
+
+    Sets up:
+    - Unbuffered output for immediate log visibility
+    - JSON or text formatting based on environment
+    - Appropriate log level from LOG_LEVEL env var
+
+    This should be called once at application startup, typically
+    before any other logging configuration.
+
+    Args:
+        level: Log level (DEBUG, INFO, WARNING, ERROR). Defaults to LOG_LEVEL env var or INFO.
+        json_format: If True, use JSON format; if False, use text. Defaults to LOG_FORMAT env var.
+        stream: Output stream. Defaults to sys.stderr.
+
+    Environment Variables:
+        LOG_LEVEL: Default log level (default: INFO)
+        LOG_FORMAT: "text" for human-readable, anything else for JSON (default: JSON)
+        PYTHONUNBUFFERED: Set to "1" for unbuffered output
+
+    Example:
+        >>> from svc_infra.logging import configure_for_container
+        >>>
+        >>> # In your app's main.py or __init__.py
+        >>> configure_for_container()
+        >>>
+        >>> # Or with explicit settings
+        >>> configure_for_container(level="DEBUG", json_format=False)
+    """
+    # Determine settings
+    log_level = level or DEFAULT_LOG_LEVEL
+    use_json = json_format if json_format is not None else USE_JSON_FORMAT
+    output_stream = stream or sys.stderr
+
+    # Force unbuffered output
+    os.environ["PYTHONUNBUFFERED"] = "1"
+
+    # Get root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(getattr(logging, log_level, logging.INFO))
+
+    # Remove existing handlers to avoid duplicates
+    root_logger.handlers.clear()
+
+    # Create handler with appropriate formatter
+    handler = logging.StreamHandler(output_stream)
+    handler.setLevel(getattr(logging, log_level, logging.INFO))
+
+    if use_json:
+        handler.setFormatter(JsonFormatter())
+    else:
+        handler.setFormatter(TextFormatter())
+
+    root_logger.addHandler(handler)
+
+    # Also configure uvicorn loggers to use our format
+    for logger_name in ("uvicorn", "uvicorn.access", "uvicorn.error"):
+        uvicorn_logger = logging.getLogger(logger_name)
+        uvicorn_logger.handlers.clear()
+        uvicorn_logger.addHandler(handler)
+        uvicorn_logger.propagate = False
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a pre-configured logger instance.
+
+    Returns a logger that respects the configuration set by
+    `configure_for_container()`. If that hasn't been called,
+    the logger will use Python's default configuration.
+
+    Args:
+        name: Logger name, typically `__name__` of the module.
+
+    Returns:
+        Configured logger instance.
+
+    Example:
+        >>> from svc_infra.logging import get_logger
+        >>>
+        >>> logger = get_logger(__name__)
+        >>> logger.info("Processing request", extra={"user_id": 123})
+    """
+    return logging.getLogger(name)
+
+
+@contextmanager
+def with_context(**kwargs: Any) -> Iterator[None]:
+    """
+    Context manager for adding structured context to log messages.
+
+    All log messages within the context will include the specified
+    key-value pairs, making it easy to trace requests or operations
+    across multiple log statements.
+
+    Args:
+        **kwargs: Key-value pairs to add to log context.
+
+    Yields:
+        None
+
+    Example:
+        >>> from svc_infra.logging import with_context, get_logger
+        >>>
+        >>> logger = get_logger(__name__)
+        >>>
+        >>> with with_context(request_id="abc-123", user_id=42):
+        ...     logger.info("Processing request")
+        ...     # Output includes: {"request_id": "abc-123", "user_id": 42, ...}
+        ...     do_something()
+        ...     logger.info("Request complete")
+        >>>
+        >>> # Context is automatically cleared after the block
+        >>> logger.info("No context here")
+    """
+    # Get current context and merge with new values
+    current = _log_context.get()
+    new_context = {**current, **kwargs}
+
+    # Set new context
+    token = _log_context.set(new_context)
+    try:
+        yield
+    finally:
+        # Restore previous context
+        _log_context.reset(token)
+
+
+def set_context(**kwargs: Any) -> None:
+    """
+    Set persistent log context (not scoped like with_context).
+
+    Use this for context that should persist across multiple operations,
+    like tenant_id or user_id for the entire request lifecycle.
+
+    Args:
+        **kwargs: Key-value pairs to add to log context.
+
+    Example:
+        >>> from svc_infra.logging import set_context, clear_context, get_logger
+        >>>
+        >>> logger = get_logger(__name__)
+        >>>
+        >>> # In request middleware
+        >>> set_context(request_id="abc-123", tenant_id="tenant-1")
+        >>>
+        >>> # All subsequent logs include context
+        >>> logger.info("Processing...")
+        >>>
+        >>> # Clear at end of request
+        >>> clear_context()
+    """
+    current = _log_context.get()
+    _log_context.set({**current, **kwargs})
+
+
+def clear_context() -> None:
+    """
+    Clear all log context.
+
+    Call this at the end of a request or operation to ensure
+    context doesn't leak to subsequent operations.
+
+    Example:
+        >>> from svc_infra.logging import set_context, clear_context
+        >>>
+        >>> set_context(request_id="abc-123")
+        >>> # ... handle request ...
+        >>> clear_context()  # Clean up
+    """
+    _log_context.set({})
+
+
+def get_context() -> dict[str, Any]:
+    """
+    Get the current log context.
+
+    Returns:
+        Dictionary of current context key-value pairs.
+
+    Example:
+        >>> from svc_infra.logging import set_context, get_context
+        >>>
+        >>> set_context(request_id="abc-123")
+        >>> ctx = get_context()
+        >>> print(ctx)  # {"request_id": "abc-123"}
+    """
+    return _log_context.get().copy()
+
+
+__all__ = [
+    "flush",
+    "configure_for_container",
+    "get_logger",
+    "with_context",
+    "set_context",
+    "clear_context",
+    "get_context",
+    "JsonFormatter",
+    "TextFormatter",
+]