texas-grocery-mcp 0.1.0__py3-none-any.whl

texas_grocery_mcp/observability/health.py

@@ -0,0 +1,141 @@
+"""Health check endpoints."""
+
+from datetime import UTC, datetime
+from typing import Any, Literal, cast
+
+import structlog
+
+from texas_grocery_mcp.models.health import (
+    CircuitBreakerStatus,
+    ComponentHealth,
+    HealthResponse,
+)
+
+logger = structlog.get_logger()
+
+
+def health_live() -> dict[str, str]:
+    """Liveness probe - is the process running?
+
+    Returns a simple alive status. Use for Kubernetes liveness probes.
+    """
+    return {"status": "alive"}
+
+
+def _check_redis_health_sync(redis_url: str) -> ComponentHealth:
+    """Check Redis connectivity (sync version).
+
+    Args:
+        redis_url: Redis connection URL
+
+    Returns:
+        ComponentHealth with status and optional message
+    """
+    try:
+        import redis
+
+        # Parse URL and connect with timeout
+        client = redis.from_url(  # type: ignore[no-untyped-call]
+            redis_url,
+            socket_connect_timeout=2.0,
+            socket_timeout=2.0,
+        )
+
+        try:
+            # Ping to verify connectivity
+            client.ping()
+
+            # Get basic info for health details
+            info = client.info(section="server")
+            redis_version = info.get("redis_version", "unknown")
+
+            return ComponentHealth(
+                status="up",
+                message=f"Redis {redis_version}",
+            )
+
+        finally:
+            client.close()
+
+    except ImportError:
+        return ComponentHealth(
+            status="up",
+            message="Redis client not installed (optional dependency)",
+        )
+    except Exception as e:
+        logger.warning("Redis health check failed", error=str(e))
+        return ComponentHealth(
+            status="down",
+            message=f"Connection failed: {str(e)}",
+        )
+
+
+def health_ready() -> dict[str, Any]:
+    """Readiness probe - can the server handle requests?
+
+    Returns detailed component health. Use for Kubernetes readiness probes.
+    """
+    components: dict[str, ComponentHealth] = {}
+    circuit_breakers: dict[str, CircuitBreakerStatus] = {}
+    overall_status: Literal["healthy", "degraded", "unhealthy"] = "healthy"
+
+    # Check GraphQL API status
+    try:
+        from texas_grocery_mcp.clients.graphql import HEBGraphQLClient
+
+        client = HEBGraphQLClient()
+        cb_status = client.circuit_breaker.get_status()
+
+        state_raw = cb_status.get("state")
+        state: Literal["closed", "open", "half_open"] = "closed"
+        if isinstance(state_raw, str) and state_raw in ("closed", "open", "half_open"):
+            state = cast(Literal["closed", "open", "half_open"], state_raw)
+
+        failures_raw = cb_status.get("failure_count", 0)
+        failures = int(failures_raw) if isinstance(failures_raw, int) else 0
+
+        if state == "open":
+            components["graphql_api"] = ComponentHealth(
+                status="down", message="Circuit breaker open"
+            )
+            overall_status = "degraded"
+        else:
+            components["graphql_api"] = ComponentHealth(status="up")
+
+        circuit_breakers["heb_graphql"] = CircuitBreakerStatus(
+            state=state,
+            failures=failures,
+        )
+    except Exception as e:
+        components["graphql_api"] = ComponentHealth(
+            status="down", message=str(e)
+        )
+        overall_status = "unhealthy"
+
+    # Check cache status (if configured)
+    try:
+        from texas_grocery_mcp.utils.config import get_settings
+
+        settings = get_settings()
+        if settings.redis_url:
+            # Actually check Redis connectivity
+            cache_health = _check_redis_health_sync(settings.redis_url)
+            components["cache"] = cache_health
+
+            if cache_health.status == "down" and overall_status == "healthy":
+                overall_status = "degraded"
+        else:
+            components["cache"] = ComponentHealth(
+                status="up", message="Not configured (using in-memory)"
+            )
+    except Exception as e:
+        components["cache"] = ComponentHealth(status="down", message=str(e))
+        if overall_status == "healthy":
+            overall_status = "degraded"
+
+    return HealthResponse(
+        status=overall_status,
+        timestamp=datetime.now(UTC).isoformat(),
+        components=components,
+        circuit_breakers=circuit_breakers,
+    ).model_dump()

texas_grocery_mcp/observability/logging.py

@@ -0,0 +1,73 @@
+"""Structured JSON logging configuration."""
+
+import logging
+import sys
+from collections.abc import MutableMapping
+from typing import Any
+
+import structlog
+from structlog.types import Processor
+
+from texas_grocery_mcp.utils.config import get_settings
+
+
+def add_timestamp(
+    logger: Any, method_name: str, event_dict: MutableMapping[str, Any]
+) -> MutableMapping[str, Any]:
+    """Add ISO timestamp to log entry."""
+    from datetime import UTC, datetime
+
+    event_dict["timestamp"] = datetime.now(UTC).isoformat()
+    return event_dict
+
+
+def configure_logging(log_level: str | None = None) -> None:
+    """Configure structured JSON logging.
+
+    Logs to stderr to keep stdout clean for MCP protocol.
+    """
+    settings = get_settings()
+    level = log_level or settings.log_level
+
+    # Shared processors for all loggers
+    shared_processors: list[Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        add_timestamp,
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+    ]
+
+    # Configure structlog
+    structlog.configure(
+        processors=[
+            *shared_processors,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        wrapper_class=structlog.stdlib.BoundLogger,
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    # Configure stdlib logging
+    formatter = structlog.stdlib.ProcessorFormatter(
+        foreign_pre_chain=shared_processors,
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            structlog.processors.JSONRenderer(),
+        ],
+    )
+
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(formatter)
+
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(getattr(logging, level.upper()))
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    """Get a structured logger instance."""
+    return structlog.get_logger(name)  # type: ignore[no-any-return]

texas_grocery_mcp/reliability/__init__.py

@@ -0,0 +1,23 @@
+"""Reliability patterns for production resilience."""
+
+from texas_grocery_mcp.reliability.cache import TTLCache
+from texas_grocery_mcp.reliability.circuit_breaker import (
+    CircuitBreaker,
+    CircuitBreakerConfig,
+    CircuitBreakerOpenError,
+    CircuitState,
+)
+from texas_grocery_mcp.reliability.retry import RetryConfig, with_retry
+from texas_grocery_mcp.reliability.throttle import ThrottleConfig, Throttler
+
+__all__ = [
+    "CircuitBreaker",
+    "CircuitBreakerConfig",
+    "CircuitBreakerOpenError",
+    "CircuitState",
+    "RetryConfig",
+    "TTLCache",
+    "with_retry",
+    "ThrottleConfig",
+    "Throttler",
+]

texas_grocery_mcp/reliability/cache.py

@@ -0,0 +1,116 @@
+"""Simple TTL cache for API responses."""
+
+from datetime import datetime, timedelta
+from typing import Generic, TypeVar
+
+import structlog
+
+logger = structlog.get_logger()
+
+T = TypeVar("T")
+
+
+class TTLCache(Generic[T]):
+    """Simple in-memory cache with time-to-live expiration.
+
+    Thread-safe for basic operations. Uses a dict for storage with
+    (value, timestamp) tuples.
+
+    Example:
+        cache = TTLCache[ProductDetails](ttl_hours=24)
+        cache.set("127074", product_details)
+        details = cache.get("127074")  # Returns None if expired
+    """
+
+    def __init__(self, ttl_hours: int = 24, max_size: int = 1000):
+        """Initialize the cache.
+
+        Args:
+            ttl_hours: Time-to-live in hours before entries expire
+            max_size: Maximum number of entries to store
+        """
+        self._cache: dict[str, tuple[T, datetime]] = {}
+        self._ttl = timedelta(hours=ttl_hours)
+        self._max_size = max_size
+
+    def get(self, key: str) -> T | None:
+        """Get a value from the cache.
+
+        Args:
+            key: Cache key
+
+        Returns:
+            Cached value if present and not expired, None otherwise
+        """
+        if key not in self._cache:
+            return None
+
+        value, cached_at = self._cache[key]
+        if datetime.now() - cached_at >= self._ttl:
+            # Expired - remove and return None
+            del self._cache[key]
+            logger.debug("Cache entry expired", key=key)
+            return None
+
+        logger.debug("Cache hit", key=key)
+        return value
+
+    def set(self, key: str, value: T) -> None:
+        """Store a value in the cache.
+
+        Args:
+            key: Cache key
+            value: Value to cache
+        """
+        # Evict oldest entries if at max size
+        if len(self._cache) >= self._max_size and key not in self._cache:
+            self._evict_oldest()
+
+        self._cache[key] = (value, datetime.now())
+        logger.debug("Cache set", key=key)
+
+    def invalidate(self, key: str) -> None:
+        """Remove a specific entry from the cache.
+
+        Args:
+            key: Cache key to remove
+        """
+        if key in self._cache:
+            del self._cache[key]
+            logger.debug("Cache entry invalidated", key=key)
+
+    def clear(self) -> None:
+        """Remove all entries from the cache."""
+        count = len(self._cache)
+        self._cache.clear()
+        logger.info("Cache cleared", entries_removed=count)
+
+    def _evict_oldest(self) -> None:
+        """Remove the oldest entry from the cache."""
+        if not self._cache:
+            return
+
+        oldest_key = min(self._cache, key=lambda k: self._cache[k][1])
+        del self._cache[oldest_key]
+        logger.debug("Cache evicted oldest entry", key=oldest_key)
+
+    @property
+    def size(self) -> int:
+        """Current number of entries in the cache."""
+        return len(self._cache)
+
+    def stats(self) -> dict[str, int | float]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with cache stats (size, ttl_hours, max_size)
+        """
+        now = datetime.now()
+        valid_count = sum(1 for _, (_, ts) in self._cache.items() if now - ts < self._ttl)
+        return {
+            "size": len(self._cache),
+            "valid_entries": valid_count,
+            "expired_entries": len(self._cache) - valid_count,
+            "ttl_hours": self._ttl.total_seconds() / 3600,
+            "max_size": self._max_size,
+        }

texas_grocery_mcp/reliability/circuit_breaker.py

@@ -0,0 +1,138 @@
+"""Circuit breaker pattern for preventing cascading failures."""
+
+import time
+from dataclasses import dataclass
+from enum import Enum
+from threading import Lock
+
+import structlog
+
+logger = structlog.get_logger()
+
+
+class CircuitState(Enum):
+    """Circuit breaker states."""
+
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+class CircuitBreakerOpenError(Exception):
+    """Raised when circuit breaker is open."""
+
+    def __init__(self, name: str, retry_after: float):
+        self.name = name
+        self.retry_after = retry_after
+        super().__init__(f"Circuit breaker '{name}' is open. Retry after {retry_after:.1f}s")
+
+
+@dataclass
+class CircuitBreakerConfig:
+    """Configuration for circuit breaker."""
+
+    failure_threshold: int = 5
+    recovery_timeout: float = 30.0
+    half_open_max_calls: int = 3
+
+
+class CircuitBreaker:
+    """Circuit breaker for external service calls.
+
+    Prevents cascading failures by "tripping" after repeated failures
+    and allowing the system to recover.
+    """
+
+    def __init__(self, name: str, config: CircuitBreakerConfig | None = None):
+        self.name = name
+        self.config = config or CircuitBreakerConfig()
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._success_count = 0
+        self._last_failure_time: float | None = None
+        self._half_open_calls = 0
+        self._lock = Lock()
+
+    @property
+    def state(self) -> CircuitState:
+        """Get current circuit state, checking for transition to half-open."""
+        with self._lock:
+            if self._state == CircuitState.OPEN and self._should_attempt_recovery():
+                self._state = CircuitState.HALF_OPEN
+                self._half_open_calls = 0
+                logger.info("Circuit breaker transitioning to half-open", name=self.name)
+            return self._state
+
+    @property
+    def failure_count(self) -> int:
+        """Get current failure count."""
+        return self._failure_count
+
+    def _should_attempt_recovery(self) -> bool:
+        """Check if enough time has passed to attempt recovery."""
+        if self._last_failure_time is None:
+            return False
+        return time.time() - self._last_failure_time >= self.config.recovery_timeout
+
+    def check(self) -> None:
+        """Check if request should be allowed.
+
+        Raises:
+            CircuitBreakerOpen: If circuit is open and recovery timeout hasn't elapsed.
+        """
+        current_state = self.state  # This may transition to half-open
+
+        if current_state == CircuitState.OPEN:
+            retry_after = (
+                self.config.recovery_timeout
+                - (time.time() - (self._last_failure_time or 0))
+            )
+            raise CircuitBreakerOpenError(self.name, max(0, retry_after))
+
+        if current_state == CircuitState.HALF_OPEN:
+            with self._lock:
+                if self._half_open_calls >= self.config.half_open_max_calls:
+                    raise CircuitBreakerOpenError(self.name, self.config.recovery_timeout)
+                self._half_open_calls += 1
+
+    def record_success(self) -> None:
+        """Record a successful call."""
+        with self._lock:
+            if self._state == CircuitState.HALF_OPEN:
+                self._success_count += 1
+                if self._success_count >= self.config.half_open_max_calls:
+                    self._state = CircuitState.CLOSED
+                    self._failure_count = 0
+                    self._success_count = 0
+                    logger.info("Circuit breaker closed", name=self.name)
+            else:
+                self._failure_count = 0
+                self._success_count = 0
+
+    def record_failure(self) -> None:
+        """Record a failed call."""
+        with self._lock:
+            self._failure_count += 1
+            self._last_failure_time = time.time()
+
+            if self._state == CircuitState.HALF_OPEN:
+                self._state = CircuitState.OPEN
+                logger.warning(
+                    "Circuit breaker reopened from half-open",
+                    name=self.name,
+                )
+            elif self._failure_count >= self.config.failure_threshold:
+                self._state = CircuitState.OPEN
+                logger.warning(
+                    "Circuit breaker opened",
+                    name=self.name,
+                    failure_count=self._failure_count,
+                )
+
+    def get_status(self) -> dict[str, str | int]:
+        """Get circuit breaker status for health checks."""
+        return {
+            "name": self.name,
+            "state": self.state.value,
+            "failure_count": self._failure_count,
+        }

texas_grocery_mcp/reliability/retry.py

@@ -0,0 +1,96 @@
+"""Retry logic with exponential backoff and jitter."""
+
+import asyncio
+import random
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from functools import wraps
+from typing import ParamSpec, TypeVar
+
+import structlog
+
+logger = structlog.get_logger()
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+# Exceptions that should trigger retry
+RETRYABLE_EXCEPTIONS: tuple[type[Exception], ...] = (
+    ConnectionError,
+    TimeoutError,
+    OSError,
+)
+
+
+@dataclass
+class RetryConfig:
+    """Configuration for retry behavior."""
+
+    max_attempts: int = 3
+    base_delay: float = 1.0
+    max_delay: float = 30.0
+    exponential_base: float = 2.0
+    jitter: bool = True
+    retryable_exceptions: tuple[type[Exception], ...] = field(
+        default_factory=lambda: RETRYABLE_EXCEPTIONS
+    )
+
+
+def calculate_delay(attempt: int, config: RetryConfig) -> float:
+    """Calculate delay for next retry attempt."""
+    delay = config.base_delay * (config.exponential_base ** (attempt - 1))
+    delay = min(delay, config.max_delay)
+
+    if config.jitter:
+        delay = delay * (0.5 + random.random())
+
+    return delay
+
+
+def with_retry(
+    config: RetryConfig | None = None,
+) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]:
+    """Decorator for async functions with retry logic.
+
+    Args:
+        config: Retry configuration. Uses defaults if not provided.
+
+    Returns:
+        Decorated function that retries on transient failures.
+    """
+    if config is None:
+        config = RetryConfig()
+
+    def decorator(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
+        @wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            last_exception: Exception | None = None
+
+            for attempt in range(1, config.max_attempts + 1):
+                try:
+                    return await func(*args, **kwargs)
+                except config.retryable_exceptions as e:
+                    last_exception = e
+                    if attempt < config.max_attempts:
+                        delay = calculate_delay(attempt, config)
+                        logger.warning(
+                            "Retry attempt",
+                            function=func.__name__,
+                            attempt=attempt,
+                            max_attempts=config.max_attempts,
+                            delay=delay,
+                            error=str(e),
+                        )
+                        await asyncio.sleep(delay)
+                except Exception:
+                    # Non-retryable exception, raise immediately
+                    raise
+
+            # Exhausted all retries
+            if last_exception:
+                raise last_exception
+            raise RuntimeError("Unexpected retry state")
+
+        return wrapper
+
+    return decorator

texas_grocery_mcp/reliability/throttle.py

@@ -0,0 +1,113 @@
+"""Request throttling to prevent rate limiting."""
+
+import asyncio
+import random
+import time
+from dataclasses import dataclass
+from types import TracebackType
+
+import structlog
+
+logger = structlog.get_logger()
+
+
+@dataclass
+class ThrottleConfig:
+    """Configuration for request throttling."""
+
+    max_concurrent: int = 3
+    """Maximum concurrent requests allowed."""
+
+    min_delay_ms: int = 200
+    """Minimum delay between requests in milliseconds."""
+
+    jitter_ms: int = 200
+    """Random jitter added to delay (0 to jitter_ms)."""
+
+    enabled: bool = True
+    """Whether throttling is enabled."""
+
+
+class Throttler:
+    """
+    Semaphore-based request throttler with delay and jitter.
+
+    Limits concurrent requests and enforces minimum delays between requests
+    to prevent overwhelming external APIs with burst traffic.
+
+    Usage:
+        throttler = Throttler(ThrottleConfig(max_concurrent=3))
+
+        async with throttler:
+            await make_request()
+    """
+
+    def __init__(self, config: ThrottleConfig, name: str = "default"):
+        """Initialize throttler with configuration.
+
+        Args:
+            config: Throttling configuration
+            name: Name for logging purposes
+        """
+        self._config = config
+        self._name = name
+        self._semaphore = asyncio.Semaphore(config.max_concurrent)
+        self._last_request_time: float = 0
+        self._lock = asyncio.Lock()
+
+    @property
+    def config(self) -> ThrottleConfig:
+        """Get the throttle configuration."""
+        return self._config
+
+    async def __aenter__(self) -> "Throttler":
+        """Acquire throttle slot, waiting if necessary."""
+        if not self._config.enabled:
+            return self
+
+        await self._semaphore.acquire()
+
+        try:
+            await self._wait_for_delay()
+        except Exception:
+            self._semaphore.release()
+            raise
+
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Release throttle slot and record request time."""
+        if not self._config.enabled:
+            return
+
+        async with self._lock:
+            self._last_request_time = time.monotonic()
+
+        self._semaphore.release()
+
+    async def _wait_for_delay(self) -> None:
+        """Wait for minimum delay + jitter since last request."""
+        async with self._lock:
+            now = time.monotonic()
+            min_delay = self._config.min_delay_ms / 1000.0
+            elapsed = now - self._last_request_time
+
+            wait_time = min_delay - elapsed if elapsed < min_delay else 0
+
+            # Add jitter to make traffic pattern less predictable
+            if self._config.jitter_ms > 0:
+                jitter = random.uniform(0, self._config.jitter_ms / 1000.0)
+                wait_time += jitter
+
+        if wait_time > 0:
+            logger.debug(
+                "Throttling request",
+                throttler=self._name,
+                wait_ms=int(wait_time * 1000),
+            )
+            await asyncio.sleep(wait_time)