fastapi-loopguard 0.3.1__py3-none-any.whl

fastapi_loopguard/__init__.py

@@ -0,0 +1,62 @@
+"""FastAPI LoopGuard - Detect event-loop blocking with per-request attribution.
+
+Usage:
+    from fastapi import FastAPI
+    from fastapi_loopguard import LoopGuardMiddleware, LoopGuardConfig
+
+    app = FastAPI()
+
+    # Basic usage with defaults
+    app.add_middleware(LoopGuardMiddleware)
+
+    # Or with custom config
+    config = LoopGuardConfig(
+        dev_mode=True,  # Enable X-Blocking-* headers
+        prometheus_enabled=True,  # Enable Prometheus metrics
+    )
+    app.add_middleware(LoopGuardMiddleware, config=config)
+
+v0.2.0 Changes:
+    - Pure ASGI middleware (no BaseHTTPMiddleware)
+    - Concurrent request tracking with RequestRegistry
+    - Background calibration (first request not blocked)
+    - Proper lifecycle management via ASGI lifespan
+
+v0.3.0 Changes:
+    - PEP 561 py.typed marker for type stub discovery
+    - Adaptive thresholds for high-concurrency environments
+    - Improved test coverage (logging, metrics, pytest plugin)
+    - High-concurrency configuration documentation
+"""
+
+from .config import LoopGuardConfig
+from .context import (
+    RequestContext,
+    RequestRegistry,
+    get_active_requests,
+    get_current_request,
+    get_registry,
+    register_request,
+    unregister_request,
+)
+from .middleware import LoopGuardMiddleware
+from .monitor import SentinelMonitor
+
+__version__ = "0.3.0"
+
+__all__ = [
+    # Core classes
+    "LoopGuardConfig",
+    "LoopGuardMiddleware",
+    "SentinelMonitor",
+    # Context tracking
+    "RequestContext",
+    "RequestRegistry",
+    "get_registry",
+    "register_request",
+    "unregister_request",
+    "get_active_requests",
+    "get_current_request",  # Backward compat
+    # Version
+    "__version__",
+]

fastapi_loopguard/config.py

@@ -0,0 +1,88 @@
+"""Configuration for LoopGuard middleware."""
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class LoopGuardConfig:
+    """Configuration for the LoopGuard middleware.
+
+    Attributes:
+        enabled: Whether monitoring is active. Set False to disable entirely.
+        monitor_interval_ms: How often the sentinel checks for blocking (milliseconds).
+            Default 10ms detects blocking >50ms with ~0.002% CPU overhead.
+            The sleep is non-blocking (cooperative), so it doesn't affect throughput.
+        threshold_multiplier: Blocking detected when lag > baseline × multiplier.
+        calibration_iterations: Number of samples during startup calibration.
+        fallback_threshold_ms: Used if calibration produces unreliable results.
+        dev_mode: Enable response headers with lag information.
+        log_blocking_events: Log when blocking is detected.
+        prometheus_enabled: Expose Prometheus metrics.
+        adaptive_threshold: Enable adaptive threshold based on sliding window.
+        adaptive_window_size: Number of samples in the sliding window.
+        adaptive_percentile: Percentile (0.0-1.0) for baseline calculation.
+        adaptive_min_samples: Minimum samples before adaptive mode activates.
+        adaptive_update_interval_ms: How often to recalculate threshold.
+    """
+
+    enabled: bool = True
+    monitor_interval_ms: float = 10.0
+    threshold_multiplier: float = 5.0
+    calibration_iterations: int = 100
+    fallback_threshold_ms: float = 50.0
+    dev_mode: bool = False
+    log_blocking_events: bool = True
+    prometheus_enabled: bool = False
+
+    # Adaptive threshold settings
+    adaptive_threshold: bool = False
+    adaptive_window_size: int = 1000
+    adaptive_percentile: float = 0.95
+    adaptive_min_samples: int = 100
+    adaptive_update_interval_ms: float = 1000.0
+
+    # Cumulative blocking detection
+    cumulative_blocking_enabled: bool = False
+    cumulative_blocking_threshold_ms: float = 200.0
+    cumulative_window_ms: float = 1000.0
+
+    # Internal: paths to exclude from monitoring (e.g., health checks)
+    exclude_paths: frozenset[str] = field(
+        default_factory=lambda: frozenset({"/health", "/healthz", "/ready", "/metrics"})
+    )
+
+    def __post_init__(self) -> None:
+        """Validate configuration values."""
+        if self.monitor_interval_ms <= 0:
+            raise ValueError("monitor_interval_ms must be positive")
+        if self.threshold_multiplier <= 1:
+            raise ValueError("threshold_multiplier must be greater than 1")
+        if self.calibration_iterations < 10:
+            raise ValueError("calibration_iterations must be at least 10")
+        if self.fallback_threshold_ms <= 0:
+            raise ValueError("fallback_threshold_ms must be positive")
+        # Adaptive threshold validation
+        if self.adaptive_window_size < 100:
+            raise ValueError("adaptive_window_size must be at least 100")
+        if not 0.5 <= self.adaptive_percentile <= 0.99:
+            raise ValueError("adaptive_percentile must be between 0.5 and 0.99")
+        if self.adaptive_min_samples < 10:
+            raise ValueError("adaptive_min_samples must be at least 10")
+        if self.adaptive_min_samples > self.adaptive_window_size:
+            raise ValueError(
+                "adaptive_min_samples cannot be greater than adaptive_window_size"
+            )
+        if self.adaptive_update_interval_ms <= 0:
+            raise ValueError("adaptive_update_interval_ms must be positive")
+        # Cumulative blocking validation
+        if self.cumulative_blocking_threshold_ms <= 0:
+            raise ValueError("cumulative_blocking_threshold_ms must be positive")
+        if self.cumulative_window_ms <= 0:
+            raise ValueError("cumulative_window_ms must be positive")
+        if (
+            self.cumulative_blocking_enabled
+            and self.cumulative_window_ms < self.monitor_interval_ms
+        ):
+            raise ValueError(
+                "cumulative_window_ms cannot be less than monitor_interval_ms"
+            )

fastapi_loopguard/context.py

@@ -0,0 +1,210 @@
+"""Context tracking for per-request attribution.
+
+This module provides a registry-based approach for tracking multiple concurrent
+requests. Unlike the previous single-slot design, this handles concurrent requests
+correctly by storing contexts in a dict keyed by request_id.
+
+Since asyncio is single-threaded, no locks are needed for the registry operations.
+The monitor iterates all active contexts when blocking is detected.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class RequestContext:
+    """Context information for a single request.
+
+    Attributes:
+        request_id: Unique identifier for this request.
+        path: The request path (e.g., "/api/users").
+        method: HTTP method (GET, POST, etc.).
+        start_time: Monotonic timestamp when request started.
+        blocking_events: List of (lag_ms, timestamp) tuples for blocking detected.
+    """
+
+    request_id: str
+    path: str
+    method: str
+    start_time: float = field(default_factory=time.monotonic)
+    blocking_events: list[tuple[float, float]] = field(default_factory=list)
+
+    def record_blocking(self, lag_ms: float) -> None:
+        """Record a blocking event for this request."""
+        self.blocking_events.append((lag_ms, time.monotonic()))
+
+    @property
+    def total_blocking_ms(self) -> float:
+        """Sum of all blocking event durations."""
+        return sum(lag for lag, _ in self.blocking_events)
+
+    @property
+    def blocking_count(self) -> int:
+        """Number of blocking events detected."""
+        return len(self.blocking_events)
+
+
+class RequestRegistry:
+    """Registry of active request contexts.
+
+    This class manages multiple concurrent request contexts, storing them
+    in a dict keyed by request_id. This allows the monitor to correctly
+    attribute blocking events to all active requests.
+
+    No locks are needed since asyncio is single-threaded - all operations
+    happen on the same thread within the event loop.
+    """
+
+    __slots__ = ("_contexts",)
+
+    def __init__(self) -> None:
+        """Initialize an empty registry."""
+        self._contexts: dict[str, RequestContext] = {}
+
+    def register(self, ctx: RequestContext) -> None:
+        """Register a new active request context.
+
+        Args:
+            ctx: The request context to register.
+        """
+        self._contexts[ctx.request_id] = ctx
+
+    def unregister(self, request_id: str) -> RequestContext | None:
+        """Remove a request context when the request completes.
+
+        Args:
+            request_id: The ID of the request to unregister.
+
+        Returns:
+            The removed context, or None if not found.
+        """
+        return self._contexts.pop(request_id, None)
+
+    def get(self, request_id: str) -> RequestContext | None:
+        """Get a specific request context by ID.
+
+        Args:
+            request_id: The ID of the request to retrieve.
+
+        Returns:
+            The request context, or None if not found.
+        """
+        return self._contexts.get(request_id)
+
+    def get_all_active(self) -> Iterator[RequestContext]:
+        """Iterate all currently active request contexts.
+
+        This is used by the monitor to attribute blocking events
+        to all requests that were active when blocking occurred.
+
+        Yields:
+            Each active RequestContext.
+        """
+        yield from self._contexts.values()
+
+    def active_count(self) -> int:
+        """Get the number of currently active requests.
+
+        Returns:
+            The count of active requests.
+        """
+        return len(self._contexts)
+
+    def clear(self) -> None:
+        """Clear all contexts.
+
+        Used for testing and shutdown cleanup.
+        """
+        self._contexts.clear()
+
+
+# Global registry instance
+_registry = RequestRegistry()
+
+
+def get_registry() -> RequestRegistry:
+    """Get the global request registry.
+
+    Returns:
+        The global RequestRegistry instance.
+    """
+    return _registry
+
+
+def register_request(ctx: RequestContext) -> None:
+    """Register a request context in the global registry.
+
+    Args:
+        ctx: The request context to register.
+    """
+    _registry.register(ctx)
+
+
+def unregister_request(request_id: str) -> RequestContext | None:
+    """Unregister a request context from the global registry.
+
+    Args:
+        request_id: The ID of the request to unregister.
+
+    Returns:
+        The removed context, or None if not found.
+    """
+    return _registry.unregister(request_id)
+
+
+def get_active_requests() -> Iterator[RequestContext]:
+    """Get all active request contexts.
+
+    Yields:
+        Each active RequestContext.
+    """
+    return _registry.get_all_active()
+
+
+# Backward compatibility aliases
+# These maintain the old API but now work correctly with concurrent requests
+
+
+def get_current_request() -> RequestContext | None:
+    """Get any active request context.
+
+    Note: With multiple concurrent requests, this returns an arbitrary
+    active request. For correct attribution, use get_active_requests()
+    to iterate all active contexts.
+
+    Returns:
+        An active RequestContext, or None if no requests are active.
+    """
+    for ctx in _registry.get_all_active():
+        return ctx
+    return None
+
+
+def set_current_request(ctx: RequestContext) -> str:
+    """Register a request context and return its ID for cleanup.
+
+    This is a compatibility wrapper around register_request().
+
+    Args:
+        ctx: The request context to register.
+
+    Returns:
+        The request_id (used as token for reset_current_request).
+    """
+    _registry.register(ctx)
+    return ctx.request_id
+
+
+def reset_current_request(request_id: str) -> None:
+    """Unregister a request context by its ID.
+
+    This is a compatibility wrapper around unregister_request().
+
+    Args:
+        request_id: The request ID returned by set_current_request.
+    """
+    _registry.unregister(request_id)

fastapi_loopguard/logging.py

@@ -0,0 +1,100 @@
+"""Structured logging for LoopGuard events."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import UTC, datetime
+from typing import Any
+
+logger = logging.getLogger("fastapi_loopguard")
+
+
+class StructuredFormatter(logging.Formatter):
+    """JSON formatter for structured logging."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format a log record as JSON."""
+        log_data: dict[str, Any] = {
+            "timestamp": datetime.now(UTC).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+
+        # Add extra fields if present
+        if hasattr(record, "path"):
+            log_data["path"] = record.path
+        if hasattr(record, "method"):
+            log_data["method"] = record.method
+        if hasattr(record, "lag_ms"):
+            log_data["lag_ms"] = record.lag_ms
+        if hasattr(record, "request_id"):
+            log_data["request_id"] = record.request_id
+        if hasattr(record, "blocking_count"):
+            log_data["blocking_count"] = record.blocking_count
+
+        return json.dumps(log_data)
+
+
+def configure_logging(
+    level: int = logging.INFO,
+    structured: bool = False,
+    stream: Any = None,
+) -> None:
+    """Configure logging for LoopGuard.
+
+    Args:
+        level: The logging level (default INFO).
+        structured: If True, use JSON formatting.
+        stream: Output stream (default stderr).
+    """
+    handler = logging.StreamHandler(stream or sys.stderr)
+
+    if structured:
+        handler.setFormatter(StructuredFormatter())
+    else:
+        handler.setFormatter(
+            logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        )
+
+    logger.addHandler(handler)
+    logger.setLevel(level)
+
+
+def log_blocking_event(
+    lag_ms: float,
+    path: str | None = None,
+    method: str | None = None,
+    request_id: str | None = None,
+) -> None:
+    """Log a blocking event with structured data.
+
+    Args:
+        lag_ms: The blocking duration in milliseconds.
+        path: The request path (if available).
+        method: The HTTP method (if available).
+        request_id: The request ID (if available).
+    """
+    extra = {
+        "lag_ms": lag_ms,
+        "path": path,
+        "method": method,
+        "request_id": request_id,
+    }
+
+    if path:
+        logger.warning(
+            "Event loop blocked for %.2fms during %s %s",
+            lag_ms,
+            method,
+            path,
+            extra=extra,
+        )
+    else:
+        logger.warning(
+            "Event loop blocked for %.2fms (no active request)",
+            lag_ms,
+            extra=extra,
+        )

fastapi_loopguard/metrics.py

@@ -0,0 +1,204 @@
+"""Prometheus metrics for LoopGuard.
+
+Supports custom registries for test isolation via the registry parameter.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def _get_prometheus() -> tuple[type, type, type, Any] | None:
+    """Try to import prometheus_client with registry support."""
+    try:
+        from prometheus_client import REGISTRY, Counter, Gauge, Histogram
+
+        return Counter, Histogram, Gauge, REGISTRY
+    except ImportError:
+        return None
+
+
+class LoopGuardMetrics:
+    """Prometheus metrics for loop-lag monitoring.
+
+    Metrics exposed:
+        - loopguard_blocking_total: Counter of blocking events
+        - loopguard_lag_seconds: Histogram of lag durations
+        - loopguard_requests_monitored_total: Counter of monitored requests
+        - loopguard_threshold_seconds: Gauge of current threshold
+
+    Supports custom registries for test isolation.
+    """
+
+    __slots__ = (
+        "_prefix",
+        "_registry",
+        "_blocking_total",
+        "_lag_histogram",
+        "_requests_total",
+        "_threshold_gauge",
+    )
+
+    def __init__(
+        self,
+        prefix: str = "loopguard",
+        registry: Any = None,
+    ) -> None:
+        """Initialize metrics.
+
+        Args:
+            prefix: Prefix for all metric names.
+            registry: Prometheus registry to use. If None, uses default REGISTRY.
+
+        Raises:
+            RuntimeError: If prometheus_client is not installed.
+        """
+        prometheus = _get_prometheus()
+        if prometheus is None:
+            raise RuntimeError(
+                "prometheus_client is not installed. "
+                "Install with: pip install fastapi-loopguard[prometheus]"
+            )
+
+        counter_cls, histogram_cls, gauge_cls, default_registry = prometheus
+        self._prefix = prefix
+        self._registry = registry if registry is not None else default_registry
+
+        # Create metrics with explicit registry
+        self._blocking_total: Any = counter_cls(
+            f"{prefix}_blocking_total",
+            "Total number of blocking events detected",
+            ["path", "method"],
+            registry=self._registry,
+        )
+
+        self._lag_histogram: Any = histogram_cls(
+            f"{prefix}_lag_seconds",
+            "Histogram of event loop lag durations",
+            ["path", "method"],
+            buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0),
+            registry=self._registry,
+        )
+
+        self._requests_total: Any = counter_cls(
+            f"{prefix}_requests_monitored_total",
+            "Total number of requests monitored",
+            ["path", "method"],
+            registry=self._registry,
+        )
+
+        self._threshold_gauge: Any = gauge_cls(
+            f"{prefix}_threshold_seconds",
+            "Current blocking detection threshold",
+            registry=self._registry,
+        )
+
+    @property
+    def prefix(self) -> str:
+        """The metric name prefix."""
+        return self._prefix
+
+    def record_blocking(
+        self,
+        lag_seconds: float,
+        path: str | None = None,
+        method: str | None = None,
+    ) -> None:
+        """Record a blocking event.
+
+        Args:
+            lag_seconds: The blocking duration in seconds.
+            path: The request path.
+            method: The HTTP method.
+        """
+        labels = {
+            "path": path or "unknown",
+            "method": method or "unknown",
+        }
+        self._blocking_total.labels(**labels).inc()
+        self._lag_histogram.labels(**labels).observe(lag_seconds)
+
+    def record_request(self, path: str, method: str) -> None:
+        """Record a monitored request.
+
+        Args:
+            path: The request path.
+            method: The HTTP method.
+        """
+        self._requests_total.labels(path=path, method=method).inc()
+
+    def set_threshold(self, threshold_seconds: float) -> None:
+        """Set the current threshold gauge.
+
+        Args:
+            threshold_seconds: The current threshold in seconds.
+        """
+        self._threshold_gauge.set(threshold_seconds)
+
+
+# Instance management - use regular dict since __slots__ prevents weak refs
+_instances: dict[str, LoopGuardMetrics] = {}
+
+
+def get_metrics(prefix: str = "loopguard") -> LoopGuardMetrics | None:
+    """Get existing metrics instance by prefix.
+
+    Args:
+        prefix: The prefix used when creating the metrics.
+
+    Returns:
+        The metrics instance, or None if not found.
+    """
+    return _instances.get(prefix)
+
+
+def create_metrics(
+    prefix: str = "loopguard",
+    registry: Any = None,
+) -> LoopGuardMetrics:
+    """Create or get a metrics instance.
+
+    For testing, pass a custom registry to avoid pollution.
+
+    Args:
+        prefix: Prefix for all metric names.
+        registry: Optional Prometheus registry for test isolation.
+
+    Returns:
+        The metrics instance.
+    """
+    # Use registry id to allow different registries with same prefix
+    key = f"{prefix}:{id(registry)}"
+    if key not in _instances:
+        metrics = LoopGuardMetrics(prefix, registry)
+        _instances[key] = metrics
+    return _instances[key]
+
+
+def reset_metrics() -> None:
+    """Reset all metrics instances.
+
+    For testing only - clears the instance cache.
+    """
+    _instances.clear()
+
+
+# Backward compatibility aliases
+_metrics_instance: LoopGuardMetrics | None = None
+
+
+def init_metrics(prefix: str = "loopguard") -> LoopGuardMetrics:
+    """Initialize the global metrics instance.
+
+    Deprecated: Use create_metrics() for new code.
+
+    Args:
+        prefix: Prefix for all metric names.
+
+    Returns:
+        The initialized metrics instance.
+    """
+    global _metrics_instance
+    if _metrics_instance is None:
+        _metrics_instance = create_metrics(prefix)
+    return _metrics_instance