provide-foundation 0.0.0.dev0__py3-none-any.whl

provide/foundation/logger/ratelimit/processor.py

@@ -0,0 +1,203 @@
+#
+# processor.py
+#
+"""
+Structlog processor for rate limiting log messages.
+"""
+
+import time
+from typing import Any
+
+import structlog
+
+from provide.foundation.logger.ratelimit.limiters import GlobalRateLimiter
+
+
+class RateLimiterProcessor:
+    """
+    Structlog processor that applies rate limiting to log messages.
+    Can be configured with global and per-logger rate limits.
+    """
+
+    def __init__(
+        self,
+        emit_warning_on_limit: bool = True,
+        warning_interval_seconds: float = 60.0,
+        summary_interval_seconds: float = 5.0,
+    ):
+        """
+        Initialize the rate limiter processor.
+
+        Args:
+            emit_warning_on_limit: Whether to emit a warning when rate limited
+            warning_interval_seconds: Minimum seconds between rate limit warnings
+            summary_interval_seconds: Interval for rate limit summary reports
+        """
+        self.rate_limiter = GlobalRateLimiter()
+        self.emit_warning_on_limit = emit_warning_on_limit
+        self.warning_interval_seconds = warning_interval_seconds
+
+        # Track last warning time per logger
+        self.last_warning_times: dict[str, float] = {}
+
+        # Track suppressed message counts
+        self.suppressed_counts: dict[str, int] = {}
+        self.last_summary_time = time.monotonic()
+        self.summary_interval = summary_interval_seconds  # Emit summary periodically
+
+    def __call__(
+        self, logger: Any, method_name: str, event_dict: structlog.types.EventDict
+    ) -> structlog.types.EventDict:
+        """
+        Process a log event, applying rate limiting.
+
+        Args:
+            logger: The logger instance
+            method_name: The log method name (debug, info, etc.)
+            event_dict: The event dictionary
+
+        Returns:
+            The event dictionary if allowed, or raises DropEvent if rate limited
+        """
+        logger_name = event_dict.get("logger_name", "unknown")
+
+        # Check if this log is allowed (pass event_dict for tracking)
+        allowed, reason = self.rate_limiter.is_allowed(logger_name, event_dict)
+
+        if not allowed:
+            # Track suppressed count
+            if logger_name not in self.suppressed_counts:
+                self.suppressed_counts[logger_name] = 0
+            self.suppressed_counts[logger_name] += 1
+
+            # Optionally emit a warning about rate limiting
+            if self.emit_warning_on_limit:
+                now = time.monotonic()
+                last_warning = self.last_warning_times.get(logger_name, 0)
+
+                if now - last_warning >= self.warning_interval_seconds:
+                    # Create a rate limit warning event
+                    self.last_warning_times[logger_name] = now
+
+                    # Return a modified event indicating rate limiting
+                    return {
+                        "event": f"⚠️ Rate limit: {reason}",
+                        "level": "warning",
+                        "logger_name": "provide.foundation.ratelimit",
+                        "suppressed_count": self.suppressed_counts[logger_name],
+                        "original_logger": logger_name,
+                        "_rate_limit_warning": True,
+                    }
+
+            # Drop the event
+            raise structlog.DropEvent
+
+        # Check if we should emit a summary
+        now = time.monotonic()
+        if now - self.last_summary_time >= self.summary_interval:
+            # Always check and emit summary if there's been any rate limiting
+            self._emit_summary()
+            self.last_summary_time = now
+
+        return event_dict
+
+    def _emit_summary(self):
+        """Emit a summary of rate-limited messages."""
+        # Get current stats first to check if any rate limiting has occurred
+        stats = self.rate_limiter.get_stats()
+
+        # Check if there's been any rate limiting activity
+        global_stats = stats.get("global", {})
+        total_denied = global_stats.get("total_denied", 0)
+
+        if not self.suppressed_counts and total_denied == 0:
+            return  # No rate limiting activity to report
+
+        total_suppressed = sum(self.suppressed_counts.values())
+
+        # Get a logger for rate limit summaries
+        try:
+            from provide.foundation.logger import get_logger
+
+            summary_logger = get_logger("provide.foundation.ratelimit.summary")
+
+            # Calculate rate limiting percentage
+            total_allowed = global_stats.get("total_allowed", 0)
+            total_attempts = total_allowed + total_denied
+            if total_attempts > 0:
+                denial_rate = (total_denied / total_attempts) * 100
+            else:
+                denial_rate = 0
+
+            # Format the summary message
+            summary_logger.warning(
+                f"⚠️ Rate limiting active: {total_suppressed:,} logs dropped in last {self.summary_interval}s | "
+                f"Denial rate: {denial_rate:.1f}% | "
+                f"Tokens: {global_stats.get('tokens_available', 0):.0f}/{global_stats.get('capacity', 0):.0f}",
+                suppressed_by_logger=dict(self.suppressed_counts)
+                if self.suppressed_counts
+                else {},
+                total_suppressed=total_suppressed,
+                total_denied_overall=total_denied,
+                total_allowed_overall=total_allowed,
+                denial_rate_percent=denial_rate,
+                tokens_available=global_stats.get("tokens_available", 0),
+                capacity=global_stats.get("capacity", 0),
+                refill_rate=global_stats.get("refill_rate", 0),
+            )
+
+            # Reset counts after summary
+            self.suppressed_counts.clear()
+        except Exception:
+            # If we can't log the summary, just clear counts
+            self.suppressed_counts.clear()
+
+
+def create_rate_limiter_processor(
+    global_rate: float | None = None,
+    global_capacity: float | None = None,
+    per_logger_rates: dict[str, tuple[float, float]] | None = None,
+    emit_warnings: bool = True,
+    summary_interval: float = 5.0,
+    max_queue_size: int = 1000,
+    max_memory_mb: float | None = None,
+    overflow_policy: str = "drop_oldest",
+) -> RateLimiterProcessor:
+    """
+    Factory function to create and configure a rate limiter processor.
+
+    Args:
+        global_rate: Global logs per second limit
+        global_capacity: Global burst capacity
+        per_logger_rates: Dict of logger_name -> (rate, capacity) tuples
+        emit_warnings: Whether to emit warnings when rate limited
+        summary_interval: Seconds between rate limit summary reports
+        max_queue_size: Maximum queue size when buffering
+        max_memory_mb: Maximum memory for buffered logs
+        overflow_policy: Policy when queue is full
+
+    Returns:
+        Configured RateLimiterProcessor instance
+    """
+    processor = RateLimiterProcessor(
+        emit_warning_on_limit=emit_warnings, summary_interval_seconds=summary_interval
+    )
+
+    # Determine if we should use buffered rate limiting
+    use_buffered = max_queue_size > 0 and overflow_policy in (
+        "drop_oldest",
+        "drop_newest",
+    )
+
+    # Configure the global rate limiter
+    processor.rate_limiter.configure(
+        global_rate=global_rate,
+        global_capacity=global_capacity,
+        per_logger_rates=per_logger_rates,
+        use_buffered=use_buffered,
+        max_queue_size=max_queue_size,
+        max_memory_mb=max_memory_mb,
+        overflow_policy=overflow_policy,
+    )
+
+    return processor

provide/foundation/logger/ratelimit/queue_limiter.py

@@ -0,0 +1,305 @@
+#
+# queue_limiter.py
+#
+"""
+Queue-based rate limiter with overflow protection for Foundation's logging system.
+"""
+
+from collections import deque
+import sys
+import threading
+import time
+from typing import Any, Literal
+
+
+class QueuedRateLimiter:
+    """
+    Rate limiter with a queue for buffering logs.
+    Drops oldest messages when queue is full (FIFO overflow).
+    """
+
+    def __init__(
+        self,
+        capacity: float,
+        refill_rate: float,
+        max_queue_size: int = 1000,
+        max_memory_mb: float | None = None,
+        overflow_policy: Literal["drop_oldest", "drop_newest", "block"] = "drop_oldest",
+    ):
+        """
+        Initialize the queued rate limiter.
+
+        Args:
+            capacity: Maximum tokens (burst capacity)
+            refill_rate: Tokens per second
+            max_queue_size: Maximum number of items in queue
+            max_memory_mb: Maximum memory usage in MB (estimated)
+            overflow_policy: What to do when queue is full
+        """
+        if capacity <= 0:
+            raise ValueError("Capacity must be positive")
+        if refill_rate <= 0:
+            raise ValueError("Refill rate must be positive")
+        if max_queue_size <= 0:
+            raise ValueError("Max queue size must be positive")
+
+        self.capacity = float(capacity)
+        self.refill_rate = float(refill_rate)
+        self.tokens = float(capacity)
+        self.last_refill = time.monotonic()
+
+        # Queue management
+        self.max_queue_size = max_queue_size
+        self.max_memory_bytes = (
+            int(max_memory_mb * 1024 * 1024) if max_memory_mb else None
+        )
+        self.overflow_policy = overflow_policy
+
+        # Use deque for efficient FIFO operations
+        self.pending_queue = deque(
+            maxlen=max_queue_size if overflow_policy == "drop_oldest" else None
+        )
+        self.queue_lock = threading.Lock()
+
+        # Track statistics
+        self.total_queued = 0
+        self.total_dropped = 0
+        self.total_processed = 0
+        self.estimated_memory = 0
+
+        # Worker thread for processing queue
+        self.running = True
+        self.worker_thread = threading.Thread(target=self._process_queue, daemon=True)
+        self.worker_thread.start()
+
+    def _estimate_size(self, item: Any) -> int:
+        """Estimate memory size of an item."""
+        # Simple estimation - can be made more sophisticated
+        return sys.getsizeof(item)
+
+    def _refill_tokens(self) -> None:
+        """Refill tokens based on elapsed time."""
+        now = time.monotonic()
+        elapsed = now - self.last_refill
+
+        if elapsed > 0:
+            tokens_to_add = elapsed * self.refill_rate
+            self.tokens = min(self.capacity, self.tokens + tokens_to_add)
+            self.last_refill = now
+
+    def enqueue(self, item: Any) -> tuple[bool, str | None]:
+        """
+        Add item to queue for rate-limited processing.
+
+        Returns:
+            Tuple of (accepted, reason) where reason is set if rejected
+        """
+        with self.queue_lock:
+            # Check memory limit
+            if self.max_memory_bytes:
+                item_size = self._estimate_size(item)
+                if self.estimated_memory + item_size > self.max_memory_bytes:
+                    self.total_dropped += 1
+                    return (
+                        False,
+                        f"Memory limit exceeded ({self.estimated_memory / 1024 / 1024:.1f}MB)",
+                    )
+
+            # Check queue size
+            if len(self.pending_queue) >= self.max_queue_size:
+                if self.overflow_policy == "drop_newest":
+                    self.total_dropped += 1
+                    return False, f"Queue full ({self.max_queue_size} items)"
+                elif self.overflow_policy == "drop_oldest":
+                    # deque with maxlen automatically drops oldest
+                    if len(self.pending_queue) > 0:
+                        old_item = (
+                            self.pending_queue[0]
+                            if len(self.pending_queue) == self.max_queue_size
+                            else None
+                        )
+                        if old_item and self.max_memory_bytes:
+                            self.estimated_memory -= self._estimate_size(old_item)
+                        self.total_dropped += 1
+                elif self.overflow_policy == "block":
+                    # In block mode, we would need to wait
+                    # For now, just reject
+                    return False, "Queue full (blocking not implemented)"
+
+            # Add to queue
+            self.pending_queue.append(item)
+            self.total_queued += 1
+
+            if self.max_memory_bytes:
+                self.estimated_memory += self._estimate_size(item)
+
+            return True, None
+
+    def _process_queue(self):
+        """Worker thread that processes queued items."""
+        while self.running:
+            with self.queue_lock:
+                self._refill_tokens()
+
+                # Process items while we have tokens
+                while self.tokens >= 1.0 and self.pending_queue:
+                    item = self.pending_queue.popleft()
+                    self.tokens -= 1.0
+                    self.total_processed += 1
+
+                    if self.max_memory_bytes:
+                        self.estimated_memory -= self._estimate_size(item)
+
+                    # Here we would actually process the item
+                    # For logging, this would mean emitting the log
+                    self._process_item(item)
+
+            # Sleep briefly to avoid busy waiting
+            time.sleep(0.01)
+
+    def _process_item(self, item: Any):
+        """Process a single item from the queue."""
+        # This would be overridden to actually emit the log
+        pass
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get queue statistics."""
+        with self.queue_lock:
+            return {
+                "queue_size": len(self.pending_queue),
+                "max_queue_size": self.max_queue_size,
+                "tokens_available": self.tokens,
+                "capacity": self.capacity,
+                "refill_rate": self.refill_rate,
+                "total_queued": self.total_queued,
+                "total_dropped": self.total_dropped,
+                "total_processed": self.total_processed,
+                "estimated_memory_mb": self.estimated_memory / 1024 / 1024
+                if self.max_memory_bytes
+                else None,
+                "max_memory_mb": self.max_memory_bytes / 1024 / 1024
+                if self.max_memory_bytes
+                else None,
+                "overflow_policy": self.overflow_policy,
+            }
+
+    def shutdown(self):
+        """Shutdown the worker thread."""
+        self.running = False
+        if self.worker_thread.is_alive():
+            self.worker_thread.join(timeout=1.0)
+
+
+class BufferedRateLimiter:
+    """
+    Simple synchronous rate limiter with overflow buffer.
+    Does not use a worker thread - processes inline.
+    """
+
+    def __init__(
+        self,
+        capacity: float,
+        refill_rate: float,
+        buffer_size: int = 100,
+        track_dropped: bool = True,
+    ):
+        """
+        Initialize buffered rate limiter.
+
+        Args:
+            capacity: Maximum tokens (burst capacity)
+            refill_rate: Tokens per second
+            buffer_size: Number of recently dropped items to track
+            track_dropped: Whether to keep dropped items for debugging
+        """
+        if capacity <= 0:
+            raise ValueError("Capacity must be positive")
+        if refill_rate <= 0:
+            raise ValueError("Refill rate must be positive")
+
+        self.capacity = float(capacity)
+        self.refill_rate = float(refill_rate)
+        self.tokens = float(capacity)
+        self.last_refill = time.monotonic()
+        self.lock = threading.Lock()
+
+        # Track dropped items
+        self.buffer_size = buffer_size
+        self.track_dropped = track_dropped
+        self.dropped_buffer = deque(maxlen=buffer_size) if track_dropped else None
+
+        # Statistics
+        self.total_allowed = 0
+        self.total_denied = 0
+        self.total_bytes_dropped = 0
+
+    def is_allowed(self, item: Any | None = None) -> tuple[bool, str | None]:
+        """
+        Check if item is allowed based on rate limit.
+
+        Args:
+            item: Optional item to track if dropped
+
+        Returns:
+            Tuple of (allowed, reason)
+        """
+        with self.lock:
+            now = time.monotonic()
+            elapsed = now - self.last_refill
+
+            # Refill tokens
+            if elapsed > 0:
+                tokens_to_add = elapsed * self.refill_rate
+                self.tokens = min(self.capacity, self.tokens + tokens_to_add)
+                self.last_refill = now
+
+            # Try to consume token
+            if self.tokens >= 1.0:
+                self.tokens -= 1.0
+                self.total_allowed += 1
+                return True, None
+            else:
+                self.total_denied += 1
+
+                # Track dropped item
+                if self.track_dropped and item is not None:
+                    self.dropped_buffer.append(
+                        {
+                            "time": now,
+                            "item": item,
+                            "size": sys.getsizeof(item),
+                        }
+                    )
+                    self.total_bytes_dropped += sys.getsizeof(item)
+
+                return False, f"Rate limit exceeded (tokens: {self.tokens:.1f})"
+
+    def get_dropped_samples(self, count: int = 10) -> list[Any]:
+        """Get recent dropped items for debugging."""
+        if not self.track_dropped or not self.dropped_buffer:
+            return []
+
+        with self.lock:
+            return list(self.dropped_buffer)[-count:]
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get statistics."""
+        with self.lock:
+            stats = {
+                "tokens_available": self.tokens,
+                "capacity": self.capacity,
+                "refill_rate": self.refill_rate,
+                "total_allowed": self.total_allowed,
+                "total_denied": self.total_denied,
+                "total_bytes_dropped": self.total_bytes_dropped,
+            }
+
+            if self.track_dropped and self.dropped_buffer:
+                stats["dropped_buffer_size"] = len(self.dropped_buffer)
+                stats["oldest_dropped_age"] = (
+                    time.monotonic() - self.dropped_buffer[0]["time"]
+                    if self.dropped_buffer
+                    else 0
+                )
+
+            return stats

provide/foundation/logger/setup/__init__.py

@@ -0,0 +1,29 @@
+#
+# __init__.py
+#
+"""
+Foundation Logger Setup Module.
+
+Handles structured logging configuration, processor setup, and emoji resolution.
+Provides the core setup functionality for the Foundation logging system.
+"""
+
+from provide.foundation.logger.setup.coordinator import internal_setup
+
+# Import testing utilities conditionally
+try:
+    from provide.foundation.logger.setup.testing import (
+        reset_foundation_setup_for_testing as reset_for_testing,
+    )
+
+    _has_testing = True
+except ImportError:
+    _has_testing = False
+    reset_for_testing = None
+
+__all__ = [
+    "internal_setup",
+]
+
+if _has_testing:
+    __all__.append("reset_for_testing")

provide/foundation/logger/setup/coordinator.py

@@ -0,0 +1,138 @@
+#
+# coordinator.py
+#
+"""
+Main setup coordination for Foundation Telemetry.
+Handles the core setup logic, state management, and setup logger creation.
+"""
+
+import logging as stdlib_logging
+import threading
+from typing import Any
+
+import structlog
+
+from provide.foundation.logger.config import LoggingConfig, TelemetryConfig
+from provide.foundation.logger.core import (
+    _LAZY_SETUP_STATE,
+    logger as foundation_logger,
+)
+from provide.foundation.logger.emoji.sets import BUILTIN_EMOJI_SETS
+from provide.foundation.logger.setup.emoji_resolver import resolve_active_emoji_config
+from provide.foundation.logger.setup.processors import (
+    configure_structlog_output,
+    handle_globally_disabled_setup,
+)
+from provide.foundation.streams import get_log_stream
+from provide.foundation.utils.streams import get_foundation_log_stream, get_safe_stderr
+
+_PROVIDE_SETUP_LOCK = threading.Lock()
+_CORE_SETUP_LOGGER_NAME = "provide.foundation.core_setup"
+_EXPLICIT_SETUP_DONE = False
+_FOUNDATION_LOG_LEVEL: int | None = None
+
+
+def get_foundation_log_level() -> int:
+    """Get the Foundation log level from LoggingConfig, checking only once."""
+    global _FOUNDATION_LOG_LEVEL
+    if _FOUNDATION_LOG_LEVEL is None:
+        # Use the proper config system to get the Foundation setup log level
+        logging_config = LoggingConfig.from_env(strict=False)
+        level_str = logging_config.foundation_setup_log_level.upper()
+        _FOUNDATION_LOG_LEVEL = getattr(
+            stdlib_logging,
+            level_str,
+            stdlib_logging.INFO,  # Default fallback
+        )
+    return _FOUNDATION_LOG_LEVEL
+
+
+def create_core_setup_logger(globally_disabled: bool = False) -> Any:
+    """Create a structlog logger for core setup messages."""
+    if globally_disabled:
+        # Configure structlog to be a no-op for core setup logger
+        structlog.configure(
+            processors=[],
+            logger_factory=structlog.ReturnLoggerFactory(),
+            wrapper_class=structlog.BoundLogger,
+            cache_logger_on_first_use=True,
+        )
+        return structlog.get_logger(_CORE_SETUP_LOGGER_NAME)
+    else:
+        # Get the foundation log output stream
+        try:
+            logging_config = LoggingConfig.from_env(strict=False)
+            foundation_stream = get_foundation_log_stream(
+                logging_config.foundation_log_output
+            )
+        except Exception:
+            # Fallback to stderr if config loading fails
+            foundation_stream = get_safe_stderr()
+
+        # Configure structlog for core setup logger
+        structlog.configure(
+            processors=[
+                structlog.processors.add_log_level,
+                structlog.processors.TimeStamper(fmt="iso"),
+                structlog.dev.ConsoleRenderer(),
+            ],
+            logger_factory=structlog.PrintLoggerFactory(file=foundation_stream),
+            wrapper_class=structlog.BoundLogger,
+            cache_logger_on_first_use=True,
+        )
+
+        return structlog.get_logger(_CORE_SETUP_LOGGER_NAME)
+
+
+def internal_setup(
+    config: TelemetryConfig | None = None, is_explicit_call: bool = False
+) -> None:
+    """
+    The single, internal setup function that both explicit and lazy setup call.
+    It is protected by the _PROVIDE_SETUP_LOCK in its callers.
+    """
+    # This function assumes the lock is already held.
+    structlog.reset_defaults()
+    foundation_logger._is_configured_by_setup = False
+    foundation_logger._active_config = None
+    foundation_logger._active_resolved_emoji_config = None
+    _LAZY_SETUP_STATE.update({"done": False, "error": None, "in_progress": False})
+
+    current_config = config if config is not None else TelemetryConfig.from_env()
+    core_setup_logger = create_core_setup_logger(
+        globally_disabled=current_config.globally_disabled
+    )
+
+    if not current_config.globally_disabled:
+        core_setup_logger.debug(
+            "⚙️➡️🚀 Starting Foundation (structlog) setup",
+            service_name=current_config.service_name,
+            log_level=current_config.logging.default_level,
+            formatter=current_config.logging.console_formatter,
+        )
+
+    resolved_emoji_config = resolve_active_emoji_config(
+        current_config.logging, BUILTIN_EMOJI_SETS
+    )
+
+    if current_config.globally_disabled:
+        handle_globally_disabled_setup()
+    else:
+        configure_structlog_output(
+            current_config, resolved_emoji_config, get_log_stream()
+        )
+
+    foundation_logger._is_configured_by_setup = is_explicit_call
+    foundation_logger._active_config = current_config
+    foundation_logger._active_resolved_emoji_config = resolved_emoji_config
+    _LAZY_SETUP_STATE["done"] = True
+
+    if not current_config.globally_disabled:
+        field_definitions, emoji_sets = resolved_emoji_config
+        core_setup_logger.debug(
+            "⚙️➡️✅ Foundation (structlog) setup completed",
+            emoji_sets_enabled=len(field_definitions) > 0,
+            emoji_sets_count=len(emoji_sets),
+            processors_configured=True,
+            log_file_enabled=current_config.logging.log_file is not None,
+        )