provide-foundation 0.0.0.dev0__py3-none-any.whl

provide/foundation/logger/processors/main.py

@@ -0,0 +1,254 @@
+#
+# processors.py
+#
+"""
+Structlog processors for Foundation Telemetry.
+"""
+
+import json
+import logging as stdlib_logging
+from typing import TYPE_CHECKING, Any, TextIO, cast
+
+import structlog
+
+from provide.foundation.logger.config import LoggingConfig, TelemetryConfig
+from provide.foundation.logger.custom_processors import (
+    StructlogProcessor,
+    add_log_level_custom,
+    add_logger_name_emoji_prefix,
+    filter_by_level_custom,
+)
+
+# Import trace context processor
+from provide.foundation.logger.processors.trace import inject_trace_context
+from provide.foundation.types import (
+    TRACE_LEVEL_NUM,
+    LogLevelStr,
+)
+
+if TYPE_CHECKING:
+    from provide.foundation.logger.setup.emoji_resolver import ResolvedEmojiConfig
+
+_LEVEL_TO_NUMERIC: dict[LogLevelStr, int] = {
+    "CRITICAL": stdlib_logging.CRITICAL,
+    "ERROR": stdlib_logging.ERROR,
+    "WARNING": stdlib_logging.WARNING,
+    "INFO": stdlib_logging.INFO,
+    "DEBUG": stdlib_logging.DEBUG,
+    "TRACE": TRACE_LEVEL_NUM,
+    "NOTSET": stdlib_logging.NOTSET,
+}
+
+
+def _config_create_service_name_processor(
+    service_name: str | None,
+) -> StructlogProcessor:
+    def processor(
+        _logger: Any, _method_name: str, event_dict: structlog.types.EventDict
+    ) -> structlog.types.EventDict:
+        if service_name is not None:
+            event_dict["service_name"] = service_name
+        return event_dict
+
+    return cast(StructlogProcessor, processor)
+
+
+def _config_create_timestamp_processors(
+    omit_timestamp: bool,
+) -> list[StructlogProcessor]:
+    processors: list[StructlogProcessor] = [
+        structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S.%f", utc=False)
+    ]
+    if omit_timestamp:
+
+        def pop_timestamp_processor(
+            _logger: Any, _method_name: str, event_dict: structlog.types.EventDict
+        ) -> structlog.types.EventDict:
+            event_dict.pop("timestamp", None)
+            return event_dict
+
+        processors.append(cast(StructlogProcessor, pop_timestamp_processor))
+    return processors
+
+
+def _config_create_emoji_processors(
+    logging_config: LoggingConfig, resolved_emoji_config: "ResolvedEmojiConfig"
+) -> list[StructlogProcessor]:
+    processors: list[StructlogProcessor] = []
+    if logging_config.logger_name_emoji_prefix_enabled:
+        processors.append(cast(StructlogProcessor, add_logger_name_emoji_prefix))
+    if logging_config.das_emoji_prefix_enabled:
+        # FIX: Create the processor as a closure with the resolved config
+        resolved_field_definitions, resolved_emoji_sets_lookup = resolved_emoji_config
+
+        def add_das_emoji_prefix_closure(
+            _logger: Any, _method_name: str, event_dict: structlog.types.EventDict
+        ) -> structlog.types.EventDict:
+            # This inner function now has access to the resolved config from its closure scope
+            from provide.foundation.logger.emoji.matrix import (
+                PRIMARY_EMOJI,
+                SECONDARY_EMOJI,
+                TERTIARY_EMOJI,
+            )
+
+            final_das_prefix_parts: list[str] = []
+
+            if resolved_field_definitions:  # New Layered Emoji System is active
+                for field_def in resolved_field_definitions:
+                    value_from_event = event_dict.get(field_def.log_key)
+                    if value_from_event is not None and field_def.emoji_set_name:
+                        event_dict.pop(field_def.log_key, None)
+                        emoji_set = resolved_emoji_sets_lookup.get(
+                            field_def.emoji_set_name
+                        )
+                        if emoji_set:
+                            value_str_lower = str(value_from_event).lower()
+                            specific_emoji = emoji_set.emojis.get(value_str_lower)
+                            default_key = (
+                                field_def.default_emoji_override_key
+                                or emoji_set.default_emoji_key
+                            )
+                            default_emoji = emoji_set.emojis.get(default_key, "❓")
+                            chosen_emoji = (
+                                specific_emoji
+                                if specific_emoji is not None
+                                else default_emoji
+                            )
+                            final_das_prefix_parts.append(f"[{chosen_emoji}]")
+                        else:
+                            final_das_prefix_parts.append("[❓]")
+            else:  # Fallback to Core DAS System
+                domain = event_dict.pop("domain", None)
+                action = event_dict.pop("action", None)
+                status = event_dict.pop("status", None)
+                if domain or action or status:
+                    domain_emoji = (
+                        PRIMARY_EMOJI.get(str(domain).lower(), PRIMARY_EMOJI["default"])
+                        if domain
+                        else PRIMARY_EMOJI["default"]
+                    )
+                    action_emoji = (
+                        SECONDARY_EMOJI.get(
+                            str(action).lower(), SECONDARY_EMOJI["default"]
+                        )
+                        if action
+                        else SECONDARY_EMOJI["default"]
+                    )
+                    status_emoji = (
+                        TERTIARY_EMOJI.get(
+                            str(status).lower(), TERTIARY_EMOJI["default"]
+                        )
+                        if status
+                        else TERTIARY_EMOJI["default"]
+                    )
+                    final_das_prefix_parts.extend(
+                        [f"[{domain_emoji}]", f"[{action_emoji}]", f"[{status_emoji}]"]
+                    )
+
+            if final_das_prefix_parts:
+                final_das_prefix_str = "".join(final_das_prefix_parts)
+                event_msg = event_dict.get("event")
+                event_dict["event"] = (
+                    f"{final_das_prefix_str} {event_msg}"
+                    if event_msg is not None
+                    else final_das_prefix_str
+                )
+            return event_dict
+
+        processors.append(cast(StructlogProcessor, add_das_emoji_prefix_closure))
+    return processors
+
+
+def _build_core_processors_list(
+    config: TelemetryConfig, resolved_emoji_config: "ResolvedEmojiConfig"
+) -> list[StructlogProcessor]:
+    log_cfg = config.logging
+    processors: list[StructlogProcessor] = [
+        structlog.contextvars.merge_contextvars,
+        cast(StructlogProcessor, add_log_level_custom),
+        cast(
+            StructlogProcessor,
+            filter_by_level_custom(
+                default_level_str=log_cfg.default_level,
+                module_levels=log_cfg.module_levels,
+                level_to_numeric_map=_LEVEL_TO_NUMERIC,
+            ),
+        ),
+        structlog.processors.StackInfoRenderer(),
+        structlog.dev.set_exc_info,
+    ]
+
+    # Add rate limiting processor if enabled
+    if log_cfg.rate_limit_enabled:
+        from provide.foundation.logger.ratelimit import create_rate_limiter_processor
+
+        rate_limiter_processor = create_rate_limiter_processor(
+            global_rate=log_cfg.rate_limit_global,
+            global_capacity=log_cfg.rate_limit_global_capacity,
+            per_logger_rates=log_cfg.rate_limit_per_logger,
+            emit_warnings=log_cfg.rate_limit_emit_warnings,
+            summary_interval=log_cfg.rate_limit_summary_interval,
+            max_queue_size=log_cfg.rate_limit_max_queue_size,
+            max_memory_mb=log_cfg.rate_limit_max_memory_mb,
+            overflow_policy=log_cfg.rate_limit_overflow_policy,
+        )
+        processors.append(cast(StructlogProcessor, rate_limiter_processor))
+
+    processors.extend(_config_create_timestamp_processors(log_cfg.omit_timestamp))
+    if config.service_name is not None:
+        processors.append(_config_create_service_name_processor(config.service_name))
+
+    # Add trace context injection if tracing is enabled
+    if config.tracing_enabled and not config.globally_disabled:
+        processors.append(cast(StructlogProcessor, inject_trace_context))
+
+    processors.extend(_config_create_emoji_processors(log_cfg, resolved_emoji_config))
+    return processors
+
+
+def _config_create_json_formatter_processors() -> list[StructlogProcessor]:
+    return [
+        structlog.processors.format_exc_info,
+        structlog.processors.JSONRenderer(serializer=json.dumps, sort_keys=False),
+    ]
+
+
+def _config_create_keyvalue_formatter_processors(
+    output_stream: TextIO,
+) -> list[StructlogProcessor]:
+    def pop_logger_name_processor(
+        _logger: object, _method_name: str, event_dict: structlog.types.EventDict
+    ) -> structlog.types.EventDict:
+        event_dict.pop("logger_name", None)
+        return event_dict
+
+    is_tty = hasattr(output_stream, "isatty") and output_stream.isatty()
+    return [
+        cast(StructlogProcessor, pop_logger_name_processor),
+        structlog.dev.ConsoleRenderer(
+            colors=is_tty, exception_formatter=structlog.dev.plain_traceback
+        ),
+    ]
+
+
+def _build_formatter_processors_list(
+    logging_config: LoggingConfig, output_stream: TextIO
+) -> list[StructlogProcessor]:
+    match logging_config.console_formatter:
+        case "json":
+            return _config_create_json_formatter_processors()
+        case "key_value":
+            return _config_create_keyvalue_formatter_processors(output_stream)
+        case _:
+            # Unknown formatter, warn and default to key_value
+            # Use setup coordinator logger
+            from provide.foundation.logger.setup.coordinator import (
+                create_core_setup_logger,
+            )
+
+            setup_logger = create_core_setup_logger()
+            setup_logger.warning(
+                f"Unknown formatter '{logging_config.console_formatter}', using default 'key_value'. "
+                f"Valid formatters: ['json', 'key_value']"
+            )
+            return _config_create_keyvalue_formatter_processors(output_stream)

provide/foundation/logger/processors/trace.py

@@ -0,0 +1,113 @@
+"""Trace context processor for injecting trace/span IDs into logs."""
+
+from typing import Any
+
+# Note: Cannot import get_logger here due to circular dependency during setup
+# Use structlog directly for foundation-internal logging
+import structlog
+
+log = structlog.get_logger(__name__)
+
+# Note: Internal trace injection logging removed to avoid circular dependencies
+# and level registration issues during logger setup
+
+# OpenTelemetry feature detection
+try:
+    from opentelemetry import trace as otel_trace
+
+    _HAS_OTEL = True
+except ImportError:
+    otel_trace = None
+    _HAS_OTEL = False
+
+
+def inject_trace_context(
+    logger: Any, method_name: str, event_dict: dict[str, Any]
+) -> dict[str, Any]:
+    """Processor to inject trace context into log records.
+
+    Args:
+        logger: Logger instance
+        method_name: Method name being called
+        event_dict: Current event dictionary
+
+    Returns:
+        Event dictionary with trace context added
+    """
+    # Try OpenTelemetry trace context first
+    if _HAS_OTEL:
+        try:
+            current_span = otel_trace.get_current_span()
+            if current_span and current_span.is_recording():
+                span_context = current_span.get_span_context()
+
+                # Add OpenTelemetry trace and span IDs (only if not already present)
+                if "trace_id" not in event_dict:
+                    event_dict["trace_id"] = f"{span_context.trace_id:032x}"
+                if "span_id" not in event_dict:
+                    event_dict["span_id"] = f"{span_context.span_id:016x}"
+
+                # Add trace flags if present
+                if span_context.trace_flags:
+                    event_dict["trace_flags"] = span_context.trace_flags
+
+                # Trace context injected successfully
+                return event_dict
+        except Exception:
+            # OpenTelemetry trace context unavailable - continue to fallback
+            pass
+
+    # Fallback to Foundation's simple tracer context
+    try:
+        from provide.foundation.tracer.context import (
+            get_current_span,
+            get_current_trace_id,
+        )
+
+        current_span = get_current_span()
+        current_trace_id = get_current_trace_id()
+
+        if current_span:
+            if "trace_id" not in event_dict:
+                event_dict["trace_id"] = current_span.trace_id
+            if "span_id" not in event_dict:
+                event_dict["span_id"] = current_span.span_id
+            # Foundation trace context injected successfully
+        elif current_trace_id and "trace_id" not in event_dict:
+            event_dict["trace_id"] = current_trace_id
+            # Foundation trace ID injected successfully
+
+    except Exception:
+        # Foundation trace context unavailable - skip injection
+        pass
+
+    return event_dict
+
+
+def should_inject_trace_context() -> bool:
+    """Check if trace context injection is available.
+
+    Returns:
+        True if trace context can be injected
+    """
+    # Check if OpenTelemetry is available and has active span
+    if _HAS_OTEL:
+        try:
+            current_span = otel_trace.get_current_span()
+            if current_span and current_span.is_recording():
+                return True
+        except Exception:
+            pass
+
+    # Check if Foundation tracer has active context
+    try:
+        from provide.foundation.tracer.context import (
+            get_current_span,
+            get_current_trace_id,
+        )
+
+        return get_current_span() is not None or get_current_trace_id() is not None
+    except Exception:
+        pass
+
+    return False

provide/foundation/logger/ratelimit/__init__.py

@@ -0,0 +1,31 @@
+#
+# __init__.py
+#
+"""
+Rate limiting subcomponent for Foundation's logging system.
+Provides rate limiters and processors for controlling log output rates.
+"""
+
+from provide.foundation.logger.ratelimit.limiters import (
+    AsyncRateLimiter,
+    GlobalRateLimiter,
+    SyncRateLimiter,
+)
+from provide.foundation.logger.ratelimit.processor import (
+    RateLimiterProcessor,
+    create_rate_limiter_processor,
+)
+from provide.foundation.logger.ratelimit.queue_limiter import (
+    BufferedRateLimiter,
+    QueuedRateLimiter,
+)
+
+__all__ = [
+    "AsyncRateLimiter",
+    "BufferedRateLimiter",
+    "GlobalRateLimiter",
+    "QueuedRateLimiter",
+    "RateLimiterProcessor",
+    "SyncRateLimiter",
+    "create_rate_limiter_processor",
+]

provide/foundation/logger/ratelimit/limiters.py

@@ -0,0 +1,294 @@
+#
+# limiters.py
+#
+"""
+Rate limiter implementations for Foundation's logging system.
+"""
+
+import asyncio
+import threading
+import time
+from typing import Any
+
+
+class SyncRateLimiter:
+    """
+    Synchronous token bucket rate limiter for controlling log output rates.
+    Thread-safe implementation suitable for synchronous logging operations.
+    """
+
+    def __init__(self, capacity: float, refill_rate: float):
+        """
+        Initialize the rate limiter.
+
+        Args:
+            capacity: Maximum number of tokens (burst capacity)
+            refill_rate: Tokens refilled per second
+        """
+        if capacity <= 0:
+            raise ValueError("Capacity must be positive")
+        if refill_rate <= 0:
+            raise ValueError("Refill rate must be positive")
+
+        self.capacity = float(capacity)
+        self.refill_rate = float(refill_rate)
+        self.tokens = float(capacity)
+        self.last_refill = time.monotonic()
+        self.lock = threading.Lock()
+
+        # Track statistics
+        self.total_allowed = 0
+        self.total_denied = 0
+        self.last_denied_time = None
+
+    def is_allowed(self) -> bool:
+        """
+        Check if a log message is allowed based on available tokens.
+
+        Returns:
+            True if the log should be allowed, False if rate limited
+        """
+        with self.lock:
+            now = time.monotonic()
+            elapsed = now - self.last_refill
+
+            # Refill tokens based on elapsed time
+            if elapsed > 0:
+                tokens_to_add = elapsed * self.refill_rate
+                self.tokens = min(self.capacity, self.tokens + tokens_to_add)
+                self.last_refill = now
+
+            # Try to consume a token
+            if self.tokens >= 1.0:
+                self.tokens -= 1.0
+                self.total_allowed += 1
+                return True
+            else:
+                self.total_denied += 1
+                self.last_denied_time = now
+                return False
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get rate limiter statistics."""
+        with self.lock:
+            return {
+                "tokens_available": self.tokens,
+                "capacity": self.capacity,
+                "refill_rate": self.refill_rate,
+                "total_allowed": self.total_allowed,
+                "total_denied": self.total_denied,
+                "last_denied_time": self.last_denied_time,
+            }
+
+
+class AsyncRateLimiter:
+    """
+    Asynchronous token bucket rate limiter.
+    Uses asyncio.Lock for thread safety in async contexts.
+    """
+
+    def __init__(self, capacity: float, refill_rate: float):
+        """
+        Initialize the async rate limiter.
+
+        Args:
+            capacity: Maximum number of tokens (burst capacity)
+            refill_rate: Tokens refilled per second
+        """
+        if capacity <= 0:
+            raise ValueError("Capacity must be positive")
+        if refill_rate <= 0:
+            raise ValueError("Refill rate must be positive")
+
+        self.capacity = float(capacity)
+        self.refill_rate = float(refill_rate)
+        self.tokens = float(capacity)
+        self.last_refill = time.monotonic()
+        self.lock = asyncio.Lock()
+
+        # Track statistics
+        self.total_allowed = 0
+        self.total_denied = 0
+        self.last_denied_time = None
+
+    async def is_allowed(self) -> bool:
+        """
+        Check if a log message is allowed based on available tokens.
+
+        Returns:
+            True if the log should be allowed, False if rate limited
+        """
+        async with self.lock:
+            now = time.monotonic()
+            elapsed = now - self.last_refill
+
+            # Refill tokens based on elapsed time
+            if elapsed > 0:
+                tokens_to_add = elapsed * self.refill_rate
+                self.tokens = min(self.capacity, self.tokens + tokens_to_add)
+                self.last_refill = now
+
+            # Try to consume a token
+            if self.tokens >= 1.0:
+                self.tokens -= 1.0
+                self.total_allowed += 1
+                return True
+            else:
+                self.total_denied += 1
+                self.last_denied_time = now
+                return False
+
+    async def get_stats(self) -> dict[str, Any]:
+        """Get rate limiter statistics."""
+        async with self.lock:
+            return {
+                "tokens_available": self.tokens,
+                "capacity": self.capacity,
+                "refill_rate": self.refill_rate,
+                "total_allowed": self.total_allowed,
+                "total_denied": self.total_denied,
+                "last_denied_time": self.last_denied_time,
+            }
+
+
+class GlobalRateLimiter:
+    """
+    Global rate limiter singleton for Foundation's logging system.
+    Manages per-logger and global rate limits.
+    """
+
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+
+        self._initialized = True
+        self.global_limiter = None
+        self.logger_limiters: dict[str, SyncRateLimiter] = {}
+        self.lock = threading.Lock()
+
+        # Default configuration (can be overridden)
+        self.global_rate = None
+        self.global_capacity = None
+        self.per_logger_rates: dict[str, tuple[float, float]] = {}
+
+        # Queue configuration
+        self.use_buffered = False
+        self.max_queue_size = 1000
+        self.max_memory_mb = None
+        self.overflow_policy = "drop_oldest"
+
+    def configure(
+        self,
+        global_rate: float | None = None,
+        global_capacity: float | None = None,
+        per_logger_rates: dict[str, tuple[float, float]] | None = None,
+        use_buffered: bool = False,
+        max_queue_size: int = 1000,
+        max_memory_mb: float | None = None,
+        overflow_policy: str = "drop_oldest",
+    ):
+        """
+        Configure the global rate limiter.
+
+        Args:
+            global_rate: Global logs per second limit
+            global_capacity: Global burst capacity
+            per_logger_rates: Dict of logger_name -> (rate, capacity) tuples
+            use_buffered: Use buffered rate limiter with tracking
+            max_queue_size: Maximum queue size for buffered limiter
+            max_memory_mb: Maximum memory for buffered limiter
+            overflow_policy: What to do when queue is full
+        """
+        with self.lock:
+            self.use_buffered = use_buffered
+            self.max_queue_size = max_queue_size
+            self.max_memory_mb = max_memory_mb
+            self.overflow_policy = overflow_policy
+
+            if global_rate is not None and global_capacity is not None:
+                self.global_rate = global_rate
+                self.global_capacity = global_capacity
+
+                if use_buffered:
+                    from provide.foundation.logger.ratelimit.queue_limiter import (
+                        BufferedRateLimiter,
+                    )
+
+                    self.global_limiter = BufferedRateLimiter(
+                        capacity=global_capacity,
+                        refill_rate=global_rate,
+                        buffer_size=max_queue_size,
+                        track_dropped=True,
+                    )
+                else:
+                    self.global_limiter = SyncRateLimiter(global_capacity, global_rate)
+
+            if per_logger_rates:
+                self.per_logger_rates = per_logger_rates
+                # Create rate limiters for configured loggers
+                for logger_name, (rate, capacity) in per_logger_rates.items():
+                    self.logger_limiters[logger_name] = SyncRateLimiter(capacity, rate)
+
+    def is_allowed(
+        self, logger_name: str, item: Any | None = None
+    ) -> tuple[bool, str | None]:
+        """
+        Check if a log from a specific logger is allowed.
+
+        Args:
+            logger_name: Name of the logger
+            item: Optional item for buffered tracking
+
+        Returns:
+            Tuple of (allowed, reason) where reason is set if denied
+        """
+        with self.lock:
+            # Check per-logger limit first
+            if logger_name in self.logger_limiters:
+                if not self.logger_limiters[logger_name].is_allowed():
+                    return False, f"Logger '{logger_name}' rate limit exceeded"
+
+            # Check global limit
+            if self.global_limiter:
+                if self.use_buffered:
+                    # BufferedRateLimiter returns tuple
+                    from provide.foundation.logger.ratelimit.queue_limiter import (
+                        BufferedRateLimiter,
+                    )
+
+                    if isinstance(self.global_limiter, BufferedRateLimiter):
+                        allowed, reason = self.global_limiter.is_allowed(item)
+                        if not allowed:
+                            return False, reason or "Global rate limit exceeded"
+                else:
+                    # SyncRateLimiter returns bool
+                    if not self.global_limiter.is_allowed():
+                        return False, "Global rate limit exceeded"
+
+            return True, None
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get comprehensive rate limiting statistics."""
+        with self.lock:
+            stats = {
+                "global": self.global_limiter.get_stats()
+                if self.global_limiter
+                else None,
+                "per_logger": {},
+            }
+
+            for logger_name, limiter in self.logger_limiters.items():
+                stats["per_logger"][logger_name] = limiter.get_stats()
+
+            return stats