elven-logs-interceptor-python 0.1.2__py3-none-any.whl

logs_interceptor/config.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Literal
+
+from .types import LogLevel
+
+CompressionType = Literal["none", "gzip", "brotli", "snappy"]
+DLQType = Literal["memory", "file"]
+
+
+@dataclass(slots=True)
+class TransportConfig:
+    url: str = ""
+    tenant_id: str = ""
+    auth_token: str | None = None
+    timeout: int | None = None
+    max_retries: int | None = None
+    retry_delay: int | None = None
+    compression: CompressionType | bool | None = None
+    compression_level: int | None = None
+    compression_threshold: int | None = None
+    use_workers: bool | None = None
+    max_workers: int | None = None
+    worker_timeout: int | None = None
+    enable_connection_pooling: bool | None = None
+    max_sockets: int | None = None
+
+
+@dataclass(slots=True)
+class BufferConfig:
+    max_size: int | None = None
+    flush_interval: int | None = None
+    max_age: int | None = None
+    auto_flush: bool | None = None
+    max_memory_mb: int | None = None
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    levels: list[LogLevel] | None = None
+    patterns: list[str] | None = None
+    sampling_rate: float | None = None
+    max_message_length: int | None = None
+    sanitize: bool | None = None
+    sensitive_patterns: list[str] | None = None
+
+
+@dataclass(slots=True)
+class CircuitBreakerConfig:
+    enabled: bool | None = None
+    failure_threshold: int | None = None
+    reset_timeout: int | None = None
+    half_open_requests: int | None = None
+
+
+@dataclass(slots=True)
+class PerformanceConfig:
+    use_workers: bool | None = None
+    max_concurrent_flushes: int | None = None
+    compression_level: int | None = None
+    max_workers: int | None = None
+    worker_timeout: int | None = None
+
+
+@dataclass(slots=True)
+class DeadLetterQueueConfig:
+    enabled: bool | None = None
+    type: DLQType | None = None
+    max_size: int | None = None
+    max_file_size_mb: int | None = None
+    max_retries: int | None = None
+    base_path: str | None = None
+
+
+@dataclass(slots=True)
+class WinstonIntegrationConfig:
+    enabled: bool = True
+    levels: dict[str, LogLevel] | None = None
+
+
+@dataclass(slots=True)
+class MorganIntegrationConfig:
+    enabled: bool = True
+    format: str | None = None
+
+
+@dataclass(slots=True)
+class IntegrationsConfig:
+    winston: bool | WinstonIntegrationConfig | None = None
+    morgan: bool | MorganIntegrationConfig | None = None
+
+
+@dataclass(slots=True)
+class LogsInterceptorConfig:
+    transport: TransportConfig = field(default_factory=TransportConfig)
+    app_name: str = ""
+    version: str | None = None
+    environment: str | None = None
+    labels: dict[str, str] | None = None
+    dynamic_labels: dict[str, Callable[[], str | int]] | None = None
+    buffer: BufferConfig | None = None
+    filter: FilterConfig | None = None
+    circuit_breaker: CircuitBreakerConfig | None = None
+    integrations: IntegrationsConfig | None = None
+    performance: PerformanceConfig | None = None
+    dead_letter_queue: DeadLetterQueueConfig | None = None
+    enable_metrics: bool | None = None
+    enable_health_check: bool | None = None
+    intercept_console: bool | None = None
+    preserve_original_console: bool | None = None
+    debug: bool | None = None
+    silent_errors: bool | None = None
+
+
+@dataclass(slots=True)
+class ResolvedTransportConfig:
+    url: str
+    tenant_id: str
+    auth_token: str
+    timeout: int
+    max_retries: int
+    retry_delay: int
+    compression: CompressionType
+    compression_level: int
+    compression_threshold: int
+    use_workers: bool
+    max_workers: int | None
+    worker_timeout: int
+    enable_connection_pooling: bool
+    max_sockets: int
+
+
+@dataclass(slots=True)
+class ResolvedBufferConfig:
+    max_size: int
+    flush_interval: int
+    max_age: int
+    auto_flush: bool
+    max_memory_mb: int
+
+
+@dataclass(slots=True)
+class ResolvedFilterConfig:
+    levels: list[LogLevel]
+    patterns: list[str]
+    sampling_rate: float
+    max_message_length: int
+    sanitize: bool
+    sensitive_patterns: list[str]
+
+
+@dataclass(slots=True)
+class ResolvedCircuitBreakerConfig:
+    enabled: bool
+    failure_threshold: int
+    reset_timeout: int
+    half_open_requests: int
+
+
+@dataclass(slots=True)
+class ResolvedPerformanceConfig:
+    use_workers: bool
+    max_concurrent_flushes: int
+    compression_level: int
+    max_workers: int | None
+    worker_timeout: int
+
+
+@dataclass(slots=True)
+class ResolvedLogsInterceptorConfig:
+    transport: ResolvedTransportConfig
+    app_name: str
+    version: str
+    environment: str
+    labels: dict[str, str]
+    dynamic_labels: dict[str, Callable[[], str | int]]
+    buffer: ResolvedBufferConfig
+    filter: ResolvedFilterConfig
+    circuit_breaker: ResolvedCircuitBreakerConfig
+    integrations: IntegrationsConfig
+    performance: ResolvedPerformanceConfig
+    dead_letter_queue: DeadLetterQueueConfig | None
+    enable_metrics: bool
+    enable_health_check: bool
+    intercept_console: bool
+    preserve_original_console: bool
+    debug: bool
+    silent_errors: bool

logs_interceptor/domain/__init__.py

@@ -0,0 +1,25 @@
+from .entities import LogEntryEntity
+from .interfaces import (
+    ICircuitBreaker,
+    IContextProvider,
+    IDeadLetterQueue,
+    ILogBuffer,
+    ILogFilter,
+    ILogger,
+    ILogInterceptor,
+    ILogTransport,
+)
+from .value_objects import LogLevelVO
+
+__all__ = [
+    "LogEntryEntity",
+    "LogLevelVO",
+    "ILogger",
+    "ILogTransport",
+    "ILogBuffer",
+    "ILogFilter",
+    "IContextProvider",
+    "ICircuitBreaker",
+    "IDeadLetterQueue",
+    "ILogInterceptor",
+]

logs_interceptor/domain/entities.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ..types import LogEntry, LogLevel
+
+
+@dataclass(slots=True)
+class LogEntryEntity:
+    id: str
+    timestamp: str
+    level: LogLevel
+    message: str
+    context: dict[str, Any] | None = None
+    trace_id: str | None = None
+    span_id: str | None = None
+    request_id: str | None = None
+    labels: dict[str, str] | None = None
+    metadata: dict[str, Any] | None = None
+
+    def to_dict(self) -> LogEntry:
+        payload: LogEntry = {
+            "id": self.id,
+            "timestamp": self.timestamp,
+            "level": self.level,
+            "message": self.message,
+        }
+        if self.context is not None:
+            payload["context"] = self.context
+        if self.trace_id:
+            payload["trace_id"] = self.trace_id
+        if self.span_id:
+            payload["span_id"] = self.span_id
+        if self.request_id:
+            payload["request_id"] = self.request_id
+        if self.labels:
+            payload["labels"] = self.labels
+        if self.metadata:
+            payload["metadata"] = self.metadata
+        return payload

logs_interceptor/domain/interfaces.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+from ..types import (
+    CircuitBreakerState,
+    HealthStatus,
+    LoggerMetrics,
+    LogLevel,
+    TransportHealth,
+    TransportMetrics,
+)
+from .entities import LogEntryEntity
+
+
+class BufferMetrics(Protocol):
+    size: int
+    max_size: int
+    oldest_entry: float | None
+    newest_entry: float | None
+    memory_usage_mb: float
+    dropped_entries: int
+
+
+class ILogBuffer(Protocol):
+    def add(self, entry: LogEntryEntity) -> None: ...
+
+    def flush(self) -> list[LogEntryEntity]: ...
+
+    def peek(self) -> list[LogEntryEntity]: ...
+
+    def size(self) -> int: ...
+
+    def is_full(self) -> bool: ...
+
+    def should_flush(self) -> bool: ...
+
+    def clear(self) -> None: ...
+
+    def destroy(self) -> None: ...
+
+    def get_metrics(self) -> dict[str, Any]: ...
+
+
+class ILogFilter(Protocol):
+    def should_process(self, entry: LogEntryEntity) -> bool: ...
+
+    def filter(self, entry: LogEntryEntity) -> LogEntryEntity: ...
+
+    def is_level_enabled(self, level: LogLevel) -> bool: ...
+
+
+class ILogTransport(Protocol):
+    def send(self, entries: list[LogEntryEntity]) -> None: ...
+
+    def is_available(self) -> bool: ...
+
+    def get_health(self) -> TransportHealth: ...
+
+    def get_metrics(self) -> TransportMetrics | None: ...
+
+    def destroy(self) -> None: ...
+
+
+class IDeadLetterQueue(Protocol):
+    def add(self, entry: LogEntryEntity, reason: str) -> dict[str, int]: ...
+
+    def add_batch(self, entries: list[LogEntryEntity], reason: str) -> dict[str, int]: ...
+
+    def flush(self) -> int: ...
+
+    def size(self) -> int: ...
+
+    def clear(self) -> None: ...
+
+    def get_entries(self, limit: int = 100) -> list[dict[str, Any]]: ...
+
+    def get_stats(self) -> dict[str, int]: ...
+
+
+class ICircuitBreaker(Protocol):
+    def execute(self, operation: Any) -> Any: ...
+
+    def record_success(self) -> None: ...
+
+    def record_failure(self, error: Exception | None = None) -> None: ...
+
+    def get_state(self) -> dict[str, Any]: ...
+
+    def reset(self) -> None: ...
+
+
+class IContextProvider(Protocol):
+    def get_context(self) -> dict[str, Any]: ...
+
+    def run_with_context(self, context: dict[str, Any], fn: Any) -> Any: ...
+
+    async def run_with_context_async(self, context: dict[str, Any], fn: Any) -> Any: ...
+
+    def set(self, key: str, value: Any) -> None: ...
+
+    def get(self, key: str, default: Any = None) -> Any: ...
+
+    def clear(self) -> None: ...
+
+
+@runtime_checkable
+class ILogger(Protocol):
+    def debug(self, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def info(self, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def warn(self, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def error(self, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def fatal(self, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def log(self, level: LogLevel, message: str, context: dict[str, Any] | None = None) -> None: ...
+
+    def track_event(self, event_name: str, properties: dict[str, Any] | None = None) -> None: ...
+
+    def flush(self) -> None: ...
+
+    async def aflush(self) -> None: ...
+
+    def with_context(self, context: dict[str, Any], fn: Any) -> Any: ...
+
+    async def with_context_async(self, context: dict[str, Any], fn: Any) -> Any: ...
+
+    def get_metrics(self) -> LoggerMetrics: ...
+
+    def get_health(self) -> HealthStatus: ...
+
+    def destroy(self) -> None: ...
+
+    async def adestroy(self) -> None: ...
+
+
+class ILogInterceptor(Protocol):
+    def enable(self) -> None: ...
+
+    def restore(self) -> None: ...
+
+    def is_enabled(self) -> bool: ...
+
+
+class CircuitBreakerSnapshot(Protocol):
+    state: CircuitBreakerState

logs_interceptor/domain/value_objects.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..types import LogLevel
+
+VALID_LOG_LEVELS: tuple[LogLevel, ...] = ("debug", "info", "warn", "error", "fatal")
+LEVEL_PRIORITY: dict[LogLevel, int] = {
+    "debug": 0,
+    "info": 1,
+    "warn": 2,
+    "error": 3,
+    "fatal": 4,
+}
+
+
+@dataclass(frozen=True)
+class LogLevelVO:
+    value: LogLevel
+
+    def __post_init__(self) -> None:
+        if not self.is_valid(self.value):
+            raise ValueError(f"Invalid log level: {self.value}")
+
+    @staticmethod
+    def is_valid(level: str) -> bool:
+        return level in VALID_LOG_LEVELS
+
+    @staticmethod
+    def from_string(level: str) -> LogLevelVO:
+        normalized = level.lower().strip()
+        if not LogLevelVO.is_valid(normalized):
+            raise ValueError(f"Invalid log level: {level}")
+        return LogLevelVO(normalized)  # type: ignore[arg-type]
+
+    def compare_to(self, other: LogLevelVO) -> int:
+        return LEVEL_PRIORITY[self.value] - LEVEL_PRIORITY[other.value]
+
+    def is_greater_than_or_equal(self, other: LogLevelVO) -> bool:
+        return self.compare_to(other) >= 0

logs_interceptor/infrastructure/__init__.py

@@ -0,0 +1,48 @@
+from .buffer import MemoryBuffer
+from .circuit_breaker import CircuitBreaker
+from .compression import (
+    BrotliCompressor,
+    Compressor,
+    CompressorConfig,
+    CompressorFactory,
+    GzipCompressor,
+    NoOpCompressor,
+)
+from .context import ContextVarProvider
+from .dlq import FileDeadLetterQueue, MemoryDeadLetterQueue
+from .filter import LogFilter
+from .interceptors import RuntimeInterceptor
+from .memory import MemoryTracker
+from .metrics import MetricsCollector
+from .transport import (
+    LokiJsonTransport,
+    LokiProtobufTransport,
+    ResilientTransport,
+    ResilientTransportConfig,
+    TransportFactory,
+)
+from .workers import WorkerPool
+
+__all__ = [
+    "MemoryBuffer",
+    "CircuitBreaker",
+    "Compressor",
+    "CompressorConfig",
+    "CompressorFactory",
+    "GzipCompressor",
+    "BrotliCompressor",
+    "NoOpCompressor",
+    "ContextVarProvider",
+    "FileDeadLetterQueue",
+    "MemoryDeadLetterQueue",
+    "LogFilter",
+    "RuntimeInterceptor",
+    "MemoryTracker",
+    "MetricsCollector",
+    "LokiJsonTransport",
+    "LokiProtobufTransport",
+    "ResilientTransport",
+    "ResilientTransportConfig",
+    "TransportFactory",
+    "WorkerPool",
+]

logs_interceptor/infrastructure/buffer/__init__.py

@@ -0,0 +1,3 @@
+from .memory_buffer import MemoryBuffer
+
+__all__ = ["MemoryBuffer"]

logs_interceptor/infrastructure/buffer/memory_buffer.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from datetime import datetime
+
+from ...config import ResolvedBufferConfig
+from ...domain.entities import LogEntryEntity
+from ..memory.memory_tracker import MemoryTracker
+
+
+class MemoryBuffer:
+    def __init__(self, config: ResolvedBufferConfig) -> None:
+        self._config = config
+        self._entries: list[LogEntryEntity] = []
+        self._last_flush_time = time.time()
+        self._flush_timer: threading.Timer | None = None
+        self._memory_tracker = MemoryTracker()
+        self._flush_callback: Callable[[], None] | None = None
+        self._dropped_entries = 0
+        self._destroyed = False
+        self._lock = threading.RLock()
+
+        if self._config.auto_flush:
+            self._schedule_flush()
+
+    def set_flush_callback(self, callback: Callable[[], None]) -> None:
+        self._flush_callback = callback
+
+    def add(self, entry: LogEntryEntity) -> None:
+        with self._lock:
+            if self._destroyed:
+                return
+
+            self._entries.append(entry)
+            self._memory_tracker.add_entry(entry)
+
+            self._enforce_max_size()
+
+            if self._memory_tracker.get_total_size_mb() > self._config.max_memory_mb:
+                self._remove_old_entries()
+                self._enforce_max_size()
+
+            if self._config.auto_flush:
+                self._schedule_flush()
+
+            if len(self._entries) >= self._config.max_size and self._config.auto_flush:
+                self._trigger_immediate_flush()
+
+    def flush(self) -> list[LogEntryEntity]:
+        with self._lock:
+            if self._flush_timer:
+                self._flush_timer.cancel()
+                self._flush_timer = None
+
+            flushed = list(self._entries)
+            self._memory_tracker.remove_entries(flushed)
+            self._entries.clear()
+            self._last_flush_time = time.time()
+
+            if self._config.auto_flush and not self._destroyed:
+                self._schedule_flush()
+
+            return flushed
+
+    def peek(self) -> list[LogEntryEntity]:
+        with self._lock:
+            return list(self._entries)
+
+    def size(self) -> int:
+        with self._lock:
+            return len(self._entries)
+
+    def is_full(self) -> bool:
+        with self._lock:
+            return len(self._entries) >= self._config.max_size
+
+    def should_flush(self) -> bool:
+        with self._lock:
+            elapsed_ms = int((time.time() - self._last_flush_time) * 1000)
+            return len(self._entries) >= self._config.max_size or (
+                self._config.auto_flush and elapsed_ms >= self._config.flush_interval
+            )
+
+    def clear(self) -> None:
+        with self._lock:
+            self._memory_tracker.remove_entries(self._entries)
+            self._entries.clear()
+            if self._flush_timer:
+                self._flush_timer.cancel()
+                self._flush_timer = None
+
+    def destroy(self) -> None:
+        with self._lock:
+            self._destroyed = True
+            if self._flush_timer:
+                self._flush_timer.cancel()
+                self._flush_timer = None
+            self.clear()
+            self._memory_tracker.reset()
+            self._flush_callback = None
+
+    def get_metrics(self) -> dict[str, float | int | None]:
+        with self._lock:
+            oldest = self._entries[0].timestamp if self._entries else None
+            newest = self._entries[-1].timestamp if self._entries else None
+            oldest_ts = self._to_timestamp(oldest)
+            newest_ts = self._to_timestamp(newest)
+            return {
+                "size": len(self._entries),
+                "max_size": self._config.max_size,
+                "oldest_entry": oldest_ts,
+                "newest_entry": newest_ts,
+                "memory_usage_mb": self._memory_tracker.get_total_size_mb(),
+                "dropped_entries": self._dropped_entries,
+            }
+
+    def _enforce_max_size(self) -> None:
+        if len(self._entries) <= self._config.max_size:
+            return
+        remove_count = len(self._entries) - self._config.max_size
+        self._drop_oldest(remove_count)
+
+    def _drop_oldest(self, count: int) -> None:
+        if count <= 0 or not self._entries:
+            return
+        dropped = self._entries[:count]
+        self._entries = self._entries[count:]
+        self._memory_tracker.remove_entries(dropped)
+        self._dropped_entries += len(dropped)
+
+    def _trigger_immediate_flush(self) -> None:
+        if self._destroyed or self._flush_callback is None:
+            return
+        callback = self._flush_callback
+        timer = threading.Timer(0.001, callback)
+        timer.daemon = True
+        timer.start()
+
+    def _remove_old_entries(self) -> None:
+        now = time.time()
+        max_age_s = self._config.max_age / 1000
+
+        kept: list[LogEntryEntity] = []
+        removed: list[LogEntryEntity] = []
+        for entry in self._entries:
+            ts = self._to_timestamp(entry.timestamp)
+            if ts is None or (now - ts) < max_age_s:
+                kept.append(entry)
+            else:
+                removed.append(entry)
+
+        self._entries = kept
+        if removed:
+            self._memory_tracker.remove_entries(removed)
+            self._dropped_entries += len(removed)
+
+        if self._memory_tracker.get_total_size_mb() > self._config.max_memory_mb and self._entries:
+            remove_count = max(1, len(self._entries) // 10)
+            self._drop_oldest(remove_count)
+
+    def _schedule_flush(self) -> None:
+        if self._destroyed or self._flush_timer is not None:
+            return
+
+        def _on_timer() -> None:
+            with self._lock:
+                self._flush_timer = None
+                if self._destroyed:
+                    return
+                if self._flush_callback and self._entries:
+                    self._flush_callback()
+
+        self._flush_timer = threading.Timer(self._config.flush_interval / 1000, _on_timer)
+        self._flush_timer.daemon = True
+        self._flush_timer.start()
+
+    @staticmethod
+    def _to_timestamp(value: str | None) -> float | None:
+        if value is None:
+            return None
+        try:
+            normalized = value.replace("Z", "+00:00")
+            return datetime.fromisoformat(normalized).timestamp()
+        except Exception:
+            return None

logs_interceptor/infrastructure/circuit_breaker/__init__.py

@@ -0,0 +1,3 @@
+from .circuit_breaker import CircuitBreaker
+
+__all__ = ["CircuitBreaker"]