elven-logs-interceptor-python 0.1.2__py3-none-any.whl

logs_interceptor/infrastructure/interceptors/runtime_interceptor.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import builtins
+import logging
+import sys
+import traceback as traceback_module
+from types import TracebackType
+from typing import Any
+
+from ...domain.interfaces import ILogger
+from ...types import LogLevel
+from ...utils import safe_stringify
+
+
+class _BridgeLoggingHandler(logging.Handler):
+    def __init__(self, logger: ILogger) -> None:
+        super().__init__()
+        self._logger = logger
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            level = self._map_level(record.levelno)
+            msg = record.getMessage()
+            context = {
+                "source": "logging",
+                "logger_name": record.name,
+                "module": record.module,
+                "function": record.funcName,
+                "line": record.lineno,
+            }
+            if record.exc_info:
+                context["exc_info"] = "".join(traceback_module.format_exception(*record.exc_info))
+            self._logger.log(level, msg, context)
+        except Exception:
+            return
+
+    @staticmethod
+    def _map_level(level_no: int) -> LogLevel:
+        if level_no >= logging.CRITICAL:
+            return "fatal"
+        if level_no >= logging.ERROR:
+            return "error"
+        if level_no >= logging.WARNING:
+            return "warn"
+        if level_no >= logging.INFO:
+            return "info"
+        return "debug"
+
+
+class RuntimeInterceptor:
+    def __init__(self, logger: ILogger, preserve_original: bool = True) -> None:
+        self._logger = logger
+        self._preserve_original = preserve_original
+        self._enabled = False
+
+        self._original_print = builtins.print
+        self._original_excepthook = sys.excepthook
+        self._root_logger = logging.getLogger()
+        self._bridge_handler = _BridgeLoggingHandler(logger)
+        self._original_handlers: list[logging.Handler] = []
+
+    def enable(self) -> None:
+        if self._enabled:
+            return
+        self._enabled = True
+
+        self._patch_print()
+        self._patch_excepthook()
+        self._patch_logging()
+
+    def restore(self) -> None:
+        if not self._enabled:
+            return
+
+        builtins.print = self._original_print
+        sys.excepthook = self._original_excepthook
+
+        try:
+            self._root_logger.removeHandler(self._bridge_handler)
+        except ValueError:
+            pass
+
+        if not self._preserve_original:
+            self._root_logger.handlers = self._original_handlers
+
+        self._enabled = False
+
+    def is_enabled(self) -> bool:
+        return self._enabled
+
+    def _patch_print(self) -> None:
+        def intercepted_print(*args: Any, **kwargs: Any) -> None:
+            try:
+                message = " ".join(
+                    [arg if isinstance(arg, str) else safe_stringify(arg) for arg in args]
+                )
+                self._logger.info(message, {"source": "print"})
+            except Exception:
+                pass
+
+            if self._preserve_original:
+                self._original_print(*args, **kwargs)
+
+        builtins.print = intercepted_print
+
+    def _patch_excepthook(self) -> None:
+        def intercepted_excepthook(
+            exc_type: type[BaseException],
+            exc_value: BaseException,
+            traceback: TracebackType | None,
+        ) -> None:
+            try:
+                self._logger.fatal(
+                    f"Uncaught exception: {exc_value}",
+                    {
+                        "source": "sys.excepthook",
+                        "exception_type": exc_type.__name__,
+                        "traceback": "".join(traceback_module.format_tb(traceback))
+                        if traceback
+                        else None,
+                    },
+                )
+                self._logger.flush()
+            except Exception:
+                pass
+
+            if self._preserve_original and self._original_excepthook is not None:
+                self._original_excepthook(exc_type, exc_value, traceback)
+
+        sys.excepthook = intercepted_excepthook
+
+    def _patch_logging(self) -> None:
+        if not self._preserve_original:
+            self._original_handlers = list(self._root_logger.handlers)
+            self._root_logger.handlers = []
+
+        self._bridge_handler.setLevel(logging.DEBUG)
+        self._root_logger.addHandler(self._bridge_handler)
+        self._root_logger.setLevel(min(self._root_logger.level, logging.DEBUG) or logging.DEBUG)

logs_interceptor/infrastructure/memory/__init__.py

@@ -0,0 +1,3 @@
+from .memory_tracker import MemoryStats, MemoryTracker
+
+__all__ = ["MemoryTracker", "MemoryStats"]

logs_interceptor/infrastructure/memory/memory_tracker.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ...domain.entities import LogEntryEntity
+
+
+@dataclass(slots=True)
+class MemoryStats:
+    total_bytes: int
+    total_mb: float
+    entry_count: int
+    avg_entry_size: float
+
+
+class MemoryTracker:
+    def __init__(self) -> None:
+        self._total_size = 0
+        self._entry_sizes: dict[int, int] = {}
+        self._entry_count = 0
+
+    def add_entry(self, entry: LogEntryEntity) -> None:
+        size = self._estimate_size(entry)
+        self._entry_sizes[id(entry)] = size
+        self._total_size += size
+        self._entry_count += 1
+
+    def remove_entry(self, entry: LogEntryEntity) -> None:
+        key = id(entry)
+        size = self._entry_sizes.pop(key, None)
+        if size is None:
+            return
+        self._total_size -= size
+        self._entry_count -= 1
+
+    def remove_entries(self, entries: list[LogEntryEntity]) -> None:
+        for entry in entries:
+            self.remove_entry(entry)
+
+    def get_total_size(self) -> int:
+        return self._total_size
+
+    def get_total_size_mb(self) -> float:
+        return self._total_size / 1024 / 1024
+
+    def get_entry_count(self) -> int:
+        return self._entry_count
+
+    def get_avg_entry_size(self) -> float:
+        if self._entry_count <= 0:
+            return 0.0
+        return self._total_size / self._entry_count
+
+    def get_stats(self) -> MemoryStats:
+        return MemoryStats(
+            total_bytes=self._total_size,
+            total_mb=self.get_total_size_mb(),
+            entry_count=self._entry_count,
+            avg_entry_size=self.get_avg_entry_size(),
+        )
+
+    def reset(self) -> None:
+        self._total_size = 0
+        self._entry_sizes.clear()
+        self._entry_count = 0
+
+    def _estimate_size(self, entry: LogEntryEntity) -> int:
+        size = 0
+        size += len(entry.id) * 2
+        size += len(entry.timestamp) * 2
+        size += len(entry.level) * 2
+        size += len(entry.message) * 2
+
+        if entry.trace_id:
+            size += len(entry.trace_id) * 2
+        if entry.span_id:
+            size += len(entry.span_id) * 2
+        if entry.request_id:
+            size += len(entry.request_id) * 2
+
+        if entry.context:
+            keys = list(entry.context.keys())
+            size += len(keys) * 20
+            size += len(keys) * 50
+
+        if entry.labels:
+            for key, value in entry.labels.items():
+                size += len(key) * 2
+                size += len(value) * 2
+
+        if entry.metadata:
+            size += 50
+
+        size += 200
+        return size

logs_interceptor/infrastructure/metrics/__init__.py

@@ -0,0 +1,3 @@
+from .metrics_collector import MetricsCollector
+
+__all__ = ["MetricsCollector"]

logs_interceptor/infrastructure/metrics/metrics_collector.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from statistics import mean
+from time import time
+
+
+class MetricsCollector:
+    def __init__(self, max_samples: int = 10_000) -> None:
+        self._latencies: list[float] = []
+        self._compression_ratios: list[float] = []
+        self._compression_times: list[float] = []
+        self._operation_timestamps: list[float] = []
+        self._total_original_bytes = 0
+        self._total_compressed_bytes = 0
+        self._max_samples = max_samples
+
+    def record_latency(self, ms: float) -> None:
+        self._latencies.append(ms)
+        self._operation_timestamps.append(time())
+
+        if len(self._latencies) > self._max_samples:
+            self._latencies.pop(0)
+        if len(self._operation_timestamps) > self._max_samples:
+            self._operation_timestamps.pop(0)
+
+    def record_compression(self, original_size: int, compressed_size: int, time_ms: float) -> None:
+        self._total_original_bytes += original_size
+        self._total_compressed_bytes += compressed_size
+
+        if original_size > 0:
+            ratio = (1 - compressed_size / original_size) * 100
+            self._compression_ratios.append(ratio)
+            if len(self._compression_ratios) > self._max_samples:
+                self._compression_ratios.pop(0)
+
+        self._compression_times.append(time_ms)
+        if len(self._compression_times) > self._max_samples:
+            self._compression_times.pop(0)
+
+    def get_latency_metrics(self) -> dict[str, float]:
+        if not self._latencies:
+            return {
+                "p50": 0.0,
+                "p95": 0.0,
+                "p99": 0.0,
+                "p999": 0.0,
+                "min": 0.0,
+                "max": 0.0,
+                "avg": 0.0,
+                "count": 0.0,
+            }
+
+        sorted_values = sorted(self._latencies)
+        count = len(sorted_values)
+
+        return {
+            "p50": self._get_percentile(sorted_values, 50),
+            "p95": self._get_percentile(sorted_values, 95),
+            "p99": self._get_percentile(sorted_values, 99),
+            "p999": self._get_percentile(sorted_values, 99.9),
+            "min": sorted_values[0],
+            "max": sorted_values[-1],
+            "avg": mean(sorted_values),
+            "count": float(count),
+        }
+
+    def get_compression_metrics(self) -> dict[str, float | int]:
+        avg_ratio = mean(self._compression_ratios) if self._compression_ratios else 0.0
+        avg_time = mean(self._compression_times) if self._compression_times else 0.0
+
+        return {
+            "avg_ratio": avg_ratio,
+            "avg_time": avg_time,
+            "total_original_bytes": self._total_original_bytes,
+            "total_compressed_bytes": self._total_compressed_bytes,
+            "total_saved_bytes": self._total_original_bytes - self._total_compressed_bytes,
+            "count": len(self._compression_ratios),
+        }
+
+    def get_throughput(self, window_seconds: int = 60) -> float:
+        if window_seconds <= 0:
+            return 0.0
+        if not self._operation_timestamps:
+            return 0.0
+
+        now = time()
+        window_start = now - window_seconds
+        in_window = len([ts for ts in self._operation_timestamps if ts >= window_start])
+        return in_window / window_seconds
+
+    def reset(self) -> None:
+        self._latencies.clear()
+        self._compression_ratios.clear()
+        self._compression_times.clear()
+        self._operation_timestamps.clear()
+        self._total_original_bytes = 0
+        self._total_compressed_bytes = 0
+
+    def _get_percentile(self, sorted_values: list[float], percentile: float) -> float:
+        if not sorted_values:
+            return 0.0
+        index = int((percentile / 100) * len(sorted_values))
+        bounded = min(max(index, 1), len(sorted_values)) - 1
+        return sorted_values[bounded]

logs_interceptor/infrastructure/transport/__init__.py

@@ -0,0 +1,12 @@
+from .loki_json_transport import LokiJsonTransport
+from .loki_protobuf_transport import LokiProtobufTransport
+from .resilient_transport import ResilientTransport, ResilientTransportConfig
+from .transport_factory import TransportFactory
+
+__all__ = [
+    "LokiJsonTransport",
+    "LokiProtobufTransport",
+    "ResilientTransport",
+    "ResilientTransportConfig",
+    "TransportFactory",
+]

logs_interceptor/infrastructure/transport/loki_json_transport.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import json
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, cast
+
+import httpx
+
+from ...config import ResolvedTransportConfig
+from ...domain.entities import LogEntryEntity
+from ...types import TransportHealth, TransportMetrics
+from ..compression import CompressorConfig, CompressorFactory
+
+try:
+    import orjson  # type: ignore[import-not-found]
+except Exception:  # pragma: no cover - optional dependency
+    orjson = None
+
+
+@dataclass(slots=True)
+class RetryableTransportError(Exception):
+    message: str
+    status_code: int | None = None
+    retryable: bool = False
+
+    def __str__(self) -> str:
+        return self.message
+
+
+class LokiJsonTransport:
+    def __init__(
+        self,
+        config: ResolvedTransportConfig,
+        extra_headers: dict[str, str] | None = None,
+    ) -> None:
+        self._config = config
+        self._extra_headers = extra_headers or {}
+        self._timeout = config.timeout / 1000
+        self._compression_threshold = config.compression_threshold
+        self._compressor = CompressorFactory.create(
+            config.compression,
+            CompressorConfig(level=config.compression_level, threshold=config.compression_threshold),
+        )
+        limits = httpx.Limits(max_keepalive_connections=config.max_sockets, max_connections=config.max_sockets)
+        self._client: httpx.Client | None = None
+        if config.enable_connection_pooling:
+            self._client = httpx.Client(timeout=self._timeout, limits=limits)
+
+        self._health: TransportHealth = {
+            "healthy": True,
+            "consecutive_failures": 0,
+        }
+        self._metrics: TransportMetrics = {
+            "total_sends": 0,
+            "successful_sends": 0,
+            "failed_sends": 0,
+            "avg_latency": 0.0,
+            "avg_compression_time": 0.0,
+            "avg_compression_ratio": 0.0,
+            "total_bytes_sent": 0,
+            "total_bytes_compressed": 0,
+        }
+
+    def send(self, entries: list[LogEntryEntity]) -> None:
+        if not entries:
+            return
+
+        start = time.perf_counter()
+        self._metrics["total_sends"] = self._metrics.get("total_sends", 0) + 1
+
+        try:
+            payload = self._format_for_loki(entries)
+            raw_bytes = self._dumps(payload)
+            raw_size = len(raw_bytes)
+
+            body = raw_bytes
+            compression_time = 0.0
+            was_compressed = False
+            if self._compressor.get_name() != "none" and raw_size >= self._compression_threshold:
+                compression_start = time.perf_counter()
+                body = self._compressor.compress(raw_bytes)
+                compression_time = (time.perf_counter() - compression_start) * 1000
+                was_compressed = True
+
+            headers: dict[str, str] = {
+                "Content-Type": "application/json",
+                "X-Scope-OrgID": self._config.tenant_id,
+                "User-Agent": "elven-logs-interceptor-python/0.1.2",
+                **self._extra_headers,
+            }
+            if self._config.auth_token:
+                headers["Authorization"] = f"Bearer {self._config.auth_token}"
+            if was_compressed:
+                encoding = self._compressor.get_content_encoding()
+                if encoding:
+                    headers["Content-Encoding"] = encoding
+
+            response = self._request(headers, body)
+
+            if response.status_code >= 300:
+                raise RetryableTransportError(
+                    message=f"Loki responded with {response.status_code}: {response.text}",
+                    status_code=response.status_code,
+                    retryable=response.status_code == 429 or response.status_code >= 500,
+                )
+
+            latency = (time.perf_counter() - start) * 1000
+            self._record_success(latency)
+            self._metrics["total_bytes_sent"] = self._metrics.get("total_bytes_sent", 0) + len(body)
+            self._metrics["total_bytes_compressed"] = self._metrics.get("total_bytes_compressed", 0) + raw_size
+
+            if was_compressed:
+                self._update_compression_metrics(compression_time, raw_size, len(body))
+        except Exception as exc:
+            self._record_failure(exc)
+            raise
+
+    def _request(self, headers: dict[str, str], body: bytes) -> httpx.Response:
+        if self._client is not None:
+            return self._client.post(self._config.url, headers=headers, content=body)
+
+        with httpx.Client(timeout=self._timeout) as client:
+            return client.post(self._config.url, headers=headers, content=body)
+
+    def is_available(self) -> bool:
+        return bool(self._health.get("healthy", False))
+
+    def get_health(self) -> TransportHealth:
+        return cast(TransportHealth, dict(self._health))
+
+    def get_metrics(self) -> TransportMetrics:
+        return cast(TransportMetrics, dict(self._metrics))
+
+    def destroy(self) -> None:
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+    def _record_success(self, duration_ms: float) -> None:
+        self._health = {
+            "healthy": True,
+            "consecutive_failures": 0,
+            "last_successful_send": time.time(),
+        }
+        successful = self._metrics.get("successful_sends", 0) + 1
+        self._metrics["successful_sends"] = successful
+        current_avg = float(self._metrics.get("avg_latency", 0.0))
+        self._metrics["avg_latency"] = ((current_avg * (successful - 1)) + duration_ms) / successful
+
+    def _record_failure(self, error: Exception) -> None:
+        failures = int(self._health.get("consecutive_failures", 0)) + 1
+        self._health = {
+            "healthy": False,
+            "consecutive_failures": failures,
+            "error_message": str(error),
+        }
+        self._metrics["failed_sends"] = self._metrics.get("failed_sends", 0) + 1
+
+    def _update_compression_metrics(self, duration_ms: float, raw_size: int, compressed_size: int) -> None:
+        successful = max(1, self._metrics.get("successful_sends", 1))
+        current_time = float(self._metrics.get("avg_compression_time", 0.0))
+        self._metrics["avg_compression_time"] = (
+            (current_time * (successful - 1)) + duration_ms
+        ) / successful
+
+        ratio = compressed_size / raw_size if raw_size > 0 else 1.0
+        current_ratio = float(self._metrics.get("avg_compression_ratio", 0.0))
+        self._metrics["avg_compression_ratio"] = (
+            (current_ratio * (successful - 1)) + ratio
+        ) / successful
+
+    def _format_for_loki(self, entries: list[LogEntryEntity]) -> dict[str, Any]:
+        streams: dict[str, dict[str, Any]] = {}
+        values_map: dict[str, list[list[str]]] = defaultdict(list)
+
+        for entry in entries:
+            labels = entry.labels or {}
+            key = json.dumps(labels, sort_keys=True)
+
+            payload: dict[str, Any] = {
+                "id": entry.id,
+                "level": entry.level,
+                "message": entry.message,
+                "context": entry.context,
+            }
+            if entry.trace_id:
+                payload["traceId"] = entry.trace_id
+            if entry.span_id:
+                payload["spanId"] = entry.span_id
+            if entry.request_id:
+                payload["requestId"] = entry.request_id
+            if entry.metadata:
+                payload["metadata"] = entry.metadata
+
+            timestamp_ns = self._timestamp_to_ns(entry.timestamp)
+            values_map[key].append([str(timestamp_ns), self._dumps(payload).decode("utf-8")])
+            streams[key] = labels
+
+        formatted_streams: list[dict[str, Any]] = []
+        for key, stream_labels in streams.items():
+            values = values_map[key]
+            values.sort(key=lambda item: item[0])
+            formatted_streams.append({"stream": stream_labels, "values": values})
+
+        return {"streams": formatted_streams}
+
+    @staticmethod
+    def _timestamp_to_ns(iso_timestamp: str) -> int:
+        try:
+            normalized = iso_timestamp.replace("Z", "+00:00")
+            dt_obj = datetime.fromisoformat(normalized)
+            if dt_obj.tzinfo is None:
+                dt_obj = dt_obj.replace(tzinfo=timezone.utc)
+            seconds = int(dt_obj.timestamp())
+            return (seconds * 1_000_000_000) + (dt_obj.microsecond * 1000)
+        except Exception:
+            return int(time.time() * 1_000_000_000)
+
+    @staticmethod
+    def _dumps(payload: Any) -> bytes:
+        if orjson is not None:
+            return cast(bytes, orjson.dumps(payload))
+        return json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")