elven-logs-interceptor-python 0.1.2__py3-none-any.whl

logs_interceptor/infrastructure/transport/loki_protobuf_transport.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import json
+import os
+import time
+from dataclasses import dataclass
+from typing import Any, cast
+
+import httpx
+
+from ...config import ResolvedTransportConfig
+from ...domain.entities import LogEntryEntity
+from ...types import TransportHealth, TransportMetrics
+
+try:
+    import snappy  # type: ignore[import-not-found]
+except Exception:  # pragma: no cover - optional dependency
+    snappy = None
+
+@dataclass(slots=True)
+class RetryableTransportError(Exception):
+    message: str
+    status_code: int | None = None
+    retryable: bool = False
+
+    def __str__(self) -> str:
+        return self.message
+
+
+class LokiProtobufTransport:
+    """
+    Protobuf/Snappy transport with graceful fallback semantics.
+
+    The runtime payload is encoded as JSON bytes and then Snappy-compressed
+    when protobuf schemas are unavailable in environment.
+    """
+
+    def __init__(
+        self,
+        config: ResolvedTransportConfig,
+        extra_headers: dict[str, str] | None = None,
+    ) -> None:
+        # This transport requires a real protobuf payload implementation.
+        # Keep it explicitly opt-in to avoid silent breakage in production
+        # when users install `python-snappy` but don't expect experimental behavior.
+        experimental_enabled = (os.getenv("LOGS_ENABLE_EXPERIMENTAL_PROTOBUF") or "").strip().lower()
+        if experimental_enabled not in {"1", "true", "yes", "on"}:
+            raise RuntimeError("Set LOGS_ENABLE_EXPERIMENTAL_PROTOBUF=true to enable LokiProtobufTransport")
+
+        if snappy is None:
+            raise RuntimeError("python-snappy is required for compression='snappy'")
+
+        self._config = config
+        self._extra_headers = extra_headers or {}
+        self._timeout = config.timeout / 1000
+        self._client: httpx.Client | None = None
+        if config.enable_connection_pooling:
+            limits = httpx.Limits(max_keepalive_connections=config.max_sockets, max_connections=config.max_sockets)
+            self._client = httpx.Client(timeout=self._timeout, limits=limits)
+
+        self._health: TransportHealth = {
+            "healthy": True,
+            "consecutive_failures": 0,
+        }
+        self._metrics: TransportMetrics = {
+            "total_sends": 0,
+            "successful_sends": 0,
+            "failed_sends": 0,
+            "avg_latency": 0.0,
+            "avg_compression_time": 0.0,
+            "avg_compression_ratio": 0.0,
+            "total_bytes_sent": 0,
+            "total_bytes_compressed": 0,
+        }
+
+    def send(self, entries: list[LogEntryEntity]) -> None:
+        if not entries:
+            return
+
+        start = time.perf_counter()
+        self._metrics["total_sends"] = self._metrics.get("total_sends", 0) + 1
+
+        try:
+            payload = self._format_payload(entries)
+            raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+            raw_size = len(raw)
+
+            compression_start = time.perf_counter()
+            compressed = cast(bytes, snappy.compress(raw))
+            compression_time = (time.perf_counter() - compression_start) * 1000
+            compressed_size = len(compressed)
+
+            headers = {
+                "Content-Type": "application/x-protobuf",
+                "Content-Encoding": "snappy",
+                "X-Scope-OrgID": self._config.tenant_id,
+                "User-Agent": "elven-logs-interceptor-python/0.1.2",
+                **self._extra_headers,
+            }
+            if self._config.auth_token:
+                headers["Authorization"] = f"Bearer {self._config.auth_token}"
+
+            response = self._request(headers, compressed)
+            if response.status_code >= 300:
+                raise RetryableTransportError(
+                    message=f"Loki responded with {response.status_code}: {response.text}",
+                    status_code=response.status_code,
+                    retryable=response.status_code == 429 or response.status_code >= 500,
+                )
+
+            latency = (time.perf_counter() - start) * 1000
+            self._record_success(latency)
+            self._update_compression_metrics(compression_time, raw_size, compressed_size)
+            self._metrics["total_bytes_sent"] = self._metrics.get("total_bytes_sent", 0) + compressed_size
+            self._metrics["total_bytes_compressed"] = self._metrics.get("total_bytes_compressed", 0) + raw_size
+        except Exception as exc:
+            self._record_failure(exc)
+            raise
+
+    def _request(self, headers: dict[str, str], body: bytes) -> httpx.Response:
+        if self._client is not None:
+            return self._client.post(self._config.url, headers=headers, content=body)
+        with httpx.Client(timeout=self._timeout) as client:
+            return client.post(self._config.url, headers=headers, content=body)
+
+    def is_available(self) -> bool:
+        return bool(self._health.get("healthy", False))
+
+    def get_health(self) -> TransportHealth:
+        return cast(TransportHealth, dict(self._health))
+
+    def get_metrics(self) -> TransportMetrics:
+        return cast(TransportMetrics, dict(self._metrics))
+
+    def destroy(self) -> None:
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+    def _record_success(self, duration_ms: float) -> None:
+        self._health = {
+            "healthy": True,
+            "consecutive_failures": 0,
+            "last_successful_send": time.time(),
+        }
+        successful = self._metrics.get("successful_sends", 0) + 1
+        self._metrics["successful_sends"] = successful
+        current_avg = float(self._metrics.get("avg_latency", 0.0))
+        self._metrics["avg_latency"] = ((current_avg * (successful - 1)) + duration_ms) / successful
+
+    def _record_failure(self, error: Exception) -> None:
+        failures = int(self._health.get("consecutive_failures", 0)) + 1
+        self._health = {
+            "healthy": False,
+            "consecutive_failures": failures,
+            "error_message": str(error),
+        }
+        self._metrics["failed_sends"] = self._metrics.get("failed_sends", 0) + 1
+
+    def _update_compression_metrics(self, duration_ms: float, raw_size: int, compressed_size: int) -> None:
+        successful = max(1, self._metrics.get("successful_sends", 1))
+        current_time = float(self._metrics.get("avg_compression_time", 0.0))
+        self._metrics["avg_compression_time"] = (
+            (current_time * (successful - 1)) + duration_ms
+        ) / successful
+
+        ratio = compressed_size / raw_size if raw_size > 0 else 1.0
+        current_ratio = float(self._metrics.get("avg_compression_ratio", 0.0))
+        self._metrics["avg_compression_ratio"] = (
+            (current_ratio * (successful - 1)) + ratio
+        ) / successful
+
+    def _format_payload(self, entries: list[LogEntryEntity]) -> dict[str, Any]:
+        streams: dict[str, dict[str, Any]] = {}
+        values: dict[str, list[dict[str, Any]]] = {}
+
+        for entry in entries:
+            labels = entry.labels or {}
+            key = json.dumps(labels, sort_keys=True)
+            if key not in streams:
+                streams[key] = labels
+                values[key] = []
+            values[key].append(
+                {
+                    "timestamp": entry.timestamp,
+                    "line": json.dumps(
+                        {
+                            "level": entry.level,
+                            "message": entry.message,
+                            "context": entry.context,
+                            "traceId": entry.trace_id,
+                            "spanId": entry.span_id,
+                            "requestId": entry.request_id,
+                            "metadata": entry.metadata,
+                        },
+                        separators=(",", ":"),
+                    ),
+                }
+            )
+
+        return {
+            "streams": [
+                {
+                    "labels": stream_labels,
+                    "entries": values[key],
+                }
+                for key, stream_labels in streams.items()
+            ]
+        }

logs_interceptor/infrastructure/transport/resilient_transport.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import random
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import cast
+
+from ...domain.entities import LogEntryEntity
+from ...domain.interfaces import ICircuitBreaker, IDeadLetterQueue, ILogTransport
+from ...types import TransportHealth, TransportMetrics
+from ...utils import internal_warn
+
+
+@dataclass(slots=True)
+class ResilientTransportConfig:
+    max_retries: int = 3
+    retry_delay: int = 1000
+
+
+class ResilientTransport:
+    def __init__(
+        self,
+        transport: ILogTransport,
+        config: ResilientTransportConfig,
+        circuit_breaker: ICircuitBreaker | None = None,
+        dlq: IDeadLetterQueue | None = None,
+    ) -> None:
+        self._transport = transport
+        self._config = config
+        self._circuit_breaker = circuit_breaker
+        self._dlq = dlq
+        self._metrics: TransportMetrics = {
+            "total_sends": 0,
+            "successful_sends": 0,
+            "failed_sends": 0,
+            "avg_latency": 0.0,
+            "retry_attempts": 0,
+            "retried_requests": 0,
+            "dlq_dropped_entries": 0,
+        }
+        self._last_dlq_warning = 0.0
+
+    def send(self, entries: list[LogEntryEntity]) -> None:
+        if not entries:
+            return
+
+        self._metrics["total_sends"] = self._metrics.get("total_sends", 0) + 1
+
+        def operation() -> None:
+            self._retry_operation(lambda: self._transport.send(entries))
+
+        try:
+            if self._circuit_breaker is not None:
+                self._circuit_breaker.execute(operation)
+            else:
+                operation()
+            self._metrics["successful_sends"] = self._metrics.get("successful_sends", 0) + 1
+        except Exception as exc:
+            self._metrics["failed_sends"] = self._metrics.get("failed_sends", 0) + 1
+            self._enqueue_dlq(entries, exc)
+            raise
+
+    def _enqueue_dlq(self, entries: list[LogEntryEntity], error: Exception) -> None:
+        if self._dlq is None:
+            return
+        try:
+            result = self._dlq.add_batch(entries, str(error))
+            self._metrics["dlq_dropped_entries"] = self._metrics.get("dlq_dropped_entries", 0) + result.get(
+                "dropped", 0
+            )
+        except Exception as dlq_error:
+            now = time.time()
+            if now - self._last_dlq_warning > 10:
+                self._last_dlq_warning = now
+                internal_warn("Failed to enqueue logs to DLQ", dlq_error)
+
+    def _retry_operation(self, operation: Callable[[], None]) -> None:
+        max_retries = self._config.max_retries
+        delay = self._config.retry_delay
+        total_attempts = max_retries + 1
+        request_retried = False
+
+        for attempt in range(total_attempts):
+            try:
+                operation()
+                return
+            except Exception as exc:
+                should_retry = attempt < total_attempts - 1 and self._is_retryable_error(exc)
+                if not should_retry:
+                    raise
+
+                if not request_retried:
+                    request_retried = True
+                    self._metrics["retried_requests"] = self._metrics.get("retried_requests", 0) + 1
+                self._metrics["retry_attempts"] = self._metrics.get("retry_attempts", 0) + 1
+
+                base_delay = delay * (2**attempt)
+                jitter = random.randint(0, 250)
+                time.sleep((base_delay + jitter) / 1000)
+
+    @staticmethod
+    def _is_retryable_error(error: Exception) -> bool:
+        retryable = getattr(error, "retryable", None)
+        if isinstance(retryable, bool):
+            return retryable
+
+        status_code = getattr(error, "status_code", None)
+        if isinstance(status_code, int):
+            return status_code == 429 or status_code >= 500
+
+        message = str(error).lower()
+        markers = [
+            "timeout",
+            "socket",
+            "connect",
+            "network",
+            "429",
+            "502",
+            "503",
+            "504",
+        ]
+        return any(marker in message for marker in markers)
+
+    def is_available(self) -> bool:
+        return self._transport.is_available()
+
+    def get_health(self) -> TransportHealth:
+        if self._circuit_breaker is not None:
+            state = self._circuit_breaker.get_state()
+            state_name = state.get("state")
+            if state_name == "open":
+                return {
+                    "healthy": False,
+                    "consecutive_failures": int(state.get("failures", 0)),
+                    "error_message": f"CircuitBreaker is OPEN. Last error: {state.get('last_error')}",
+                }
+            if state_name == "half-open":
+                return {
+                    "healthy": True,
+                    "consecutive_failures": int(state.get("failures", 0)),
+                    "error_message": "CircuitBreaker is HALF_OPEN",
+                }
+
+        return self._transport.get_health()
+
+    def get_metrics(self) -> TransportMetrics | None:
+        base = self._transport.get_metrics()
+        if base is None:
+            return cast(TransportMetrics, dict(self._metrics))
+
+        merged = dict(base)
+        merged["retry_attempts"] = base.get("retry_attempts", 0) + self._metrics.get("retry_attempts", 0)
+        merged["retried_requests"] = base.get("retried_requests", 0) + self._metrics.get("retried_requests", 0)
+        merged["dlq_dropped_entries"] = base.get("dlq_dropped_entries", 0) + self._metrics.get(
+            "dlq_dropped_entries", 0
+        )
+        return cast(TransportMetrics, merged)
+
+    def destroy(self) -> None:
+        self._transport.destroy()

logs_interceptor/infrastructure/transport/transport_factory.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from ...config import ResolvedLogsInterceptorConfig
+from ...domain.interfaces import ICircuitBreaker, IDeadLetterQueue, ILogTransport
+from ...utils import internal_debug, internal_warn
+from .loki_json_transport import LokiJsonTransport
+from .loki_protobuf_transport import LokiProtobufTransport
+from .resilient_transport import ResilientTransport, ResilientTransportConfig
+
+
+class TransportFactory:
+    @staticmethod
+    def create(
+        config: ResolvedLogsInterceptorConfig,
+        circuit_breaker: ICircuitBreaker | None = None,
+        dlq: IDeadLetterQueue | None = None,
+    ) -> ILogTransport:
+        base_transport: ILogTransport
+
+        if config.transport.compression == "snappy":
+            try:
+                internal_debug("Selected LokiProtobufTransport")
+                base_transport = LokiProtobufTransport(config.transport)
+            except Exception as exc:
+                internal_warn("Falling back to LokiJsonTransport because snappy/protobuf is unavailable", exc)
+                base_transport = LokiJsonTransport(config.transport)
+        else:
+            internal_debug("Selected LokiJsonTransport")
+            base_transport = LokiJsonTransport(config.transport)
+
+        return ResilientTransport(
+            base_transport,
+            ResilientTransportConfig(
+                max_retries=config.transport.max_retries,
+                retry_delay=config.transport.retry_delay,
+            ),
+            circuit_breaker,
+            dlq,
+        )

logs_interceptor/infrastructure/workers/__init__.py

@@ -0,0 +1,3 @@
+from .worker_pool import WorkerMetrics, WorkerPool
+
+__all__ = ["WorkerPool", "WorkerMetrics"]

logs_interceptor/infrastructure/workers/worker_pool.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from threading import Lock
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass(slots=True)
+class WorkerMetrics:
+    active_workers: int
+    queue_length: int
+    total_tasks: int
+    completed_tasks: int
+    failed_tasks: int
+
+
+class WorkerPool:
+    def __init__(self, max_workers: int | None = None) -> None:
+        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._lock = Lock()
+        self._total_tasks = 0
+        self._completed_tasks = 0
+        self._failed_tasks = 0
+
+    def execute(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> Future[T]:
+        with self._lock:
+            self._total_tasks += 1
+
+        future = self._executor.submit(fn, *args, **kwargs)
+
+        def _done_callback(done: Future[T]) -> None:
+            with self._lock:
+                if done.exception() is None:
+                    self._completed_tasks += 1
+                else:
+                    self._failed_tasks += 1
+
+        future.add_done_callback(_done_callback)
+        return future
+
+    def get_metrics(self) -> WorkerMetrics:
+        with self._lock:
+            queue_length = self._total_tasks - self._completed_tasks - self._failed_tasks
+            return WorkerMetrics(
+                active_workers=0,
+                queue_length=max(0, queue_length),
+                total_tasks=self._total_tasks,
+                completed_tasks=self._completed_tasks,
+                failed_tasks=self._failed_tasks,
+            )
+
+    def destroy(self) -> None:
+        self._executor.shutdown(wait=True, cancel_futures=True)

logs_interceptor/integrations/__init__.py

@@ -0,0 +1,17 @@
+from .celery import CelerySignals
+from .django import DjangoMiddleware
+from .fastapi import FastAPIMiddleware
+from .flask import FlaskExtension
+from .logging_handler import LoggingHandler
+from .loguru import LoguruSink
+from .structlog import StructlogProcessor
+
+__all__ = [
+    "LoggingHandler",
+    "FastAPIMiddleware",
+    "DjangoMiddleware",
+    "FlaskExtension",
+    "CelerySignals",
+    "StructlogProcessor",
+    "LoguruSink",
+]

logs_interceptor/integrations/celery.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ..domain.interfaces import ILogger
+
+
+class CelerySignals:
+    def __init__(self, logger: ILogger) -> None:
+        self.logger = logger
+
+    def register(self, app: Any) -> None:
+        try:
+            from celery import signals
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("celery extra is not installed") from exc
+
+        @signals.task_prerun.connect
+        def _task_prerun(task_id: str | None = None, task: Any = None, *args: Any, **kwargs: Any) -> None:
+            self.logger.info(
+                "Celery task started",
+                {
+                    "source": "celery",
+                    "task_id": task_id,
+                    "task_name": getattr(task, "name", None),
+                },
+            )
+
+        @signals.task_postrun.connect
+        def _task_postrun(task_id: str | None = None, task: Any = None, retval: Any = None, *args: Any, **kwargs: Any) -> None:
+            self.logger.info(
+                "Celery task completed",
+                {
+                    "source": "celery",
+                    "task_id": task_id,
+                    "task_name": getattr(task, "name", None),
+                    "retval": str(retval),
+                },
+            )
+
+        @signals.task_failure.connect
+        def _task_failure(task_id: str | None = None, task: Any = None, exception: Exception | None = None, *args: Any, **kwargs: Any) -> None:
+            self.logger.error(
+                "Celery task failed",
+                {
+                    "source": "celery",
+                    "task_id": task_id,
+                    "task_name": getattr(task, "name", None),
+                    "error": str(exception),
+                },
+            )
+
+        app.log.get_default_logger()

logs_interceptor/integrations/django.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from ..domain.interfaces import ILogger
+
+
+class DjangoMiddleware:
+    def __init__(self, get_response: Any, logger: ILogger) -> None:
+        self.get_response = get_response
+        self.logger = logger
+
+    def __call__(self, request: Any) -> Any:
+        start = time.time()
+        request_id = request.headers.get("X-Request-Id") if hasattr(request, "headers") else None
+        if not request_id:
+            request_id = f"req-{int(start * 1000)}"
+
+        def _run() -> Any:
+            return self.get_response(request)
+
+        response = self.logger.with_context({"request_id": request_id}, _run)
+
+        duration_ms = int((time.time() - start) * 1000)
+        status_code = int(getattr(response, "status_code", 200))
+        level = "info"
+        if status_code >= 500:
+            level = "error"
+        elif status_code >= 400:
+            level = "warn"
+
+        self.logger.log(
+            level,  # type: ignore[arg-type]
+            f"{request.method} {request.path}",
+            {
+                "type": "http_request",
+                "source": "django",
+                "status_code": status_code,
+                "duration_ms": duration_ms,
+            },
+        )
+
+        return response

logs_interceptor/integrations/fastapi.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from ..domain.interfaces import ILogger
+
+
+class FastAPIMiddleware:
+    def __init__(self, app: Any, logger: ILogger) -> None:
+        self.app = app
+        self.logger = logger
+
+    async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
+        if scope.get("type") != "http":
+            await self.app(scope, receive, send)
+            return
+
+        start = time.time()
+        headers = {k.decode("latin-1"): v.decode("latin-1") for k, v in scope.get("headers", [])}
+        request_id = headers.get("x-request-id") or f"req-{int(start * 1000)}"
+
+        status_code_holder = {"status": 200}
+
+        async def send_wrapper(message: dict[str, Any]) -> None:
+            if message.get("type") == "http.response.start":
+                status_code_holder["status"] = int(message.get("status", 200))
+            await send(message)
+
+        async def run_request() -> None:
+            await self.app(scope, receive, send_wrapper)
+
+        await self.logger.with_context_async({"request_id": request_id}, run_request)
+
+        duration_ms = int((time.time() - start) * 1000)
+        level = "info"
+        if status_code_holder["status"] >= 500:
+            level = "error"
+        elif status_code_holder["status"] >= 400:
+            level = "warn"
+
+        self.logger.log(
+            level,  # type: ignore[arg-type]
+            f"{scope.get('method', 'GET')} {scope.get('path', '/')}",
+            {
+                "type": "http_request",
+                "path": scope.get("path"),
+                "method": scope.get("method"),
+                "status_code": status_code_holder["status"],
+                "duration_ms": duration_ms,
+                "source": "fastapi",
+            },
+        )