digitalkin 0.3.2.dev7__py3-none-any.whl → 0.3.2.dev10__py3-none-any.whl

monitoring/digitalkin_observability/__init__.py

@@ -0,0 +1,46 @@
+"""Standalone observability module for DigitalKin.
+
+This module can be copied into your project and used independently.
+It has no dependencies on the digitalkin package.
+
+Usage:
+    from digitalkin_observability import (
+        MetricsCollector,
+        MetricsServer,
+        MetricsServerInterceptor,
+        PrometheusExporter,
+        get_metrics,
+        start_metrics_server,
+        stop_metrics_server,
+    )
+
+    # Start metrics HTTP server
+    start_metrics_server(port=8081)
+
+    # Track metrics
+    metrics = get_metrics()
+    metrics.inc_jobs_started("my_module")
+    metrics.inc_jobs_completed("my_module", duration=1.5)
+
+    # Export to Prometheus format
+    print(PrometheusExporter.export())
+"""
+
+from digitalkin_observability.http_server import (
+    MetricsServer,
+    start_metrics_server,
+    stop_metrics_server,
+)
+from digitalkin_observability.interceptors import MetricsServerInterceptor
+from digitalkin_observability.metrics import MetricsCollector, get_metrics
+from digitalkin_observability.prometheus import PrometheusExporter
+
+__all__ = [
+    "MetricsCollector",
+    "MetricsServer",
+    "MetricsServerInterceptor",
+    "PrometheusExporter",
+    "get_metrics",
+    "start_metrics_server",
+    "stop_metrics_server",
+]

monitoring/digitalkin_observability/http_server.py

@@ -0,0 +1,150 @@
+"""Simple HTTP server for exposing Prometheus metrics.
+
+This module provides an HTTP server that exposes metrics at /metrics endpoint.
+No external dependencies required beyond Python standard library.
+"""
+
+from __future__ import annotations
+
+import logging
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from threading import Thread
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from typing import Self
+
+logger = logging.getLogger(__name__)
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    """HTTP request handler for metrics endpoint."""
+
+    def do_GET(self) -> None:
+        """Handle GET requests."""
+        if self.path == "/metrics":
+            self._serve_metrics()
+        elif self.path == "/health":
+            self._serve_health()
+        else:
+            self.send_error(404, "Not Found")
+
+    def _serve_metrics(self) -> None:
+        """Serve Prometheus metrics."""
+        from digitalkin_observability.prometheus import PrometheusExporter
+
+        content = PrometheusExporter.export()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; charset=utf-8")
+        self.send_header("Content-Length", str(len(content)))
+        self.end_headers()
+        self.wfile.write(content.encode("utf-8"))
+
+    def _serve_health(self) -> None:
+        """Serve health check."""
+        content = '{"status": "ok"}'
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(content)))
+        self.end_headers()
+        self.wfile.write(content.encode("utf-8"))
+
+    def log_message(self, format: str, *args: object) -> None:
+        """Suppress default logging."""
+
+
+class MetricsServer:
+    """HTTP server for exposing metrics to Prometheus.
+
+    Usage:
+        server = MetricsServer(port=8081)
+        server.start()
+        # ... run your application ...
+        server.stop()
+
+    Or as context manager:
+        with MetricsServer(port=8081):
+            # ... run your application ...
+
+    Or as async context manager:
+        async with MetricsServer(port=8081):
+            # ... run your application ...
+    """
+
+    instance: ClassVar["MetricsServer | None"] = None
+
+    def __init__(self, host: str = "0.0.0.0", port: int = 8081) -> None:
+        """Initialize the metrics server.
+
+        Args:
+            host: Host to bind to (default: 0.0.0.0 for all interfaces).
+            port: Port to listen on (default: 8081).
+        """
+        self.host = host
+        self.port = port
+        self._server: HTTPServer | None = None
+        self._thread: Thread | None = None
+
+    def start(self) -> None:
+        """Start the metrics server in a background thread."""
+        if self._server is not None:
+            logger.warning("Metrics server already running")
+            return
+
+        self._server = HTTPServer((self.host, self.port), MetricsHandler)
+        self._thread = Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+        logger.info(
+            "Metrics server started on http://%s:%s/metrics",
+            self.host,
+            self.port,
+        )
+
+    def stop(self) -> None:
+        """Stop the metrics server."""
+        if self._server is not None:
+            self._server.shutdown()
+            self._server = None
+            self._thread = None
+            logger.info("Metrics server stopped")
+
+    async def __aenter__(self) -> "Self":
+        """Async context manager entry."""
+        self.start()
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        """Async context manager exit."""
+        self.stop()
+
+    def __enter__(self) -> "Self":
+        """Context manager entry."""
+        self.start()
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        """Context manager exit."""
+        self.stop()
+
+
+def start_metrics_server(host: str = "0.0.0.0", port: int = 8081) -> MetricsServer:
+    """Start a metrics server singleton.
+
+    Args:
+        host: Host to bind to.
+        port: Port to listen on.
+
+    Returns:
+        The MetricsServer instance.
+    """
+    if MetricsServer.instance is None:
+        MetricsServer.instance = MetricsServer(host, port)
+        MetricsServer.instance.start()
+    return MetricsServer.instance
+
+
+def stop_metrics_server() -> None:
+    """Stop the metrics server singleton."""
+    if MetricsServer.instance is not None:
+        MetricsServer.instance.stop()
+        MetricsServer.instance = None

monitoring/digitalkin_observability/interceptors.py

@@ -0,0 +1,176 @@
+"""gRPC interceptors for automatic metrics collection.
+
+This module provides gRPC server interceptors that automatically track
+request duration and errors. Requires grpcio package.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from typing import Any
+
+    import grpc
+
+from digitalkin_observability.metrics import get_metrics
+
+
+class MetricsServerInterceptor:
+    """Intercepts all gRPC calls to collect metrics.
+
+    This interceptor automatically tracks:
+    - Request duration (histogram)
+    - Error counts
+
+    Usage:
+        import grpc
+        from digitalkin_observability import MetricsServerInterceptor
+
+        interceptors = [MetricsServerInterceptor()]
+        server = grpc.aio.server(interceptors=interceptors)
+    """
+
+    async def intercept_service(
+        self,
+        continuation: Callable[["grpc.HandlerCallDetails"], Awaitable["grpc.RpcMethodHandler"]],
+        handler_call_details: "grpc.HandlerCallDetails",
+    ) -> "grpc.RpcMethodHandler":
+        """Intercept a gRPC service call to collect metrics.
+
+        Args:
+            continuation: The next interceptor or the actual handler.
+            handler_call_details: Details about the call being intercepted.
+
+        Returns:
+            The RPC method handler.
+        """
+        start = time.perf_counter()
+        metrics = get_metrics()
+
+        try:
+            handler = await continuation(handler_call_details)
+            return _MetricsWrappedHandler(handler, start, handler_call_details.method)
+        except Exception:
+            metrics.inc_errors()
+            metrics.observe_grpc_duration(time.perf_counter() - start)
+            raise
+
+
+class _MetricsWrappedHandler:
+    """Wrapper that measures actual handler execution time."""
+
+    def __init__(
+        self,
+        handler: "grpc.RpcMethodHandler",
+        start_time: float,
+        method: str,
+    ) -> None:
+        self._handler = handler
+        self._start_time = start_time
+        self._method = method
+
+        # Copy attributes from original handler
+        self.request_streaming = handler.request_streaming
+        self.response_streaming = handler.response_streaming
+        self.request_deserializer = handler.request_deserializer
+        self.response_serializer = handler.response_serializer
+
+        # Wrap the appropriate method based on streaming type
+        if handler.unary_unary:
+            self.unary_unary = self._wrap_unary_unary(handler.unary_unary)
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = None
+        elif handler.unary_stream:
+            self.unary_unary = None
+            self.unary_stream = self._wrap_unary_stream(handler.unary_stream)
+            self.stream_unary = None
+            self.stream_stream = None
+        elif handler.stream_unary:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = self._wrap_stream_unary(handler.stream_unary)
+            self.stream_stream = None
+        elif handler.stream_stream:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = self._wrap_stream_stream(handler.stream_stream)
+        else:
+            self.unary_unary = None
+            self.unary_stream = None
+            self.stream_unary = None
+            self.stream_stream = None
+
+    def _wrap_unary_unary(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]]:
+        """Wrap a unary-unary handler."""
+        async def wrapped(request: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                return await handler(request, context)
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_unary_stream(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], "Any"],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], "Any"]:
+        """Wrap a unary-stream handler."""
+        async def wrapped(request: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                async for response in handler(request, context):
+                    yield response
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_stream_unary(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], Awaitable["Any"]]:
+        """Wrap a stream-unary handler."""
+        async def wrapped(request_iterator: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                return await handler(request_iterator, context)
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped
+
+    def _wrap_stream_stream(
+        self,
+        handler: Callable[["Any", "grpc.aio.ServicerContext"], "Any"],
+    ) -> Callable[["Any", "grpc.aio.ServicerContext"], "Any"]:
+        """Wrap a stream-stream handler."""
+        async def wrapped(request_iterator: "Any", context: "grpc.aio.ServicerContext") -> "Any":
+            metrics = get_metrics()
+            try:
+                async for response in handler(request_iterator, context):
+                    yield response
+            except Exception:
+                metrics.inc_errors()
+                raise
+            finally:
+                metrics.observe_grpc_duration(time.perf_counter() - self._start_time)
+
+        return wrapped

monitoring/digitalkin_observability/metrics.py

@@ -0,0 +1,201 @@
+"""Core metrics collection for DigitalKin.
+
+This module provides a thread-safe singleton MetricsCollector that tracks
+various metrics about job execution, gRPC requests, and system performance.
+
+No external dependencies required.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from threading import Lock
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@dataclass
+class Histogram:
+    """Simple histogram with configurable buckets."""
+
+    buckets: tuple[float, ...] = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
+    counts: dict[float, int] = field(default_factory=lambda: defaultdict(int))
+    total_sum: float = 0.0
+    count: int = 0
+
+    def observe(self, value: float) -> None:
+        """Record an observation in the histogram."""
+        self.total_sum += value
+        self.count += 1
+        for bucket in self.buckets:
+            if value <= bucket:
+                self.counts[bucket] += 1
+
+    def reset(self) -> None:
+        """Reset histogram state."""
+        self.counts = defaultdict(int)
+        self.total_sum = 0.0
+        self.count = 0
+
+
+class MetricsCollector:
+    """Thread-safe singleton metrics collector.
+
+    Collects various metrics about job execution, gRPC requests,
+    and system performance. Designed to be stateless per-request
+    while maintaining aggregate counters.
+
+    Usage:
+        metrics = MetricsCollector()  # or get_metrics()
+        metrics.inc_jobs_started("my_module")
+        metrics.inc_jobs_completed("my_module", duration=1.5)
+        print(metrics.snapshot())
+    """
+
+    _instance: ClassVar[MetricsCollector | None] = None
+    _lock: ClassVar[Lock] = Lock()
+
+    def __new__(cls) -> "MetricsCollector":
+        """Create or return the singleton instance."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    instance = super().__new__(cls)
+                    instance._init_metrics()
+                    cls._instance = instance
+        return cls._instance
+
+    def _init_metrics(self) -> None:
+        """Initialize all metric storage."""
+        # Counters
+        self.jobs_started_total: int = 0
+        self.jobs_completed_total: int = 0
+        self.jobs_failed_total: int = 0
+        self.jobs_cancelled_total: int = 0
+        self.messages_sent_total: int = 0
+        self.heartbeats_sent_total: int = 0
+        self.errors_total: int = 0
+
+        # Gauges
+        self.active_jobs: int = 0
+        self.active_connections: int = 0
+        self.queue_depth: dict[str, int] = {}
+
+        # Histograms
+        self.job_duration_seconds = Histogram()
+        self.message_latency_seconds = Histogram()
+        self.grpc_request_duration_seconds = Histogram()
+
+        # Labels for breakdown
+        self._by_module: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+        self._by_protocol: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+        # Instance lock for thread safety
+        self._instance_lock = Lock()
+
+    def inc_jobs_started(self, module_name: str) -> None:
+        """Increment jobs started counter."""
+        with self._instance_lock:
+            self.jobs_started_total += 1
+            self.active_jobs += 1
+            self._by_module[module_name]["started"] += 1
+
+    def inc_jobs_completed(self, module_name: str, duration: float) -> None:
+        """Increment jobs completed counter and record duration."""
+        with self._instance_lock:
+            self.jobs_completed_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["completed"] += 1
+            self.job_duration_seconds.observe(duration)
+
+    def inc_jobs_failed(self, module_name: str) -> None:
+        """Increment jobs failed counter."""
+        with self._instance_lock:
+            self.jobs_failed_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["failed"] += 1
+
+    def inc_jobs_cancelled(self, module_name: str) -> None:
+        """Increment jobs cancelled counter."""
+        with self._instance_lock:
+            self.jobs_cancelled_total += 1
+            self.active_jobs = max(0, self.active_jobs - 1)
+            self._by_module[module_name]["cancelled"] += 1
+
+    def inc_messages_sent(self, protocol: str | None = None) -> None:
+        """Increment messages sent counter."""
+        with self._instance_lock:
+            self.messages_sent_total += 1
+            if protocol:
+                self._by_protocol[protocol]["messages"] += 1
+
+    def inc_heartbeats_sent(self) -> None:
+        """Increment heartbeats sent counter."""
+        with self._instance_lock:
+            self.heartbeats_sent_total += 1
+
+    def inc_errors(self) -> None:
+        """Increment errors counter."""
+        with self._instance_lock:
+            self.errors_total += 1
+
+    def set_queue_depth(self, job_id: str, depth: int) -> None:
+        """Set the queue depth for a job."""
+        with self._instance_lock:
+            self.queue_depth[job_id] = depth
+
+    def clear_queue_depth(self, job_id: str) -> None:
+        """Clear queue depth tracking for a job."""
+        with self._instance_lock:
+            self.queue_depth.pop(job_id, None)
+
+    def observe_grpc_duration(self, duration: float) -> None:
+        """Record a gRPC request duration."""
+        with self._instance_lock:
+            self.grpc_request_duration_seconds.observe(duration)
+
+    def observe_message_latency(self, latency: float) -> None:
+        """Record a message latency."""
+        with self._instance_lock:
+            self.message_latency_seconds.observe(latency)
+
+    def snapshot(self) -> dict[str, Any]:
+        """Return current metrics as dict for export."""
+        with self._instance_lock:
+            return {
+                "jobs_started_total": self.jobs_started_total,
+                "jobs_completed_total": self.jobs_completed_total,
+                "jobs_failed_total": self.jobs_failed_total,
+                "jobs_cancelled_total": self.jobs_cancelled_total,
+                "active_jobs": self.active_jobs,
+                "messages_sent_total": self.messages_sent_total,
+                "heartbeats_sent_total": self.heartbeats_sent_total,
+                "errors_total": self.errors_total,
+                "active_connections": self.active_connections,
+                "total_queue_depth": sum(self.queue_depth.values()),
+                "job_duration_seconds": {
+                    "count": self.job_duration_seconds.count,
+                    "sum": self.job_duration_seconds.total_sum,
+                    "buckets": dict(self.job_duration_seconds.counts),
+                },
+                "grpc_request_duration_seconds": {
+                    "count": self.grpc_request_duration_seconds.count,
+                    "sum": self.grpc_request_duration_seconds.total_sum,
+                    "buckets": dict(self.grpc_request_duration_seconds.counts),
+                },
+                "by_module": {k: dict(v) for k, v in self._by_module.items()},
+                "by_protocol": {k: dict(v) for k, v in self._by_protocol.items()},
+            }
+
+    def reset(self) -> None:
+        """Reset all metrics. Useful for testing."""
+        with self._instance_lock:
+            self._init_metrics()
+
+
+def get_metrics() -> MetricsCollector:
+    """Get the global MetricsCollector instance."""
+    return MetricsCollector()