021-python-observability 0.3.0__py3-none-any.whl

observability_021/logging.py

@@ -0,0 +1,209 @@
+"""JSON stdout logs. Only structured, approved fields cross the logging boundary."""
+
+import logging
+import re
+import sys
+from collections import deque
+from pathlib import Path
+from traceback import walk_tb
+from types import TracebackType
+from typing import cast
+from uuid import UUID
+
+import structlog
+from opentelemetry import trace
+
+_CORRELATION_FIELDS = (
+    "brain_id",
+    "job_id",
+    "file_id",
+    "file_version_id",
+    "entity_id",
+    "event_id",
+    "correlation_id",
+)
+_FIELDS = frozenset(
+    {
+        *_CORRELATION_FIELDS,
+        "event",
+        "level",
+        "timestamp",
+        "service_name",
+        "environment",
+        "service_version",
+        "request_id",
+        "trace_id",
+        "span_id",
+        "method",
+        "route",
+        "status_code",
+        "outcome",
+        "duration_ms",
+        "dependency",
+        "integration",
+        "operation",
+        "action",
+        "op",
+        "status",
+        "error_type",
+        "reason",
+        "signal",
+        "logger",
+    }
+)
+
+
+def _exception_details(exc_info):
+    if exc_info is True:
+        exc_info = sys.exc_info()
+    if isinstance(exc_info, BaseException):
+        exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
+    if not isinstance(exc_info, tuple) or len(exc_info) != 3:
+        return {}
+    _, error, tb = exc_info
+    if not isinstance(error, BaseException):
+        return {}
+    chain, seen = [], set()
+    # Bound log size while retaining the frames nearest each failure. Never call
+    # format_exception/extract_tb: source lines, locals and error messages stay out.
+    while error is not None and id(error) not in seen and len(chain) < 8:
+        seen.add(id(error))
+        frames = deque(maxlen=32)
+        count = 0
+        if isinstance(tb, TracebackType):
+            for frame, line in walk_tb(tb):
+                frames.append(
+                    {
+                        "file": Path(frame.f_code.co_filename).name[:256],
+                        "function": frame.f_code.co_name[:128],
+                        "line": line,
+                    }
+                )
+                count += 1
+        chain.append(
+            {
+                "type": type(error).__name__[:128],
+                "frames": list(frames),
+                "frames_truncated": count > len(frames),
+            }
+        )
+        cause = error.__cause__
+        error = (
+            cause
+            if cause is not None
+            else (None if error.__suppress_context__ else error.__context__)
+        )
+        tb = error.__traceback__ if error is not None else None
+    return {
+        "error_type": chain[0]["type"],
+        "exception": chain,
+        "exception_truncated": error is not None,
+    }
+
+
+def _sanitize(_logger, _method, event):
+    context = trace.get_current_span().get_span_context()
+    if context.is_valid:
+        event["trace_id"] = f"{context.trace_id:032x}"
+        event["span_id"] = f"{context.span_id:016x}"
+    # Never serialize raw exceptions, arbitrary extras, bodies, headers or SQL. Standard
+    # logging messages may interpolate secrets; retain only their source category.
+    if not event.get("_from_structlog") and "_record" in event:
+        record = event["_record"]
+        event["event"] = getattr(record, "event", "library.log")
+        if record.name.startswith("opentelemetry.exporter") and record.levelno >= logging.WARNING:
+            event["event"] = "telemetry.export_failed"
+            event["reason"] = "export_failed"
+            event["signal"] = "metrics" if "metric" in record.name else "traces"
+    name = event.get("event", "log")
+    if not isinstance(name, str) or not re.fullmatch(r"[a-z][a-z0-9_.]{0,95}", name):
+        event["event"] = "log"
+    result = {key: value for key, value in event.items() if key in _FIELDS}
+    for key in _CORRELATION_FIELDS:
+        value = result.pop(key, None)
+        if isinstance(value, UUID):
+            result[key] = str(value)
+        elif isinstance(value, str) and len(value) <= 36:
+            try:
+                result[key] = str(UUID(value))
+            except ValueError:
+                pass
+    # Diagnostic fields are generated here, never accepted from arbitrary extras.
+    result.update(_exception_details(event.get("exc_info")))
+    result["message"] = " ".join(
+        f"{key}={result[key]}"
+        for key in (
+            "event",
+            "request_id",
+            "trace_id",
+            "span_id",
+            *_CORRELATION_FIELDS,
+            "method",
+            "route",
+            "status_code",
+            "duration_ms",
+            "operation",
+            "action",
+            "op",
+            "integration",
+            "error_type",
+            "reason",
+        )
+        if key in result
+    )
+    return result
+
+
+def configure_logging(level: str = "INFO") -> None:
+    shared = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
+    ]
+    structlog.configure(
+        processors=[
+            structlog.stdlib.filter_by_level,
+            *shared,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=False,
+    )
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(
+        structlog.stdlib.ProcessorFormatter(
+            processors=[_sanitize, structlog.processors.JSONRenderer(sort_keys=True)],
+            foreign_pre_chain=[*shared, structlog.stdlib.ExtraAdder(allow=_FIELDS)],
+        )
+    )
+    root = logging.getLogger()
+    root.handlers[:] = [handler]
+    root.setLevel(level)
+    # Uvicorn installs non-propagating handlers of its own. Route runtime errors
+    # through the same formatter so exception diagnostics use the safe frame schema.
+    for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
+        runtime_logger = logging.getLogger(name)
+        runtime_logger.handlers.clear()
+        runtime_logger.propagate = True
+    for name in (
+        "httpx",
+        "httpcore",
+        "httpx2",
+        "httpcore2",
+        "neo4j",
+        "sqlalchemy.engine",
+        "uvicorn.access",
+    ):
+        logging.getLogger(name).setLevel(logging.WARNING)
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    """Use static event names and structured fields; never interpolate user data."""
+    return cast(structlog.stdlib.BoundLogger, structlog.get_logger(name))
+
+
+def current_request_id() -> str | None:
+    value = structlog.contextvars.get_contextvars().get("request_id")
+    return value if isinstance(value, str) else None

observability_021/runtime.py

@@ -0,0 +1,360 @@
+"""Providers, instruments and bounded lifecycle for one application."""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import re
+import resource
+import sys
+import threading
+import time
+from collections.abc import Callable, Coroutine, Iterator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import httpx
+
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.metrics import Observation
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+
+from .config import Settings, identifier
+from .logging import get_logger
+
+# The package version, kept in step by release-please.
+INSTRUMENTATION_VERSION = "0.3.0"  # x-release-please-version
+
+# Seconds, with finite boundaries for both ordinary API traffic and long ingestion.
+DURATION_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300)
+BackgroundTask = Callable[[Coroutine[Any, Any, None]], Any]
+logger = get_logger(__name__)
+current_observability: ContextVar[Observability | None] = ContextVar(
+    "observability_021", default=None
+)
+
+
+def error_kind(error: BaseException) -> str:
+    if isinstance(error, asyncio.CancelledError):
+        return "cancelled"
+    if isinstance(error, TimeoutError):
+        return "timeout"
+    if isinstance(error, OSError):
+        return "connection_error"
+    return "internal_error"
+
+
+class Observability:
+    def __init__(
+        self,
+        settings: Settings,
+        *,
+        background_task: BackgroundTask | None = None,
+        prometheus: bool = False,
+    ):
+        self.settings = settings
+        self.background_task = background_task
+        self._closed = False
+        self._instrument_lock = threading.Lock()
+        self._instruments: dict[tuple[str, str], Any] = {}
+        self._cleanup: list[Callable[[], None]] = []
+        self._pools: list[tuple[str, Any]] = []
+        self._started = time.monotonic()
+        self.resource = Resource(
+            {
+                "service.name": settings.service_name,
+                "service.version": settings.service_version,
+                "service.instance.id": settings.instance_id,
+                "deployment.environment.name": settings.environment,
+            }
+        )
+        self.tracer_provider = TracerProvider(
+            resource=self.resource,
+            sampler=ParentBased(TraceIdRatioBased(settings.trace_sample_ratio)),
+            shutdown_on_exit=False,
+        )
+        # A nonempty explicit mapping prevents SDK fallback to ambient OTLP credentials.
+        # Content-Type is already required by OTLP/HTTP, so no extra header is introduced.
+        if settings.traces_endpoint:
+            self.tracer_provider.add_span_processor(
+                BatchSpanProcessor(
+                    OTLPSpanExporter(
+                        endpoint=settings.traces_endpoint,
+                        headers={
+                            "Content-Type": "application/x-protobuf",
+                            **settings.traces_headers,
+                        },
+                        timeout=settings.export_timeout_seconds,
+                    )
+                )
+            )
+        readers = []
+        self.prometheus_registry = None
+        if prometheus:
+            from opentelemetry.exporter.prometheus import PrometheusMetricReader
+            from prometheus_client import CollectorRegistry
+
+            self.prometheus_registry = CollectorRegistry()
+            readers.append(PrometheusMetricReader(registry=self.prometheus_registry))
+        if settings.metrics_endpoint:
+            readers.append(
+                PeriodicExportingMetricReader(
+                    OTLPMetricExporter(
+                        endpoint=settings.metrics_endpoint,
+                        headers={
+                            "Content-Type": "application/x-protobuf",
+                            **settings.metrics_headers,
+                        },
+                        timeout=settings.export_timeout_seconds,
+                    ),
+                    export_interval_millis=settings.metric_interval_ms,
+                    export_timeout_millis=settings.export_timeout_seconds * 1000,
+                )
+            )
+        self.meter_provider = MeterProvider(
+            resource=self.resource,
+            metric_readers=readers,
+            shutdown_on_exit=False,
+            views=[
+                View(
+                    instrument_name="*.duration",
+                    aggregation=ExplicitBucketHistogramAggregation(DURATION_BUCKETS),
+                )
+            ],
+        )
+        self.meter = self.meter_provider.get_meter("observability_021", INSTRUMENTATION_VERSION)
+        self.tracer = self.tracer_provider.get_tracer("observability_021", INSTRUMENTATION_VERSION)
+        self.requests = self.meter.create_counter("http.server.request.count", unit="{request}")
+        self.duration = self.meter.create_histogram("http.server.request.duration", unit="s")
+        self.active = self.meter.create_up_down_counter(
+            "http.server.active_requests", unit="{request}"
+        )
+        self.client_requests = self.meter.create_counter(
+            "http.client.request.count", unit="{request}"
+        )
+        self.client_duration = self.meter.create_histogram("http.client.request.duration", unit="s")
+        self.db_requests = self.meter.create_counter(
+            "db.client.operation.count", unit="{operation}"
+        )
+        self.db_duration = self.meter.create_histogram("db.client.operation.duration", unit="s")
+        self.operation_count = self.meter.create_counter("app.operation.count", unit="{operation}")
+        self.operation_duration = self.meter.create_histogram("app.operation.duration", unit="s")
+        self.meter.create_observable_counter(
+            "process.cpu.time", [lambda _: [Observation(time.process_time())]], unit="s"
+        )
+        self.meter.create_observable_gauge(
+            "process.uptime", [lambda _: [Observation(time.monotonic() - self._started)]], unit="s"
+        )
+        self.meter.create_observable_gauge(
+            "process.memory.peak_usage",
+            [
+                lambda _: [
+                    Observation(
+                        resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+                        * (1 if sys.platform == "darwin" else 1024)
+                    )
+                ]
+            ],
+            unit="By",
+        )
+        self.meter.create_observable_gauge(
+            "process.thread.count",
+            [lambda _: [Observation(threading.active_count())]],
+            unit="{thread}",
+        )
+        # Keep providers local to this application. OTel's process-global setter is
+        # write-once and would silently bind subsequent create_app() calls elsewhere.
+
+    def increment(
+        self,
+        name: str,
+        value: int = 1,
+        *,
+        operation: str,
+        status: str,
+        attributes: Mapping[str, str] | None = None,
+    ) -> None:
+        if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+            raise ValueError("A counter increment cannot be negative")
+        self._instrument(name, "counter", "1").add(
+            value, self._attributes(operation, status, attributes)
+        )
+
+    def observe(
+        self,
+        name: str,
+        seconds: float,
+        *,
+        operation: str,
+        status: str,
+        attributes: Mapping[str, str] | None = None,
+        unit: str = "s",
+    ) -> None:
+        if not math.isfinite(seconds) or seconds < 0:
+            raise ValueError("Duration must be finite and nonnegative")
+        self._instrument(name, "histogram", unit).record(
+            seconds, self._attributes(operation, status, attributes)
+        )
+
+    def _instrument(self, name: str, kind: str, unit: str):
+        identifier(name)
+        if not name.startswith("app."):
+            raise ValueError("Domain metric names must start with app.")
+        if unit not in {"s", "1", "USD"}:
+            raise ValueError("Unsupported domain metric unit")
+        if kind == "histogram" and unit == "s" and not name.endswith(".duration"):
+            raise ValueError("Duration metric names must end with .duration")
+        with self._instrument_lock:
+            if (name, kind) not in self._instruments:
+                create = (
+                    self.meter.create_counter if kind == "counter" else self.meter.create_histogram
+                )
+                self._instruments[name, kind] = create(name, unit=unit)
+            elif self._instruments[name, kind].unit != unit:
+                raise ValueError("A metric's unit cannot change")
+            return self._instruments[name, kind]
+
+    @staticmethod
+    def _attributes(operation: str, status: str, attributes=None) -> dict[str, str]:
+        identifier(operation)
+        identifier(status)
+        # Static domain dimensions used by the APIs. Never accept identifiers or payloads.
+        allowed = {
+            "event",
+            "code",
+            "scope",
+            "permission",
+            "action",
+            "reason",
+            "mode",
+            "channel",
+            "provider",
+            "dry_run",
+            "error_type",
+        }
+        safe = {
+            k: v if re.fullmatch(r"[A-Za-z0-9_.:/-]{1,96}", v) else "_OTHER"
+            for k, v in (attributes or {}).items()
+            if k in allowed
+        }
+        return {**safe, "operation": operation, "status": status}
+
+    def prometheus_latest(self) -> tuple[bytes, str]:
+        from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
+
+        if self.prometheus_registry is None:
+            raise RuntimeError("Enable the prometheus extra and reader at setup")
+        return generate_latest(self.prometheus_registry), CONTENT_TYPE_LATEST
+
+    def http_request(self, *, method: str, dependency: str, headers):
+        from .dependencies import observe_http_request
+
+        return observe_http_request(self, method=method, dependency=dependency, headers=headers)
+
+    @contextmanager
+    def operation(self, name: str) -> Iterator[None]:
+        """Works around sync work or around `await` inside an async function."""
+        identifier(name)
+        started, status = time.perf_counter(), "ok"
+        try:
+            yield
+        except BaseException as exc:
+            status = "cancelled" if isinstance(exc, asyncio.CancelledError) else "error"
+            raise
+        finally:
+            attributes = self._attributes(name, status)
+            self.operation_count.add(1, attributes)
+            self.operation_duration.record(time.perf_counter() - started, attributes)
+
+    def schedule_flush(self) -> None:
+        if self.background_task is None or self._closed:
+            return
+        coroutine = self.flush_async()
+        try:
+            self.background_task(coroutine)
+        except Exception:
+            coroutine.close()
+            logger.warning("telemetry.flush_failed", reason="schedule_failed")
+
+    async def flush_async(self) -> None:
+        # Export off the application event loop. Both signals share one wall-clock
+        # wait budget. Transport timeouts also bound the SDK's work in its threads.
+        timeout = self.settings.export_timeout_seconds
+        try:
+            await asyncio.wait_for(
+                asyncio.gather(
+                    asyncio.to_thread(self._flush_signal, "traces", self.tracer_provider),
+                    asyncio.to_thread(self._flush_signal, "metrics", self.meter_provider),
+                ),
+                timeout=timeout,
+            )
+        except TimeoutError:
+            logger.warning("telemetry.flush_failed", reason="timeout")
+
+    def _flush_signal(self, signal: str, provider) -> None:
+        try:
+            if not provider.force_flush(
+                timeout_millis=int(self.settings.export_timeout_seconds * 1000)
+            ):
+                logger.warning("telemetry.flush_failed", signal=signal, reason="timeout")
+        except Exception:
+            logger.warning("telemetry.flush_failed", signal=signal, reason="export_failed")
+
+    async def shutdown(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        await self.flush_async()
+        for cleanup in self._cleanup:
+            cleanup()
+        # The SDK has synchronous shutdown methods; never block the ASGI event loop.
+        await asyncio.to_thread(
+            self.meter_provider.shutdown,
+            timeout_millis=int(self.settings.export_timeout_seconds * 1000),
+        )
+        await asyncio.to_thread(self.tracer_provider.shutdown)
+
+    def instrument_sqlalchemy(self, engine, *, dependency: str = "database") -> None:
+        from .dependencies import instrument_sqlalchemy
+
+        instrument_sqlalchemy(self, engine, dependency=dependency)
+
+    def httpx_transport(
+        self, *, dependency: str, transport: httpx.BaseTransport | None = None
+    ) -> httpx.BaseTransport:
+        from .dependencies import HTTPTransport
+
+        return HTTPTransport(self, dependency=dependency, transport=transport)
+
+    def async_httpx_transport(
+        self, *, dependency: str, transport: httpx.AsyncBaseTransport | None = None
+    ) -> httpx.AsyncBaseTransport:
+        from .dependencies import AsyncHTTPTransport
+
+        return AsyncHTTPTransport(self, dependency=dependency, transport=transport)
+
+    def add_health_routes(self, app, *, readiness=None) -> None:
+        from .asgi import add_health_routes
+
+        add_health_routes(app, self, readiness=readiness or {})
+
+
+def vercel_background_task(settings: Settings) -> BackgroundTask | None:
+    if not settings.serverless or not (settings.traces_endpoint or settings.metrics_endpoint):
+        return None
+    try:
+        from vercel.functions import wait_until
+    except ImportError:
+        raise ValueError(
+            "Install 021-python-observability[vercel] for the Vercel runtime"
+        ) from None
+    return wait_until