021-python-observability 0.3.0__py3-none-any.whl

observability_021/asgi.py

@@ -0,0 +1,266 @@
+"""Outer ASGI observer: sees the response produced by FastAPI's error handlers."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Awaitable, Callable, Mapping
+from contextlib import asynccontextmanager
+from typing import Literal
+from uuid import UUID, uuid4
+
+import structlog
+from fastapi import FastAPI
+from opentelemetry.propagators.textmap import DefaultGetter
+from opentelemetry.trace import SpanKind, StatusCode
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from pydantic import BaseModel
+from starlette.responses import JSONResponse
+
+from .config import Settings, identifier
+from .logging import configure_logging, get_logger
+from .runtime import (
+    BackgroundTask,
+    Observability,
+    current_observability,
+    error_kind,
+    vercel_background_task,
+)
+
+_METHODS = frozenset(
+    {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT", "TRACE"}
+)
+_PROPAGATOR = TraceContextTextMapPropagator()
+logger = get_logger(__name__)
+
+
+def request_method(method: str) -> str:
+    return method if method in _METHODS else "_OTHER"
+
+
+class ObservabilityMiddleware:
+    def __init__(self, app, observability: Observability):
+        self.app, self.obs = app, observability
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+        started = time.perf_counter()
+        method = request_method(scope.get("method", ""))
+        headers = {k.decode("latin1").lower(): v.decode("latin1") for k, v in scope["headers"]}
+        try:
+            request_id = str(UUID(headers.get("x-request-id", "")))
+        except (ValueError, AttributeError):
+            request_id = str(uuid4())
+        context = _PROPAGATOR.extract(headers, getter=DefaultGetter())
+        settings = self.obs.settings
+        tokens = structlog.contextvars.bind_contextvars(
+            request_id=request_id,
+            service_name=settings.service_name,
+            environment=settings.environment,
+            service_version=settings.service_version,
+        )
+        obs_token = current_observability.set(self.obs)
+        scope.setdefault("state", {})["request_id"] = request_id
+        status = None
+        completed, disconnected = False, False
+        failure = None
+        failure_class = None
+        self.obs.active.add(1, {"http.request.method": method})
+
+        async def observed_receive():
+            nonlocal disconnected
+            message = await receive()
+            if message["type"] == "http.disconnect":
+                disconnected = True
+            return message
+
+        async def observed_send(message):
+            nonlocal status, completed
+            if message["type"] == "http.response.start":
+                message = dict(message)
+                message["headers"] = [
+                    (k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"
+                ] + [(b"x-request-id", request_id.encode("ascii"))]
+            await send(message)
+            if message["type"] == "http.response.start" and message["status"] >= 200:
+                status = message["status"]
+            if message["type"] == "http.response.body" and not message.get("more_body", False):
+                completed = status is not None
+
+        try:
+            with self.obs.tracer.start_as_current_span(
+                method,
+                context=context,
+                kind=SpanKind.SERVER,
+                record_exception=False,
+                set_status_on_exception=False,
+            ) as span:
+                try:
+                    await self.app(scope, observed_receive, observed_send)
+                except BaseException as exc:
+                    failure = error_kind(exc)
+                    failure_class = type(exc).__name__
+                    raise
+                finally:
+                    duration = time.perf_counter() - started
+                    # FastAPI 0.141 keeps the original route in scope["route"].
+                    # Its effective route includes APIRouter prefixes (never raw paths).
+                    effective = scope.get("fastapi", {}).get("effective_route_context")
+                    route = (
+                        getattr(effective, "path", None)
+                        or getattr(scope.get("route"), "path", None)
+                        or "__unresolved__"
+                    )
+                    outcome = (
+                        "cancelled"
+                        if failure == "cancelled"
+                        else "completed"
+                        if completed
+                        else "disconnected"
+                        if disconnected
+                        else "error"
+                    )
+                    attrs = {
+                        "http.request.method": method,
+                        "http.route": route,
+                        "response.outcome": outcome,
+                    }
+                    if status is not None:
+                        attrs["http.response.status_code"] = status
+                    if failure:
+                        attrs["error.type"] = failure
+                    span.update_name(f"{method} {route}")
+                    span.set_attributes(attrs)
+                    if failure or outcome != "completed" or (status is not None and status >= 500):
+                        span.set_status(StatusCode.ERROR)
+                    self.obs.requests.add(1, attrs)
+                    self.obs.duration.record(duration, attrs)
+                    self.obs.active.add(-1, {"http.request.method": method})
+                    log = (
+                        logger.warning
+                        if failure or status is not None and status >= 500
+                        else logger.info
+                    )
+                    log(
+                        "request.completed",
+                        method=method,
+                        route=route,
+                        status_code=status,
+                        outcome=outcome,
+                        duration_ms=round(duration * 1000, 3),
+                        error_type=failure_class,
+                        reason=failure,
+                        exc_info=bool(failure),
+                    )
+        finally:
+            # The span is closed BEFORE scheduling the Vercel continuation.
+            self.obs.schedule_flush()
+            structlog.contextvars.reset_contextvars(**tokens)
+            current_observability.reset(obs_token)
+
+
+def setup_observability(
+    app: FastAPI,
+    *,
+    service_name: str | None = None,
+    settings: Settings | None = None,
+    background_task: BackgroundTask | None = None,
+    prometheus: bool = False,
+) -> Observability:
+    """Call once during application construction, before serving the first request."""
+    existing = getattr(app.state, "observability", None)
+    if existing is not None:
+        return existing
+    if app.middleware_stack is not None:
+        raise RuntimeError("Configure observability before the application starts")
+    settings = settings or Settings.from_env(service_name=service_name)
+    configure_logging(settings.log_level)
+    obs = Observability(
+        settings,
+        prometheus=prometheus,
+        background_task=(
+            background_task if background_task is not None else vercel_background_task(settings)
+        ),
+    )
+    original_build = app.build_middleware_stack
+    # FastAPI wraps user middleware inside ServerErrorMiddleware. Wrapping the built
+    # stack preserves the FastAPI object used by Vercel and observes real error responses.
+    app.build_middleware_stack = lambda: ObservabilityMiddleware(original_build(), obs)
+    original_lifespan = app.router.lifespan_context
+
+    @asynccontextmanager
+    async def lifespan(application):
+        # Servers accepting an already-built app may configure logging after setup.
+        configure_logging(settings.log_level)
+        token = current_observability.set(obs)
+        try:
+            async with original_lifespan(application) as state:
+                yield state
+        finally:
+            try:
+                await obs.shutdown()
+            finally:
+                current_observability.reset(token)
+
+    app.router.lifespan_context = lifespan
+    app.state.observability = obs
+    return obs
+
+
+class HealthResponse(BaseModel):
+    status: Literal["ok"] = "ok"
+
+
+class ReadyResponse(BaseModel):
+    status: Literal["ok", "unavailable"]
+    checks: dict[str, Literal["ok", "unavailable", "timeout"]]
+
+
+def add_health_routes(
+    app: FastAPI,
+    obs: Observability,
+    *,
+    readiness: Mapping[str, Callable[[], Awaitable[object]]],
+) -> None:
+    if any(getattr(route, "path", None) in {"/health", "/ready"} for route in app.routes):
+        raise ValueError("Health routes already exist; migrate their semantics explicitly")
+    checks = {identifier(name): callback for name, callback in readiness.items()}
+
+    async def liveness():
+        return {"status": "ok"}
+
+    async def ready():
+        async def check(name, callback):
+            try:
+                result = await asyncio.wait_for(callback(), obs.settings.readiness_timeout_seconds)
+                return name, "unavailable" if result is False else "ok"
+            except TimeoutError:
+                return name, "timeout"
+            except Exception:
+                return name, "unavailable"
+
+        result = dict(await asyncio.gather(*(check(n, f) for n, f in checks.items())))
+        healthy = all(value == "ok" for value in result.values())
+        return JSONResponse(
+            {"status": "ok" if healthy else "unavailable", "checks": result},
+            status_code=200 if healthy else 503,
+        )
+
+    app.add_api_route(
+        "/health",
+        liveness,
+        methods=["GET"],
+        tags=["health"],
+        response_model=HealthResponse,
+        summary="Process liveness",
+    )
+    app.add_api_route(
+        "/ready",
+        ready,
+        methods=["GET"],
+        tags=["health"],
+        response_model=ReadyResponse,
+        responses={503: {"model": ReadyResponse}},
+        summary="Dependency readiness",
+    )

observability_021/config.py

@@ -0,0 +1,132 @@
+"""OTLP/HTTP configuration. No remote export without an explicit endpoint."""
+
+from __future__ import annotations
+
+import math
+import os
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from urllib.parse import unquote, urlsplit
+from uuid import uuid4
+
+
+def identifier(value: str) -> str:
+    if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,95}", value):
+        raise ValueError("Expected a static identifier of 1–96 characters")
+    return value
+
+
+def _endpoint(value: str | None) -> str | None:
+    if not value:
+        return None
+    parsed = urlsplit(value)
+    if (
+        parsed.scheme not in {"http", "https"}
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.query
+        or parsed.fragment
+    ):
+        raise ValueError("OTLP endpoint must be an HTTP(S) URL without credentials or query")
+    return value
+
+
+def _headers(value: str) -> dict[str, str]:
+    headers: dict[str, str] = {}
+    for pair in value.split(","):
+        if not pair.strip():
+            continue
+        key, sep, val = pair.strip().partition("=")
+        if not sep or not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", key):
+            raise ValueError("Invalid OTLP header configuration")
+        val = unquote(val.strip())
+        if "\r" in val or "\n" in val:
+            raise ValueError("Invalid OTLP header configuration")
+        headers[key] = val
+    return headers
+
+
+@dataclass(frozen=True)
+class Settings:
+    service_name: str
+    environment: str = "local"
+    service_version: str = "unknown"
+    instance_id: str = field(default_factory=lambda: str(uuid4()))
+    traces_endpoint: str | None = field(default=None, repr=False)
+    metrics_endpoint: str | None = field(default=None, repr=False)
+    traces_headers: Mapping[str, str] = field(default_factory=dict, repr=False)
+    metrics_headers: Mapping[str, str] = field(default_factory=dict, repr=False)
+    export_timeout_seconds: float = 2.0
+    metric_interval_ms: float = 60000
+    trace_sample_ratio: float = 1.0
+    log_level: str = "INFO"
+    serverless: bool = False
+    readiness_timeout_seconds: float = 2.0
+
+    def __post_init__(self) -> None:
+        identifier(self.service_name)
+        identifier(self.environment)
+        if not self.service_version or len(self.service_version) > 128:
+            raise ValueError("service_version must contain 1–128 characters")
+        _endpoint(self.traces_endpoint)
+        _endpoint(self.metrics_endpoint)
+        for value in (
+            self.export_timeout_seconds,
+            self.metric_interval_ms,
+            self.readiness_timeout_seconds,
+        ):
+            if not math.isfinite(value) or value <= 0:
+                raise ValueError("Timeouts and intervals must be finite and positive")
+        if not 0 <= self.trace_sample_ratio <= 1:
+            raise ValueError("trace_sample_ratio must be between zero and one")
+        if self.log_level not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}:
+            raise ValueError("Invalid log level")
+
+    @classmethod
+    def from_env(
+        cls, *, service_name: str | None = None, env: Mapping[str, str] | None = None
+    ) -> Settings:
+        env = os.environ if env is None else env
+        name = service_name or env.get("OTEL_SERVICE_NAME")
+        if not name:
+            raise ValueError("Set OTEL_SERVICE_NAME or pass service_name")
+        for signal in ("_TRACES", "_METRICS"):
+            protocol = env.get(
+                f"OTEL_EXPORTER_OTLP{signal}_PROTOCOL",
+                env.get("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"),
+            )
+            if protocol != "http/protobuf":
+                raise ValueError("021-python-observability supports OTLP http/protobuf")
+        base = _endpoint(env.get("OTEL_EXPORTER_OTLP_ENDPOINT"))
+
+        def endpoint(signal: str) -> str | None:
+            return _endpoint(env.get(f"OTEL_EXPORTER_OTLP_{signal.upper()}_ENDPOINT")) or (
+                f"{base.rstrip('/')}/v1/{signal}" if base else None
+            )
+
+        return cls(
+            service_name=name,
+            environment=env.get("OTEL_DEPLOYMENT_ENVIRONMENT", env.get("VERCEL_ENV", "local")),
+            service_version=env.get(
+                "OTEL_SERVICE_VERSION", env.get("VERCEL_GIT_COMMIT_SHA", "unknown")
+            ),
+            traces_endpoint=endpoint("traces"),
+            metrics_endpoint=endpoint("metrics"),
+            traces_headers=_headers(
+                env.get(
+                    "OTEL_EXPORTER_OTLP_TRACES_HEADERS", env.get("OTEL_EXPORTER_OTLP_HEADERS", "")
+                )
+            ),
+            metrics_headers=_headers(
+                env.get(
+                    "OTEL_EXPORTER_OTLP_METRICS_HEADERS", env.get("OTEL_EXPORTER_OTLP_HEADERS", "")
+                )
+            ),
+            export_timeout_seconds=float(env.get("OTEL_EXPORTER_OTLP_TIMEOUT", "2")),
+            metric_interval_ms=float(env.get("OTEL_METRIC_EXPORT_INTERVAL", "60000")),
+            trace_sample_ratio=float(env.get("OBS_TRACE_SAMPLE_RATIO", "1")),
+            log_level=env.get("LOG_LEVEL", "INFO").upper(),
+            serverless=env.get("VERCEL") == "1",
+        )

observability_021/dependencies.py

@@ -0,0 +1,165 @@
+"""Explicit dependency instrumentation: stable names, no URLs or SQL text."""
+
+from __future__ import annotations
+
+import time
+from contextlib import contextmanager
+
+import httpx
+import structlog
+from opentelemetry.trace import SpanKind, StatusCode
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from sqlalchemy import event
+from sqlalchemy.pool import QueuePool
+
+from .config import identifier
+from .runtime import error_kind
+
+
+@contextmanager
+def observe_http_request(obs, *, dependency, method, headers):
+    from .asgi import request_method
+
+    attributes = {
+        "dependency": identifier(dependency),
+        "http.request.method": request_method(method),
+    }
+    started = time.perf_counter()
+    with obs.tracer.start_as_current_span(
+        f"HTTP {dependency}",
+        kind=SpanKind.CLIENT,
+        attributes=attributes,
+        record_exception=False,
+        set_status_on_exception=False,
+    ) as span:
+        request_id = structlog.contextvars.get_contextvars().get("request_id")
+        # Plain dicts are case-sensitive; HTTP header names are not. Replace the
+        # caller's context instead of sending conflicting duplicate headers.
+        replaced = {"traceparent", "tracestate"}
+        if request_id:
+            replaced.add("x-request-id")
+        for key in list(headers):
+            if key.lower() in replaced:
+                del headers[key]
+        TraceContextTextMapPropagator().inject(headers)
+        if request_id:
+            headers["x-request-id"] = request_id
+        try:
+            yield attributes
+        except BaseException as exc:
+            attributes["error.type"] = (
+                "timeout" if isinstance(exc, httpx.TimeoutException) else error_kind(exc)
+            )
+            span.set_status(StatusCode.ERROR)
+            raise
+        finally:
+            if attributes.get("http.response.status_code", 0) >= 400:
+                span.set_status(StatusCode.ERROR)
+            span.set_attributes(attributes)
+            obs.client_requests.add(1, attributes)
+            obs.client_duration.record(time.perf_counter() - started, attributes)
+
+
+class HTTPTransport(httpx.BaseTransport):
+    def __init__(self, obs, *, dependency: str, transport=None):
+        self.obs, self.dependency = obs, identifier(dependency)
+        self.transport = transport if transport is not None else httpx.HTTPTransport()
+
+    def handle_request(self, request):
+        with observe_http_request(
+            self.obs, dependency=self.dependency, method=request.method, headers=request.headers
+        ) as attrs:
+            response = self.transport.handle_request(request)
+            attrs["http.response.status_code"] = response.status_code
+            return response
+
+    def close(self):
+        self.transport.close()
+
+
+class AsyncHTTPTransport(httpx.AsyncBaseTransport):
+    def __init__(self, obs, *, dependency: str, transport=None):
+        self.obs, self.dependency = obs, identifier(dependency)
+        self.transport = transport if transport is not None else httpx.AsyncHTTPTransport()
+
+    async def handle_async_request(self, request):
+        with observe_http_request(
+            self.obs, dependency=self.dependency, method=request.method, headers=request.headers
+        ) as attrs:
+            response = await self.transport.handle_async_request(request)
+            attrs["http.response.status_code"] = response.status_code
+            return response
+
+    async def aclose(self):
+        await self.transport.aclose()
+
+
+def instrument_sqlalchemy(obs, engine, *, dependency: str):
+    """Supports sync engines and AsyncEngine via its public sync_engine property."""
+    identifier(dependency)
+    engine = getattr(engine, "sync_engine", engine)
+    existing = getattr(engine, "_observability_021", None)
+    if existing is obs:
+        return
+    if existing is not None:
+        raise ValueError("This engine is already observed by another application")
+    engine._observability_021 = obs
+
+    def before(_connection, _cursor, _statement, _parameters, context, _executemany):
+        context._observability_021 = (
+            time.perf_counter(),
+            obs.tracer.start_span(f"SQL {dependency}", kind=SpanKind.CLIENT),
+        )
+
+    def finish(context, error=None):
+        state = getattr(context, "_observability_021", None)
+        if state is None:
+            return
+        context._observability_021 = None
+        started, span = state
+        attrs = {"dependency": dependency, "status": "error" if error else "ok"}
+        if error:
+            attrs["error.type"] = error_kind(error)
+            span.set_status(StatusCode.ERROR)
+        span.set_attributes(attrs)
+        span.end()
+        obs.db_requests.add(1, attrs)
+        obs.db_duration.record(time.perf_counter() - started, attrs)
+
+    def after(_connection, _cursor, _statement, _parameters, context, _executemany):
+        finish(context)
+
+    def failed(context):
+        finish(context.execution_context, context.original_exception)
+
+    listeners = [
+        ("before_cursor_execute", before),
+        ("after_cursor_execute", after),
+        ("handle_error", failed),
+    ]
+    for name, fn in listeners:
+        event.listen(engine, name, fn)
+
+    def cleanup():
+        for name, fn in listeners:
+            event.remove(engine, name, fn)
+        del engine._observability_021
+
+    obs._cleanup.append(cleanup)
+
+    if isinstance(engine.pool, QueuePool):
+        from opentelemetry.metrics import Observation
+
+        obs._pools.append((dependency, engine.pool))
+        if len(obs._pools) == 1:
+
+            def pool_observations(_options):
+                return [
+                    Observation(value, {"dependency": name, "state": state})
+                    for name, pool in obs._pools
+                    for state, value in (("used", pool.checkedout()), ("idle", pool.checkedin()))
+                ]
+
+            obs.meter.create_observable_gauge(
+                "db.client.connection.count", [pool_observations], unit="{connection}"
+            )