snake-orm 0.1.0b1__py3-none-any.whl

snakeorm/debug/otel/__init__.py

@@ -0,0 +1,68 @@
+"""The `otel` channel: the debug report as OpenTelemetry spans, over OTLP/HTTP.
+
+It is the one delivery meant for PRODUCTION, and the only one whose consumer is a tool rather than a
+person. Its audience is `OPERATOR`: it goes out sideways to infrastructure the operator already
+runs, instead of riding on the response. That is why it is NOT in `RISKY_CHANNELS`: putting it there
+would drop it exactly where it justifies existing.
+
+Riding on the response is NOT what makes a channel risky — `timing` rides it and is harmless. What
+decides it is whether the requester ends up holding the SQL, which is what `ChannelAudience` asks.
+
+What travels is the parametrised SQL and never the values. The convention collects `db.query.text`
+by default "because parametrising is a strong signal from the user that anything sensitive is in
+the values", and SnakeORM never interpolates. The values are opt-in, key by key, from code.
+
+Four pieces:
+
+- `context` — the only thing `opentelemetry-api` is for: is a span already open, and which.
+- `summary` — `db.query.summary`, which NAMES the span (`SELECT orders`, not the whole SQL).
+- `spans`   — the pure mapping from a `DebugReport` to the hybrid shape.
+- `payload` / `exporter` — OTLP's JSON and the background queue that posts it.
+"""
+
+from snakeorm.debug.otel.context import TraceContext as TraceContext
+from snakeorm.debug.otel.context import active_context as active_context
+from snakeorm.debug.otel.exporter import DEFAULT_ENDPOINT as DEFAULT_ENDPOINT
+from snakeorm.debug.otel.exporter import DEFAULT_SERVICE_NAME as DEFAULT_SERVICE_NAME
+from snakeorm.debug.otel.exporter import ENDPOINT_ENV_KEY as ENDPOINT_ENV_KEY
+from snakeorm.debug.otel.exporter import OtelExporter as OtelExporter
+from snakeorm.debug.otel.exporter import SERVICE_NAME_ENV_KEY as SERVICE_NAME_ENV_KEY
+from snakeorm.debug.otel.exporter import (
+    TRACES_ENDPOINT_ENV_KEY as TRACES_ENDPOINT_ENV_KEY,
+)
+from snakeorm.debug.otel.exporter import Transport as Transport
+from snakeorm.debug.otel.exporter import current_exporter as current_exporter
+from snakeorm.debug.otel.exporter import endpoint_from_env as endpoint_from_env
+from snakeorm.debug.otel.exporter import post_json as post_json
+from snakeorm.debug.otel.exporter import service_name_from_env as service_name_from_env
+from snakeorm.debug.otel.exporter import set_exporter as set_exporter
+from snakeorm.debug.otel.payload import SCOPE_NAME as SCOPE_NAME
+from snakeorm.debug.otel.payload import encode_payload as encode_payload
+from snakeorm.debug.otel.payload import otlp_payload as otlp_payload
+from snakeorm.debug.otel.report import export_report as export_report
+from snakeorm.debug.otel.spans import CODE_FILE_PATH as CODE_FILE_PATH
+from snakeorm.debug.otel.spans import CODE_FUNCTION_NAME as CODE_FUNCTION_NAME
+from snakeorm.debug.otel.spans import CODE_LINE_NUMBER as CODE_LINE_NUMBER
+from snakeorm.debug.otel.spans import DB_COLLECTION_NAME as DB_COLLECTION_NAME
+from snakeorm.debug.otel.spans import DB_NAMESPACE as DB_NAMESPACE
+from snakeorm.debug.otel.spans import DB_OPERATION_NAME as DB_OPERATION_NAME
+from snakeorm.debug.otel.spans import (
+    DB_QUERY_PARAMETER_PREFIX as DB_QUERY_PARAMETER_PREFIX,
+)
+from snakeorm.debug.otel.spans import DB_QUERY_SUMMARY as DB_QUERY_SUMMARY
+from snakeorm.debug.otel.spans import DB_QUERY_TEXT as DB_QUERY_TEXT
+from snakeorm.debug.otel.spans import (
+    DB_RESPONSE_RETURNED_ROWS as DB_RESPONSE_RETURNED_ROWS,
+)
+from snakeorm.debug.otel.spans import DB_SYSTEM_NAME as DB_SYSTEM_NAME
+from snakeorm.debug.otel.spans import SNAKEORM_HAS_N_PLUS_ONE as SNAKEORM_HAS_N_PLUS_ONE
+from snakeorm.debug.otel.spans import AttributeValue as AttributeValue
+from snakeorm.debug.otel.spans import IdSource as IdSource
+from snakeorm.debug.otel.spans import RandomIds as RandomIds
+from snakeorm.debug.otel.spans import SnakeSpan as SnakeSpan
+from snakeorm.debug.otel.spans import SpanEvent as SpanEvent
+from snakeorm.debug.otel.spans import SpanKind as SpanKind
+from snakeorm.debug.otel.spans import monotonic_epoch_ns as monotonic_epoch_ns
+from snakeorm.debug.otel.spans import spans_from_report as spans_from_report
+from snakeorm.debug.otel.summary import QuerySummary as QuerySummary
+from snakeorm.debug.otel.summary import summarise as summarise

snakeorm/debug/otel/context.py

@@ -0,0 +1,43 @@
+"""Reading the trace context the APPLICATION already has open, if it has one.
+
+This is the ONLY thing `opentelemetry-api` is needed for, and it is the only thing that cannot be
+reimplemented: a process-wide context variable owned by a library we do not control. Everything else
+in this channel —the payload, the HTTP POST, the queue— is stdlib.
+
+It degrades three ways and none of them raises: no library installed, no provider active, or a
+provider that misbehaves. All three give `None`, which turns our span into a loose root instead of a
+child.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class TraceContext:
+    """The active span, in the hex form OTLP/JSON wants: 32 characters of trace, 16 of span."""
+
+    trace_id: str
+    span_id: str
+
+
+def active_context() -> TraceContext | None:
+    """The application's active span, or `None` if there is none to hang off.
+
+    `ImportError` is the expected path for an application that never installed the extra, so it is
+    caught rather than guarded with a flag. The broad catch underneath is deliberate too: a
+    telemetry read has no business turning into a 500.
+    """
+    try:
+        from opentelemetry import trace
+
+        span_context = trace.get_current_span().get_span_context()
+        if not span_context.is_valid:
+            return None
+        return TraceContext(
+            trace_id=format(span_context.trace_id, "032x"),
+            span_id=format(span_context.span_id, "016x"),
+        )
+    except Exception:
+        return None

snakeorm/debug/otel/exporter.py

@@ -0,0 +1,277 @@
+"""Sending the spans: OTLP over HTTP, from a BACKGROUND QUEUE, with `urllib` and nothing else.
+
+The queue is not an optimisation and it is not deferred to a later version. Measured against
+localhost, exporting in line adds ~210 ms to a request of 503 queries; on the asynchronous path that
+does not slow one request down, it blocks the WHOLE event loop. So `submit` reads the active trace
+context —which has to happen on the calling thread, because that is where the context lives— hands
+the report to a worker and returns.
+
+Two failure modes, and both are declared rather than swallowed. A full queue DROPS (telemetry must
+never make an application wait) and counts the drop. A transport that fails counts the failure and
+warns ONCE, naming the endpoint: a silence would be the exact fault this ORM exists not to commit,
+and a warning per request would be a log nobody reads.
+"""
+
+from __future__ import annotations
+
+import atexit
+import os
+import queue
+import threading
+import warnings
+from collections.abc import Callable, Mapping
+from typing import TYPE_CHECKING
+
+from snakeorm.core.exceptions import SnakeWarning
+from snakeorm.debug.otel.context import TraceContext, active_context
+from snakeorm.debug.otel.payload import encode_payload, otlp_payload
+from snakeorm.debug.otel.spans import spans_from_report
+
+if TYPE_CHECKING:
+    from snakeorm.debug.report import DebugReport
+
+Transport = Callable[[str, bytes], None]
+"""How the body actually leaves: `(endpoint, body) -> None`. Injected, so it is testable offline."""
+
+DEFAULT_ENDPOINT = "http://localhost:4318/v1/traces"
+"""The local OTLP/HTTP port, which is the one `docker compose --profile tracing` publishes."""
+
+DEFAULT_SERVICE_NAME = "snakeorm"
+"""What Jaeger's Service dropdown says when the application did not name itself."""
+
+TRACES_ENDPOINT_ENV_KEY = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
+ENDPOINT_ENV_KEY = "OTEL_EXPORTER_OTLP_ENDPOINT"
+SERVICE_NAME_ENV_KEY = "OTEL_SERVICE_NAME"
+# The STANDARD variable names, not ours. Anyone who has configured any other exporter has already
+# set them, and a private spelling would be one more thing to discover.
+
+_TRACES_PATH = "/v1/traces"
+_HTTP_TIMEOUT_SECONDS = 5.0
+_DEFAULT_QUEUE_SIZE = 256
+
+
+def post_json(endpoint: str, body: bytes) -> None:
+    """POST the payload as `application/json`. The response body is read and dropped: only the send matters.
+
+    `urllib.request` is imported HERE and not at the top, which is the one lazy import in this
+    package and the one that earns it. `snakeorm/__init__` re-exports `snakeorm.debug`, which
+    re-exports `export_report`, which lives in this module — so a module-level import would put an
+    HTTP client (plus `http.client`, `email.*` and `ssl`, ~17 ms measured) into the startup of every
+    application that uses the ORM, including the ones that never turn this channel on. The channel
+    promises that switching it off costs nothing; an import is part of that promise.
+    """
+    import urllib.request
+
+    request = urllib.request.Request(
+        endpoint,
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=_HTTP_TIMEOUT_SECONDS) as response:
+        response.read()
+
+
+def endpoint_from_env(environ: Mapping[str, str] | None = None) -> str:
+    """The traces endpoint: the specific variable, then the generic one plus `/v1/traces`, then local."""
+    env = _environ(environ)
+    specific = env.get(TRACES_ENDPOINT_ENV_KEY, "").strip()
+    if specific:
+        return specific
+    generic = env.get(ENDPOINT_ENV_KEY, "").strip()
+    if generic:
+        return f"{generic.rstrip('/')}{_TRACES_PATH}"
+    return DEFAULT_ENDPOINT
+
+
+def service_name_from_env(environ: Mapping[str, str] | None = None) -> str:
+    """The service name (`OTEL_SERVICE_NAME`), or ours when the application did not declare one."""
+    return (
+        _environ(environ).get(SERVICE_NAME_ENV_KEY, "").strip() or DEFAULT_SERVICE_NAME
+    )
+
+
+def _environ(environ: Mapping[str, str] | None) -> Mapping[str, str]:
+    """The given mapping, or the real environment. Injected in tests, read for real everywhere else."""
+    return os.environ if environ is None else environ
+
+
+class OtelExporter:
+    """Turns reports into OTLP spans and posts them, from a worker thread of its own.
+
+    One per process (see `current_exporter`): one thread and one queue, not one per request.
+    """
+
+    __slots__ = (
+        "_dropped",
+        "_endpoint",
+        "_failures",
+        "_lock",
+        "_parameter_keys",
+        "_pending",
+        "_queue",
+        "_service_name",
+        "_transport",
+        "_warned",
+        "_worker",
+    )
+
+    def __init__(
+        self,
+        *,
+        endpoint: str,
+        service_name: str,
+        transport: Transport = post_json,
+        queue_size: int = _DEFAULT_QUEUE_SIZE,
+        parameter_keys: frozenset[str] = frozenset(),
+    ) -> None:
+        """`parameter_keys` names the query parameters to send, and there is NO environment variable for it.
+
+        That omission is the decision: an environment variable is precisely the switch somebody
+        flips in production by accident, and this one would send user data to a tracing backend. It
+        takes an explicit line of code, key by key.
+        """
+        self._endpoint = endpoint
+        self._service_name = service_name
+        self._transport = transport
+        self._parameter_keys = parameter_keys
+        self._queue: queue.Queue[tuple[DebugReport, TraceContext | None] | None] = (
+            queue.Queue(maxsize=queue_size)
+        )
+        self._lock = threading.Condition()
+        self._pending = 0
+        self._dropped = 0
+        self._failures = 0
+        self._warned = False
+        self._worker: threading.Thread | None = None
+
+    @property
+    def endpoint(self) -> str:
+        """Where the spans are posted."""
+        return self._endpoint
+
+    @property
+    def service_name(self) -> str:
+        """What the resource calls this service."""
+        return self._service_name
+
+    @property
+    def dropped(self) -> int:
+        """How many reports were discarded because the queue was full."""
+        return self._dropped
+
+    @property
+    def failures(self) -> int:
+        """How many sends the transport refused."""
+        return self._failures
+
+    def submit(self, report: DebugReport) -> None:
+        """Queue a report and RETURN. Nothing is serialised or sent on the calling thread.
+
+        The trace context is read HERE and not in the worker: it lives in a `ContextVar` belonging
+        to the request's task, and a worker thread cannot see it.
+        """
+        item = (report, active_context())
+        with self._lock:
+            self._start_worker()
+            try:
+                self._queue.put_nowait(item)
+            except queue.Full:
+                self._dropped += 1
+                return
+            self._pending += 1
+
+    def send_now(self, report: DebugReport, *, parent: TraceContext | None) -> None:
+        """Build the payload and post it, synchronously. The worker's body, and the test's seam."""
+        try:
+            spans = spans_from_report(
+                report, parent=parent, parameter_keys=self._parameter_keys
+            )
+            body = encode_payload(otlp_payload(spans, service_name=self._service_name))
+            self._transport(self._endpoint, body)
+        except (
+            Exception
+        ) as error:  # a collector that is down is not an application error
+            self._record_failure(error)
+
+    def flush(self, timeout: float = 5.0) -> bool:
+        """Wait until the queue has drained. `True` if it did, `False` if the wait ran out."""
+        with self._lock:
+            return self._lock.wait_for(lambda: self._pending == 0, timeout=timeout)
+
+    def shutdown(self, timeout: float = 5.0) -> None:
+        """Drain what is queued and stop the worker. Registered at exit for the process exporter."""
+        self.flush(timeout)
+        with self._lock:
+            worker, self._worker = self._worker, None
+        if worker is None:
+            return
+        self._queue.put(None)
+        worker.join(timeout)
+
+    def _start_worker(self) -> None:
+        """Start the worker on first use. Daemon: a stuck send never keeps the process alive."""
+        if self._worker is not None:
+            return
+        self._worker = threading.Thread(
+            target=self._drain, name="snakeorm-otel", daemon=True
+        )
+        self._worker.start()
+
+    def _drain(self) -> None:
+        """Take reports off the queue and send them, one at a time, until the sentinel arrives."""
+        while True:
+            item = self._queue.get()
+            if item is None:
+                return
+            report, parent = item
+            try:
+                self.send_now(report, parent=parent)
+            finally:
+                with self._lock:
+                    self._pending -= 1
+                    self._lock.notify_all()
+
+    def _record_failure(self, error: Exception) -> None:
+        """Count the failure and warn about the FIRST one, naming the endpoint and the reason."""
+        with self._lock:
+            self._failures += 1
+            already_warned, self._warned = self._warned, True
+        if already_warned:
+            return
+        warnings.warn(
+            f"The debug channel 'otel' could not reach {self._endpoint}: {error}. "
+            f"The spans of this request are lost; the following failures stay quiet.",
+            SnakeWarning,
+            stacklevel=2,
+        )
+
+
+_process_exporter: OtelExporter | None = None
+_process_lock = threading.Lock()
+
+
+def current_exporter() -> OtelExporter:
+    """The process's exporter, built from the standard environment variables on first use."""
+    global _process_exporter
+    with _process_lock:
+        if _process_exporter is None:
+            _process_exporter = OtelExporter(
+                endpoint=endpoint_from_env(), service_name=service_name_from_env()
+            )
+            atexit.register(_shutdown_process_exporter)
+        return _process_exporter
+
+
+def set_exporter(exporter: OtelExporter | None) -> None:
+    """Install (or clear) the process exporter. `None` makes the next call build a fresh one."""
+    global _process_exporter
+    with _process_lock:
+        _process_exporter = exporter
+
+
+def _shutdown_process_exporter() -> None:
+    """Drain whatever is queued before the interpreter goes away, so the last request is not lost."""
+    exporter = _process_exporter
+    if exporter is not None:
+        exporter.shutdown()

snakeorm/debug/otel/payload.py

@@ -0,0 +1,97 @@
+"""The OTLP/HTTP JSON body: the ~60 lines of stdlib that stand in for the whole SDK.
+
+The shape is protobuf's JSON mapping, which is picky in ways plain JSON is not, and each pickiness
+is a 400 from the collector rather than a wrong picture:
+
+- A 64-bit field travels as a STRING (`startTimeUnixNano`, `intValue`). A JSON number loses
+  precision above 2^53, and a Unix nanosecond is well past that.
+- Every attribute is a `{key, value}` pair with the value TAGGED by type.
+- An absent parent is an ABSENT field, not an empty one.
+"""
+
+from __future__ import annotations
+
+import json
+
+from snakeorm.debug.otel.spans import AttributeValue, SnakeSpan, SpanEvent
+
+SCOPE_NAME = "snakeorm"
+"""Who produced these spans. It is how a reader tells ours from the application's own."""
+
+SERVICE_NAME_KEY = "service.name"
+"""The resource attribute Jaeger lists in its Service dropdown."""
+
+
+def otlp_payload(
+    spans: tuple[SnakeSpan, ...], *, service_name: str
+) -> dict[str, object]:
+    """The complete `resourceSpans` document for one export, ready to be serialised."""
+    return {
+        "resourceSpans": [
+            {
+                "resource": {
+                    "attributes": [_attribute(SERVICE_NAME_KEY, service_name)]
+                },
+                "scopeSpans": [
+                    {
+                        "scope": {"name": SCOPE_NAME},
+                        "spans": [_span(span) for span in spans],
+                    }
+                ],
+            }
+        ]
+    }
+
+
+def encode_payload(payload: dict[str, object]) -> bytes:
+    """The bytes the POST sends. Compact and UTF-8: nothing here is read by a person."""
+    return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()
+
+
+def _span(span: SnakeSpan) -> dict[str, object]:
+    """One span in OTLP's JSON shape, omitting the parent when there is none."""
+    encoded: dict[str, object] = {
+        "traceId": span.trace_id,
+        "spanId": span.span_id,
+        "name": span.name,
+        "kind": int(span.kind),
+        "startTimeUnixNano": str(span.start_unix_nano),
+        "endTimeUnixNano": str(span.end_unix_nano),
+        "attributes": [_attribute(key, value) for key, value in span.attributes],
+    }
+    if span.parent_span_id:
+        encoded["parentSpanId"] = span.parent_span_id
+    if span.events:
+        encoded["events"] = [_event(event) for event in span.events]
+    return encoded
+
+
+def _event(event: SpanEvent) -> dict[str, object]:
+    """One span event: its instant, its name and its own attributes."""
+    return {
+        "name": event.name,
+        "timeUnixNano": str(event.time_unix_nano),
+        "attributes": [_attribute(key, value) for key, value in event.attributes],
+    }
+
+
+def _attribute(key: str, value: AttributeValue) -> dict[str, object]:
+    """A `{key, value}` pair with the value tagged by its type."""
+    return {"key": key, "value": _value(value)}
+
+
+def _value(value: AttributeValue) -> dict[str, object]:
+    """Tag a value with its OTLP type.
+
+    `bool` is checked FIRST on purpose: in Python it is a subclass of `int`, so the obvious order
+    ships every `True` as the integer `1` and the flag stops being a flag in the backend's index.
+    """
+    if isinstance(value, bool):
+        return {"boolValue": value}
+    if isinstance(value, int):
+        return {"intValue": str(value)}
+    if isinstance(value, float):
+        return {"doubleValue": value}
+    if isinstance(value, tuple):
+        return {"arrayValue": {"values": [{"stringValue": item} for item in value]}}
+    return {"stringValue": value}

snakeorm/debug/otel/report.py

@@ -0,0 +1,29 @@
+"""`export_report`: the ONE call the three web adapters make, and where the channel is checked.
+
+It lives here and not in `contrib/deliver.py` on purpose. `plan_delivery` is a PURE function that
+answers `(headers, envelope)` — two things that modify the response — and `Delivery` is exactly that
+pair. Sending bytes over the network is neither, and widening that dataclass to hold an I/O side
+effect would dissolve the reason `deliver.py` exists. So this is shared the way `index_advice`
+already is: one function, called from all three adapters right after `report.with_request(...)`.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from snakeorm.debug.channel import SnakeDebugChannel
+from snakeorm.debug.otel.exporter import current_exporter
+
+if TYPE_CHECKING:
+    from snakeorm.debug.report import DebugReport
+
+
+def export_report(report: DebugReport, channels: frozenset[SnakeDebugChannel]) -> None:
+    """Queue the report as spans when the `otel` channel is on; do nothing at all when it is not.
+
+    With the channel off the whole cost is one frozenset membership test: no exporter is built, no
+    thread is started and `opentelemetry` is never imported.
+    """
+    if SnakeDebugChannel.OTEL not in channels:
+        return
+    current_exporter().submit(report)