devbench 0.1.0__py3-none-any.whl

devbench/celery.py

@@ -0,0 +1,152 @@
+"""Celery: failed tasks, and the trace from the request that enqueued them::
+
+    # where your Celery app is created (tasks.py, celery.py, ...)
+    import devbench.celery
+    devbench.celery.install()
+
+Under Django this is automatic (``"devbench.django"`` installs it when
+Celery is loaded). Signals only: nothing wraps your tasks.
+
+* ``before_task_publish``: the active trace travels in the task message's
+  headers, so a task's failure and its log lines join the request that
+  enqueued it.
+* ``task_prerun`` / ``task_postrun``: the task runs in its own scope (its
+  trace; identity set inside it ends with it).
+* ``task_failure``: reported as a ``job`` exception, symbol = the task name.
+  A retry (``self.retry``) is not a failure; the last attempt's failure is.
+* ``after_setup_logger`` / ``after_setup_task_logger``: log capture is put
+  back after the worker replaces the root logger's handlers.
+* ``worker_process_shutdown``: a prefork child flushes before it exits (the
+  pool ends children without running atexit).
+
+A worker process reports as "<service>-celery" unless DEVBENCH_SERVICE is
+set, so an app's web and worker processes are told apart.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Dict, Optional
+
+from celery import signals
+
+from . import _core, _integration, context, logs, reporter
+from .trace import Trace
+
+# The task-message header the trace travels in.
+HEADER = "adt_trace"
+WORKER_SUFFIX = "celery"
+
+_installed = False
+_lock = threading.Lock()
+_tokens: Dict[str, Any] = {}
+
+
+def install() -> bool:
+    """Connects the hooks. Safe to call more than once. Returns True when
+    installed."""
+    global _installed
+    with _lock:
+        if _installed:
+            return True
+        try:
+            signals.before_task_publish.connect(_before_publish, weak=False, dispatch_uid="devbench.publish")
+            signals.task_prerun.connect(_prerun, weak=False, dispatch_uid="devbench.prerun")
+            signals.task_postrun.connect(_postrun, weak=False, dispatch_uid="devbench.postrun")
+            signals.task_failure.connect(_failure, weak=False, dispatch_uid="devbench.failure")
+            signals.worker_init.connect(_worker_init, weak=False, dispatch_uid="devbench.worker_init")
+            signals.worker_process_shutdown.connect(_worker_shutdown, weak=False, dispatch_uid="devbench.shutdown")
+            signals.after_setup_logger.connect(_after_logging, weak=False, dispatch_uid="devbench.logger")
+            signals.after_setup_task_logger.connect(_after_logging, weak=False, dispatch_uid="devbench.task_logger")
+            _installed = True
+        except Exception:
+            return False
+    if _in_worker():
+        _mark_worker()
+    _integration.setup()
+    return True
+
+
+def _in_worker() -> bool:
+    return _integration.celery_worker_command()
+
+
+def _mark_worker() -> None:
+    try:
+        cfg = _core.config()
+        if not cfg.worker:
+            cfg.worker = WORKER_SUFFIX
+    except Exception:
+        pass
+
+
+def _worker_init(**_: Any) -> None:
+    _mark_worker()
+
+
+def _after_logging(logger: Any = None, **_: Any) -> None:
+    """A worker replaces the root logger's handlers when it sets up logging
+    (worker_hijack_root_logger), and its task logger does not propagate:
+    capture goes back on both."""
+    try:
+        if _core.capture_logs() and isinstance(logger, logging.Logger):
+            logs.install(logger)
+    except Exception:
+        pass
+
+
+def _before_publish(headers: Optional[dict] = None, **_: Any) -> None:
+    try:
+        trace = context.current_trace()
+        if trace is not None and isinstance(headers, dict) and HEADER not in headers:
+            headers[HEADER] = str(trace)
+    except Exception:
+        pass
+
+
+def _prerun(task_id: Optional[str] = None, task: Any = None, **_: Any) -> None:
+    try:
+        trace = Trace.parse(_header(task))
+        _scope, token = context.enter(trace)
+        if task_id:
+            _tokens[task_id] = token
+    except Exception:
+        pass
+
+
+def _postrun(task_id: Optional[str] = None, **_: Any) -> None:
+    try:
+        token = _tokens.pop(task_id, None) if task_id else None
+        if token is not None:
+            context.leave(token)
+    except Exception:
+        pass
+
+
+def _failure(sender: Any = None, exception: Optional[BaseException] = None, **_: Any) -> None:
+    try:
+        reporter.capture(exception, "job", False, _task_name(sender))
+    except Exception:
+        pass
+
+
+def _worker_shutdown(**_: Any) -> None:
+    _core.flush(timeout=2.0)
+
+
+def _header(task: Any) -> Optional[str]:
+    request = getattr(task, "request", None)
+    if request is None:
+        return None
+    value = getattr(request, HEADER, None)
+    if value is None:
+        headers = getattr(request, "headers", None)
+        if isinstance(headers, dict):
+            value = headers.get(HEADER)
+    return value if isinstance(value, str) else None
+
+
+def _task_name(task: Any) -> Optional[str]:
+    name = getattr(task, "name", None)
+    return name if isinstance(name, str) and name else None

devbench/config.py

@@ -0,0 +1,189 @@
+"""Where reports go, parsed from DEVBENCH_DSN, and everything else read at
+start (docs/SERVER_SDK_SPEC.md, "Dev Bench naming and configuration";
+DECISIONS #161)."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Mapping, Optional
+from urllib.parse import quote, unquote, urlsplit
+
+SHAPE = "https://<public>:<secret>@<host>"
+_LOOPBACK = re.compile(r"\A(?:localhost|127(?:\.[0-9]{1,3}){3}|::1|\[::1\])\Z|\.localhost\Z", re.IGNORECASE)
+
+
+class InvalidDSN(ValueError):
+    """A DSN that does not parse. The message says what to fix and never
+    contains either key."""
+
+
+class DSN:
+    """``https://<public>:<secret>@<host>[:port]`` from ``adt dsn create``, or
+    a single-key ``https://<key>@<host>``.
+
+    ``scheme://host[:port]`` is the ingest base. The secret (a server key) is
+    what this process authenticates with; the public part (a browser key) is
+    only ever handed to a page, via ``browser_dsn``. A path or query, if
+    present, is ignored: the keys alone identify tenant, environment and
+    source.
+    """
+
+    __slots__ = ("base", "key", "public_key")
+
+    def __init__(self, base: str, key: str, public_key: Optional[str] = None) -> None:
+        self.base = base
+        self.key = key
+        self.public_key = public_key
+
+    @property
+    def flush_url(self) -> str:
+        return self.base + "/v1/flush"
+
+    @property
+    def check_url(self) -> str:
+        """The self-test endpoint: validates the key, stores nothing."""
+        return self.base + "/v1/check"
+
+    @property
+    def logs_url(self) -> str:
+        return self.base + "/v1/logs"
+
+    @property
+    def browser_dsn(self) -> Optional[str]:
+        """The DSN a browser may hold: https://<public>@host, or None when
+        there is no public part."""
+        if self.public_key is None:
+            return None
+        scheme, rest = self.base.split("://", 1)
+        return "%s://%s@%s" % (scheme, quote(self.public_key, safe=""), rest)
+
+    def __str__(self) -> str:
+        # The DSN without its keys. The secret must never be printed.
+        return self.base
+
+    __repr__ = __str__
+
+    @classmethod
+    def parse(cls, raw: object) -> "DSN":
+        text = ("" if raw is None else str(raw)).strip()
+        if not text:
+            raise InvalidDSN("is empty")
+        try:
+            parts = urlsplit(text)
+            port = parts.port
+        except ValueError:
+            raise InvalidDSN("is not a URL (expected %s)" % SHAPE) from None
+
+        scheme = (parts.scheme or "").lower()
+        if scheme not in ("http", "https"):
+            raise InvalidDSN("must start with https:// (expected %s)" % SHAPE)
+        host = parts.hostname or ""
+        if not host:
+            raise InvalidDSN("has no host (expected %s)" % SHAPE)
+
+        user = unquote(parts.username or "").strip()
+        secret = unquote(parts.password or "").strip()
+        if not user and not secret:
+            raise InvalidDSN("has no key (expected %s)" % SHAPE)
+
+        # The secret is a real secret on a server. Over plain http it would
+        # cross the network readable by anything on the path, so http is
+        # accepted only for a loopback ingest (local development and tests).
+        if scheme == "http" and not _LOOPBACK.search(host):
+            raise InvalidDSN("uses http:// for %s: the key would cross the network in plaintext; use https://" % host)
+
+        default_port = 443 if scheme == "https" else 80
+        port_text = ":%d" % port if port is not None and port != default_port else ""
+        if ":" in host and not host.startswith("["):
+            host = "[%s]" % host
+        base = "%s://%s%s" % (scheme, host, port_text)
+        if not secret:
+            return cls(base, user)
+        return cls(base, secret, user or None)
+
+
+# Tried in order after the REVISION file when no release is configured. Common
+# deploy tools and platforms only; never an app's own variable. The same order
+# as the Ruby and Go SDKs (#23).
+RELEASE_FALLBACKS = ("HEROKU_SLUG_COMMIT", "KAMAL_VERSION", "GITHUB_SHA", "GIT_SHA", "SOURCE_VERSION", "RENDER_GIT_COMMIT")
+# Capistrano-style deploys write the deployed commit here, in the app root.
+REVISION_FILE = "REVISION"
+# A commit or tag, not a document: read no more than this.
+REVISION_MAX_BYTES = 255
+_OFF = ("false", "0", "no", "off")
+
+
+def _presence(value: object) -> Optional[str]:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+class Configuration:
+    """Everything Dev Bench reads at start. From the environment by default;
+    ``devbench.configure(...)`` sets the same fields from code.
+
+    * ``DEVBENCH_DSN``: where to send (fallback ``ADT_DSN``). Unset: nothing
+      is sent (this SDK has no sidecar mode).
+    * ``DEVBENCH_SERVICE``: this app's name. Default: what the framework
+      knows (the Django project, the Flask app's import name), else "app";
+      with "-celery" appended in a Celery worker.
+    * ``DEVBENCH_RELEASE``: the deployed version. Default, first found: a
+      REVISION file in the app root, then RELEASE_FALLBACKS, else "".
+    * ``DEVBENCH_ENABLED``: "false" turns everything off.
+    """
+
+    def __init__(self, env: Optional[Mapping[str, str]] = None) -> None:
+        env = os.environ if env is None else env
+        self.dsn: Optional[str] = _presence(env.get("DEVBENCH_DSN")) or _presence(env.get("ADT_DSN"))
+        self.service: Optional[str] = _presence(env.get("DEVBENCH_SERVICE"))
+        self.release: Optional[str] = _presence(env.get("DEVBENCH_RELEASE"))
+        self.enabled: bool = (env.get("DEVBENCH_ENABLED") or "").strip().lower() not in _OFF
+        # Seconds between background flushes (spec: 60).
+        self.flush_interval: float = 60.0
+        self._release_env = next((v for v in (_presence(env.get(n)) for n in RELEASE_FALLBACKS) if v), None)
+        # Set by the integrations, never by the app: the framework's own name
+        # for the app, the app root (REVISION, relative frame paths), and
+        # whether this process is a Celery worker.
+        self.framework_service: Optional[str] = None
+        self.app_root: Optional[str] = None
+        self.worker: Optional[str] = None
+        self._detected_release: Optional[str] = None
+
+    def resolved_service(self) -> str:
+        """The service name sent with every flush and folded into every
+        fingerprint. An explicit service always wins; a worker process
+        defaults to "<app>-<worker>", so an app's web and job processes are
+        told apart with no DEVBENCH_SERVICE at all."""
+        explicit = _presence(self.service)
+        if explicit:
+            return explicit
+        base = _presence(self.framework_service) or "app"
+        return "%s-%s" % (base, self.worker) if self.worker else base
+
+    def resolved_root(self) -> str:
+        return _presence(self.app_root) or os.getcwd()
+
+    def resolved_release(self) -> str:
+        """Explicit (configure or DEVBENCH_RELEASE) wins; else the REVISION
+        file in the app root, read once; else RELEASE_FALLBACKS; else ""."""
+        explicit = _presence(self.release)
+        if explicit:
+            return explicit
+        if self._detected_release is None:
+            self._detected_release = self._revision_file() or self._release_env or ""
+        return self._detected_release
+
+    def _revision_file(self) -> Optional[str]:
+        try:
+            path = os.path.join(self.resolved_root(), REVISION_FILE)
+            if not os.path.isfile(path):
+                return None
+            with open(path, "rb") as f:
+                head = f.read(REVISION_MAX_BYTES)
+            lines = head.decode("utf-8", "replace").splitlines()
+            return _presence(lines[0]) if lines else None
+        except Exception:
+            return None

devbench/context.py

@@ -0,0 +1,94 @@
+"""Per-request (and per-job) state: the active trace, who the request is for,
+and how many failures were handled during it.
+
+One mutable ``Scope`` object per request, held in a ContextVar. Mutable on
+purpose: frameworks run parts of a request in a *copy* of the context (a
+FastAPI/Starlette sync endpoint runs in a thread pool; asgiref's
+sync_to_async copies too), and a ``ContextVar.set`` made in a copy never
+reaches the middleware that reports the request's failure. Setting a field on
+the shared Scope does.
+
+Scopes nest and restore rather than clear: servers reuse threads, and
+clearing would leak one request's absence into the next one's presence;
+inheriting would attribute one user's failure to whoever this thread served
+before.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+from typing import Iterator, Optional
+
+from .trace import Trace
+
+
+class Scope:
+    __slots__ = ("trace", "user", "handled", "__weakref__")
+
+    def __init__(self, trace: Optional[Trace] = None) -> None:
+        self.trace = trace
+        # A frozen dict {"email": ..., "account": ...} (empty fields dropped)
+        # or None. Normalised before it gets here (identity.normalize).
+        self.user: Optional[dict] = None
+        # Failures handled during this request: the middleware turns a
+        # non-zero count into the x-adt-handled response header.
+        self.handled = 0
+
+
+_SCOPE: "contextvars.ContextVar[Optional[Scope]]" = contextvars.ContextVar("devbench_scope", default=None)
+
+
+def current() -> Optional[Scope]:
+    return _SCOPE.get()
+
+
+def current_trace() -> Optional[Trace]:
+    scope = _SCOPE.get()
+    return scope.trace if scope is not None else None
+
+
+def current_user() -> Optional[dict]:
+    scope = _SCOPE.get()
+    return scope.user if scope is not None else None
+
+
+def ensure() -> Scope:
+    """The current scope, creating one (outside any request: a script, a
+    shell) when there is none."""
+    scope = _SCOPE.get()
+    if scope is None:
+        scope = Scope()
+        _SCOPE.set(scope)
+    return scope
+
+
+@contextlib.contextmanager
+def scoped(trace: Optional[Trace]) -> Iterator[Scope]:
+    """Runs the block with a fresh scope for ``trace`` (which may be None),
+    restoring whatever was there before."""
+    scope = Scope(trace)
+    token = _SCOPE.set(scope)
+    try:
+        yield scope
+    finally:
+        try:
+            _SCOPE.reset(token)
+        except ValueError:
+            # Reset from another context (a generator finalised elsewhere):
+            # fall back to clearing, never raising into the app.
+            _SCOPE.set(None)
+
+
+def enter(trace: Optional[Trace]) -> "tuple[Scope, contextvars.Token]":
+    """For hooks whose start and end are separate calls (Celery's prerun and
+    postrun): returns the new scope and the token ``leave`` needs."""
+    scope = Scope(trace)
+    return scope, _SCOPE.set(scope)
+
+
+def leave(token: "contextvars.Token") -> None:
+    try:
+        _SCOPE.reset(token)
+    except (ValueError, RuntimeError):
+        _SCOPE.set(None)