devbench 0.1.0__py3-none-any.whl

devbench/logs.py

@@ -0,0 +1,282 @@
+"""In-process log capture (docs/SERVER_SDK_SPEC.md, "In-process log capture
+(direct mode)"): recent log lines kept in memory, keyed by the trace that was
+current when each was written, so triage can be handed the server lines of
+one user action without a sidecar.
+
+One way in: ``LogHandler``, a ``logging.Handler`` on the root logger
+(``devbench.capture_logs()``; every integration installs it). It sees exactly
+what the app's loggers already let through, so it never turns on a level the
+app has off; it writes nothing anywhere, it only remembers.
+
+Never harms the host: a record with no current trace costs one ContextVar
+read; the buffer lock is held for a deque append, never across I/O or
+redaction; nothing raises out of a logging call. Fork-safe: a child starts
+with an empty buffer.
+"""
+
+from __future__ import annotations
+
+import collections
+import logging
+import os
+import threading
+import time
+from typing import Any, Callable, Deque, Dict, List, Optional
+
+from . import context
+
+MAX_LINES = 10_000
+MAX_BYTES = 4 * 1024 * 1024
+MAX_AGE = 15 * 60
+MAX_LINE_BYTES = 4096
+# Lines held from one multi-line message (an exception and its traceback):
+# the top of it, so one framework traceback cannot push the rest of the
+# request out of a 200-line slice.
+MAX_MESSAGE_LINES = 50
+
+
+class _Entry:
+    __slots__ = ("key", "at", "text", "size")
+
+    def __init__(self, key: str, at: float, text: str) -> None:
+        self.key = key
+        self.at = at
+        self.text = text
+        self.size = len(text.encode("utf-8", "replace"))
+
+
+class Buffer:
+    """Bounded, thread-safe, per-trace store of recent lines. Oldest evicted
+    first, by count, bytes and age."""
+
+    def __init__(self, max_lines: int = MAX_LINES, max_bytes: int = MAX_BYTES, max_age: float = MAX_AGE,
+                 clock: Callable[[], float] = time.monotonic) -> None:
+        self.max_lines = max_lines
+        self.max_bytes = max_bytes
+        self.max_age = max_age
+        self._clock = clock
+        self._lock = threading.Lock()
+        self._reset()
+
+    def _reset(self) -> None:
+        self._pid = os.getpid()
+        self._entries: Deque[_Entry] = collections.deque()
+        self._by_key: Dict[str, Deque[_Entry]] = {}
+        self._bytes = 0
+
+    def push(self, key: str, text: str) -> None:
+        """Keeps one line under a trace key."""
+        now = self._clock()
+        entry = _Entry(key, now, text)
+        with self._lock:
+            if self._pid != os.getpid():
+                self._reset()
+            self._entries.append(entry)
+            self._by_key.setdefault(key, collections.deque()).append(entry)
+            self._bytes += entry.size
+            self._evict(now)
+
+    def lookup(self, key: str, limit: int) -> List[str]:
+        """The lines held for a key, oldest first, at most ``limit`` (the most
+        recent ones when there are more). A copy: the caller redacts it
+        outside the lock."""
+        with self._lock:
+            if self._pid != os.getpid():
+                self._reset()
+            self._evict(self._clock())
+            held = self._by_key.get(key)
+            if not held:
+                return []
+            items = list(held)
+        return [e.text for e in items[-limit:]]
+
+    def any(self) -> bool:
+        """Whether any line younger than the age bound is held (by this
+        process)."""
+        with self._lock:
+            if self._pid != os.getpid():
+                self._reset()
+            self._evict(self._clock())
+            return bool(self._entries)
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._entries)
+
+    @property
+    def bytes(self) -> int:
+        with self._lock:
+            return self._bytes
+
+    def clear(self) -> None:
+        with self._lock:
+            self._reset()
+
+    # Every entry is the oldest of its own key when it is the oldest overall,
+    # so removing it from its key's deque is a popleft.
+    def _evict(self, now: float) -> None:
+        cutoff = now - self.max_age
+        entries = self._entries
+        while entries and (len(entries) > self.max_lines or self._bytes > self.max_bytes or entries[0].at < cutoff):
+            oldest = entries.popleft()
+            held = self._by_key.get(oldest.key)
+            if held:
+                held.popleft()
+                if not held:
+                    del self._by_key[oldest.key]
+            self._bytes -= oldest.size
+
+
+_buffer = Buffer()
+_active = False
+_polling_pid: Optional[int] = None
+_state_lock = threading.Lock()
+_formatter = logging.Formatter()
+
+
+def buffer() -> Buffer:
+    return _buffer
+
+
+def set_buffer(value: Buffer) -> None:
+    """For tests: a buffer with other bounds or a controllable clock."""
+    global _buffer
+    _buffer = value
+
+
+def active() -> bool:
+    return _active
+
+
+def activate() -> None:
+    global _active, _polling_pid
+    _polling_pid = None
+    _active = True
+
+
+def deactivate() -> None:
+    global _active, _polling_pid
+    _polling_pid = None
+    _active = False
+
+
+def holding() -> bool:
+    """Whether this process holds any traced line: the transport polls ingest
+    for log requests only then."""
+    try:
+        return _buffer.any()
+    except Exception:
+        return False
+
+
+def lookup(key: str, limit: int) -> List[str]:
+    try:
+        return _buffer.lookup(key, limit)
+    except Exception:
+        return []
+
+
+class LogHandler(logging.Handler):
+    """Keeps every record logged under a trace, one entry per line, as
+    ``[v1/<session>/<intent>/<hop>] LEVEL message``. Records with no current
+    trace are not kept."""
+
+    def __init__(self) -> None:
+        super().__init__(logging.NOTSET)
+
+    def handle(self, record: logging.LogRecord) -> bool:
+        # No handler lock and no filters: emit touches only the buffer, which
+        # has its own lock, and must cost nothing for an untraced record.
+        self.emit(record)
+        return True
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            # Our own warnings are never held: logging one while the
+            # transport is being built must not start building it again.
+            if not _active or record.name == "devbench":
+                return
+            trace = context.current_trace()
+            if trace is None:
+                return
+            for line in lines_for(str(trace), record.levelname, _message(record)):
+                _buffer.push(trace.key, line)
+            _start_polling()
+        except Exception:
+            # Never handleError(): that prints to the app's stderr.
+            pass
+
+
+def install(logger: Optional[logging.Logger] = None) -> bool:
+    """Adds the capturing handler to a logger (default: the root logger).
+    Idempotent. Returns True when it is in place."""
+    target = logger if logger is not None else logging.getLogger()
+    with _state_lock:
+        if not any(isinstance(h, LogHandler) for h in target.handlers):
+            target.addHandler(LogHandler())
+    return True
+
+
+def lines_for(trace: str, level: Any, text: str) -> List[str]:
+    """One held line per line of the message, each at most MAX_LINE_BYTES of
+    valid UTF-8. A multi-line message (an exception and its traceback) is
+    split so each line reaches triage on its own, blank lines dropped; at most
+    MAX_MESSAGE_LINES, the last saying how many were left out."""
+    prefix = "[%s] %s " % (trace, level) if level else "[%s] " % trace
+    if text.endswith("\n"):
+        text = text[:-1]
+    if "\n" not in text:
+        return [_bound(prefix + text)]
+    parts = [p[:-1] if p.endswith("\r") else p for p in text.split("\n")]
+    parts = [p for p in parts if p.strip()]
+    if not parts:
+        return []
+    if len(parts) > MAX_MESSAGE_LINES:
+        omitted = len(parts) - (MAX_MESSAGE_LINES - 1)
+        parts = parts[: MAX_MESSAGE_LINES - 1] + ["... (%d more lines)" % omitted]
+    return [_bound(prefix + p) for p in parts]
+
+
+def _message(record: logging.LogRecord) -> str:
+    # The record's own message, as Formatter.format builds it before
+    # adding the app's format string: the text, then the traceback.
+    text = record.getMessage()
+    if record.exc_info:
+        exc = _formatter.formatException(record.exc_info)
+        text = text + "\n" + exc if text else exc
+    elif record.exc_text:
+        text = text + "\n" + record.exc_text if text else record.exc_text
+    if record.stack_info:
+        text = text + "\n" + record.stack_info
+    return _valid(text)
+
+
+def _valid(text: str) -> str:
+    from .fingerprint import utf8
+
+    return utf8(text)
+
+
+def _bound(line: str) -> str:
+    data = line.encode("utf-8", "replace")
+    if len(data) <= MAX_LINE_BYTES:
+        return line
+    return data[:MAX_LINE_BYTES].decode("utf-8", "ignore")
+
+
+def _start_polling() -> None:
+    """The first line this process holds starts the transport's flush
+    thread, so a process with nothing to report still polls for log requests
+    (a forked worker included: the PID is per process)."""
+    global _polling_pid
+    pid = os.getpid()
+    if _polling_pid == pid:
+        return
+    _polling_pid = pid
+    from . import _core
+
+    t = _core.transport()
+    start = getattr(t, "start", None)
+    if start is not None:
+        start()
+

devbench/reporter.py

@@ -0,0 +1,233 @@
+"""Turns a failure into a report (the ``exception`` / ``handled_failure``
+message of docs/SERVER_SDK_SPEC.md, as the Ruby and Go SDKs build it) and
+hands it to the active transport.
+
+Every failure here is swallowed. This is the one place where that is correct:
+everywhere else it is the exact behaviour the product exists to find.
+"""
+
+from __future__ import annotations
+
+import collections
+import os
+import threading
+from types import TracebackType
+from typing import Any, List, Optional
+
+from . import context
+from .fingerprint import utf8
+from .version import PROTOCOL_VERSION
+
+# docs/SERVER_SDK_SPEC.md, "Control socket — v1".
+MAX_MESSAGE = 2000
+MAX_HANDLED_MESSAGE = 500
+MAX_FRAMES = 50
+# Bounds one frame so 50 of them stay small.
+MAX_FIELD = 1024
+
+CONTEXTS = ("request", "job", "explicit")
+
+# Status codes and framework control flow, not failures. Matched by name
+# against the exception's class and its bases (module.QualName), so none of
+# these packages needs to be importable. Most never reach a hook (Django,
+# Flask and Starlette answer them before any error signal), but an app may
+# pass one to capture_exception. Extend with devbench.ignore_exceptions.
+DEFAULT_IGNORED = (
+    "django.http.response.Http404",
+    "django.core.exceptions.PermissionDenied",
+    "django.core.exceptions.SuspiciousOperation",
+    "django.core.exceptions.BadRequest",
+    "werkzeug.exceptions.HTTPException",
+    "starlette.exceptions.HTTPException",
+    "celery.exceptions.Retry",
+    "celery.exceptions.Ignore",
+    "celery.exceptions.Reject",
+)
+
+_MARK = "__devbench_reported__"
+# Exceptions that refuse an attribute: remembered by id, bounded.
+_UNMARKABLE_MAX = 1024
+
+_ignored: List[str] = list(DEFAULT_IGNORED)
+_unmarkable: "collections.OrderedDict[int, bool]" = collections.OrderedDict()
+_unmarkable_lock = threading.Lock()
+
+
+def ignore_exceptions() -> List[str]:
+    """The class names never reported (a list the app may extend)::
+
+        devbench.ignore_exceptions().append("myapp.errors.NotAuthorized")
+    """
+    return _ignored
+
+
+def class_name(cls: type) -> str:
+    """``module.QualName``; builtins by their plain name (``ValueError``)."""
+    module = getattr(cls, "__module__", None) or ""
+    qual = getattr(cls, "__qualname__", None) or getattr(cls, "__name__", "Exception")
+    return qual if module in ("builtins", "exceptions", "") else "%s.%s" % (module, qual)
+
+
+def ignored(error: BaseException) -> bool:
+    # Process control, not application failures: SystemExit,
+    # KeyboardInterrupt, GeneratorExit, asyncio.CancelledError.
+    if not isinstance(error, Exception):
+        return True
+    try:
+        names = {class_name(c) for c in type(error).__mro__}
+        return any(name in names for name in _ignored)
+    except Exception:
+        return False
+
+
+def reported(error: BaseException) -> bool:
+    """Whether any hook already sent this exception object."""
+    try:
+        if getattr(error, _MARK, False) is True:
+            return True
+    except Exception:
+        pass
+    with _unmarkable_lock:
+        return id(error) in _unmarkable
+
+
+def _mark(error: BaseException) -> None:
+    try:
+        setattr(error, _MARK, True)
+        if getattr(error, _MARK, False) is True:
+            return
+    except Exception:
+        pass
+    with _unmarkable_lock:
+        _unmarkable[id(error)] = True
+        while len(_unmarkable) > _UNMARKABLE_MAX:
+            _unmarkable.popitem(last=False)
+
+
+def capture(error: Any, context_name: str, handled: bool, symbol: Optional[str] = None) -> None:
+    """Sends an ``exception`` report, once per exception object. Never
+    raises: this runs inside the app's own error paths, and anything raised
+    here would replace the application's exception."""
+    try:
+        if not isinstance(error, BaseException):
+            return
+        if reported(error) or ignored(error):
+            return
+        # Marked before sending, so a second hook reached while this one is
+        # still sending does not send it again.
+        _mark(error)
+        _emit(exception_payload(error, context_name, handled, symbol))
+    except Exception:
+        pass
+
+
+def report_handled(error: Any, symbol: str, reason: Optional[str] = None) -> None:
+    """Records that a failure was handled here. Does not change behaviour."""
+    # Counted first, and regardless of whether anything after it works: the
+    # count is what the browser is told.
+    try:
+        context.ensure().handled += 1
+    except Exception:
+        pass
+    try:
+        _emit(handled_payload(error, symbol, reason))
+    except Exception:
+        pass
+
+
+def exception_payload(error: BaseException, context_name: str, handled: bool, symbol: Optional[str]) -> dict:
+    payload = {
+        "v": PROTOCOL_VERSION,
+        "kind": "exception",
+        "context": context_name if context_name in CONTEXTS else "explicit",
+        "handled": bool(handled),
+        "error": _safely(lambda: class_name(type(error))) or "Exception",
+        "message": _safely(lambda: truncate(_message(error), MAX_MESSAGE)),
+        "symbol": _safely(lambda: utf8(symbol) if symbol is not None else "") or "",
+        "frames": _safely(lambda: frames(error.__traceback__, _app_root())) or [],
+        "trace": _safely(lambda: _trace_text()),
+        "user": _safely(context.current_user),
+    }
+    return {k: v for k, v in payload.items() if v is not None}
+
+
+def handled_payload(error: Any, symbol: Any, reason: Any) -> dict:
+    # Each field is read on its own: an exception whose __str__ raises still
+    # reports its class and site, which are the fingerprint.
+    payload = {
+        "v": PROTOCOL_VERSION,
+        "kind": "handled_failure",
+        "symbol": _safely(lambda: utf8(symbol)) if symbol is not None else None,
+        "error": _safely(lambda: class_name(type(error))),
+        "message": _safely(lambda: truncate(_message(error), MAX_HANDLED_MESSAGE)),
+        "reason": _safely(lambda: utf8(reason)) if reason is not None else None,
+        "trace": _safely(lambda: _trace_text()),
+        "user": _safely(context.current_user),
+    }
+    return {k: v for k, v in payload.items() if v is not None}
+
+
+def frames(tb: Optional[TracebackType], root: Optional[str]) -> list:
+    """A traceback as ``[{function, file, line}]``, innermost first, at most
+    MAX_FRAMES, with files under ``root`` made relative to it."""
+    window: "collections.deque" = collections.deque(maxlen=MAX_FRAMES)
+    while tb is not None:
+        window.append(tb)
+        tb = tb.tb_next
+    prefix = _root_prefix(root)
+    out = []
+    for item in reversed(window):
+        code = item.tb_frame.f_code
+        function = getattr(code, "co_qualname", None) or code.co_name
+        file = utf8(code.co_filename)
+        if prefix and file.startswith(prefix):
+            file = file[len(prefix):]
+        out.append({"function": _clip(utf8(function)), "file": _clip(file), "line": int(item.tb_lineno or 0)})
+    return out
+
+
+def truncate(text: Any, limit: int) -> str:
+    """At most ``limit`` characters, including the ellipsis."""
+    text = utf8(text)
+    return text[: limit - 1] + "…" if len(text) > limit else text
+
+
+def _message(error: Any) -> str:
+    return str(error)
+
+
+def _trace_text() -> Optional[str]:
+    trace = context.current_trace()
+    return str(trace) if trace is not None else None
+
+
+def _app_root() -> Optional[str]:
+    from . import _core
+
+    return _core.config().resolved_root()
+
+
+def _root_prefix(root: Optional[str]) -> Optional[str]:
+    if not root:
+        return None
+    return root if root.endswith(os.sep) else root + os.sep
+
+
+def _clip(text: str) -> str:
+    return text[:MAX_FIELD] if len(text) > MAX_FIELD else text
+
+
+def _safely(fn: Any) -> Any:
+    try:
+        return fn()
+    except Exception:
+        return None
+
+
+def _emit(payload: dict) -> None:
+    from . import _core
+
+    try:
+        _core.transport().deliver(payload)
+    except Exception:
+        pass