crumbtrail-python 0.1.0__py3-none-any.whl

crumbtrail/__init__.py

@@ -0,0 +1,4 @@
+from .core import Client, Sender
+from .middleware import ASGIMiddleware, WSGIMiddleware
+
+__all__ = ["Client", "Sender", "ASGIMiddleware", "WSGIMiddleware"]

crumbtrail/core.py

@@ -0,0 +1,373 @@
+"""Request scoped evidence and bounded asynchronous HTTP delivery."""
+import atexit
+import contextvars
+import datetime
+import email.utils
+import ipaddress
+import json
+import logging
+import os
+import queue
+import re
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import weakref
+
+from .privacy import capture_body, is_json, redaction, MAX_BYTES
+
+log = logging.getLogger("crumbtrail")
+
+current_capture = contextvars.ContextVar("crumbtrail_capture", default=None)
+_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
+_SECONDS = re.compile(r"[0-9]{1,5}")
+_MAX_RETRY_AFTER = 30.0
+
+
+def now():
+    return int(time.time() * 1000)
+
+
+def _loopback(hostname):
+    """Local stacks are served over plain HTTP, so exempt loopback from the HTTPS rule."""
+    if not hostname:
+        return False
+    host = hostname.lower().strip("[]")
+    if host == "localhost" or host.endswith(".localhost"):
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+# Senders are tracked weakly so the fork and exit hooks below can reach every live
+# sender without keeping a dead one alive. The lock is taken before every fork and
+# released on both sides, otherwise a child could inherit it already held.
+_registry_lock = threading.Lock()
+_registry = []
+
+
+def _register(sender):
+    with _registry_lock:
+        _registry[:] = [ref for ref in _registry if ref() is not None]
+        _registry.append(weakref.ref(sender))
+
+
+def _live_senders():
+    with _registry_lock:
+        refs = list(_registry)
+    return [sender for sender in (ref() for ref in refs) if sender is not None]
+
+
+def _before_fork():
+    _registry_lock.acquire()
+
+
+def _after_fork_parent():
+    _registry_lock.release()
+
+
+def _after_fork_child():
+    try:
+        senders = [ref() for ref in _registry]
+    finally:
+        _registry_lock.release()
+    for sender in senders:
+        if sender is not None:
+            sender._adopt_fork()
+
+
+def _flush_at_exit():
+    for sender in _live_senders():
+        try:
+            sender.close(2)
+        except Exception:
+            log.debug("Crumbtrail: flush at interpreter exit failed", exc_info=True)
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=_before_fork, after_in_parent=_after_fork_parent, after_in_child=_after_fork_child)
+atexit.register(_flush_at_exit)
+
+
+class Sender:
+    """One worker per process. Rebuilt automatically in a forked child; close on shutdown."""
+    def __init__(self, endpoint, key):
+        parsed = urllib.parse.urlsplit(endpoint)
+        if parsed.scheme not in ("https", "http") or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise ValueError("Crumbtrail endpoint must be HTTPS without credentials, query or fragment")
+        if parsed.scheme == "http" and not _loopback(parsed.hostname):
+            raise ValueError("Crumbtrail endpoint must be HTTPS unless it is a loopback address")
+        if not key or any(ord(c) < 32 or ord(c) > 126 for c in key):
+            raise ValueError("Crumbtrail ingest key must be nonempty printable ASCII")
+        self.url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, "/api/events", "", ""))
+        self.key = key
+        self.dropped = 0
+        self.failed = 0
+        self._counters = threading.Lock()
+        self._closed = False
+        self._warned_fork = False
+        self.queue = queue.Queue(maxsize=64)
+        self._lock = threading.Lock()
+        self._worker = None
+        self._pid = os.getpid()
+        self._http = urllib.request.build_opener(_NoRedirect())
+        _register(self)
+
+    def _adopt_fork(self):
+        """Runs single threaded in the child, so state is replaced by assignment.
+
+        Only this thread survives fork, so the queue, the worker and both locks are
+        rebuilt. `_closed` is inherited on purpose: a sender the parent shut down
+        stays shut down.
+        """
+        self.queue = queue.Queue(maxsize=64)
+        self._lock = threading.Lock()
+        self._counters = threading.Lock()
+        self._worker = None
+        self._pid = os.getpid()
+
+    def _count_drop(self):
+        with self._counters:
+            self.dropped += 1
+
+    def _count_failure(self):
+        with self._counters:
+            self.failed += 1
+
+    def enqueue(self, batch):
+        if os.getpid() != self._pid:
+            self._count_drop()
+            if not self._warned_fork:
+                self._warned_fork = True
+                log.warning("Crumbtrail: dropping evidence in process %d from a sender built in process %d; build the client after the worker forks", os.getpid(), self._pid)
+            return
+        with self._lock:
+            if self._closed:
+                self._count_drop()
+                return
+            if self._worker is None:
+                self._worker = threading.Thread(target=self._run, name="crumbtrail-delivery", daemon=True)
+                self._worker.start()
+            try:
+                self.queue.put_nowait(batch)
+            except queue.Full:
+                self._count_drop()
+                log.debug("Crumbtrail: delivery queue is full, dropping a batch")
+
+    @staticmethod
+    def _retryable(status):
+        # A 404 means the session is not one this key owns, which no retry changes.
+        return status == 429 or status >= 500
+
+    @staticmethod
+    def _retry_after(headers, status):
+        if status != 429 or headers is None:
+            return None
+        try:
+            value = headers.get("Retry-After")
+        except Exception:
+            return None
+        if value is None:
+            return None
+        value = str(value).strip()
+        if _SECONDS.fullmatch(value):
+            return min(float(value), _MAX_RETRY_AFTER)
+        try:
+            stamp = email.utils.parsedate_to_datetime(value)
+        except (TypeError, ValueError, IndexError):
+            return None
+        if stamp is None:
+            return None
+        if stamp.tzinfo is None:
+            stamp = stamp.replace(tzinfo=datetime.timezone.utc)
+        seconds = (stamp - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
+        return max(0.0, min(seconds, _MAX_RETRY_AFTER))
+
+    def _send(self, batch):
+        body = json.dumps(batch, allow_nan=False).encode()
+        for attempt in range(4):
+            request = urllib.request.Request(self.url, data=body, headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json"}, method="POST")
+            delay = None
+            try:
+                with self._http.open(request, timeout=5) as response:
+                    if response.status == 200:
+                        return True
+                    if not self._retryable(response.status):
+                        log.warning("Crumbtrail: ingest rejected a batch with status %s", response.status)
+                        return False
+                    delay = self._retry_after(getattr(response, "headers", None), response.status)
+            except urllib.error.HTTPError as error:
+                code, headers = error.code, error.headers
+                error.close()
+                if not self._retryable(code):
+                    log.warning("Crumbtrail: ingest rejected a batch with status %s", code)
+                    return False
+                delay = self._retry_after(headers, code)
+            except (OSError, urllib.error.URLError) as failure:
+                log.debug("Crumbtrail: delivery attempt %d failed: %r", attempt + 1, failure)
+            if attempt < 3:
+                time.sleep(0.25 * (attempt + 1) if delay is None else delay)
+        log.warning("Crumbtrail: dropping %d events after 4 delivery attempts", len(batch.get("events") or []) if isinstance(batch, dict) else 0)
+        return False
+
+    def _run(self):
+        while True:
+            try:
+                batch = self.queue.get(timeout=0.1)
+            except queue.Empty:
+                if self._closed:
+                    return
+                continue
+            try:
+                if not self._send(batch):
+                    self._count_failure()
+            except Exception:
+                self._count_failure()
+                log.debug("Crumbtrail: delivery worker raised", exc_info=True)
+            finally:
+                self.queue.task_done()
+
+    def close(self, timeout=5):
+        """Stop accepting evidence and wait at most timeout seconds for queued delivery."""
+        if os.getpid() != self._pid:
+            return False
+        with self._lock:
+            self._closed = True
+        if self._worker:
+            self._worker.join(max(0, timeout))
+        return self.queue.unfinished_tasks == 0
+
+
+class _Disabled:
+    """Sink used when configuration is unusable. Capture stays off and nothing is sent."""
+    def __init__(self):
+        self.dropped = 0
+        self.failed = 0
+
+    def enqueue(self, batch):
+        self.dropped += 1
+
+    def close(self, timeout=5):
+        return True
+
+
+def _build_sender(endpoint, key):
+    try:
+        return Sender(endpoint, key)
+    except ValueError as reason:
+        log.warning("Crumbtrail: capture is disabled, %s", reason)
+        return _Disabled()
+
+
+class Capture:
+    def __init__(self, session, request, service, method):
+        self.session, self.request, self.service, self.method = session, request, service, method
+        self.started = now()
+        self.clock = time.monotonic()
+        self.events = []
+        self.dropped = 0
+        self.request_bytes = bytearray()
+        self.response_bytes = bytearray()
+        self.request_complete = False
+        self.request_count = 0
+        self.request_length = None
+        self.response_complete = False
+        self.status = 500
+        self.response_started = False
+        self.response_type = ""
+        self.response_count = 0
+        self.response_length = None
+        self.sequence = 0
+
+    def keep(self, target, chunk):
+        remaining = MAX_BYTES + 1 - len(target)
+        if remaining > 0:
+            target.extend(chunk[:remaining])
+
+    @staticmethod
+    def content_length(headers):
+        lengths = [v for k, v in headers if k.lower() == "content-length"]
+        if not lengths:
+            return None
+        value = lengths[0].strip()
+        return int(value) if len(lengths) == 1 and re.fullmatch(r"[0-9]{1,20}", value) else -1
+
+    def keep_request(self, chunk):
+        self.request_count += len(chunk)
+        self.keep(self.request_bytes, chunk)
+
+    def request_body(self, content_type):
+        if not is_json(content_type):
+            return None, "missing"
+        complete = self.request_complete and (self.request_length is None or self.request_count == self.request_length)
+        return capture_body(self.request_bytes, not complete)
+
+    def response_headers(self, headers):
+        self.response_type = next((v for k, v in headers if k.lower() == "content-type"), "")
+        self.response_length = self.content_length(headers)
+
+    def keep_response(self, chunk):
+        self.response_count += len(chunk)
+        self.keep(self.response_bytes, chunk)
+
+    def response_body(self):
+        no_body = self.method.upper() == "HEAD" or 100 <= self.status < 200 or self.status in (204, 205, 304) or (self.method.upper() == "CONNECT" and 200 <= self.status < 300)
+        if no_body or not is_json(self.response_type):
+            return None, "missing"
+        complete = self.response_complete and (self.response_length is None or self.response_count == self.response_length)
+        return capture_body(self.response_bytes, not complete)
+
+    def add(self, kind, data):
+        if len(self.events) < 198:
+            self.events.append({"t": now(), "k": kind, "d": data})
+        else:
+            self.dropped += 1
+
+    def finish(self, sink, request_type, route="/", error=None):
+        try:
+            correlation = {"status": "linked", "sessionIdSource": "header", "requestIdSource": "header"}
+            common = {"requestId": self.request, "sessionId": self.session, "method": self.method, "url": route, "pathname": route, "route": route, "service": self.service, "correlation": correlation}
+            body, state = self.request_body(request_type)
+            start = {"t": self.started, "k": "backend.req.start", "d": dict(common, body=body, requestBodyState=state, redaction=redaction("body", state))}
+            if error:
+                self.add("backend.req.error", dict(common, error={"name": type(error).__name__}))
+            body, state = self.response_body()
+            end = {"t": now(), "k": "backend.req.end", "d": dict(common, statusCode=self.status if error is None or self.response_started else 500, durationMs=(time.monotonic() - self.clock) * 1000, responseBody=body, responseBodyState=state, responseBodyTruncated=state == "truncated", redaction=redaction("responseBody", state))}
+            events = [start] + self.events + [end]
+            if self.dropped:
+                events.append({"t": now(), "k": "capture_gap", "d": {"kind": "capture_gap", "surface": "backend_request", "reason": "scan_budget_exceeded", "requestId": self.request, "detail": "Event limit reached", "droppedEvents": self.dropped}})
+            for i in range(0, len(events), 20):
+                sink.enqueue({"sessionId": self.session, "events": events[i:i + 20]})
+        except Exception:
+            log.debug("Crumbtrail: could not assemble request evidence", exc_info=True)
+
+
+class Client:
+    def __init__(self, *, service="python", should_capture=None, endpoint=None, key=None, sink=None):
+        self.service = service
+        self.should_capture = should_capture or (lambda path: False)
+        self.sink = sink if sink is not None else _build_sender(endpoint or os.environ.get("CRUMBTRAIL_ENDPOINT", ""), key or os.environ.get("CRUMBTRAIL_INGEST_KEY", ""))
+        self.enabled = not isinstance(self.sink, _Disabled)
+
+    def begin(self, path, method, session, request):
+        if not self.enabled:
+            return None
+        try:
+            if _ID.fullmatch(session) and _ID.fullmatch(request) and self.should_capture(path):
+                return Capture(session, request, self.service, method)
+        except Exception:
+            log.debug("Crumbtrail: capture eligibility check raised", exc_info=True)
+        return None
+
+    def close(self, timeout=5):
+        close = getattr(self.sink, "close", None)
+        return close(timeout) if close else True

crumbtrail/database.py

@@ -0,0 +1,73 @@
+"""Query metadata without SQL text, parameters, result rows or exception messages."""
+import logging
+import re
+import time
+from .core import current_capture, now
+
+log = logging.getLogger("crumbtrail")
+
+
+def _record(capture, sql, engine, start, rows=None, error=None):
+    if capture is None:
+        return
+    try:
+        # Only a leading SQL operation is retained. Arbitrary dialect literals cannot leak.
+        first = re.match(r"\s*(select|insert|update|delete)\b", sql[:64], re.I)
+        operation = first.group(1).lower() if first else "other"
+        capture.sequence += 1
+        payload = {"engine": engine, "op": operation, "table": None, "shape": "[statement omitted]", "requestId": capture.request, "t": now(), "durationMs": (time.monotonic() - start) * 1000, "seq": capture.sequence}
+        if error is None:
+            payload.update(rowCount=rows if isinstance(rows, int) and rows >= 0 else None, rowEvidence="not_captured")
+        else:
+            payload.update(code=None, category="unknown", errorName=type(error).__name__)
+        capture.add("db.error" if error else "db.statement", payload)
+    except Exception:
+        log.debug("Crumbtrail: could not record a database statement", exc_info=True)
+
+
+def instrument_sqlalchemy(engine):
+    """Register on a SQLAlchemy 2.x Engine (or AsyncEngine.sync_engine). Returns uninstall."""
+    from sqlalchemy import event
+    if getattr(engine, "_crumbtrail_uninstall", None):
+        return engine._crumbtrail_uninstall
+    engine_name = {"postgresql": "postgres", "sqlite": "sqlite", "mysql": "mysql"}.get(engine.dialect.name, "unknown")
+
+    def before(conn, cursor, statement, parameters, context, executemany):
+        context._crumbtrail = (current_capture.get(), time.monotonic())
+
+    def after(conn, cursor, statement, parameters, context, executemany):
+        capture, start = getattr(context, "_crumbtrail", (None, 0))
+        _record(capture, statement, engine_name, start, cursor.rowcount)
+
+    def error(context):
+        execution = context.execution_context
+        capture, start = getattr(execution, "_crumbtrail", (None, 0))
+        _record(capture, context.statement or "", engine_name, start, error=context.original_exception)
+
+    listeners = [("before_cursor_execute", before), ("after_cursor_execute", after), ("handle_error", error)]
+    for name, fn in listeners:
+        event.listen(engine, name, fn)
+
+    def uninstall():
+        for name, fn in listeners:
+            if event.contains(engine, name, fn):
+                event.remove(engine, name, fn)
+        if getattr(engine, "_crumbtrail_uninstall", None) is uninstall:
+            del engine._crumbtrail_uninstall
+    engine._crumbtrail_uninstall = uninstall
+    return uninstall
+
+
+def django_execute(execute, sql, params, many, context):
+    capture = current_capture.get()
+    started = time.monotonic()
+    error = None
+    try:
+        return execute(sql, params, many, context)
+    except BaseException as failure:
+        error = failure
+        raise
+    finally:
+        vendor = context["connection"].vendor
+        engine = {"postgresql": "postgres", "sqlite": "sqlite", "mysql": "mysql"}.get(vendor, "unknown")
+        _record(capture, sql, engine, started, getattr(context["cursor"], "rowcount", None), error)

crumbtrail/django.py

@@ -0,0 +1,31 @@
+"""Django synchronous WSGI integration with request scoped database wrappers."""
+from contextlib import ExitStack
+from .database import django_execute
+from .middleware import WSGIMiddleware
+
+
+def wrap_wsgi(application, client):
+    """Wrap get_wsgi_application() in wsgi.py; queries captured through response iteration."""
+    from django.db import connections
+
+    def app(environ, start_response):
+        def response():
+            with ExitStack() as stack:
+                for connection in connections.all():
+                    stack.enter_context(connection.execute_wrapper(django_execute))
+                iterable = application(environ, start_response)
+                try:
+                    yield from iterable
+                finally:
+                    close = getattr(iterable, "close", None)
+                    if close:
+                        close()
+        return response()
+
+    def route(environ):
+        from django.urls import resolve, Resolver404
+        try:
+            return "/" + resolve(environ.get("PATH_INFO", "/")).route
+        except Resolver404:
+            return "/"
+    return WSGIMiddleware(app, client, route)

crumbtrail/flask.py

@@ -0,0 +1,16 @@
+from .middleware import WSGIMiddleware
+
+
+def install(app, client):
+    """Register once, before Flask handles its first request."""
+    if "crumbtrail" in app.extensions:
+        return app.extensions["crumbtrail"]
+    from flask import request
+
+    @app.before_request
+    def route_template():
+        request.environ["crumbtrail.route"] = request.url_rule.rule if request.url_rule else "/"
+
+    app.wsgi_app = WSGIMiddleware(app.wsgi_app, client)
+    app.extensions["crumbtrail"] = client
+    return client

crumbtrail/middleware.py

@@ -0,0 +1,221 @@
+"""Transparent stream tees: capture never pre-reads or consumes application input."""
+import logging
+
+from .core import current_capture
+
+log = logging.getLogger("crumbtrail")
+
+
+class _Input:
+    def __init__(self, stream, capture, length):
+        self._stream, self._capture, self._length = stream, capture, length
+        self._count = 0
+
+    def _keep(self, data, eof=False):
+        self._capture.keep_request(data)
+        self._count += len(data)
+        if eof or (self._length is not None and self._count >= self._length):
+            self._capture.request_complete = True
+        return data
+
+    def read(self, size=-1):
+        value = self._stream.read(size)
+        return self._keep(value, size < 0 or (size != 0 and not value))
+
+    def readline(self, size=-1):
+        value = self._stream.readline(size)
+        return self._keep(value, size != 0 and not value)
+
+    def readlines(self, hint=-1):
+        lines = []
+        count = 0
+        while True:
+            line = self.readline()
+            if not line:
+                break
+            lines.append(line)
+            count += len(line)
+            if hint > 0 and count >= hint:
+                break
+        return lines
+
+    def readinto(self, buffer):
+        value = self.read(len(buffer))
+        buffer[:len(value)] = value
+        return len(value)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        line = self.readline()
+        if not line:
+            raise StopIteration
+        return line
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+
+class WSGIMiddleware:
+    def __init__(self, app, client, route=None):
+        self.app, self.client = app, client
+        self.route = route or (lambda environ: environ.get("crumbtrail.route", "/"))
+
+    def __call__(self, environ, start_response):
+        capture = self.client.begin(environ.get("PATH_INFO", "/"), environ.get("REQUEST_METHOD", "GET"), environ.get("HTTP_X_CRUMBTRAIL_SESSION_ID", ""), environ.get("HTTP_X_CRUMBTRAIL_REQUEST_ID", ""))
+        if capture is None:
+            return self.app(environ, start_response)
+        declared = environ.get("CONTENT_LENGTH")
+        capture.request_length = capture.content_length([("content-length", declared)] if declared not in (None, "") else [])
+        length = capture.request_length
+        def safe_route():
+            try:
+                value = self.route(environ)
+                return value if isinstance(value, str) and len(value) <= 2048 else "/"
+            except Exception:
+                log.debug("Crumbtrail: route resolution raised, reporting /", exc_info=True)
+                return "/"
+
+        original_input = environ["wsgi.input"]
+        environ["wsgi.input"] = _Input(original_input, capture, length)
+        capture.request_complete = length == 0
+
+        def wrapped_start(status, headers, exc_info=None):
+            write = start_response(status, headers, exc_info)
+            try:
+                capture.status = int(str(status).split(" ", 1)[0])
+                capture.response_headers(headers)
+            except Exception:
+                log.debug("Crumbtrail: could not read the response status line or headers", exc_info=True)
+
+            def wrapped_write(chunk):
+                result = write(chunk)
+                capture.response_started = True
+                try:
+                    capture.keep_response(chunk)
+                except Exception:
+                    log.debug("Crumbtrail: could not retain a written response chunk", exc_info=True)
+                return result
+            return wrapped_write
+
+        token = current_capture.set(capture)
+        try:
+            iterable = self.app(environ, wrapped_start)
+        except BaseException as error:
+            capture.finish(self.client.sink, environ.get("CONTENT_TYPE", ""), safe_route(), error)
+            environ["wsgi.input"] = original_input
+            raise
+        finally:
+            current_capture.reset(token)
+
+        middleware = self
+
+        class Response:
+            def __init__(self):
+                self.iterator = None
+                self.closed = False
+                self.error = None
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                token = current_capture.set(capture)
+                try:
+                    if self.iterator is None:
+                        self.iterator = iter(iterable)
+                    chunk = next(self.iterator)
+                    if chunk:
+                        capture.response_started = True
+                    try:
+                        capture.keep_response(chunk)
+                    except Exception:
+                        log.debug("Crumbtrail: could not retain a yielded response chunk", exc_info=True)
+                    return chunk
+                except StopIteration:
+                    capture.response_complete = True
+                    self.close()
+                    raise
+                except BaseException as error:
+                    self.error = error
+                    self.close()
+                    raise
+                finally:
+                    current_capture.reset(token)
+
+            def close(self):
+                if self.closed:
+                    return
+                self.closed = True
+                token = current_capture.set(capture)
+                try:
+                    close = getattr(iterable, "close", None)
+                    if close:
+                        close()
+                except BaseException as failure:
+                    if self.error is None:
+                        self.error = failure
+                        raise
+                    # Preserve the primary application exception when cleanup also fails.
+                finally:
+                    try:
+                        capture.finish(middleware.client.sink, environ.get("CONTENT_TYPE", ""), safe_route(), self.error)
+                    finally:
+                        environ["wsgi.input"] = original_input
+                        current_capture.reset(token)
+
+        return Response()
+
+
+class ASGIMiddleware:
+    def __init__(self, app, client):
+        self.app, self.client = app, client
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+        headers = {}
+        for key, value in scope.get("headers", []):
+            key = key.lower()
+            # Duplicate identity headers must not be treated as authoritative.
+            headers[key] = value.decode("latin1") if key not in headers else ""
+        capture = self.client.begin(scope.get("path", "/"), scope.get("method", "GET"), headers.get(b"x-crumbtrail-session-id", ""), headers.get(b"x-crumbtrail-request-id", ""))
+        if capture is None:
+            return await self.app(scope, receive, send)
+
+        capture.request_length = capture.content_length([(k.decode("latin1"), v.decode("latin1")) for k, v in scope.get("headers", [])])
+        capture.request_complete = capture.request_length == 0
+
+        async def wrapped_receive():
+            message = await receive()
+            if message["type"] == "http.request":
+                capture.keep_request(message.get("body", b""))
+                if not message.get("more_body", False):
+                    capture.request_complete = True
+            return message
+
+        async def wrapped_send(message):
+            await send(message)
+            if message["type"] == "http.response.start":
+                capture.status = message["status"]
+                capture.response_started = True
+                capture.response_headers([(k.decode("latin1"), v.decode("latin1")) for k, v in message.get("headers", [])])
+            elif message["type"] == "http.response.body":
+                capture.keep_response(message.get("body", b""))
+                if not message.get("more_body", False):
+                    capture.response_complete = True
+
+        token = current_capture.set(capture)
+        error = None
+        try:
+            return await self.app(scope, wrapped_receive, wrapped_send)
+        except BaseException as failure:
+            error = failure
+            raise
+        finally:
+            try:
+                route = getattr(scope.get("route"), "path", "/")
+                capture.finish(self.client.sink, headers.get(b"content-type", ""), route, error)
+            finally:
+                current_capture.reset(token)

crumbtrail/privacy.py

@@ -0,0 +1,113 @@
+"""Bounded conservative backend structured profile; never captures free text."""
+import functools
+import json
+import logging
+import math
+import re
+
+log = logging.getLogger("crumbtrail")
+
+MAX_BYTES = 16384
+POLICY = "crumbtrail.backend-redaction.v1"
+REDACTED = "[REDACTED]"
+MAX_KEYS = 64
+MAX_ARRAY = 40
+MAX_DEPTH = 8
+# Substring match against the key with separators stripped, so only terms that cannot
+# appear inside an unrelated word belong here. Short location terms are matched as whole
+# words by _WORD instead, otherwise "lat" would redact "latency" and "platform".
+_DENIED = re.compile(r"password|passwd|passphrase|passcode|secret|token|auth|card|cvv|cvc|ssn|email|phone|address|iban|account|birth|credential|creds|cookie|session|privatekey|apikey|accesskey|securitycode|verificationcode|connection|routingnumber|taxid|nationalid|sortcode|name|postal|payload|beforejson|afterjson|latitude|longitude|geolocation|coordinate", re.I)
+_WORD = re.compile(r"^(pwd|pin|pan|otp|pass|sid|dob|zip|jwt|mfa|csrf|xsrf|lat|lon|lng|geo|coord|coords)[0-9]*$", re.I)
+_KEY = re.compile(r"[a-zA-Z_][a-zA-Z0-9_.-]{0,63}")
+_CAMEL = re.compile(r"([a-z0-9])([A-Z])")
+_STRIP = re.compile(r"[^a-zA-Z0-9]")
+_SPLIT = re.compile(r"[^a-zA-Z0-9]+")
+_ENUM = re.compile(r"(?:[a-z][a-z_]{0,22}|[A-Z]{3}|[0-9]{1,12})")
+_DIGITS = re.compile(r"[0-9]{13,19}")
+
+
+class _Unsupported(ValueError):
+    """Well formed JSON whose shape the conservative profile will not export."""
+
+
+@functools.lru_cache(maxsize=4096)
+def _sensitive(key):
+    words = _CAMEL.sub(r"\1 \2", key)
+    return bool(_DENIED.search(_STRIP.sub("", key))) or any(_WORD.fullmatch(w) for w in _SPLIT.split(words))
+
+
+def _card(number):
+    digits = str(int(number)) if isinstance(number, float) and number.is_integer() else str(number)
+    if not _DIGITS.fullmatch(digits):
+        return False
+    total = 0
+    for i, ch in enumerate(reversed(digits)):
+        n = int(ch) * (2 if i % 2 else 1)
+        total += n - 9 if n > 9 else n
+    return total % 10 == 0
+
+
+def _object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result or len(result) >= MAX_KEYS or not _KEY.fullmatch(key):
+            raise _Unsupported("ambiguous or unsupported object")
+        result[key] = value
+    return result
+
+
+def capture_body(raw, truncated=False):
+    if truncated or len(raw) > MAX_BYTES:
+        return None, "truncated"
+    if not raw:
+        return None, "missing"
+    removed = False
+
+    def walk(value, key="", depth=0, check_key=True):
+        nonlocal removed
+        if depth > MAX_DEPTH:
+            raise _Unsupported("depth")
+        if check_key and _sensitive(key):
+            removed = True
+            return REDACTED
+        if isinstance(value, dict):
+            return {k: walk(v, k, depth + 1) for k, v in value.items()}
+        if isinstance(value, list):
+            if len(value) > MAX_ARRAY:
+                raise _Unsupported("array size")
+            # The key is constant across elements and was cleared above, so skip it.
+            return [walk(v, key, depth + 1, False) for v in value]
+        if value is None or isinstance(value, bool):
+            return value
+        if isinstance(value, (float, int)):
+            if math.isfinite(value) and abs(value) <= 9007199254740991 and not _card(value):
+                return value
+        elif isinstance(value, str) and _ENUM.fullmatch(value):
+            return value
+        removed = True
+        return REDACTED
+
+    try:
+        value = json.loads(raw.decode("utf-8"), object_pairs_hook=_object, parse_constant=lambda _: (_ for _ in ()).throw(ValueError("number")))
+        body = json.dumps(walk(value), separators=(",", ":"), ensure_ascii=True, allow_nan=False)
+        if len(body.encode()) > MAX_BYTES:
+            return None, "truncated"
+        return body, "redacted" if removed else "captured"
+    except _Unsupported as reason:
+        # The bundle consumer accepts only captured, redacted, missing, invalid and
+        # truncated, so an unsupported shape has to report "invalid". The log carries
+        # the distinction a new state value would have carried.
+        log.debug("Crumbtrail: body not exported, unsupported shape (%s)", reason)
+        return None, "invalid"
+    except (ValueError, RecursionError, OverflowError) as reason:
+        log.debug("Crumbtrail: body not exported, invalid JSON (%s)", reason)
+        return None, "invalid"
+
+
+def redaction(field, state):
+    return {"policy": POLICY, "fields": [{"path": field, "reason": "backend_structured_profile", "action": "redacted"}] if state == "redacted" else []}
+
+
+def is_json(content_type):
+    media = content_type.split(";", 1)[0].strip().lower()
+    return media == "application/json" or (media.startswith("application/") and media.endswith("+json"))

crumbtrail_python-0.1.0.dist-info/METADATA

@@ -0,0 +1,124 @@
+Metadata-Version: 2.4
+Name: crumbtrail-python
+Version: 0.1.0
+Summary: Maintained Crumbtrail request and database capture for Python services
+License-Expression: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Requires-Dist: Flask<4,>=3; extra == "test"
+Requires-Dist: fastapi<1,>=0.115; extra == "test"
+Requires-Dist: httpx<1,>=0.27; extra == "test"
+Requires-Dist: Django<5,>=4.2; extra == "test"
+Requires-Dist: SQLAlchemy<3,>=2; extra == "test"
+Requires-Dist: build<2,>=1; extra == "test"
+Dynamic: license-file
+
+# Capture Python backend evidence
+
+Requires Python 3.9 or later. Package version 0.1.0 is source available and pending publication. Build and install the wheel locally until a release is available:
+
+```sh
+python -m pip install build
+python -m build packages/python
+python -m pip install packages/python/dist/crumbtrail_python-0.1.0-py3-none-any.whl
+```
+
+Configure `CRUMBTRAIL_ENDPOINT` with the HTTPS Crumbtrail origin and `CRUMBTRAIL_INGEST_KEY` with the project's backend ingest key. A loopback origin such as `http://localhost:19890` may use plain HTTP so the package can point at a local stack. Every other origin must be HTTPS.
+
+When the configuration is missing or unusable the client does not raise. It logs one warning on the `crumbtrail` logger, disables capture, and returns the application unchanged, so a client built at module scope cannot crash the host at import.
+
+A client may be built before the server forks its workers. The package rebuilds its queue, its worker thread and its locks in each child, so `gunicorn --preload`, uwsgi with a master process, and a client created at module scope in `wsgi.py` all capture normally. Evidence produced in a process the sender was never rebuilt for is dropped, and the first such drop logs a warning.
+
+Capture is disabled for every route unless `should_capture` explicitly selects it. Requests must include valid `x-crumbtrail-session-id` and `x-crumbtrail-request-id` headers from an existing browser session.
+
+```python
+from crumbtrail import Client
+
+client = Client(
+    service="orders-api",
+    should_capture=lambda path: path.startswith("/api/") and not path.startswith("/api/auth"),
+)
+```
+
+## Flask
+
+Register before serving requests:
+
+```python
+from flask import Flask
+from crumbtrail.flask import install
+
+app = Flask(__name__)
+install(app, client)
+```
+
+## FastAPI and other ASGI applications
+
+```python
+from fastapi import FastAPI
+from crumbtrail import ASGIMiddleware
+
+app = FastAPI()
+app.add_middleware(ASGIMiddleware, client=client)
+```
+
+Non HTTP scopes pass through. FastAPI route templates are retained. Generic ASGI integrations without a route object report `/` to avoid exporting path parameters. SQLAlchemy instrumentation also works with FastAPI synchronous handlers because context is propagated to its thread pool.
+
+## Django WSGI
+
+In `wsgi.py`, wrap the application returned by Django:
+
+```python
+from django.core.wsgi import get_wsgi_application
+from crumbtrail.django import wrap_wsgi
+
+application = wrap_wsgi(get_wsgi_application(), client)
+```
+
+The wrapper captures queries on configured Django database connections throughout response iteration. This Django database adapter supports synchronous WSGI. For Django ASGI, the generic `ASGIMiddleware` provides request evidence but does not instrument Django's separate synchronous database threads.
+
+## SQLAlchemy
+
+Register once on an engine. For an asynchronous engine, pass its `sync_engine`:
+
+```python
+from sqlalchemy import create_engine
+from crumbtrail.database import instrument_sqlalchemy
+
+engine = create_engine("sqlite://")
+uninstall = instrument_sqlalchemy(engine)
+```
+
+The maintained adapter records query operation, duration, row count when available, and error type. It never captures SQL text, parameters, row values, connection strings or exception messages. `shape` is explicitly `[statement omitted]`, and `rowEvidence` is `not_captured`. Row diffs and transaction state capture are not implemented. Call `uninstall()` when disposing instrumentation.
+
+## Verify and shut down
+
+Exercise an eligible JSON route with browser correlation headers. The existing session should contain `backend.req.start`, `backend.req.end` and any instrumented `db.statement` or `db.error` events. This package appends to existing sessions. It does not create browser sessions.
+
+Call `client.close(timeout=5)` from the server's worker shutdown hook. It stops accepting evidence and waits up to five seconds for queued delivery. It returns `False` if work remains. ASGI lifecycle scopes pass through, so register your shutdown hook explicitly. A hook registered with `atexit` flushes queued evidence for up to two seconds at interpreter exit, which covers an ordinary shutdown but not a signal that kills the process outright.
+
+The background worker uses a queue of 64 batches, each containing at most 20 events. A request retains at most 198 intermediate events and emits a capture gap when that limit is exceeded. Delivery retries 429, server failures and network failures up to four attempts with five second request timeouts, and honours a `Retry-After` header on 429 up to 30 seconds. A 404 means the session does not belong to this ingest key, which no retry changes, so it is not retried. Redirects are disabled. Other rejection statuses are not retried. `client.sink.dropped` and `client.sink.failed` report overflow or failed delivery counts when using the default sender. Capture failures do not change application response bytes or exceptions, including when an application breaks the WSGI contract by passing a malformed status line or yielding `str` instead of `bytes`. A failure after response transmission begins records `backend.req.error` and retains the emitted HTTP status.
+
+JSON request and response bodies retain at most 16 KiB each. The conservative profile exports numbers, booleans, null, short lowercase enums and three letter uppercase units. It also exports object key names verbatim. Read that plainly: a numeric value under a key the profile does not recognise as sensitive leaves your process, so `{"salary":185000}` is exported in full, and so is every key name in the body. Values under keys matching the sensitive list, including location keys such as `lat`, `lng` and `latitude`, are replaced with `[REDACTED]`, as is every other string.
+
+Object keys must match `[a-zA-Z_][a-zA-Z0-9_.-]{0,63}`, so hyphens and dots are accepted. Duplicate keys, keys outside that pattern, more than 64 keys in one object, depth beyond 8 and arrays longer than 40 elements are not exported. Bodies have explicit `captured`, `redacted`, `missing`, `invalid` or `truncated` states. A body the profile will not export reports `invalid` whether the JSON was malformed or merely an unsupported shape, because the analysis pipeline accepts no other state. The `crumbtrail` logger records which of the two it was at debug level. Bodies the application does not fully read are marked truncated. Request capture never pre reads a stream. Response streaming remains incremental. A JSON request or response whose observed byte count differs from its declared `Content-Length` is marked truncated even if its stream ends normally. Malformed length declarations also prevent claiming complete body evidence. HEAD responses and statuses that do not carry a response body report missing body evidence. Query strings, headers, raw path parameters and exception messages are not exported.
+
+## Diagnostics
+
+The package logs on the `crumbtrail` logger and never raises into the host application. Failures inside capture are logged at debug. Capture being disabled, a batch dropped after every delivery attempt, an ingest rejection, and evidence produced in a process the sender was not rebuilt for are logged at warning.
+
+```python
+import logging
+
+logging.getLogger("crumbtrail").setLevel(logging.DEBUG)
+```
+
+## Run package tests
+
+```sh
+python -m pip install -e 'packages/python[test]'
+python -m pytest packages/python/tests
+```

crumbtrail_python-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+crumbtrail/__init__.py,sha256=KrsoZgoScbG4WYJMQ3f85j9EJ1HKTHEbsa-h9WeAKug,156
+crumbtrail/core.py,sha256=fJKGUdEpWrmogq3S1hh8u3sMbFzcbSkXQE9Xh6irpdI,14634
+crumbtrail/database.py,sha256=QxqhTfa5cwk2qh_w5blovlnUUFrP6ZQXuH6T6EFM3c0,3278
+crumbtrail/django.py,sha256=zhRmaHpa2qS4cIAuzjgBWJDxghdM5OyBJeaerbU4L68,1157
+crumbtrail/flask.py,sha256=_ZcQ6E-S8TkZK23nOV8YKVZ2fX6l4JBa-dKchnJg4Yk,510
+crumbtrail/middleware.py,sha256=TlJyH92LH87Q8CbayXrlWU_fw30pisZFJ2T4if_WxPM,8520
+crumbtrail/privacy.py,sha256=39p0gbeup40cgUV_GKdyAwui2-PzBCL2tAxeVl2TT8E,4775
+crumbtrail_python-0.1.0.dist-info/licenses/LICENSE,sha256=TtFbQp-yLe609hewRKljsptCHZmuFS_qZpGHBWF-_50,1067
+crumbtrail_python-0.1.0.dist-info/METADATA,sha256=nYjmngIMkOkCG3-hyS46WQKC3r2WGUZ7lBHJbxOm4Jo,8075
+crumbtrail_python-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+crumbtrail_python-0.1.0.dist-info/top_level.txt,sha256=83C0OZLxG0IgSg-lsi8NXT0-1rTTsPHALWznbca0suA,11
+crumbtrail_python-0.1.0.dist-info/RECORD,,

crumbtrail_python-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

crumbtrail_python-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Crumbtrail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crumbtrail_python-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+crumbtrail