restless-sdk 0.1.0__py3-none-any.whl

restless/__init__.py

@@ -0,0 +1,161 @@
+"""Restless Python SDK.
+
+Capture your API traffic and send it to Restless.
+
+    import restless
+
+    client = restless.Restless(os.environ["RESTLESS_KEY"])
+
+    @client.setup
+    def _(request):
+        return {
+            "api_key": client.mask(request.headers.get("authorization")),
+            "owner": {
+                "id": current_workspace_id(),
+                "enrich": lambda owner_id: {"label": lookup_name(owner_id)},
+            },
+        }
+
+    app.wsgi_app = client.wsgi(app.wsgi_app)
+
+Conformance: implements spec version 1.0.0 of the Restless SDK Contract at
+level L2. Run the shared harness against ``python -m restless._conformance``
+to verify. See CONFORMANCE.md.
+"""
+
+import os
+import sys
+from typing import Any, Callable, Dict, List, Optional
+
+from ._capture import CaptureEngine
+from ._mask import mask
+from ._request import Headers, RequestInfo
+from ._request_id import (
+    format_request_id,
+    is_valid_request_id,
+    new_request_id,
+    strip_request_id_prefix,
+)
+from ._settings import load_settings, resolve_api
+from ._uploader import is_test_run, resolve_base_url
+from ._version import CONFORMANCE_LEVEL, SDK_NAME, SDK_VERSION, SPEC_VERSION
+
+__all__ = [
+    "Restless",
+    "restless",
+    "mask",
+    "RequestInfo",
+    "Headers",
+    "new_request_id",
+    "format_request_id",
+    "strip_request_id_prefix",
+    "is_valid_request_id",
+    "SDK_NAME",
+    "SDK_VERSION",
+    "SPEC_VERSION",
+    "CONFORMANCE_LEVEL",
+]
+
+__version__ = SDK_VERSION
+
+
+class Restless:
+    """A Restless client.
+
+    Naming here is idiomatic Python rather than a transliteration of the Node
+    SDK. CONTRACT.md section 14 makes the public surface explicitly
+    per-language: what is normative is the SEMANTIC surface (construct with
+    an explicit key, register a per-request callback, mask a key, flush), not
+    the method names.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api: Optional[str] = None,
+        redact: Optional[Dict[str, List[str]]] = None,
+        base_url: Optional[str] = None,
+        transport: Optional[Callable] = None,
+    ):
+        # CONFIG-001. Explicit key wins, then RESTLESS_KEY, then README_API_KEY.
+        resolved = (
+            api_key
+            or os.environ.get("RESTLESS_KEY")
+            or os.environ.get("README_API_KEY")
+            or ""
+        )
+        # CONFIG-002. Capture still runs; only upload is disabled.
+        if not resolved and not is_test_run():
+            print(
+                "[restless] no API key found - set RESTLESS_KEY in your environment "
+                "or pass it to Restless(). Captured requests will not be uploaded.",
+                file=sys.stderr,
+            )
+
+        request_id_prefix = None
+        settings_redact = None
+        entry = resolve_api(load_settings(), api)  # CONFIG-013/014 may raise
+        if entry:
+            request_id_prefix = entry.get("requestIdPrefix")
+            settings_redact = entry.get("redact")
+
+        # REDACT-015. Both sources are additive on top of the defaults.
+        merged: Dict[str, List[str]] = {}
+        for field in ("headers", "bodyKeys", "queryParams"):
+            merged[field] = list((settings_redact or {}).get(field) or []) + list(
+                (redact or {}).get(field) or []
+            )
+
+        self.engine = CaptureEngine(
+            api_key=resolved,
+            base_url=resolve_base_url(base_url),
+            request_id_prefix=request_id_prefix,
+            redact=merged,
+            transport=transport,
+        )
+
+    # ------------------------------------------------------------ public API
+
+    @staticmethod
+    def mask(api_key: Optional[str]) -> Optional[str]:
+        """Mask an end-user API key (MASK-001).
+
+        Pass the raw header value straight through. Do NOT substitute a
+        placeholder like ``"anonymous"``: its last 4 characters would become
+        the tail and cluster unrelated callers together.
+        """
+        return mask(api_key)
+
+    def setup(self, callback: Callable) -> Callable:
+        """Register the per-request callback. Usable as a decorator.
+
+        The callback receives the framework-native request object and returns
+        a dict: ``api_key`` (masked), ``owner`` (``id`` plus ``enrich``), and
+        optionally ``block``.
+        """
+        self.engine.set_callback(callback)
+        return callback
+
+    def flush(self) -> None:
+        """Force-upload the queued batch. Call before process exit if you
+        care about in-flight captures."""
+        self.engine.flush()
+
+    # ------------------------------------------------------------- adapters
+
+    def wsgi(self, app: Callable) -> Callable:
+        """Wrap a WSGI app (Flask, Django, Pyramid, Bottle)."""
+        from .adapters.wsgi import wrap_wsgi
+
+        return wrap_wsgi(app, self.engine)
+
+    def asgi(self, app: Callable) -> Callable:
+        """Wrap an ASGI app (FastAPI, Starlette, Quart)."""
+        from .adapters.asgi import wrap_asgi
+
+        return wrap_asgi(app, self.engine)
+
+
+def restless(api_key: Optional[str] = None, **kwargs: Any) -> Restless:
+    """Functional constructor, mirroring the Node SDK's ``restless(key)``."""
+    return Restless(api_key, **kwargs)

restless/_caches.py

@@ -0,0 +1,128 @@
+"""In-process caches.
+
+Implements CONTRACT.md section 11.
+"""
+
+import time
+from typing import Any, Dict, Optional
+
+# CACHE-004, CACHE-014
+_ENRICH_TTL_MS = 60 * 60 * 1000
+_RECOVERY_TTL_MS = 60 * 60 * 1000
+_RECOVERY_NEGATIVE_TTL_MS = 5 * 60 * 1000
+
+
+def _now_ms() -> float:
+    return time.monotonic() * 1000.0
+
+
+class EnrichCache:
+    """CACHE-001..007.
+
+    Stores the enriched VALUE, not merely a freshness flag (CACHE-003). Every
+    upload has to carry owner metadata, including uploads that skipped the
+    callback, because the ingest cannot backfill it: without the value, every
+    request after the first would land in the dashboard as unauthenticated.
+    """
+
+    def __init__(self, ttl_ms: float = _ENRICH_TTL_MS):
+        self._cache: Dict[str, Any] = {}
+        self._ttl_ms = ttl_ms
+
+    def get(self, key: str) -> Optional[Dict[str, Any]]:
+        entry = self._cache.get(key)
+        if entry is None:
+            return None
+        value, ts = entry
+        if _now_ms() - ts > self._ttl_ms:
+            self._cache.pop(key, None)
+            return None
+        return value
+
+    def set(self, key: str, value: Dict[str, Any]) -> None:
+        self._cache[key] = (value, _now_ms())
+
+    def invalidate(self, key: str) -> None:
+        """CACHE-006. Next request from this key re-runs enrich."""
+        self._cache.pop(key, None)
+
+    def clear(self) -> None:
+        self._cache.clear()
+
+    def size(self) -> int:
+        return len(self._cache)
+
+
+class RecoveryCache:
+    """CACHE-010..015. Fingerprint key to recovery message.
+
+    Every read is synchronous and in-process. The lookup sits on the hot path
+    of every 4xx/5xx, so it never performs I/O and never blocks the response;
+    a cold miss simply injects nothing.
+
+    "No message for this fingerprint" is itself cached (as ``None``) so a cold
+    miss does not stay a cold miss on every subsequent request. The negative
+    TTL is shorter so a freshly-attached dashboard message starts working
+    within minutes.
+    """
+
+    def __init__(
+        self,
+        ttl_ms: float = _RECOVERY_TTL_MS,
+        negative_ttl_ms: float = _RECOVERY_NEGATIVE_TTL_MS,
+    ):
+        self._cache: Dict[str, Any] = {}
+        self._ttl_ms = ttl_ms
+        self._negative_ttl_ms = negative_ttl_ms
+
+    _MISS = object()
+
+    def get(self, key: str) -> Any:
+        """Returns the message, ``None`` for a confirmed absence, or ``_MISS``.
+
+        The three-way result matters: ``None`` means the server told us there
+        is no message (do not ask again yet), while ``_MISS`` means we have
+        never asked.
+        """
+        entry = self._cache.get(key)
+        if entry is None:
+            return self._MISS
+        message, ts = entry
+        ttl = self._negative_ttl_ms if message is None else self._ttl_ms
+        if _now_ms() - ts > ttl:
+            self._cache.pop(key, None)
+            return self._MISS
+        return message
+
+    def set(self, key: str, message: Optional[str]) -> None:
+        self._cache[key] = (message, _now_ms())
+
+    def lookup(self, key: str) -> Optional[str]:
+        """CACHE-010. The hot-path read: a message to inject, or None."""
+        value = self.get(key)
+        return value if isinstance(value, str) else None
+
+    def invalidate(self, key: str) -> None:
+        self._cache.pop(key, None)
+
+    def clear(self) -> None:
+        self._cache.clear()
+
+    def size(self) -> int:
+        return len(self._cache)
+
+
+class Blocklist:
+    """Masked keys to reject. O(1) lookup, per-process state."""
+
+    def __init__(self) -> None:
+        self._blocked = set()
+
+    def replace(self, masked_keys) -> None:
+        self._blocked = set(masked_keys)
+
+    def has(self, masked_key: Optional[str]) -> bool:
+        return bool(masked_key) and masked_key in self._blocked
+
+    def size(self) -> int:
+        return len(self._blocked)

restless/_capture.py

@@ -0,0 +1,274 @@
+"""The capture engine: the single choke point every adapter goes through.
+
+Implements CONTRACT.md sections 10, 11 and 13.
+"""
+
+import datetime
+import inspect
+from typing import Any, Callable, Dict, List, Optional
+
+from ._caches import Blocklist, EnrichCache, RecoveryCache
+from ._fingerprint import Fingerprint, fingerprint
+from ._injection import apply_internal_body_mods, build_debug_injection
+from ._redact import redact_body, redact_headers, redact_url, truncate_body
+from ._uploader import Uploader, _debug
+
+#: REDACT-030. 256 KiB, in UTF-8 bytes.
+MAX_BODY_BYTES = 256 * 1024
+
+
+def now_iso() -> str:
+    """PRIM-040. Exactly three fractional digits and a literal Z.
+
+    NOT ``datetime.isoformat()``, which emits microseconds and ``+00:00``.
+    The ingest parses this permissively and silently falls back to server
+    time when it cannot, so a wrong format loses real request timing with no
+    error anywhere in the system.
+    """
+    now = datetime.datetime.now(datetime.timezone.utc)
+    return "{}.{:03d}Z".format(now.strftime("%Y-%m-%dT%H:%M:%S"), now.microsecond // 1000)
+
+
+class CaptureEngine:
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str,
+        request_id_prefix: Optional[str] = None,
+        redact: Optional[Dict[str, List[str]]] = None,
+        transport: Optional[Callable] = None,
+    ):
+        self.blocklist = Blocklist()
+        self.enrich_cache = EnrichCache()
+        self.recovery_cache = RecoveryCache()
+        self._redact = redact or {}
+        self._callback: Optional[Callable] = None
+        self._docs_url: Optional[str] = None
+        self.uploader = Uploader(
+            api_key=api_key,
+            base_url=base_url,
+            request_id_prefix=request_id_prefix,
+            on_response=self._handle_server_response,
+            transport=transport,
+        )
+
+    # ---------------------------------------------------------------- setup
+
+    def set_callback(self, callback: Callable) -> None:
+        self._callback = callback
+
+    @property
+    def docs_url(self) -> Optional[str]:
+        """INJECT-006. Server-learned origin for injected log links."""
+        return self._docs_url
+
+    @property
+    def request_id_prefix(self) -> Optional[str]:
+        return self.uploader.request_id_prefix
+
+    @property
+    def base_url(self) -> str:
+        return self.uploader.base_url
+
+    @property
+    def has_api_key(self) -> bool:
+        return self.uploader.has_api_key
+
+    # ------------------------------------------------------- server signals
+
+    def _handle_server_response(self, body: Any, batch_fingerprints: List[str]) -> None:
+        """WIRE-021..023, CACHE-006, CACHE-012, CACHE-013."""
+        if not isinstance(body, dict):
+            return
+
+        needs = body.get("needsEnrichment")
+        if isinstance(needs, list):
+            for key in needs:
+                if isinstance(key, str):
+                    self.enrich_cache.invalidate(key)
+
+        docs = body.get("docsUrl")
+        if isinstance(docs, str) and docs:
+            self._docs_url = docs.rstrip("/")
+
+        messages = body.get("recoveryMessages")
+        if not isinstance(messages, dict):
+            messages = {}
+        for key in batch_fingerprints:
+            value = messages.get(key, ...)
+            if isinstance(value, str):
+                self.recovery_cache.set(key, value)
+            elif value is None and key in messages:
+                self.recovery_cache.set(key, None)
+            elif self.recovery_cache.get(key) is RecoveryCache._MISS:
+                # CACHE-012 with CACHE-013: negative-cache what the server
+                # did not answer, but never clobber a positive entry.
+                self.recovery_cache.set(key, None)
+
+    def lookup_recovery(self, fingerprint_key: str) -> Optional[str]:
+        """CACHE-010. Synchronous, in-process, never touches the network."""
+        return self.recovery_cache.lookup(fingerprint_key)
+
+    # ----------------------------------------------------------- per-request
+
+    def compute_fingerprint(self, captured: Dict[str, Any]) -> Optional[Fingerprint]:
+        """FP-002. Errors only."""
+        response = captured["response"]
+        if response["status"] < 400:
+            return None
+        parsed: Any = response.get("body")
+        if isinstance(parsed, str):
+            try:
+                import json
+
+                parsed = json.loads(parsed)
+            except ValueError:
+                pass  # fingerprint() handles a raw string too
+        return fingerprint(
+            status=response["status"],
+            method=captured["request"]["method"],
+            route=captured.get("routePattern"),
+            response_headers=response.get("headers"),
+            response_body=parsed,
+            stack_trace=captured.get("stackTrace"),
+        )
+
+    def resolve(self, request: Any) -> Dict[str, Any]:
+        """SETUP-001..005, CACHE-001..007, SAFETY-002."""
+        if self._callback is None:
+            return {}
+        try:
+            result = self._callback(request) or {}
+        except Exception:  # noqa: BLE001 - SAFETY-002
+            return {}
+        if not isinstance(result, dict):
+            return {}
+
+        rest = {k: v for k, v in result.items() if k not in ("owner", "project")}
+        raw = result.get("owner") or result.get("project")
+        if not raw:
+            return rest
+
+        # SETUP-003. `id` is the only field carried off the raw owner; all
+        # other metadata flows through enrich.
+        owner_id = raw.get("id")
+        enrich = raw.get("enrich")
+        base = {"id": owner_id} if owner_id is not None else {}
+        cache_key = owner_id or rest.get("apiKey")
+
+        if callable(enrich) and owner_id and cache_key:
+            cached = self.enrich_cache.get(cache_key)
+            if cached is not None:
+                merged = dict(base)
+                merged.update(cached)
+                return dict(rest, project=merged)
+            try:
+                enriched = enrich(owner_id)
+                if inspect.isawaitable(enriched):  # pragma: no cover
+                    enriched = None  # async enrich is resolved by the ASGI adapter
+                if isinstance(enriched, dict):
+                    self.enrich_cache.set(cache_key, enriched)
+                    merged = dict(base)
+                    merged.update(enriched)
+                    return dict(rest, project=merged)
+            except Exception:  # noqa: BLE001 - CACHE-005, SAFETY-003
+                pass  # failures are NOT cached; the next request retries
+
+        # CACHE-007. Ship the bare id so the dashboard can still group by it.
+        return dict(rest, project=base)
+
+    def record(self, captured: Dict[str, Any]) -> None:
+        """The single choke point. Redact, truncate, fingerprint, enqueue."""
+        request = captured["request"]
+        response = captured["response"]
+        redact_opts = self._redact
+
+        sanitized = dict(captured)
+        sanitized["request"] = dict(
+            request,
+            url=redact_url(request["url"], redact_opts.get("queryParams")),
+            headers=redact_headers(request.get("headers") or {}, redact_opts.get("headers")),
+            # REDACT-033: truncation runs AFTER redaction, so a secret cannot
+            # survive by sitting past the byte limit.
+            body=truncate_body(
+                redact_body(
+                    request.get("body"),
+                    (request.get("headers") or {}).get("content-type"),
+                    redact_opts.get("bodyKeys"),
+                ),
+                MAX_BODY_BYTES,
+            ),
+        )
+        sanitized["response"] = dict(
+            response,
+            headers=redact_headers(response.get("headers") or {}, redact_opts.get("headers")),
+            body=truncate_body(
+                redact_body(
+                    response.get("body"),
+                    (response.get("headers") or {}).get("content-type"),
+                    redact_opts.get("bodyKeys"),
+                ),
+                MAX_BODY_BYTES,
+            ),
+        )
+
+        # INJECT-010. Reuse the fingerprint the adapter already computed for
+        # its recovery lookup rather than repeating the work.
+        if not sanitized.get("errorFingerprint") and response["status"] >= 400:
+            fp = self.compute_fingerprint(sanitized)
+            if fp is not None:
+                sanitized["errorFingerprint"] = fp.to_wire()
+        sanitized.pop("stackTrace", None)
+
+        try:
+            self.uploader.push(sanitized)
+        except Exception as err:  # noqa: BLE001 - SAFETY-001
+            _debug("record failed:", err)
+
+    def flush(self) -> None:
+        self.uploader.flush()
+
+    # ------------------------------------------------------------ injection
+
+    def build_injection(
+        self,
+        status: int,
+        request_id: str,
+        method: Optional[str],
+        route: Optional[str],
+        raw_body: Optional[str],
+        content_type: Optional[str],
+        response_headers: Dict[str, str],
+    ):
+        """INJECT-001..009. Returns (headers, new_body, fingerprint_wire).
+
+        The fingerprint is computed against the customer's RAW response,
+        before any injected header or body field is layered on (INJECT-009).
+        """
+        if status < 400:
+            return {}, raw_body, None
+
+        fp = self.compute_fingerprint(
+            {
+                "request": {"method": method or "GET", "url": "", "headers": {}},
+                "response": {
+                    "status": status,
+                    "headers": response_headers,
+                    "body": raw_body,
+                },
+                "routePattern": route,
+            }
+        )
+        recovery = self.lookup_recovery(fp.key) if fp else None
+        headers, mutate = build_debug_injection(
+            status=status,
+            request_id=request_id,
+            base_url=self.base_url,
+            prefix=self.request_id_prefix,
+            recovery=recovery,
+            method=method,
+            path=route,
+            docs_url=self.docs_url,
+        )
+        new_body = apply_internal_body_mods(raw_body, content_type, mutate)
+        return headers, new_body, (fp.to_wire() if fp else None)

restless/_conformance/__main__.py

@@ -0,0 +1,125 @@
+"""Python conformance driver. See node-sdk/spec/driver/PROTOCOL.md.
+
+Dev-only: not part of the public API, not imported by customer code.
+
+    python -m restless._conformance
+
+Reads JSON Lines on stdin, writes JSON Lines on stdout. The body is a thin
+shell over the real modules in ``restless``; anything it reimplemented would
+be testing itself instead of the SDK.
+"""
+
+import json
+import re
+import sys
+from typing import Any, Dict
+
+from .._fingerprint import (
+    fingerprint,
+    normalize_message,
+    normalize_route,
+    project_relative,
+)
+from .._har import to_har_entry
+from .._injection import recovery_slug
+from .._mask import mask
+from .._redact import (
+    redact_body,
+    redact_headers,
+    redact_url,
+    redact_value,
+    truncate_body,
+)
+from .._request_id import (
+    format_request_id,
+    request_id_response_headers,
+    strip_request_id_prefix,
+)
+
+# FP-044. The stack strategy's frame format is language-specific: only the
+# OUTPUT shape is contract surface. The shared vectors carry v8-shaped
+# stacks, which a Python traceback parser cannot read, so we report those
+# cases as an unsupported dialect. The harness records them as SKIPPED
+# rather than failed, and the SDK covers Python tracebacks in its own
+# tests (tests/test_stack_frames.py).
+_V8_FRAME_RE = re.compile(r"^\s*at\s+\S", re.MULTILINE)
+
+
+class UnsupportedDialect(Exception):
+    pass
+
+
+def _op_fingerprint(i: Dict[str, Any]) -> Dict[str, str]:
+    stack = i.get("stackTrace")
+    if stack:
+        text = stack if isinstance(stack, str) else "\n".join(str(s) for s in stack)
+        if _V8_FRAME_RE.search(text):
+            raise UnsupportedDialect(
+                "unsupported stack dialect: v8 (this SDK parses Python tracebacks; FP-044)"
+            )
+    fp = fingerprint(
+        status=i["status"],
+        method=i.get("method"),
+        route=i.get("route"),
+        response_headers=i.get("responseHeaders"),
+        response_body=i.get("responseBody"),
+        stack_trace=stack,
+    )
+    # FP-003: `reason` is prose, not contract surface.
+    return {"strategy": fp.strategy, "key": fp.key}
+
+
+OPS = {
+    "mask": lambda i: mask(i.get("apiKey")),
+    "redactValue": lambda i: redact_value(i["value"]),
+    "redactHeaders": lambda i: redact_headers(i["headers"], i.get("extra")),
+    "redactUrl": lambda i: redact_url(i["url"], i.get("extra")),
+    "redactBody": lambda i: redact_body(
+        i.get("body"), i.get("contentType"), i.get("extra")
+    ),
+    "truncateBody": lambda i: truncate_body(i.get("body"), i["maxBytes"]),
+    "fingerprint": _op_fingerprint,
+    "normalizeRoute": lambda i: normalize_route(i.get("route")),
+    "normalizeMessage": lambda i: normalize_message(i.get("message")),
+    "projectRelative": lambda i: project_relative(i["file"]),
+    "formatRequestId": lambda i: format_request_id(i["rawId"], i.get("prefix")),
+    "stripRequestIdPrefix": lambda i: strip_request_id_prefix(i["requestId"]),
+    "requestIdHeaders": lambda i: request_id_response_headers(
+        i["ourId"], i.get("incomingHeaders") or {}, i.get("prefix"), i["hasApiKey"]
+    ),
+    "recoverySlug": lambda i: recovery_slug(i.get("method"), i.get("path")),
+    "harEntry": lambda i: to_har_entry(i["captured"]),
+}
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        req_id = None
+        try:
+            req = json.loads(line)
+            req_id = req.get("id")
+            op = req.get("op")
+            fn = OPS.get(op)
+            if fn is None:
+                raise KeyError("unknown op: {}".format(op))
+            out = {"id": req_id, "result": fn(req.get("input") or {})}
+        except Exception as err:  # noqa: BLE001 - the protocol reports errors
+            out = {"id": req_id, "error": "{}".format(err)}
+        # ensure_ascii=True here, deliberately. This is the DRIVER's own
+        # transport, not a captured body: PRIM-032's "emit UTF-8 literally"
+        # governs payloads, while stdout has to survive values the fuzzer
+        # generates - an unpaired surrogate cannot be UTF-8 encoded at all
+        # and would kill the process mid-run. JSON \uXXXX escapes carry it
+        # losslessly and the harness decodes it back.
+        sys.stdout.write(
+            json.dumps(out, separators=(",", ":"), ensure_ascii=True) + "\n"
+        )
+        sys.stdout.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())