trailscope 0.1.0__tar.gz

trailscope-0.1.0/.gitignore

@@ -0,0 +1,34 @@
+/target
+node_modules/
+dist/
+*.tsbuildinfo
+.env
+.env.local
+trailscope.toml
+!deploy/**/trailscope.toml
+__pycache__/
+*.py[cod]
+.venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+*.egg-info/
+build/
+coverage/
+test-results/
+playwright-report/
+crates/trailscope-query/fuzz/target/
+crates/trailscope-query/fuzz/corpus/parse/*
+!crates/trailscope-query/fuzz/corpus/parse/seed*
+
+crates/trailscope-query/fuzz/artifacts/
+.DS_Store
+
+# Website: generated by website/scripts/sync-docs.mjs and astro
+website/.astro/
+website/src/content/docs/
+website/public/og.png
+website/public/favicon.svg
+
+# Personal Claude Code notes (machine paths, working instructions)
+CLAUDE.local.md

trailscope-0.1.0/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.5
+Name: trailscope
+Version: 0.1.0
+Summary: Trailscope SDK for Python: non-blocking API request logging for Django, FastAPI/Starlette and Celery
+Project-URL: Homepage, https://trailscope.pro
+Project-URL: Source, https://github.com/karaa1122/trailscope
+Project-URL: Documentation, https://trailscope.pro/docs/sdk-python/
+Author: Trailscope contributors
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Django
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Logging
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: celery
+Requires-Dist: celery>=5.3; extra == 'celery'
+Provides-Extra: dev
+Requires-Dist: celery>=5.6; extra == 'dev'
+Requires-Dist: django<6,>=5.2; (python_version < '3.12') and extra == 'dev'
+Requires-Dist: django>=6.1; (python_version >= '3.12') and extra == 'dev'
+Requires-Dist: fastapi>=0.162; extra == 'dev'
+Requires-Dist: httpx>=0.28; extra == 'dev'
+Requires-Dist: mypy>=2.4; extra == 'dev'
+Requires-Dist: pytest-asyncio>=1.4; extra == 'dev'
+Requires-Dist: pytest>=9.1; extra == 'dev'
+Requires-Dist: ruff>=0.17; extra == 'dev'
+Requires-Dist: starlette<1.9,>=1.0; (python_version < '3.11') and extra == 'dev'
+Requires-Dist: starlette>=1.9; (python_version >= '3.11') and extra == 'dev'
+Provides-Extra: django
+Requires-Dist: django>=4.2; extra == 'django'
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.110; extra == 'fastapi'
+Provides-Extra: starlette
+Requires-Dist: starlette>=0.37; extra == 'starlette'
+Description-Content-Type: text/markdown
+
+# trailscope
+
+The [Trailscope](https://github.com/karaa1122/trailscope) SDK for Python 3.10+: logs every request with its route, timing, user, journey keys and (on errors) bodies and stack, plus Celery jobs and log lines, without blocking your request path. No dependencies beyond the standard library.
+
+```bash
+pip install trailscope
+```
+
+## FastAPI / Starlette
+
+```python
+import trailscope
+from trailscope.asgi import TrailscopeMiddleware
+
+trailscope.init(service="orders-api")  # reads TRAILSCOPE_ENDPOINT and TRAILSCOPE_INGEST_KEY
+app = FastAPI()
+app.add_middleware(TrailscopeMiddleware)  # add it last, so it runs first
+```
+
+## Django
+
+```python
+MIDDLEWARE = ["trailscope.django.TrailscopeMiddleware", ...]  # first
+TRAILSCOPE = {"service": "orders-api"}
+```
+
+## Celery and logging
+
+```python
+from trailscope.celery import instrument
+
+instrument()  # in the web process and the worker, after trailscope.init()
+logging.getLogger().addHandler(trailscope.TrailscopeHandler())
+```
+
+In request code: `trailscope.set_user(user_id)`, `trailscope.set_correlation("order_id", order_id)`.
+
+Full documentation: [docs/sdk-python.md](https://github.com/karaa1122/trailscope/blob/main/docs/sdk-python.md).
+
+## License
+
+Apache-2.0

trailscope-0.1.0/README.md

@@ -0,0 +1,42 @@
+# trailscope
+
+The [Trailscope](https://github.com/karaa1122/trailscope) SDK for Python 3.10+: logs every request with its route, timing, user, journey keys and (on errors) bodies and stack, plus Celery jobs and log lines, without blocking your request path. No dependencies beyond the standard library.
+
+```bash
+pip install trailscope
+```
+
+## FastAPI / Starlette
+
+```python
+import trailscope
+from trailscope.asgi import TrailscopeMiddleware
+
+trailscope.init(service="orders-api")  # reads TRAILSCOPE_ENDPOINT and TRAILSCOPE_INGEST_KEY
+app = FastAPI()
+app.add_middleware(TrailscopeMiddleware)  # add it last, so it runs first
+```
+
+## Django
+
+```python
+MIDDLEWARE = ["trailscope.django.TrailscopeMiddleware", ...]  # first
+TRAILSCOPE = {"service": "orders-api"}
+```
+
+## Celery and logging
+
+```python
+from trailscope.celery import instrument
+
+instrument()  # in the web process and the worker, after trailscope.init()
+logging.getLogger().addHandler(trailscope.TrailscopeHandler())
+```
+
+In request code: `trailscope.set_user(user_id)`, `trailscope.set_correlation("order_id", order_id)`.
+
+Full documentation: [docs/sdk-python.md](https://github.com/karaa1122/trailscope/blob/main/docs/sdk-python.md).
+
+## License
+
+Apache-2.0

trailscope-0.1.0/bench/overhead.py

@@ -0,0 +1,166 @@
+"""Measures what the ASGI and Django middleware add to the request path.
+
+Context setup, request ID header, body buffering and recording the request
+(redaction, serialization and gzip happen later in the flush thread). Synthetic
+requests keep network noise out. Run: python bench/overhead.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import statistics
+import sys
+import time
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+import trailscope
+from trailscope.asgi import TrailscopeMiddleware as AsgiMiddleware
+
+N = 50_000
+client = trailscope.Client(
+    ingest_key="trs_ingest_bench",
+    service="bench",
+    endpoint="http://127.0.0.1:9",  # never reached: flushing is paused below
+    flush_interval=3600,
+    flush_at=10**9,
+    max_queue_size=N * 3,
+    remote_config=False,
+)
+PAYLOAD = json.dumps([{"id": "t1", "title": "Write the brief", "done": False}]).encode()
+HEADERS = [
+    (b"host", b"api.example.com"),
+    (b"content-type", b"application/json"),
+    (b"authorization", b"Bearer eyJhbGciOiJIUzI1NiJ9.bench"),
+    (b"user-agent", b"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15"),
+    (b"content-length", b"64"),
+]
+
+
+class Route:
+    path_format = "/projects/{project_id}/tasks"
+
+
+async def app(
+    scope: dict[str, Any],
+    receive: Callable[[], Awaitable[Any]],
+    send: Callable[[Any], Awaitable[None]],
+) -> None:
+    await receive()
+    scope["route"] = Route()
+    status = 500 if scope["path"].endswith("/fail") else 200
+    await send(
+        {
+            "type": "http.response.start",
+            "status": status,
+            "headers": [(b"content-type", b"application/json")],
+        }
+    )
+    await send({"type": "http.response.body", "body": PAYLOAD})
+
+
+def pct(samples: list[float], p: float) -> float:
+    return samples[min(len(samples) - 1, int(len(samples) * p))] * 1000
+
+
+async def bench_asgi(wrapped: bool) -> list[float]:
+    target = AsgiMiddleware(app, client=client) if wrapped else app
+    body = {
+        "type": "http.request",
+        "body": b'{"title":"Draft roadmap","tags":["a","b"]}',
+        "more_body": False,
+    }
+
+    async def receive() -> dict[str, Any]:
+        return body
+
+    async def send(_: Any) -> None:
+        return None
+
+    samples = []
+    for i in range(N):
+        scope = {
+            "type": "http",
+            "method": "POST",
+            "path": f"/projects/prj_{i % 100}/tasks" + ("/fail" if i % 50 == 0 else ""),
+            "query_string": b"expand=owner",
+            "headers": HEADERS,
+            "client": ("198.51.100.7", 50000),
+            "root_path": "",
+        }
+        t = time.perf_counter()
+        await target(scope, receive, send)
+        samples.append(time.perf_counter() - t)
+    return sorted(samples)
+
+
+def bench_django(wrapped: bool) -> list[float]:
+    from django.conf import settings
+
+    if not settings.configured:
+        settings.configure(ALLOWED_HOSTS=["*"], USE_TZ=True)
+    import django
+
+    django.setup()
+    from django.http import HttpResponse
+    from django.test import RequestFactory
+
+    import trailscope.django as lbd
+
+    def view(request: Any) -> HttpResponse:
+        status = 500 if request.path.endswith("/fail") else 200
+        return HttpResponse(PAYLOAD, status=status, content_type="application/json")
+
+    trailscope.init(  # the Django middleware uses the process-wide client
+        ingest_key="trs_ingest_bench",
+        service="bench",
+        endpoint="http://127.0.0.1:9",
+        flush_interval=3600,
+        flush_at=10**9,
+        max_queue_size=N * 3,
+        remote_config=False,
+    )
+    target: Callable[[Any], Any] = lbd.TrailscopeMiddleware(view) if wrapped else view
+    factory = RequestFactory()
+    samples = []
+    for i in range(N):
+        path = f"/projects/prj_{i % 100}/tasks" + ("/fail" if i % 50 == 0 else "")
+        request = factory.post(
+            path + "?expand=owner", data=b'{"title":"Draft"}', content_type="application/json"
+        )
+        t = time.perf_counter()
+        target(request)
+        samples.append(time.perf_counter() - t)
+    return sorted(samples)
+
+
+def report(name: str, base: list[float], sdk: list[float]) -> dict[str, Any]:
+    return {
+        "name": name,
+        "iterations": N,
+        "baseline_p50_ms": round(pct(base, 0.5), 4),
+        "baseline_p99_ms": round(pct(base, 0.99), 4),
+        "with_sdk_p50_ms": round(pct(sdk, 0.5), 4),
+        "with_sdk_p99_ms": round(pct(sdk, 0.99), 4),
+        "added_p50_ms": round(pct(sdk, 0.5) - pct(base, 0.5), 4),
+        "added_p99_ms": round(pct(sdk, 0.99) - pct(base, 0.99), 4),
+        "added_mean_ms": round((statistics.fmean(sdk) - statistics.fmean(base)) * 1000, 4),
+    }
+
+
+def main() -> int:
+    results = []
+    asyncio.run(bench_asgi(True))  # warm up
+    results.append(report("asgi", asyncio.run(bench_asgi(False)), asyncio.run(bench_asgi(True))))
+    try:
+        bench_django(True)  # warm up
+        results.append(report("django", bench_django(False), bench_django(True)))
+    except ImportError:
+        print("django not installed; skipping", file=sys.stderr)
+    print(json.dumps({"python": sys.version.split()[0], "results": results}, indent=2))
+    return 1 if any(r["added_p99_ms"] >= 0.2 for r in results) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

trailscope-0.1.0/pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["hatchling>=1.32"]
+build-backend = "hatchling.build"
+
+[project]
+name = "trailscope"
+version = "0.1.0"
+description = "Trailscope SDK for Python: non-blocking API request logging for Django, FastAPI/Starlette and Celery"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+authors = [{ name = "Trailscope contributors" }]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Framework :: Django",
+  "Framework :: FastAPI",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Topic :: System :: Logging",
+  "Typing :: Typed",
+]
+dependencies = []
+
+[project.optional-dependencies]
+django = ["django>=4.2"]
+starlette = ["starlette>=0.37"]
+fastapi = ["fastapi>=0.110"]
+celery = ["celery>=5.3"]
+dev = [
+  "pytest>=9.1",
+  "pytest-asyncio>=1.4",
+  "ruff>=0.17",
+  "mypy>=2.4",
+  # Newest releases need newer Pythons; 3.10 and 3.11 test against the previous lines.
+  "django>=6.1; python_version >= '3.12'",
+  "django>=5.2,<6; python_version < '3.12'",
+  "starlette>=1.9; python_version >= '3.11'",
+  "starlette>=1.0,<1.9; python_version < '3.11'",
+  "fastapi>=0.162",
+  "httpx>=0.28",
+  "celery>=5.6",
+]
+
+[project.urls]
+Homepage = "https://trailscope.pro"
+Source = "https://github.com/karaa1122/trailscope"
+Documentation = "https://trailscope.pro/docs/sdk-python/"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/trailscope"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
+
+[tool.mypy]
+strict = true
+python_version = "3.10"
+mypy_path = "src"
+packages = ["trailscope"]
+
+[[tool.mypy.overrides]]
+module = ["django.*", "celery.*"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

trailscope-0.1.0/src/trailscope/__init__.py

@@ -0,0 +1,55 @@
+"""Trailscope SDK for Python: non-blocking request logging for Django, ASGI apps and Celery."""
+
+from ._context import (
+    TrailscopeContext,
+    current_context,
+    deserialize_context,
+    record_error,
+    serialize_context,
+    set_attribute,
+    set_correlation,
+    set_user,
+    use_context,
+)
+from ._ids import format_traceparent, new_request_id, new_span_id, new_trace_id, parse_traceparent
+from ._redact import REDACTED, Redactor, mask_ip
+from ._version import __version__
+from .client import (
+    DEFAULT_IGNORE_ROUTES,
+    Client,
+    ClientStats,
+    HttpExchange,
+    JobScope,
+    get_client,
+    init,
+)
+from .handler import TrailscopeHandler
+
+__all__ = [
+    "DEFAULT_IGNORE_ROUTES",
+    "REDACTED",
+    "Client",
+    "ClientStats",
+    "HttpExchange",
+    "JobScope",
+    "Redactor",
+    "TrailscopeContext",
+    "TrailscopeHandler",
+    "__version__",
+    "current_context",
+    "deserialize_context",
+    "format_traceparent",
+    "get_client",
+    "init",
+    "mask_ip",
+    "new_request_id",
+    "new_span_id",
+    "new_trace_id",
+    "parse_traceparent",
+    "record_error",
+    "serialize_context",
+    "set_attribute",
+    "set_correlation",
+    "set_user",
+    "use_context",
+]

trailscope-0.1.0/src/trailscope/_capture.py

@@ -0,0 +1,108 @@
+"""Body capture policy, serialization and truncation."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping
+from typing import Any, Literal
+
+BodyCapture = Literal["errors", "always", "never"]
+
+_BINARY_TYPES = (
+    "multipart/",
+    "application/octet-stream",
+    "image/",
+    "audio/",
+    "video/",
+    "font/",
+    "application/pdf",
+    "application/zip",
+    "application/gzip",
+    "application/x-protobuf",
+    "application/grpc",
+)
+
+
+def is_binary_content_type(content_type: str | None) -> bool:
+    """Whether a body with this content type should never be captured."""
+    if not content_type:
+        return False
+    return content_type.lower().startswith(_BINARY_TYPES)
+
+
+def should_capture_bodies(
+    policy: BodyCapture, status: int, duration_ms: float, slow_ms: float
+) -> bool:
+    if policy == "never":
+        return False
+    if policy == "always":
+        return True
+    return status >= 400 or duration_ms >= slow_ms
+
+
+def truncate(text: str, limit: int) -> tuple[str, bool]:
+    data = text.encode("utf-8")
+    if len(data) <= limit:
+        return text, False
+    # Cut at a byte boundary and drop a trailing partial character.
+    return data[:limit].decode("utf-8", errors="ignore"), True
+
+
+def serialize_body(value: Any, content_type: str | None, limit: int) -> tuple[str, bool] | None:
+    """Serializes a body-like value and truncates it to ``limit`` bytes.
+
+    Returns None for binary or empty bodies. Raw bodies of binary and multipart
+    requests are skipped; already parsed form fields (a dict) are kept.
+    """
+    if value is None:
+        return None
+    if isinstance(value, (bytes, bytearray, memoryview, str)) and is_binary_content_type(
+        content_type
+    ):
+        return None
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        raw = bytes(value[: limit + 4])
+        if b"\x00" in raw[:512]:
+            return None
+        text = raw.decode("utf-8", errors="replace")
+        if len(value) > limit:
+            return truncate(text, limit)[0], True
+    elif isinstance(value, str):
+        text = value
+    else:
+        try:
+            text = json.dumps(value, ensure_ascii=False, default=str, separators=(",", ":"))
+        except (TypeError, ValueError):
+            return None
+    if not text:
+        return None
+    return truncate(text, limit)
+
+
+def split_url(url: str) -> tuple[str, str]:
+    """``/a/b?x=1`` -> (``/a/b``, ``x=1``)."""
+    path, _, query = url.partition("?")
+    return path or "/", query
+
+
+def flatten_headers(
+    headers: Mapping[str, Any] | Iterable[tuple[str, Any]] | None,
+) -> dict[str, str]:
+    """Lower-cased names; repeated headers joined with ``, ``."""
+    out: dict[str, str] = {}
+    if headers is None:
+        return out
+    items = headers.items() if isinstance(headers, Mapping) else headers
+    for k, v in items:
+        if v is None:
+            continue
+        name = (k.decode("latin-1") if isinstance(k, bytes) else str(k)).lower()
+        val = (
+            v.decode("latin-1")
+            if isinstance(v, bytes)
+            else ", ".join(v)
+            if isinstance(v, list)
+            else str(v)
+        )
+        out[name] = f"{out[name]}, {val}" if name in out else val
+    return out

trailscope-0.1.0/src/trailscope/_context.py

@@ -0,0 +1,125 @@
+"""Request-scoped context carried through threads and async tasks with contextvars."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar, Token
+from dataclasses import dataclass, field
+from typing import Any
+
+Scalar = str | int | float | bool
+
+
+@dataclass
+class TrailscopeContext:
+    request_id: str
+    trace_id: str
+    span_id: str
+    parent_span_id: str | None = None
+    user_id: str | None = None
+    #: Journey keys, e.g. ``{"account_id": "acc_1"}``.
+    correlation: dict[str, str] = field(default_factory=dict)
+    attributes: dict[str, Scalar] = field(default_factory=dict)
+    #: Route pattern, once the framework has matched one.
+    route: str | None = None
+    #: The exception that failed the request, recorded by an error hook.
+    error: BaseException | None = None
+
+
+_current: ContextVar[TrailscopeContext | None] = ContextVar("trailscope_context", default=None)
+
+
+def current_context() -> TrailscopeContext | None:
+    return _current.get()
+
+
+def set_context(ctx: TrailscopeContext | None) -> Token[TrailscopeContext | None]:
+    return _current.set(ctx)
+
+
+def reset_context(token: Token[TrailscopeContext | None]) -> None:
+    try:
+        _current.reset(token)
+    except ValueError:
+        # Reset from another context (e.g. a generator finalized elsewhere).
+        _current.set(None)
+
+
+@contextmanager
+def use_context(ctx: TrailscopeContext) -> Iterator[TrailscopeContext]:
+    """Runs a block inside ``ctx``."""
+    token = _current.set(ctx)
+    try:
+        yield ctx
+    finally:
+        reset_context(token)
+
+
+def set_user(user_id: str | int | None) -> None:
+    """Attaches a user ID to the current request."""
+    ctx = _current.get()
+    if ctx is not None and user_id is not None and user_id != "":
+        ctx.user_id = str(user_id)
+
+
+def set_correlation(key: str, value: str | int) -> None:
+    """Adds a correlation (journey) key to the current request and everything it triggers."""
+    ctx = _current.get()
+    if ctx is not None and key:
+        ctx.correlation[key] = str(value)
+
+
+def set_attribute(key: str, value: Scalar) -> None:
+    """Adds a free-form attribute to the current request."""
+    ctx = _current.get()
+    if ctx is not None and key:
+        ctx.attributes[key] = value
+
+
+def record_error(error: BaseException, ctx: TrailscopeContext | None = None) -> None:
+    """Attaches the exception that failed the request (kept when the response is a 5xx)."""
+    ctx = ctx or _current.get()
+    if ctx is not None and ctx.error is None:
+        ctx.error = error
+
+
+def serialize_context(ctx: TrailscopeContext | None = None) -> dict[str, Any] | None:
+    """The part of a context that travels with a background job."""
+    ctx = ctx or _current.get()
+    if ctx is None:
+        return None
+    out: dict[str, Any] = {
+        "requestId": ctx.request_id,
+        "traceId": ctx.trace_id,
+        "spanId": ctx.span_id,
+    }
+    if ctx.user_id:
+        out["userId"] = ctx.user_id
+    if ctx.correlation:
+        out["correlation"] = dict(ctx.correlation)
+    return out
+
+
+def deserialize_context(value: object) -> dict[str, Any] | None:
+    """Validates a serialized context (from job data or message headers)."""
+    if not isinstance(value, dict):
+        return None
+
+    def s(key: str) -> str | None:
+        v = value.get(key)
+        return v if isinstance(v, str) and len(v) <= 128 else None
+
+    correlation: dict[str, str] = {}
+    raw = value.get("correlation")
+    if isinstance(raw, dict):
+        for k, v in raw.items():
+            if isinstance(k, str) and isinstance(v, (str, int)):
+                correlation[k] = str(v)
+    return {
+        "request_id": s("requestId"),
+        "trace_id": s("traceId"),
+        "span_id": s("spanId"),
+        "user_id": s("userId"),
+        "correlation": correlation,
+    }