traffic-diff 0.1.0__py3-none-any.whl

shadowdiff/__init__.py

@@ -0,0 +1,3 @@
+"""shadowdiff: record real API traffic, replay it, and diff the responses that matter."""
+
+__version__ = "0.1.0"

shadowdiff/cli.py

@@ -0,0 +1,131 @@
+"""Command-line interface."""
+
+from enum import StrEnum
+from pathlib import Path
+
+import typer
+
+from shadowdiff import __version__
+from shadowdiff.config import ConfigError, load_config
+from shadowdiff.report import build_report, render
+from shadowdiff.store import Store
+
+app = typer.Typer(
+    help="Record API traffic, replay it, and report meaningful diffs.",
+    no_args_is_help=True,
+)
+
+
+@app.callback()
+def main() -> None:
+    """shadowdiff: record → replay → smart diff → report."""
+
+
+@app.command()
+def version() -> None:
+    """Print the shadowdiff version."""
+    typer.echo(__version__)
+
+
+DbOption = typer.Option("shadowdiff.db", "--db", help="SQLite database file.")
+
+
+@app.command()
+def record(
+    target: str = typer.Option(..., help="Upstream base URL to forward to."),
+    session: str = typer.Option("default", help="Session name to group recordings under."),
+    host: str = typer.Option("127.0.0.1", help="Interface to listen on."),
+    port: int = typer.Option(8080, help="Port to listen on."),
+    db: str = DbOption,
+) -> None:
+    """Run a recording proxy in front of TARGET."""
+    import uvicorn
+
+    from shadowdiff.proxy import create_app
+
+    store = Store(db)
+    typer.echo(f"Recording session '{session}': http://{host}:{port} -> {target} (db: {db})")
+    uvicorn.run(
+        create_app(target, store, session),
+        host=host,
+        port=port,
+        log_level="warning",
+        access_log=False,
+    )
+    store.close()
+
+
+class Format(StrEnum):
+    md = "md"
+    html = "html"
+
+
+def _write_report(store: Store, run_id: int, fmt: Format, output: Path | None) -> int:
+    report = build_report(store, run_id)
+    output = output or Path(f"report.{fmt.value}")
+    output.write_text(render(report, fmt.value))
+    verdict = "FAIL" if report.exit_code else "PASS"
+    typer.echo(
+        f"{verdict}: {len(report.breaking)} breaking change(s), "
+        f"{len(report.warnings)} warning(s), {sum(report.errors.values())} error(s). "
+        f"Report: {output}"
+    )
+    for g in report.breaking:
+        typer.echo(f"  BREAKING {g.endpoint} {g.path}: {g.detail} on {g.count} request(s)")
+    return report.exit_code
+
+
+@app.command()
+def replay(
+    session: str = typer.Option(..., help="Recorded session to replay."),
+    target: str = typer.Option(..., help="Base URL of the version under test."),
+    concurrency: int = typer.Option(20, min=1, help="Maximum requests in flight."),
+    timeout: float = typer.Option(10.0, min=0.1, help="Per-request timeout in seconds."),
+    config: Path = typer.Option(Path("shadowdiff.yaml"), help="Ignore-rules file."),
+    output: Path | None = typer.Option(None, "--output", "-o", help="Report file path."),
+    fmt: Format = typer.Option(Format.md, "--format", help="Report format."),
+    db: str = DbOption,
+) -> None:
+    """Replay SESSION against TARGET, diff responses and write a report.
+
+    Exits 1 if any breaking difference or request error is found.
+    """
+    from shadowdiff.replay import run_replay
+
+    try:
+        rules = load_config(config)
+    except ConfigError as exc:
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(2) from None
+    store = Store(db)
+    try:
+        summary = run_replay(store, session, target, rules, concurrency, timeout)
+    except ValueError as exc:
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(2) from None
+    typer.echo(
+        f"Run {summary.run_id}: replayed {summary.total} request(s) in "
+        f"{summary.replay_seconds:.2f}s ({summary.requests_per_second:.0f} req/s), "
+        f"{summary.passed} passed, {summary.failed} failed"
+    )
+    code = _write_report(store, summary.run_id, fmt, output)
+    store.close()
+    raise typer.Exit(code)
+
+
+@app.command()
+def report(
+    run: int | None = typer.Option(None, "--run", help="Run ID (default: latest run)."),
+    output: Path | None = typer.Option(None, "--output", "-o", help="Report file path."),
+    fmt: Format = typer.Option(Format.md, "--format", help="Report format."),
+    db: str = DbOption,
+) -> None:
+    """Write the report for a replay run. Exits 1 if it has breaking diffs."""
+    store = Store(db)
+    run_id = run if run is not None else store.latest_run_id()
+    if run_id is None or store.run(run_id) is None:
+        typer.echo("error: no such run" if run is not None else "error: no runs yet", err=True)
+        raise typer.Exit(2)
+    code = _write_report(store, run_id, fmt, output)
+    store.close()
+    raise typer.Exit(code)

shadowdiff/config.py

@@ -0,0 +1,57 @@
+"""Loading of shadowdiff.yaml ignore rules."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+VALUE_PATTERNS = ("uuid", "iso_timestamp")
+
+
+class ConfigError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class Config:
+    # Globs matched against a field name (e.g. "*_at"), or against the full
+    # JSON path when the pattern starts with "$" (e.g. "$.meta.*").
+    ignore_fields: tuple[str, ...] = ()
+    # Named value patterns: a change between two values matching the same
+    # pattern is noise (e.g. one UUID replaced by another).
+    ignore_values: tuple[str, ...] = ()
+    # JSON path of a list (indices written as [*]) -> key to match items by,
+    # or None to compare the list as a sorted multiset.
+    unordered_lists: dict[str, str | None] = field(default_factory=dict)
+
+
+def load_config(path: str | Path | None) -> Config:
+    if path is None:
+        return Config()
+    path = Path(path)
+    if not path.exists():
+        return Config()
+    data = yaml.safe_load(path.read_text()) or {}
+    if not isinstance(data, dict):
+        raise ConfigError(f"{path}: expected a mapping at the top level")
+
+    unknown = set(data) - {"ignore_fields", "ignore_values", "unordered_lists"}
+    if unknown:
+        raise ConfigError(f"{path}: unknown keys: {', '.join(sorted(unknown))}")
+
+    ignore_values = tuple(data.get("ignore_values") or ())
+    bad = [v for v in ignore_values if v not in VALUE_PATTERNS]
+    if bad:
+        raise ConfigError(
+            f"{path}: unknown ignore_values {bad}; supported: {', '.join(VALUE_PATTERNS)}"
+        )
+
+    unordered = data.get("unordered_lists") or {}
+    if not isinstance(unordered, dict):
+        raise ConfigError(f"{path}: unordered_lists must map a list path to a key")
+
+    return Config(
+        ignore_fields=tuple(str(f) for f in data.get("ignore_fields") or ()),
+        ignore_values=ignore_values,
+        unordered_lists={str(k): (None if v is None else str(v)) for k, v in unordered.items()},
+    )

shadowdiff/diff.py

@@ -0,0 +1,240 @@
+"""Structural response diff with noise filtering.
+
+Walks two JSON documents and reports each difference with a JSON path and a
+kind. Ignore rules from ``Config`` drop known noise (timestamps, generated IDs,
+reordered lists) so only meaningful changes are reported.
+"""
+
+import json
+import re
+from collections import Counter
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+from shadowdiff.config import Config
+
+VALUE_CHANGED = "value_changed"
+TYPE_CHANGED = "type_changed"
+FIELD_ADDED = "field_added"
+FIELD_REMOVED = "field_removed"
+STATUS_CHANGED = "status_changed"
+
+BREAKING_KINDS = frozenset({TYPE_CHANGED, FIELD_REMOVED, STATUS_CHANGED})
+
+VALUE_REGEXES = {
+    "uuid": re.compile(
+        r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
+    ),
+    "iso_timestamp": re.compile(
+        r"^\d{4}-\d{2}-\d{2}"
+        r"([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$"
+    ),
+}
+
+_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_INDEX = re.compile(r"\[(\d+|[^\[\]]*=[^\[\]]*)\]")
+_MISSING = object()
+
+
+@dataclass(frozen=True)
+class Difference:
+    path: str
+    kind: str
+    old: Any = None
+    new: Any = None
+    old_type: str | None = None
+    new_type: str | None = None
+
+    @property
+    def breaking(self) -> bool:
+        return self.kind in BREAKING_KINDS
+
+    def to_dict(self) -> dict[str, Any]:
+        d = {"path": self.path, "kind": self.kind, "old": self.old, "new": self.new}
+        if self.kind == TYPE_CHANGED:
+            d["old_type"] = self.old_type
+            d["new_type"] = self.new_type
+        d["breaking"] = self.breaking
+        return d
+
+
+def compare(
+    old_status: int, old_body: bytes, new_status: int, new_body: bytes, config: Config
+) -> list[Difference]:
+    """Compare a recorded response with a replayed one."""
+    if old_status != new_status:
+        return [Difference("$", STATUS_CHANGED, old_status, new_status)]
+
+    old_doc, old_is_json = _parse(old_body)
+    new_doc, new_is_json = _parse(new_body)
+    if old_is_json and new_is_json:
+        return diff_json(old_doc, new_doc, config)
+    if old_is_json != new_is_json:
+        old_t = json_type(old_doc) if old_is_json else "text"
+        new_t = json_type(new_doc) if new_is_json else "text"
+        return [Difference("$", TYPE_CHANGED, old_doc, new_doc, old_t, new_t)]
+    if old_doc == new_doc or _same_value_pattern(old_doc, new_doc, config):
+        return []
+    return [Difference("$", VALUE_CHANGED, old_doc, new_doc)]
+
+
+def diff_json(old: Any, new: Any, config: Config, path: str = "$") -> list[Difference]:
+    """Recursively diff two parsed JSON values."""
+    out: list[Difference] = []
+    _walk(old, new, path, config, out)
+    return out
+
+
+def json_type(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int | float):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    return "object"
+
+
+def normalize_path(path: str) -> str:
+    """Replace concrete list positions/keys with [*] so paths can be grouped."""
+    return _INDEX.sub("[*]", path)
+
+
+def _parse(body: bytes) -> tuple[Any, bool]:
+    if body:
+        try:
+            return json.loads(body), True
+        except ValueError:
+            pass
+    return body.decode("utf-8", errors="replace"), False
+
+
+def _walk(old: Any, new: Any, path: str, config: Config, out: list[Difference]) -> None:
+    old_t, new_t = json_type(old), json_type(new)
+    if old_t != new_t:
+        out.append(Difference(path, TYPE_CHANGED, old, new, old_t, new_t))
+    elif old_t == "object":
+        _walk_object(old, new, path, config, out)
+    elif old_t == "array":
+        _walk_array(old, new, path, config, out)
+    elif old != new and not _same_value_pattern(old, new, config):
+        out.append(Difference(path, VALUE_CHANGED, old, new))
+
+
+def _walk_object(old: dict, new: dict, path: str, config: Config, out: list[Difference]) -> None:
+    for key in sorted(old.keys() | new.keys()):
+        child = _child_path(path, key)
+        if _field_ignored(key, child, config.ignore_fields):
+            continue
+        o, n = old.get(key, _MISSING), new.get(key, _MISSING)
+        if n is _MISSING:
+            out.append(Difference(child, FIELD_REMOVED, old=o))
+        elif o is _MISSING:
+            out.append(Difference(child, FIELD_ADDED, new=n))
+        else:
+            _walk(o, n, child, config, out)
+
+
+def _walk_array(old: list, new: list, path: str, config: Config, out: list[Difference]) -> None:
+    rule = _unordered_rule(path, config)
+    if rule is not _MISSING:
+        key = rule
+        if key is None:
+            _walk_multiset(old, new, path, out)
+            return
+        old_by_key, new_by_key = _index_by(old, key), _index_by(new, key)
+        if old_by_key is not None and new_by_key is not None:
+            for k in sorted(old_by_key.keys() | new_by_key.keys(), key=json.dumps):
+                child = f"{path}[{key}={json.dumps(k)}]"
+                o, n = old_by_key.get(k, _MISSING), new_by_key.get(k, _MISSING)
+                if n is _MISSING:
+                    out.append(Difference(child, FIELD_REMOVED, old=o))
+                elif o is _MISSING:
+                    out.append(Difference(child, FIELD_ADDED, new=n))
+                else:
+                    _walk(o, n, child, config, out)
+            return
+        # Items without a unique key: fall back to positional comparison.
+
+    for i in range(max(len(old), len(new))):
+        child = f"{path}[{i}]"
+        if i >= len(new):
+            out.append(Difference(child, FIELD_REMOVED, old=old[i]))
+        elif i >= len(old):
+            out.append(Difference(child, FIELD_ADDED, new=new[i]))
+        else:
+            _walk(old[i], new[i], child, config, out)
+
+
+def _walk_multiset(old: list, new: list, path: str, out: list[Difference]) -> None:
+    canon = [json.dumps(x, sort_keys=True) for x in old]
+    new_canon = [json.dumps(x, sort_keys=True) for x in new]
+    removed = Counter(canon) - Counter(new_canon)
+    added = Counter(new_canon) - Counter(canon)
+    for i, c in enumerate(canon):
+        if removed[c]:
+            removed[c] -= 1
+            out.append(Difference(f"{path}[{i}]", FIELD_REMOVED, old=old[i]))
+    for i, c in enumerate(new_canon):
+        if added[c]:
+            added[c] -= 1
+            out.append(Difference(f"{path}[{i}]", FIELD_ADDED, new=new[i]))
+
+
+def _index_by(items: list, key: str) -> dict | None:
+    """Map item[key] -> item, or None if any item lacks a unique, hashable key."""
+    index: dict = {}
+    for item in items:
+        if not isinstance(item, dict) or key not in item:
+            return None
+        k = item[key]
+        if isinstance(k, dict | list) or k in index:
+            return None
+        index[k] = item
+    return index
+
+
+def _unordered_rule(path: str, config: Config) -> Any:
+    if not config.unordered_lists:
+        return _MISSING
+    if path in config.unordered_lists:
+        return config.unordered_lists[path]
+    return config.unordered_lists.get(normalize_path(path), _MISSING)
+
+
+def _field_ignored(key: str, path: str, patterns: tuple[str, ...]) -> bool:
+    for pattern in patterns:
+        if pattern.startswith("$"):
+            rx = _glob(pattern)
+            if rx.match(path) or rx.match(normalize_path(path)):
+                return True
+        elif _glob(pattern).match(key):
+            return True
+    return False
+
+
+@lru_cache(maxsize=256)
+def _glob(pattern: str) -> re.Pattern:
+    """Compile a glob where only * and ? are special (brackets are literal)."""
+    parts = (".*" if c == "*" else "." if c == "?" else re.escape(c) for c in pattern)
+    return re.compile("".join(parts) + r"\Z")
+
+
+def _same_value_pattern(old: Any, new: Any, config: Config) -> bool:
+    if not (isinstance(old, str) and isinstance(new, str)):
+        return False
+    return any(
+        VALUE_REGEXES[name].match(old) and VALUE_REGEXES[name].match(new)
+        for name in config.ignore_values
+    )
+
+
+def _child_path(path: str, key: str) -> str:
+    if _IDENTIFIER.match(key):
+        return f"{path}.{key}"
+    return f"{path}[{json.dumps(key)}]"

shadowdiff/proxy.py

@@ -0,0 +1,191 @@
+"""Recording reverse proxy.
+
+Forwards every request to an upstream unchanged and records ``GET`` exchanges
+with 2xx-4xx responses. Recording happens off the response path: exchanges go
+onto an in-memory queue that a background task flushes to SQLite in batches.
+"""
+
+import asyncio
+import contextlib
+import time
+
+import httpx
+from fastapi import FastAPI, Request, Response
+
+from shadowdiff.store import Exchange, Store
+
+MAX_BODY_BYTES = 1024 * 1024
+HOP_BY_HOP = frozenset(
+    {
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "te",
+        "trailer",
+        "trailers",
+        "transfer-encoding",
+        "upgrade",
+        "host",
+        "content-length",
+    }
+)
+# httpx transparently decodes compressed bodies, so the encoding header no longer applies.
+STRIP_FROM_RESPONSE = HOP_BY_HOP | {"content-encoding"}
+
+
+def strip_headers(headers, drop: frozenset[str] = HOP_BY_HOP) -> dict[str, str]:
+    return {k: v for k, v in headers.items() if k.lower() not in drop}
+
+
+def should_record(method: str, status: int, req_body: bytes, resp_body: bytes) -> bool:
+    return (
+        method == "GET"
+        and 200 <= status < 500
+        and len(req_body) <= MAX_BODY_BYTES
+        and len(resp_body) <= MAX_BODY_BYTES
+    )
+
+
+class Recorder:
+    """Buffers exchanges and writes them to the store in batches."""
+
+    _STOP = object()
+
+    def __init__(self, store: Store, batch_size: int = 200):
+        self.store = store
+        self.batch_size = batch_size
+        self.queue: asyncio.Queue = asyncio.Queue()
+        self.recorded = 0
+
+    def submit(self, exchange: Exchange) -> None:
+        self.queue.put_nowait(exchange)
+
+    def stop(self) -> None:
+        """Ask run() to flush everything submitted so far, then return."""
+        self.queue.put_nowait(self._STOP)
+
+    async def run(self) -> None:
+        # Stopping with a sentinel rather than cancelling means a batch is never
+        # abandoned mid-write and nothing submitted before stop() is lost.
+        stopping = False
+        while not stopping:
+            batch = []
+            item = await self.queue.get()
+            while True:
+                if item is self._STOP:
+                    stopping = True
+                    break
+                batch.append(item)
+                if len(batch) >= self.batch_size or self.queue.empty():
+                    break
+                item = self.queue.get_nowait()
+            if batch:
+                await asyncio.to_thread(self.store.add_exchanges, batch)
+                self.recorded += len(batch)
+
+
+class ClientPool:
+    """A growable LIFO pool of single-connection httpx clients.
+
+    One shared client with a large connection pool spends CPU per request
+    proportional to the pool size, which added tens of milliseconds of p95
+    latency under concurrent load. Single-connection clients avoid that, and
+    LIFO reuse keeps the warmest connections busy.
+    """
+
+    def __init__(self, factory, max_size: int = 200):
+        self.factory = factory
+        self.max_size = max_size
+        self.idle: asyncio.LifoQueue[httpx.AsyncClient] = asyncio.LifoQueue()
+        self.clients: list[httpx.AsyncClient] = []
+
+    @contextlib.asynccontextmanager
+    async def client(self):
+        if not self.idle.empty():
+            client = self.idle.get_nowait()
+        elif len(self.clients) < self.max_size:
+            client = self.factory()
+            self.clients.append(client)
+        else:
+            client = await self.idle.get()
+        try:
+            yield client
+        finally:
+            self.idle.put_nowait(client)
+
+    async def aclose(self) -> None:
+        for client in self.clients:
+            await client.aclose()
+
+
+def create_app(
+    target: str,
+    store: Store,
+    session: str,
+    transport: httpx.AsyncBaseTransport | None = None,
+) -> FastAPI:
+    target = target.rstrip("/")
+
+    @contextlib.asynccontextmanager
+    async def lifespan(app: FastAPI):
+        recorder = Recorder(store)
+        limits = httpx.Limits(max_connections=1, max_keepalive_connections=1)
+        pool = ClientPool(
+            lambda: httpx.AsyncClient(
+                base_url=target, limits=limits, timeout=30, transport=transport
+            )
+        )
+        app.state.pool = pool
+        app.state.recorder = recorder
+        writer = asyncio.create_task(recorder.run())
+        try:
+            yield
+        finally:
+            recorder.stop()
+            await writer
+            await pool.aclose()
+
+    app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
+
+    @app.api_route(
+        "/{path:path}",
+        methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"],
+    )
+    async def forward(request: Request, path: str) -> Response:
+        pool: ClientPool = request.app.state.pool
+        req_body = await request.body()
+        req_headers = strip_headers(request.headers)
+        query = request.url.query
+        url = request.url.path + (f"?{query}" if query else "")
+
+        start = time.perf_counter()
+        try:
+            async with pool.client() as client:
+                upstream = await client.request(
+                    request.method, url, headers=req_headers, content=req_body
+                )
+        except httpx.HTTPError as exc:
+            return Response(f"shadowdiff proxy: upstream error: {exc!r}", status_code=502)
+        latency_ms = (time.perf_counter() - start) * 1000
+
+        resp_headers = strip_headers(upstream.headers, STRIP_FROM_RESPONSE)
+        resp_body = upstream.content
+        if should_record(request.method, upstream.status_code, req_body, resp_body):
+            request.app.state.recorder.submit(
+                Exchange(
+                    session=session,
+                    method=request.method,
+                    path=request.url.path,
+                    query=query,
+                    req_headers=req_headers,
+                    req_body=req_body,
+                    status=upstream.status_code,
+                    resp_headers=resp_headers,
+                    resp_body=resp_body,
+                    latency_ms=latency_ms,
+                )
+            )
+        return Response(content=resp_body, status_code=upstream.status_code, headers=resp_headers)
+
+    return app