traffic-diff 0.1.0__py3-none-any.whl

shadowdiff/replay.py

@@ -0,0 +1,141 @@
+"""Async replay of recorded exchanges against a target, plus run orchestration."""
+
+import asyncio
+import contextlib
+import time
+from dataclasses import dataclass
+
+import httpx
+
+from shadowdiff.config import Config
+from shadowdiff.diff import compare
+from shadowdiff.proxy import HOP_BY_HOP, STRIP_FROM_RESPONSE, strip_headers
+from shadowdiff.store import Exchange, Result, Store
+
+DEFAULT_TIMEOUT = 10.0
+
+
+@dataclass
+class Replayed:
+    exchange_id: int
+    status: int | None
+    headers: dict[str, str]
+    body: bytes | None
+    latency_ms: float | None
+    error: str | None = None
+
+
+async def replay_exchanges(
+    exchanges: list[Exchange],
+    target: str,
+    concurrency: int = 20,
+    timeout: float = DEFAULT_TIMEOUT,
+    transport: httpx.AsyncBaseTransport | None = None,
+) -> list[Replayed]:
+    """Send each recorded request to TARGET; failures are captured, never raised."""
+    sem = asyncio.Semaphore(concurrency)
+    # One single-connection client per concurrency slot. A single httpx client
+    # with a large pool spends CPU per request proportional to the pool size,
+    # which capped replay at ~250 req/s at concurrency 50; small pools scale.
+    limits = httpx.Limits(max_connections=1, max_keepalive_connections=1)
+    async with contextlib.AsyncExitStack() as stack:
+        free = [
+            await stack.enter_async_context(
+                httpx.AsyncClient(
+                    base_url=target.rstrip("/"), timeout=timeout, limits=limits, transport=transport
+                )
+            )
+            for _ in range(min(concurrency, len(exchanges)))
+        ]
+
+        async def one(ex: Exchange) -> Replayed:
+            url = ex.path + (f"?{ex.query}" if ex.query else "")
+            async with sem:
+                # The semaphore guarantees a free client is available.
+                client = free.pop()
+                start = time.perf_counter()
+                try:
+                    resp = await client.request(
+                        ex.method,
+                        url,
+                        headers=strip_headers(ex.req_headers, HOP_BY_HOP),
+                        content=ex.req_body or None,
+                    )
+                except httpx.TimeoutException:
+                    return Replayed(ex.id, None, {}, None, None, f"timeout after {timeout:g}s")
+                except httpx.HTTPError as exc:
+                    return Replayed(ex.id, None, {}, None, None, f"{type(exc).__name__}: {exc}")
+                finally:
+                    free.append(client)
+                latency_ms = (time.perf_counter() - start) * 1000
+            return Replayed(
+                ex.id,
+                resp.status_code,
+                strip_headers(resp.headers, STRIP_FROM_RESPONSE),
+                resp.content,
+                latency_ms,
+            )
+
+        return await asyncio.gather(*(one(ex) for ex in exchanges))
+
+
+@dataclass
+class RunSummary:
+    run_id: int
+    total: int
+    passed: int
+    failed: int
+    replay_seconds: float
+
+    @property
+    def requests_per_second(self) -> float:
+        return self.total / self.replay_seconds if self.replay_seconds else 0.0
+
+
+def run_replay(
+    store: Store,
+    session: str,
+    target: str,
+    config: Config,
+    concurrency: int = 20,
+    timeout: float = DEFAULT_TIMEOUT,
+    transport: httpx.AsyncBaseTransport | None = None,
+) -> RunSummary:
+    """Replay a recorded session against TARGET, diff every response and store the run."""
+    exchanges = store.exchanges(session)
+    if not exchanges:
+        raise ValueError(f"no recorded exchanges for session {session!r}")
+    run_id = store.create_run(session, target, concurrency, total=len(exchanges))
+
+    start = time.perf_counter()
+    try:
+        replayed = asyncio.run(replay_exchanges(exchanges, target, concurrency, timeout, transport))
+    except Exception:
+        store.finish_run(run_id, "failed", done=0, passed=0, failed=0)
+        raise
+    replay_seconds = time.perf_counter() - start
+
+    results = []
+    for ex, rep in zip(exchanges, replayed, strict=True):
+        if rep.error is not None:
+            diffs, breaking = [], True
+        else:
+            diffs = compare(ex.status, ex.resp_body, rep.status, rep.body, config)
+            breaking = any(d.breaking for d in diffs)
+        results.append(
+            Result(
+                exchange_id=ex.id,
+                status=rep.status,
+                resp_body=rep.body,
+                latency_ms=rep.latency_ms,
+                error=rep.error,
+                diffs=[d.to_dict() for d in diffs],
+                breaking=breaking,
+            )
+        )
+    store.add_results(run_id, results)
+
+    failed = sum(r.breaking for r in results)
+    passed = len(results) - failed
+    store.finish_run(run_id, "done", done=len(results), passed=passed, failed=failed)
+    return RunSummary(run_id, len(results), passed, failed, replay_seconds)

shadowdiff/report.py

@@ -0,0 +1,299 @@
+"""Grouped regression report for a replay run, rendered as Markdown or HTML."""
+
+import html
+import json
+import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+from shadowdiff.diff import normalize_path
+from shadowdiff.store import Exchange, Result, Run, Store
+
+_ID_SEGMENT = re.compile(
+    r"^(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24,})$",
+    re.IGNORECASE,
+)
+_EXAMPLE_CHARS = 80
+
+
+def endpoint_template(method: str, path: str) -> str:
+    """'GET /products/42' -> 'GET /products/{id}'."""
+    segments = ["{id}" if _ID_SEGMENT.match(s) else s for s in path.split("/")]
+    return f"{method} {'/'.join(segments)}"
+
+
+@dataclass
+class DiffGroup:
+    endpoint: str
+    path: str
+    kind: str
+    breaking: bool
+    detail: str
+    count: int = 0
+    example_url: str = ""
+    example_old: Any = None
+    example_new: Any = None
+
+
+@dataclass
+class EndpointStats:
+    requests: int = 0
+    passed: int = 0
+    failed: int = 0
+
+
+@dataclass
+class Report:
+    run: Run
+    breaking: list[DiffGroup] = field(default_factory=list)
+    warnings: list[DiffGroup] = field(default_factory=list)
+    errors: dict[str, int] = field(default_factory=dict)
+    endpoints: dict[str, EndpointStats] = field(default_factory=dict)
+
+    @property
+    def exit_code(self) -> int:
+        return 1 if self.run.failed else 0
+
+
+def build_report(store: Store, run_id: int) -> Report:
+    run = store.run(run_id)
+    if run is None:
+        raise ValueError(f"run {run_id} not found")
+    results = store.results(run_id)
+    exchanges = store.exchanges_by_id([r.exchange_id for r in results])
+    return group_results(run, results, exchanges)
+
+
+def group_results(run: Run, results: list[Result], exchanges: dict[int, Exchange]) -> Report:
+    groups: dict[tuple, DiffGroup] = {}
+    errors: dict[str, int] = defaultdict(int)
+    endpoints: dict[str, EndpointStats] = defaultdict(EndpointStats)
+
+    for result in results:
+        ex = exchanges[result.exchange_id]
+        endpoint = endpoint_template(ex.method, ex.path)
+        stats = endpoints[endpoint]
+        stats.requests += 1
+        if result.breaking:
+            stats.failed += 1
+        else:
+            stats.passed += 1
+        if result.error:
+            errors[result.error] += 1
+
+        # Count each group at most once per request, even if a list repeats the diff.
+        seen: set[tuple] = set()
+        for d in result.diffs:
+            detail = _detail(d)
+            key = (endpoint, normalize_path(d["path"]), d["kind"], detail)
+            if key in seen:
+                continue
+            seen.add(key)
+            group = groups.get(key)
+            if group is None:
+                group = groups[key] = DiffGroup(
+                    endpoint=endpoint,
+                    path=key[1],
+                    kind=d["kind"],
+                    breaking=d["breaking"],
+                    detail=detail,
+                    example_url=ex.path + (f"?{ex.query}" if ex.query else ""),
+                    example_old=d.get("old"),
+                    example_new=d.get("new"),
+                )
+            group.count += 1
+
+    ordered = sorted(groups.values(), key=lambda g: (-g.count, g.endpoint, g.path))
+    return Report(
+        run=run,
+        breaking=[g for g in ordered if g.breaking],
+        warnings=[g for g in ordered if not g.breaking],
+        errors=dict(sorted(errors.items(), key=lambda kv: -kv[1])),
+        endpoints=dict(sorted(endpoints.items())),
+    )
+
+
+def _detail(d: dict) -> str:
+    kind = d["kind"]
+    if kind == "type_changed":
+        return f"type changed {d['old_type']} → {d['new_type']}"
+    if kind == "status_changed":
+        return f"status changed {d['old']} → {d['new']}"
+    if kind == "field_removed":
+        return "field removed"
+    if kind == "field_added":
+        return "field added"
+    return "value changed"
+
+
+# --- rendering -------------------------------------------------------------------
+#
+# The report is built once as a list of simple blocks, then rendered to either
+# format. Inline `code` spans in text are honoured by both renderers.
+
+
+def _blocks(report: Report) -> list[tuple]:
+    run = report.run
+    started = datetime.fromtimestamp(run.started_at or 0, UTC).strftime("%Y-%m-%d %H:%M UTC")
+    duration = (run.finished_at or 0) - (run.started_at or 0)
+    errors = sum(report.errors.values())
+
+    if report.breaking or errors:
+        verdict = (
+            f"**FAIL**: {len(report.breaking)} breaking change(s) across {run.failed} "
+            f"request(s)" + (f", {errors} request error(s)" if errors else "")
+        )
+    else:
+        verdict = "**PASS**: no breaking changes"
+
+    blocks: list[tuple] = [
+        ("h1", f"shadowdiff report: run {run.id}"),
+        (
+            "p",
+            f"Session `{run.session}` replayed against `{run.target}` "
+            f"at {started} (concurrency {run.concurrency}).",
+        ),
+        ("p", verdict),
+        (
+            "table",
+            ["Replayed", "Passed", "Failed", "Breaking changes", "Warnings", "Duration"],
+            [
+                [
+                    run.done,
+                    run.passed,
+                    run.failed,
+                    len(report.breaking),
+                    len(report.warnings),
+                    f"{duration:.2f}s",
+                ]
+            ],
+        ),
+    ]
+
+    for title, groups in [("Breaking changes", report.breaking), ("Warnings", report.warnings)]:
+        blocks.append(("h2", title))
+        if not groups:
+            blocks.append(("p", "None."))
+            continue
+        blocks.append(
+            (
+                "table",
+                ["Endpoint", "Path", "Change", "Requests", "Example"],
+                [
+                    [
+                        f"`{g.endpoint}`",
+                        f"`{g.path}`",
+                        g.detail,
+                        g.count,
+                        f"`{g.example_url}`: {_example(g)}",
+                    ]
+                    for g in groups
+                ],
+            )
+        )
+
+    if report.errors:
+        blocks.append(("h2", "Request errors"))
+        blocks.append(
+            ("table", ["Error", "Requests"], [[f"`{e}`", n] for e, n in report.errors.items()])
+        )
+
+    blocks.append(("h2", "By endpoint"))
+    blocks.append(
+        (
+            "table",
+            ["Endpoint", "Requests", "Passed", "Failed"],
+            [[f"`{name}`", s.requests, s.passed, s.failed] for name, s in report.endpoints.items()],
+        )
+    )
+    return blocks
+
+
+def _example(g: DiffGroup) -> str:
+    if g.kind == "field_added":
+        return f"added `{_short(g.example_new)}`"
+    if g.kind == "field_removed":
+        return f"was `{_short(g.example_old)}`"
+    return f"`{_short(g.example_old)}` → `{_short(g.example_new)}`"
+
+
+def _short(value: Any) -> str:
+    text = json.dumps(value, ensure_ascii=False)
+    text = text if len(text) <= _EXAMPLE_CHARS else text[: _EXAMPLE_CHARS - 1] + "…"
+    # Keep table cells intact in Markdown.
+    return text.replace("|", "\\|").replace("`", "'")
+
+
+def render_markdown(report: Report) -> str:
+    out = []
+    for block in _blocks(report):
+        kind = block[0]
+        if kind == "h1":
+            out.append(f"# {block[1]}")
+        elif kind == "h2":
+            out.append(f"## {block[1]}")
+        elif kind == "p":
+            out.append(block[1])
+        elif kind == "table":
+            headers, rows = block[1], block[2]
+            lines = ["| " + " | ".join(headers) + " |", "|" + " --- |" * len(headers)]
+            lines += ["| " + " | ".join(str(c) for c in row) + " |" for row in rows]
+            out.append("\n".join(lines))
+    return "\n\n".join(out) + "\n"
+
+
+_CODE = re.compile(r"`([^`]*)`")
+_BOLD = re.compile(r"\*\*([^*]+)\*\*")
+
+_HTML_STYLE = """
+body { font: 14px/1.5 system-ui, sans-serif; margin: 2rem auto; max-width: 1100px;
+       padding: 0 1rem; color: #1f2328; background: #fff; }
+table { border-collapse: collapse; margin: 0.5rem 0 1.5rem; width: 100%; }
+th, td { border: 1px solid #d0d7de; padding: 6px 10px; text-align: left; vertical-align: top; }
+th { background: #f6f8fa; }
+code { background: #f6f8fa; padding: 1px 4px; border-radius: 4px; font-size: 13px; }
+@media (prefers-color-scheme: dark) {
+  body { color: #e6edf3; background: #0d1117; }
+  th, code { background: #161b22; }
+  th, td { border-color: #30363d; }
+}
+"""
+
+
+def _inline_html(text: Any) -> str:
+    text = html.escape(str(text).replace("\\|", "|"), quote=False)
+    text = _CODE.sub(r"<code>\1</code>", text)
+    return _BOLD.sub(r"<strong>\1</strong>", text)
+
+
+def render_html(report: Report) -> str:
+    body = []
+    for block in _blocks(report):
+        kind = block[0]
+        if kind in ("h1", "h2"):
+            body.append(f"<{kind}>{_inline_html(block[1])}</{kind}>")
+        elif kind == "p":
+            body.append(f"<p>{_inline_html(block[1])}</p>")
+        elif kind == "table":
+            head = "".join(f"<th>{_inline_html(h)}</th>" for h in block[1])
+            rows = "".join(
+                "<tr>" + "".join(f"<td>{_inline_html(c)}</td>" for c in row) + "</tr>"
+                for row in block[2]
+            )
+            body.append(f"<table><thead><tr>{head}</tr></thead><tbody>{rows}</tbody></table>")
+    return (
+        '<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'
+        '<meta name="viewport" content="width=device-width, initial-scale=1">'
+        f"<title>shadowdiff run {report.run.id}</title><style>{_HTML_STYLE}</style></head>"
+        f"<body>{''.join(body)}</body></html>\n"
+    )
+
+
+def render(report: Report, fmt: str = "md") -> str:
+    if fmt == "md":
+        return render_markdown(report)
+    if fmt == "html":
+        return render_html(report)
+    raise ValueError(f"unknown report format {fmt!r}; use md or html")

shadowdiff/store.py

@@ -0,0 +1,242 @@
+"""SQLite storage for recorded exchanges, replay runs and their results."""
+
+import json
+import sqlite3
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS exchanges (
+    id INTEGER PRIMARY KEY,
+    session TEXT NOT NULL,
+    method TEXT NOT NULL,
+    path TEXT NOT NULL,
+    query TEXT NOT NULL DEFAULT '',
+    req_headers TEXT NOT NULL DEFAULT '{}',
+    req_body BLOB,
+    status INTEGER NOT NULL,
+    resp_headers TEXT NOT NULL DEFAULT '{}',
+    resp_body BLOB,
+    latency_ms REAL,
+    recorded_at REAL NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_exchanges_session_path ON exchanges (session, path);
+
+CREATE TABLE IF NOT EXISTS runs (
+    id INTEGER PRIMARY KEY,
+    session TEXT NOT NULL,
+    target TEXT NOT NULL,
+    concurrency INTEGER NOT NULL,
+    status TEXT NOT NULL DEFAULT 'queued',
+    total INTEGER NOT NULL DEFAULT 0,
+    done INTEGER NOT NULL DEFAULT 0,
+    passed INTEGER NOT NULL DEFAULT 0,
+    failed INTEGER NOT NULL DEFAULT 0,
+    started_at REAL,
+    finished_at REAL
+);
+
+CREATE TABLE IF NOT EXISTS results (
+    id INTEGER PRIMARY KEY,
+    run_id INTEGER NOT NULL REFERENCES runs (id),
+    exchange_id INTEGER NOT NULL REFERENCES exchanges (id),
+    status INTEGER,
+    resp_body BLOB,
+    latency_ms REAL,
+    error TEXT,
+    diffs TEXT NOT NULL DEFAULT '[]',
+    breaking INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_results_run_breaking ON results (run_id, breaking);
+"""
+
+
+@dataclass
+class Exchange:
+    session: str
+    method: str
+    path: str
+    query: str
+    req_headers: dict[str, str]
+    req_body: bytes
+    status: int
+    resp_headers: dict[str, str]
+    resp_body: bytes
+    latency_ms: float
+    recorded_at: float = field(default_factory=time.time)
+    id: int | None = None
+
+
+@dataclass
+class Result:
+    exchange_id: int
+    status: int | None
+    resp_body: bytes | None
+    latency_ms: float | None
+    error: str | None = None
+    diffs: list[dict[str, Any]] = field(default_factory=list)
+    breaking: bool = False
+
+
+@dataclass
+class Run:
+    id: int
+    session: str
+    target: str
+    concurrency: int
+    status: str
+    total: int
+    done: int
+    passed: int
+    failed: int
+    started_at: float | None
+    finished_at: float | None
+
+
+class Store:
+    def __init__(self, path: str = "shadowdiff.db"):
+        self.path = path
+        # The proxy writes from worker threads; callers serialize writes themselves.
+        self.conn = sqlite3.connect(path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA synchronous=NORMAL")
+        self.conn.executescript(SCHEMA)
+
+    def close(self) -> None:
+        self.conn.close()
+
+    # exchanges
+
+    def add_exchanges(self, exchanges: list[Exchange]) -> None:
+        with self.conn:
+            self.conn.executemany(
+                "INSERT INTO exchanges (session, method, path, query, req_headers, req_body,"
+                " status, resp_headers, resp_body, latency_ms, recorded_at)"
+                " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                [
+                    (
+                        e.session,
+                        e.method,
+                        e.path,
+                        e.query,
+                        json.dumps(e.req_headers),
+                        e.req_body,
+                        e.status,
+                        json.dumps(e.resp_headers),
+                        e.resp_body,
+                        e.latency_ms,
+                        e.recorded_at,
+                    )
+                    for e in exchanges
+                ],
+            )
+
+    def exchanges(self, session: str) -> list[Exchange]:
+        rows = self.conn.execute(
+            "SELECT * FROM exchanges WHERE session = ? ORDER BY id", (session,)
+        ).fetchall()
+        return [_exchange_from_row(r) for r in rows]
+
+    def exchanges_by_id(self, ids: list[int]) -> dict[int, Exchange]:
+        if not ids:
+            return {}
+        out: dict[int, Exchange] = {}
+        # Stay under SQLite's bound-parameter limit.
+        for i in range(0, len(ids), 500):
+            chunk = ids[i : i + 500]
+            placeholders = ",".join("?" * len(chunk))
+            rows = self.conn.execute(
+                f"SELECT * FROM exchanges WHERE id IN ({placeholders})", chunk
+            ).fetchall()
+            out.update({r["id"]: _exchange_from_row(r) for r in rows})
+        return out
+
+    def sessions(self) -> list[dict[str, Any]]:
+        rows = self.conn.execute(
+            "SELECT session, COUNT(*) AS exchanges, MIN(recorded_at) AS first_seen,"
+            " MAX(recorded_at) AS last_seen FROM exchanges GROUP BY session ORDER BY session"
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    # runs and results
+
+    def create_run(self, session: str, target: str, concurrency: int, total: int) -> int:
+        with self.conn:
+            cur = self.conn.execute(
+                "INSERT INTO runs (session, target, concurrency, status, total, started_at)"
+                " VALUES (?, ?, ?, 'running', ?, ?)",
+                (session, target, concurrency, total, time.time()),
+            )
+        return cur.lastrowid
+
+    def add_results(self, run_id: int, results: list[Result]) -> None:
+        with self.conn:
+            self.conn.executemany(
+                "INSERT INTO results (run_id, exchange_id, status, resp_body, latency_ms,"
+                " error, diffs, breaking) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [
+                    (
+                        run_id,
+                        r.exchange_id,
+                        r.status,
+                        r.resp_body,
+                        r.latency_ms,
+                        r.error,
+                        json.dumps(r.diffs),
+                        int(r.breaking),
+                    )
+                    for r in results
+                ],
+            )
+
+    def finish_run(self, run_id: int, status: str, done: int, passed: int, failed: int) -> None:
+        with self.conn:
+            self.conn.execute(
+                "UPDATE runs SET status = ?, done = ?, passed = ?, failed = ?, finished_at = ?"
+                " WHERE id = ?",
+                (status, done, passed, failed, time.time(), run_id),
+            )
+
+    def run(self, run_id: int) -> Run | None:
+        row = self.conn.execute("SELECT * FROM runs WHERE id = ?", (run_id,)).fetchone()
+        return Run(**dict(row)) if row else None
+
+    def latest_run_id(self) -> int | None:
+        row = self.conn.execute("SELECT MAX(id) FROM runs").fetchone()
+        return row[0]
+
+    def results(self, run_id: int) -> list[Result]:
+        rows = self.conn.execute(
+            "SELECT * FROM results WHERE run_id = ? ORDER BY id", (run_id,)
+        ).fetchall()
+        return [
+            Result(
+                exchange_id=r["exchange_id"],
+                status=r["status"],
+                resp_body=r["resp_body"],
+                latency_ms=r["latency_ms"],
+                error=r["error"],
+                diffs=json.loads(r["diffs"]),
+                breaking=bool(r["breaking"]),
+            )
+            for r in rows
+        ]
+
+
+def _exchange_from_row(r: sqlite3.Row) -> Exchange:
+    return Exchange(
+        id=r["id"],
+        session=r["session"],
+        method=r["method"],
+        path=r["path"],
+        query=r["query"],
+        req_headers=json.loads(r["req_headers"]),
+        req_body=r["req_body"] or b"",
+        status=r["status"],
+        resp_headers=json.loads(r["resp_headers"]),
+        resp_body=r["resp_body"] or b"",
+        latency_ms=r["latency_ms"],
+        recorded_at=r["recorded_at"],
+    )