pytest-wirecheck 0.1.0__py3-none-any.whl

wirecheck/plugin.py

@@ -0,0 +1,314 @@
+"""pytest plugin: ``pytest --wirecheck``."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+from collections import Counter
+from collections.abc import Sequence
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from . import __version__, clock, report, rules, versions
+from .capture import Recorder
+from .rules import Captured
+
+STATE_KEY = pytest.StashKey["WireCheck"]()
+FAIL_ON = ("error", "warn", "never")
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("wirecheck", "Claude API request checks")
+    group.addoption(
+        "--wirecheck",
+        action="store_true",
+        default=None,
+        help="Capture requests to the Claude API and check them against API version 2026-10-21.",
+    )
+    group.addoption(
+        "--wirecheck-base-url",
+        action="append",
+        default=[],
+        metavar="URL",
+        help="Extra base URL to capture (for gateways and proxies). Repeatable. "
+        "api.anthropic.com and $ANTHROPIC_BASE_URL are always captured.",
+    )
+    group.addoption(
+        "--wirecheck-fail-on",
+        choices=FAIL_ON,
+        default=None,
+        help="Fail tests on 'error' findings (default), on any finding ('warn'), or 'never'.",
+    )
+    group.addoption("--wirecheck-report", metavar="PATH", default=None, help="Write findings as JSON.")
+    group.addoption("--wirecheck-markdown", metavar="PATH", default=None, help="Write findings as Markdown.")
+    parser.addini("wirecheck", type="bool", default=False, help="Turn WireCheck on for every run.")
+    parser.addini("wirecheck_base_urls", type="linelist", default=[], help="Extra base URLs to capture.")
+    parser.addini("wirecheck_fail_on", default="error", help="error, warn or never.")
+
+
+def _enabled(config: pytest.Config) -> bool:
+    flag = config.getoption("wirecheck")
+    if flag is not None:
+        return bool(flag)
+    if os.environ.get("WIRECHECK", "").strip().lower() in ("1", "true", "yes", "on"):
+        return True
+    return bool(config.getini("wirecheck"))
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "wirecheck_skip: do not check Claude API requests made by this test")
+    if not _enabled(config):
+        return
+    state = WireCheck(config)
+    config.stash[STATE_KEY] = state
+    config.pluginmanager.register(state, "wirecheck-state")
+    state.start()
+
+
+def pytest_unconfigure(config: pytest.Config) -> None:
+    state = config.stash.get(STATE_KEY, None)
+    if state is not None:
+        state.stop()
+
+
+class CapturedView(Sequence):
+    """Live, read-only list of the requests one test has sent so far."""
+
+    def __init__(self, state: "WireCheck | None", nodeid: str) -> None:
+        self._state = state
+        self._nodeid = nodeid
+
+    def _items(self) -> list[Captured]:
+        if self._state is None:
+            return []
+        return [c for c in self._state.captured if c.test == self._nodeid]
+
+    def __getitem__(self, index: Any) -> Any:
+        return self._items()[index]
+
+    def __len__(self) -> int:
+        return len(self._items())
+
+    def __repr__(self) -> str:
+        return f"CapturedView({self._items()!r})"
+
+
+@pytest.fixture
+def wirecheck_requests(request: pytest.FixtureRequest) -> CapturedView:
+    """Requests to the Claude API captured during this test (always empty if WireCheck is off)."""
+    return CapturedView(request.config.stash.get(STATE_KEY, None), request.node.nodeid)
+
+
+@pytest.fixture
+def wirecheck_stub(request: pytest.FixtureRequest) -> Any:
+    """Answer every Claude API request in this test with a canned 200 reply, without the network.
+
+    This mocks the response, not the client, so WireCheck still sees what your code sends.
+    Works with or without --wirecheck. Non-streaming calls only.
+    """
+    state = request.config.stash.get(STATE_KEY, None)
+    own: Recorder | None = None
+    if state is None:
+        own = Recorder(on_capture=lambda c: None)
+        own.install()
+        recorder = own
+    else:
+        recorder = state.recorder
+    stub = {"text": "stub reply"}
+    recorder.stub = stub
+    try:
+        yield stub
+    finally:
+        recorder.stub = None
+        if own is not None:
+            own.uninstall()
+
+
+class FailureRepr:
+    """Test failure text with a one-line summary for pytest's short test summary."""
+
+    def __init__(self, text: str, summary: str) -> None:
+        self.text = text
+        self.reprcrash = SimpleNamespace(message=summary, path="", lineno=0)
+
+    def toterminal(self, tw: Any) -> None:
+        for line in self.text.splitlines():
+            tw.line(line)
+
+    def __str__(self) -> str:
+        return self.text
+
+
+def _one_line(findings: list[rules.Finding]) -> str:
+    parts = []
+    for f in findings:
+        item = f"{f.rule_id} {f.where}"
+        if item not in parts:
+            parts.append(item)
+    shown = ", ".join(parts[:3]) + (f" and {len(parts) - 3} more" if len(parts) > 3 else "")
+    return f"WireCheck: {shown} (breaks under Claude API 2026-10-21)"
+
+
+class WireCheck:
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        workerinput = getattr(config, "workerinput", None)
+        self.worker_id: str | None = workerinput.get("workerid") if workerinput else None
+        if workerinput:
+            self.share_dir: str | None = workerinput.get("wirecheck_dir")
+            self.own_share_dir = False
+        else:
+            self.share_dir = tempfile.mkdtemp(prefix="wirecheck-")
+            self.own_share_dir = True
+        self.merged: dict[str, Any] | None = None
+        base_urls = list(config.getoption("wirecheck_base_url") or []) + list(config.getini("wirecheck_base_urls"))
+        env_base = os.environ.get("ANTHROPIC_BASE_URL")
+        if env_base:
+            base_urls.append(env_base)
+        fail_on = config.getoption("wirecheck_fail_on") or config.getini("wirecheck_fail_on") or "error"
+        if fail_on not in FAIL_ON:
+            raise pytest.UsageError(f"wirecheck_fail_on must be one of {', '.join(FAIL_ON)}, not {fail_on!r}")
+        self.fail_on = fail_on
+        self.sdk_version = versions.installed_sdk_version()
+        self.recorder = Recorder(self._on_capture, base_urls=base_urls, root=str(config.rootpath))
+        self.captured: list[Captured] = []
+        self.findings: list[rules.Finding] = []
+        self.pending: dict[str, list[rules.Finding]] = {}
+        self.skip_tests: set[str] = set()
+
+    # -- lifecycle -------------------------------------------------------------
+    def start(self) -> None:
+        self.recorder.install()
+
+    def stop(self) -> None:
+        self.recorder.uninstall()
+        if self.own_share_dir and self.share_dir:
+            shutil.rmtree(self.share_dir, ignore_errors=True)
+
+    @pytest.hookimpl(optionalhook=True)
+    def pytest_configure_node(self, node: Any) -> None:
+        """pytest-xdist: tell each worker where to leave its findings."""
+        if self.share_dir:
+            node.workerinput["wirecheck_dir"] = self.share_dir
+
+    def _on_capture(self, captured: Captured) -> None:
+        if captured.test in self.skip_tests:
+            return
+        self.captured.append(captured)
+        ctx = rules.Context(sdk_version=self.sdk_version, today=clock.today())
+        found = rules.check(captured, ctx)
+        self.findings.extend(found)
+        if captured.test:
+            self.pending.setdefault(captured.test, []).extend(found)
+
+    def _fails(self, finding: rules.Finding) -> bool:
+        if self.fail_on == "never":
+            return False
+        if self.fail_on == "warn":
+            return True
+        return finding.severity == rules.ERROR
+
+    # -- pytest hooks ----------------------------------------------------------
+    @pytest.hookimpl(wrapper=True)
+    def pytest_runtest_protocol(self, item: pytest.Item, nextitem: Any) -> Any:
+        if item.get_closest_marker("wirecheck_skip"):
+            self.skip_tests.add(item.nodeid)
+        self.recorder.current_test = item.nodeid
+        try:
+            return (yield)
+        finally:
+            self.recorder.current_test = None
+
+    @pytest.hookimpl(wrapper=True)
+    def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo) -> Any:
+        rep = yield
+        new = self.pending.pop(item.nodeid, [])
+        failing = [f for f in new if self._fails(f)]
+        if failing:
+            text = report.render_test_failure([f.to_dict() for f in failing])
+            if rep.passed:
+                rep.outcome = "failed"
+                rep.longrepr = FailureRepr(text, _one_line(failing))
+            else:
+                rep.sections.append(("wirecheck", text))
+        return rep
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None:
+        if self.worker_id:
+            # pytest-xdist worker: hand the findings to the controller, which reports them.
+            if self.share_dir and os.path.isdir(self.share_dir):
+                report.dump_json(self.data(), os.path.join(self.share_dir, f"worker-{self.worker_id}.json"))
+            return
+        data = self.merged = self._merge_workers(self.data())
+        outside = [f for f in data["findings"] if not f.get("test") and self._fails_dict(f)]
+        if outside and session.exitstatus == 0:
+            session.exitstatus = pytest.ExitCode.TESTS_FAILED
+        path = self.config.getoption("wirecheck_report")
+        if path:
+            report.dump_json(data, path)
+        md = self.config.getoption("wirecheck_markdown")
+        if md:
+            with open(md, "w", encoding="utf-8") as fh:
+                fh.write(report.render_markdown(data, None))
+
+    def pytest_report_header(self, config: pytest.Config) -> str:
+        libs = ", ".join(self.recorder.libraries) or "none (install httpx2 or httpx)"
+        sdk = self.sdk_version or "not installed"
+        return f"wirecheck {__version__}: hooked {libs}; anthropic SDK {sdk}; fail on {self.fail_on}"
+
+    def pytest_terminal_summary(self, terminalreporter: Any) -> None:
+        if self.worker_id:
+            return
+        terminalreporter.write_sep("=", "wirecheck")
+        terminalreporter.write_line(report.render_text(self.merged or self.data()))
+
+    def _fails_dict(self, finding: dict[str, Any]) -> bool:
+        if self.fail_on == "never":
+            return False
+        return self.fail_on == "warn" or finding.get("severity") == rules.ERROR
+
+    def _merge_workers(self, own: dict[str, Any]) -> dict[str, Any]:
+        if not self.share_dir or not os.path.isdir(self.share_dir):
+            return own
+        merged = dict(
+            own,
+            findings=list(own["findings"]),
+            api_versions=Counter(own["api_versions"]),
+            internal_errors=list(own.get("internal_errors", [])),
+        )
+        for name in sorted(os.listdir(self.share_dir)):
+            if not (name.startswith("worker-") and name.endswith(".json")):
+                continue
+            try:
+                with open(os.path.join(self.share_dir, name), encoding="utf-8") as fh:
+                    part = json.load(fh)
+            except (OSError, ValueError):
+                continue
+            merged["requests_checked"] += part.get("requests_checked", 0)
+            merged["api_versions"].update(part.get("api_versions", {}))
+            merged["findings"].extend(part.get("findings", []))
+            merged["internal_errors"].extend(part.get("internal_errors", []))
+            merged["sdk_version"] = merged.get("sdk_version") or part.get("sdk_version")
+        merged["api_versions"] = dict(merged["api_versions"])
+        return merged
+
+    def data(self) -> dict[str, Any]:
+        versions_seen = Counter(
+            versions.resolve(c.headers, self.sdk_version).api_version for c in self.captured
+        )
+        return {
+            "tool": "wirecheck",
+            "version": __version__,
+            "kind": "pytest",
+            "requests_checked": len(self.captured),
+            "sdk_version": self.sdk_version,
+            "api_versions": dict(versions_seen),
+            "fail_on": self.fail_on,
+            "findings": [f.to_dict() for f in self.findings],
+            "internal_errors": list(self.recorder.errors[:50]),
+        }

wirecheck/report.py

@@ -0,0 +1,183 @@
+"""Turn findings into a terminal table, JSON, or a Markdown PR comment."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter, OrderedDict
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from . import __version__
+from .rules import RULES
+
+COMMENT_MARKER = "<!-- wirecheck-comment -->"
+REPO_URL = "https://github.com/ishanyash/wirecheck"
+SCAN_RULE_DOCS: dict[str, str] = {}  # filled by scan.py to avoid an import cycle
+
+
+@dataclass
+class Row:
+    rule_id: str
+    severity: str
+    test: str
+    call_site: str
+    count: int = 0
+    where: list[str] = field(default_factory=list)
+
+
+def group_findings(findings: Iterable[dict[str, Any]]) -> list[Row]:
+    rows: "OrderedDict[tuple, Row]" = OrderedDict()
+    for f in findings:
+        key = (f["rule_id"], f["severity"], f.get("test") or "(outside tests)", f.get("call_site") or "?")
+        row = rows.get(key)
+        if row is None:
+            row = rows[key] = Row(*key)
+        row.count += 1
+        if f.get("where") and f["where"] not in row.where:
+            row.where.append(f["where"])
+    order = {"error": 0, "warn": 1}
+    return sorted(rows.values(), key=lambda r: (order.get(r.severity, 2), r.rule_id, r.test, r.call_site))
+
+
+def _counts(findings: Iterable[dict[str, Any]]) -> tuple[int, int]:
+    c = Counter(f["severity"] for f in findings)
+    return c.get("error", 0), c.get("warn", 0)
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+# -- terminal -------------------------------------------------------------------
+
+def render_text(data: dict[str, Any]) -> str:
+    findings = data["findings"]
+    errors, warns = _counts(findings)
+    lines = [
+        f"{_plural(data['requests_checked'], 'request')} to the Claude API checked: "
+        f"{_plural(errors, 'error')}, {_plural(warns, 'warning')}",
+        f"anthropic SDK: {data.get('sdk_version') or 'not installed'}; "
+        f"API versions seen: {', '.join(f'{k} x{v}' for k, v in data.get('api_versions', {}).items()) or 'none'}",
+    ]
+    internal = data.get("internal_errors") or []
+    if internal:
+        lines.append(
+            f"WARNING: WireCheck hit {len(internal)} internal error(s), so some requests may be missing. "
+            f"First: {internal[0]}. Please report it at {REPO_URL}/issues"
+        )
+    if data["requests_checked"] == 0:
+        lines.append(
+            "No requests were seen. If your tests replace client.messages.create with a mock, no request is "
+            "ever built. Mock the HTTP response instead (see the wirecheck_stub fixture)."
+        )
+    rows = group_findings(findings)
+    if rows:
+        table = [("RULE", "LEVEL", "COUNT", "TEST", "CALL SITE", "WHAT")]
+        for r in rows:
+            table.append((r.rule_id, r.severity, str(r.count), r.test, r.call_site, ", ".join(r.where)))
+        widths = [max(len(row[i]) for row in table) for i in range(len(table[0]) - 1)]
+        lines.append("")
+        for row in table:
+            cells = [row[i].ljust(widths[i]) for i in range(len(widths))] + [row[-1]]
+            lines.append("  ".join(cells).rstrip())
+        lines.append("")
+        for rule_id in sorted({r.rule_id for r in rows}):
+            rule = RULES[rule_id]
+            lines.append(f"{rule_id} {rule.name}: {rule.summary}")
+            lines.append(f"      {rule.doc}")
+    return "\n".join(lines)
+
+
+def render_test_failure(findings: list[dict[str, Any]]) -> str:
+    out = ["WireCheck: this test sent a request that breaks under Claude API 2026-10-21", ""]
+    for f in findings:
+        out.append(f"  {f['rule_id']} [{f['severity']}] {f['where']}  (api {f['api_version']}, from {f['api_version_source']})")
+        out.append(f"      call site: {f.get('call_site') or '?'}")
+        out.append(f"      {f['message']}")
+        out.append(f"      {f['doc']}")
+    return "\n".join(out)
+
+
+# -- markdown (PR comment and step summary) -------------------------------------
+
+def _md(text: str) -> str:
+    return str(text).replace("|", "\\|").replace("\n", " ")
+
+
+def render_markdown(
+    pytest_data: dict[str, Any] | None,
+    scan_data: dict[str, Any] | None,
+    pytest_exit: int | None = None,
+) -> str:
+    runtime = (pytest_data or {}).get("findings", [])
+    static = (scan_data or {}).get("findings", [])
+    errors, warns = _counts(list(runtime) + list(static))
+    if errors:
+        headline = f"**{_plural(errors, 'error')}, {_plural(warns, 'warning')}.** These requests return 400 under API version 2026-10-21."
+    elif warns:
+        headline = f"**No errors, {_plural(warns, 'warning')}.**"
+    else:
+        headline = "**No problems found.**"
+
+    parts = [COMMENT_MARKER, "### WireCheck: Claude API 2026-10-21", "", headline, ""]
+    facts = []
+    if pytest_data is not None:
+        facts.append(f"{_plural(pytest_data.get('requests_checked', 0), 'request')} captured in tests")
+        if pytest_data.get("sdk_version"):
+            facts.append(f"anthropic {pytest_data['sdk_version']}")
+    if scan_data is not None:
+        facts.append(f"{_plural(scan_data.get('files_scanned', 0), 'file')} scanned")
+    if facts:
+        parts += ["<sub>" + " · ".join(facts) + "</sub>", ""]
+    internal = (pytest_data or {}).get("internal_errors") or []
+    if internal:
+        parts += [f"> WireCheck hit {len(internal)} internal error(s), so some requests may be missing. "
+                  f"First: `{_md(internal[0])}`", ""]
+    if pytest_exit not in (None, 0, 1, 5):
+        parts += [f"> pytest exited with code {pytest_exit}, so some tests may not have run. Check the job log.", ""]
+    elif pytest_data is not None and pytest_data.get("requests_checked", 0) == 0:
+        parts += [
+            "> Your tests sent no requests to the Claude API. If they replace `client.messages.create` with a mock, "
+            "no request is ever built. Mock the HTTP response instead (see the `wirecheck_stub` fixture).",
+            "",
+        ]
+
+    rows = group_findings(runtime)
+    if rows:
+        parts += [
+            "#### Requests your tests sent",
+            "",
+            "| Rule | Level | Count | Test | Call site | What |",
+            "|---|---|---:|---|---|---|",
+        ]
+        for r in rows:
+            parts.append(
+                f"| {r.rule_id} | {r.severity} | {r.count} | `{_md(r.test)}` | `{_md(r.call_site)}` | {_md(', '.join(r.where))} |"
+            )
+        parts.append("")
+
+    if static:
+        parts += ["#### Static scan", "", "| Rule | Level | File | What |", "|---|---|---|---|"]
+        for f in sorted(static, key=lambda x: ({"error": 0, "warn": 1}.get(x["severity"], 2), x["path"], x["line"])):
+            parts.append(f"| {f['rule_id']} | {f['severity']} | `{_md(f['path'])}:{f['line']}` | {_md(f['message'])} |")
+        parts.append("")
+
+    used = sorted({f["rule_id"] for f in list(runtime) + list(static)})
+    if used:
+        parts += ["<details><summary>What the rules mean</summary>", ""]
+        for rule_id in used:
+            if rule_id in RULES:
+                rule = RULES[rule_id]
+                parts.append(f"- **{rule_id}** {_md(rule.summary)} [Docs]({rule.doc})")
+            elif rule_id in SCAN_RULE_DOCS:
+                parts.append(f"- **{rule_id}** {SCAN_RULE_DOCS[rule_id]}")
+        parts += ["", "</details>", ""]
+
+    parts.append(f"<sub>[WireCheck]({REPO_URL}) v{__version__}</sub>")
+    return "\n".join(parts) + "\n"
+
+
+def dump_json(data: dict[str, Any], path: str) -> None:
+    with open(path, "w", encoding="utf-8") as fh:
+        json.dump(data, fh, indent=2)
+        fh.write("\n")