donkey-kit 0.1.0__py3-none-any.whl

donkey_kit/conformance/harness.py

@@ -0,0 +1,320 @@
+"""The observable ``Donkey`` the conformance suite runs against (#191, `BG §1.5`).
+
+The suite (:mod:`donkey_kit.conformance.suite`) says *what* to check; this
+module owns *how* it is observed. For each scenario it builds a fresh ``Donkey``,
+lets the customer's factory build an agent against it, arms the gateway to serve
+one captured fixture for every send, runs ``agent.run(...)`` while capturing what
+it raised and logged, and hands the resulting :class:`~.suite.Observation` to
+the scenario's check.
+
+It is the in-process sibling of the local gateway simulator, and it reuses the
+simulator's machinery so the injected refusal is the *same captured bytes*
+``classify()`` is tested against (:func:`donkey_kit.simulator.inject._resolve`
+gives us that fixture and asserts the classify() round-trip). The one deliberate
+difference from ``simulate()``'s transport: this one counts *every* wire send
+rather than deduping retries, because counting re-sends is exactly how the retry
+scenario is observed.
+
+Framework isolation (the layered architecture): imports only ``httpx``, the framework-free
+``core``/``donkey`` surface, and the sibling simulator loader — never a web
+framework and never ``openai`` at module top. It is a dev-only simulator-style
+sibling, kept out of the five production layers by an import-linter contract.
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+from collections.abc import Callable
+from contextlib import nullcontext
+from dataclasses import replace
+from typing import Any
+
+import httpx
+
+from ..core.config import DonkeyConfig
+from ..core.errors import DonkeyError
+from ..donkey import Donkey
+from ..simulator.app import SIMULATOR_HEADER
+from ..simulator.fixtures import Fixture, load, replay_headers
+
+# Reuse the simulator's single exception->fixture mapping (and its classify()
+# round-trip assertion) rather than duplicating the table: if the mapping ever
+# drifts, simulate() and this harness fail together — the "same files" rule.
+from ..simulator.inject import _resolve as _refusal_fixture
+from .suite import SCENARIOS, Observation, Outcome, Result, Scenario, validate_known_limitations
+
+__all__ = [
+    "DEFAULT_RUN_INPUT",
+    "ConformanceHarness",
+    "ConformanceUsageError",
+    "run_conformance",
+]
+
+# A neutral prompt handed to every agent under test. It is never actually sent
+# upstream — the transport is swapped — but agents that inspect their input want
+# a real string.
+DEFAULT_RUN_INPUT = "This is a Donkey conformance probe. Reply briefly."
+
+# Filled in only when the customer's environment does not already supply the LLM
+# proxy config. The harness never connects (the transport is swapped for every
+# scenario), so these just need to be non-empty to satisfy validated(need="llm").
+_PLACEHOLDER = "donkey-conformance"
+_PLACEHOLDER_URL = "https://donkey-conformance.invalid"
+
+AgentFactory = Callable[..., Any]
+
+
+class ConformanceUsageError(Exception):
+    """The agent under test does not meet the harness contract (e.g. no callable
+    ``run()``). A setup problem to fix, not a scenario finding — it is surfaced
+    as a hard error, never a fail verdict."""
+
+
+def _offline_config() -> DonkeyConfig:
+    """The customer's own config, with placeholder LLM credentials filled in
+    only where absent. Real config (base URL, model catalog) is preserved so the
+    agent is built as it would be in production; the placeholders exist purely so
+    ``donkey.openai()`` constructs without live credentials, since nothing here
+    ever reaches the network."""
+    cfg = DonkeyConfig.from_env()
+    # Replace with named str fields (not **dict) so the types stay checkable;
+    # ``x or <placeholder>`` keeps any value the customer has set and fills only
+    # the ones they have not.
+    return replace(
+        cfg,
+        llm_proxy_url=cfg.llm_proxy_url or _PLACEHOLDER_URL,
+        llm_proxy_client_id=cfg.llm_proxy_client_id or _PLACEHOLDER,
+        llm_proxy_client_secret=cfg.llm_proxy_client_secret or _PLACEHOLDER,
+    )
+
+
+def _fixture_responder(
+    fixture: Fixture, *, strip_budget: bool = False
+) -> Callable[[httpx.Request], httpx.Response]:
+    """Build a responder that replays one captured fixture, honesty-stamped.
+
+    Uses the same header allow-list (:func:`replay_headers`) and honesty stamp
+    (``x-donkey-simulator: true``) as ``simulate()`` and the mock server, so a
+    conformance response is never mistaken for a live gateway response. With
+    ``strip_budget`` the ``x-token-*`` headers are dropped, so the run observes a
+    genuinely unobserved budget even if a future capture grows them."""
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        headers = replay_headers(fixture)
+        if strip_budget:
+            headers = {k: v for k, v in headers.items() if not k.startswith("x-token-")}
+        if fixture.content_type is not None:
+            headers["content-type"] = fixture.content_type
+        headers[SIMULATOR_HEADER] = "true"
+        return httpx.Response(
+            status_code=fixture.status,
+            headers=headers,
+            content=fixture.body,
+            request=request,
+        )
+
+    return respond
+
+
+class _ProbeTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
+    """Serves a fixed captured response for every send, counting wire sends.
+
+    Deliberately does NOT dedupe by request identity the way ``simulate()``'s
+    transport does: the retry scenario is observed by counting re-sends, so every
+    ``handle_request`` increments :attr:`wire_sends`. Implements both transport
+    protocols so one instance fronts a ``DonkeyClient`` and a
+    ``DonkeyAsyncClient`` at once (and its count spans both)."""
+
+    def __init__(self, responder: Callable[[httpx.Request], httpx.Response]) -> None:
+        self._responder = responder
+        self.wire_sends = 0
+
+    def _serve(self, request: httpx.Request) -> httpx.Response:
+        self.wire_sends += 1
+        return self._responder(request)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        return self._serve(request)
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        return self._serve(request)
+
+
+# Standard LogRecord attribute names, so :func:`_render_records` can tell an
+# ``extra={...}`` field apart from the built-in ones and include it in the scan.
+_STD_LOGRECORD_KEYS = frozenset(vars(logging.makeLogRecord({})).keys())
+
+
+class _CaptureHandler(logging.Handler):
+    """Collects every record emitted anywhere during a run, for the correlation
+    scenario to grep."""
+
+    def __init__(self, sink: list[logging.LogRecord]) -> None:
+        super().__init__(level=logging.DEBUG)
+        self._sink = sink
+
+    def emit(self, record: logging.LogRecord) -> None:
+        self._sink.append(record)
+
+
+def _render_records(records: list[logging.LogRecord]) -> str:
+    """Flatten captured records into one searchable string: the formatted
+    message plus the stringified value of every ``extra=`` field, so an id logged
+    either in the message or via ``extra={"correlation_id": ...}`` is found."""
+    parts: list[str] = []
+    for record in records:
+        try:
+            parts.append(record.getMessage())
+        except Exception:  # noqa: BLE001 — a bad format string must not abort capture
+            parts.append(str(record.msg))
+        parts.extend(
+            str(value)
+            for key, value in record.__dict__.items()
+            if key not in _STD_LOGRECORD_KEYS
+        )
+    return "\n".join(parts)
+
+
+def _build_agent(factory: AgentFactory, donkey: Donkey) -> Any:
+    """Call the customer's factory, passing the ``Donkey`` iff it declares a slot
+    for it — the app-factory / dependency-injection pattern (``gunicorn
+    app:create_app``). ``build(donkey)`` and ``build(*, donkey=...)`` receive it;
+    a zero-argument ``build()`` does not."""
+    try:
+        params = inspect.signature(factory).parameters
+    except (TypeError, ValueError):
+        # Un-introspectable (some builtins/C callables): assume it wants donkey.
+        return factory(donkey)
+    if "donkey" in params and params["donkey"].kind is inspect.Parameter.KEYWORD_ONLY:
+        return factory(donkey=donkey)
+    accepts_positional = any(
+        p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        for p in params.values()
+    ) or any(p.kind is inspect.Parameter.VAR_POSITIONAL for p in params.values())
+    return factory(donkey) if accepts_positional else factory()
+
+
+class ConformanceHarness:
+    """Runs one scenario against a freshly built agent. Implements
+    :class:`~.suite.ScenarioContext`: the scenario check calls
+    :meth:`serve_refusal`/:meth:`serve_success` to arm the gateway and
+    :meth:`run` to drive the agent."""
+
+    def __init__(self, agent_factory: AgentFactory, *, run_input: str = DEFAULT_RUN_INPUT) -> None:
+        self._factory = agent_factory
+        self._run_input = run_input
+        self._donkey: Donkey | None = None
+        self._agent: Any = None
+        self._probe: _ProbeTransport | None = None
+
+    async def run_scenario(self, scenario: Scenario) -> Outcome:
+        """Build a fresh ``Donkey`` + agent, run the scenario's check against
+        this harness, and tear the ``Donkey`` down — so no transport swap, budget
+        state, or agent state leaks between scenarios."""
+        self._donkey = Donkey(_offline_config())
+        self._probe = None
+        try:
+            self._agent = _build_agent(self._factory, self._donkey)
+            return await scenario.check(self)
+        finally:
+            await self._donkey.aclose()
+            self._donkey = None
+            self._agent = None
+            self._probe = None
+
+    # --- ScenarioContext surface -------------------------------------------
+    @property
+    def agent(self) -> Any:
+        return self._agent
+
+    def serve_refusal(self, error: type[DonkeyError]) -> None:
+        self._arm(_fixture_responder(_refusal_fixture(error)))
+
+    def serve_success(self, *, budget_headers: bool = True) -> None:
+        self._arm(_fixture_responder(load("success"), strip_budget=not budget_headers))
+
+    async def run(self, *, correlation_id: str | None = None) -> Observation:
+        agent = self._agent
+        run = getattr(agent, "run", None)
+        if not callable(run):
+            raise ConformanceUsageError(
+                f"the agent your factory returned has no callable run(...) method "
+                f"(got {type(agent).__name__}); the conformance suite drives "
+                f"agent.run(<input>). Adapt your agent or expose a run() shim."
+            )
+        records: list[logging.LogRecord] = []
+        handler = _CaptureHandler(records)
+        root = logging.getLogger()
+        previous_level = root.level
+        root.addHandler(handler)
+        root.setLevel(logging.DEBUG)  # let INFO/DEBUG records reach the handler
+        raised: BaseException | None = None
+        returned: Any = None
+        bind = (
+            self._donkey.run_context(correlation_id)
+            if correlation_id is not None and self._donkey is not None
+            else nullcontext()
+        )
+        try:
+            with bind:
+                result = run(self._run_input)
+                returned = await result if inspect.isawaitable(result) else result
+        except ConformanceUsageError:
+            raise
+        except Exception as exc:  # noqa: BLE001 — we observe whatever the agent lets escape
+            raised = exc
+        finally:
+            root.removeHandler(handler)
+            root.setLevel(previous_level)
+        return Observation(
+            returned=returned,
+            raised=raised,
+            wire_sends=self._probe.wire_sends if self._probe is not None else 0,
+            log_text=_render_records(records),
+        )
+
+    def _arm(self, responder: Callable[[httpx.Request], httpx.Response]) -> None:
+        """Swap a counting probe onto both transports. The sync client is built
+        eagerly (it opens no connection until used) so a blocking agent's client
+        is armed too, and both share one probe so the wire-send count spans
+        async and sync."""
+        donkey = self._donkey
+        assert donkey is not None  # set by run_scenario before any check runs
+        self._probe = _ProbeTransport(responder)
+        donkey._http._swap_transport(self._probe)
+        donkey._sync_http_client()._swap_transport(self._probe)
+
+
+async def run_conformance(
+    agent_factory: AgentFactory,
+    *,
+    known_limitations: object = None,
+    run_input: str = DEFAULT_RUN_INPUT,
+) -> list[Result]:
+    """Run every scenario against ``agent_factory`` and return the table rows.
+
+    A scenario listed in ``known_limitations`` is reported ``exempt`` with its
+    asserted reason and is *not* run (the exemption replaces the check); every
+    other scenario runs and is reported ``pass``/``fail``. ``known_limitations``
+    is validated up front, so a bad key or empty reason raises before any agent
+    is built."""
+    known = validate_known_limitations(known_limitations)
+    results: list[Result] = []
+    for scenario in SCENARIOS:
+        if scenario.name in known:
+            results.append(
+                Result(scenario.name, scenario.title, "exempt", known[scenario.name])
+            )
+            continue
+        harness = ConformanceHarness(agent_factory, run_input=run_input)
+        outcome = await harness.run_scenario(scenario)
+        results.append(
+            Result(
+                scenario.name,
+                scenario.title,
+                "pass" if outcome.passed else "fail",
+                outcome.detail,
+            )
+        )
+    return results

donkey_kit/conformance/plugin.py

@@ -0,0 +1,274 @@
+"""The public pytest plugin — ``pytest --donkey-conformance`` (#191, `BG §1.5`).
+
+This is the deliverable the milestone's centre of gravity moved to: not our
+internal adapter matrix, but a suite a **customer runs against their own agent**::
+
+    pip install "donkey-kit[test]"
+    pytest --donkey-conformance --agent=my_app.agent:build
+
+It registers via a ``pytest11`` entry point, so it auto-loads on *every* pytest
+run in an environment where ``donkey-kit[test]`` is installed — including this
+repo's own base-only job. That constraint drives the two rules this module
+follows without exception:
+
+1. **It is inert unless asked.** With no ``--donkey-conformance`` flag,
+   :func:`pytest_collection` returns ``None`` and default collection runs
+   untouched; the only footprint is three CLI options and one unused fixture.
+2. **Its module top is import-light.** It imports only stdlib, ``pytest``, and
+   the framework-free :mod:`~.suite`/:mod:`~.report` siblings. The heavy
+   harness (which imports :class:`~donkey_kit.donkey.Donkey`) and ``asyncio``
+   are imported lazily, inside the functions that need them, so merely loading
+   the plugin never drags the full SDK — or a framework — onto the import path.
+
+When the flag is set, ``--donkey-conformance`` runs the conformance suite
+*exclusively* (it replaces normal collection), turning each scenario into one
+pytest item and printing the scenario→status table in the terminal summary.
+"""
+
+from __future__ import annotations
+
+import importlib
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from .report import render_report
+from .suite import SCENARIOS, Result, Scenario, validate_known_limitations
+
+if TYPE_CHECKING:  # keep the heavy import out of the auto-loaded module body
+    from .harness import AgentFactory
+
+# Stash keys carry collection-time state to the run/summary phases without
+# module globals (pytest recommends config.stash over ad-hoc attributes).
+_FACTORY_KEY = pytest.StashKey[Callable[..., Any]]()
+_KNOWN_KEY = pytest.StashKey[dict[str, str]]()
+_RESULTS_KEY = pytest.StashKey[list[Result]]()
+
+
+class _ConformanceFailure(AssertionError):
+    """A scenario's FAIL verdict, rendered as the item's failure message with no
+    Python traceback — the finding about the agent *is* the message."""
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("donkey", "Donkey conformance")
+    group.addoption(
+        "--donkey-conformance",
+        action="store_true",
+        default=False,
+        dest="donkey_conformance",
+        help="Run the Donkey conformance suite against your agent "
+        "(requires --agent). Replaces normal test collection.",
+    )
+    group.addoption(
+        "--agent",
+        action="store",
+        default=None,
+        dest="donkey_agent",
+        metavar="MODULE:FACTORY",
+        help="Import path to your agent factory, e.g. my_app.agent:build. "
+        "Called once per scenario; receives the donkey fixture if it declares one.",
+    )
+    group.addoption(
+        "--donkey-known-limitations",
+        action="store",
+        default=None,
+        dest="donkey_known_limitations",
+        metavar="MODULE:NAME",
+        help="Import path to a KNOWN_LIMITATIONS dict of {scenario: reason}. "
+        "Defaults to a KNOWN_LIMITATIONS attribute in the --agent module if present.",
+    )
+
+
+def _import_target(spec: str, *, what: str) -> tuple[Any, Any]:
+    """Resolve a ``module:attribute`` spec, raising :class:`pytest.UsageError`
+    (a clean CLI error, no traceback) for a malformed spec, an unimportable
+    module, or a missing attribute."""
+    if ":" not in spec:
+        raise pytest.UsageError(
+            f"{what} must be given as 'module:attribute', got {spec!r}"
+        )
+    mod_name, _, attr = spec.partition(":")
+    if not mod_name or not attr:
+        raise pytest.UsageError(
+            f"{what} must be given as 'module:attribute', got {spec!r}"
+        )
+    try:
+        module = importlib.import_module(mod_name)
+    except ImportError as exc:
+        raise pytest.UsageError(f"could not import {mod_name!r} for {what}: {exc}") from exc
+    try:
+        return module, getattr(module, attr)
+    except AttributeError as exc:
+        raise pytest.UsageError(
+            f"module {mod_name!r} has no attribute {attr!r} for {what}"
+        ) from exc
+
+
+def _resolve_agent(config: pytest.Config) -> tuple[AgentFactory, dict[str, str]]:
+    """Resolve the ``--agent`` factory and its known-limitations mapping, and
+    validate the exemptions **now** (collection time) so a bad key or an empty
+    reason fails loudly before any scenario runs (the conformance kit: asserted, never silent)."""
+    agent_spec = config.getoption("donkey_agent")
+    if not agent_spec:
+        raise pytest.UsageError(
+            "--donkey-conformance requires --agent=module:factory "
+            "(the import path to your agent factory)"
+        )
+    module, factory = _import_target(agent_spec, what="the agent factory (--agent)")
+    if not callable(factory):
+        raise pytest.UsageError(f"--agent target {agent_spec!r} is not callable")
+
+    known_spec = config.getoption("donkey_known_limitations")
+    if known_spec:
+        _, known_obj = _import_target(
+            known_spec, what="known limitations (--donkey-known-limitations)"
+        )
+    else:
+        # Auto-discover: a KNOWN_LIMITATIONS attribute next to the factory.
+        known_obj = getattr(module, "KNOWN_LIMITATIONS", None)
+
+    try:
+        known = validate_known_limitations(known_obj)
+    except (TypeError, ValueError) as exc:
+        raise pytest.UsageError(str(exc)) from exc
+    return factory, known
+
+
+class _ScenarioItem(pytest.Item):
+    """One conformance scenario as a pytest item. All scenarios share a single
+    :func:`~donkey_kit.conformance.harness.run_conformance` pass (memoised on
+    the config); each item reports its own row's verdict."""
+
+    def __init__(self, *, name: str, parent: Any, scenario: Scenario) -> None:
+        super().__init__(name, parent)
+        self._scenario = scenario
+
+    def runtest(self) -> None:
+        result = _result_for(self.config, self._scenario.name)
+        if result.status == "fail":
+            raise _ConformanceFailure(f"{self._scenario.title}: {result.detail}")
+        # pass and exempt both leave the item green; the asserted exemption is
+        # validated at collection and shown in the summary table, so it is
+        # recorded, not silently skipped (the conformance kit).
+        if result.status == "exempt":
+            self.add_report_section("call", "exempt", result.detail)
+
+    def repr_failure(self, excinfo: Any, style: Any = None) -> Any:
+        if isinstance(excinfo.value, _ConformanceFailure):
+            return str(excinfo.value)
+        return super().repr_failure(excinfo)
+
+    def reportinfo(self) -> tuple[Any, int, str]:
+        return self.path, 0, f"conformance: {self._scenario.title}"
+
+
+class _ConformanceCollector(pytest.Collector):
+    """A single synthetic collector so item node ids read
+    ``donkey-conformance::<scenario>`` rather than dangling off the session."""
+
+    def collect(self) -> Any:
+        for scenario in SCENARIOS:
+            yield _ScenarioItem.from_parent(self, name=scenario.name, scenario=scenario)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_collection(session: pytest.Session) -> bool | None:
+    """When ``--donkey-conformance`` is set, replace normal collection with the
+    conformance items. Returning ``None`` (the default) leaves ordinary
+    collection untouched, so the plugin is inert on every other pytest run."""
+    config = session.config
+    if not config.getoption("donkey_conformance"):
+        return None
+    factory, known = _resolve_agent(config)  # UsageError here aborts cleanly
+    config.stash[_FACTORY_KEY] = factory
+    config.stash[_KNOWN_KEY] = known
+    collector = _ConformanceCollector.from_parent(session, name="donkey-conformance")
+    session.items = list(session.genitems(collector))
+    session.testscollected = len(session.items)
+    return True
+
+
+def _result_for(config: pytest.Config, scenario_name: str) -> Result:
+    """Return the :class:`Result` for one scenario, running the whole suite once
+    (lazily, on the first item) and caching it on the config stash so every item
+    and the terminal summary share the same single run."""
+    results = config.stash.get(_RESULTS_KEY, None)
+    if results is None:
+        import asyncio
+
+        from .harness import run_conformance
+
+        factory = config.stash[_FACTORY_KEY]
+        known = config.stash[_KNOWN_KEY]
+        results = asyncio.run(run_conformance(factory, known_limitations=known))
+        config.stash[_RESULTS_KEY] = results
+    for result in results:
+        if result.scenario == scenario_name:
+            return result
+    # Unreachable: every scenario has a row. Fail loudly rather than silently.
+    raise KeyError(f"no conformance result for scenario {scenario_name!r}")
+
+
+def pytest_terminal_summary(
+    terminalreporter: Any, exitstatus: int, config: pytest.Config
+) -> None:
+    """Print the scenario→status table once, after the run. Only fires when the
+    conformance suite actually ran (results are cached on the stash)."""
+    results = config.stash.get(_RESULTS_KEY, None)
+    if not results:
+        return
+    terminalreporter.write_line("")
+    for line in render_report(results).splitlines():
+        terminalreporter.write_line(line)
+
+
+@pytest.fixture
+def donkey() -> Any:
+    """A :class:`~donkey_kit.donkey.Donkey` for tests that drive it directly
+    (e.g. ``with donkey.simulate(PIIDetected): ...``).
+
+    Built from the environment, with placeholder LLM credentials filled in only
+    where absent — so ``donkey.openai()`` constructs and ``simulate()`` works
+    with no live gateway and no env setup, while any real config you *have* set
+    is preserved. Torn down synchronously; the async transport opens no
+    connection unless a real request is made (which ``simulate()`` intercepts)."""
+    from ..donkey import Donkey
+    from .harness import _offline_config
+
+    fab = Donkey(_offline_config())
+    try:
+        yield fab
+    finally:
+        fab.close()
+
+
+@pytest.fixture
+def gateway() -> Any:
+    """A running local gateway simulator on an ephemeral port, for tests whose
+    subject does **not** import ``donkey_kit`` — a containerised agent, a Node
+    service, an A2A client, a manual ``curl`` (#278)::
+
+        async def test_agent_stops_after_refusal(gateway):
+            gateway.set_scenarios("pii_block:every=1")
+            app = deploy(env={"DONKEY_LLM_PROXY_URL": gateway.url})
+            await app.run(ticket)
+            assert gateway.requests_received == 1  # stopped, did not retry
+
+    It serves the same captured rejection fixtures ``simulate()`` and ``donkey
+    mock`` replay, on a real port bound to ``0`` (so parallel ``pytest -n`` runs
+    never collide), and records every request in a spy the test can assert on
+    (:attr:`~.gateway.Gateway.requests_received` plus per-request method, path and
+    headers with ``client_secret`` redacted). Torn down when the test exits,
+    including on failure.
+
+    Needs the ``[local]`` extra (``starlette`` + ``uvicorn``); taking the fixture
+    without it raises an :class:`ImportError` naming the exact ``pip install``."""
+    from .gateway import start_gateway
+
+    gw = start_gateway()
+    try:
+        yield gw
+    finally:
+        gw.close()

donkey_kit/conformance/report.py

@@ -0,0 +1,85 @@
+"""Rendering the conformance :class:`~.suite.Result` list as a table (#191).
+
+Dependency-free on purpose: the pytest plugin calls this from
+``pytest_terminal_summary`` and a programmatic caller can render the same table
+without pytest installed. No colour codes, no terminal-width probing — a plain
+monospace block that reads the same in CI logs, a terminal, and a PR comment.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from .suite import Result, Status
+
+__all__ = ["StatusCounts", "count_statuses", "format_status", "render_report"]
+
+# The report is the SDK "telling the team something true about their own code",
+# so the glyphs are unambiguous in a plain CI log — no colour dependency.
+_GLYPH: dict[Status, str] = {"pass": "PASS", "fail": "FAIL", "exempt": "EXEMPT"}
+
+
+class StatusCounts(dict[Status, int]):
+    """Counts per status with convenience accessors, so callers read
+    ``counts.failed`` rather than indexing a bare dict."""
+
+    @property
+    def passed(self) -> int:
+        return self.get("pass", 0)
+
+    @property
+    def failed(self) -> int:
+        return self.get("fail", 0)
+
+    @property
+    def exempt(self) -> int:
+        return self.get("exempt", 0)
+
+    @property
+    def total(self) -> int:
+        return self.passed + self.failed + self.exempt
+
+
+def count_statuses(results: Sequence[Result]) -> StatusCounts:
+    counts = StatusCounts()
+    for result in results:
+        counts[result.status] = counts.get(result.status, 0) + 1
+    return counts
+
+
+def format_status(status: Status) -> str:
+    """The fixed-width label for one status (``PASS``/``FAIL``/``EXEMPT``)."""
+    return _GLYPH.get(status, status.upper())
+
+
+def _summary_line(counts: StatusCounts) -> str:
+    parts = [f"{counts.passed} passed", f"{counts.failed} failed"]
+    if counts.exempt:
+        parts.append(f"{counts.exempt} exempt")
+    return f"Donkey conformance: {', '.join(parts)}"
+
+
+def render_report(results: Sequence[Result], *, title: str = "Donkey conformance") -> str:
+    """Render the results as a titled, aligned status table with a summary line.
+
+    Each row is ``<STATUS>  <scenario title> — <detail>``. Rows are shown in the
+    order given (the suite's canonical scenario order), so the table reads the
+    same across runs. An empty result set renders the header and a single
+    ``(no scenarios ran)`` line rather than a bare title, so a misconfigured run
+    is obvious instead of silent."""
+    lines = [title, "=" * len(title)]
+    if not results:
+        lines.append("(no scenarios ran)")
+        return "\n".join(lines)
+
+    status_width = max(len(format_status(r.status)) for r in results)
+    title_width = max(len(r.title) for r in results)
+    for result in results:
+        label = format_status(result.status).ljust(status_width)
+        name = result.title.ljust(title_width)
+        detail = f" — {result.detail}" if result.detail else ""
+        lines.append(f"{label}  {name}{detail}")
+
+    lines.append("")
+    lines.append(_summary_line(count_statuses(results)))
+    return "\n".join(lines)