evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/core/evaluators/trajectory_rubrics.py

@@ -0,0 +1,198 @@
+"""trajectory_rubrics — checkpoint rubrics graded over the recorded trajectory.
+
+One judge call over the full ATIF trajectory: every tool call, every tool
+result and the agent's reasoning, with nothing truncated. Sized against 100
+production sessions, a single eval run renders to ~45k chars at the median and
+under 400k at the worst turn seen; compact ATIF JSON is 7-17% larger, so one
+call fits.
+
+    score = satisfied / total
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from evalkit.contracts.atif import Trajectory
+from evalkit.contracts.cases import TrajectoryRubrics
+from evalkit.core.evaluators.base import EvalContext, builtin_evaluator, neutral_skip
+from evalkit.core.fixtures import compose_input
+from evalkit.core.judge import call_judge
+from evalkit.store.base import ScoreRow
+
+LOGGER = logging.getLogger(__name__)
+
+SCORE_NAME = "trajectory_rubric_score"
+
+
+class TrajectoryVerdict(BaseModel):
+    index: int = Field(
+        description="The number this rubric was listed under in the prompt, counting from 1."
+    )
+    satisfied: bool = Field(
+        description="True only when the trajectory or the final answer contain"
+        " positive evidence for the rubric."
+    )
+    evidence: str = Field(
+        default="",
+        description='Where it shows: "main:4-5" for steps of the root trajectory,'
+        ' "<trajectory_id>:7" for a subagent, or "final answer". Empty when not'
+        " satisfied.",
+    )
+    note: str = Field(
+        default="",
+        description="One line on why this verdict, when not self-evident.",
+    )
+
+
+class TrajectoryJudgeOutput(BaseModel):
+    verdicts: list[TrajectoryVerdict]
+
+
+SYSTEM_PROMPT = """\
+You are an impartial grader for a financial-research agent. You are given the
+question the agent was asked, a numbered list of rubrics describing checkpoints
+its work should show, its recorded trajectory, and its final answer. You return
+one verdict per rubric. You are strict but fair: a rubric is satisfied only when
+the trajectory or the final answer contain positive evidence for it. Silence is
+not satisfaction; absence of contradiction is not satisfaction."""
+
+_LEGEND = """\
+The trajectory is an ATIF document. Steps run in step_id order. A tool call's
+result is in that step's observation.results, matched by source_call_id; a
+result with extra.is_error = true is a failed call. Delegated work is in
+subagent_trajectories, each a trajectory of its own.
+
+For each rubric return satisfied = true or false.
+
+  true   the trajectory, or the final answer, shows the rubric was met. Cite
+         where: "main:4-5" for steps 4-5 of the root trajectory,
+         "<trajectory_id>:7" for step 7 of a subagent (use
+         "subagent_trajectories[0]:7" when it has no trajectory_id), or
+         "final answer".
+  false  no positive evidence anywhere in the trajectory or the final answer.
+
+Match semantically: an action satisfies a rubric regardless of what the tool or
+mechanism behind it is called. A figure satisfies a rubric when it equals the
+rubric's figure after unit conversion, accepting normal rounding unless the
+rubric states its own band. Grade every rubric independently of the others."""
+
+
+def trajectory_json(trajectory: Trajectory) -> str:
+    return json.dumps(
+        trajectory.model_dump(exclude_none=True), ensure_ascii=False, separators=(",", ":")
+    )
+
+
+def user_prompt(question: str, rubrics: list[str], trajectory: str, answer: str) -> str:
+    numbered = "\n".join(f"{i}. {r}" for i, r in enumerate(rubrics, 1))
+    return "\n".join(
+        [
+            _LEGEND,
+            "",
+            "QUESTION",
+            question,
+            "",
+            "RUBRICS",
+            numbered,
+            "",
+            "TRAJECTORY (ATIF JSON)",
+            trajectory,
+            "",
+            "FINAL ANSWER",
+            answer,
+        ]
+    )
+
+
+class IncompleteVerdicts(Exception):
+    """The judge did not verdict every rubric exactly once, even after a retry."""
+
+
+def _missing(total: int, parsed: TrajectoryJudgeOutput) -> list[int]:
+    seen: dict[int, int] = {}
+    for v in parsed.verdicts:
+        seen[v.index] = seen.get(v.index, 0) + 1
+    return [i for i in range(1, total + 1) if seen.get(i, 0) != 1]
+
+
+async def _judge(ctx: EvalContext, base_prompt: str, total: int) -> TrajectoryJudgeOutput:
+    current = base_prompt
+    missing: list[int] = []
+    for attempt in (1, 2):
+        parsed, usage = await call_judge(
+            ctx.judge_cfg,
+            api_key=ctx.judge_api_key,
+            system_prompt=SYSTEM_PROMPT,
+            user_prompt=current,
+            output_type=TrajectoryJudgeOutput,
+            session_id=ctx.session_id,
+            evaluator_name="trajectory_rubrics",
+        )
+        ctx.record_judge_usage(usage)
+        missing = _missing(total, parsed)
+        if not missing:
+            return parsed
+        LOGGER.warning(
+            "evals.trajectory_rubrics.incomplete_verdicts attempt=%s missing=%s", attempt, missing
+        )
+        current = (
+            f"{base_prompt}\n\nYour previous response did not return exactly one verdict for "
+            f"rubrics {missing}. Return one verdict per rubric, for every rubric from 1 to {total}."
+        )
+    raise IncompleteVerdicts(f"judge left rubrics {missing} unverdicted after a retry")
+
+
+def _explanation(rubrics: list[str], verdicts: list[TrajectoryVerdict]) -> str:
+    by_index = {v.index: v for v in verdicts}
+    rows = []
+    for index, rubric in enumerate(rubrics, 1):
+        v = by_index[index]
+        rows.append(
+            {
+                "index": index,
+                "rubric": rubric,
+                "verdict": "satisfied" if v.satisfied else "unsatisfied",
+                "evidence": v.evidence if v.satisfied else "",
+                "note": v.note,
+            }
+        )
+    return json.dumps(
+        {
+            "satisfied": sum(1 for v in verdicts if v.satisfied),
+            "total": len(rubrics),
+            "verdicts": rows,
+        },
+        ensure_ascii=False,
+    )
+
+
+@builtin_evaluator(
+    "trajectory_rubrics",
+    requires={"trajectory", "tool_calls"},
+    emits=(SCORE_NAME,),
+    uses_judge=True,
+)
+async def trajectory_rubrics(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    cfg = TrajectoryRubrics.model_validate(config)
+    if ctx.trajectory_view is None:
+        return [neutral_skip(SCORE_NAME, "no trajectory recorded for this run")]
+
+    trajectory = trajectory_json(ctx.trajectory_view.trajectory)
+    question = str(compose_input(ctx.case.input, ctx.case.files).get("question", ""))
+    parsed = await _judge(
+        ctx, user_prompt(question, cfg.rubrics, trajectory, ctx.answer), len(cfg.rubrics)
+    )
+    score = sum(1 for v in parsed.verdicts if v.satisfied) / len(cfg.rubrics)
+    return [
+        ScoreRow(
+            name=SCORE_NAME,
+            score=score,
+            explanation=_explanation(cfg.rubrics, parsed.verdicts),
+            annotator_kind="LLM",
+        )
+    ]

evalkit/core/fixtures.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from evalkit.contracts.cases import FixtureEntry, load_registry
+from evalkit.core.lint import Finding
+from evalkit.store.base import CaseFile
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def load_registries(paths: list[Path]) -> tuple[dict[str, FixtureEntry], list[Finding]]:
+    entries: dict[str, FixtureEntry] = {}
+    findings: list[Finding] = []
+    for path in paths:
+        if not path.is_file():
+            continue
+        try:
+            registry = load_registry(path)
+        except (ValidationError, yaml.YAMLError) as exc:
+            findings.append(
+                Finding(severity="error", code="E-SCHEMA", case_id=path.name, message=str(exc))
+            )
+            continue
+        for entry in registry.fixtures:
+            if entry.key in entries:
+                findings.append(
+                    Finding(
+                        severity="error",
+                        code="E-FIXTURE-DUPLICATE-KEY",
+                        case_id=entry.key,
+                        message=f"fixture key {entry.key!r} is declared more than once ({path})",
+                    )
+                )
+                continue
+            entries[entry.key] = entry
+    return entries, findings
+
+
+ATTACHED_FILES_HEADER = "Attached files:"
+
+
+def compose_input(case_input: dict[str, Any], files: list[CaseFile]) -> dict[str, Any]:
+    out = dict(case_input)
+    question = str(case_input.get("question", ""))
+    if not files or question.startswith(ATTACHED_FILES_HEADER):
+        return out
+    lines = [ATTACHED_FILES_HEADER] + [f"- {f.key}: {f.context}" for f in files]
+    out["question"] = "\n".join(lines) + "\n\n" + question
+    return out

evalkit/core/harness.py

@@ -0,0 +1,227 @@
+import asyncio
+import json
+import logging
+import time
+from collections.abc import AsyncIterator, Sequence
+from dataclasses import dataclass
+from typing import IO
+from urllib.parse import quote
+
+import httpx
+
+from evalkit.contracts.atif import Trajectory, validate_trajectory
+from evalkit.contracts.harness import (
+    Capabilities,
+    FileList,
+    FileRef,
+    RunList,
+    RunListParams,
+    RunMetadata,
+    RunStatus,
+)
+from evalkit.core.agents import AgentDefinition
+from evalkit.core.paths import require_safe_path, require_safe_run_id
+
+LOGGER = logging.getLogger(__name__)
+
+TERMINAL_STATUSES = {"completed", "failed", "cancelled"}
+GET_ATTEMPTS = 3
+STALE_CONNECTION_ERRORS = (httpx.RemoteProtocolError, httpx.ReadError)
+# Shorter than the 5s idle keep-alive of uvicorn and Node, so evalkit drops an
+# idle connection before the harness closes it under an in-flight poll.
+KEEPALIVE_EXPIRY_SECONDS = 2
+
+
+@dataclass
+class HarnessFile:
+    size: int | None
+    chunks: AsyncIterator[bytes]
+
+
+async def _response_chunks(resp: httpx.Response) -> AsyncIterator[bytes]:
+    try:
+        async for chunk in resp.aiter_bytes():
+            yield chunk
+    finally:
+        await resp.aclose()
+
+
+class RunTimeout(Exception):
+    def __init__(self, run_id: str, last_status: RunStatus | None):
+        self.run_id = run_id
+        self.last_status = last_status
+        super().__init__(f"run {run_id} did not reach a terminal status in time")
+
+
+def _raise_for_status(resp: httpx.Response) -> None:
+    if resp.is_success:
+        return
+    message = f"harness returned {resp.status_code} for {resp.request.method} {resp.request.url}"
+    body = resp.text.strip()
+    if body:
+        message += f": {body[:1000]}"
+    raise httpx.HTTPStatusError(message, request=resp.request, response=resp)
+
+
+class HarnessClient:
+    def __init__(
+        self,
+        agent: AgentDefinition,
+        api_key: str | None,
+        *,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ):
+        assert agent.harness_url is not None
+        self.agent = agent
+        self._client = httpx.AsyncClient(
+            base_url=agent.harness_url.rstrip("/"),
+            headers={"x-api-key": api_key} if api_key else {},
+            timeout=60,
+            limits=httpx.Limits(keepalive_expiry=KEEPALIVE_EXPIRY_SECONDS),
+            transport=transport,
+        )
+        self._capabilities: Capabilities | None = None
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def _get(self, url: str, *, stream: bool = False, **kwargs) -> httpx.Response:
+        for attempt in range(1, GET_ATTEMPTS + 1):
+            try:
+                request = self._client.build_request("GET", url, **kwargs)
+                return await self._client.send(request, stream=stream)
+            except STALE_CONNECTION_ERRORS as exc:
+                if attempt == GET_ATTEMPTS:
+                    raise
+                LOGGER.warning(
+                    "harness GET %s disconnected (attempt %s/%s), retrying: %r",
+                    url,
+                    attempt,
+                    GET_ATTEMPTS,
+                    exc,
+                )
+        raise AssertionError("unreachable")
+
+    async def capabilities(self) -> Capabilities:
+        if self._capabilities is None:
+            resp = await self._get("/capabilities")
+            _raise_for_status(resp)
+            self._capabilities = Capabilities.model_validate(resp.json())
+        return self._capabilities
+
+    async def start_run(
+        self,
+        input: dict,
+        files: Sequence[tuple[str, bytes | IO[bytes]]] | None = None,
+        file_refs: list[FileRef] | None = None,
+        model: str | None = None,
+        metadata: RunMetadata | None = None,
+    ) -> str:
+        payload: dict = {"input": input}
+        if model is not None:
+            payload["model"] = model
+        if metadata is not None:
+            payload["metadata"] = metadata.model_dump(exclude_none=True)
+        if file_refs:
+            payload["files"] = [ref.model_dump(exclude_none=True) for ref in file_refs]
+        if files:
+            parts: list[
+                tuple[str, tuple[str | None, str | bytes, str] | tuple[str, bytes | IO[bytes]]]
+            ] = [("payload", (None, json.dumps(payload), "application/json"))]
+            parts += [("files", (name, data)) for name, data in files]
+            resp = await self._client.post("/runs", files=parts)
+        else:
+            resp = await self._client.post("/runs", json=payload)
+        _raise_for_status(resp)
+        return resp.json()["run_id"]
+
+    async def status(self, run_id: str) -> RunStatus:
+        resp = await self._get(f"/runs/{_run_segment(run_id)}")
+        _raise_for_status(resp)
+        return RunStatus.model_validate(resp.json())
+
+    async def poll_until_terminal(self, run_id: str) -> RunStatus:
+        deadline = time.monotonic() + self.agent.settings.run_timeout_seconds
+        last: RunStatus | None = None
+        while time.monotonic() < deadline:
+            last = await self.status(run_id)
+            if last.status in TERMINAL_STATUSES:
+                return last
+            await asyncio.sleep(self.agent.settings.poll_interval_seconds)
+        caps = await self.capabilities()
+        if caps.accepts.stop:
+            await self.stop(run_id)
+            last = await self._poll_after_stop(run_id, last)
+        else:
+            LOGGER.warning(
+                "run %s timed out and harness does not accept stop — resources may leak",
+                run_id,
+            )
+        raise RunTimeout(run_id, last)
+
+    async def _poll_after_stop(self, run_id: str, last: RunStatus | None) -> RunStatus | None:
+        deadline = time.monotonic() + self.agent.settings.stop_grace_seconds
+        while time.monotonic() < deadline:
+            last = await self.status(run_id)
+            if last.status in TERMINAL_STATUSES:
+                return last
+            await asyncio.sleep(self.agent.settings.poll_interval_seconds)
+        LOGGER.warning(
+            "run %s not terminal %ss after stop; fetching its trajectory anyway",
+            run_id,
+            self.agent.settings.stop_grace_seconds,
+        )
+        return last
+
+    async def stop(self, run_id: str) -> None:
+        resp = await self._client.post(f"/runs/{_run_segment(run_id)}/stop")
+        _raise_for_status(resp)
+
+    async def trajectory(self, run_id: str) -> Trajectory:
+        resp = await self._get(f"/runs/{_run_segment(run_id)}/trajectory")
+        _raise_for_status(resp)
+        return validate_trajectory(resp.json())
+
+    async def list_runs(self, params: RunListParams | None = None) -> RunList:
+        resp = await self._get("/runs", params=(params or RunListParams()).as_query())
+        _raise_for_status(resp)
+        return RunList.model_validate(resp.json())
+
+    async def list_files(self, run_id: str) -> FileList:
+        resp = await self._get(f"/runs/{_run_segment(run_id)}/files")
+        _raise_for_status(resp)
+        return FileList.model_validate(resp.json())
+
+    async def _file_response(self, run_id: str, path: str) -> httpx.Response:
+        return await self._get(f"/runs/{_run_segment(run_id)}/files/{_file_path(path)}")
+
+    async def find_file(self, run_id: str, path: str) -> bytes | None:
+        resp = await self._file_response(run_id, path)
+        if resp.status_code == 404:
+            return None
+        _raise_for_status(resp)
+        return resp.content
+
+    async def get_file(self, run_id: str, path: str) -> bytes:
+        resp = await self._file_response(run_id, path)
+        _raise_for_status(resp)
+        return resp.content
+
+    async def stream_file(self, run_id: str, path: str) -> HarnessFile:
+        url = f"/runs/{_run_segment(run_id)}/files/{_file_path(path)}"
+        resp = await self._get(url, stream=True)
+        if not resp.is_success:
+            await resp.aread()
+            await resp.aclose()
+            _raise_for_status(resp)
+        length = resp.headers.get("content-length")
+        size = int(length) if length and "content-encoding" not in resp.headers else None
+        return HarnessFile(size=size, chunks=_response_chunks(resp))
+
+
+def _run_segment(run_id: str) -> str:
+    return quote(require_safe_run_id(run_id), safe="")
+
+
+def _file_path(path: str) -> str:
+    return "/".join(quote(segment, safe="") for segment in require_safe_path(path).split("/"))

evalkit/core/harness_check.py

@@ -0,0 +1,191 @@
+"""Live verification of a client's Harness API implementation.
+
+Runs the spec's checks and returns a per-capability pass/fail table:
+capabilities shape, the probe input against the declared input_schema, a
+trivial run to a terminal status (carrying one input file where accepted),
+trajectory ATIF validation, then workspace files / run listing / stop where
+declared.
+"""
+
+from datetime import UTC, datetime, timedelta
+
+import httpx
+
+from evalkit.contracts.atif import TrajectoryValidationError, subagent_link_warnings
+from evalkit.contracts.harness import FileRef, RunListParams, input_errors
+from evalkit.core.agents import AgentDefinition
+from evalkit.core.blob import BlobStore
+from evalkit.core.harness import HarnessClient, RunTimeout
+from evalkit.core.paths import UnsafePathError
+from evalkit.core.traces.poll import poll_trajectory
+from evalkit.core.traces.sessions import SessionReader
+
+CHECK_INPUT = {"question": "Hi. Reply with one word."}
+CHECK_FILE = ("evalkit-check.txt", b"evalkit harness check\n")
+CHECK_FILE_CONTEXT = "A one-line text file evalkit attaches to its harness check."
+CHECK_BLOB_KEY = "checks/evalkit-check.txt"
+IGNORES_TIME_FILTER = "ignores since/until; evalkit filters the runs itself"
+
+
+def _entry(status: str, detail: str) -> dict:
+    return {"status": status, "detail": detail}
+
+
+async def check_harness(
+    agent: AgentDefinition,
+    api_key: str | None,
+    blob_store: BlobStore,
+    *,
+    transport: httpx.AsyncBaseTransport | None = None,
+    sessions: SessionReader | None = None,
+    agent_slug: str | None = None,
+) -> dict[str, dict]:
+    client = HarnessClient(agent, api_key, transport=transport)
+    report: dict[str, dict] = {}
+    try:
+        try:
+            caps = await client.capabilities()
+            detail = f"contract_version={caps.contract_version} agent={caps.agent}"
+            if caps.revision:
+                rev = caps.revision[:8]
+                detail += f" revision={rev}"
+                if caps.branch:
+                    detail += f" branch={caps.branch}"
+            report["capabilities"] = _entry("pass", detail)
+        except (httpx.HTTPError, ValueError) as exc:
+            report["capabilities"] = _entry("fail", str(exc))
+            return report
+
+        probe = caps.check_input or CHECK_INPUT
+        schema_name = "input_schema" if caps.input_schema else "the default input schema"
+        if caps.check_input is not None:
+            report["input_schema"] = _entry("pass", f"check_input valid against {schema_name}")
+        elif errors := input_errors(caps.effective_input_schema, CHECK_INPUT):
+            report["input_schema"] = _entry(
+                "fail",
+                f"default probe {CHECK_INPUT} fails input_schema ({errors[0]}); "
+                "declare check_input",
+            )
+        else:
+            report["input_schema"] = _entry("pass", f"default probe valid against {schema_name}")
+
+        input_files: list[tuple[str, bytes]] | None = None
+        input_refs: list[FileRef] | None = None
+        if caps.accepts.file_urls:
+            await blob_store.put_object(CHECK_BLOB_KEY, CHECK_FILE[1])
+            input_refs = [
+                FileRef(
+                    name=CHECK_FILE[0],
+                    url=await blob_store.object_url(CHECK_BLOB_KEY),
+                    context=CHECK_FILE_CONTEXT,
+                )
+            ]
+        elif caps.accepts.files:
+            input_files = [CHECK_FILE]
+            input_refs = [FileRef(name=CHECK_FILE[0], context=CHECK_FILE_CONTEXT)]
+        run_id: str | None = None
+        try:
+            run_id = await client.start_run(probe, files=input_files, file_refs=input_refs)
+            status = await client.poll_until_terminal(run_id)
+            carried = [
+                kind
+                for kind, value in (("an answer", status.answer), ("an output", status.output))
+                if value
+            ]
+            if status.status == "completed" and carried:
+                report["run"] = _entry(
+                    "pass", f"run {run_id} completed with {' and '.join(carried)}"
+                )
+            else:
+                report["run"] = _entry("fail", f"run ended {status.status}, error={status.error}")
+        except (httpx.HTTPError, RunTimeout) as exc:
+            report["run"] = _entry("fail", str(exc))
+        except UnsafePathError:
+            report["run"] = _entry("fail", f"run id {run_id!r} must be one plain path segment")
+            run_id = None
+
+        if not caps.accepts.file_input:
+            report["files"] = _entry("skipped", "accepts.files=false, accepts.file_urls=false")
+        elif run_id is None:
+            report["files"] = _entry("fail", report["run"]["detail"])
+        elif caps.accepts.file_urls:
+            report["files"] = _entry("pass", f"{CHECK_FILE[0]} fetched by url")
+        else:
+            report["files"] = _entry("pass", f"{CHECK_FILE[0]} accepted on the run request")
+
+        if agent.is_otel:
+            report["trajectory"] = _entry("pass", "from traces")
+            if run_id is None:
+                report["session"] = _entry("fail", "no run to poll a session for")
+            elif sessions is None:
+                report["session"] = _entry("skipped", "no session reader for this check")
+            else:
+                doc, session_status = await poll_trajectory(
+                    sessions,
+                    agent_slug or agent.ref,
+                    run_id,
+                    agent.settings.trajectory_wait_seconds,
+                )
+                session = await sessions.trajectory(agent_slug or agent.ref, run_id)
+                detail = f"{session_status}: session {run_id}, {session.span_count} spans"
+                if doc is not None:
+                    detail += f", {len(doc.get('steps') or [])} steps"
+                report["session"] = _entry(
+                    {"valid": "pass", "partial": "warn"}.get(session_status, "fail"), detail
+                )
+        elif not caps.provides.trajectory:
+            report["trajectory"] = _entry("skipped", "provides.trajectory=false")
+        elif run_id is None:
+            report["trajectory"] = _entry("fail", "no run to fetch a trajectory for")
+        else:
+            try:
+                trajectory = await client.trajectory(run_id)
+                detail = f"valid {trajectory.schema_version} with {len(trajectory.steps)} steps"
+                warnings = subagent_link_warnings(trajectory)
+                report["trajectory"] = (
+                    _entry("warn", "; ".join([detail, *warnings]))
+                    if warnings
+                    else _entry("pass", detail)
+                )
+            except (httpx.HTTPError, TrajectoryValidationError) as exc:
+                report["trajectory"] = _entry("fail", str(exc))
+
+        if not caps.provides.workspace:
+            report["workspace"] = _entry("skipped", "provides.workspace=false")
+        elif run_id is None:
+            report["workspace"] = _entry("fail", "no run to list files for")
+        else:
+            try:
+                files = await client.list_files(run_id)
+                report["workspace"] = _entry("pass", f"{len(files.files)} file(s) listed")
+            except httpx.HTTPError as exc:
+                report["workspace"] = _entry("fail", str(exc))
+
+        if not caps.provides.run_listing:
+            report["run_listing"] = _entry("skipped", "provides.run_listing=false")
+        else:
+            try:
+                runs = await client.list_runs(RunListParams(limit=5))
+                future = await client.list_runs(
+                    RunListParams(limit=5, since=datetime.now(UTC) + timedelta(days=1))
+                )
+                report["run_listing"] = (
+                    _entry("warn", f"total={runs.total}; {IGNORES_TIME_FILTER}")
+                    if future.runs
+                    else _entry("pass", f"total={runs.total}")
+                )
+            except httpx.HTTPError as exc:
+                report["run_listing"] = _entry("fail", str(exc))
+
+        if not caps.accepts.stop:
+            report["stop"] = _entry("skipped", "accepts.stop=false")
+        else:
+            try:
+                stop_run_id = await client.start_run(probe)
+                await client.stop(stop_run_id)
+                report["stop"] = _entry("pass", f"run {stop_run_id} stopped")
+            except (httpx.HTTPError, UnsafePathError) as exc:
+                report["stop"] = _entry("fail", str(exc))
+    finally:
+        await client.aclose()
+    return report