harnessweaver-cli 0.7.1__py3-none-any.whl

harnessweaver_cli/gateway.py

@@ -0,0 +1,301 @@
+"""``harnessweaver gateway litellm|mcp``: gateway configs from the same spec (ADR-0039).
+
+An app that calls models and tools straight through a gateway, with no SDK,
+still gets the spec's model aliases and its tool allow-list. A gateway can't see
+a run, though: what it has read, who approves, which tenant it is for. So a tool
+is let through only when its manifest asks for nothing that needs the run
+(approvals, egress and the trifecta, tenants, rate limits); the others are
+withheld and run only through the harness (in process, or harnessd). Policy
+rules, budgets and guards stay with the harness too.
+"""
+
+import math
+import re
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+import yaml
+
+from harnessweaver.core.errors import HarnessError
+from harnessweaver.core.gateway import AllowedTool, GatewayMetadata, ToolAllowList, WithheldTool
+from harnessweaver.core.manifests import ToolManifest, definition_digest
+from harnessweaver.modules.tools import load_manifests
+from harnessweaver.spec import EffectiveSpec, load_spec
+from harnessweaver.spec.models import HarnessSpec
+from harnessweaver_cli._load import fail, spec_path
+from harnessweaver_cli.spec import Env, Path_, ProfileDirs
+
+__all__ = [
+    "gateway_app",
+    "gateway_tools",
+    "litellm_config",
+    "render_allow_list",
+    "render_litellm",
+    "upstream_env",
+]
+
+gateway_app = typer.Typer(
+    help="Gateway configs from the spec: LiteLLM Proxy, and a tool allow-list for MCP gateways.",
+    no_args_is_help=True,
+)
+
+Out = Annotated[
+    Path | None, typer.Option("--out", "-o", help="Write to this file instead of printing.")
+]
+
+RETRIES = 2
+"""LiteLLM's retries: the gateway owns them, and SDK clients have theirs off (R5)."""
+
+
+def upstream_env(model: str) -> str:
+    """The variable that names the provider model behind a gateway model name, such as
+    ``HARNESSWEAVER_UPSTREAM_FAST_CHAT`` for ``fast-chat``. The spec names only the
+    gateway's models; which provider serves each is the gateway owner's choice."""
+    return "HARNESSWEAVER_UPSTREAM_" + re.sub(r"[^A-Za-z0-9]", "_", model).upper()
+
+
+def _withheld(manifest: ToolManifest, spec: HarnessSpec, approved: set[str]) -> str | None:
+    """Why only the harness may run this tool, or ``None`` when a gateway may.
+
+    Each reason is a manifest control that needs the run, so a gateway that
+    allowed the tool would skip it.
+    """
+    body = manifest.spec
+    if not body.enabled:
+        return "it is disabled"
+    digest = definition_digest(manifest.name, body.description, body.input_schema)
+    if body.definition_digest is not None and body.definition_digest != digest:
+        return "its pinned digest doesn't match its definition (S2)"
+    if (
+        body.approval is not None
+        or spec.tools.approval_by_risk.get(body.risk) == "required"
+        or manifest.name in approved
+    ):
+        return "it can need a person's approval, which only the harness can wait for"
+    if body.egress:
+        return "it has egress, and the trifecta check needs what the run has seen (S3)"
+    if body.tenants:
+        return "it is enabled for some tenants only"
+    if body.rate_limit is not None:
+        return "its rate limits are counted by the harness"
+    return None
+
+
+def _manifests(effective: EffectiveSpec) -> list[ToolManifest]:
+    base = effective.path.parent if effective.path is not None else Path.cwd()
+    found = load_manifests(effective.spec.tools.manifests, base)
+    return sorted(found, key=lambda manifest: manifest.name)
+
+
+def gateway_tools(effective: EffectiveSpec) -> ToolAllowList:
+    """The tools a gateway may let through, and the ones it must withhold (ADR-0039).
+
+    Raises:
+        SpecError: For a manifest that doesn't load.
+    """
+    spec = effective.spec
+    approved = {
+        tool for task in spec.tasks.values() if task.approvals for tool in task.approvals.tools
+    }
+    allowed: list[AllowedTool] = []
+    withheld: list[WithheldTool] = []
+    for manifest in _manifests(effective):
+        reason = _withheld(manifest, spec, approved)
+        if reason is not None:
+            withheld.append(WithheldTool(tool=manifest.name, reason=reason))
+            continue
+        body = manifest.spec
+        allowed.append(
+            AllowedTool(
+                tool=manifest.name,
+                transport=body.transport,
+                server=body.server,
+                # An unpinned manifest is pinned to what it says now; doctor warns about it.
+                digest=body.definition_digest
+                or definition_digest(manifest.name, body.description, body.input_schema),
+                scopes=body.scopes,
+            )
+        )
+    return ToolAllowList(
+        metadata=GatewayMetadata(name=spec.metadata.name),
+        tools=tuple(allowed),
+        withheld=tuple(withheld),
+    )
+
+
+def _param_patterns(manifest: ToolManifest) -> dict[str, str]:
+    """Regexes for the top-level string arguments the schema constrains: its ``pattern``,
+    or its ``enum`` as alternatives. Other constraints stay with the governor (step 2)."""
+    properties = manifest.spec.input_schema.get("properties")
+    patterns: dict[str, str] = {}
+    if not isinstance(properties, dict):
+        return patterns
+    for name, schema in sorted(properties.items()):
+        if not isinstance(schema, dict):
+            continue
+        pattern, enum = schema.get("pattern"), schema.get("enum")
+        if isinstance(pattern, str):
+            patterns[name] = pattern
+        elif isinstance(enum, list) and enum and all(isinstance(v, str) for v in enum):
+            patterns[name] = "^(?:" + "|".join(re.escape(str(v)) for v in enum) + ")$"
+    return patterns
+
+
+def _timeout(effective: EffectiveSpec) -> int | None:
+    """The longest deadline a run can have, in whole seconds: the gateway shouldn't keep
+    a request going after the harness has given up on it."""
+    spec = effective.spec
+    deadlines = [spec.budgets.per_run.deadline] + [
+        effective.run_budget(task).deadline for task in spec.tasks
+    ]
+    found = [deadline.total_seconds() for deadline in deadlines if deadline is not None]
+    return math.ceil(max(found)) if found else None
+
+
+def litellm_config(effective: EffectiveSpec, tools: ToolAllowList) -> dict[str, Any]:
+    """A LiteLLM Proxy config: the gateway's models, the spec's aliases, retries and a
+    timeout, content kept out of its logs, and a tool-permission guardrail on every
+    request that allows only the allow-list's tools."""
+    spec = effective.spec
+    models = spec.models
+    allowed_aliases = models.allowed_aliases
+    aliases = {
+        alias: target
+        for alias, target in sorted(models.aliases.items())
+        if allowed_aliases is None or alias in allowed_aliases
+    }
+    names = set(aliases.values())
+    if models.default is not None:
+        names.add(models.aliases.get(models.default, models.default))
+    router: dict[str, Any] = {"num_retries": RETRIES}
+    if aliases:
+        router["model_group_alias"] = aliases
+    timeout = _timeout(effective)
+    if timeout is not None:
+        router["timeout"] = timeout
+    observability = spec.observability
+    settings: dict[str, Any] = {"turn_off_message_logging": not observability.capture_content}
+    config: dict[str, Any] = {
+        "model_list": [
+            {"model_name": name, "litellm_params": {"model": f"os.environ/{upstream_env(name)}"}}
+            for name in sorted(names)
+        ],
+        "router_settings": router,
+        "litellm_settings": settings,
+    }
+    if observability.exporter == "otlp":
+        settings["callbacks"] = ["otel"]
+        config["environment_variables"] = {
+            "OTEL_EXPORTER": "otlp_http",
+            "OTEL_ENDPOINT": observability.endpoint or "http://localhost:4318",
+        }
+    manifests = {manifest.name: manifest for manifest in _manifests(effective)}
+    rules: list[dict[str, Any]] = []
+    for tool in tools.tools:
+        rule: dict[str, Any] = {
+            "id": f"allow-{tool.tool}",
+            "tool_name": f"^{re.escape(tool.tool)}$",
+            "decision": "allow",
+        }
+        patterns = _param_patterns(manifests[tool.tool])
+        if patterns:
+            rule["allowed_param_patterns"] = patterns
+        rules.append(rule)
+    rules += [
+        {"id": f"withhold-{w.tool}", "tool_name": f"^{re.escape(w.tool)}$", "decision": "deny"}
+        for w in tools.withheld
+    ]
+    config["guardrails"] = [
+        {
+            "guardrail_name": "harnessweaver-tools",
+            "litellm_params": {
+                "guardrail": "tool_permission",
+                "mode": "post_call",
+                "default_on": True,
+                "rules": rules,
+                "default_action": "deny",  # no tool runs without a manifest (S2)
+                "on_disallowed_action": "block",
+            },
+        }
+    ]
+    config["general_settings"] = {"master_key": "os.environ/LITELLM_MASTER_KEY"}
+    return config
+
+
+def _dump(data: Any) -> str:
+    return yaml.safe_dump(data, sort_keys=False, default_flow_style=False, width=100)
+
+
+def _withheld_lines(tools: ToolAllowList) -> list[str]:
+    if not tools.withheld:
+        return []
+    lines = ["Tools only the harness runs (denied here; run them through harnessd):"]
+    return lines + [f"  {w.tool}: {w.reason}" for w in tools.withheld]
+
+
+def _comment(lines: list[str]) -> str:
+    return "".join(f"# {line}".rstrip() + "\n" for line in lines)
+
+
+def render_litellm(effective: EffectiveSpec) -> str:
+    """The LiteLLM Proxy config as YAML, with a header saying what to set and what is
+    withheld."""
+    tools = gateway_tools(effective)
+    names = [entry["model_name"] for entry in litellm_config(effective, tools)["model_list"]]
+    header = [
+        f"LiteLLM Proxy config for {effective.spec.metadata.name}, generated by",
+        "`harnessweaver gateway litellm` from the spec and its tool manifests (ADR-0039).",
+        "Regenerate it when they change. In the gateway's environment (S1), set",
+        "LITELLM_MASTER_KEY, the provider keys, and the provider model behind each name:",
+        *[f"  {upstream_env(name)}" for name in names],
+        "Policy rules, budgets, guards and approvals stay with the harness.",
+        *_withheld_lines(tools),
+    ]
+    return _comment(header) + _dump(litellm_config(effective, tools))
+
+
+def render_allow_list(effective: EffectiveSpec) -> str:
+    """The neutral tool allow-list as YAML (schema ``tool-allow-list``)."""
+    tools = gateway_tools(effective)
+    header = [
+        f"Tool allow-list for {effective.spec.metadata.name}, generated by",
+        "`harnessweaver gateway mcp` (ADR-0039; schema tool-allow-list). An MCP gateway",
+        "allows these tools only, pins each digest, and requires its scopes.",
+    ]
+    return _comment(header) + _dump(tools.model_dump(mode="json", by_alias=True))
+
+
+def _emit(text: str, out: Path | None) -> None:
+    if out is None:
+        typer.echo(text, nl=False)
+        return
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(text, encoding="utf-8")
+    typer.echo(f"wrote {out}")
+
+
+@gateway_app.command("litellm")
+def litellm(
+    path: Path_ = None, env: Env = None, profile_dir: ProfileDirs = None, out: Out = None
+) -> None:
+    """A LiteLLM Proxy config: model aliases, router settings and tool permissions."""
+    try:
+        effective = load_spec(spec_path(path), env=env, profile_dirs=profile_dir or [])
+        text = render_litellm(effective)
+    except HarnessError as exc:
+        raise fail(exc) from None
+    _emit(text, out)
+
+
+@gateway_app.command("mcp")
+def mcp(
+    path: Path_ = None, env: Env = None, profile_dir: ProfileDirs = None, out: Out = None
+) -> None:
+    """A neutral tool allow-list for an MCP gateway: tool, server, digest and scopes."""
+    try:
+        effective = load_spec(spec_path(path), env=env, profile_dirs=profile_dir or [])
+        text = render_allow_list(effective)
+    except HarnessError as exc:
+        raise fail(exc) from None
+    _emit(text, out)

harnessweaver_cli/main.py

@@ -0,0 +1,62 @@
+"""Entry point for the ``harnessweaver`` command."""
+
+from typing import Annotated
+
+import typer
+
+import harnessweaver
+from harnessweaver_cli import __version__
+from harnessweaver_cli.add import add_app
+from harnessweaver_cli.dev import dev
+from harnessweaver_cli.doctor import doctor
+from harnessweaver_cli.evals import eval_app
+from harnessweaver_cli.gateway import gateway_app
+from harnessweaver_cli.plugins import plugin_app
+from harnessweaver_cli.policy import policy_app
+from harnessweaver_cli.replay import replay
+from harnessweaver_cli.scaffold import init, run_tests
+from harnessweaver_cli.sidecar import sidecar
+from harnessweaver_cli.spec import contracts_app, hooks, spec_app
+from harnessweaver_cli.tools import tools_app
+
+app = typer.Typer(
+    name="harnessweaver",
+    help="HarnessWeaver developer tooling: weave, check and test your AI harness.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+app.command()(init)
+app.add_typer(spec_app, name="spec")
+app.command()(doctor)
+app.command()(hooks)
+app.command(
+    name="test", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
+)(run_tests)
+app.add_typer(contracts_app, name="contracts")
+app.add_typer(eval_app, name="eval")
+app.add_typer(policy_app, name="policy")
+app.command()(replay)
+app.add_typer(add_app, name="add")
+app.add_typer(tools_app, name="tools")
+app.add_typer(plugin_app, name="plugin")
+app.command()(sidecar)
+app.add_typer(gateway_app, name="gateway")
+app.command()(dev)
+
+
+def _print_version(value: bool) -> None:
+    if value:
+        typer.echo(f"harnessweaver-cli {__version__} (harnessweaver {harnessweaver.__version__})")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version", callback=_print_version, is_eager=True, help="Show versions and exit."
+        ),
+    ] = False,
+) -> None:
+    """HarnessWeaver developer tooling."""

harnessweaver_cli/plugins.py

@@ -0,0 +1,105 @@
+"""``harnessweaver plugin new`` and ``harnessweaver plugin list`` (DESIGNS No. 13).
+
+``plugin new`` scaffolds a standalone plugin package, meant to live in its own
+repository: it builds on the public extension surface only and passes the
+conformance suite as written. ``plugin list`` shows what is installed.
+"""
+
+import json
+from importlib import resources
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+from harnessweaver import CONTRACT_VERSION, RESERVED_NAMESPACES
+from harnessweaver.core.errors import HarnessError
+from harnessweaver.core.plugins.names import NAMESPACE_PATTERN
+from harnessweaver.core.registry import PluginCatalog
+from harnessweaver_cli._load import fail
+from harnessweaver_cli.scaffold import copy_tree
+from harnessweaver_cli.spec import Format
+
+__all__ = ["plugin_app"]
+
+plugin_app = typer.Typer(help="Create and inspect plugins.", no_args_is_help=True)
+
+
+@plugin_app.command("new")
+def new(
+    namespace: Annotated[str, typer.Argument(help="The plugin's namespace, such as acme.")],
+    directory: Annotated[
+        Path | None, typer.Argument(help="Where to create it (default: ./<namespace>-harness).")
+    ] = None,
+    force: Annotated[bool, typer.Option("--force", help="Overwrite existing files.")] = False,
+) -> None:
+    """Scaffold a standalone plugin package: entry point, ``hw.Plugin``, a sample guard
+    and tests that also run the conformance suite (T5)."""
+    if not NAMESPACE_PATTERN.match(namespace):
+        raise fail(
+            f"invalid namespace {namespace!r}: use lowercase letters, digits and underscores, "
+            "starting with a letter"
+        )
+    if namespace in RESERVED_NAMESPACES:
+        raise fail(f"namespace {namespace!r} is reserved for the built-in plugin")
+    dist = f"{namespace.replace('_', '-')}-harness"
+    target = directory or Path(dist)
+    root = resources.files("harnessweaver_cli").joinpath("templates", "_plugin")
+    written = copy_tree(root, target, {"__dist__": dist, "__ns__": namespace}, force=force)
+    typer.echo(f"created {len(written)} files in {target} for the plugin {namespace!r} ({dist})")
+    typer.echo("next:")
+    typer.echo(f"  cd {target}")
+    typer.echo("  uv sync && uv run pytest     # its tests and the conformance suite")
+
+
+def _row(catalog: PluginCatalog, namespace: str, sources: tuple[Any, ...]) -> dict[str, Any]:
+    """What ``plugin list`` shows for one namespace. Importing a plugin does no I/O (S8)."""
+    source = sources[0]
+    row: dict[str, Any] = {
+        "namespace": namespace,
+        "distribution": source.distribution,
+        "target": source.target,
+    }
+    try:
+        plugin = catalog.load(namespace).plugin
+    except HarnessError as exc:  # several claim it, or it doesn't import
+        row.update(version=None, contract=None, compatible=False, problem=exc.message)
+        return row
+    row.update(
+        version=plugin.version,
+        contract=plugin.contract,
+        compatible=plugin.contract_range.contains(CONTRACT_VERSION),
+        contributions=len(plugin.contributions()),
+        problem=None,
+    )
+    return row
+
+
+@plugin_app.command("list")
+def list_plugins(output: Format = "text") -> None:
+    """Installed plugins: namespace, distribution, version, contract range and whether it
+    is compatible with this harnessweaver (V5)."""
+    catalog = PluginCatalog.default()
+    rows = [_row(catalog, ns, sources) for ns, sources in catalog.sources().items()]
+    if output == "json":
+        typer.echo(json.dumps({"plugins": rows, "problems": catalog.problems()}, indent=2))
+        return
+    table = Table(box=box.SIMPLE, show_header=True)
+    for column in ("namespace", "distribution", "version", "contract", "compatible"):
+        table.add_column(column)
+    for row in rows:
+        compatible = "yes" if row["compatible"] else f"no: {row['problem'] or 'contract range'}"
+        table.add_row(
+            row["namespace"],
+            row["distribution"] or "-",
+            row["version"] or "-",
+            row["contract"] or "-",
+            compatible,
+        )
+    Console(width=120, highlight=False).print(table)
+    for problem in catalog.problems():
+        typer.echo(f"warning: {problem}")
+    typer.echo(f"harnessweaver contract {CONTRACT_VERSION}")

harnessweaver_cli/policy.py

@@ -0,0 +1,143 @@
+"""``harnessweaver policy test``: table-driven tests of policy rules and tool approvals.
+
+Each row names a tool call, who makes it and what the governor must decide::
+
+    cases:
+      - id: small-refunds-pass
+        tool: refund_payment
+        args: { amount_usd: 20 }
+        principal: { id: alice, roles: [support] }
+        expect: allow
+      - id: big-refunds-need-finance
+        tool: refund_payment
+        args: { amount_usd: 900 }
+        expect: require_approval
+        approver_role: finance
+
+The governor's steps 1 to 7 decide each row as a dry run (``Harness.authorize``):
+the spec's policy engine, manifests, approval rules, the trifecta and the judge,
+with no rate limit used up and no tool executed. Exits 1 when a row's verdict
+differs and 2 when the tests can't run.
+"""
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import anyio
+import typer
+from pydantic import ValidationError
+
+from harnessweaver import Harness
+from harnessweaver.core.errors import HarnessError, SpecError
+from harnessweaver.core.policy import PolicyCase, PolicyTests
+from harnessweaver.core.tools import ToolVerdict
+from harnessweaver.modules.state import MemoryStateStore
+from harnessweaver.spec.yaml import load_yaml_file
+from harnessweaver_cli._load import fail, load_harness
+from harnessweaver_cli.spec import Env, Format, Path_, ProfileDirs
+
+__all__ = ["policy_app"]
+
+policy_app = typer.Typer(help="Test policy rules and tool approvals.", no_args_is_help=True)
+
+CANT_RUN = 2
+DEFAULT_CASES = Path("policy_tests.yaml")
+
+
+def _read(path: Path) -> PolicyTests:
+    if not path.is_file():
+        raise SpecError(f"no policy tests at {path}")
+    try:
+        return PolicyTests.model_validate(load_yaml_file(path))
+    except ValidationError as exc:
+        problems = "; ".join(
+            f"{'.'.join(str(p) for p in e['loc']) or '(file)'}: {e['msg']}" for e in exc.errors()
+        )
+        raise SpecError(f"{path}: {problems}") from None
+
+
+def _mismatch(case: PolicyCase, verdict: ToolVerdict) -> str | None:
+    wanted = [
+        ("decision", case.expect, verdict.decision),
+        ("code", case.code, verdict.code),
+        ("step", case.step, verdict.step),
+        ("approver_role", case.approver_role, verdict.approver_role),
+    ]
+    wrong = [
+        f"{name} {got!r}, not {want!r}"
+        for name, want, got in wanted
+        if want is not None and want != got
+    ]
+    return "; ".join(wrong) or None
+
+
+async def _check(harness: Harness, tests: PolicyTests) -> list[tuple[PolicyCase, ToolVerdict]]:
+    return [
+        (
+            case,
+            await harness.authorize(
+                case.tool,
+                case.args,
+                principal=case.principal,
+                tenant=case.tenant,
+                task=case.task,
+                labels=case.labels,
+            ),
+        )
+        for case in tests.cases
+    ]
+
+
+@policy_app.command("test")
+def run_policy_tests(
+    path: Path_ = None,
+    env: Env = None,
+    profile_dir: ProfileDirs = None,
+    cases: Annotated[
+        list[Path] | None,
+        typer.Option(
+            "--cases", "-c", help="Test files (default: policy_tests.yaml next to the spec)."
+        ),
+    ] = None,
+    output: Format = "text",
+) -> None:
+    """Check each row's tool call against the governor, without running anything.
+
+    Exits 1 when a verdict differs from the row's expectation; 2 when the tests
+    can't run.
+    """
+    harness = load_harness(path, env, profile_dir or [], state=MemoryStateStore(), code=CANT_RUN)
+    base = harness.spec.path.parent if harness.spec.path is not None else Path.cwd()
+    files = cases or [base / DEFAULT_CASES]
+    results: list[tuple[Path, PolicyCase, ToolVerdict]] = []
+    try:
+        for file in files:
+            checked = anyio.run(_check, harness, _read(file))
+            results += [(file, case, verdict) for case, verdict in checked]
+    except HarnessError as exc:
+        raise fail(exc, CANT_RUN) from None
+    failures = [(f, c, v) for f, c, v in results if _mismatch(c, v) is not None]
+    if output == "json":
+        body = [
+            {
+                "file": str(file),
+                "id": case.id,
+                "ok": _mismatch(case, verdict) is None,
+                "expect": case.expect,
+                "verdict": verdict.model_dump(mode="json"),
+            }
+            for file, case, verdict in results
+        ]
+        typer.echo(json.dumps(body, indent=2))
+    else:
+        for _, case, verdict in results:
+            problem = _mismatch(case, verdict)
+            shown = f"{verdict.decision} at step {verdict.step}"
+            if problem is None:
+                typer.echo(f"ok   {case.id}: {shown}")
+            else:
+                typer.echo(f"FAIL {case.id}: {problem} ({verdict.reason})")
+        typer.echo(f"{len(results) - len(failures)} of {len(results)} policy tests passed")
+    if failures:
+        raise typer.Exit(code=1)

harnessweaver_cli/replay.py

@@ -0,0 +1,75 @@
+"""``harnessweaver replay <run_id>``: run a recorded run again against the current code
+and spec (ADR-0035).
+
+The run must have been recorded (``state.record: true``). Its model answers and
+tool results come from the recording, so nothing calls out and no tool runs;
+the replay's own runs stay in memory. Exits 1 when the status or the output
+changed, and 2 when the run can't be replayed.
+"""
+
+import json
+
+import anyio
+import typer
+
+from harnessweaver import Harness
+from harnessweaver.core.errors import HarnessError
+from harnessweaver.core.replay import ReplayReport
+from harnessweaver_cli._load import fail, load_harness
+from harnessweaver_cli.spec import Env, Format, Path_, ProfileDirs
+
+__all__ = ["replay"]
+
+CANT_RUN = 2
+
+
+async def _replay(harness: Harness, run_id: str) -> ReplayReport:
+    try:
+        return await harness.replay(run_id)
+    finally:
+        await harness.aclose()
+
+
+def _print(report: ReplayReport) -> None:
+    before, after = report.original, report.replayed
+    typer.echo(
+        f"{'changed' if report.differs else 'same'}: run {report.run_id} ({report.task}) "
+        f"ended {before.status.value}; the replay ended {after.status.value}"
+    )
+    typer.echo(f"  output: {'same' if report.same_output else 'different'}")
+    typer.echo(
+        f"  model answers used: {report.model_answers_used} of {report.model_answers}"
+        + (f"; {report.changed_requests} request(s) changed" if report.changed_requests else "")
+    )
+    if report.missing_tool_results:
+        typer.echo(f"  tool calls not in the recording: {report.missing_tool_results}")
+    for change in report.changes:
+        sign = "+" if change.change == "added" else "-"
+        where = change.hook.value if change.hook is not None else "-"
+        what = change.code or (change.outcome.value if change.outcome else "")
+        typer.echo(f"  {sign} {where} {change.source} {what}: {change.reason or ''}".rstrip(": "))
+
+
+def replay(
+    run_id: str = typer.Argument(help="The recorded run."),
+    path: Path_ = None,
+    env: Env = None,
+    profile_dir: ProfileDirs = None,
+    output: Format = "text",
+) -> None:
+    """Replay a recorded run against the current code and spec, and show what changed.
+
+    Exits 1 when the status or output changed; 2 when the run can't be replayed.
+    """
+    harness = load_harness(path, env, profile_dir or [], code=CANT_RUN)
+    try:
+        report = anyio.run(_replay, harness, run_id)
+    except HarnessError as exc:
+        raise fail(exc, CANT_RUN) from None
+    if output == "json":
+        body = {**report.model_dump(mode="json"), "differs": report.differs}
+        typer.echo(json.dumps(body, indent=2))
+    else:
+        _print(report)
+    if report.differs:
+        raise typer.Exit(code=1)