outerloop-science 0.1.0__py3-none-any.whl

outerloop/contract.py

@@ -0,0 +1,492 @@
+"""Contract schema and loader for a target repo's contract file (`.outerloop.yaml`).
+
+The contract is the opt-in declaration: benchmarks, budgets, scope. The loader
+enforces invariants no YAML can override (see the threat model in
+docs/design/architecture.md): autoresearch is never a target of itself, and the
+contract file, the target's roadmap, and `.github/` are always forbidden write
+paths, regardless of what `scope.allowed` says.
+
+Contracts live in target repos and are therefore untrusted input: parsing
+rejects aliases, duplicate keys, and oversized documents.
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+from collections.abc import Callable
+from pathlib import Path, PurePosixPath
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+SELF_REPO = "outerloop-science/outerloop"
+# The contract's filename in the target repo. New adopters write `.outerloop.yaml`;
+# `.autoresearch.yaml` (targets written before the rename) is still honored. Every
+# read goes through `find_contract`, which tries the new name first. Neither is
+# ever a writable path for the agent.
+# The pre-rename name is dropped in the release after 0.1.
+CONTRACT_NAMES: tuple[str, ...] = (".outerloop.yaml", ".autoresearch.yaml")
+CONTRACT_NAME = CONTRACT_NAMES[0]  # what the docs and new contracts use
+ALWAYS_FORBIDDEN: tuple[str, ...] = (".github", *CONTRACT_NAMES)
+MAX_CONTRACT_BYTES = 64 * 1024
+_GLOB_CHARS = set("*?[]!")
+
+
+def find_contract(read: Callable[[str], str | None]) -> tuple[str, str] | None:
+    """(name, text) of the first contract file `read` yields, new name first; None
+    when the target has neither. `read(name)` returns the file's text, or None when
+    that name is absent — wrap a reader that raises instead."""
+    for name in CONTRACT_NAMES:
+        text = read(name)
+        if text is not None:
+            return name, text
+    return None
+
+
+def contract_in_tree(tree: Path) -> tuple[str, str] | None:
+    """The contract in a checked-out tree: (name, text), or None."""
+    return find_contract(lambda n: (tree / n).read_text() if (tree / n).is_file() else None)
+
+
+def contract_text_in_tree(tree: Path) -> str:
+    """The contract's text in a checked-out tree; raises FileNotFoundError (as a
+    direct read did) naming both candidates when the tree has neither."""
+    found = contract_in_tree(tree)
+    if found is None:
+        raise FileNotFoundError(f"no contract in {tree} ({' or '.join(CONTRACT_NAMES)})")
+    return found[1]
+
+
+class ContractError(ValueError):
+    """Base class for contract rejections."""
+
+
+class SelfTargetError(ContractError):
+    """Raised when a contract names autoresearch itself as the target."""
+
+
+class ScopeError(ContractError):
+    """Raised when an allowed path is unsafe or overlaps a forbidden path."""
+
+
+class _SafeLoader(yaml.SafeLoader):
+    """SafeLoader that refuses alias expansion and duplicate mapping keys."""
+
+    def compose_node(self, parent: Any, index: Any) -> Any:
+        if self.check_event(yaml.events.AliasEvent):
+            raise ContractError("YAML aliases are not allowed in contracts")
+        return super().compose_node(parent, index)
+
+    def construct_mapping(self, node: Any, deep: bool = False) -> dict[Any, Any]:
+        seen = set()
+        for key_node, _ in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in seen:
+                raise ContractError(f"duplicate key in contract: {key!r}")
+            seen.add(key)
+        return super().construct_mapping(node, deep=deep)
+
+
+class _StrictModel(BaseModel):
+    # Typos in a contract must fail loudly, never be silently ignored.
+    model_config = ConfigDict(extra="forbid")
+
+
+class Benchmark(_StrictModel):
+    # Slug shape only: the name reaches branch names, ledger keys, and log
+    # labels — contract text must not shape refs or paths beyond a slug.
+    name: str = Field(min_length=1, pattern=r"^[A-Za-z0-9_.-]{1,64}$")
+    command: str = Field(min_length=1)
+    metric: str = Field(min_length=1)
+    direction: Literal["min", "max"]
+    # Significant digits for HUMAN surfaces (PR tables, BENCHMARKS.md,
+    # replies) per the benchmark's community convention. Full precision
+    # lives only in results/leader.json, the machine ledger (maintainer
+    # decision 2026-08-09). Display-only: comparisons always use full
+    # floats, so this can never hide or fake an improvement.
+    display_digits: int | None = Field(default=None, ge=2, le=12)
+    # Resampled-pool benchmarks: the env var the eval reads its run seed
+    # from (e.g. PILOT_REACH_SEED). When set, the orchestrator draws ONE
+    # fresh seed per measurement pass and pins BOTH sides of a comparison
+    # to it (paired, common random numbers), then records it in the ledger
+    # row — the number becomes re-derivable instead of pool luck. Strict
+    # env-var shape: this string reaches a subprocess environment. RULER
+    # INVARIANT the eval must uphold: emit the seed to stdout only, never
+    # persist it into the tree — a seed artifact in the workspace would be
+    # readable by the solver session that runs between the paired evals
+    # (the verifier's ruler read covers this).
+    seed_env: str | None = Field(default=None, pattern=r"^[A-Z][A-Z0-9_]{0,63}$")
+    # Expected eval duration, minutes — a HINT that turns on dispatched
+    # evals (docs/design/dispatcher.md): above the in-job threshold the
+    # orchestrator runs this benchmark's evals as their own jobs. Clamped
+    # by dispatch.EVAL_JOB_MINUTES_CEILING (our spend cap); it governs only
+    # the first eval once measured durations exist. None = in-job (today's
+    # behavior).
+    eval_minutes: int | None = Field(default=None, ge=1)
+    # GPUs for every dispatched job of this benchmark — gate measures and
+    # author launches alike. 0 (default) = CPU. A GPU benchmark needs the
+    # deployment to name a GPU lane (OUTERLOOP_GPU_PARTITION, optionally
+    # OUTERLOOP_GPU_ACCOUNT); without one the tick refuses to launch
+    # attempts on it rather than queue evals that can never run. Bounded at
+    # one node's worth: multi-node evals are not a shape the jail supports.
+    gpus: int = Field(default=0, ge=0, le=8)
+    # How the gate obtains the baseline number it compares a candidate to.
+    #   paired (default): re-measure the base tree next to every candidate,
+    #     both under one fresh seed (common random numbers) — the noise-
+    #     minimal comparison, at two evals per attempt.
+    #   cached: measure the base tree ONCE per (benchmark, base sha) into a
+    #     cache shared by every attempt on that base, then run only the
+    #     candidate at a fresh seed. Halves eval spend; the comparison is
+    #     unpaired, so the contract's min_delta must cover cross-seed noise
+    #     on its own (calibrate it). The ledger row says which baseline
+    #     measurement (and seed) a credited delta was taken against.
+    baseline: Literal["paired", "cached"] = "paired"
+    # Depth budget (docs/design/research-loop.md, "one syscall, author-directed"):
+    # how many external experiment jobs the author may LAUNCH within one attempt.
+    # A generous meter on actions, not a loop the kernel drives — the author
+    # decides what each launch is for. Per-benchmark so "does depth pay here?"
+    # is answerable per benchmark. The syscalls are CONTRACT-DRIVEN and on by
+    # default wherever the deployment can deliver them (dispatch coords + a
+    # resumable backend); `depth_k: 0` is a benchmark's opt-out — the tool is
+    # then not offered at all. Weekly spend stays bounded by `runs_per_week`.
+    depth_k: int = Field(default=10, ge=0, le=16)
+    # Sleep budget, the sibling knob: how many dispatch->sleep->wake cycles the
+    # author may spend. Independent of depth_k — launches meter external compute,
+    # sleeps meter wake cycles (without this an author could checkpoint-refresh
+    # its session clock forever and never launch). Batching is rewarded: many
+    # launches under one sleep burn one sleep; a `submit` also rides one sleep.
+    sleep_k: int = Field(default=20, ge=1, le=32)
+    # Research lines (docs/design/research-lines.md): each agent slot works on
+    # its own persistent branch `agents/<agent-id>` — checked out at run start
+    # with the base branch merged in and instruction-bearing files reset to the
+    # base branch's reviewed versions. Off (default) = today's fork-main-only
+    # behavior; the branch substrate is inert until a contract opts in.
+    lines: bool = False
+
+    # Pure loop-steering dials: how often/deep the fleet iterates and how a
+    # number renders. Everything else — the command, metric, seed, GPUs,
+    # walltime (it selects the execution route), direction, floors, and
+    # baseline protocol — defines the measurement or the claim's meaning.
+    _WORKFLOW_DIALS = frozenset({"lines", "depth_k", "sleep_k", "display_digits"})
+
+    def measurement_signature(self) -> tuple:
+        """The fields that determine how this benchmark is measured and what
+        a claim about it means — every field EXCEPT the pure workflow dials,
+        so a future field joins the signature by default and the base-sync
+        skip fails toward re-measuring."""
+        data = self.model_dump()
+        return tuple(sorted((k, repr(v)) for k, v in data.items() if k not in self._WORKFLOW_DIALS))
+
+    @field_validator("seed_env")
+    @classmethod
+    def _seed_env_never_managed(cls, value: str | None) -> str | None:
+        from outerloop.orchestrator import managed_eval_env
+
+        if value is not None and managed_eval_env(value):
+            raise ValueError(
+                f"seed_env must not name or prefix the evaluator's managed environment ({value!r})"
+            )
+        return value
+
+    @model_validator(mode="after")
+    def _gpu_benchmarks_dispatch(self) -> Benchmark:
+        # GPUs only exist on dispatched jobs: the in-job evaluator runs inside
+        # the CPU climb job with no allocation and no --nv, so a GPU benchmark
+        # under the in-job threshold would measure on a machine with no GPU
+        # (terra #174 r2). Make the contract say so up front.
+        from outerloop.dispatch import should_dispatch
+
+        if self.gpus > 0 and not should_dispatch(self.eval_minutes):
+            raise ValueError(
+                f"benchmark {self.name!r} asks for {self.gpus} GPU(s) but its evals "
+                "would run in-job: set eval_minutes above the in-job threshold so they dispatch"
+            )
+        floor_set = (self.min_delta or 0) > 0 or (self.min_delta_rel or 0) > 0
+        if self.baseline == "cached" and not floor_set:
+            # an unpaired comparison has no built-in noise cancellation: the
+            # floor is the only thing standing between seed luck and a record
+            # — and benchmark_floor treats 0 as "no floor", so it must be > 0
+            raise ValueError(
+                f"benchmark {self.name!r} uses a cached baseline (unpaired comparison) "
+                "but declares no positive significance floor: set min_delta or "
+                "min_delta_rel (> 0) from a seed-variance calibration"
+            )
+        return self
+
+    # Cross-seed noise floor. A comparison against the RECORDED best was
+    # measured under a different seed, so a delta inside the floor is noise,
+    # not progress; same-seed paired comparisons are exempt by construction.
+    # Two forms. min_delta is absolute metric units, right for a bounded
+    # metric (a success rate). min_delta_rel is a fraction of the recorded
+    # level, right for an unbounded metric (wall-clock timing) whose level
+    # drifts with hardware, so an absolute number goes stale. Set either or
+    # both; both means the more conservative of the two applies.
+    min_delta: float | None = Field(default=None, ge=0)
+    # capped at 1.0: a noise floor is a small fraction (e.g. 0.13), so a
+    # value above the level itself is a typo (13 meaning 13%), and a floor
+    # of 13x would freeze the benchmark. A relative floor assumes a
+    # non-zero level; pair it with a small absolute min_delta as a backstop
+    # for a metric that can reach 0.
+    min_delta_rel: float | None = Field(default=None, ge=0, le=1.0)
+
+
+class SuiteAggregate(_StrictModel):
+    """Unified-benchmark targets: one change is evaluated on every benchmark
+    and reported per-env plus this aggregate (no cherry-picking)."""
+
+    metric: str = Field(min_length=1)
+    direction: Literal["min", "max"]
+
+
+class Budgets(_StrictModel):
+    # The attempt's compute allowance, METERED at the syscall for GPU
+    # benchmarks: every author launch (minutes x gpus) and a submit's two
+    # paired gate evals (2 x eval walltime x gpus) draw on it, and an
+    # over-budget request is refused with the numbers. The author may
+    # declare its own eval walltime at submit (`submit --minutes`), so a
+    # candidate whose eval runs longer is paid for out of this budget rather
+    # than killed by a fixed limit — walltime is never the metric; compute
+    # is priced here. 0 for CPU benchmarks (nothing to meter).
+    gpu_hours_per_run: float = Field(ge=0)
+    runs_per_week: int = Field(gt=0)
+    # Optional per-repo shaping of the orchestrator's session/job limits.
+    # These are WISHES, not grants: limits.effective_limits clamps every
+    # value into orchestrator-side [floor, ceiling] bounds, so a target can
+    # spend less of us, never more. Absent = orchestrator defaults.
+    session_max_turns: int | None = Field(default=None, gt=0)
+    session_minutes: int | None = Field(default=None, gt=0)
+    attempt_job_minutes: int | None = Field(default=None, gt=0)
+    followup_job_minutes: int | None = Field(default=None, gt=0)
+    # Per-benchmark self-initiated cooldown, in minutes. Default (unset) is
+    # the orchestrator's 6h — right for a standard research repo. An RSI /
+    # hot-loop target sets 0: the loop re-dispatches back-to-back and
+    # `runs_per_week` becomes the spend guard (Mengye: "for the RSI
+    # experiment — no cooldown; make sure the cluster is hot"). Still
+    # SERIAL per target until the width dial lands.
+    attempt_cooldown_minutes: int | None = Field(default=None, ge=0)
+    # THE WIDTH DIAL: concurrent self-initiated attempts per target. Default
+    # (unset) = 1, today's serial behavior. A hot-loop target raises it
+    # to run N authors abreast — each slot gets its own agent identity
+    # (agent-01..agent-0N), so branches, ledger rows, and reports stay
+    # distinct. runs_per_week and gpu_hours_per_run remain the spend guards.
+    max_active_attempts: int | None = Field(default=None, ge=1)
+    # The pace ceiling for an author's sweeps, in GPUs: one launch may hold at
+    # most this many at once, so a sweep of N tasks runs
+    # max_concurrent_gpus // gpus of them at a time (`--array=0-N%K`). Lenient
+    # by design — not the cap divided by the agent count (agents rarely launch
+    # at the same moment, and an idle share is wasted GPU); on a 16-GPU cap 12
+    # lets one sweep use most of the machine while a sibling's job still gets
+    # in. Unset: the author's own pace, the whole array by default.
+    max_concurrent_gpus: int | None = Field(default=None, ge=1)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _accept_legacy_climb_job_minutes(cls, data: Any) -> Any:
+        # TRANSITIONAL: the field was `climb_job_minutes`. Map the legacy key
+        # to the new name before validation (new name wins if BOTH appear, so
+        # a mid-migration contract never fails), and consume it so extra=forbid
+        # does not reject it. Drop this once the (two) live contracts migrate.
+        if isinstance(data, dict) and "climb_job_minutes" in data:
+            data = dict(data)
+            legacy = data.pop("climb_job_minutes")
+            data.setdefault("attempt_job_minutes", legacy)
+        return data
+
+
+class Scope(_StrictModel):
+    allowed: list[str] = Field(min_length=1)
+    # Shared code paths (encoder / world model / training loop — code every
+    # benchmark exercises, as opposed to env-specific solver code). A solver
+    # diff touching any of these is suite-gated: the orchestrator re-measures
+    # EVERY sibling benchmark on both sides and refuses credit if one
+    # regresses beyond its own floor. Env-specific diffs stay cheap — only
+    # their benchmark is measured.
+    shared: list[str] = Field(default_factory=list)
+
+
+# The orchestrator's ledger: no agent scope — solver OR steward — may
+# contain these; their numbers carry orchestrator provenance only.
+RECORD_PATHS = ("BENCHMARKS.md", "results/leader.json")
+
+
+class StewardScope(_StrictModel):
+    """Paths the BENCHMARK STEWARD may edit: env generators, the eval
+    harness, tests, and reference data — NEVER the record ledger
+    (BENCHMARKS.md, results/leader.json; the orchestrator writes those
+    with its own measurements, and load_contract rejects a steward scope
+    that includes them). The solver's `scope.allowed` is implicitly
+    forbidden to the steward — the roles' territories must not overlap
+    (collusion structure, design/meta.md) — and the always-forbidden set
+    (this contract, `.github/`, the roadmap) binds the steward too."""
+
+    allowed: list[str] = Field(min_length=1)
+
+
+class Contract(_StrictModel):
+    benchmarks: list[Benchmark] = Field(min_length=1)
+    budgets: Budgets
+    scope: Scope
+    roadmap: str = Field(min_length=1)
+    suite: SuiteAggregate | None = None
+    steward: StewardScope | None = None
+    # MERGE POLICY — the autonomy mode dial (docs/design/headline.md), the
+    # target owner's declaration like a harness permission mode:
+    #   manual (default): the bot opens PRs and arms auto-merge only when a
+    #     required human review stands between arming and merging.
+    #   auto: a gate+panel-clean PR merges itself. Repo prerequisites the
+    #     owner sets alongside this knob: "Allow auto-merge" on; branch
+    #     protection whose required checks are the repo's own CI with
+    #     STRICT up-to-date enforcement (the branch must match the base —
+    #     this is what closes the race where the base moves between our
+    #     freshness check and a direct merge: GitHub itself refuses a
+    #     stale-base merge); NO required review. The gate is the whole
+    #     conscience in auto mode — the floor (min_delta), the suite
+    #     no-regression phase, and the panel's taste rubric all bind
+    #     BEFORE publish.
+    merge: Literal["manual", "auto"] = "manual"
+
+    @field_validator("benchmarks")
+    @classmethod
+    def _unique_names(cls, benchmarks: list[Benchmark]) -> list[Benchmark]:
+        # Benchmark names are IDENTITIES: they key branch names, ledger rows,
+        # and dispatched measure/job names. A duplicate silently collides all
+        # three (two measures share an eval dir; one result is lost).
+        names = [b.name for b in benchmarks]
+        dupes = sorted({n for n in names if names.count(n) > 1})
+        if dupes:
+            raise ValueError(f"duplicate benchmark name(s): {', '.join(dupes)}")
+        return benchmarks
+
+
+_HOST_PREFIX = re.compile(r"^(?:[a-z+]+://)?(?:[^@/]*@)?(?:www\.)?github\.com[:/]+")
+
+
+def normalize_repo(target_repo: str) -> str:
+    """Reduce a repo reference to `owner/name`, casefolded.
+
+    Accepts bare `owner/name`, trailing `/` or `.git`, and any scheme/userinfo
+    URL spelling, so the self-target check can't be dodged by spelling.
+    """
+    ref = target_repo.strip().casefold()
+    ref = _HOST_PREFIX.sub("", ref)
+    ref = re.sub(r"/{2,}", "/", ref).strip("/")
+    if ref.endswith(".git"):
+        ref = ref[: -len(".git")]
+    return ref
+
+
+def normalize_path(entry: str) -> PurePosixPath:
+    """Normalize a scope entry, rejecting anything that can escape the repo."""
+    raw = entry.strip()
+    if not raw or raw in {".", "./"}:
+        raise ScopeError("allowing the repository root is never permitted")
+    if raw.startswith("/") or ":" in raw or "\\" in raw:
+        raise ScopeError(f"path must be repo-relative and POSIX: {entry!r}")
+    if _GLOB_CHARS & set(raw):
+        raise ScopeError(f"glob patterns are not allowed in scope: {entry!r}")
+    normalized = posixpath.normpath(raw)
+    path = PurePosixPath(normalized)
+    if normalized in {".", ""} or any(part == ".." for part in path.parts):
+        raise ScopeError(f"path escapes the repository: {entry!r}")
+    if any(part.casefold() == ".git" for part in path.parts):
+        raise ScopeError(f"the git directory is never writable: {entry!r}")
+    return path
+
+
+def forbidden_paths(contract: Contract) -> tuple[str, ...]:
+    """Write paths forbidden for this contract: the hard-coded set plus the
+    target's roadmap."""
+    return (*ALWAYS_FORBIDDEN, str(normalize_path(contract.roadmap)))
+
+
+def _fold(path: PurePosixPath) -> PurePosixPath:
+    """Casefold components: `.GITHUB/x` must be as forbidden as `.github/x`."""
+    return PurePosixPath(*[part.casefold() for part in path.parts])
+
+
+def path_is_forbidden(candidate: str, contract: Contract) -> bool:
+    """True if `candidate` (a repo-relative file path) may not be written.
+
+    Component-wise and case-insensitive, so `README.mdx` is not shadowed by
+    roadmap `README.md` but `.GITHUB/` is still blocked. Anything
+    unnormalizable counts as forbidden.
+    """
+    try:
+        path = _fold(normalize_path(candidate))
+    except ScopeError:
+        return True
+    if any(part == ".git" for part in path.parts):
+        return True  # never write into the git directory itself
+    for forbidden in forbidden_paths(contract):
+        f = _fold(PurePosixPath(forbidden))
+        if path == f or f in path.parents:
+            return True
+    return False
+
+
+def _overlaps(allowed: PurePosixPath, forbidden: PurePosixPath) -> bool:
+    a, f = _fold(allowed), _fold(forbidden)
+    return a == f or f in a.parents or a in f.parents
+
+
+def load_contract(text: str, target_repo: str) -> Contract:
+    """Parse and validate a contract for `target_repo`."""
+    if normalize_repo(target_repo) == SELF_REPO:
+        raise SelfTargetError("autoresearch is never a valid target of itself")
+    if len(text.encode()) > MAX_CONTRACT_BYTES:
+        raise ContractError(f"contract exceeds {MAX_CONTRACT_BYTES} bytes")
+    try:
+        data = yaml.load(text, Loader=_SafeLoader)
+    except ContractError:
+        raise
+    except (yaml.YAMLError, TypeError, ValueError) as exc:
+        raise ContractError(f"unparseable contract: {type(exc).__name__}") from None
+    if not isinstance(data, dict):
+        raise ContractError("contract must be a YAML mapping")
+    contract = Contract.model_validate(data)
+    forbidden = [PurePosixPath(p) for p in forbidden_paths(contract)]
+    for entry in contract.scope.allowed:
+        allowed = normalize_path(entry)
+        for path in forbidden:
+            if _overlaps(allowed, path):
+                raise ScopeError(f"allowed path {entry!r} overlaps forbidden {str(path)!r}")
+    # Shared paths route the suite gate off changed paths, which are already
+    # scope-checked — but a malformed entry would silently never match, so
+    # the same load-time rigor applies. A shared path the agent can never
+    # touch (no overlap with any allowed path) is dead config that reads as
+    # protection: refused loudly, like any contract typo.
+    solver_allowed = [normalize_path(entry) for entry in contract.scope.allowed]
+    for entry in contract.scope.shared:
+        shared = normalize_path(entry)
+        for path in forbidden:
+            if _overlaps(shared, path):
+                raise ScopeError(f"shared path {entry!r} overlaps forbidden {str(path)!r}")
+        if not any(_overlaps(shared, a) for a in solver_allowed):
+            raise ScopeError(
+                f"shared path {entry!r} overlaps no allowed path — the suite "
+                f"gate could never trigger on it"
+            )
+    if contract.steward is not None:
+        # same rigor as the solver scope, plus the role-separation invariant:
+        # steward and solver territories must not overlap AT LOAD TIME — a
+        # malformed or colliding entry is a contract error, never a
+        # mid-run surprise
+        solver = [normalize_path(entry) for entry in contract.scope.allowed]
+        records = [PurePosixPath(p) for p in RECORD_PATHS]
+        for entry in contract.steward.allowed:
+            allowed = normalize_path(entry)
+            for path in forbidden:
+                if _overlaps(allowed, path):
+                    raise ScopeError(f"steward path {entry!r} overlaps forbidden {str(path)!r}")
+            for sp in solver:
+                if _overlaps(allowed, sp):
+                    raise ScopeError(f"steward path {entry!r} overlaps solver scope {str(sp)!r}")
+            for rp in records:
+                if _overlaps(allowed, rp):
+                    raise ScopeError(
+                        f"steward path {entry!r} overlaps the record ledger "
+                        f"{str(rp)!r} (orchestrator-owned)"
+                    )
+    return contract

outerloop/contract_cli.py

@@ -0,0 +1,63 @@
+"""Validate a contract file (`.outerloop.yaml`) before you push it.
+
+    uv run python -m outerloop.contract_cli .outerloop.yaml
+
+Prints what the agent would be allowed to do, or exactly what is wrong.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from outerloop.contract import (
+    CONTRACT_NAME,
+    CONTRACT_NAMES,
+    ContractError,
+    forbidden_paths,
+    load_contract,
+)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = argv if argv is not None else sys.argv[1:]
+    # no arg: whichever contract name the cwd has, new name first
+    path = (
+        Path(args[0])
+        if args
+        else next((Path(n) for n in CONTRACT_NAMES if Path(n).is_file()), Path(CONTRACT_NAME))
+    )
+    repo = args[1] if len(args) > 1 else "your-org/your-repo"
+
+    if not path.is_file():
+        print(f"✗ no contract at {path}")
+        return 2
+    try:
+        contract = load_contract(path.read_text(), repo)
+    except ContractError as exc:
+        print(f"✗ {path}: {exc}")
+        return 1
+
+    print(f"✓ {path} is valid\n")
+    print("Benchmarks the agent will try to improve:")
+    for benchmark in contract.benchmarks:
+        arrow = "↓ lower is better" if benchmark.direction == "min" else "↑ higher is better"
+        print(f"  • {benchmark.name}: {benchmark.metric} ({arrow})")
+        print(f"    $ {benchmark.command}")
+    if contract.suite is not None:
+        print(f"\nSuite aggregate: {contract.suite.metric} ({contract.suite.direction})")
+    print("\nThe agent may write only to:")
+    for allowed in contract.scope.allowed:
+        print(f"  • {allowed}")
+    print("\nAlways forbidden, whatever the contract says:")
+    for forbidden in forbidden_paths(contract):
+        print(f"  • {forbidden}")
+    print(
+        f"\nBudget: {contract.budgets.gpu_hours_per_run} GPU-hours per run, "
+        f"{contract.budgets.runs_per_week} runs per week"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())