outerloop-science 0.1.0.dev0__py3-none-any.whl

outerloop/review_agent.py

@@ -0,0 +1,263 @@
+"""Agent-session reviewer: runs the reviewer as an agent over a PR-head
+checkout, recording its verdict through the installed syscall tool.
+
+`run_agent_review` is the orchestration core, testable with a fake harness and
+client. It builds on the shared vocabulary in `review`: the same skip rules,
+the same rubric (via `build_agent_brief`), the same result policy
+(`review_result_from_role`), and the same inline-posting path (`format_review`
+-> `post_round_review`).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from outerloop.github import GitHubClient
+from outerloop.harness import (
+    Harness,
+    backend_id,
+    budget_exhausted,
+    outage,
+)
+from outerloop.posting import (
+    EXPECTED_FAILURES,
+    post_round_review,
+    post_skip_stub,
+)
+from outerloop.review import (
+    MARKER,
+    PullRequest,
+    build_agent_brief,
+    format_comment,
+    format_review,
+    skip_reason,
+)
+from outerloop.role_runner import run_role
+from outerloop.roles import review_result_from_role, reviewer_spec
+from outerloop.rolespec import RoleSpec
+
+log = logging.getLogger(__name__)
+
+# Files an agent harness may auto-load as INSTRUCTIONS from a checkout. In an
+# untrusted tree they are attack surface (a PR-authored CLAUDE.md, or
+# .claude/settings.json hooks that execute commands), so the CLIs rename them
+# before any session starts. Renamed — not deleted — so a judge can still read
+# them as data. Backend-agnostic defense in depth behind claude's --bare.
+INSTRUCTION_FILES = ("CLAUDE.md", "AGENTS.md", ".claude", ".mcp.json")
+SANITIZED_SUFFIX = ".pr-data"
+
+
+def sanitize_checkout(tree: Path) -> tuple[int, int]:
+    """Rename instruction-bearing files/dirs anywhere under `tree` so no agent
+    backend auto-loads untrusted content as instructions. Returns
+    (renamed, failed). A non-zero `failed` means an instruction file is still
+    live (e.g. a crafted name collision blocking the rename) — callers must
+    FAIL CLOSED and skip the session rather than judge an unsanitized tree.
+    Never raises."""
+    renamed = failed = 0
+    if not tree.is_dir():
+        return 0, 0
+    # bottom-up so a renamed directory doesn't orphan paths found beneath it
+    for path in sorted(tree.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+        if path.name in INSTRUCTION_FILES:
+            try:
+                path.rename(path.with_name(path.name + SANITIZED_SUFFIX))
+                renamed += 1
+            except OSError as exc:
+                log.warning("could not sanitize %s: %s", path, exc)
+                failed += 1
+    return renamed, failed
+
+
+def _pull_request(client: GitHubClient, repo: str, number: int) -> tuple[PullRequest, dict]:
+    pr_data = client.get_pull_request(repo, number)
+    diff = client.get_pull_request_diff(repo, number)
+    pr = PullRequest(
+        repo=repo,
+        number=number,
+        title=str(pr_data.get("title", "")),
+        body=str(pr_data.get("body") or ""),
+        diff=diff,
+        author=str((pr_data.get("user") or {}).get("login", "")),
+        # `or []`: the labels field may be null in the payload, not just absent
+        labels=tuple(
+            str(label.get("name", ""))
+            for label in (pr_data.get("labels") or [])
+            if isinstance(label, dict)
+        ),
+    )
+    return pr, pr_data
+
+
+def _emit(
+    path: Path,
+    repo: str,
+    number: int,
+    *,
+    kind: str,
+    data: dict[str, Any] | None = None,
+    detail: str = "",
+    reviewed_by: str = "",
+    lens: str = "",
+) -> None:
+    """Write the posting step's input. repo/number ride along so the poster
+    can refuse an envelope that does not match its own PR reference;
+    reviewed_by (backend/model) rides along for the round stamp."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(
+            {
+                "repo": repo,
+                "number": number,
+                "kind": kind,
+                "data": data,
+                "detail": detail,
+                "reviewed_by": reviewed_by,
+                "lens": lens,
+            }
+        )
+    )
+
+
+def run_agent_review(
+    client: GitHubClient,
+    repo: str,
+    number: int,
+    harness: Harness,
+    workspace: Path,
+    *,
+    bot_login: str,
+    spec: RoleSpec | None = None,
+    today: str | None = None,
+    emit_path: Path | None = None,
+    lens: str = "",
+) -> str | None:
+    """Review PR #`number` as an agent session over `workspace` (a
+    PR-head checkout the caller prepared). Post the findings inline via the
+    Reviews API. Returns the round label, or None when it skipped or could not
+    produce a verdict. Advisory: never raises the expected failures, so it can
+    never turn a target repo's CI red.
+
+    With `emit_path`, nothing is posted: the raw findings (or a skip
+    envelope) are written there for a separate posting step — the least-token
+    split (docs/design/reviewer-infra.md). The session job runs with read-only
+    permissions; the posting job holds the write token but runs no session.
+    EVERY outcome writes an envelope — findings, a PR-visible skip-stub
+    (missing key, errored session), or skip-clean (bot PR, opt-out; posts
+    nothing) — so the posting job can treat a missing artifact as a broken
+    session, loudly.
+    """
+    spec = spec or reviewer_spec()
+    today = today or datetime.now(UTC).date().isoformat()
+    try:
+        pr, pr_data = _pull_request(client, repo, number)
+        skip = skip_reason(pr, bot_login)
+        if skip is not None:
+            log.info("skipping agent review of %s#%s: %s", repo, number, skip)
+            if emit_path is not None:
+                # even a clean skip leaves an envelope: the posting job can
+                # then REQUIRE an artifact, so "no artifact" always means a
+                # broken session, never an ambiguous quiet day
+                _emit(emit_path, repo, number, kind="skip-clean", detail=skip)
+            return None
+
+        from outerloop.syscall import tool_command
+
+        brief = build_agent_brief(pr, today, syscall_cmd=tool_command(workspace), lens=lens)
+        role_result = run_role(spec, harness, brief, workspace)
+        review = review_result_from_role(role_result)
+        if review is None:
+            # No verdict: an errored or refused session, not a clean read. An
+            # API outage or a budget-exhausted session (walltime/turns) says so
+            # on the thread; other failures are logged, advisory-silent.
+            detail = role_result.error or role_result.session.stop_reason
+            log.warning("agent review produced no verdict on %s#%s: %s", repo, number, detail)
+            # `detail` is already api-key-redacted by the harness (it owns
+            # its own secret), so no secrets are passed here.
+            if emit_path is not None:
+                # EVERY errored session surfaces on the PR in the split
+                # topology: this job's log is not the record — the stub the
+                # post job publishes is.
+                _emit(
+                    emit_path,
+                    repo,
+                    number,
+                    kind="skip-stub",
+                    detail=detail,
+                    reviewed_by=backend_id(harness),
+                )
+            elif outage(role_result.session) or budget_exhausted(role_result.session):
+                post_skip_stub(client, repo, number, "advisory review", RuntimeError(detail))
+            return None
+
+        if emit_path is not None:
+            # raw data, not rendered text: the posting step re-validates and
+            # sanitizes at the render boundary, so the artifact crossing the
+            # job boundary carries no pre-trusted markup
+            _emit(
+                emit_path,
+                repo,
+                number,
+                kind="findings",
+                data=role_result.data,
+                reviewed_by=backend_id(harness),
+                lens=lens,
+            )
+            cost = role_result.session.cost_usd
+            log.info(
+                "emitted findings for %s#%s (cost=%s turns=%d)",
+                repo,
+                number,
+                f"${cost:.2f}" if cost else "unreported",
+                role_result.session.num_turns,
+            )
+            return "emitted"
+
+        rendered = format_review(review, pr.diff)
+        full = format_comment(review)
+        if rendered is None or full is None:
+            log.info("nothing to post")
+            return None
+        body, inline = rendered
+        round_label = post_round_review(
+            client,
+            repo,
+            number,
+            MARKER,
+            body,
+            inline,
+            pr_data,
+            fallback_body=full,
+            reviewed_by=backend_id(harness),
+        )
+        cost = role_result.session.cost_usd
+        log.info(
+            "posted agent review (%s) on %s#%s (cost=%s turns=%d)",
+            round_label,
+            repo,
+            number,
+            f"${cost:.2f}" if cost else "unreported",
+            role_result.session.num_turns,
+        )
+        return round_label
+    except EXPECTED_FAILURES as exc:  # advisory: never fail the target repo's CI
+        log.warning("agent review did not complete: %s: %s", type(exc).__name__, exc)
+        if emit_path is not None:
+            # the invariant holds here too: the workflow backstop would cover
+            # a missing file, but with a generic detail — the real failure is
+            # the one worth reading on the PR
+            with contextlib.suppress(Exception):
+                _emit(
+                    emit_path,
+                    repo,
+                    number,
+                    kind="skip-stub",
+                    detail=f"{type(exc).__name__}: {exc}",
+                    reviewed_by=backend_id(harness),
+                )
+        return None

outerloop/review_agent_cli.py

@@ -0,0 +1,209 @@
+"""Entry point for the agent-session advisory reviewer.
+
+Runs the reviewer as an agent over a PR-head checkout the workflow prepared
+(REVIEW_CHECKOUT), and posts the findings inline. Exits 0 even on skip
+or failure — an advisory reviewer must never turn a target repo's CI red.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+from outerloop.github import EnvTokenProvider, GitHubClient
+from outerloop.harness import Harness
+from outerloop.review_agent import (
+    _emit,
+    run_agent_review,
+    sanitize_checkout,
+)
+from outerloop.role_runner import build_harness
+from outerloop.roles import reviewer_spec
+from outerloop.rolespec import RoleSpec
+
+log = logging.getLogger(__name__)
+
+
+def _skip_stub(emit_env: str, repo: str, number: int, detail: str, reviewed_by: str) -> None:
+    """A fail-closed skip still leaves an envelope when emitting: the post job
+    REQUIRES an artifact, and a standing reviewer that cannot run must say so
+    on the PR rather than fail into silence."""
+    log.warning("%s; skipping review", detail)
+    if emit_env:
+        _emit(
+            Path(emit_env).resolve(),
+            repo,
+            number,
+            kind="skip-stub",
+            detail=detail,
+            reviewed_by=reviewed_by,
+        )
+
+
+def resolve_reviewer_harness(spec: RoleSpec) -> tuple[Harness | None, str, str]:
+    """Resolve the reviewer-env backend contract into a built harness:
+    `(harness, "", backend_label)` on success, `(None, why, backend_label)`
+    on a config that must skip. The ONE owner of the REVIEW_BACKEND /
+    REVIEW_MODEL / REVIEW_HERMES_* / key-var contract, shared by the
+    reviewer and the summarizer CLIs — free-form caller inputs are compared
+    with EXACTLY GitHub's expression semantics (case-insensitive, never
+    trimmed) so this code and the workflow's key-injection expressions can
+    never disagree about a value."""
+    backend = os.environ.get("REVIEW_BACKEND", "claude").lower()
+    review_model = os.environ.get("REVIEW_MODEL", "").strip()
+    hermes_provider = os.environ.get("REVIEW_HERMES_PROVIDER", "").lower() or "openrouter"
+    key_var = {
+        "claude": "ANTHROPIC_REVIEWER_KEY",
+        "codex": "OPENAI_REVIEWER_KEY",
+        "hermes": {"openrouter": "OPENROUTER_API_KEY", "openai": "OPENAI_REVIEWER_KEY"}.get(
+            hermes_provider
+        ),
+    }.get(backend)
+    if key_var is None:
+        what = (
+            f"unknown REVIEW_HERMES_PROVIDER {hermes_provider!r}"
+            if backend == "hermes"
+            else f"unknown REVIEW_BACKEND {backend!r}"
+        )
+        return None, what, backend
+    from outerloop.harness import vertex_from_env
+
+    api_key = os.environ.get(key_var, "").strip()
+    # an ADC-only deployment holds no Anthropic key: Vertex covering the
+    # claude backend stands in for it (same tolerance as role_key)
+    vertex_covers = backend == "claude" and vertex_from_env() is not None
+    if not api_key and not vertex_covers:
+        return None, f"{key_var} is unset or empty", backend
+    hermes_repo: Path | None = None
+    provider = ""
+    if backend == "hermes":
+        hermes_repo_env = os.environ.get("REVIEW_HERMES_REPO", "").strip()
+        if not hermes_repo_env:
+            return None, "REVIEW_HERMES_REPO is unset (hermes needs its pinned clone)", backend
+        hermes_repo = Path(hermes_repo_env).resolve()
+        provider = hermes_provider
+        if provider == "openrouter" and review_model and "/" not in review_model:
+            return (
+                None,
+                "hermes+openrouter requires an OpenRouter-shaped REVIEW_MODEL "
+                "(openai/gpt-5.6-terra, with the provider prefix)",
+                backend,
+            )
+        if provider == "openai" and (not review_model or "/" in review_model):
+            return (
+                None,
+                "hermes+openai requires a provider-native REVIEW_MODEL "
+                "(gpt-5.6-terra, no openai/ prefix)",
+                backend,
+            )
+    harness = build_harness(
+        api_key,
+        spec,
+        backend=backend,
+        binary=os.environ.get("REVIEW_BINARY") or None,  # else the backend default on PATH
+        model=review_model or None,
+        hermes_repo=hermes_repo,
+        hermes_provider=provider,
+    )
+    return harness, "", backend
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    # Fail closed on a missing/invalid PR reference too, so a misconfigured
+    # workflow skips cleanly rather than exiting nonzero (never red the CI).
+    repo = os.environ.get("PR_REPO", "").strip()
+    number_raw = os.environ.get("PR_NUMBER", "").strip()
+    if not repo or not number_raw.isdigit():
+        log.warning("PR_REPO/PR_NUMBER unset or invalid; skipping")
+        return 0
+    number = int(number_raw)
+    emit_env = os.environ.get("REVIEW_EMIT_FILE", "").strip()
+    # Fail closed: without the bot login we cannot honor "never review
+    # bot-authored PRs", so we do not review at all.
+    bot_login = os.environ.get("REVIEW_BOT_LOGIN", "").strip()
+    if not bot_login:
+        _skip_stub(emit_env, repo, number, "REVIEW_BOT_LOGIN is unset", "")
+        return 0
+    # Backend is a deployment choice, not baked in: the shared resolver owns
+    # the REVIEW_BACKEND/REVIEW_MODEL/REVIEW_HERMES_*/key-var contract (also
+    # used by the summarizer CLI); any config that must skip becomes a
+    # PR-visible stub, never a silent log line or a traceback.
+    # the backend label for pre-harness stubs (attribution only; the resolver
+    # below re-derives it authoritatively)
+    backend = os.environ.get("REVIEW_BACKEND", "claude").lower()
+    # The workflow checks out the PR head into REVIEW_CHECKOUT for the agent
+    # to investigate (it records its verdict via the syscall tool). Fail closed:
+    # defaulting to cwd would silently review the wrong tree (the reviewer's
+    # own repo) if the checkout step were misconfigured.
+    checkout = os.environ.get("REVIEW_CHECKOUT", "").strip()
+    if not checkout:
+        _skip_stub(
+            emit_env,
+            repo,
+            number,
+            "REVIEW_CHECKOUT is unset (won't review the wrong tree)",
+            backend,
+        )
+        return 0
+    workspace = Path(checkout).resolve()
+    # The checkout is untrusted: rename instruction files (CLAUDE.md, .claude/
+    # hooks, ...) so no backend auto-loads PR content as instructions. A rename
+    # failure means an instruction file is still live — fail closed.
+    renamed, failed = sanitize_checkout(workspace)
+    if failed:
+        _skip_stub(
+            emit_env,
+            repo,
+            number,
+            f"checkout could not be fully sanitized ({failed} instruction files left)",
+            backend,
+        )
+        return 0
+    if renamed:
+        log.info("sanitized %d instruction file(s) in the checkout", renamed)
+
+    lens = os.environ.get("REVIEW_LENS", "").strip()
+    if lens:
+        from outerloop.review import REVIEW_LENSES
+
+        if lens != "general" and lens not in REVIEW_LENSES:
+            # a typo'd lens must be a PR-visible stub, never a silent
+            # default-review (a configured lens must never quietly vanish)
+            _skip_stub(
+                emit_env,
+                repo,
+                number,
+                f"unknown REVIEW_LENS {lens!r} (have: {sorted(REVIEW_LENSES)})",
+                backend,
+            )
+            return 0
+
+    harness, why, backend = resolve_reviewer_harness(reviewer_spec())
+    if harness is None:
+        _skip_stub(emit_env, repo, number, why, backend)
+        return 0
+
+    client = GitHubClient(auth=EnvTokenProvider("GITHUB_TOKEN"))
+    # Least-token split: with REVIEW_EMIT_FILE set, findings are written there
+    # instead of posted — this job then needs only READ permissions, and a
+    # separate posting job (review_post_cli) holds the write token with no
+    # session next to it. This is what makes non-Claude backends safe on the
+    # auto path (docs/design/reviewer-infra.md).
+    run_agent_review(
+        client,
+        repo,
+        number,
+        harness,
+        workspace,
+        bot_login=bot_login,
+        emit_path=Path(emit_env).resolve() if emit_env else None,
+        lens=lens,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

outerloop/review_post_cli.py

@@ -0,0 +1,162 @@
+"""Posting half of the least-token split.
+
+Reads the findings file the session job emitted (`REVIEW_EMIT_FILE`),
+re-validates it, and posts through the normal advisory path. This job holds
+the write token; no model session runs next to it. The artifact crosses a
+job boundary, so nothing in it is trusted: the envelope must name this PR,
+the skip rules are re-checked, and every string passes the same sanitizing
+render as the single-job path. Exits 0 on every failure — advisory means
+advisory.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+from outerloop.github import EnvTokenProvider, GitHubClient
+from outerloop.posting import EXPECTED_FAILURES, post_round_review, post_skip_stub
+from outerloop.review import (
+    MARKER,
+    format_comment,
+    format_review,
+    result_from_data,
+    sanitize,
+    skip_reason,
+)
+from outerloop.review_agent import _pull_request
+
+log = logging.getLogger(__name__)
+
+# An opinion label rides into the round body so a human can tell the second
+# opinion from the primary reviewer at a glance. Caller-configured, but
+# rendered into a comment: length-capped and newline-stripped.
+MAX_OPINION_LABEL = 80
+
+
+def post_from_file(
+    client: GitHubClient,
+    repo: str,
+    number: int,
+    bot_login: str,
+    path: Path,
+    opinion_label: str = "",
+) -> str | None:
+    """Post the emitted findings (or skip stub). Returns the round label, or
+    None when there was nothing to post or the envelope was refused."""
+    try:
+        envelope = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError) as exc:
+        log.warning("findings file unreadable (%s); nothing posted", exc)
+        return None
+    if not isinstance(envelope, dict):
+        log.warning("findings file is not an object; nothing posted")
+        return None
+    if envelope.get("repo") != repo or envelope.get("number") != number:
+        log.warning("findings envelope names a different PR; refused")
+        return None
+    kind = envelope.get("kind")
+    if kind == "skip-clean":
+        # a clean skip (bot PR, opt-out) posts nothing by design; the
+        # envelope exists so a MISSING artifact always means a broken session
+        log.info("session skipped cleanly (%s); nothing to post", envelope.get("detail", ""))
+        return None
+    if kind not in ("skip-stub", "findings"):
+        log.warning("unknown envelope kind %r; nothing posted", kind)
+        return None
+    try:
+        # The write authority re-checks the skip rules for EVERY kind: the
+        # session job decided them once, but this side of the artifact
+        # boundary is the one that must never post on a bot PR — a forged
+        # stub envelope is still a post.
+        pr, pr_data = _pull_request(client, repo, number)
+        skip = skip_reason(pr, bot_login)
+        if skip is not None:
+            log.info("skipping post on %s#%s: %s", repo, number, skip)
+            return None
+        if kind == "skip-stub":
+            # sanitize: the detail crossed the job boundary and lands in a
+            # comment — collapse newlines, neutralize markdown, cap length
+            detail = sanitize(str(envelope.get("detail", "")), 300)
+            # name WHOSE round failed: with two standing reviewers, an
+            # unattributed stub is ambiguous exactly when a key dies
+            who = " ".join(opinion_label.split())[:MAX_OPINION_LABEL] or str(
+                envelope.get("reviewed_by", "")
+            )
+            role = f"advisory review ({who})" if who else "advisory review"
+            post_skip_stub(client, repo, number, role, RuntimeError(detail))
+            return "skip-stub"
+        data = envelope.get("data")
+        review = result_from_data(data if isinstance(data, dict) else {})
+        rendered = format_review(review, pr.diff)
+        full = format_comment(review)
+        if rendered is None or full is None:
+            log.info("nothing to post")
+            return None
+        body, inline = rendered
+        if opinion_label:
+            # AFTER the marker, never before it: round counting and the
+            # quote-reply defense both match marker-FIRST bodies
+            label = " ".join(opinion_label.split())[:MAX_OPINION_LABEL]
+            body = body.replace(MARKER, f"{MARKER}\n*{label}*", 1)
+            full = full.replace(MARKER, f"{MARKER}\n*{label}*", 1)
+        round_label = post_round_review(
+            client,
+            repo,
+            number,
+            MARKER,
+            body,
+            inline,
+            pr_data,
+            fallback_body=full,
+            reviewed_by=str(envelope.get("reviewed_by", "")),
+        )
+        log.info(
+            "posted review (%s) on %s#%s (reviewer=%s)",
+            round_label,
+            repo,
+            number,
+            str(envelope.get("reviewed_by", "")) or "unattributed",
+        )
+        return round_label
+    except EXPECTED_FAILURES as exc:  # advisory: never fail the target repo's CI
+        log.warning("posting did not complete: %s: %s", type(exc).__name__, exc)
+        return None
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    repo = os.environ.get("PR_REPO", "").strip()
+    number_raw = os.environ.get("PR_NUMBER", "").strip()
+    if not repo or not number_raw.isdigit():
+        log.warning("PR_REPO/PR_NUMBER unset or invalid; skipping")
+        return 0
+    bot_login = os.environ.get("REVIEW_BOT_LOGIN", "").strip()
+    if not bot_login:
+        log.warning("REVIEW_BOT_LOGIN is unset; skipping (cannot re-check the bot skip)")
+        return 0
+    emit_file = os.environ.get("REVIEW_EMIT_FILE", "").strip()
+    if not emit_file:
+        log.warning("REVIEW_EMIT_FILE is unset; skipping")
+        return 0
+    path = Path(emit_file).resolve()
+    if not path.is_file():
+        log.info("no findings file at %s (clean skip upstream); nothing to post", path)
+        return 0
+    client = GitHubClient(auth=EnvTokenProvider("GITHUB_TOKEN"))
+    post_from_file(
+        client,
+        repo,
+        int(number_raw),
+        bot_login,
+        path,
+        opinion_label=os.environ.get("REVIEW_OPINION_LABEL", "").strip(),
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())