ctrlrelay 0.10.0__py3-none-any.whl

ctrlrelay/core/pr_verifier.py

@@ -0,0 +1,230 @@
+"""PR verification: wait for CI and confirm mergeability before hand-off."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any
+
+from ctrlrelay.core.github import GitHubCLI, GitHubError
+from ctrlrelay.core.obs import get_logger, log_event
+
+_logger = get_logger("core.pr_verifier")
+
+# `gh pr checks --json bucket` returns one of: pass, fail, pending, skipping, cancel.
+# Treat skipping as pass (skipped jobs don't block a merge) and everything except
+# pending as "terminal".
+_PENDING_BUCKETS = frozenset({"pending"})
+_PASSING_BUCKETS = frozenset({"pass", "skipping"})
+_TERMINAL_MERGEABLE_VALUES = frozenset({"MERGEABLE", "CONFLICTING"})
+
+# The dev pipeline's contract is "open a PR and hand it to humans for review",
+# not "merge the PR". So the verifier does NOT require the PR be in a directly-
+# mergeable state — human-gated states (awaiting review, unresolved comments,
+# pending deployments, merge-queue requirements) are all expected terminal
+# states for us. What we DO reject is state that (a) indicates broken code
+# (failing checks) or (b) the orchestrator can itself fix before hand-off
+# (conflicts, behind base). All other mergeStateStatus values are accepted.
+_REBASE_REQUIRED_MERGE_STATE_STATUS = frozenset({"BEHIND"})
+
+# Require 2 consecutive empty check-list responses separated by a poll interval
+# before concluding "no CI configured". GitHub registers check runs a few
+# seconds after `gh pr create`, so a single-shot empty read is unreliable.
+_EMPTY_CHECKS_CONFIRM_POLLS = 2
+
+
+@dataclass
+class VerificationResult:
+    """Outcome of verifying a PR is ready for hand-off."""
+
+    ready: bool
+    reason: str = ""
+    failing_checks: list[dict[str, Any]] = field(default_factory=list)
+    pending_checks: list[dict[str, Any]] = field(default_factory=list)
+    # Set True when wait_for_checks returned with pending entries (timeout hit
+    # while CI was still running). Callers use this to distinguish "needs a
+    # fix" from "just slow CI" and avoid burning retry budget on the latter.
+    timed_out: bool = False
+    mergeable: str | None = None
+    merge_state_status: str | None = None
+
+
+@dataclass
+class PRVerifier:
+    """Verifies a PR is green and conflict-free before declaring a dev task done."""
+
+    github: GitHubCLI
+    poll_interval: int = 30
+    check_timeout: int = 1800
+    mergeable_poll_attempts: int = 10
+
+    async def wait_for_checks(
+        self,
+        repo: str,
+        pr_number: int,
+        timeout: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Poll PR checks until every check has left the 'pending' bucket or
+        the timeout is reached.
+
+        Empty-check handling: GitHub registers check runs asynchronously after
+        `gh pr create` so a single empty read is ambiguous — the repo might
+        have no CI, or CI just hasn't registered yet. We require
+        `_EMPTY_CHECKS_CONFIRM_POLLS` consecutive empty reads separated by
+        `poll_interval` before concluding "no CI configured".
+
+        Transient `gh` failures (subprocess timeout, GitHubError) are
+        treated as "still pending" and retried on the next iteration —
+        a flaky network or a GitHub rate-limit back-off shouldn't abort
+        the wait. The outer `limit` deadline is the safety net.
+
+        Sleep cap: when called with `timeout < poll_interval` the loop
+        used to block the full `poll_interval` before noticing the
+        deadline (issue #90). Now caps `sleep` at `max(0, limit -
+        elapsed)` so a 1-second timeout with a 15-second interval
+        returns within ~1 second.
+        """
+        limit = self.check_timeout if timeout is None else timeout
+        loop = asyncio.get_event_loop()
+        # Wall-clock deadline (monotonic) so the loop terminates even
+        # if `poll_interval` is 0 and every poll errors — accumulating
+        # sleep durations would loop forever in that case because
+        # max(0, 0) is still 0.
+        deadline = loop.time() + limit
+        empty_streak = 0
+        checks: list[dict[str, Any]] = []
+        had_successful_read = False
+        last_transient_error: Exception | None = None
+        while True:
+            try:
+                checks = await self.github.get_pr_checks(repo, pr_number)
+            except (GitHubError, asyncio.TimeoutError) as e:
+                # Transient gh failure — treat as still-pending, retry
+                # on next iteration. Unconditional re-raise of
+                # asyncio.TimeoutError used to surface as an ugly
+                # traceback in `ctrlrelay ci wait` (issue #90).
+                last_transient_error = e
+                log_event(
+                    _logger,
+                    "pr_verifier.transient_gh_error",
+                    repo=repo,
+                    pr_number=pr_number,
+                    reason=type(e).__name__,
+                    error=str(e)[:200],
+                )
+            else:
+                had_successful_read = True
+                last_transient_error = None
+                if not checks:
+                    empty_streak += 1
+                    if empty_streak >= _EMPTY_CHECKS_CONFIRM_POLLS:
+                        return checks
+                else:
+                    empty_streak = 0
+                    if all(
+                        c.get("bucket") not in _PENDING_BUCKETS
+                        for c in checks
+                    ):
+                        return checks
+            remaining = deadline - loop.time()
+            if remaining <= 0:
+                # Fail closed if every poll failed: an empty `checks`
+                # list would be misread as "no CI configured" and the
+                # caller would silently greenlight the PR while
+                # GitHub was actually unavailable. Surface the last
+                # transient error so the CLI / verify() can react —
+                # the widened except clause in `ci_wait` catches it
+                # cleanly. (Codex P1 caught on the first PR pass.)
+                if not had_successful_read and last_transient_error is not None:
+                    raise last_transient_error
+                return checks
+            sleep_for = min(self.poll_interval, remaining)
+            await asyncio.sleep(sleep_for)
+
+    async def verify(
+        self,
+        repo: str,
+        pr_number: int,
+        timeout: int | None = None,
+    ) -> VerificationResult:
+        """Wait for CI, then check mergeability. Report ready only when both are green."""
+        checks = await self.wait_for_checks(repo, pr_number, timeout=timeout)
+        pending = [c for c in checks if c.get("bucket") in _PENDING_BUCKETS]
+        failing = [
+            c for c in checks
+            if c.get("bucket") not in _PENDING_BUCKETS
+            and c.get("bucket") not in _PASSING_BUCKETS
+        ]
+        # Failing checks take priority over pending. A matrix where lint
+        # already failed but a long integration run is still pending must be
+        # reported as broken, not timed out — otherwise the caller would hand
+        # off a known-bad PR.
+        if failing:
+            names = ", ".join(c.get("name", "?") for c in failing)
+            return VerificationResult(
+                ready=False,
+                reason=f"{len(failing)} check(s) failing: {names}",
+                failing_checks=failing,
+                pending_checks=pending,
+            )
+        if pending:
+            # All failing paths ruled out; we simply hit the timeout while
+            # everything still in flight was healthy. Don't ask Claude to
+            # "fix" slow CI — surface it as a distinct outcome so the caller
+            # hands off the PR as-is.
+            names = ", ".join(c.get("name", "?") for c in pending)
+            return VerificationResult(
+                ready=False,
+                timed_out=True,
+                reason=(
+                    f"CI still running after timeout: {len(pending)} "
+                    f"check(s) pending ({names})"
+                ),
+                pending_checks=pending,
+            )
+
+        mergeable: str | None = None
+        merge_state: str | None = None
+        for _ in range(self.mergeable_poll_attempts):
+            state = await self.github.get_pr_state(repo, pr_number)
+            mergeable = state.get("mergeable")
+            merge_state = state.get("mergeStateStatus")
+            if mergeable in _TERMINAL_MERGEABLE_VALUES:
+                break
+            await asyncio.sleep(self.poll_interval)
+
+        if mergeable == "CONFLICTING":
+            return VerificationResult(
+                ready=False,
+                reason="PR has merge conflicts with the base branch",
+                mergeable=mergeable,
+                merge_state_status=merge_state,
+            )
+        if mergeable != "MERGEABLE":
+            return VerificationResult(
+                ready=False,
+                reason=f"PR mergeable state unresolved: {mergeable}",
+                mergeable=mergeable,
+                merge_state_status=merge_state,
+            )
+        if merge_state in _REBASE_REQUIRED_MERGE_STATE_STATUS:
+            return VerificationResult(
+                ready=False,
+                reason=(
+                    "PR is behind the base branch and must be rebased before "
+                    "merge (mergeStateStatus=BEHIND)"
+                ),
+                mergeable=mergeable,
+                merge_state_status=merge_state,
+            )
+
+        # Any remaining state (CLEAN, HAS_HOOKS, BLOCKED, UNSTABLE, DRAFT,
+        # etc.) is accepted. CI is verified green above, conflicts and
+        # behind-base are handled explicitly, so what's left is either
+        # directly mergeable or human-gated — both are valid hand-off states
+        # for a pipeline that never auto-merges.
+        return VerificationResult(
+            ready=True,
+            mergeable=mergeable,
+            merge_state_status=merge_state,
+        )

ctrlrelay/core/pr_watcher.py

@@ -0,0 +1,121 @@
+"""PR merge watcher for monitoring PR state."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Awaitable, Callable
+
+from ctrlrelay.core.github import GitHubCLI, GitHubError
+from ctrlrelay.core.obs import get_logger, log_event
+
+_logger = get_logger("core.pr_watcher")
+
+# After this many CONSECUTIVE transient failures we give up on the watch.
+# gh can raise "transient-looking" errors (GitHubError, TimeoutError,
+# OSError) for permanent problems too — bad repo, expired auth,
+# permission change, missing gh binary. Without a cap, those would
+# silently loop for the full 7-day timeout and never surface the
+# problem.
+#
+# Sizing: at the default 60s poll interval, 60 consecutive failures
+# covers a ~1-hour outage window. That's enough slack for routine VPN
+# flaps, GitHub incidents, or auth token rotations without abandoning
+# the watch, while still bounding truly permanent failures (deleted
+# repo, revoked credentials) to an hour instead of the full 7-day
+# timeout. A successful poll resets the counter, so genuinely
+# intermittent failures never accumulate.
+_TRANSIENT_FAILURE_CAP = 60
+
+
+@dataclass
+class PRWatcher:
+    """Watches PRs for merge events."""
+
+    github: GitHubCLI
+    poll_interval: int = 60
+
+    async def check_merged(self, repo: str, pr_number: int) -> bool:
+        """Check if a PR has been merged.
+
+        Args:
+            repo: Repository name (owner/repo)
+            pr_number: PR number
+
+        Returns:
+            True if merged, False otherwise
+        """
+        pr_state = await self.github.get_pr_state(repo, pr_number)
+        return pr_state.get("state") == "MERGED"
+
+    async def wait_for_merge(
+        self,
+        repo: str,
+        pr_number: int,
+        timeout: int = 86400,
+        on_poll: Callable[[], Awaitable[None]] | None = None,
+    ) -> bool:
+        """Wait for a PR to be merged.
+
+        Args:
+            repo: Repository name
+            pr_number: PR number
+            timeout: Max seconds to wait (default 24h)
+            on_poll: Optional callback after each poll
+
+        Returns:
+            True if merged within timeout, False otherwise
+
+        Transient-failure handling: individual ``gh`` failures
+        (``GitHubError``, ``TimeoutError``, network-level ``OSError``)
+        during a multi-day watch MUST NOT abort the loop — otherwise a
+        single flaky poll cycle permanently stops monitoring the PR.
+        Log a structured ``pr_watch.transient_error`` event and keep
+        polling. ``asyncio.CancelledError`` is always re-raised so a
+        clean shutdown propagates.
+        """
+        elapsed = 0
+        consecutive_failures = 0
+        while elapsed < timeout:
+            try:
+                if await self.check_merged(repo, pr_number):
+                    return True
+                consecutive_failures = 0  # successful poll resets the counter
+            except asyncio.CancelledError:
+                raise
+            except (GitHubError, TimeoutError, OSError) as e:
+                consecutive_failures += 1
+                log_event(
+                    _logger, "pr_watch.transient_error",
+                    repo=repo, pr_number=pr_number,
+                    reason=type(e).__name__,
+                    error=str(e)[:200],
+                    elapsed=elapsed,
+                    consecutive_failures=consecutive_failures,
+                )
+                if consecutive_failures >= _TRANSIENT_FAILURE_CAP:
+                    # Likely permanent: bad repo, expired auth, 404,
+                    # missing gh binary. Fail fast instead of zombie-
+                    # sleeping for 7 days.
+                    log_event(
+                        _logger, "pr_watch.abandoned_after_too_many_errors",
+                        repo=repo, pr_number=pr_number,
+                        consecutive_failures=consecutive_failures,
+                        last_reason=type(e).__name__,
+                        last_error=str(e)[:200],
+                    )
+                    raise
+                # Fall through to the sleep + retry.
+
+            if on_poll:
+                try:
+                    await on_poll()
+                except asyncio.CancelledError:
+                    raise
+                except Exception:
+                    pass  # on_poll is best-effort diagnostic plumbing
+
+            await asyncio.sleep(self.poll_interval)
+            elapsed += self.poll_interval
+
+        return False

ctrlrelay/core/question_expiry.py

@@ -0,0 +1,193 @@
+"""Retirement of BLOCKED questions the operator can no longer usefully answer.
+
+A ``pending_resumes`` row deliberately outlives the in-session wait: the
+transport gives up after ``ask_timeout_seconds``, but the row survives so
+a reply arriving hours later still drives the resume. Nothing ever
+retired those rows, so two failure modes accumulated:
+
+- **Nobody answered.** Real numbers from one deployment: 46 questions
+  posted, 4 ever answered. The unanswered 42 stayed live targets for
+  orphan reply routing, and each sweep re-posted the same question.
+- **Somebody acted outside ctrlrelay.** The PR got merged by hand, or
+  the issue was closed by a teammate. The question is now not just
+  unanswered but meaningless, and an answer to it would drive a resume
+  against state that has already moved on.
+
+This module retires both. It only ever sets ``expired_at`` on rows that
+are still unanswered, so a reply that lands in the same tick wins.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from typing import Any
+
+from ctrlrelay.core.github import GitHubCLI, GitHubError
+from ctrlrelay.core.obs import get_logger, log_event
+from ctrlrelay.core.state import StateDB
+
+_logger = get_logger("core.question_expiry")
+
+# Captures both "PR #60" and "#60" forms the agent uses interchangeably
+# in BLOCKED questions. The negative-lookahead on trailing digits avoids
+# matching CVE-2026-... style identifiers; PR numbers are always >= 1.
+_PR_NUM_RE = re.compile(r"(?:PR\s*)?#(\d+)(?!\d)", re.IGNORECASE)
+
+# Per-call cap on the `gh api` probe. The sweep runs on a schedule and
+# holds no repo lock, but a hung gh would still stall the whole pass;
+# a single REST read has no business taking longer than this.
+_STATE_PROBE_TIMEOUT_SECONDS = 15
+
+# Ceiling on probes per run, so one pass cannot outlive its own cron
+# interval. Unbounded, the cost is one serialized `gh api` call per
+# cited number: 2000 backlogged rows against a slow GitHub is
+# 2000 x 15s = 8.3h of a job that fires hourly, and the sweep would
+# spend all day overlapping itself. Rows past the cap are simply
+# examined on the next run — expiry is never urgent, and the TTL pass
+# retires them on age regardless of whether they were ever probed.
+# 200 x 15s worst case = 50min, inside the hour with room to spare.
+_MAX_RESOLVED_PROBES_PER_RUN = 200
+
+EXPIRY_REASON_TTL = "ttl"
+EXPIRY_REASON_RESOLVED = "resolved_elsewhere"
+
+
+def extract_referenced_numbers(question: str) -> list[str]:
+    """Pull deduplicated issue/PR numbers out of a question, in the
+    order the agent listed them."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for n in _PR_NUM_RE.findall(question or ""):
+        if n in seen:
+            continue
+        seen.add(n)
+        out.append(n)
+    return out
+
+
+async def _all_references_resolved(
+    github: GitHubCLI,
+    *,
+    repo: str,
+    numbers: list[str],
+) -> bool:
+    """True when every cited number is closed or merged.
+
+    Conservative on purpose. One still-open reference means the question
+    retains something to decide, so the row stays. Any probe error at
+    all — network, auth, a number that names nothing — returns False:
+    we would rather re-ask a dead question than silently drop a live
+    one, since the operator can ignore noise but cannot recover a
+    question that was never posted.
+    """
+    if not numbers:
+        return False
+
+    for number in numbers:
+        try:
+            state = await github.get_issue_or_pr_state(
+                repo,
+                int(number),
+                timeout=_STATE_PROBE_TIMEOUT_SECONDS,
+            )
+        except (GitHubError, TimeoutError, OSError, ValueError):
+            return False
+        except Exception:
+            return False
+
+        if (state.get("state") or "").lower() != "closed":
+            return False
+
+    return True
+
+
+async def expire_stale_questions(
+    state_db: StateDB,
+    github: GitHubCLI | None,
+    *,
+    ttl_seconds: int,
+    now: int | None = None,
+) -> list[dict[str, Any]]:
+    """Retire questions that aged out or whose subject is already
+    resolved. Returns the rows actually expired.
+
+    TTL runs first and needs no network, so a GitHub outage still lets
+    the age-based sweep make progress. ``github=None`` disables only the
+    resolved-elsewhere pass.
+    """
+    now = int(time.time()) if now is None else now
+    expired: list[dict[str, Any]] = []
+
+    for row in state_db.list_expirable_pending_resumes(now - ttl_seconds):
+        if state_db.expire_pending_resume(row["session_id"], EXPIRY_REASON_TTL):
+            expired.append({**row, "expired_reason": EXPIRY_REASON_TTL})
+            log_event(
+                _logger,
+                "question.expired",
+                session_id=row["session_id"],
+                repo=row["repo"],
+                pipeline=row["pipeline"],
+                reason=EXPIRY_REASON_TTL,
+                age_seconds=now - int(row["created_at"]),
+            )
+
+    if github is None:
+        return expired
+
+    probes_used = 0
+    for row in state_db.list_unanswered_pending_resumes():
+        numbers = extract_referenced_numbers(row["question"])
+        # Rows citing nothing are skipped without spending budget —
+        # they can never be resolution-expired anyway.
+        if not numbers:
+            continue
+
+        # Check the budget against what this row will actually COST,
+        # before spending any of it. Testing `probes_used` alone lets a
+        # single row overshoot without limit: one question citing 300
+        # numbers passes a `0 >= 200` check and then issues all 300
+        # calls, ~75min of a job that fires hourly.
+        if len(numbers) > _MAX_RESOLVED_PROBES_PER_RUN:
+            # Bigger than any run's whole budget, so it can never be
+            # resolution-checked — expiry needs EVERY reference closed.
+            # Skip rather than break: one pathological row must not
+            # starve the rest. The TTL pass still retires it on age.
+            log_event(
+                _logger,
+                "question.expiry.row_exceeds_budget",
+                session_id=row["session_id"],
+                repo=row["repo"],
+                references=len(numbers),
+                budget=_MAX_RESOLVED_PROBES_PER_RUN,
+            )
+            continue
+        if probes_used + len(numbers) > _MAX_RESOLVED_PROBES_PER_RUN:
+            log_event(
+                _logger,
+                "question.expiry.probe_budget_exhausted",
+                budget=_MAX_RESOLVED_PROBES_PER_RUN,
+                probes_used=probes_used,
+                reason="remaining rows deferred to the next run",
+            )
+            break
+        probes_used += len(numbers)
+        if not await _all_references_resolved(
+            github, repo=row["repo"], numbers=numbers
+        ):
+            continue
+        if state_db.expire_pending_resume(
+            row["session_id"], EXPIRY_REASON_RESOLVED
+        ):
+            expired.append({**row, "expired_reason": EXPIRY_REASON_RESOLVED})
+            log_event(
+                _logger,
+                "question.expired",
+                session_id=row["session_id"],
+                repo=row["repo"],
+                pipeline=row["pipeline"],
+                reason=EXPIRY_REASON_RESOLVED,
+                references=",".join(numbers),
+            )
+
+    return expired