code-coordinator 0.5.204__tar.gz → 0.5.205__tar.gz

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.204
+Version: 0.5.205
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.204
+Version: 0.5.205
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/code_coordinator.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ coord/brain.py
 coord/branch_model.py
 coord/bug_intake.py
 coord/cargo_cache.py
+coord/ci_fix.py
 coord/ci_github.py
 coord/ci_gitlab.py
 coord/ci_store.py
@@ -383,6 +384,7 @@ tests/test_build_worker_image_env_update.py
 tests/test_cargo_cache.py
 tests/test_check_gh_argv_containment.py
 tests/test_ci_acceptance_gate_1950.py
+tests/test_ci_fix.py
 tests/test_ci_gitlab.py
 tests/test_ci_parity_2091.py
 tests/test_ci_store.py

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/code_coordinator.egg-info/scm_file_list.json

@@ -40,6 +40,7 @@
     "coord/branch_model.py",
     "coord/bug_intake.py",
     "coord/cargo_cache.py",
+    "coord/ci_fix.py",
     "coord/ci_github.py",
     "coord/ci_gitlab.py",
     "coord/ci_store.py",
@@ -445,6 +446,7 @@
     "tests/test_cargo_cache.py",
     "tests/test_check_gh_argv_containment.py",
     "tests/test_ci_acceptance_gate_1950.py",
+    "tests/test_ci_fix.py",
     "tests/test_ci_gitlab.py",
     "tests/test_ci_parity_2091.py",
     "tests/test_ci_store.py",

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.204",
+  "tag": "0.5.205",
   "distance": 0,
-  "node": "g471c14e8906b4f0ebf43a5b08365c101a0b02156",
+  "node": "g464b6f72ce8e60436c28ceba3d8453892108cb88",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-21"

code_coordinator-0.5.205/coord/ci_fix.py

@@ -0,0 +1,209 @@
+"""Auto-dispatch a fix worker for a CONFIRMED CI check failure at the Merge
+gate (#2510).
+
+``coord/merge_queue.py``'s CI gate already classifies a failing check into
+three shapes: a verdictless (infra) failure auto-reruns up to
+``MAX_CI_INFRA_RERUNS``; a verdicted failure gets one flake re-check up to
+``MAX_CI_FLAKY_RERUNS``; and once both budgets are exhausted/inapplicable the
+failure is CONFIRMED — real, not infra, not (as far as one re-run can tell)
+flaky. Both exhaustion points emit a ``"checks_failed"`` :class:`~coord.
+merge_queue.MergeEvent`. Before this module existed that event did nothing
+but set ``entry.error`` — the entry stayed ``PENDING`` in the merge queue
+forever, invisible to any status view that only surfaces ``HUMAN_REQUIRED``
+rows, with no automated path back to a fix.
+
+Why a separate module, following ``coord/conflict_fix.py``'s lead: triggered
+by a merge_queue event, not a planner proposal, so it shares little with
+``coord.dispatch``.
+
+Unlike a merge CONFLICT (mechanical, content-preserving, handled by a
+dedicated ``type="conflict-fix"`` worker whose success needs no fresh
+Test/Review since the diff is unchanged), a confirmed CI failure needs a
+worker that can genuinely change code — the same shape as a review
+``request-changes`` bounce. So this dispatches via
+:func:`coord.auto_loop._dispatch_fix` itself (not a bespoke HTTP POST): the
+new commit lands on the SAME branch/PR, and the existing generic Work→Test→
+Review pipeline (already built to re-drive Test then Review for any bounced
+``WORK_LIKE_TYPES`` fix) picks the new commit up on its own — no separate
+re-review wiring needed here. The merge queue's own ``branch_head_sha``/
+``branch_patch_id`` staleness check (see ``has_approved_review``) already
+invalidates the stale approval once the new commit lands, so the PR
+naturally cannot merge again until it clears Test + Review + CI fresh.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from coord.config import Config
+from coord.merge_queue import QueuedMerge, _chain_work_ids
+from coord.models import Assignment, Board
+
+_log = logging.getLogger(__name__)
+
+
+# #2510: bounded the same shape as MAX_CI_INFRA_RERUNS/MAX_CI_FLAKY_RERUNS —
+# a couple of genuine attempts is enough to tell "this is fixable by a
+# worker" from "this needs a human"; a higher cap just burns more of the
+# same exhausted budget the way an unbounded conflict-fix retry would.
+MAX_CI_FIX_DISPATCHES = 2
+
+# Prefix on the dispatched fix worker's issue_title — mirrors
+# `conflict_fix.SEMANTIC_FIX_TITLE_PREFIX`'s visibility purpose: the TUI
+# Pipeline row and the fix briefing itself both make it obvious WHY this
+# round was dispatched (a CI failure, not a review request-changes bounce).
+CI_FIX_TITLE_PREFIX = "[ci-fix]"
+
+
+def build_ci_fix_briefing(
+    *,
+    entry: QueuedMerge,
+    checks_summary: str,
+    attempt: int,
+) -> str:
+    """Assemble the CI-fix worker's briefing. Pure function — testable."""
+    lines: list[str] = [
+        f"# CI failure fix: {entry.repo_github} branch `{entry.branch}`",
+        "",
+        f"Issue #{entry.issue_number} — {entry.issue_title}",
+        "",
+        "Review already approved this PR and it reached the Merge gate, "
+        "where GitHub Actions CI reported a REAL failure (not infra, and "
+        "not resolved by one automatic re-run) on the checks below:",
+        "",
+        f"    {checks_summary}",
+        "",
+        f"This is fix attempt {attempt}/{MAX_CI_FIX_DISPATCHES} for this "
+        "failure streak — the coordinator will escalate to a human if this "
+        "many attempts don't produce a green run.",
+        "",
+        "## What to do",
+        "",
+        "1. You are already on this issue's existing branch — continue on "
+        "it, do NOT start a fresh branch.",
+        "2. Reproduce the failure locally if the project's test command "
+        "covers it (`coord`'s Test stage already runs the same diff-scoped "
+        "suite CI reruns in full — a failure CI alone caught may live in a "
+        "suite that only runs in CI, e.g. a sealed acceptance suite).",
+        "3. Fix the actual regression. Do not edit the failing check's "
+        "workflow config to make it pass without fixing the underlying "
+        "issue, and do not skip/xfail the failing test unless the test "
+        "itself is proven wrong.",
+        "4. Commit and push to the SAME branch. The coordinator re-runs "
+        "Test, Review, and CI from scratch on the new commit — this is not "
+        "a force-merge shortcut.",
+        "",
+        "You will NOT use `gh` — the coordinator owns PR/issue interaction "
+        "and CI status reads.",
+        "",
+        f"Last merge-gate error: {entry.error or checks_summary}",
+    ]
+    return "\n".join(lines)
+
+
+def _has_active_fix(board: Board, entry: QueuedMerge) -> bool:
+    """True when a WORK_LIKE fix for *entry*'s chain is already running or
+    pending — regardless of whether it was dispatched for this CI failure or
+    an unrelated review bounce; either way a second dispatch here would race
+    it onto the same branch.
+    """
+    pool = list(board.completed) + list(board.active)
+    chain_ids = _chain_work_ids(entry, pool)
+    active_ids = {
+        getattr(a, "assignment_id", None) for a in board.active
+    } - {None}
+    for a in pool:
+        aid = getattr(a, "assignment_id", None)
+        if aid is None or aid not in chain_ids or aid == entry.assignment_id:
+            continue
+        if aid in active_ids and getattr(a, "status", None) in ("running", "pending"):
+            return True
+    return False
+
+
+def dispatch_ci_fix(
+    entry: QueuedMerge,
+    board: Board,
+    config: Config,
+    *,
+    checks_summary: str | None = None,
+    http_client: httpx.Client | None = None,
+) -> Assignment | None:
+    """Dispatch a fix worker for *entry*'s confirmed CI failure.
+
+    Returns the new ``Assignment``, or ``None`` when dispatch couldn't
+    proceed: the retry cap (:data:`MAX_CI_FIX_DISPATCHES`) is already spent,
+    a fix for this chain is already in flight, the original work assignment
+    can't be found on *board*, or the underlying ``_dispatch_fix`` call
+    itself declined (no capable machine, agent unreachable, …). The caller
+    (``coord.commands.merge._dispatch_ci_fixes``) is responsible for
+    promoting *entry* to ``HUMAN_REQUIRED`` when this returns ``None`` with
+    the retry cap exhausted, and for persisting the board/queue.
+
+    Does NOT increment ``entry.ci_fix_dispatches`` on a ``None`` return —
+    only a successful dispatch spends the budget, mirroring how
+    ``ci_infra_reruns``/``ci_flaky_reruns`` are only bumped when their
+    respective remedy actually fired.
+    """
+    if entry.ci_fix_dispatches >= MAX_CI_FIX_DISPATCHES:
+        return None
+    if _has_active_fix(board, entry):
+        return None
+    if entry.assignment_id is None:
+        return None
+    work = board.find_by_id(entry.assignment_id)
+    if work is None:
+        _log.warning(
+            "ci_fix: cannot find original work assignment %s for %s#%d — "
+            "no fix dispatched",
+            entry.assignment_id, entry.repo_name, entry.issue_number,
+        )
+        return None
+
+    summary = checks_summary or entry.error or "CI checks failed"
+    briefing = build_ci_fix_briefing(
+        entry=entry, checks_summary=summary, attempt=entry.ci_fix_dispatches + 1,
+    )
+
+    from coord.auto_loop import _dispatch_fix  # noqa: PLC0415
+
+    fix = _dispatch_fix(
+        work, briefing, board, config, entry.ci_fix_dispatches + 1,
+        http_client=http_client,
+    )
+    if fix is None:
+        return None
+
+    # #2510 visibility: mark the row so the TUI/board can tell a CI-triggered
+    # round apart from an ordinary review-bounce fix at a glance, same
+    # purpose as conflict_fix's SEMANTIC_FIX_TITLE_PREFIX.
+    fix.issue_title = f"{CI_FIX_TITLE_PREFIX} {work.issue_title}"
+
+    entry.ci_fix_dispatches += 1
+
+    from coord.audit import record_audit  # noqa: PLC0415
+
+    record_audit(
+        tier="operational",
+        category="merge",
+        event_type="ci_fix_dispatched",
+        actor="daemon",
+        summary=(
+            f"ci-fix dispatched ({entry.ci_fix_dispatches}/"
+            f"{MAX_CI_FIX_DISPATCHES}): {entry.repo_name}#{entry.issue_number} "
+            f"-> {fix.machine_name}"
+        ),
+        repo=entry.repo_name,
+        issue=entry.issue_number,
+        assignment_id=fix.assignment_id,
+        machine=fix.machine_name,
+        details={
+            "merge_entry_id": entry.assignment_id,
+            "ci_fix_dispatches": entry.ci_fix_dispatches,
+            "checks_summary": summary,
+        },
+    )
+
+    return fix

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/coord/commands/merge.py

@@ -539,6 +539,85 @@ def _dispatch_conflict_fixes(events, config, *, dry_run: bool) -> None:
         save_board(fix_board)
 
 
+def _dispatch_ci_fixes(events, config, *, dry_run: bool) -> None:
+    """#2510: classify any CONFIRMED ``checks_failed`` events and dispatch a
+    bounded CI-fix worker for the eligible ones, escalating to
+    ``HUMAN_REQUIRED`` once the retry cap (``coord.ci_fix.
+    MAX_CI_FIX_DISPATCHES``) is spent.
+
+    Mutates each qualifying event's ``ev.entry.state``/``ev.entry.error`` in
+    place — same shape as :func:`_dispatch_conflict_fixes` — so the caller's
+    own subsequent save-queue step picks the mutation up naturally.
+
+    ``process()`` emits a bare ``"checks_failed"`` event kind at exactly two
+    points, both reached only once the OTHER auto-remedies (the #1892 infra
+    rerun budget, the #2252 one-shot flake re-check) are exhausted or
+    inapplicable — i.e. both ARE the "confirmed real failure" case this leg
+    exists for (see ``coord.ci_fix``'s module docstring). Every other CI
+    outcome (``checks_pending``, ``ci_infra_rerun``, ``ci_flaky_rerun``,
+    ``checks_unreadable``, ``checks_stale``) is still self-healing or
+    mid-retry and must not be touched here.
+
+    Shared by the whole-queue path and the ``--only`` surgical path, same
+    reasoning as ``_dispatch_conflict_fixes``: a bare ``checks_failed`` entry
+    is never reprocessed by a later ``process()`` call (it only acts on
+    ``PENDING`` entries) so whichever caller sees the event first must also
+    be the one to act on it.
+    """
+    ci_events = [ev for ev in events if ev.kind == "checks_failed"]
+    if not ci_events or dry_run:
+        return
+
+    from coord.audit import record_audit  # noqa: PLC0415
+    from coord.ci_fix import MAX_CI_FIX_DISPATCHES, dispatch_ci_fix  # noqa: PLC0415
+    from coord.merge_queue import HUMAN_REQUIRED  # noqa: PLC0415
+    from coord.state import load_board, save_board  # noqa: PLC0415
+
+    fix_board = load_board()
+    if fix_board is None:
+        return
+    dispatched_any = False
+    for ev in ci_events:
+        entry = ev.entry
+        fix = dispatch_ci_fix(entry, fix_board, config, checks_summary=ev.message)
+        if fix is not None:
+            click.echo(
+                f"  {entry.repo_name} #{entry.issue_number}: "
+                f"ci-fix dispatched to {fix.machine_name} "
+                f"({entry.ci_fix_dispatches}/{MAX_CI_FIX_DISPATCHES})"
+            )
+            dispatched_any = True
+            continue
+        if entry.ci_fix_dispatches >= MAX_CI_FIX_DISPATCHES:
+            entry.state = HUMAN_REQUIRED
+            click.echo(
+                f"  {entry.repo_name} #{entry.issue_number}: "
+                f"ci-fix retry cap hit ({entry.ci_fix_dispatches}/"
+                f"{MAX_CI_FIX_DISPATCHES}) — manual resolution required"
+            )
+            record_audit(
+                tier="operational",
+                category="merge",
+                event_type="ci_fix_human_required",
+                actor="daemon",
+                summary=(
+                    f"ci-fix retry cap hit: {entry.repo_name}#"
+                    f"{entry.issue_number} — manual resolution required"
+                ),
+                repo=entry.repo_name,
+                issue=entry.issue_number,
+                assignment_id=entry.assignment_id,
+                details={"reason": "ci_fix_retry_cap", "error": entry.error},
+            )
+            dispatched_any = True  # the HUMAN_REQUIRED mutation needs saving
+        # else: budget remains but dispatch declined for another reason (no
+        # machine, an unrelated fix already in flight, agent unreachable) —
+        # leave the entry PENDING for the next tick to retry, exactly like a
+        # declined conflict-fix dispatch does.
+    if dispatched_any:
+        save_board(fix_board)
+
+
 @click.command(
     "verify-merge",
     help=(
@@ -1927,6 +2006,10 @@ def merge(
         # before the save below so a retry-cap/non-rebaseable HUMAN_REQUIRED
         # mutation on only_entry.state is persisted, not lost.
         _dispatch_conflict_fixes(events_only, cfg_only, dry_run=dry_run)
+        # #2510: same reasoning, for a CONFIRMED checks_failed event — dispatch
+        # a bounded ci-fix worker or escalate to HUMAN_REQUIRED before the
+        # save below, so the mutation on only_entry.state is persisted too.
+        _dispatch_ci_fixes(events_only, cfg_only, dry_run=dry_run)
         # #2246: `--only` merges one entry, but the branch it just moved is
         # shared — every OTHER queued PR based on it may have become
         # CONFLICTING a second ago. This is the drive-queue's path (`coord
@@ -2251,6 +2334,11 @@ def merge(
     # review, so the --only path below can share it).
     _dispatch_conflict_fixes(events, cfg, dry_run=dry_run)
 
+    # #2510: classify any CONFIRMED checks_failed events and dispatch a
+    # bounded ci-fix worker for the eligible ones, escalating to
+    # HUMAN_REQUIRED once the retry cap is spent.
+    _dispatch_ci_fixes(events, cfg, dry_run=dry_run)
+
     # #2246: whatever just landed may have invalidated its siblings — ask
     # GitHub now, while the merge that caused it is still the obvious
     # explanation, rather than letting the next drive attempt discover it as

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/coord/db.py

@@ -192,7 +192,8 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
             ci_stale_reruns INTEGER NOT NULL DEFAULT 0,
             ci_flaky_reruns INTEGER NOT NULL DEFAULT 0,
             ci_flaky_pending TEXT NOT NULL DEFAULT '',
-            ci_unreadable_reruns INTEGER NOT NULL DEFAULT 0
+            ci_unreadable_reruns INTEGER NOT NULL DEFAULT 0,
+            ci_fix_dispatches INTEGER NOT NULL DEFAULT 0
         );
 
         CREATE TABLE IF NOT EXISTS plans (
@@ -934,6 +935,13 @@ def _migrate_add_columns(conn: sqlite3.Connection) -> None:
         # NULL for a hand dispatch, a coordinator/brain-proposed dispatch,
         # and every row predating this column.
         "ALTER TABLE assignments ADD COLUMN dispatched_by_assignment_id TEXT",
+        # #2510: count of `coord.ci_fix.dispatch_ci_fix` fix-worker dispatches
+        # issued for this entry's CURRENT confirmed (non-infra, non-first-
+        # flake) `checks_failed` streak — see `QueuedMerge.ci_fix_dispatches`'s
+        # docstring and `coord.ci_fix.MAX_CI_FIX_DISPATCHES`. 0 for every row
+        # predating this migration, same as the column's own default for a
+        # freshly-enqueued entry.
+        "ALTER TABLE merge_queue ADD COLUMN ci_fix_dispatches INTEGER NOT NULL DEFAULT 0",
     ]
     for sql in migrations:
         try:

{code_coordinator-0.5.204 → code_coordinator-0.5.205}/coord/merge_queue.py

@@ -2524,6 +2524,29 @@ class QueuedMerge:
     # any of the other three ("what did CI say?"). 0 for every entry that
     # has never hit a fetch failure, and for rows predating this column.
     ci_unreadable_reruns: int = 0
+    # #2510: count of `type="ci-fix"`-shaped fix-worker dispatches the
+    # coordinator has issued for this entry's confirmed (non-infra,
+    # non-first-flake) `checks_failed` streak — capped at
+    # `coord.ci_fix.MAX_CI_FIX_DISPATCHES`. A CONFIRMED failure (see
+    # `process()`'s `msg = f"checks failed: {summary}"` block, reached only
+    # once the infra/flaky retry budgets above are exhausted or
+    # inapplicable) used to just leave the entry `PENDING` forever with no
+    # path back to a fix — this is the durable ceiling that lets
+    # `coord.commands.merge._dispatch_ci_fixes` dispatch a bounded number of
+    # fix attempts before giving up and promoting the entry to
+    # `HUMAN_REQUIRED`, mirroring how `ci_infra_reruns`/`ci_flaky_reruns`
+    # bound their own auto-remedies. 0 for every entry that has never hit a
+    # confirmed CI failure, and for rows predating this column. Unlike the
+    # rerun counters above (which reset once CI resolves cleanly — see the
+    # "genuinely resolved" reset block later in `process()`), this one is
+    # intentionally NOT auto-reset there: a fix dispatch's payoff is a FUTURE
+    # green run on a NEW commit, which as a side effect also resets
+    # ci_infra_reruns/ci_flaky_reruns/ci_unreadable_reruns — so by the time
+    # this entry could reach that reset block again, the fix either worked
+    # (merge proceeds, entry leaves the queue) or the retry budget is what
+    # stopped the loop; resetting it on an unrelated green tick would just
+    # reopen a budget a still-broken PR could re-exhaust forever.
+    ci_fix_dispatches: int = 0
 
 
 class GhOps(Protocol):
@@ -2848,6 +2871,10 @@ def load_queue() -> list[QueuedMerge]:
             # #2347: same NULL-to-0 decoding as ci_infra_reruns above, for
             # rows predating this column.
             ci_unreadable_reruns=row["ci_unreadable_reruns"] or 0,
+            # #2510: column added via migration; NULL (pre-migration rows)
+            # decodes to 0 — no CI-fix dispatches spent yet, same as a fresh
+            # entry.
+            ci_fix_dispatches=row["ci_fix_dispatches"] or 0,
         )
         for row in rows
     ]
@@ -2866,8 +2893,8 @@ def save_queue(items: list[QueuedMerge]) -> None:
                     pr_number, pr_url, size, last_attempt, error, enqueued_at,
                     assignment_type, required_gates, ci_infra_reruns,
                     ci_stale_reruns, ci_flaky_reruns, ci_flaky_pending,
-                    ci_unreadable_reruns
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    ci_unreadable_reruns, ci_fix_dispatches
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                 (
                     item.assignment_id, item.repo_name, item.repo_github,
                     item.branch, item.target_branch, item.issue_number,
@@ -2876,7 +2903,7 @@ def save_queue(items: list[QueuedMerge]) -> None:
                     item.assignment_type, json.dumps(list(item.required_gates or [])),
                     item.ci_infra_reruns, item.ci_stale_reruns,
                     item.ci_flaky_reruns, item.ci_flaky_pending,
-                    item.ci_unreadable_reruns,
+                    item.ci_unreadable_reruns, item.ci_fix_dispatches,
                 ),
             )