code-coordinator 0.5.219__tar.gz → 0.5.220__tar.gz

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.219
+Version: 0.5.220
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.219
+Version: 0.5.220
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.219",
+  "tag": "0.5.220",
   "distance": 0,
-  "node": "g570324652ea19cd658d0b5503959bae5bc90a452",
+  "node": "g3b354927ade8c4466bfcb6981c100e1c23f7c41c",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-22"

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/coord/commands/lifecycle.py

@@ -130,8 +130,15 @@ def notify(config_path: Path) -> None:
     from coord.notify import run as run_notify
 
     cfg = _load_config(config_path)
-    posted, stuck, needs_attention, stalled, liveness = run_notify(cfg)
-    if not posted and not stuck and not needs_attention and not stalled and not liveness:
+    posted, stuck, needs_attention, stalled, liveness, phantom_healed = run_notify(cfg)
+    if (
+        not posted
+        and not stuck
+        and not needs_attention
+        and not stalled
+        and not liveness
+        and not phantom_healed
+    ):
         click.echo("No new transitions to notify.")
         return
     if posted:
@@ -173,6 +180,15 @@ def notify(config_path: Path) -> None:
                 f"#{lv.issue_number} (assignment {lv.assignment_id}, "
                 f"{lv.consecutive_blocked} consecutive blocked verdicts)"
             )
+    if phantom_healed:
+        click.echo(f"Auto-healed {len(phantom_healed)} phantom row(s) (#2536):")
+        for h in phantom_healed:
+            click.echo(
+                f"  [phantom-healed:{h.stage}] {h.machine_name} → {h.repo_name} "
+                f"#{h.issue_number} (assignment {h.assignment_id})"
+            )
+            click.echo(f"    {h.detail}")
+            click.echo(f"    → {h.action}")
     board = read_board()
 
     if is_round_complete(board) and cfg.hooks.on_round_complete:

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/coord/comments.py

@@ -53,6 +53,16 @@ EVENT_STALLED = "stalled_pipeline"
 # surfacing only, exactly like EVENT_NEEDS_ATTENTION/EVENT_STALLED: never
 # gates status/review_state/test_state.
 EVENT_LIVENESS_STALL = "liveness_stall"
+# #2536: a `running` row the fleet-wide sweep
+# (`coord.diagnose.sweep_dead_running_rows`) confirmed dead via its own
+# recorded machine's live status, found aged well past its own
+# EVENT_NEEDS_ATTENTION wall-clock threshold, and auto-healed with the same
+# non-destructive recovery `coord diagnose --reset` runs by hand — branch
+# and commits preserved, stage re-dispatchable, and (as a side effect of the
+# row leaving `running`) its repo's drive-queue concurrency slot freed.
+# Unlike every other EVENT_* above, this one narrates an action ALREADY
+# TAKEN, not only a finding.
+EVENT_PHANTOM_HEALED = "phantom_row_healed"
 
 
 @dataclass
@@ -479,6 +489,55 @@ def format_liveness_stall(
     return "\n".join(lines)
 
 
+def format_phantom_row_healed(
+    *,
+    assignment_id: str,
+    machine_name: str,
+    repo_name: str,
+    issue_number: int,
+    stage: str,
+    detail: str,
+    action: str,
+) -> str:
+    """Format a "phantom row auto-healed" comment (#2536).
+
+    Mirrors :func:`format_needs_attention`'s shape (same headings, same
+    "detection ... human should look" register), with one deliberate
+    difference: this one is posted AFTER the automatic, non-destructive
+    ``coord diagnose --reset``-equivalent recovery already ran, not before
+    it — *detail* explains what was confirmed (dead session, aged past
+    threshold), *action* what was actually done.
+    """
+    marker = _marker(
+        EVENT_PHANTOM_HEALED,
+        assignment=assignment_id,
+        machine=machine_name,
+        repo=repo_name,
+        stage=stage,
+    )
+    lines = [
+        marker,
+        "## 🩹 Phantom row auto-healed",
+        f"**Machine:** {machine_name}",
+        f"**Assignment:** {assignment_id}",
+        f"**Issue:** #{issue_number}",
+        f"**Stage:** {stage}",
+        "",
+        detail.strip(),
+        "",
+        f"**Action taken:** {action.strip()}",
+        "",
+        "This session's own recorded machine reported nothing active for "
+        "it, and the row had already sat well past its own needs-attention "
+        "threshold — the same phantom-row recovery `coord diagnose "
+        "--reset` performs by hand ran automatically. The branch and any "
+        "commits are preserved, the stage is re-dispatchable, and this "
+        "repo's drive-queue slot is freed. If this looks wrong, `coord log "
+        "<id>` / `coord diagnose <repo> <issue>` can confirm what happened.",
+    ]
+    return "\n".join(lines)
+
+
 _STALLED_REASON_LABELS = {
     "review_request_changes_no_fix": "Review requested changes, no fix dispatched",
     "review_done_no_verdict": "Review finished with no verdict ever captured (#812)",

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/coord/config.py

@@ -989,6 +989,25 @@ class PipelineConfig:
     stands. Costs one real suite run per Test-stage completion where it
     *can* apply; set ``false`` (or ``COORD_CONFIRM_TEST_VERDICT=0``) to opt
     back out.
+
+    ``auto_heal_phantom_rows`` (#2536) controls whether
+    ``coord.notify._sweep_phantom_rows`` gets to act, on
+    ``coord-notify.timer``'s existing cadence, on a ``running`` board row
+    its own recorded machine confirms is dead (see
+    ``coord.diagnose.sweep_dead_running_rows``). **Defaults to ``True``**,
+    the same posture as ``confirm_test_verdict`` and for the same reason:
+    this closes a real bug (a phantom row silently holding its repo's
+    entire drive-queue concurrency slot indefinitely — #2536) rather than
+    adding a new capability, and every action it takes is gated behind a
+    CONFIRMED-dead liveness read (never an ambiguous "unknown" one — #1870)
+    plus a wall-clock buffer past the row's own ``needs_attention``
+    threshold, so it never races a session that is merely idle between
+    turns. Unlike ``auto_dispatch_stalled``/``escalate_semantic_conflicts``,
+    it never dispatches new work and never touches a branch — the recovery
+    is the same non-destructive one ``coord diagnose --reset`` already
+    performs by hand (branch/commits preserved, stage re-dispatchable). Set
+    ``false`` to require a human to run ``coord diagnose --reset``
+    themselves, as before #2536.
     """
 
     default_gates: list[str] = field(default_factory=lambda: ["test", "review", "merge"])
@@ -1009,6 +1028,8 @@ class PipelineConfig:
     liveness_auditor: LivenessAuditorConfig = field(default_factory=LivenessAuditorConfig)
     # #2464 — DEFAULT ON. See the class docstring.
     confirm_test_verdict: bool = True
+    # #2536 — DEFAULT ON. See the class docstring.
+    auto_heal_phantom_rows: bool = True
 
     def attention_threshold_for(
         self,
@@ -2763,6 +2784,12 @@ def _parse_pipeline(raw: Any) -> PipelineConfig:
             raise ConfigError("pipeline.confirm_test_verdict must be a boolean")
         cfg.confirm_test_verdict = value
 
+    if "auto_heal_phantom_rows" in raw:
+        value = raw["auto_heal_phantom_rows"]
+        if not isinstance(value, bool):
+            raise ConfigError("pipeline.auto_heal_phantom_rows must be a boolean")
+        cfg.auto_heal_phantom_rows = value
+
     if "max_review_iterations" in raw:
         value = raw["max_review_iterations"]
         if not isinstance(value, int) or value < 1:

{code_coordinator-0.5.219 → code_coordinator-0.5.220/coord}/deploy/coord-notify.service

@@ -2,9 +2,15 @@
 #
 # A systemd *user* unit (mirrors coord-agent/coord-serve/coord-web), paired
 # with coord-notify.timer. Runs `coord notify` on a schedule: polls each
-# agent for completion, posts the GH completion/failure comments, and
-# triggers the auto-loop (review-on-completion, fix-on-request-changes,
-# re-review-on-fix-completion). Install ONLY on the box that owns
+# agent for completion, posts the GH completion/failure comments, triggers
+# the auto-loop (review-on-completion, fix-on-request-changes,
+# re-review-on-fix-completion), and — #2536 — auto-heals any `running` board
+# row whose recorded machine confirms is dead and aged well past its own
+# needs-attention threshold (the same non-destructive recovery `coord
+# diagnose --reset` performs by hand; see
+# docs/AGENT_OPERATIONS.md → "Periodic coord notify" for the guardrails, and
+# `pipeline.auto_heal_phantom_rows` in coordinator.yml to opt out). Install
+# ONLY on the box that owns
 # ~/.coord/coord.db (dellserver in production — the same host that runs
 # coord-serve/coord-web) — it must NOT run on a thin-client machine, since
 # `coord notify` there would just re-route itself back over HTTP to the

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/coord/diagnose.py

@@ -1454,6 +1454,163 @@ def _is_stale(assignment: "Assignment", *, max_age_hours: float = 12.0) -> bool:
     return (time.time() - assignment.dispatched_at) > max_age_hours * 3600.0
 
 
+# ── #2536: fleet-wide phantom-row self-heal sweep ───────────────────────────
+#
+# `_cleanup_issue` above already recovers phantom `running` rows — but only
+# ONE issue at a time, and only when a human happens to run `coord diagnose
+# <repo> <issue> --reset`. Nothing runs that recovery on its own, so a
+# phantom row can sit indefinitely holding its repo's entire drive-queue
+# concurrency slot (`coord/drive_queue.py`'s own capacity comment: "a drive
+# whose observer died still holds its repo's slot until something reconciles
+# it"). :func:`sweep_dead_running_rows` is that "something" — a fleet-wide,
+# board-scanning counterpart callable periodically (see
+# `coord.notify._sweep_phantom_rows`, which piggybacks it on
+# `coord-notify.timer`'s existing 5-minute cadence) rather than only on
+# operator demand.
+
+# How far past a row's own needs-attention wall-clock threshold it must sit
+# before this sweep will treat a CONFIRMED-dead session as safe to
+# auto-heal. Two full `coord-notify.timer` cycles (5min cadence) of margin:
+# large enough that `coord.notify.detect_needs_attention` has already had a
+# full extra tick to flag (and a human a full extra tick to notice) the same
+# row via the existing `needs_attention` comment before this sweep ever
+# touches it, small enough that a genuinely dead row doesn't sit much longer
+# than it already would have. Never a substitute for the liveness check
+# below — a row can be arbitrarily old and still left alone if its session
+# reads "live" or "unknown".
+PHANTOM_HEAL_BUFFER_SECONDS = 600.0
+
+
+@dataclass
+class PhantomRowHeal:
+    """One board row this sweep confirmed dead, aged out past its own
+    needs-attention threshold, and healed automatically (or, in a dry run,
+    WOULD heal) — see :func:`sweep_dead_running_rows`."""
+
+    assignment_id: str
+    machine_name: str
+    repo_name: str
+    issue_number: int
+    stage: str
+    detail: str
+    action: str = ""
+
+
+def sweep_dead_running_rows(
+    board: "Board",
+    config: "Config",
+    *,
+    now: float | None = None,
+    dry_run: bool = False,
+) -> list[PhantomRowHeal]:
+    """The automatic counterpart to a human running ``coord diagnose <repo>
+    <issue> --reset`` on a phantom ``running`` row (#2536).
+
+    Scans every ``running``/``pending`` row on *board* (fleet-wide, not
+    scoped to one issue — unlike :func:`_cleanup_issue`) and, for each,
+    applies the same two guards a careful human would before touching a
+    row that merely *looks* wedged:
+
+    1. **Confirmed dead, never ambiguous** — liveness is read via
+       :func:`_session_state` (tmux first, local or remote, falling back to
+       the assignment's own recorded machine's ``/status``), and only a
+       ``"dead"`` verdict is actionable. ``"unknown"`` (an unresolvable or
+       unconfigured machine, an unreachable agent, a probe error) is always
+       left alone, exactly like ``"live"`` — the same #1870 caution
+       ``coord/drive_queue.py`` documents ("liveness cannot be verified from
+       here, so this is UNKNOWN, not dead") applies here too: only the
+       assignment's own recorded machine can make this call.
+    2. **Aged out, not merely between turns** — the row must be running
+       longer than its own ``config.pipeline.attention_threshold_for(...)``
+       (the same wall-clock threshold
+       :func:`coord.notify.detect_needs_attention` uses) PLUS
+       :data:`PHANTOM_HEAL_BUFFER_SECONDS` of margin, so a session that is
+       merely idle between turns or briefly disconnected is never raced.
+
+    Recovery, once both guards pass, is byte-for-byte
+    :func:`_finalize_dead` — the exact non-destructive action ``coord
+    diagnose --reset`` runs for a phantom row: branch and commits are always
+    preserved, and the stage becomes re-dispatchable. Writes go through the
+    same seam (``issue_store``, not ``save_board``) :func:`_cleanup_issue`
+    already uses, so this function never mutates *board* and the caller
+    does not need to persist it. Freeing the row's repo's drive-queue slot
+    is then a side effect of the write, not something this function or its
+    caller has to do explicitly — the next board read simply no longer
+    counts the row as ``running``.
+
+    Returns one :class:`PhantomRowHeal` per row healed (or, when *dry_run*,
+    per row that WOULD be healed), so the caller can post a GitHub comment
+    recording it — mirroring the existing ``needs_attention`` comment's
+    posture, but reporting an action already taken rather than only a
+    finding.
+    """
+    if now is None:
+        now = time.time()
+
+    healed: list[PhantomRowHeal] = []
+    seen_ids: set[str] = set()
+    for a in (board.active + board.completed):
+        if a.status not in ("running", "pending"):
+            continue
+        if not a.assignment_id or a.assignment_id in seen_ids:
+            continue
+        seen_ids.add(a.assignment_id)
+        if not a.dispatched_at:
+            continue  # nothing to compute an age from — never guess
+
+        threshold = config.pipeline.attention_threshold_for(
+            a.type or "work",
+            provider_name=a.provider_name,
+            review_of_assignment_id=a.review_of_assignment_id,
+        )
+        if threshold == float("inf"):
+            continue  # interactive session type — no wall-clock concept
+        running_for = now - a.dispatched_at
+        if running_for <= threshold + PHANTOM_HEAL_BUFFER_SECONDS:
+            continue  # not aged out yet — don't race a session between turns
+
+        if _session_state(a, config) != "dead":
+            # "unknown" or "live" — only ever act on a CONFIRMED-dead read,
+            # never an ambiguous one (#1870/#2536).
+            continue
+
+        detail = (
+            f"{a.type or 'work'} row running {running_for / 60.0:.0f}m "
+            f"(past its {threshold / 60.0:.0f}m needs-attention threshold + "
+            f"{PHANTOM_HEAL_BUFFER_SECONDS / 60.0:.0f}m buffer) — session "
+            f"confirmed dead on machine {a.machine_name!r}"
+        )
+        if dry_run:
+            healed.append(PhantomRowHeal(
+                assignment_id=a.assignment_id,
+                machine_name=a.machine_name or "unknown",
+                repo_name=a.repo_name,
+                issue_number=a.issue_number,
+                stage=a.type or "work",
+                detail=detail,
+                action="(dry-run) would finalize the phantom session",
+            ))
+            continue
+
+        try:
+            action = f"finalized phantom session ({_finalize_dead(a, config)})"
+        except Exception as exc:  # noqa: BLE001 — fall back to a direct terminal mark
+            _mark_terminal(a, config)
+            action = f"finalize failed ({exc}); marked row terminal directly"
+
+        healed.append(PhantomRowHeal(
+            assignment_id=a.assignment_id,
+            machine_name=a.machine_name or "unknown",
+            repo_name=a.repo_name,
+            issue_number=a.issue_number,
+            stage=a.type or "work",
+            detail=detail,
+            action=action,
+        ))
+
+    return healed
+
+
 # ── #618: orphaned worktree detection + pruning ──────────────────────────────
 
 

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/coord/notify.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
 import httpx
 
 if TYPE_CHECKING:
+    from coord.diagnose import PhantomRowHeal
     from coord.merge_queue import QueuedMerge
     from coord.models import Assignment, Board
     from coord.progress import SmokeVerdict
@@ -3544,6 +3545,106 @@ def _sweep_stalled_pipeline(
     return posted
 
 
+# ── #2536: fleet-wide phantom-row self-heal sweep ───────────────────────────
+#
+# See `coord.diagnose.sweep_dead_running_rows`'s docstring for the full
+# rationale. This is the thin `coord notify` glue around it: read the board,
+# run the sweep (which does the actual recovery write per row), post one
+# comment per row it healed, and audit-log it. No board write-back is needed
+# here — the sweep writes through `issue_store` directly (the same seam
+# `_cleanup_issue` uses), never mutating the in-memory `board` object.
+
+
+def post_phantom_row_healed(heal: "PhantomRowHeal", config: Config) -> None:
+    """Post the #2536 auto-heal comment for one board row
+    :func:`_sweep_phantom_rows` just recovered."""
+    from coord.comments import (  # noqa: PLC0415
+        format_phantom_row_healed,
+    )
+
+    repo = config.repo(heal.repo_name)
+    repo_github = repo.github if repo is not None else None
+    if not repo_github:
+        return
+    body = format_phantom_row_healed(
+        assignment_id=heal.assignment_id,
+        machine_name=heal.machine_name,
+        repo_name=heal.repo_name,
+        issue_number=heal.issue_number,
+        stage=heal.stage,
+        detail=heal.detail,
+        action=heal.action,
+    )
+    github_ops.post_issue_comment(repo_github, heal.issue_number, body)
+
+
+def _sweep_phantom_rows(config: Config) -> list["PhantomRowHeal"]:
+    """Scan the board for ``running``/``pending`` rows whose session is
+    CONFIRMED dead and aged well past their own needs-attention threshold,
+    auto-heal them with the same non-destructive recovery ``coord diagnose
+    --reset`` performs by hand, and post one comment per row recovered
+    (#2536).
+
+    Gated by ``config.pipeline.auto_heal_phantom_rows`` (**default
+    ``True``** — see that field's docstring for why this ships lit unlike
+    ``auto_dispatch_stalled``/``escalate_semantic_conflicts``: every action
+    here is gated behind a confirmed-dead liveness read plus an aged-out
+    wall-clock buffer, and the recovery itself never dispatches work or
+    touches a branch).
+
+    Best-effort per row, mirroring :func:`_sweep_stalled_pipeline`'s
+    contract: a comment-posting failure for one row must not stop the sweep
+    from reaching the rest, and does not block the row's own recovery
+    (already durably written by the time this posts) — it just means the
+    GitHub comment is missing, which a future manual `coord diagnose` can
+    still explain.
+    """
+    if not config.pipeline.auto_heal_phantom_rows:
+        return []
+
+    from coord.board_service import read_board  # noqa: PLC0415
+    from coord.diagnose import sweep_dead_running_rows  # noqa: PLC0415
+
+    board = read_board()
+    healed = sweep_dead_running_rows(board, config)
+
+    posted: list["PhantomRowHeal"] = []
+    for heal in healed:
+        try:
+            post_phantom_row_healed(heal, config)
+        except Exception:  # noqa: BLE001
+            log.exception(
+                "post_phantom_row_healed failed for %s", heal.assignment_id,
+            )
+            continue
+        posted.append(heal)
+
+        try:
+            from coord.audit import record_audit  # noqa: PLC0415
+
+            record_audit(
+                tier="business",
+                category="pipeline",
+                event_type="phantom_row_auto_healed",
+                actor="coordinator",
+                summary=(
+                    f"phantom-row sweep healed {heal.stage} row "
+                    f"{heal.assignment_id} for {heal.repo_name}#{heal.issue_number}"
+                ),
+                repo=heal.repo_name,
+                issue=heal.issue_number,
+                assignment_id=heal.assignment_id,
+                machine=heal.machine_name,
+                details={"detail": heal.detail, "action": heal.action},
+            )
+        except Exception:  # noqa: BLE001
+            log.exception(
+                "record_audit failed for phantom-row heal %s", heal.assignment_id,
+            )
+
+    return posted
+
+
 @dataclass(frozen=True)
 class DrainResult:
     """What one :func:`run_drain` pass actually did.
@@ -3828,23 +3929,27 @@ def run(
     list[NeedsAttentionDetection],
     list[StalledDetection],
     list[LivenessStallDetection],
+    list["PhantomRowHeal"],
 ]:
     """Detect and post all pending transitions, stuck signals, #846
-    needs-attention detections, #1441 stalled-pipeline detections, and
-    #2048 liveness-auditor stalls.
+    needs-attention detections, #1441 stalled-pipeline detections, #2048
+    liveness-auditor stalls, and #2536 phantom-row auto-heals.
 
     Also dispatches any pending reviews found on the saved board so that
     ``coord notify`` acts as a reliable review-dispatch trigger in addition
     to ``coord status --reconcile``.
 
     Returns (posted_transitions, posted_stuck, posted_needs_attention,
-    posted_stalled, posted_liveness). The liveness entry is new in #2048 —
-    appended rather than inserted, following #1441's own precedent: any
-    existing caller unpacking a 4-tuple positionally breaks loudly, which
-    is a good thing (it means the CLI/board/TUI surfacing was actually
-    wired up, not silently skipped). ``posted_liveness`` is always ``[]``
-    when ``config.pipeline.liveness_auditor.enabled`` is ``False`` (the
-    default).
+    posted_stalled, posted_liveness, posted_phantom_healed). Each of the
+    liveness and phantom-healed entries was appended rather than inserted,
+    following #1441's own precedent: any existing caller unpacking a
+    shorter tuple positionally breaks loudly, which is a good thing (it
+    means the CLI/board/TUI surfacing was actually wired up, not silently
+    skipped). ``posted_liveness`` is always ``[]`` when
+    ``config.pipeline.liveness_auditor.enabled`` is ``False`` (the
+    default); ``posted_phantom_healed`` is always ``[]`` when
+    ``config.pipeline.auto_heal_phantom_rows`` is ``False`` (not the
+    default — see that field's docstring).
     """
     # Refresh the agent-host cache so _try_parse_and_post_review (and any
     # other helper using _agent_host) can resolve hostnames without
@@ -3929,6 +4034,20 @@ def run(
     except Exception:  # noqa: BLE001
         log.exception("detect_needs_attention: unexpected error")
 
+    # #2536: fleet-wide phantom-row self-heal — a `running` row whose
+    # session its own recorded machine confirms is dead, aged well past its
+    # own needs-attention threshold, gets the same non-destructive recovery
+    # `coord diagnose --reset` performs by hand, automatically. Runs right
+    # after the needs-attention scan above (same "long-running row" concern,
+    # acted on rather than only narrated) and before the dispatch steps
+    # below, so a slot this sweep frees is visible to them in the same pass.
+    # Best-effort, non-fatal — mirrors every other sweep in this function.
+    phantom_healed_posted: list["PhantomRowHeal"] = []
+    try:
+        phantom_healed_posted = _sweep_phantom_rows(config)
+    except Exception:  # noqa: BLE001
+        log.exception("_sweep_phantom_rows: unexpected error")
+
     # Dispatch pending Test-stage smoke from the saved board (#1426;
     # best-effort, non-fatal). Runs BEFORE review dispatch to mirror the
     # pipeline's Work -> Test -> Review order, though ordering isn't load-
@@ -4036,4 +4155,11 @@ def run(
     except Exception:  # noqa: BLE001
         log.exception("detect_liveness_stall: unexpected error")
 
-    return posted, stuck_posted, needs_attention_posted, stalled_posted, liveness_posted
+    return (
+        posted,
+        stuck_posted,
+        needs_attention_posted,
+        stalled_posted,
+        liveness_posted,
+        phantom_healed_posted,
+    )

{code_coordinator-0.5.219/coord → code_coordinator-0.5.220}/deploy/coord-notify.service

@@ -2,9 +2,15 @@
 #
 # A systemd *user* unit (mirrors coord-agent/coord-serve/coord-web), paired
 # with coord-notify.timer. Runs `coord notify` on a schedule: polls each
-# agent for completion, posts the GH completion/failure comments, and
-# triggers the auto-loop (review-on-completion, fix-on-request-changes,
-# re-review-on-fix-completion). Install ONLY on the box that owns
+# agent for completion, posts the GH completion/failure comments, triggers
+# the auto-loop (review-on-completion, fix-on-request-changes,
+# re-review-on-fix-completion), and — #2536 — auto-heals any `running` board
+# row whose recorded machine confirms is dead and aged well past its own
+# needs-attention threshold (the same non-destructive recovery `coord
+# diagnose --reset` performs by hand; see
+# docs/AGENT_OPERATIONS.md → "Periodic coord notify" for the guardrails, and
+# `pipeline.auto_heal_phantom_rows` in coordinator.yml to opt out). Install
+# ONLY on the box that owns
 # ~/.coord/coord.db (dellserver in production — the same host that runs
 # coord-serve/coord-web) — it must NOT run on a thin-client machine, since
 # `coord notify` there would just re-route itself back over HTTP to the

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/docs/AGENT_OPERATIONS.md

@@ -827,6 +827,28 @@ auto-loop dispatch path. Low risk today (flat headless-only dispatch has no
 than blocking this timer — flag it if it ever bites (a fix worker firing while
 someone is mid-`--fix-of` on the same issue).
 
+**Phantom-row self-heal rides the same timer (#2536).** Every `coord notify`
+pass this timer fires also sweeps the board for a `status='running'` row
+whose recorded machine's own `/status` shows nothing active for it — a
+session that finished or died without the board ever finding out. Left
+alone, that phantom row sits forever (nothing else notices) and — per
+`coord/drive_queue.py`'s own capacity comment — holds its repo's *entire*
+drive-queue concurrency slot, blocking every other queued issue in that
+repo. The sweep only ever acts on a **confirmed**-dead session (never an
+"unknown" read — an unresolvable machine, an unreachable agent, a probe
+error all count as "leave it alone", same as "live"), and only once the row
+is aged well past its own `needs_attention` wall-clock threshold plus a
+buffer, so it can never race a session that's merely idle between turns.
+When both hold, it runs the exact same non-destructive recovery a human
+would via `coord diagnose <repo> <issue> --reset` — branch and commits are
+always preserved, the stage becomes re-dispatchable — and posts a GitHub
+comment recording that it happened (`coord.diagnose.sweep_dead_running_rows`,
+`coord.notify._sweep_phantom_rows`). Governed by
+`pipeline.auto_heal_phantom_rows` in `coordinator.yml` (**default `true`**,
+unlike most other auto-dispatch flags in this file — see
+`PipelineConfig`'s docstring in `coord/config.py` for why); set it `false`
+to go back to requiring a human to run `--reset` by hand.
+
 ## Periodic `coord drive-queue tick` (`coord-drive-queue` timer, #1756)
 
 Same shape as the `coord-notify` timer above — a `Type=oneshot` unit fired on

{code_coordinator-0.5.219 → code_coordinator-0.5.220}/tests/test_auto_loop.py

@@ -1272,6 +1272,47 @@ class TestPipelineConfigParsing:
         ):
             load(p)
 
+    def test_auto_heal_phantom_rows_defaults_to_true(self, tmp_path) -> None:
+        """#2536: unlike auto_dispatch_stalled, the phantom-row self-heal
+        sweep ships lit — every action it takes is gated behind a
+        confirmed-dead liveness read plus an aged-out wall-clock buffer, and
+        it never dispatches work or touches a branch."""
+        from coord.config import load
+
+        p = tmp_path / "coordinator.yml"
+        p.write_text(
+            "repos:\n  - name: api\n    github: acme/api\n"
+            "machines:\n  - name: laptop\n    host: laptop.tail\n    repos: [api]\n"
+        )
+        cfg = load(p)
+        assert cfg.pipeline.auto_heal_phantom_rows is True
+
+    def test_can_disable_auto_heal_phantom_rows(self, tmp_path) -> None:
+        from coord.config import load
+
+        p = tmp_path / "coordinator.yml"
+        p.write_text(
+            "repos:\n  - name: api\n    github: acme/api\n"
+            "machines:\n  - name: laptop\n    host: laptop.tail\n    repos: [api]\n"
+            "pipeline:\n  auto_heal_phantom_rows: false\n"
+        )
+        cfg = load(p)
+        assert cfg.pipeline.auto_heal_phantom_rows is False
+
+    def test_invalid_auto_heal_phantom_rows_raises(self, tmp_path) -> None:
+        from coord.config import ConfigError, load
+
+        p = tmp_path / "coordinator.yml"
+        p.write_text(
+            "repos:\n  - name: api\n    github: acme/api\n"
+            "machines:\n  - name: laptop\n    host: laptop.tail\n    repos: [api]\n"
+            "pipeline:\n  auto_heal_phantom_rows: maybe\n"
+        )
+        with pytest.raises(
+            ConfigError, match="pipeline.auto_heal_phantom_rows must be a boolean"
+        ):
+            load(p)
+
 
 # ── Unit tests: Assignment.review_iteration persistence ─────────────────────