code-coordinator 0.5.500__tar.gz → 0.5.502__tar.gz

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.500
+Version: 0.5.502
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.500
+Version: 0.5.502
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.500",
+  "tag": "0.5.502",
   "distance": 0,
-  "node": "gd2476088b8684b908e08f1fbde1d3d82d3f02271",
+  "node": "g1215b2a207119f23b06fb38bbb2e77c613c7062e",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-19"

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/coord/agent.py

@@ -450,6 +450,45 @@ HOST_SLEEP_EXIT = 127
 # #2638.
 _DEFAULT_RUNTIME_CEILING_S = 6.0 * 60.0 * 60.0  # 6 hours
 
+# #3363: safety margin added on top of the resolved runtime ceiling before
+# `AgentServer._phantom_age_ceiling_s` treats a PENDING/RUNNING record's
+# timestamp as stale enough to investigate. Kept as its own constant so the
+# margin itself doesn't have to be re-derived at both the class-default and
+# per-instance/per-assignment call sites.
+_PHANTOM_AGE_MARGIN_S = 60.0 * 60.0  # 1 hour
+
+# #3363: DEFAULT max age a PENDING/RUNNING assignment record is trusted at
+# face value before it is even a *candidate* for phantom finalization —
+# i.e. the floor `AgentServer._phantom_age_ceiling_s` applies when neither
+# this agent's own configured `runtime_ceiling_s` nor the assignment's own
+# `spec.runtime_ceiling_s` raises it further. Nothing but a process restart
+# ever clears `_assignments` — so an entry that never reaches a terminal
+# status (worker killed out from under it, dispatch crashed between insert
+# and spawn, a stop that didn't write back) stays "active" forever, which
+# is exactly what makes it dangerous: every restart path (`/update`,
+# `/rollback`, the #2139 idle watcher) refuses to act while ANY assignment
+# looks active, so a single phantom permanently blocks the one thing — a
+# restart — that would clear it.
+#
+# #3363 review: this constant alone is NOT the last word on whether a
+# record is a phantom, for two reasons `_phantom_age_ceiling_s` and
+# `AgentServer._active_assignment_ids` handle explicitly:
+#   1. An operator can legitimately raise or disable the runtime ceiling
+#      per agent (`self.runtime_ceiling_s`) or per leg
+#      (`AssignmentSpec.runtime_ceiling_s`) — both documented, supported
+#      overrides (`coord/config.py:252-263`). A hardcoded module constant
+#      here would silently override that choice and finalize a real,
+#      healthy, still-running leg. `_phantom_age_ceiling_s` resolves the
+#      actual per-assignment ceiling instead of trusting this default
+#      unconditionally.
+#   2. Age is unconfirmed death, never proof of it. `_active_assignment_ids`
+#      only finalizes a record whose process is CONFIRMED not alive
+#      (`self._processes.get(aid)` is missing or `proc.poll() is not None`)
+#      — a record whose process is still alive is left alone regardless of
+#      how stale its timestamp looks, exactly the ground-truth check
+#      `cancel()`/`_reap` already use elsewhere in this class.
+_PHANTOM_ASSIGNMENT_MAX_AGE_S = _DEFAULT_RUNTIME_CEILING_S + _PHANTOM_AGE_MARGIN_S  # 7 hours
+
 # Minimum wall-vs-monotonic divergence measured over a SINGLE poll interval
 # that is unambiguously a host suspend rather than ordinary thread-scheduling
 # jitter, a GC pause, or a loaded box briefly starving this thread. Comfortably
@@ -3402,6 +3441,15 @@ class AgentAssignment:
     spec: AssignmentSpec
     status: str = PENDING
     pid: int | None = None
+    # #3363: stamped once, at construction, so a PENDING assignment (no
+    # `started_at` yet — that's only set once it reaches RUNNING) still has
+    # an age `active_assignment_count()`'s phantom check can measure.
+    # Reloaded PENDING/RUNNING entries from a persisted state file never
+    # keep this value in practice: `_load_state` already force-fails any
+    # PENDING/RUNNING record it finds (the owning subprocess is provably
+    # gone across a restart), so a missing/stale `created_at` on an old
+    # state file is harmless.
+    created_at: float = field(default_factory=time.time)
     started_at: float | None = None
     finished_at: float | None = None
     exit_code: int | None = None
@@ -7356,6 +7404,146 @@ class AgentServer:
         cargo_cache.write_gc_status(self.state_dir, result)
         return result
 
+    def active_assignment_count(self, *, now: float | None = None) -> int:
+        """How many assignments are genuinely still in flight right now.
+
+        #3363: the ONE predicate every "is this host busy" surface must
+        call. Before this, `_idle_restart_target`, `/update`'s post-swap
+        busy check, and `/rollback`'s busy check (all in
+        `coord/agent_app.py`) each independently re-derived
+        ``status in (PENDING, RUNNING)`` straight off ``self._assignments``
+        — three copies of the same question, agreeing right up until one of
+        them needed to disagree. A PENDING/RUNNING entry that never reaches
+        a terminal status (worker killed out from under it, a dispatch that
+        crashed between insert and spawn, a stop that didn't write back) is
+        "active" by that raw predicate forever, since nothing but a process
+        restart clears ``_assignments`` — and every restart path above
+        refuses to act while count > 0, so the phantom permanently blocks
+        the one thing that would clear it (see #3363's six-day macmini
+        deadlock).
+
+        Delegates the actual scan (and phantom finalization) to
+        :meth:`_active_assignment_ids`.
+        """
+        return len(self._active_assignment_ids(now=now))
+
+    def _phantom_age_ceiling_s(self, assignment: AgentAssignment) -> float:
+        """Resolve the phantom-age ceiling for *this specific* assignment.
+
+        #3363 review: the ceiling used to be the hardcoded module constant
+        ``_PHANTOM_ASSIGNMENT_MAX_AGE_S``, sized only off the compile-time
+        ``_DEFAULT_RUNTIME_CEILING_S``. That ignored an operator who
+        legitimately raised (or disabled) the runtime ceiling — either
+        fleet-wide (``self.runtime_ceiling_s``) or for this one leg
+        (``AssignmentSpec.runtime_ceiling_s``), both documented, supported
+        overrides (`coord/config.py:252-263`). Mirrors ``_reap``'s own
+        per-leg resolution (~line 9588): a positive ``spec.runtime_ceiling_s``
+        raises the base above this agent's own configured default, then the
+        result is floored at ``_DEFAULT_RUNTIME_CEILING_S`` and the same
+        margin is added — so a higher configured ceiling only ever pushes
+        this watchdog further OUT, never in.
+
+        This only bounds how stale a record is allowed to look before it
+        even becomes a *candidate* for phantom finalization — a record
+        whose process is confirmed still alive is never finalized by
+        ``_active_assignment_ids`` regardless of this value (see there), so
+        a leg that legitimately disables its own runtime ceiling is still
+        protected even though this method always returns a finite number.
+        """
+        base = self.runtime_ceiling_s or 0.0
+        raw_spec_ceiling = getattr(
+            getattr(assignment, "spec", None), "runtime_ceiling_s", None
+        )
+        if isinstance(raw_spec_ceiling, (int, float)) and raw_spec_ceiling > 0:
+            base = max(base, float(raw_spec_ceiling))
+        return max(base, _DEFAULT_RUNTIME_CEILING_S) + _PHANTOM_AGE_MARGIN_S
+
+    def _active_assignment_ids(self, *, now: float | None = None) -> list[str]:
+        """IDs of assignments still genuinely active, aging out phantoms.
+
+        A PENDING/RUNNING entry older than
+        :meth:`_phantom_age_ceiling_s` (measured from ``started_at`` once
+        RUNNING, ``created_at`` while still PENDING) is a *candidate* for
+        being a leaked bookkeeping record rather than live work — but age
+        alone is never proof of death (#2096: an unconfirmed verdict, in
+        either direction, is a defect). Before finalizing, this consults
+        the same ground truth ``cancel()``/``_reap`` already use elsewhere
+        in this class: ``self._processes.get(aid)``. Only when the process
+        is CONFIRMED not alive (no entry, or ``proc.poll()`` returns a real
+        exit code) is the record excluded from the returned IDs AND
+        finalized in place (flipped to FAILED with an explanatory
+        ``error``, ``finished_at`` stamped) so it stops looking active on
+        every other surface too — `/status`, `coord usage`, the board
+        reconciler — not merely in this count. A stale-looking record whose
+        process is still confirmed alive is left completely alone (still
+        counted active, status untouched) — force-finalizing it here would
+        orphan the real subprocess with no WIP push, since a FAILED record
+        no longer shows up in `/restart`'s `pending_ids` scan that would
+        otherwise `cancel(aid, push_mode="branch")` it gracefully.
+
+        Logged at ERROR so it shows up in the agent's own log, since
+        finalizing a phantom is itself a symptom worth an operator noticing
+        (see #3363 suggested fix item 3 — persistent phantoms should
+        eventually surface in `coord release verify`, not just here).
+        """
+        now = time.time() if now is None else now
+        active: list[str] = []
+        # #3363 review: iterate `.items()` and key off the DICT key, not
+        # `a.id` — a couple of existing tests plant a bare status-only stand-
+        # in object straight into `_assignments` (no `id`/`started_at`/
+        # `created_at` at all) to exercise the busy-veto in isolation.
+        # `getattr(..., None)` below is the same defensiveness applied to
+        # the timestamp reads: a stand-in with no age info reads as age 0
+        # (never a phantom), which preserves every pre-#3363 test's
+        # "PENDING/RUNNING always counts as busy" expectation unchanged.
+        to_finalize: list[tuple[str, AgentAssignment, str, float]] = []
+        with self._lock:
+            for aid, a in self._assignments.items():
+                if a.status not in (PENDING, RUNNING):
+                    continue
+                age_ref = (
+                    getattr(a, "started_at", None) if a.status == RUNNING
+                    else getattr(a, "created_at", None)
+                )
+                age = now - age_ref if age_ref is not None else 0.0
+                if age <= self._phantom_age_ceiling_s(a):
+                    active.append(aid)
+                    continue
+                # #3363 review: unconfirmed death is the same anti-pattern
+                # (epic #2096) as unconfirmed success — consult the actual
+                # process before declaring this record dead. A live PID
+                # here means real work is still in flight no matter how
+                # stale the bookkeeping timestamp looks, so it stays active
+                # and untouched; the normal `_reap` thread watching that
+                # same process (with its own correctly-resolved runtime
+                # ceiling) remains the one path that ever finalizes it.
+                proc = self._processes.get(aid)
+                if proc is not None and proc.poll() is None:
+                    active.append(aid)
+                    continue
+                to_finalize.append((aid, a, a.status, age))
+            for _aid, a, _prior_status, age in to_finalize:
+                ceiling = self._phantom_age_ceiling_s(a)
+                a.status = FAILED
+                a.finished_at = now
+                a.error = (
+                    f"phantom assignment (#3363): stuck in {_prior_status!r} "
+                    f"for {age / 3600.0:.1f}h with no live process behind it, "
+                    f"past the {ceiling / 3600.0:.1f}h bookkeeping ceiling "
+                    "with no terminal status update — finalized "
+                    "automatically so it stops blocking restart"
+                )
+        if to_finalize:
+            for aid, a, prior_status, age in to_finalize:
+                _log.error(
+                    "finalizing phantom assignment %s (was %s for %.0fs, "
+                    "repo=%s) so it stops blocking restart (#3363)",
+                    aid, prior_status, age,
+                    getattr(getattr(a, "spec", None), "repo_name", "?"),
+                )
+            self._persist()
+        return active
+
     def list_assignments(self) -> dict:
         from coord.worker_events import is_stream_json, parse_log
 

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/coord/agent_app.py

@@ -593,11 +593,12 @@ def _idle_restart_target(server: "AgentServer", venv_dir: Path) -> Path | None:
         # or it's already running the slot the symlink points at — no swap
         # is waiting.
         return None
-    with server._lock:
-        active_count = sum(
-            1 for a in server._assignments.values() if a.status in (PENDING, RUNNING)
-        )
-    if active_count:
+    # #3363: shared predicate — see `AgentServer.active_assignment_count`'s
+    # docstring for why this must never be re-derived independently here.
+    # It also ages out (and finalizes) any phantom PENDING/RUNNING entry,
+    # so a leaked bookkeeping record can no longer keep this watcher —
+    # the deferred restart's own advertised fallback — permanently idle.
+    if server.active_assignment_count():
         return None
     if _host_has_live_interactive_session():
         # #2139 blocking review fix: an interactive pane is not an
@@ -1538,12 +1539,14 @@ def build_app(
                 # started. Re-read fresh: `perform_update` above can take
                 # tens of seconds (venv creation, pip, smoke check), so the
                 # count at the top of this request is stale by now.
-                with server._lock:
-                    still_active = sum(
-                        1
-                        for a in server._assignments.values()
-                        if a.status in (PENDING, RUNNING)
-                    )
+                #
+                # #3363: `active_assignment_count()` is the single shared
+                # predicate (also used by `/rollback` and the idle
+                # watcher's `_idle_restart_target`) — it ages out and
+                # finalizes any phantom PENDING/RUNNING entry instead of
+                # trusting raw status forever, which is what let a single
+                # leaked assignment defer this restart for six days.
+                still_active = server.active_assignment_count()
                 if still_active and not force:
                     payload["result"] = "staged"
                     payload["error"] = (
@@ -1815,12 +1818,9 @@ def build_app(
             body = {}
         force = bool(body.get("force"))
 
-        with server._lock:
-            active_count = sum(
-                1
-                for a in server._assignments.values()
-                if a.status in (PENDING, RUNNING)
-            )
+        # #3363: shared predicate with `/update` and the idle watcher — see
+        # `AgentServer.active_assignment_count`'s docstring.
+        active_count = server.active_assignment_count()
         if active_count and not force:
             payload = {
                 "mode": "rollback",
@@ -1999,34 +1999,36 @@ def build_app(
         cancel_timeout = float(body.get("cancel_timeout", 30))
         saved_argv = list(sys.argv)
 
-        with server._lock:
-            active_count = sum(
-                1
-                for a in server._assignments.values()
-                if a.status in (PENDING, RUNNING)
-            )
+        # #3363: shared predicate with `/update`/`/rollback`/the idle
+        # watcher — see `AgentServer.active_assignment_count`'s docstring.
+        # A side benefit here specifically: a phantom entry gets finalized
+        # (flipped to FAILED) the first time this is called, so it no
+        # longer shows up in `pending_ids` below either — this restart
+        # stops trying to `cancel()` a record with no real process behind
+        # it.
+        active_count = server.active_assignment_count()
 
         def _do_restart() -> None:
             # Wait for workers to drain.
             deadline = time.time() + cancel_timeout
             while time.time() < deadline:
-                with server._lock:
-                    still_active = sum(
-                        1
-                        for a in server._assignments.values()
-                        if a.status in (PENDING, RUNNING)
-                    )
+                still_active = server.active_assignment_count()
                 if still_active == 0:
                     break
                 time.sleep(1)
 
-            # Cancel any workers that are still running.
-            with server._lock:
-                pending_ids = [
-                    aid
-                    for aid, a in server._assignments.items()
-                    if a.status in (PENDING, RUNNING)
-                ]
+            # Cancel any workers that are still running. #3363 review: reuse
+            # the shared predicate rather than re-deriving PENDING/RUNNING
+            # via a second raw scan over `server._assignments` — by now any
+            # true phantom (confirmed-dead process, no terminal status) has
+            # already been finalized to FAILED by the `active_assignment_count()`
+            # polls above, so a raw scan would agree with this today, but a
+            # second copy of the same question right next to the shared
+            # predicate is exactly the split `active_assignment_count()` was
+            # introduced to eliminate — and `_active_assignment_ids()` is the
+            # one that also knows to leave a still-alive process's record
+            # alone rather than mis-scan it as gone.
+            pending_ids = list(server._active_assignment_ids())
             for aid in pending_ids:
                 try:
                     # #1567: this is an infra-triggered restart, not an

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/coord/notify.py

@@ -4171,12 +4171,26 @@ def _dispatch_board_pending_smoke(config: Config) -> None:
     passed/failed/skipped verdict and re-dispatch instead of skipping it
     forever — without it, the staleness check fails open and this call would
     be no more capable than before that fix.
+
+    #3375: also passes a real *issue_liveness_fetcher*
+    (`coord.dispatch_liveness.github_issue_liveness_fetcher`) so the Test
+    stage — which dispatches over `httpx` directly rather than through
+    `coord.dispatch.dispatch()` — gets the SAME "is this issue closed / has
+    this branch already merged" precondition #3376 wired into every other
+    dispatch chokepoint. Without it, this stage stays the one path #3376
+    missed, and the five-smoke-agents-after-merge incident can still recur.
     """
     from coord.board_service import read_board, write_board
+    from coord.dispatch_liveness import github_issue_liveness_fetcher
     from coord.smoke import dispatch_pending_smoke
 
     board = read_board()
-    dispatched = dispatch_pending_smoke(board, config, gh_ops=github_ops)
+    dispatched = dispatch_pending_smoke(
+        board,
+        config,
+        gh_ops=github_ops,
+        issue_liveness_fetcher=github_issue_liveness_fetcher(config),
+    )
     if dispatched:
         write_board(board)
 

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/coord/reconcile.py

@@ -2699,10 +2699,22 @@ def reconcile(board: Board, config: Config) -> list[str]:
     # passed/failed/skipped verdict (a rebase moved the base or branch out
     # from under it) is re-dispatched instead of skipped forever — without
     # it, the staleness check inside `dispatch_pending_smoke` fails open.
+    #
+    # #3375: also passes a real *issue_liveness_fetcher* so `coord resume`
+    # gets the same "issue closed / branch already merged" precondition
+    # #3376 wired into every other dispatch chokepoint — the Test stage
+    # bypasses `coord.dispatch.dispatch()` entirely (a direct `httpx` POST
+    # to `/assign`), so it was the one path #3376 didn't reach.
     from coord import github_ops  # noqa: PLC0415
+    from coord.dispatch_liveness import github_issue_liveness_fetcher  # noqa: PLC0415
     from coord.smoke import dispatch_pending_smoke
 
-    for smoke in dispatch_pending_smoke(board, config, gh_ops=github_ops):
+    for smoke in dispatch_pending_smoke(
+        board,
+        config,
+        gh_ops=github_ops,
+        issue_liveness_fetcher=github_issue_liveness_fetcher(config),
+    ):
         if smoke.assignment_id is not None:
             changed.append(smoke.assignment_id)
 

{code_coordinator-0.5.500 → code_coordinator-0.5.502}/coord/smoke.py

@@ -3033,6 +3033,7 @@ def dispatch_pending_smoke(
     *,
     now: float | None = None,
     gh_ops: "GhOps | None" = None,
+    issue_liveness_fetcher: Callable[[str, int], tuple[bool, bool]] | None = None,
 ) -> list[Assignment]:
     """Bulk Test-stage dispatch — the smoke analogue of
     :func:`coord.review.dispatch_pending_reviews`.
@@ -3064,6 +3065,23 @@ def dispatch_pending_smoke(
     real :mod:`coord.github_ops` explicitly, the same module every other
     live gate check in this codebase hands `merge_queue`'s gate functions.
 
+    #3375: *issue_liveness_fetcher* is the SAME opt-in
+    ``(repo_name, issue_number) -> (issue_closed, branch_merged)`` contract
+    `coord.dispatch_liveness.check_dispatch_liveness` already gates
+    `coord.dispatch.dispatch()` with (#3376) — but this stage never went
+    through that call path. `dispatch_smoke`/`_dispatch_smoke_legs` POST
+    straight to a machine's `/assign` over `httpx`, so #3376's dispatch-time
+    gate never saw a Test-stage dispatch at all, and the five-smoke-agents
+    incident that motivated both issues could still reproduce after #3376
+    landed. Checked once per eligible row here, BEFORE any candidate is
+    walked or any machine chosen (`_dispatch_smoke_legs` needs no machine
+    name to know the issue is closed or the branch already merged) — a hit
+    records a `skipped` verdict (there is nothing left to gate) via
+    `record_test_verdict` and an audit row via
+    `coord.dispatch_liveness.record_dispatch_refusal`, and the row is left
+    alone rather than walking the candidate list for nothing. Default
+    ``None`` refuses nothing — identical to every pre-#3375 caller.
+
     A confirmed-stale verdict is **cleared** (``record_test_verdict(...,
     test_state=None)``, mirrored in-memory on ``completed.test_state``)
     *before* the re-dispatch, not left in place — so the row reads
@@ -3264,6 +3282,59 @@ def dispatch_pending_smoke(
             )
             continue
 
+        # #3375: refuse to dispatch a Test stage for a row whose issue has
+        # already closed or whose branch has already merged — the liveness
+        # precondition #3376 built for `coord.dispatch.dispatch()` but never
+        # reached this stage, because smoke dispatch never calls it (see the
+        # docstring above). Checked here, right before candidate walking,
+        # because it needs no machine selection to answer. A hit records a
+        # `skipped` verdict (there is nothing left for a Test verdict to
+        # gate) instead of `None`/leaving it unset — leaving it unset would
+        # re-enter this exact row on the very next tick and re-probe GitHub
+        # forever for an answer that cannot change back.
+        if issue_liveness_fetcher is not None:
+            from coord.dispatch_liveness import (  # noqa: PLC0415
+                check_dispatch_liveness,
+                record_dispatch_refusal,
+            )
+
+            issue_closed, branch_merged = issue_liveness_fetcher(
+                completed.repo_name, completed.issue_number
+            )
+            refusal = check_dispatch_liveness(
+                repo_name=completed.repo_name,
+                issue_number=completed.issue_number,
+                machine_name="(smoke: no machine selected yet)",
+                issue_closed=issue_closed,
+                branch_merged=branch_merged,
+            )
+            if refusal is not None:
+                if completed.assignment_id is not None:
+                    from coord.state import record_test_verdict  # noqa: PLC0415
+
+                    record_test_verdict(
+                        assignment_id=completed.assignment_id,
+                        test_state="skipped",
+                        test_reason=(
+                            f"Test stage skipped — {refusal.reason} (#3375)"
+                        ),
+                    )
+                    completed.test_state = "skipped"
+                record_dispatch_refusal(
+                    refusal,
+                    repo_name=completed.repo_name,
+                    issue_number=completed.issue_number,
+                    machine_name="(smoke: no machine selected yet)",
+                    assignment_type="smoke",
+                )
+                logger.info(
+                    "dispatch_pending_smoke: refused to dispatch for %s#%s "
+                    "row %s — %s",
+                    completed.repo_name, completed.issue_number,
+                    completed.assignment_id, refusal.reason,
+                )
+                continue
+
         # #3182: the full-list implementation — a capability fan-out
         # dispatches more than one leg per completion, so this loop cannot
         # go through the compat-wrapping `dispatch_smoke` (which only ever