code-coordinator 0.5.139__tar.gz → 0.5.141__tar.gz

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.139
+Version: 0.5.141
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.139
+Version: 0.5.141
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/code_coordinator.egg-info/SOURCES.txt

@@ -112,6 +112,7 @@ coord/prereqs.py
 coord/progress.py
 coord/queue_diagnose.py
 coord/reconcile.py
+coord/reconcile_tick.py
 coord/refine_chat.py
 coord/release_cordon.py
 coord/release_propagate.py
@@ -431,6 +432,7 @@ tests/test_agent_branch_capture.py
 tests/test_agent_config_free_1712.py
 tests/test_agent_deploy_units_endpoint.py
 tests/test_agent_dirty_worktree.py
+tests/test_agent_drive_queue_reconcile_2373.py
 tests/test_agent_graph_self_heal.py
 tests/test_agent_health_h3.py
 tests/test_agent_idle_restart.py
@@ -682,6 +684,7 @@ tests/test_reconcile_branch.py
 tests/test_reconcile_completions.py
 tests/test_reconcile_merges.py
 tests/test_reconcile_no_agent_record.py
+tests/test_reconcile_tick.py
 tests/test_reconcile_usage_limit_done.py
 tests/test_refine_chat.py
 tests/test_release_cordon_2101.py

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/code_coordinator.egg-info/scm_file_list.json

@@ -293,6 +293,7 @@
     "coord/providers/opencode.py",
     "coord/queue_diagnose.py",
     "coord/reconcile.py",
+    "coord/reconcile_tick.py",
     "coord/refine_chat.py",
     "coord/release_cordon.py",
     "coord/release_propagate.py",
@@ -474,6 +475,7 @@
     "tests/test_agent_config_free_1712.py",
     "tests/test_agent_deploy_units_endpoint.py",
     "tests/test_agent_dirty_worktree.py",
+    "tests/test_agent_drive_queue_reconcile_2373.py",
     "tests/test_agent_graph_self_heal.py",
     "tests/test_agent_health_h3.py",
     "tests/test_agent_idle_restart.py",
@@ -725,6 +727,7 @@
     "tests/test_reconcile_completions.py",
     "tests/test_reconcile_merges.py",
     "tests/test_reconcile_no_agent_record.py",
+    "tests/test_reconcile_tick.py",
     "tests/test_reconcile_usage_limit_done.py",
     "tests/test_refine_chat.py",
     "tests/test_release_cordon_2101.py",

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.139",
+  "tag": "0.5.141",
   "distance": 0,
-  "node": "g30ba3872d2157cfdc9832bc57cca0722f858a55a",
+  "node": "gef1e71db0f0c6bc0d0346e3020abaeabac680993",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-18"

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/coord/agent.py

@@ -6102,6 +6102,48 @@ class AgentServer:
             self._local_health_cache = None
         return {**result.to_dict(), "repo": repo_name}
 
+    def reconcile_drive_queue(self, *, timeout: float = 120.0) -> dict:
+        """``coord drive-queue tick --reconcile-only`` — run on THIS machine
+        (#2373 — ``coord release propagate``'s drain-deadline escalation
+        fans this out to a host it has cordoned but that will not drain, via
+        `_reconcile_launch_host` in `coord/commands/release.py`).
+
+        Only the machine that launched a `running` drive-queue entry can
+        resolve it (the #1870 cross-host guard in
+        `coord.drive_queue._reconcile_running` is keyed on a LOCAL tmux
+        read); this method is that resolution, reachable over this agent's
+        HTTP API instead of a human's SSH session. Full incident writeup
+        (claude-coordinator#2360) and rationale: `_reconcile_launch_host`'s
+        docstring and docs/AGENT_OPERATIONS.md's #2373 section.
+
+        Delegates to :func:`coord.reconcile_tick.run_reconcile_tick` — the
+        same shared implementation `coord release propagate`'s own
+        `_run_reconcile_tick` (`coord/commands/release.py`) calls, so there
+        is exactly one implementation of "what a reconcile-only tick does",
+        not two that can drift apart.
+
+        Returns ``{"ok": bool, "detail": str}``. ``ok=False`` with a
+        ``no local coordinator.yml`` detail when this agent has no on-disk
+        config to pass as ``--config`` — thin-client/config-free mode
+        (docs/EPHEMERAL_WORKERS.md) has nothing for a queue tick to resolve
+        against here. Never raises: the caller (`/drive-queue-reconcile`) is
+        a best-effort self-heal step inside a larger escalation, not a gate
+        that may take propagation down with it.
+        """
+        from coord.reconcile_tick import run_reconcile_tick  # noqa: PLC0415
+
+        config_path = getattr(self._health_config, "path", None)
+        if not config_path:
+            return {
+                "ok": False,
+                "detail": (
+                    "no local coordinator.yml on this agent — cannot resolve "
+                    "a --config path for the tick"
+                ),
+            }
+        ok, detail = run_reconcile_tick(config_path, timeout=timeout, detail_limit=2000)
+        return {"ok": ok, "detail": detail}
+
     def list_repos(self) -> dict[str, dict]:
         """Return local HEAD / branch / dirty flag for each configured repo.
 

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/coord/agent_app.py

@@ -1020,6 +1020,42 @@ def _openapi_spec() -> dict:
                 },
             }
         },
+        "/drive-queue-reconcile": {
+            "post": {
+                "summary": (
+                    "Run `coord drive-queue tick --reconcile-only` on this "
+                    "machine (#2373)"
+                ),
+                "description": (
+                    "The per-machine self-heal `coord release propagate`'s "
+                    "drain-deadline escalation calls before it escalates "
+                    "loudly: only the host that actually launched a "
+                    "`running` drive-queue entry can resolve the #1870 "
+                    "cross-host liveness guard for it. Fleet-queue-wide, "
+                    "not scoped to a repo."
+                ),
+                "requestBody": {
+                    "required": False,
+                    "content": {
+                        "application/json": {
+                            "schema": {
+                                "type": "object",
+                                "properties": {
+                                    "timeout": {
+                                        "type": "number",
+                                        "description": (
+                                            "seconds for the tick subprocess; "
+                                            "defaults to 120."
+                                        ),
+                                    },
+                                },
+                            }
+                        }
+                    },
+                },
+                "responses": {"200": {"description": "OK (including a failed tick)"}},
+            }
+        },
         "/artifact/{repo}/{branch}": {
             "get": {
                 "summary": "Manifest of stashed build artifacts for a (repo, branch) pair",
@@ -1948,6 +1984,36 @@ def build_app(
         result = await asyncio.to_thread(server.fix_graph, repo, timeout=timeout)
         return JSONResponse(result)
 
+    async def drive_queue_reconcile(request: Request) -> JSONResponse:
+        """Run ``coord drive-queue tick --reconcile-only`` on THIS machine
+        (#2373). ``coord release propagate``'s drain-deadline escalation
+        POSTs here — before it escalates loudly — on whichever host it has
+        cordoned but that will not drain, so the one machine that can
+        actually resolve the #1870 cross-host liveness ambiguity for an
+        entry it launched gets asked to, automatically, instead of a human
+        needing to SSH in and run this by hand (the live 2026-08-18 incident
+        this closes).
+
+        Body: optional ``{"timeout": 120}``. No required fields, unlike
+        ``/graph-fix`` — a reconcile-only tick is fleet-queue-wide, not
+        scoped to one repo.
+
+        Runs off the event loop: the subprocess holds the drive-queue file
+        lock for the length of one tick, which must never stall this
+        agent's ``/health`` for the rest of the fleet.
+        """
+        body: dict = {}
+        try:
+            body = await request.json()
+        except Exception:
+            pass
+        try:
+            timeout = float(body.get("timeout", 120.0))
+        except (TypeError, ValueError):
+            timeout = 120.0
+        result = await asyncio.to_thread(server.reconcile_drive_queue, timeout=timeout)
+        return JSONResponse(result)
+
     routes = [
         Route("/health", health, methods=["GET"]),
         Route("/status", status, methods=["GET"]),
@@ -1973,6 +2039,10 @@ def build_app(
         # missing graph and set core.hooksPath, on the machines that
         # actually run workers rather than only on the operator's laptop.
         Route("/graph-fix", graph_fix, methods=["POST"]),
+        # #2373: `coord release propagate`'s drain-deadline escalation's
+        # per-machine self-heal — resolve a `running` drive-queue entry this
+        # host launched before the escalation fires loudly.
+        Route("/drive-queue-reconcile", drive_queue_reconcile, methods=["POST"]),
         # #305: artifact stash manifest (GET /artifact/<repo>/<branch>)
         Route("/artifact/{repo}/{branch}", artifact_manifest, methods=["GET"]),
         # #207: CPU + memory snapshot for TUI sparklines

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/coord/commands/drive_queue.py

@@ -2761,11 +2761,18 @@ def drive_queue_tick(
                 gates=" | ".join(plan.alert.details),
                 command=plan.alert.command,
             )
-        elif any(h.outcome == "released" for h in plan.holds):
-            # A probe just auto-released the gate.  Drop the "QUEUE HELD"
-            # record in the same tick, or `status` keeps shouting HELD while
-            # the queue is demonstrably running again — the contradiction that
-            # teaches an operator to stop reading alerts.
+        else:
+            # #2381: ANY resolved alert-raising condition — a released hold,
+            # a lapsed/cleared cordon, a fixed editable-drift checkout — must
+            # drop the stale escalation record in the SAME tick it resolves,
+            # or `status` (and the TUI/`decisions` report reading the same
+            # record) keeps shouting the old reason while the queue is
+            # demonstrably running again. This used to be gated on `any(h.
+            # outcome == "released" for h in plan.holds)` — the deploy-gate
+            # case only — so a cleared CORDON (no `plan.holds` entry at all)
+            # never cleared its own stale alert. `_clear_queue_alert()` is a
+            # no-op when there is nothing to dismiss, so calling it on every
+            # alert-free tick is safe and cheap.
             _clear_queue_alert()
 
         target = plan.launch

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/coord/commands/release.py

@@ -503,6 +503,77 @@ def _post(url: str, payload: dict, *, timeout: float) -> tuple[int | None, dict,
     return resp.status_code, (body if isinstance(body, dict) else {}), ""
 
 
+#: #2373: default timeout for the remote-reconcile POST below. Generous, but
+#: still small next to `DEFAULT_DRAIN_DEADLINE_SECONDS` (90 minutes), so a
+#: wedged/unreachable agent never turns "ask before escalating" into a
+#: second, longer wait before the loud message an operator is actually
+#: watching for. The POST body forwards a *shorter* remote-subprocess
+#: timeout (see `_RECONCILE_TIMEOUT_MARGIN_SECONDS` below) so the remote
+#: agent's own tick (`AgentServer.reconcile_drive_queue`, otherwise 120s by
+#: default) can never outlive this wait — without that, a tick legitimately
+#: taking longer than this timeout would surface here as a misleading
+#: "unreachable" even though it was still running and would have resolved
+#: correctly on its own.
+DEFAULT_RECONCILE_TIMEOUT_SECONDS = 30.0
+
+#: Margin subtracted from *timeout* before it is forwarded as the remote
+#: subprocess's own timeout (see `_reconcile_launch_host`) — leaves the
+#: agent's HTTP handler room to marshal and return its response before
+#: this side's own `httpx` deadline fires.
+_RECONCILE_TIMEOUT_MARGIN_SECONDS = 5.0
+
+
+def _reconcile_launch_host(
+    machine_host: str,
+    *,
+    agent_port: int,
+    timeout: float = DEFAULT_RECONCILE_TIMEOUT_SECONDS,
+    post=None,
+) -> tuple[bool | None, str]:
+    """Ask *machine_host*'s own agent to run a local reconcile-only tick
+    (#2373), over the same ``POST /<verb>`` HTTP pattern ``/graph-fix``
+    already uses for a per-machine self-heal fanned out from elsewhere.
+
+    Why this has to be a REMOTE call rather than something run locally by
+    whichever host happens to be executing `coord release propagate`: the
+    #1870 cross-host guard (`coord.drive_queue._reconcile_running`) is keyed
+    on a LOCAL tmux read, so only the machine that actually launched a
+    `running` drive-queue entry can ever resolve whether it is still alive.
+    A non-daemon launch host has no periodic tick of its own — confirmed
+    live 2026-08-18 (claude-coordinator#2360): elitebook ran only
+    `coord-agent.service`, dellserver's `coord-drive-queue.timer` correctly
+    refused to declare a foreign-host entry dead every time it ticked, and
+    the ambiguity sat wedging the fleet's release drain for ~17h until a
+    human happened to SSH in and run this exact command by hand. Routing it
+    through the agent HTTP API that already runs on every machine capable of
+    launching a drive closes that gap without a new timer unit anywhere.
+
+    Returns ``(ok, detail)``. ``ok`` is ``None`` when the call could not even
+    be attempted (never treated as a *failure* by the caller — see
+    :func:`_escalate_drain`), ``False`` on an unreachable agent or a non-200
+    response, ``True`` when the agent ran the tick (its own exit code, not
+    "and it resolved the entry" — a genuinely still-busy host is expected to
+    report ``ok=True`` with nothing to reconcile).
+
+    The POST body forwards a remote subprocess timeout derived from
+    *timeout* itself (minus `_RECONCILE_TIMEOUT_MARGIN_SECONDS`) rather than
+    leaving the remote agent to fall back to its own 120s default
+    (`AgentServer.reconcile_drive_queue`) — otherwise a tick that legitimately
+    takes longer than this call's own HTTP wait reports here as
+    `ok=False, detail="unreachable: ..."` even though it is still running and
+    will resolve correctly on its own, just not attributed to this call.
+    """
+    poster = post or _post
+    url = f"http://{machine_host}:{agent_port}/drive-queue-reconcile"
+    remote_timeout = max(1.0, timeout - _RECONCILE_TIMEOUT_MARGIN_SECONDS)
+    status, body, err = poster(url, {"timeout": remote_timeout}, timeout=timeout)
+    if err:
+        return False, f"unreachable: {err}"
+    if status != 200:
+        return False, f"HTTP {status}: {body.get('detail') or body}"
+    return bool(body.get("ok")), str(body.get("detail") or "")
+
+
 def _lane_versions_by_host(report) -> dict[str, list[str | None]]:
     out: dict[str, list[str | None]] = {}
     for lane in report.lanes:
@@ -791,6 +862,11 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
         max_deferrals=cordon_max_deferrals,
         release_cooldown=cordon_cooldown,
         daemon_host=daemon_name,
+        # #2373: lets a blown drain deadline ask the escalated host's own
+        # agent to resolve the #1870 cross-host liveness ambiguity locally
+        # before the loud message goes out — see `_reconcile_launch_host`.
+        machines=getattr(config, "machines", ()) or (),
+        agent_port=AGENT_PORT,
     )
 
     if fully_busy and not force:
@@ -1162,6 +1238,9 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
     max_deferrals: int | None = None,
     release_cooldown: float | None = None,
     daemon_host: str | None = None,
+    machines=None,
+    agent_port: int | None = None,
+    reconcile_timeout: float | None = None,
 ) -> dict:
     """Plan and apply this run's cordons. Returns the journal fragment.
 
@@ -1185,6 +1264,13 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
     pre-#2240 behaviour) rather than to a spurious release: dropping the
     fleet's cordon because a file could not be read is the wrong direction to
     fail in.
+
+    #2373: *machines*/*agent_port* let each :class:`~coord.release_cordon.
+    DrainEscalation` ask the escalated host's OWN agent to run a local
+    reconcile-only tick before the loud message goes out — see
+    :func:`_reconcile_launch_host`. ``machines=None`` (a caller with no fleet
+    topology, e.g. a unit test exercising `plan_cordons` output directly)
+    skips the remote call entirely and escalates exactly as before #2373.
     """
     import time  # noqa: PLC0415
 
@@ -1304,9 +1390,39 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
                 outcome.errors.append(f"release cordon {name}: {exc}")
                 click.echo(f"  ✗ release cordon {name}: {exc}", err=True)
 
+    machines_by_name = {m.name: m for m in (machines or ())}
+    # #2373: each escalation below makes one synchronous `_reconcile_launch_host`
+    # HTTP call, serially, each blocking up to ~`DEFAULT_RECONCILE_TIMEOUT_SECONDS`.
+    # With multiple simultaneously-overdue hosts this adds up to N * timeout of
+    # latency inside one `release propagate` invocation. Fine today — escalations
+    # are rare and few at once — but if that ever stops being true, parallelize
+    # this loop (e.g. a thread pool over `plan.escalations`) rather than letting
+    # a single propagate call silently grow with fleet size.
     for escalation in plan.escalations:
-        _escalate_drain(escalation)
-        outcome.escalated.append(escalation.to_dict())
+        # #2373: before the loud message, give the escalated host's OWN
+        # agent one chance to resolve the ambiguity that is actually holding
+        # it — see `_reconcile_launch_host`'s docstring. `machine` missing
+        # from the fleet map, or no `agent_port` supplied (a caller with no
+        # HTTP topology), skips the call rather than guessing an address.
+        reconcile_ok: bool | None = None
+        reconcile_detail = ""
+        target = machines_by_name.get(escalation.machine)
+        if target is not None and agent_port:
+            reconcile_ok, reconcile_detail = _reconcile_launch_host(
+                target.host,
+                agent_port=agent_port,
+                timeout=(
+                    DEFAULT_RECONCILE_TIMEOUT_SECONDS
+                    if reconcile_timeout is None
+                    else reconcile_timeout
+                ),
+            )
+        _escalate_drain(escalation, reconcile_ok=reconcile_ok, reconcile_detail=reconcile_detail)
+        outcome.escalated.append({
+            **escalation.to_dict(),
+            "reconcile_ok": reconcile_ok,
+            "reconcile_detail": reconcile_detail,
+        })
 
     return outcome.to_dict()
 
@@ -1321,7 +1437,9 @@ DRAIN_ALERT_ISSUE = 0
 DRAIN_ALERT_STAGE = "release-cordon"
 
 
-def _escalate_drain(escalation) -> None:
+def _escalate_drain(
+    escalation, *, reconcile_ok: bool | None = None, reconcile_detail: str = ""
+) -> None:
     """Surface a host that will not drain — loudly, in three places.
 
     stderr (the timer's journal), the escalation table (the TUI and
@@ -1329,21 +1447,43 @@ def _escalate_drain(escalation) -> None:
     criterion 4 is explicitly about the surfaced MESSAGE rather than an
     internal state change, because a silent forever-wait is the failure this
     whole mechanism replaces.
+
+    #2373: *reconcile_ok*/*reconcile_detail* are the outcome of the remote
+    reconcile-only tick `_apply_cordons` asked the escalated host's own
+    agent to run just before calling this — ``None`` when that call was
+    never attempted (no fleet topology / agent port available), so the
+    message stays byte-identical to before #2373 in that case. When it WAS
+    attempted, the outcome is folded into the same message rather than a
+    second alert, so an operator reading one escalation sees both "this is
+    overdue" and "here is what the self-heal attempt already found" without
+    correlating two records.
     """
-    click.echo(f"  ! {escalation.message}", err=True)
+    reconcile_note = ""
+    if reconcile_ok is not None:
+        verb = "ok" if reconcile_ok else "failed"
+        detail = f" ({reconcile_detail})" if reconcile_detail else ""
+        reconcile_note = (
+            f" — asked {escalation.machine}'s own agent to run a local "
+            f"reconcile-only tick first (#2373): {verb}{detail}"
+        )
+    message = f"{escalation.message}{reconcile_note}"
+    click.echo(f"  ! {message}", err=True)
     try:
         from coord.state import record_drive_escalation  # noqa: PLC0415
 
+        gate_readings = (
+            f"machine={escalation.machine} | "
+            f"waited={escalation.waited_seconds:.0f}s | "
+            f"deadline={escalation.deadline_seconds:.0f}s"
+        )
+        if reconcile_ok is not None:
+            gate_readings += f" | remote_reconcile={'ok' if reconcile_ok else 'failed'}"
         record_drive_escalation(
             DRAIN_ALERT_REPO,
             DRAIN_ALERT_ISSUE,
             stage=DRAIN_ALERT_STAGE,
-            reason=escalation.message,
-            gate_readings=(
-                f"machine={escalation.machine} | "
-                f"waited={escalation.waited_seconds:.0f}s | "
-                f"deadline={escalation.deadline_seconds:.0f}s"
-            ),
+            reason=message,
+            gate_readings=gate_readings,
             proposed_command=escalation.command,
         )
     except Exception as exc:  # noqa: BLE001 — the stderr line above is the
@@ -2181,24 +2321,19 @@ def _run_reconcile_tick(config_path: Path, *, runner=None) -> tuple[bool, str]:
     *after* the timer stopped would stay stuck ``running`` for the rest of
     the drain no matter how generous the deadline is — the exact deadlock
     #2110 fixed reconciliation to no longer require the timer for.
+
+    Thin wrapper around :func:`coord.reconcile_tick.run_reconcile_tick`, the
+    #2373-extracted shared implementation also used by
+    `AgentServer.reconcile_drive_queue` (`coord/agent.py`) — see that
+    module's docstring for why there must be exactly one implementation of
+    "what a reconcile-only tick does". A failure here is not fatal on its
+    own: the quiescence check right after this call is the real signal a
+    stuck reconcile eventually clears on a later poll, not this one call's
+    job to guarantee.
     """
-    import subprocess  # noqa: PLC0415
+    from coord.reconcile_tick import run_reconcile_tick  # noqa: PLC0415
 
-    run = runner or subprocess.run
-    try:
-        proc = run(
-            [sys.executable, "-m", "coord.cli", "drive-queue", "tick",
-             "--reconcile-only", "--config", str(config_path)],
-            capture_output=True, text=True, timeout=120,
-        )
-    except Exception as exc:  # noqa: BLE001 — best effort; the quiescence
-        # check right after this call is the real signal a stuck reconcile
-        # eventually clears on a later poll, not this one call's job to
-        # guarantee.
-        return False, f"{type(exc).__name__}: {exc}"
-    ok = getattr(proc, "returncode", 1) == 0
-    detail = (getattr(proc, "stderr", "") or getattr(proc, "stdout", "") or "").strip()
-    return ok, detail[:200]
+    return run_reconcile_tick(config_path, timeout=120, detail_limit=200, runner=runner)
 
 
 def _drain(

code_coordinator-0.5.141/coord/reconcile_tick.py

@@ -0,0 +1,65 @@
+"""Shared primitive: run ``coord drive-queue tick --reconcile-only`` as a
+subprocess and report ``(ok, detail)``.
+
+Two independent surfaces need to run this exact command — the real CLI path,
+with its own file lock (`coord.filelock.drive_queue_lock_path`) against a
+concurrently-running timer tick on the same host, and its own ``--config``
+resolution:
+
+* ``coord release propagate``'s drain loop (`coord/commands/release.py`,
+  `_run_reconcile_tick`, #2110) — polls this on every iteration so a drive
+  that finishes *after* `coord-drive-queue.timer` has been stopped for the
+  drain still gets reconciled from ``running`` to ``done``.
+* `AgentServer.reconcile_drive_queue` (`coord/agent.py`, #2373) — reachable
+  over the agent's HTTP API so `coord release propagate`'s drain-deadline
+  escalation can ask a non-daemon ``launch_host`` to resolve its own
+  cross-host liveness ambiguity (`coord.drive_queue._reconcile_running`'s
+  #1870 guard) before escalating loudly. See ``docs/AGENT_OPERATIONS.md``'s
+  #2373 section for the full incident writeup.
+
+Per this repo's rule (CLAUDE.md, epic #2096): if two surfaces answer the
+same question, they must call the same function. This is that function —
+not two hand-maintained copies that quietly drift apart (as #2373's review
+found: a hardcoded 120s timeout vs. a caller-supplied one, and a 200-char
+detail truncation vs. a 2000-char one).
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def run_reconcile_tick(
+    config_path: Path | str,
+    *,
+    timeout: float = 120.0,
+    detail_limit: int = 2000,
+    runner=None,
+) -> tuple[bool, str]:
+    """Run ``coord drive-queue tick --reconcile-only --config <config_path>``.
+
+    Never raises: a timeout or any other subprocess failure comes back as
+    ``(False, "<ExceptionType>: <message>")`` rather than propagating — both
+    callers treat this as a best-effort step inside a larger loop/escalation,
+    not a gate that may take the caller down with it.
+
+    *runner* is the `subprocess.run`-shaped seam tests substitute; *timeout*
+    and *detail_limit* let each caller keep its own existing behavior
+    (`_run_reconcile_tick`'s 120s/200-char vs. `reconcile_drive_queue`'s
+    caller-supplied timeout/2000-char) without re-implementing the command
+    itself twice.
+    """
+    run = runner or subprocess.run
+    try:
+        proc = run(
+            [sys.executable, "-m", "coord.cli", "drive-queue", "tick",
+             "--reconcile-only", "--config", str(config_path)],
+            capture_output=True, text=True, timeout=timeout,
+        )
+    except Exception as exc:  # noqa: BLE001 — best effort, see docstring
+        return False, f"{type(exc).__name__}: {exc}"
+    ok = getattr(proc, "returncode", 1) == 0
+    detail = (getattr(proc, "stderr", "") or getattr(proc, "stdout", "") or "").strip()
+    return ok, detail[:detail_limit]

{code_coordinator-0.5.139 → code_coordinator-0.5.141}/docs/AGENT_OPERATIONS.md

@@ -1028,6 +1028,48 @@ the gate's dependents stop themselves (the whole queue too, if the entry was
 declared `--scope=fleet` — #2186), propagation sees a *fired* gate as the
 opposite of busy, rolls, and releases the hold.
 
+**#2373 (fixed 2026-08-18): a launch-host-only liveness ambiguity had no
+machine assigned to resolve it.** `_reconcile_running`'s #1870 cross-host
+guard (`coord/drive_queue.py`) is correct and load-bearing: a tick's tmux
+read is always LOCAL, so a `running` entry launched on a *different* host
+than the one ticking must read as `unknown`, never dead — declaring it dead
+from the wrong machine is exactly the false-positive #1870 exists to
+prevent. But only `coord-drive-queue.timer` on the daemon host ticks the
+queue at all; a machine that only runs `coord-agent.service` (precision,
+elitebook) never runs its own tick, so an entry launched there sits
+`running`/`unknown` forever if it actually died — there is no periodic
+process anywhere whose job is to resolve it. Confirmed live 2026-08-18
+(claude-coordinator#2360): an entry launched on elitebook sat `running` for
+~17h; every dellserver tick correctly refused to declare it dead; elitebook's
+release-cordon (which #2101/#2136 renews on every tick that finds the host
+"still busy") kept renewing past its drain deadline and re-escalated with
+nothing ever resolving the ambiguity. Running `coord drive-queue tick
+--reconcile-only` locally on elitebook resolved it in one call — the entry
+moved to `parked` (waiting on a CI re-check, #1891), not actually dead.
+
+The fix closes the loop inside the drain-deadline escalation itself, so it
+needs no new timer unit anywhere: before `_apply_cordons` surfaces a
+`DrainEscalation` (the loud `DRAIN OVERDUE: ...` message), it first `POST`s
+to the escalated host's own agent — `POST /drive-queue-reconcile`
+(`coord/agent_app.py`, handled by `AgentServer.reconcile_drive_queue` in
+`coord/agent.py`) — which shells out to `coord drive-queue tick
+--reconcile-only` *on that machine*, so the #1870 guard's own local-tmux-read
+requirement is satisfied by construction: the host asked is the host whose
+evidence actually counts. The outcome (`ok`/`detail`) is folded into the SAME
+escalation message and escalation-table record rather than a second alert —
+"DRAIN OVERDUE: elitebook has been cordoned for 91m ... — asked elitebook's
+own agent to run a local reconcile-only tick first (#2373): ok (moved to
+parked)". An unreachable agent does not suppress the escalation; the loud
+message still goes out, with the failed self-heal attempt named rather than
+hidden. Every machine capable of launching a drive already runs
+`coord-agent.service`, so no fleet ever needs a new unit installed for this
+to work; a machine with no local `coordinator.yml` (thin-client/config-free
+mode) refuses the call cleanly (`no local coordinator.yml on this agent`)
+since it has nothing to pass as `--config`. See
+`tests/test_release_cordon_2101.py`'s `#2373` section for the seeded
+end-to-end shape (a `running` entry pinned to a non-daemon host, cordoned
+past the drain deadline, resolved without an operator SSH session).
+
 ## `coord.db` backups to the external SSD (interim — #1822 owns the real thing)
 
 `~/.coord/coord.db` on the daemon host is the fleet's canonical state and had