code-coordinator 0.5.84__tar.gz → 0.5.85__tar.gz

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.84
+Version: 0.5.85
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.84
+Version: 0.5.85
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/code_coordinator.egg-info/SOURCES.txt

@@ -668,6 +668,7 @@ tests/test_reconcile_merges.py
 tests/test_reconcile_usage_limit_done.py
 tests/test_refine_chat.py
 tests/test_release_cordon_2101.py
+tests/test_release_cordon_deadlock_2240.py
 tests/test_release_propagate.py
 tests/test_release_unified_1242.py
 tests/test_release_verify.py

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/code_coordinator.egg-info/scm_file_list.json

@@ -698,6 +698,7 @@
     "tests/test_reconcile_usage_limit_done.py",
     "tests/test_refine_chat.py",
     "tests/test_release_cordon_2101.py",
+    "tests/test_release_cordon_deadlock_2240.py",
     "tests/test_release_propagate.py",
     "tests/test_release_unified_1242.py",
     "tests/test_release_verify.py",

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.84",
+  "tag": "0.5.85",
   "distance": 0,
-  "node": "g9d455921f6488cf863cf829624bf9cd29658a2f4",
+  "node": "g87a8be6dfe549ad5607352dbdda991a3ed87f1fd",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-14"

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/coord/auto_loop.py

@@ -958,8 +958,16 @@ def _dispatch_fix(
     machine = next(
         (m for m in config.machines if m.name == work.machine_name), None
     )
-    from coord.machine_pause import paused_set
-    paused = paused_set(config.machines)
+    # #2240: the pause set here is `follow_on_paused_set()`, NOT `paused_set()`
+    # — a fix leg is the tail of work that is already running (dispatched
+    # after a `request-changes` review verdict on a row that has not gone
+    # anywhere new), so a release cordon ("route no NEW work here") must not
+    # filter its host out. This is the same fix as `coord/review.py`'s
+    # reviewer-selection change; leaving this call on `paused_set()` would
+    # reproduce the fleet-wide deadlock for the fix leg instead of the
+    # review leg. Explicit pauses and quiet hours still apply.
+    from coord.machine_pause import follow_on_paused_set
+    paused = follow_on_paused_set(config.machines)
     if (
         machine is None
         or not machine.can_work_on(work.repo_name)

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/coord/commands/release.py

@@ -506,6 +506,17 @@ def _lane_versions_by_host(report) -> dict[str, list[str | None]]:
 @click.option("--drain-deadline", default=None, type=float,
               help="Seconds a host may fail to drain before the cordon "
                    "escalates loudly (default 5400).")
+@click.option("--cordon-max-deferrals", default=None, type=int,
+              help="#2240: consecutive DEFERRED runs that may hold a cordon "
+                   "for the same target before it is released outright "
+                   "(default 2). A cordon that has failed to produce a window "
+                   "twice running is not draining anything — it is blocking "
+                   "the review dispatch it is waiting for. 0 disables the "
+                   "bound and re-arms the deadlock.")
+@click.option("--cordon-cooldown", default=None, type=float,
+              help="Seconds after a #2240 release before cordoning may resume "
+                   "(default 1800). Without it the next run re-cordons — the "
+                   "hosts are still behind — and the deadlock re-arms.")
 @click.option("--json", "as_json", is_flag=True, help="Emit the propagation record as JSON.")
 def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions are elsewhere
     config_path: Path,
@@ -522,6 +533,8 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
     cordon_after: int | None,
     cordon_ttl: float | None,
     drain_deadline: float | None,
+    cordon_max_deferrals: int | None,
+    cordon_cooldown: float | None,
     as_json: bool,
 ) -> None:
     """One propagation attempt. Exit 0 on deferral, 1 on red, 2 on rollback.
@@ -559,6 +572,16 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
     run that defers without cordoning is a run that will defer again in 20
     minutes for exactly the same reason, which is how the fleet sat eleven
     releases behind for a day with elitebook idle and rollable throughout.
+
+    #2240: and a deferral is not passive either. The cordon it leaves behind
+    is what blocks the review dispatch that would let the between-legs entry
+    finish, and that unfinished entry is what defers the next run — four
+    cycles, 70 minutes, three idle machines. So the cordon now has a bound
+    that nothing else has to be running for: ``--cordon-max-deferrals``
+    consecutive deferrals for one target and every cordon is dropped, with
+    ``--cordon-cooldown`` seconds before any may be set again. The counter
+    lives in the propagation journal, because the process holding it is
+    restarted by the roll it gates.
     """
     import json as _json  # noqa: PLC0415
     import time  # noqa: PLC0415
@@ -686,6 +709,9 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
         ttl_seconds=cordon_ttl,
         drain_deadline=drain_deadline,
         dry_run=dry_run,
+        state_dir=state_dir,
+        max_deferrals=cordon_max_deferrals,
+        release_cooldown=cordon_cooldown,
     )
 
     if fully_busy and not force:
@@ -1042,7 +1068,7 @@ def _oldest_version(versions: list[str | None]) -> str | None:
     return min(readable, key=_key)
 
 
-def _apply_cordons(
+def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
     *,
     hosts: list[str],
     report,
@@ -1053,6 +1079,9 @@ def _apply_cordons(
     ttl_seconds: float | None,
     drain_deadline: float | None,
     dry_run: bool,
+    state_dir: Path | None = None,
+    max_deferrals: int | None = None,
+    release_cooldown: float | None = None,
 ) -> dict:
     """Plan and apply this run's cordons. Returns the journal fragment.
 
@@ -1060,16 +1089,27 @@ def _apply_cordons(
     error and the roll continues on its existing (quiescence-based) rules.
     Failing the whole propagation because the cordon could not be renewed
     would make #2101's fix strictly worse than not having it.
+
+    #2240: *state_dir* is where the propagation journal lives, and the journal
+    is where "how many runs in a row has this cordon now deferred?" is stored
+    — the process holding the answer in memory is restarted by the roll it is
+    gating, so an in-memory counter would reset exactly when the deadlock is
+    at its worst. An unreadable journal degrades to "no pressure" (the
+    pre-#2240 behaviour) rather than to a spurious release: dropping the
+    fleet's cordon because a file could not be read is the wrong direction to
+    fail in.
     """
     import time  # noqa: PLC0415
 
     from coord import release_cordon as rc  # noqa: PLC0415
+    from coord import release_propagate as rp  # noqa: PLC0415
     from coord.machine_pause import (  # noqa: PLC0415
         clear_cordon,
         cordons as read_cordons,
         set_cordon,
     )
 
+    now = time.time()
     outcome = rc.CordonOutcome()
     try:
         existing = read_cordons()
@@ -1077,11 +1117,31 @@ def _apply_cordons(
         outcome.errors.append(f"could not read the cordon store: {exc}")
         return outcome.to_dict()
 
+    pressure = rc.DeferralPressure()
+    if state_dir is not None:
+        try:
+            pressure = rc.deferral_pressure(
+                rp.read_records(state_dir), target_version=target_version
+            )
+        except Exception as exc:  # noqa: BLE001 — see docstring
+            outcome.errors.append(f"could not read the propagation journal: {exc}")
+    outcome.pressure = pressure.to_dict()
+
+    # #2240 review: resolved once and journaled on every run (not just a
+    # releasing one) via `outcome.max_deferrals`, so `deferral_pressure()`
+    # can read the operator's actual `--cordon-max-deferrals` back out of
+    # the newest record instead of `describe_deferral_pressure()` callers
+    # silently assuming `DEFAULT_MAX_DEFERRALS`.
+    resolved_max_deferrals = (
+        rc.DEFAULT_MAX_DEFERRALS if max_deferrals is None else max_deferrals
+    )
+    outcome.max_deferrals = resolved_max_deferrals
+
     plan = rc.plan_cordons(
         target_version=target_version,
         host_versions=_python_lane_versions(report, hosts, target_version),
         existing=existing,
-        now=time.time(),
+        now=now,
         ttl_seconds=(
             rc.DEFAULT_TTL_SECONDS if ttl_seconds is None else ttl_seconds
         ),
@@ -1095,9 +1155,23 @@ def _apply_cordons(
         ),
         busy_reasons=busy_reasons,
         enabled=enabled,
+        pressure=pressure,
+        max_deferrals=resolved_max_deferrals,
+        release_cooldown=(
+            rc.DEFAULT_RELEASE_COOLDOWN_SECONDS
+            if release_cooldown is None
+            else release_cooldown
+        ),
     )
     for line in plan.render():
         click.echo(line)
+    outcome.cooling_seconds = plan.cooling_seconds
+    if plan.released is not None:
+        outcome.released = plan.released.to_dict()
+        # stderr as well as stdout: on the timer host this line IS the
+        # operator's only notice that the fleet just spent 40 minutes
+        # cordoned for nothing, and the timer's journal is stderr.
+        click.echo(f"warning: {plan.released.message}", err=True)
     if dry_run:
         # `--dry-run` promises to change nothing, cordon store included. The
         # plan above is still printed, so a dry run answers "and what would
@@ -1126,6 +1200,20 @@ def _apply_cordons(
             outcome.errors.append(f"uncordon {name}: {exc}")
             click.echo(f"  ✗ uncordon {name}: {exc}", err=True)
 
+    if plan.released is not None:
+        # #2240: stamped even when every clear_cordon() below fails or finds
+        # nothing — `released_at` is what resets the counter and starts the
+        # cooldown, and a release the journal does not record is a release
+        # that happens again on the very next tick, forever.
+        outcome.released_at = now
+        for name in plan.released.hosts:
+            try:
+                if clear_cordon(name):
+                    outcome.uncordoned.append(name)
+            except Exception as exc:  # noqa: BLE001 — see docstring
+                outcome.errors.append(f"release cordon {name}: {exc}")
+                click.echo(f"  ✗ release cordon {name}: {exc}", err=True)
+
     for escalation in plan.escalations:
         _escalate_drain(escalation)
         outcome.escalated.append(escalation.to_dict())
@@ -1829,6 +1917,27 @@ def release_cordon(
             f"[{record.age(now) / 60.0:.0f}m draining, lapses in "
             f"{remaining:.0f}m, owner={record.owner}]{overdue}"
         )
+    # #2240: the same stall count `coord status` appends, on the surface an
+    # operator reaches for once they have noticed the fleet is quiet. The
+    # journal is a local file (the propagate timer's host); no journal → no
+    # line, never a fabricated one.
+    target = next(
+        (c.target_version for c in active.values() if c.target_version), None
+    )
+    try:
+        pressure = rc.deferral_pressure(
+            _read_propagation_records(), target_version=target
+        )
+    except Exception:  # noqa: BLE001 — a listing must not fail on the journal
+        pressure = rc.DeferralPressure()
+    if pressure.consecutive:
+        click.echo(
+            f"\n! {rc.describe_deferral_pressure(pressure)}: these cordons "
+            "have not produced a rollable window. A cordon that has failed "
+            "twice running is blocking the work it is waiting for (#2240) — "
+            f"`coord release propagate` releases it by itself after "
+            f"{rc.DEFAULT_MAX_DEFERRALS}."
+        )
     click.echo(
         f"\nclear one with `coord release cordon --clear <machine>` "
         f"(default lifetime {rc.DEFAULT_TTL_SECONDS / 60:.0f}m — a cordon "
@@ -1836,6 +1945,16 @@ def release_cordon(
     )
 
 
+def _read_propagation_records() -> list[dict]:
+    """The propagation journal, or ``[]`` when this host does not have one."""
+    from coord import release_propagate as rp  # noqa: PLC0415
+
+    try:
+        return rp.read_records(_state_dir())
+    except Exception:  # noqa: BLE001 — see every caller: never load-bearing
+        return []
+
+
 @release_group.command(
     "rollback",
     help=(

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/coord/commands/status.py

@@ -18,6 +18,43 @@ if TYPE_CHECKING:
     from coord.config import Config
 
 
+def _cordon_stall_suffix(cordons: dict) -> str:
+    """The " (deferred N runs — NOT DRAINING)" tag for a stalled cordon (#2240).
+
+    `coord status` renders "CORDONED: DRAINING FOR V0.5.77" for a fleet that
+    is upgrading itself in thirty seconds and for one that has been unable to
+    dispatch anything for seventy minutes, identically. That is the whole
+    observability half of #2240: "nothing surfaced it either — `coord status`
+    shows `CORDONED: DRAINING FOR V0.5.77`, which reads as normal in-progress
+    behaviour rather than a 70-minute stall."
+
+    The count lives in the propagation journal, which is a file on whichever
+    host runs the propagate timer. A thin client has no such file, so this
+    degrades to "" — a missing journal must never turn a normal drain into a
+    reported stall, and the surface it is missing from is precisely the one
+    that is not running the loop.
+    """
+    if not cordons:
+        return ""
+    from coord import release_cordon as rc  # noqa: PLC0415
+    from coord import release_propagate as rp  # noqa: PLC0415
+    from coord.commands.release import _state_dir  # noqa: PLC0415
+
+    target = next(
+        (getattr(c, "target_version", None) for c in cordons.values()
+         if getattr(c, "target_version", None)),
+        None,
+    )
+    try:
+        pressure = rc.deferral_pressure(
+            rp.read_records(_state_dir()), target_version=target
+        )
+    except Exception:  # noqa: BLE001 — a status render must never fail on this
+        return ""
+    described = rc.describe_deferral_pressure(pressure)
+    return f" ({described})" if described else ""
+
+
 def _live_advisory_entries(
     entries: list[dict],
     cfg: "Config",
@@ -144,6 +181,11 @@ def status(config_path: Path, machine_filter: str | None, no_reconcile: bool, ti
     )
     paused = paused_set(cfg.machines)
     cordons = fetch_cordons()
+    # #2240: how many consecutive propagate runs this cordon has now failed to
+    # produce a window for. Computed once, appended to every cordoned
+    # machine's label below — "draining" and "deadlocked" must not render the
+    # same, which is why the 70-minute stall was invisible.
+    cordon_stall = _cordon_stall_suffix(cordons)
     quiet_windows = effective_quiet_hours(cfg.machines)
 
     statuses = check_all(machines, timeout=timeout)
@@ -211,7 +253,9 @@ def status(config_path: Path, machine_filter: str | None, no_reconcile: bool, ti
             # #2101 trap E: name the version it is draining for, so a stopped
             # machine reads as "the fleet is upgrading itself" rather than as
             # a mystery.
-            label = f"{pause_state.detail.upper()} — {label}"
+            # #2240: ...and say when it has stopped being that, so it does
+            # not read as normal in-progress behaviour for an hour.
+            label = f"{pause_state.detail.upper()}{cordon_stall} — {label}"
         elif pause_state is not None and pause_state.kind == "hand":
             label = f"PAUSED — {label}"
         elif pause_state is not None and pause_state.kind == "quiet":

{code_coordinator-0.5.84 → code_coordinator-0.5.85}/coord/machine_pause.py

@@ -76,6 +76,26 @@ else:
   for a cordon to lapse; the live loop renews on every run while the host is
   still behind (#2101 trap B).
 
+#2240: A CORDON MEANS "NO NEW WORK", AND A FOLLOW-ON LEG IS NOT NEW WORK.
+Folding cordons into the one `paused_set()` is what made #2101 buildable, but
+it also gave a cordon a reach a *drain* must not have.  Observed 2026-08-14:
+the fleet cordoned to drain for v0.5.77; an entry that had finished Work and
+Test needed its REVIEW dispatched; `pick_reviewer_machine` filtered every
+cordoned host out of `paused_set()` and returned "no eligible reviewer
+machine configured"; the entry therefore stayed `running` forever; a
+perpetually-running entry defers the roll; a deferred roll leaves the cordon
+up.  Four cycles, 70 minutes, three idle machines, no exit without a human.
+
+The cordon was blocking the completion of the very work it was waiting to
+drain.  So `paused_set(..., include_cordons=False)` — reached through
+`follow_on_paused_set()` — is the set a dispatch consults when it is
+finishing work already in flight rather than starting new work.  It still
+honours explicit `coord pause` and quiet hours: those are an operator's or a
+policy's decision about a machine and mean what they say.  A cordon is this
+fleet's own drain asking a host to go idle, and refusing to dispatch the leg
+that would make it idle is self-defeating (a `--merge-of`/review dispatch
+onto a cordoned host is what the drain WANTS — it is what ends the work).
+
 #2146: OPERATOR-SET quiet hours.  #1862's window can only be declared in
 `coordinator.yml`, which on a thin client is a read-only cache that is
 re-fetched and overwritten on essentially every command, and on the daemon
@@ -142,7 +162,10 @@ def _state_path() -> Path:
 
 
 def paused_set(
-    machines: Sequence["Machine"] | None = None, *, now: datetime | None = None,
+    machines: Sequence["Machine"] | None = None,
+    *,
+    now: datetime | None = None,
+    include_cordons: bool = True,
 ) -> set[str]:
     """Read the current set of paused machine names (#1563: daemon-aware).
 
@@ -160,16 +183,54 @@ def paused_set(
     call site) to fold quiet hours into the LOCAL computation — the daemon's
     own in-process tick-loop calls (no board service configured for
     itself), and any solo/local use with no daemon at all.
+
+    *include_cordons* (#2240) is False only for a dispatch that FINISHES work
+    already in flight — see `follow_on_paused_set()`, which is the name every
+    caller should use, and the module docstring for the 70-minute fleet-wide
+    deadlock that made this necessary. It never widens the set: explicit
+    pauses and quiet hours are untouched, and a cordon read that fails is
+    left IN (a cordon we could not resolve stays a pause, the same direction
+    every other read here fails in).
     """
     svc = _resolve_service()
     if svc is not None:
         from coord.client import fetch_paused_machines  # noqa: PLC0415
 
         try:
-            return fetch_paused_machines(svc)
+            names = fetch_paused_machines(svc)
         except Exception:  # noqa: BLE001 — fail-soft read, see module docstring
             return set()
-    return local_paused_set(machines, now=now)
+        if include_cordons:
+            return names
+        # The daemon publishes ONE union (`local_paused_set`), so the cordon
+        # half has to be subtracted here from its own endpoint — a second
+        # round trip, paid only on the follow-on path, rather than a second
+        # union the daemon would have to learn to publish.
+        try:
+            return names - set(cordons())
+        except Exception:  # noqa: BLE001 — see docstring: cordons stay paused
+            return names
+    return local_paused_set(machines, now=now, include_cordons=include_cordons)
+
+
+def follow_on_paused_set(
+    machines: Sequence["Machine"] | None = None, *, now: datetime | None = None,
+) -> set[str]:
+    """The pause set for a dispatch that COMPLETES in-flight work (#2240).
+
+    `paused_set()` minus release cordons. Use this — and only this — for a
+    review / smoke / fix leg of an assignment that is already running: a
+    cordon means "route no NEW work here", and the tail of the work the
+    cordon is explicitly waiting to drain is not new work. Blocking it is
+    self-defeating in the precise way #2240 observed, because the entry can
+    then never finish, so the host never drains, so the cordon never lifts,
+    so the roll defers and re-cordons.
+
+    Everything else still applies: an explicit `coord pause` and a quiet-hours
+    window both mean "this machine is unavailable, full stop", and neither is
+    this fleet's own drain talking to itself.
+    """
+    return paused_set(machines, now=now, include_cordons=False)
 
 
 def is_paused(
@@ -259,7 +320,10 @@ def _resolve_service():  # -> coord.client.ServiceConfig | None
 
 
 def local_paused_set(
-    machines: Sequence["Machine"] | None = None, *, now: datetime | None = None,
+    machines: Sequence["Machine"] | None = None,
+    *,
+    now: datetime | None = None,
+    include_cordons: bool = True,
 ) -> set[str]:
     """The local, effective paused-machine set: explicit pauses UNION any
     machine currently inside its quiet-hours window (#1862) UNION any machine
@@ -279,8 +343,16 @@ def local_paused_set(
     the daemon's `/pause` endpoint handler calls (passing `config.machines`
     so quiet hours apply), and what `paused_set()` itself falls through to
     when no board service is configured.
+
+    *include_cordons* (#2240) drops the cordon half for a dispatch that
+    finishes work already in flight — see `follow_on_paused_set()`. The
+    daemon's own `/pause` handler must never pass it: what that endpoint
+    publishes is the full routing set, and the subtraction is the caller's
+    decision to make, per dispatch.
     """
-    effective = _explicit_paused_set() | cordoned_names(now=_epoch(now))
+    effective = _explicit_paused_set()
+    if include_cordons:
+        effective |= cordoned_names(now=_epoch(now))
     if not machines:
         return effective
     return effective | _quiet_covered_names(machines, now=now)