code-coordinator 0.5.497__tar.gz → 0.5.498__tar.gz

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.497
+Version: 0.5.498
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.497
+Version: 0.5.498
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.497",
+  "tag": "0.5.498",
   "distance": 0,
-  "node": "gd05a81b74292333679389936cf7cdb3edea6fd12",
+  "node": "g99081c11bad4947401aaeef6ccd4301a2b450952",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-19"

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/coord/commands/release.py

@@ -1722,8 +1722,32 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
         # never touched would undo somebody else's deliberate state.
         from coord.agent_update import cli_initiator  # noqa: PLC0415
 
+        # #3364: narrow further, to the hosts the RED GATE ITSELF names.
+        # `--rollback-on-red` used to revert every host this run updated
+        # the moment ANY of them went red — four hosts that rolled and
+        # verified clean, reverted because a fifth could not roll at all.
+        # A host that rolled and verified is not part of the failure just
+        # because a sibling host's roll was; partial fleet convergence is
+        # strictly better than reverting everyone plus the fifth's own
+        # damage. `blocking_hosts` is `None` when the gate cannot attribute
+        # to specific hosts (a fleet-wide finding) — that must fail toward
+        # rolling back every updated host, the previous (safe) behavior,
+        # never toward rolling back none of them.
+        implicated = gate.blocking_hosts
+        targets = (
+            updated_hosts if implicated is None
+            else [h for h in updated_hosts if h in implicated]
+        )
+        spared = [h for h in updated_hosts if h not in targets]
+        if spared:
+            record.spared_rollback = spared
+            _out(
+                "  spared (rolled and verified, not named by the red gate): "
+                + ", ".join(spared)
+            )
+
         down: list[str] = []
-        for host in updated_hosts:
+        for host in targets:
             machine = by_name.get(host)
             if machine is None:
                 continue
@@ -2948,6 +2972,20 @@ def _rollback_host(
     take under systemd), and only then gives up — loudly, naming the host as
     DOWN rather than reporting a tidy "rolling back".
 
+    #3364: the agent answering ``/health`` again is not the end of the job
+    either. ``POST /rollback`` (``agent_app.py``'s ``rollback`` handler)
+    flips ``~/.coord-venv`` back and re-execs the AGENT — and, same gap
+    #2069 closed for the forward ``/update`` path, nothing about that
+    restarts ``coord-serve``/``coord-web``/``coord-drive-queue``. Left as
+    they were, those siblings keep running out of the slot the rollback
+    just deactivated — permanently orphaned on the slot the next ``/update``
+    must delete and rebuild, which is exactly the #2121 occupied-slot
+    deadlock this issue is about. So a confirmed agent restart here also
+    triggers the same ``_restart_sibling_services`` call `_roll_python` uses,
+    and a sibling that fails to come back on the reverted slot fails this
+    rollback's own verdict — "the venv flipped back" is not "the host is
+    back", same reasoning as #2095 for the forward roll.
+
     #2121: *initiator* names this call on the target host's audit trail, the
     same as ``_roll_python``'s own ``/update`` POST a few lines up — both are
     automation that fires during a fleet roll with nobody at a keyboard, so
@@ -2968,9 +3006,23 @@ def _rollback_host(
     if status != 202:
         return False, str(body.get("error") or f"HTTP {status}")
 
+    def _back_on_target(note: str) -> tuple[bool, str]:
+        """The agent answered ``/health`` on the reverted slot — re-home the
+        siblings onto it too (#3364) before this counts as done."""
+        sib_ok, sib_detail, _sib_failed = _restart_sibling_services(
+            machine, agent_port=agent_port
+        )
+        if sib_ok is False:
+            return False, f"{note}, but {sib_detail}"
+        # True (nothing failed) and None (this agent predates
+        # /restart-services — no channel to have restarted anything
+        # through) both still count as a rollback success: neither is a
+        # service THIS call took down.
+        return True, f"{note}; {sib_detail}"
+
     back, version = _wait_agent_back(machine, agent_port=agent_port, timeout=timeout)
     if back:
-        return True, f"rolled back; agent is serving again on v{version}"
+        return _back_on_target(f"rolled back; agent is serving again on v{version}")
 
     # The re-exec did not take. This is the documented systemd stall, and it
     # has a documented fix — apply it rather than handing the operator a
@@ -2981,7 +3033,7 @@ def _rollback_host(
             machine, agent_port=agent_port, timeout=min(timeout, 60.0)
         )
         if back:
-            return True, (
+            return _back_on_target(
                 f"rolled back; agent needed an SSH `systemctl --user restart "
                 f"coord-agent` but is serving again on v{version}"
             )

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/coord/release_propagate.py

@@ -419,6 +419,14 @@ OUT_OF_REACH_LANES: frozenset[str] = frozenset({"~/.coord-cli-venv", "webapp bun
 #: for a lane, and the only form a grouped finding names its lanes by.
 _LANE_LABEL = re.compile(r"^(?P<lane>.+?)\s+\((?P<host>[^()]+)\)$")
 
+#: The pseudo-host ``coord/release_verify.py``'s ``verify()`` stamps on a
+#: finding it cannot pin to one machine (a fleet-wide version-skew or
+#: no-data finding — every ``host="(fleet)"`` call there). Named here so
+#: :attr:`GateVerdict.blocking_hosts` recognises it as "not a real host" the
+#: same way :func:`lane_is_out_of_reach`'s allow-list keeps a lane's gaps
+#: explicit rather than inferred.
+_UNATTRIBUTABLE_HOST = "(fleet)"
+
 
 def parse_lane_label(label: str) -> tuple[str, str] | None:
     """``"~/.coord-venv (precision)"`` -> ``("precision", "~/.coord-venv")``.
@@ -487,6 +495,36 @@ class GateVerdict:
     def red(self) -> bool:
         return self.severity == "crit"
 
+    @property
+    def blocking_hosts(self) -> frozenset[str] | None:
+        """Real hosts a BLOCKING finding actually names (#3364).
+
+        ``--rollback-on-red`` used to revert every host a run updated the
+        instant ANY of them went red — a fifth host's failure rolling back
+        four that verified clean. This is what lets a caller narrow to just
+        the hosts the red gate is actually about, the same way
+        :func:`attempted_scope` already narrows which lanes count.
+
+        ``None`` means "could not attribute to specific hosts" — a
+        fleet-wide finding like the no-``--expected``-given skew case
+        (``host="(fleet)"``, see ``coord/release_verify.py``'s ``verify()``)
+        or a finding with no host at all. A caller must fail toward that
+        meaning "implicates everyone this run touched", never toward
+        "implicates nobody" — the same "an unattributable signal blocks
+        every host, never none of them" rule :attr:`Quiescence.
+        fleet_wide_busy` already follows for busy signals. A concrete
+        (possibly empty, when there are no blocking findings at all) set
+        means every blocking finding named a real host, and rollback may
+        scope to exactly those.
+        """
+        hosts: set[str] = set()
+        for finding in self.blocking:
+            for host, _lane in _finding_pairs(finding):
+                if not host or host == _UNATTRIBUTABLE_HOST:
+                    return None
+                hosts.add(host)
+        return frozenset(hosts)
+
     def to_dict(self) -> dict:
         return {
             "severity": self.severity,
@@ -1485,6 +1523,12 @@ class PropagationRecord:
     #: done so, or the resulting quiet fleet is indistinguishable from #2082.
     cordons: dict = field(default_factory=dict)
     rolled_back: list[str] = field(default_factory=list)
+    #: #3364: hosts this run updated and verified clean but did NOT roll back
+    #: even though the gate went red — because the red gate's blocking
+    #: findings (:attr:`GateVerdict.blocking_hosts`) named some OTHER host,
+    #: not this one. A host that rolled and verified is not part of the
+    #: failure just because a sibling host's roll was.
+    spared_rollback: list[str] = field(default_factory=list)
     released_holds: list[str] = field(default_factory=list)
     #: #2583: the min-releases-behind gate's own readings for this run —
     #: ``min_releases_behind`` is whatever this run resolved (flag > config >
@@ -1763,6 +1807,11 @@ def render_record(record: PropagationRecord | Mapping[str, Any]) -> list[str]:
 
     if data.get("rolled_back"):
         lines.append(f"    rolled back: {', '.join(data['rolled_back'])}")
+    if data.get("spared_rollback"):
+        lines.append(
+            "    spared (rolled and verified, not named by the red gate): "
+            + ", ".join(data["spared_rollback"])
+        )
     if data.get("released_holds"):
         lines.append(f"    released deploy gates: {', '.join(data['released_holds'])}")
     if data.get("error"):

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/tests/test_cli_release_propagate.py

@@ -1170,6 +1170,77 @@ def test_a_host_whose_python_lane_failed_is_not_rolled_back(valid_config_path,
     assert rolled_back == []
 
 
+def test_a_red_gate_naming_one_host_spares_the_others(valid_config_path, state_dir,
+                                                       no_network, monkeypatch):
+    """#3364: `--rollback-on-red` used to revert every host this run updated
+    the instant ANY of them went red — four hosts that rolled and verified
+    clean, reverted because a fifth could not roll at all. A host that
+    rolled and verified is not part of the failure just because a sibling
+    host's roll was; partial fleet convergence beats reverting everyone and
+    re-orphaning a daemon host's serve/web for nothing."""
+    from coord import release_verify as rv
+
+    _stub_lanes(monkeypatch)
+    _stub_verify(
+        monkeypatch,
+        versions={"laptop": ["0.4.110"], "server": ["0.4.110"]},
+        findings=[
+            rv.Finding(severity="crit", host="laptop",
+                       lane="~/.coord-venv (laptop)", summary="stubbed"),
+        ],
+    )
+    rolled_back: list[str] = []
+    monkeypatch.setattr(
+        release_cmd, "_rollback_host",
+        lambda machine, **k: (rolled_back.append(machine.name), (True, "rolling back"))[1],
+    )
+    result = CliRunner().invoke(
+        main,
+        ["release", "propagate", "--config", str(valid_config_path),
+         "--target", "0.4.111", "--daemon-host", "server"],
+    )
+    assert result.exit_code == 2, result.output
+    assert rolled_back == ["laptop"], "server rolled and verified clean — it must be spared"
+    record = _records(state_dir)[0]
+    assert record["status"] == rp.STATUS_ROLLED_BACK
+    assert record["spared_rollback"] == ["server"]
+    assert "spared" in result.output
+
+
+def test_a_fleet_wide_finding_still_rolls_back_every_updated_host(
+    valid_config_path, state_dir, no_network, monkeypatch
+):
+    """A finding `blocking_hosts` cannot attribute to a real host (`(fleet)`
+    — no per-lane breakdown) must fail toward the old, safe behavior: roll
+    back every host this run touched, not toward sparing everyone because
+    nothing could be pinned down."""
+    from coord import release_verify as rv
+
+    _stub_lanes(monkeypatch)
+    _stub_verify(
+        monkeypatch,
+        versions={"laptop": ["0.4.110"], "server": ["0.4.110"]},
+        findings=[
+            rv.Finding(severity="crit", host="(fleet)", lane="(version skew)",
+                       summary="2 versions live across the fleet"),
+        ],
+    )
+    rolled_back: list[str] = []
+    monkeypatch.setattr(
+        release_cmd, "_rollback_host",
+        lambda machine, **k: (rolled_back.append(machine.name), (True, "rolling back"))[1],
+    )
+    result = CliRunner().invoke(
+        main,
+        ["release", "propagate", "--config", str(valid_config_path),
+         "--target", "0.4.111", "--daemon-host", "server"],
+    )
+    assert result.exit_code == 2, result.output
+    assert sorted(rolled_back) == ["laptop", "server"]
+    record = _records(state_dir)[0]
+    assert record["spared_rollback"] == []
+
+
 def test_a_failed_daemon_python_roll_skips_other_hosts_python_lane(
     valid_config_path, state_dir, no_network, monkeypatch
 ):
@@ -1362,6 +1433,12 @@ def test_release_rollback_hits_every_machine(valid_config_path, monkeypatch):
     monkeypatch.setattr(release_cmd, "_post", _fake_post)
     monkeypatch.setattr(release_cmd, "_get",
                         lambda url, *, timeout: (200, {"version": "0.4.110"}))
+    # #3364: the sibling-restart call this test isn't about — assert it's
+    # made in the dedicated `_rollback_host` tests below instead.
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda *a, **k: (True, "restarted coord-serve", {}),
+    )
     result = CliRunner().invoke(
         main, ["release", "rollback", "--config", str(valid_config_path), "--yes"]
     )
@@ -1414,6 +1491,10 @@ def test_a_rollback_rescued_by_the_ssh_restart_is_a_success(valid_config_path,
         "coord.commands.agent_ops._escalate_restart",
         lambda machine: revived.__setitem__("yes", True) or True,
     )
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda *a, **k: (True, "restarted coord-serve", {}),
+    )
     result = CliRunner().invoke(
         main, ["release", "rollback", "--config", str(valid_config_path), "--yes",
                "--wait", "1"]
@@ -1737,7 +1818,14 @@ def test_a_stale_unit_advisory_names_the_units_remedy_not_agent_update(
 
 def test_a_crit_on_a_lane_this_run_rolled_still_reverts(valid_config_path, state_dir,
                                                         no_network, monkeypatch):
-    """Scoping the gate is not removing it."""
+    """Scoping the gate is not removing it: a crit on `laptop`'s own
+    python lane — a lane THIS run rolled — must still turn the gate red and
+    trigger a rollback, never get waved through as advisory.
+
+    #3364: which HOST that rollback then touches is a separate, narrower
+    question. The finding names only `laptop`, so only `laptop` — not
+    `server`, which rolled and verified clean — gets rolled back; see
+    `test_a_red_gate_naming_one_host_spares_the_others` for that half."""
     from coord import release_verify as rv
 
     _stub_lanes(monkeypatch)
@@ -1759,7 +1847,7 @@ def test_a_crit_on_a_lane_this_run_rolled_still_reverts(valid_config_path, state
          "--target", "0.4.111", "--daemon-host", "server"],
     )
     assert result.exit_code == 2, result.output
-    assert sorted(rolled_back) == ["laptop", "server"]
+    assert rolled_back == ["laptop"]
 
 
 def test_the_remote_tui_lane_is_recorded_as_unrollable_not_failed(
@@ -2409,6 +2497,12 @@ def test_rollback_host_posts_the_caller_supplied_initiator(monkeypatch):
     monkeypatch.setattr(
         release_cmd, "_wait_agent_back", lambda *a, **k: (True, "0.4.110")
     )
+    # #3364: `_rollback_host` also restarts the daemon-host siblings once the
+    # agent is back — not this test's concern, so stub it as clean.
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda *a, **k: (True, "restarted coord-serve", {}),
+    )
 
     ok, detail = release_cmd._rollback_host(
         _machine(), agent_port=7433, timeout=5.0,
@@ -2438,6 +2532,10 @@ def test_rollback_host_without_an_initiator_omits_the_field(monkeypatch):
     monkeypatch.setattr(
         release_cmd, "_wait_agent_back", lambda *a, **k: (True, "0.4.110")
     )
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda *a, **k: (True, "restarted coord-serve", {}),
+    )
 
     release_cmd._rollback_host(_machine(), agent_port=7433, timeout=5.0)
 
@@ -2445,6 +2543,54 @@ def test_rollback_host_without_an_initiator_omits_the_field(monkeypatch):
     assert "initiator" not in payload
 
 
+def test_rollback_host_restarts_the_siblings_once_the_agent_is_back(monkeypatch):
+    """#3364's actual defect: `POST /rollback` (`agent_app.py`) flips the venv
+    symlink and re-execs only the AGENT. Left there, `coord-serve`/
+    `coord-web` keep running the slot the rollback just deactivated —
+    permanently orphaned on the slot the next `/update` must delete and
+    rebuild, the exact #2121 deadlock this issue reports. `_rollback_host`
+    must therefore restart the siblings too, the same `_restart_sibling_
+    services` call `_roll_python` makes for the forward roll."""
+    monkeypatch.setattr(release_cmd, "_post", lambda *a, **k: (202, {}, ""))
+    monkeypatch.setattr(
+        release_cmd, "_wait_agent_back", lambda *a, **k: (True, "0.4.110")
+    )
+    calls: list[str] = []
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda machine, **k: (calls.append(machine.name), (True, "restarted coord-serve, coord-web", {}))[1],
+    )
+
+    ok, detail = release_cmd._rollback_host(_machine(), agent_port=7433, timeout=5.0)
+
+    assert ok
+    assert calls == ["server"], "the siblings must be restarted on the rolled-back host"
+    assert "restarted coord-serve, coord-web" in detail
+
+
+def test_rollback_host_fails_when_a_sibling_stays_orphaned(monkeypatch):
+    """A sibling that fails to restart onto the reverted slot leaves the
+    fleet in exactly the orphaned-slot shape #3364 describes — that is not a
+    successful rollback, whatever the agent itself did."""
+    monkeypatch.setattr(release_cmd, "_post", lambda *a, **k: (202, {}, ""))
+    monkeypatch.setattr(
+        release_cmd, "_wait_agent_back", lambda *a, **k: (True, "0.4.110")
+    )
+    monkeypatch.setattr(
+        release_cmd, "_restart_sibling_services",
+        lambda *a, **k: (
+            False,
+            "restarted coord-web; FAILED to restart: coord-serve (timed out)",
+            {"coord-serve": "timed out"},
+        ),
+    )
+
+    ok, detail = release_cmd._rollback_host(_machine(), agent_port=7433, timeout=5.0)
+
+    assert not ok, "a sibling orphaned on the reverted slot is not a success"
+    assert "coord-serve" in detail
+
+
 def test_release_propagate_red_gate_rollback_names_itself(
     valid_config_path, state_dir, no_network, monkeypatch
 ):

{code_coordinator-0.5.497 → code_coordinator-0.5.498}/tests/test_release_propagate.py

@@ -1112,6 +1112,68 @@ def test_no_verification_at_all_is_not_a_pass_or_a_failure():
     assert set(verdict.unrollable) == {"tui@precision", "tui@elitebook"}
 
 
+# ──────────────────────────────────────────────────────────────────────────
+# #3364: `--rollback-on-red` must scope to the hosts the red gate actually
+# names, not to every host the run happened to update
+# ──────────────────────────────────────────────────────────────────────────
+
+
+def test_blocking_hosts_names_only_the_host_a_finding_blames():
+    """A per-host crit finding (the common case: `coord release verify`
+    stamping ``lane (host)`` for one offending machine) must scope
+    `--rollback-on-red` to exactly that host — not every host this run
+    updated. #3364's second defect was rolling back four clean hosts because
+    a fifth went red."""
+    verification = {
+        "severity": "crit",
+        "findings": [_finding("crit", "dellserver", "~/.coord-venv (dellserver)")],
+    }
+    verdict = rp.scope_verification(verification, lanes=_run_lanes())
+    assert verdict.red
+    assert verdict.blocking_hosts == frozenset({"dellserver"})
+
+
+def test_blocking_hosts_unions_a_grouped_finding():
+    """The grouped ``--expected`` mismatch names several real lanes at once
+    (see `test_a_grouped_finding_blocks_when_ANY_lane_in_it_is_in_scope`) —
+    `blocking_hosts` must pick every real host out of it, not just the first."""
+    verification = {
+        "severity": "crit",
+        "findings": [
+            _finding("crit", "elitebook, precision",
+                     "~/.coord-cli-venv (elitebook), ~/.coord-venv (precision)"),
+        ],
+    }
+    verdict = rp.scope_verification(verification, lanes=_run_lanes())
+    assert verdict.red
+    assert verdict.blocking_hosts == frozenset({"elitebook", "precision"})
+
+
+def test_blocking_hosts_is_none_for_a_fleet_wide_finding():
+    """A skew/no-data finding `coord release verify` cannot pin to one
+    machine is stamped ``host="(fleet)"`` (see ``release_verify.verify``'s
+    ``"(version skew)"``/``"(expected version)"`` findings). `blocking_hosts`
+    must read that as "cannot attribute" (``None``), not as a literal host
+    named "(fleet)" — a caller must fail toward rolling back every host it
+    touched, never toward a rollback that silently touches nobody."""
+    verification = {
+        "severity": "crit",
+        "findings": [
+            {"severity": "crit", "host": "(fleet)", "lane": "(version skew)",
+             "summary": "2 versions live across the fleet", "detail": ""},
+        ],
+    }
+    verdict = rp.scope_verification(verification, lanes=_run_lanes())
+    assert verdict.red
+    assert verdict.blocking_hosts is None
+
+
+def test_blocking_hosts_is_empty_when_the_gate_is_clean():
+    verdict = rp.scope_verification(None, lanes=_run_lanes())
+    assert not verdict.red
+    assert verdict.blocking_hosts == frozenset()
+
+
 def test_the_gate_verdict_is_rendered_so_a_scoped_gate_is_never_invisible():
     """Scoping the gate is only safe if the scoping is legible afterwards —
     a check that quietly stopped checking is the failure this whole module