code-coordinator 0.5.498__tar.gz → 0.5.499__tar.gz

{code_coordinator-0.5.498 → code_coordinator-0.5.499}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.498
+Version: 0.5.499
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.498 → code_coordinator-0.5.499}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.498
+Version: 0.5.499
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.498 → code_coordinator-0.5.499}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.498",
+  "tag": "0.5.499",
   "distance": 0,
-  "node": "g99081c11bad4947401aaeef6ccd4301a2b450952",
+  "node": "gf300cdf9d6d4ef1871e0b1016a2875ac0572ea95",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-19"

{code_coordinator-0.5.498 → code_coordinator-0.5.499}/coord/commands/release.py

@@ -1746,6 +1746,14 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
                 + ", ".join(spared)
             )
 
+        # #3365: the version each targeted host was on BEFORE this run's
+        # forward roll — i.e. exactly the version a rollback puts it back
+        # on. `before` is the pre-roll verify sweep already paid for in
+        # step 3, so this is a read, not a new probe. Passed through so
+        # `_wait_agent_back` can assert on that version rather than on mere
+        # liveness — see its docstring for why liveness alone lied here.
+        pre_roll_versions = _python_lane_versions(before, targets, record.target_version)
+
         down: list[str] = []
         for host in targets:
             machine = by_name.get(host)
@@ -1756,6 +1764,7 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
                 initiator=cli_initiator(
                     f"coord release propagate -> {machine.name} rollback (red gate)"
                 ),
+                expected_version=pre_roll_versions.get(host),
             )
             record.rolled_back.append(f"{host}: {detail}")
             if not ok:
@@ -2935,34 +2944,98 @@ def _get(url: str, *, timeout: float) -> tuple[int | None, dict]:
     return resp.status_code, (body if isinstance(body, dict) else {})
 
 
-def _wait_agent_back(machine, *, agent_port: int, timeout: float) -> tuple[bool, str]:
-    """Poll ``/health`` until the agent answers again. ``(back, version)``.
+def _wait_agent_back(
+    machine, *, agent_port: int, timeout: float,
+    expected_version: str | None = None, require_change: bool = True,
+) -> tuple[bool, str]:
+    """Poll ``/health`` until the agent answers again ON THE POST-ROLLBACK
+    VERSION — never on the first 200, which may still be the not-yet-re-execed
+    OLD process. ``(back, version)``.
 
     A rollback re-execs the agent process, and #2052 fault 1 is what happens
     when that re-exec does not take: precision's ``coord-agent`` went
     ``inactive (dead)`` at the moment of the rollback and stayed there until
     a human noticed. "The POST was accepted" is therefore not an outcome —
     the outcome is whether the service is serving again.
+
+    #3365: "serving again" turned out not to be a strong enough condition
+    either. Between the accepted ``POST /rollback`` and the re-exec actually
+    taking, the OLD process is still listening and answers 200 with the
+    *pre-rollback* version — a liveness signal indistinguishable from success
+    at the moment it's observed. Reporting the first 200's version asserted
+    the opposite of what happened: four hosts rolled back to 0.5.479 all
+    reported "serving again on v0.5.482", the version they'd just been rolled
+    away from. Liveness alone cannot tell "the new process is up" from "the
+    old process hasn't gone down yet" — both are a 200 — so this now asserts
+    on version identity instead, the same fix #1568 made for the forward
+    ``/update`` wait (:func:`coord.commands.agent_ops._wait_agents_updated`):
+
+    * When the caller knows the destination version — the rollback path in
+      ``coord release propagate`` reads it straight out of the pre-roll
+      ``before`` snapshot, the same version this host is rolling back TO —
+      pass it as *expected_version* and this polls until ``/health`` reports
+      exactly that, never returning True for anything else. The version
+      printed by the caller is then the version verified, by construction.
+    * When the caller has no such snapshot (``coord release rollback``, the
+      manual break-glass command, has no forward-roll record to read a
+      target version from) AND *require_change* is left at its default
+      ``True``, the first 200 seen is recorded as a baseline to RULE OUT
+      rather than proof of success — it is exactly the reading a still-live
+      old process would produce — and this waits for a *different* version
+      to show up before returning True. A process that never re-execs keeps
+      answering the same baseline version for the whole window and
+      correctly times out False, instead of the old bug's instant (wrong)
+      True on the first answer.
+    * *require_change* exists for :func:`_rollback_host`'s retry-after-SSH-
+      escalation call. That escalation only ever runs after a FULL prior
+      window of nothing but non-200s — the ambiguity above (an old process
+      that never went away) cannot apply, because the polling already
+      proved for one whole ``timeout`` that nothing was answering. Reusing
+      the version-change requirement there would demand a signal
+      (a version DIFFERENT from a baseline that was never even set) this
+      call has no way to produce, and wrongly fail a host a hard
+      ``systemctl --user restart`` genuinely revived.
     """
     import time  # noqa: PLC0415
 
     deadline = time.time() + max(timeout, 1.0)
     poll = min(2.0, max(timeout / 10, 0.05))
+    baseline: str | None = None  # only used when expected_version is None
+    last_version = "?"
     while True:
         status, body = _get(f"http://{machine.host}:{agent_port}/health", timeout=3.0)
         if status == 200:
-            return True, str(body.get("version") or "?")
+            version = str(body.get("version") or "?")
+            last_version = version
+            if expected_version is not None:
+                if version == expected_version:
+                    return True, version
+            elif not require_change:
+                return True, version
+            elif baseline is None:
+                baseline = version
+            elif version != baseline:
+                return True, version
         if time.time() >= deadline:
-            return False, "?"
+            return False, last_version
         time.sleep(poll)
 
 
 def _rollback_host(
-    machine, *, agent_port: int, timeout: float = 90.0, initiator: str | None = None
+    machine, *, agent_port: int, timeout: float = 90.0, initiator: str | None = None,
+    expected_version: str | None = None,
 ) -> tuple[bool, str]:
     """POST /rollback — back to the previous blue/green generation (#1241) —
     and then put the service back on its feet.
 
+    #3365: *expected_version*, when the caller has one (the pre-roll version
+    read for this host before the forward roll that's now being undone —
+    see the ``coord release propagate`` rollback loop), is threaded straight
+    through to :func:`_wait_agent_back` so the wait asserts on THAT version
+    rather than on mere liveness. Without it, ``_wait_agent_back`` falls
+    back to its own baseline-diff check — see its docstring — rather than
+    this function inventing a second version-identity scheme.
+
     #2052 fault 1: "a rollback that stops a service and does not restore it
     leaves the fleet worse off than the failed roll did." This used to return
     True the instant the agent answered 202, which is a statement about the
@@ -3020,7 +3093,10 @@ def _rollback_host(
         # service THIS call took down.
         return True, f"{note}; {sib_detail}"
 
-    back, version = _wait_agent_back(machine, agent_port=agent_port, timeout=timeout)
+    back, version = _wait_agent_back(
+        machine, agent_port=agent_port, timeout=timeout,
+        expected_version=expected_version,
+    )
     if back:
         return _back_on_target(f"rolled back; agent is serving again on v{version}")
 
@@ -3030,7 +3106,8 @@ def _rollback_host(
     escalated = _escalate_restart(machine)
     if escalated:
         back, version = _wait_agent_back(
-            machine, agent_port=agent_port, timeout=min(timeout, 60.0)
+            machine, agent_port=agent_port, timeout=min(timeout, 60.0),
+            expected_version=expected_version, require_change=False,
         )
         if back:
             return _back_on_target(

{code_coordinator-0.5.498 → code_coordinator-0.5.499}/tests/test_cli_release_propagate.py

@@ -1424,15 +1424,26 @@ def test_no_verify_stops_before_the_gate(valid_config_path, state_dir, no_networ
 
 
 def test_release_rollback_hits_every_machine(valid_config_path, monkeypatch):
+    """#3365: `/health` answering with the SAME version on every poll — the
+    still-live OLD process, not yet re-execed — must never read as success.
+    Only a version CHANGE (here: the pre-rollback "0.4.110" giving way to
+    the post-rollback "0.4.109") proves the new process actually took over."""
     hit: list[str] = []
+    poll_count: dict[str, int] = {}
 
     def _fake_post(url, payload, *, timeout):
         hit.append(url)
         return 202, {}, ""
 
+    def _fake_get(url, *, timeout):
+        # First poll per host still sees the not-yet-re-execed old process;
+        # the next one sees the new, reverted generation.
+        n = poll_count.get(url, 0)
+        poll_count[url] = n + 1
+        return 200, {"version": "0.4.110" if n == 0 else "0.4.109"}
+
     monkeypatch.setattr(release_cmd, "_post", _fake_post)
-    monkeypatch.setattr(release_cmd, "_get",
-                        lambda url, *, timeout: (200, {"version": "0.4.110"}))
+    monkeypatch.setattr(release_cmd, "_get", _fake_get)
     # #3364: the sibling-restart call this test isn't about — assert it's
     # made in the dedicated `_rollback_host` tests below instead.
     monkeypatch.setattr(
@@ -1448,6 +1459,10 @@ def test_release_rollback_hits_every_machine(valid_config_path, monkeypatch):
     # #2052 fault 1: "rolling back" is a statement about the request. The
     # outcome is whether the service is serving again.
     assert "serving again" in result.output
+    # #3365: the version reported must be the one VERIFIED after the
+    # version changed, never the stale first answer.
+    assert "0.4.109" in result.output
+    assert "v0.4.110" not in result.output
 
 
 def test_a_rollback_that_leaves_the_agent_dead_says_so(valid_config_path, monkeypatch):
@@ -2483,6 +2498,90 @@ def test_roll_units_reload_never_attempted_is_not_a_failure(monkeypatch):
     assert "daemon-reload FAILED" not in detail
 
 
+def test_wait_agent_back_rejects_a_stuck_old_process(monkeypatch):
+    """#3365: the reported bug, reproduced directly. A rollback re-execs the
+    agent, but between the accepted `POST /rollback` and the re-exec taking,
+    the OLD process is still listening and keeps answering 200 with the
+    PRE-rollback version. If `/health` never advances past that version for
+    the whole window, this must time out False — not report the stale
+    version as "serving again", which is what actually happened on 4 hosts
+    across 2 runs in the field."""
+    monkeypatch.setattr(
+        release_cmd, "_get", lambda url, *, timeout: (200, {"version": "0.5.482"})
+    )
+    back, _version = release_cmd._wait_agent_back(_machine(), agent_port=7433, timeout=0.3)
+    assert back is False, "a version that never changes must never read as 'back'"
+
+
+def test_wait_agent_back_with_expected_version_ignores_the_old_answer(monkeypatch):
+    """The `coord release propagate` rollback path knows exactly which
+    version it's rolling back TO (the pre-roll snapshot) — passing it as
+    `expected_version` must reject the stale first answer and wait for the
+    real one, rather than accepting anything that merely answers 200."""
+    calls = {"n": 0}
+
+    def _fake_get(url, *, timeout):
+        calls["n"] += 1
+        # First couple of polls: the old, not-yet-re-execed process.
+        # Only later does the reverted generation actually answer.
+        return 200, {"version": "0.5.482" if calls["n"] <= 2 else "0.5.479"}
+
+    monkeypatch.setattr(release_cmd, "_get", _fake_get)
+    back, version = release_cmd._wait_agent_back(
+        _machine(), agent_port=7433, timeout=5.0, expected_version="0.5.479",
+    )
+    assert back is True
+    assert version == "0.5.479"
+
+
+def test_wait_agent_back_with_expected_version_times_out_if_never_reached(monkeypatch):
+    """If the fleet never actually reaches the expected version — e.g. the
+    rollback itself failed silently — `_wait_agent_back` must not be fooled
+    by a live process answering on the WRONG (old) version."""
+    monkeypatch.setattr(
+        release_cmd, "_get", lambda url, *, timeout: (200, {"version": "0.5.482"})
+    )
+    back, _version = release_cmd._wait_agent_back(
+        _machine(), agent_port=7433, timeout=0.3, expected_version="0.5.479",
+    )
+    assert back is False
+
+
+def test_wait_agent_back_accepts_a_version_change_without_an_expected_target(monkeypatch):
+    """`coord release rollback`, the manual break-glass command, has no
+    forward-roll record to read a destination version from. Lacking one,
+    a version CHANGE away from the first-seen (baseline) reading is itself
+    proof the new process took over — the same "went away, came back"
+    signal, expressed through version identity rather than a connectivity
+    gap that a same-PID `execv` restart may never actually produce."""
+    calls = {"n": 0}
+
+    def _fake_get(url, *, timeout):
+        calls["n"] += 1
+        return 200, {"version": "0.5.482" if calls["n"] == 1 else "0.5.479"}
+
+    monkeypatch.setattr(release_cmd, "_get", _fake_get)
+    back, version = release_cmd._wait_agent_back(_machine(), agent_port=7433, timeout=5.0)
+    assert back is True
+    assert version == "0.5.479"
+
+
+def test_wait_agent_back_require_change_false_trusts_the_first_answer(monkeypatch):
+    """`_rollback_host`'s retry-after-SSH-escalation call passes
+    `require_change=False`: that escalation only ever runs after a FULL
+    prior window of nothing but non-200s, so the ambiguity a still-live old
+    process would create cannot apply — a hard `systemctl --user restart`
+    genuinely revived a dead service, and the first answer is trustworthy."""
+    monkeypatch.setattr(
+        release_cmd, "_get", lambda url, *, timeout: (200, {"version": "0.5.479"})
+    )
+    back, version = release_cmd._wait_agent_back(
+        _machine(), agent_port=7433, timeout=0.3, require_change=False,
+    )
+    assert back is True
+    assert version == "0.5.479"
+
+
 def test_rollback_host_posts_the_caller_supplied_initiator(monkeypatch):
     """#2121: `_rollback_host` is the sibling of `_roll_python` above — same
     fleet-automation shape, same obligation to name itself on the target
@@ -2604,7 +2703,7 @@ def test_release_propagate_red_gate_rollback_names_itself(
     captured: list[dict] = []
     monkeypatch.setattr(
         release_cmd, "_rollback_host",
-        lambda machine, *, agent_port, timeout, initiator=None: (
+        lambda machine, *, agent_port, timeout, initiator=None, expected_version=None: (
             captured.append({"machine": machine.name, "initiator": initiator}),
             (True, "back up"),
         )[1],
@@ -2621,6 +2720,41 @@ def test_release_propagate_red_gate_rollback_names_itself(
         assert row["initiator"].startswith("coord release propagate -> ")
 
 
+def test_release_propagate_red_gate_rollback_passes_the_pre_roll_version(
+    valid_config_path, state_dir, no_network, monkeypatch
+):
+    """#3365: the red-gate rollback loop knows exactly which version each
+    targeted host was on before this run's forward roll — the pre-roll
+    `before` sweep it already paid for in step 3 — and must hand that to
+    `_rollback_host` as `expected_version` so the post-rollback wait asserts
+    on the real destination rather than on mere liveness."""
+    _stub_lanes(monkeypatch)
+    _stub_verify(monkeypatch, versions={"laptop": ["0.4.110"], "server": ["0.4.110"]},
+                 severity="crit")
+    captured: list[dict] = []
+    monkeypatch.setattr(
+        release_cmd, "_rollback_host",
+        lambda machine, *, agent_port, timeout, initiator=None, expected_version=None: (
+            captured.append(
+                {"machine": machine.name, "expected_version": expected_version}
+            ),
+            (True, "back up"),
+        )[1],
+    )
+    result = CliRunner().invoke(
+        main,
+        ["release", "propagate", "--config", str(valid_config_path),
+         "--target", "0.4.111"],
+    )
+    assert result.exit_code == 2, result.output
+    assert captured, "the red gate must have rolled back at least one host"
+    for row in captured:
+        assert row["expected_version"] == "0.4.110", (
+            "must pass the pre-roll version, not the (unreachable) target "
+            "the roll never actually attained"
+        )
+
+
 def test_release_rollback_cli_names_itself(valid_config_path, monkeypatch):
     """`coord release rollback` is the other live call site of
     `_rollback_host` — same obligation."""