code-coordinator 0.5.136__tar.gz → 0.5.140__tar.gz

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.136
+Version: 0.5.140
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.136
+Version: 0.5.140
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/code_coordinator.egg-info/SOURCES.txt

@@ -112,6 +112,7 @@ coord/prereqs.py
 coord/progress.py
 coord/queue_diagnose.py
 coord/reconcile.py
+coord/reconcile_tick.py
 coord/refine_chat.py
 coord/release_cordon.py
 coord/release_propagate.py
@@ -431,6 +432,7 @@ tests/test_agent_branch_capture.py
 tests/test_agent_config_free_1712.py
 tests/test_agent_deploy_units_endpoint.py
 tests/test_agent_dirty_worktree.py
+tests/test_agent_drive_queue_reconcile_2373.py
 tests/test_agent_graph_self_heal.py
 tests/test_agent_health_h3.py
 tests/test_agent_idle_restart.py
@@ -682,6 +684,7 @@ tests/test_reconcile_branch.py
 tests/test_reconcile_completions.py
 tests/test_reconcile_merges.py
 tests/test_reconcile_no_agent_record.py
+tests/test_reconcile_tick.py
 tests/test_reconcile_usage_limit_done.py
 tests/test_refine_chat.py
 tests/test_release_cordon_2101.py
@@ -788,6 +791,7 @@ tests/acceptance/ms-51/manifest.yml
 tests/acceptance/ms-51/tsconfig.json
 tests/acceptance/ms-51/mocks/home-active.html
 tests/acceptance/ms-65/board_tabs_2282.rs
+tests/acceptance/ms-65/board_tabs_close_2283.rs
 tests/acceptance/ms-65/contract.md
 tests/acceptance/ms-65/manifest.yml
 tests/acceptance/ms-65/mocks/board-baseline-no-tabs.screen

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/code_coordinator.egg-info/scm_file_list.json

@@ -293,6 +293,7 @@
     "coord/providers/opencode.py",
     "coord/queue_diagnose.py",
     "coord/reconcile.py",
+    "coord/reconcile_tick.py",
     "coord/refine_chat.py",
     "coord/release_cordon.py",
     "coord/release_propagate.py",
@@ -431,6 +432,7 @@
     "tests/acceptance/ms-51/mocks/home-active.html",
     "tests/acceptance/ms-51/tsconfig.json",
     "tests/acceptance/ms-65/board_tabs_2282.rs",
+    "tests/acceptance/ms-65/board_tabs_close_2283.rs",
     "tests/acceptance/ms-65/contract.md",
     "tests/acceptance/ms-65/manifest.yml",
     "tests/acceptance/ms-65/mocks/board-baseline-no-tabs.screen",
@@ -473,6 +475,7 @@
     "tests/test_agent_config_free_1712.py",
     "tests/test_agent_deploy_units_endpoint.py",
     "tests/test_agent_dirty_worktree.py",
+    "tests/test_agent_drive_queue_reconcile_2373.py",
     "tests/test_agent_graph_self_heal.py",
     "tests/test_agent_health_h3.py",
     "tests/test_agent_idle_restart.py",
@@ -724,6 +727,7 @@
     "tests/test_reconcile_completions.py",
     "tests/test_reconcile_merges.py",
     "tests/test_reconcile_no_agent_record.py",
+    "tests/test_reconcile_tick.py",
     "tests/test_reconcile_usage_limit_done.py",
     "tests/test_refine_chat.py",
     "tests/test_release_cordon_2101.py",

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.136",
+  "tag": "0.5.140",
   "distance": 0,
-  "node": "g44c61fdca127a9fd4f56bc83b75d60e32e623366",
+  "node": "g99644043cc6f5e860c84b649e4a026a2d231dadb",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-18"

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/coord/agent.py

@@ -6102,6 +6102,48 @@ class AgentServer:
             self._local_health_cache = None
         return {**result.to_dict(), "repo": repo_name}
 
+    def reconcile_drive_queue(self, *, timeout: float = 120.0) -> dict:
+        """``coord drive-queue tick --reconcile-only`` — run on THIS machine
+        (#2373 — ``coord release propagate``'s drain-deadline escalation
+        fans this out to a host it has cordoned but that will not drain, via
+        `_reconcile_launch_host` in `coord/commands/release.py`).
+
+        Only the machine that launched a `running` drive-queue entry can
+        resolve it (the #1870 cross-host guard in
+        `coord.drive_queue._reconcile_running` is keyed on a LOCAL tmux
+        read); this method is that resolution, reachable over this agent's
+        HTTP API instead of a human's SSH session. Full incident writeup
+        (claude-coordinator#2360) and rationale: `_reconcile_launch_host`'s
+        docstring and docs/AGENT_OPERATIONS.md's #2373 section.
+
+        Delegates to :func:`coord.reconcile_tick.run_reconcile_tick` — the
+        same shared implementation `coord release propagate`'s own
+        `_run_reconcile_tick` (`coord/commands/release.py`) calls, so there
+        is exactly one implementation of "what a reconcile-only tick does",
+        not two that can drift apart.
+
+        Returns ``{"ok": bool, "detail": str}``. ``ok=False`` with a
+        ``no local coordinator.yml`` detail when this agent has no on-disk
+        config to pass as ``--config`` — thin-client/config-free mode
+        (docs/EPHEMERAL_WORKERS.md) has nothing for a queue tick to resolve
+        against here. Never raises: the caller (`/drive-queue-reconcile`) is
+        a best-effort self-heal step inside a larger escalation, not a gate
+        that may take propagation down with it.
+        """
+        from coord.reconcile_tick import run_reconcile_tick  # noqa: PLC0415
+
+        config_path = getattr(self._health_config, "path", None)
+        if not config_path:
+            return {
+                "ok": False,
+                "detail": (
+                    "no local coordinator.yml on this agent — cannot resolve "
+                    "a --config path for the tick"
+                ),
+            }
+        ok, detail = run_reconcile_tick(config_path, timeout=timeout, detail_limit=2000)
+        return {"ok": ok, "detail": detail}
+
     def list_repos(self) -> dict[str, dict]:
         """Return local HEAD / branch / dirty flag for each configured repo.
 

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/coord/agent_app.py

@@ -1020,6 +1020,42 @@ def _openapi_spec() -> dict:
                 },
             }
         },
+        "/drive-queue-reconcile": {
+            "post": {
+                "summary": (
+                    "Run `coord drive-queue tick --reconcile-only` on this "
+                    "machine (#2373)"
+                ),
+                "description": (
+                    "The per-machine self-heal `coord release propagate`'s "
+                    "drain-deadline escalation calls before it escalates "
+                    "loudly: only the host that actually launched a "
+                    "`running` drive-queue entry can resolve the #1870 "
+                    "cross-host liveness guard for it. Fleet-queue-wide, "
+                    "not scoped to a repo."
+                ),
+                "requestBody": {
+                    "required": False,
+                    "content": {
+                        "application/json": {
+                            "schema": {
+                                "type": "object",
+                                "properties": {
+                                    "timeout": {
+                                        "type": "number",
+                                        "description": (
+                                            "seconds for the tick subprocess; "
+                                            "defaults to 120."
+                                        ),
+                                    },
+                                },
+                            }
+                        }
+                    },
+                },
+                "responses": {"200": {"description": "OK (including a failed tick)"}},
+            }
+        },
         "/artifact/{repo}/{branch}": {
             "get": {
                 "summary": "Manifest of stashed build artifacts for a (repo, branch) pair",
@@ -1948,6 +1984,36 @@ def build_app(
         result = await asyncio.to_thread(server.fix_graph, repo, timeout=timeout)
         return JSONResponse(result)
 
+    async def drive_queue_reconcile(request: Request) -> JSONResponse:
+        """Run ``coord drive-queue tick --reconcile-only`` on THIS machine
+        (#2373). ``coord release propagate``'s drain-deadline escalation
+        POSTs here — before it escalates loudly — on whichever host it has
+        cordoned but that will not drain, so the one machine that can
+        actually resolve the #1870 cross-host liveness ambiguity for an
+        entry it launched gets asked to, automatically, instead of a human
+        needing to SSH in and run this by hand (the live 2026-08-18 incident
+        this closes).
+
+        Body: optional ``{"timeout": 120}``. No required fields, unlike
+        ``/graph-fix`` — a reconcile-only tick is fleet-queue-wide, not
+        scoped to one repo.
+
+        Runs off the event loop: the subprocess holds the drive-queue file
+        lock for the length of one tick, which must never stall this
+        agent's ``/health`` for the rest of the fleet.
+        """
+        body: dict = {}
+        try:
+            body = await request.json()
+        except Exception:
+            pass
+        try:
+            timeout = float(body.get("timeout", 120.0))
+        except (TypeError, ValueError):
+            timeout = 120.0
+        result = await asyncio.to_thread(server.reconcile_drive_queue, timeout=timeout)
+        return JSONResponse(result)
+
     routes = [
         Route("/health", health, methods=["GET"]),
         Route("/status", status, methods=["GET"]),
@@ -1973,6 +2039,10 @@ def build_app(
         # missing graph and set core.hooksPath, on the machines that
         # actually run workers rather than only on the operator's laptop.
         Route("/graph-fix", graph_fix, methods=["POST"]),
+        # #2373: `coord release propagate`'s drain-deadline escalation's
+        # per-machine self-heal — resolve a `running` drive-queue entry this
+        # host launched before the escalation fires loudly.
+        Route("/drive-queue-reconcile", drive_queue_reconcile, methods=["POST"]),
         # #305: artifact stash manifest (GET /artifact/<repo>/<branch>)
         Route("/artifact/{repo}/{branch}", artifact_manifest, methods=["GET"]),
         # #207: CPU + memory snapshot for TUI sparklines

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/coord/commands/release.py

@@ -503,6 +503,77 @@ def _post(url: str, payload: dict, *, timeout: float) -> tuple[int | None, dict,
     return resp.status_code, (body if isinstance(body, dict) else {}), ""
 
 
+#: #2373: default timeout for the remote-reconcile POST below. Generous, but
+#: still small next to `DEFAULT_DRAIN_DEADLINE_SECONDS` (90 minutes), so a
+#: wedged/unreachable agent never turns "ask before escalating" into a
+#: second, longer wait before the loud message an operator is actually
+#: watching for. The POST body forwards a *shorter* remote-subprocess
+#: timeout (see `_RECONCILE_TIMEOUT_MARGIN_SECONDS` below) so the remote
+#: agent's own tick (`AgentServer.reconcile_drive_queue`, otherwise 120s by
+#: default) can never outlive this wait — without that, a tick legitimately
+#: taking longer than this timeout would surface here as a misleading
+#: "unreachable" even though it was still running and would have resolved
+#: correctly on its own.
+DEFAULT_RECONCILE_TIMEOUT_SECONDS = 30.0
+
+#: Margin subtracted from *timeout* before it is forwarded as the remote
+#: subprocess's own timeout (see `_reconcile_launch_host`) — leaves the
+#: agent's HTTP handler room to marshal and return its response before
+#: this side's own `httpx` deadline fires.
+_RECONCILE_TIMEOUT_MARGIN_SECONDS = 5.0
+
+
+def _reconcile_launch_host(
+    machine_host: str,
+    *,
+    agent_port: int,
+    timeout: float = DEFAULT_RECONCILE_TIMEOUT_SECONDS,
+    post=None,
+) -> tuple[bool | None, str]:
+    """Ask *machine_host*'s own agent to run a local reconcile-only tick
+    (#2373), over the same ``POST /<verb>`` HTTP pattern ``/graph-fix``
+    already uses for a per-machine self-heal fanned out from elsewhere.
+
+    Why this has to be a REMOTE call rather than something run locally by
+    whichever host happens to be executing `coord release propagate`: the
+    #1870 cross-host guard (`coord.drive_queue._reconcile_running`) is keyed
+    on a LOCAL tmux read, so only the machine that actually launched a
+    `running` drive-queue entry can ever resolve whether it is still alive.
+    A non-daemon launch host has no periodic tick of its own — confirmed
+    live 2026-08-18 (claude-coordinator#2360): elitebook ran only
+    `coord-agent.service`, dellserver's `coord-drive-queue.timer` correctly
+    refused to declare a foreign-host entry dead every time it ticked, and
+    the ambiguity sat wedging the fleet's release drain for ~17h until a
+    human happened to SSH in and run this exact command by hand. Routing it
+    through the agent HTTP API that already runs on every machine capable of
+    launching a drive closes that gap without a new timer unit anywhere.
+
+    Returns ``(ok, detail)``. ``ok`` is ``None`` when the call could not even
+    be attempted (never treated as a *failure* by the caller — see
+    :func:`_escalate_drain`), ``False`` on an unreachable agent or a non-200
+    response, ``True`` when the agent ran the tick (its own exit code, not
+    "and it resolved the entry" — a genuinely still-busy host is expected to
+    report ``ok=True`` with nothing to reconcile).
+
+    The POST body forwards a remote subprocess timeout derived from
+    *timeout* itself (minus `_RECONCILE_TIMEOUT_MARGIN_SECONDS`) rather than
+    leaving the remote agent to fall back to its own 120s default
+    (`AgentServer.reconcile_drive_queue`) — otherwise a tick that legitimately
+    takes longer than this call's own HTTP wait reports here as
+    `ok=False, detail="unreachable: ..."` even though it is still running and
+    will resolve correctly on its own, just not attributed to this call.
+    """
+    poster = post or _post
+    url = f"http://{machine_host}:{agent_port}/drive-queue-reconcile"
+    remote_timeout = max(1.0, timeout - _RECONCILE_TIMEOUT_MARGIN_SECONDS)
+    status, body, err = poster(url, {"timeout": remote_timeout}, timeout=timeout)
+    if err:
+        return False, f"unreachable: {err}"
+    if status != 200:
+        return False, f"HTTP {status}: {body.get('detail') or body}"
+    return bool(body.get("ok")), str(body.get("detail") or "")
+
+
 def _lane_versions_by_host(report) -> dict[str, list[str | None]]:
     out: dict[str, list[str | None]] = {}
     for lane in report.lanes:
@@ -791,6 +862,11 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
         max_deferrals=cordon_max_deferrals,
         release_cooldown=cordon_cooldown,
         daemon_host=daemon_name,
+        # #2373: lets a blown drain deadline ask the escalated host's own
+        # agent to resolve the #1870 cross-host liveness ambiguity locally
+        # before the loud message goes out — see `_reconcile_launch_host`.
+        machines=getattr(config, "machines", ()) or (),
+        agent_port=AGENT_PORT,
     )
 
     if fully_busy and not force:
@@ -1162,6 +1238,9 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
     max_deferrals: int | None = None,
     release_cooldown: float | None = None,
     daemon_host: str | None = None,
+    machines=None,
+    agent_port: int | None = None,
+    reconcile_timeout: float | None = None,
 ) -> dict:
     """Plan and apply this run's cordons. Returns the journal fragment.
 
@@ -1185,6 +1264,13 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
     pre-#2240 behaviour) rather than to a spurious release: dropping the
     fleet's cordon because a file could not be read is the wrong direction to
     fail in.
+
+    #2373: *machines*/*agent_port* let each :class:`~coord.release_cordon.
+    DrainEscalation` ask the escalated host's OWN agent to run a local
+    reconcile-only tick before the loud message goes out — see
+    :func:`_reconcile_launch_host`. ``machines=None`` (a caller with no fleet
+    topology, e.g. a unit test exercising `plan_cordons` output directly)
+    skips the remote call entirely and escalates exactly as before #2373.
     """
     import time  # noqa: PLC0415
 
@@ -1304,9 +1390,39 @@ def _apply_cordons(  # noqa: PLR0912 — one linear apply-the-plan pass
                 outcome.errors.append(f"release cordon {name}: {exc}")
                 click.echo(f"  ✗ release cordon {name}: {exc}", err=True)
 
+    machines_by_name = {m.name: m for m in (machines or ())}
+    # #2373: each escalation below makes one synchronous `_reconcile_launch_host`
+    # HTTP call, serially, each blocking up to ~`DEFAULT_RECONCILE_TIMEOUT_SECONDS`.
+    # With multiple simultaneously-overdue hosts this adds up to N * timeout of
+    # latency inside one `release propagate` invocation. Fine today — escalations
+    # are rare and few at once — but if that ever stops being true, parallelize
+    # this loop (e.g. a thread pool over `plan.escalations`) rather than letting
+    # a single propagate call silently grow with fleet size.
     for escalation in plan.escalations:
-        _escalate_drain(escalation)
-        outcome.escalated.append(escalation.to_dict())
+        # #2373: before the loud message, give the escalated host's OWN
+        # agent one chance to resolve the ambiguity that is actually holding
+        # it — see `_reconcile_launch_host`'s docstring. `machine` missing
+        # from the fleet map, or no `agent_port` supplied (a caller with no
+        # HTTP topology), skips the call rather than guessing an address.
+        reconcile_ok: bool | None = None
+        reconcile_detail = ""
+        target = machines_by_name.get(escalation.machine)
+        if target is not None and agent_port:
+            reconcile_ok, reconcile_detail = _reconcile_launch_host(
+                target.host,
+                agent_port=agent_port,
+                timeout=(
+                    DEFAULT_RECONCILE_TIMEOUT_SECONDS
+                    if reconcile_timeout is None
+                    else reconcile_timeout
+                ),
+            )
+        _escalate_drain(escalation, reconcile_ok=reconcile_ok, reconcile_detail=reconcile_detail)
+        outcome.escalated.append({
+            **escalation.to_dict(),
+            "reconcile_ok": reconcile_ok,
+            "reconcile_detail": reconcile_detail,
+        })
 
     return outcome.to_dict()
 
@@ -1321,7 +1437,9 @@ DRAIN_ALERT_ISSUE = 0
 DRAIN_ALERT_STAGE = "release-cordon"
 
 
-def _escalate_drain(escalation) -> None:
+def _escalate_drain(
+    escalation, *, reconcile_ok: bool | None = None, reconcile_detail: str = ""
+) -> None:
     """Surface a host that will not drain — loudly, in three places.
 
     stderr (the timer's journal), the escalation table (the TUI and
@@ -1329,21 +1447,43 @@ def _escalate_drain(escalation) -> None:
     criterion 4 is explicitly about the surfaced MESSAGE rather than an
     internal state change, because a silent forever-wait is the failure this
     whole mechanism replaces.
+
+    #2373: *reconcile_ok*/*reconcile_detail* are the outcome of the remote
+    reconcile-only tick `_apply_cordons` asked the escalated host's own
+    agent to run just before calling this — ``None`` when that call was
+    never attempted (no fleet topology / agent port available), so the
+    message stays byte-identical to before #2373 in that case. When it WAS
+    attempted, the outcome is folded into the same message rather than a
+    second alert, so an operator reading one escalation sees both "this is
+    overdue" and "here is what the self-heal attempt already found" without
+    correlating two records.
     """
-    click.echo(f"  ! {escalation.message}", err=True)
+    reconcile_note = ""
+    if reconcile_ok is not None:
+        verb = "ok" if reconcile_ok else "failed"
+        detail = f" ({reconcile_detail})" if reconcile_detail else ""
+        reconcile_note = (
+            f" — asked {escalation.machine}'s own agent to run a local "
+            f"reconcile-only tick first (#2373): {verb}{detail}"
+        )
+    message = f"{escalation.message}{reconcile_note}"
+    click.echo(f"  ! {message}", err=True)
     try:
         from coord.state import record_drive_escalation  # noqa: PLC0415
 
+        gate_readings = (
+            f"machine={escalation.machine} | "
+            f"waited={escalation.waited_seconds:.0f}s | "
+            f"deadline={escalation.deadline_seconds:.0f}s"
+        )
+        if reconcile_ok is not None:
+            gate_readings += f" | remote_reconcile={'ok' if reconcile_ok else 'failed'}"
         record_drive_escalation(
             DRAIN_ALERT_REPO,
             DRAIN_ALERT_ISSUE,
             stage=DRAIN_ALERT_STAGE,
-            reason=escalation.message,
-            gate_readings=(
-                f"machine={escalation.machine} | "
-                f"waited={escalation.waited_seconds:.0f}s | "
-                f"deadline={escalation.deadline_seconds:.0f}s"
-            ),
+            reason=message,
+            gate_readings=gate_readings,
             proposed_command=escalation.command,
         )
     except Exception as exc:  # noqa: BLE001 — the stderr line above is the
@@ -2181,24 +2321,19 @@ def _run_reconcile_tick(config_path: Path, *, runner=None) -> tuple[bool, str]:
     *after* the timer stopped would stay stuck ``running`` for the rest of
     the drain no matter how generous the deadline is — the exact deadlock
     #2110 fixed reconciliation to no longer require the timer for.
+
+    Thin wrapper around :func:`coord.reconcile_tick.run_reconcile_tick`, the
+    #2373-extracted shared implementation also used by
+    `AgentServer.reconcile_drive_queue` (`coord/agent.py`) — see that
+    module's docstring for why there must be exactly one implementation of
+    "what a reconcile-only tick does". A failure here is not fatal on its
+    own: the quiescence check right after this call is the real signal a
+    stuck reconcile eventually clears on a later poll, not this one call's
+    job to guarantee.
     """
-    import subprocess  # noqa: PLC0415
+    from coord.reconcile_tick import run_reconcile_tick  # noqa: PLC0415
 
-    run = runner or subprocess.run
-    try:
-        proc = run(
-            [sys.executable, "-m", "coord.cli", "drive-queue", "tick",
-             "--reconcile-only", "--config", str(config_path)],
-            capture_output=True, text=True, timeout=120,
-        )
-    except Exception as exc:  # noqa: BLE001 — best effort; the quiescence
-        # check right after this call is the real signal a stuck reconcile
-        # eventually clears on a later poll, not this one call's job to
-        # guarantee.
-        return False, f"{type(exc).__name__}: {exc}"
-    ok = getattr(proc, "returncode", 1) == 0
-    detail = (getattr(proc, "stderr", "") or getattr(proc, "stdout", "") or "").strip()
-    return ok, detail[:200]
+    return run_reconcile_tick(config_path, timeout=120, detail_limit=200, runner=runner)
 
 
 def _drain(

{code_coordinator-0.5.136 → code_coordinator-0.5.140}/coord/drive.py

@@ -108,7 +108,11 @@ from coord.interactive import (
     tmux_session_alive,
 )
 from coord.dead_end import DeadEnd, detect_dead_end
-from coord.failure_class import classify_failure, plan_usage_limit_resume
+from coord.failure_class import (
+    classify_failure,
+    environmental_backoff_secs,
+    plan_usage_limit_resume,
+)
 from coord.models import DELIVERABLE_ANALYSIS_LABEL, POLICY_REFUSAL_MARKER
 from coord.usage_limits import PlanLimits, evaluate_usage_gate, get_plan_limits
 # Lost in the #1584-onto-#1590 rebase: _decide_review() calls this, but the
@@ -201,6 +205,19 @@ class DriveError(Exception):
 # untruncated bytes for local, on-host debugging.
 _CAPTURED_OUTPUT_LIMIT = 4000
 
+# #2360: attempt budget for a WORK-stage failure `coord.failure_class.
+# classify_failure` calls environmental (a Claude API 5xx, a dropped
+# connection, an overload — the module's own allow-listed signals), as
+# opposed to `DriveOptions.max_work_retries` (default 1: one retry, two
+# attempts total), which stays the tight budget for a genuine code defect.
+# #2335: a work assignment died on an auth hiccup, its one flat retry died
+# too, and the drive-queue entry sat `blocked` for ~19h — confirmed
+# transient after the fact (the same machine ran dozens of clean sessions in
+# that window). `max()` against `opts.max_work_retries` at the call site
+# means a run configured with a bigger flat budget than this default is
+# never *tightened* for the environmental case.
+_ENVIRONMENTAL_WORK_RETRY_BUDGET = 5
+
 
 def _bounded_tail(text: str, limit: int = _CAPTURED_OUTPUT_LIMIT) -> str:
     """*text*, or its last *limit* characters if longer — the tail, because
@@ -264,6 +281,15 @@ class DriveCounters:
     """Bounds on every retry loop.  Unbounded merge retries was a real bug."""
 
     work_retries: int = 0
+    # #2360: one-shot latch for the environmental WORK-stage backoff — the
+    # 1-based attempt number (`work_retries + 1`) `_decide_work` has already
+    # returned a backoff WAIT for. `-1` means "no backoff pending". Mirrors
+    # `review_fix_dispatched_for`'s shape below: without it, a board that
+    # hasn't changed yet (nothing was dispatched during the wait) would make
+    # `decide()` re-issue the SAME backoff WAIT forever instead of, once the
+    # sleep it already returned has elapsed, falling through to the actual
+    # `coord retry` for that attempt.
+    work_environmental_backoff_attempt: int = -1
     # ONE budget for BOTH fix arms (#1692): a failed test and a
     # request-changes review are two shapes of the same "the work needs
     # another round" loop, and a drive that spends three rounds bouncing
@@ -1507,13 +1533,25 @@ def decide(
 
     # ---- work failed: bounded retry ----------------------------------------
     if state.work_status == "failed":
-        if counters.work_retries >= opts.max_work_retries:
-            # #1590 part 6: name the actual cause. "failed 3 retries in: <prose>"
-            # sent the morning triage looking at the work even when the provider
-            # was the problem; the class is now stated up front.
-            classification = classify_failure(
-                failure_reason=state.work_failure_reason or None
-            )
+        # #1590 part 6: name the actual cause. "failed 3 retries in: <prose>"
+        # sent the morning triage looking at the work even when the provider
+        # was the problem; the class is now stated up front.
+        classification = classify_failure(
+            failure_reason=state.work_failure_reason or None
+        )
+        # #2360: an environmental failure (already established NOT a usage
+        # limit — that branch returned above) gets a wider budget than a
+        # genuine code defect, reusing the same classifier the usage-limit
+        # check just above already trusts. A NON-environmental failure keeps
+        # today's flat `opts.max_work_retries` budget completely unchanged —
+        # this is not a general increase to retry counts, only a widening
+        # scoped to `classification.is_environmental`.
+        budget = (
+            max(opts.max_work_retries, _ENVIRONMENTAL_WORK_RETRY_BUDGET)
+            if classification.is_environmental
+            else opts.max_work_retries
+        )
+        if counters.work_retries >= budget:
             return _die(
                 f"work {state.work_aid} failed {counters.work_retries} retr(ies) in: "
                 f"{state.work_failure_reason or 'no reason recorded'}\n"
@@ -1521,12 +1559,35 @@ def decide(
                 f"   inspect: coord log {state.work_aid} --machine "
                 f"{state.work_machine or machine}"
             )
-        counters.work_retries += 1
+        attempt = counters.work_retries + 1
+        if (
+            classification.is_environmental
+            and counters.work_environmental_backoff_attempt != attempt
+        ):
+            # Back off ONCE per attempt number before spending it (the same
+            # `environmental_backoff_secs` exponential curve #1590/#2275
+            # proved for the Test-stage worker-death path) — the latch means
+            # the NEXT poll, once this sleep has actually elapsed, falls
+            # through to the real `coord retry` below instead of returning
+            # the same backoff WAIT forever against a board that has not
+            # changed (nothing was dispatched during the wait).
+            counters.work_environmental_backoff_attempt = attempt
+            wait = environmental_backoff_secs(attempt)
+            return Action(
+                kind=WAIT,
+                label=(
+                    f"WORK: {state.work_aid} failed environmentally — "
+                    f"backing off {int(wait)}s before retry {attempt}/{budget}"
+                ),
+                sleep_after=wait,
+                warnings=(f"{classification.reason} (#2360)",),
+            )
+        counters.work_retries = attempt
         return Action(
             kind=RUN,
             label=(
                 f"WORK: failed → coord retry {state.work_aid} "
-                f"(attempt {counters.work_retries}/{opts.max_work_retries})"
+                f"(attempt {counters.work_retries}/{budget})"
             ),
             command=("retry", state.work_aid),
             error_message=f"coord retry failed for {state.work_aid}",