code-coordinator 0.5.238__tar.gz → 0.5.240__tar.gz

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.238
+Version: 0.5.240
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.238
+Version: 0.5.240
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/code_coordinator.egg-info/SOURCES.txt

@@ -496,6 +496,7 @@ tests/test_dist_name.py
 tests/test_done_housekeeping.py
 tests/test_drive.py
 tests/test_drive_queue.py
+tests/test_drive_queue_roll_pending.py
 tests/test_drive_state.py
 tests/test_drive_tmux.py
 tests/test_epic_up_capability_detection.py

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/code_coordinator.egg-info/scm_file_list.json

@@ -568,6 +568,7 @@
     "tests/test_done_housekeeping.py",
     "tests/test_drive.py",
     "tests/test_drive_queue.py",
+    "tests/test_drive_queue_roll_pending.py",
     "tests/test_drive_state.py",
     "tests/test_drive_tmux.py",
     "tests/test_epic_up_capability_detection.py",

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.238",
+  "tag": "0.5.240",
   "distance": 0,
-  "node": "g8d13a39e25627c1a4242ac04e7c279023d414a99",
+  "node": "gbfdc750bff3b8352eace6d4ce6ab5dde9b39771a",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-23"

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/coord/audit.py

@@ -22,16 +22,24 @@ row is best-effort observability on top of it.
 
 from __future__ import annotations
 
+import atexit
 import json
 import logging
+import sqlite3
+import threading
 import time
 from typing import Any
 
-from coord.db import get_connection
+from coord.db import get_connection, is_lock_contention_error, retry_on_locked
 
 _log = logging.getLogger(__name__)
 
-__all__ = ["record_audit", "query_audit_log"]
+__all__ = [
+    "record_audit",
+    "query_audit_log",
+    "audit_lock_contention_losses",
+    "flush_lock_contention_summary",
+]
 
 # Valid values are documented in the issue but not enforced here — callers
 # are all internal (coord.state / coord.issue_store), and rejecting an
@@ -94,7 +102,19 @@ def record_audit(
             details=details,
         )
     except Exception as exc:  # noqa: BLE001 — audit logging must never break the caller
-        _log.warning("record_audit: best-effort write failed: %s", exc)
+        if is_lock_contention_error(exc):
+            # #2597: `_record_audit_unsafe` already retried this write
+            # several times (`retry_on_locked`) before this was reached —
+            # sustained contention, not a momentary collision. Still
+            # best-effort (never raises into the caller), but a WARNING per
+            # occurrence is exactly the flood #2597 measured (1,804/day on
+            # dellserver, one indistinguishable line each) — count it
+            # instead and let `_flush_lock_contention_summary` report the
+            # aggregate once per run.
+            _record_lock_contention_loss()
+            _log.debug("record_audit: write lost to lock contention: %s", exc)
+        else:
+            _log.warning("record_audit: best-effort write failed: %s", exc)
 
 
 def _record_audit_unsafe(
@@ -115,27 +135,139 @@ def _record_audit_unsafe(
     try/except wrapper is the ONLY thing between this and the caller —
     keeps the swallow-and-log behavior in one obvious place."""
     conn = get_connection()
-    conn.execute(
-        """INSERT INTO audit_log (
-            ts, tier, category, event_type, actor,
-            repo, issue, assignment_id, machine, summary, details_json
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-        (
-            ts if ts is not None else time.time(),
-            tier,
-            category,
-            event_type,
-            actor,
-            repo,
-            issue,
-            assignment_id,
-            machine,
-            summary,
-            json.dumps(details) if details is not None else None,
-        ),
-    )
-    conn.commit()
-    _maybe_trim(conn)
+
+    def _write() -> None:
+        conn.execute(
+            """INSERT INTO audit_log (
+                ts, tier, category, event_type, actor,
+                repo, issue, assignment_id, machine, summary, details_json
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                ts if ts is not None else time.time(),
+                tier,
+                category,
+                event_type,
+                actor,
+                repo,
+                issue,
+                assignment_id,
+                machine,
+                summary,
+                json.dumps(details) if details is not None else None,
+            ),
+        )
+        conn.commit()
+
+    # #2597: this table is the coordinator's primary diagnostic surface —
+    # "coord audit" — and 1,804 rows/24h were measured lost to ordinary,
+    # momentary lock contention (a concurrent writer holding the DB for a
+    # beat) on dellserver, indistinguishable from a real gap in the trail.
+    # Give this write the same short, backed-off retry every other
+    # load-bearing write in this codebase already gets before falling
+    # through to record_audit's best-effort swallow.
+    retry_on_locked(_write)
+    # #2597-review: the trim below is opportunistic housekeeping over rows
+    # that are already durably committed above — isolate its own retry/
+    # failure from the insert's. Left unguarded, a `DELETE` that hits lock
+    # contention here would propagate out to `record_audit`'s except clause
+    # and get classified (by `is_lock_contention_error` alone, with no way
+    # to tell "the row itself didn't write" apart from "the row wrote fine
+    # but the trim afterward didn't run") as a *lost audit write* — over-
+    # counting genuine data loss for a row that made it into audit_log just
+    # fine. Give the trim the same retry budget as any other write, but
+    # keep a trim-only failure from ever touching that counter.
+    try:
+        retry_on_locked(lambda: _maybe_trim(conn))
+    except sqlite3.OperationalError as exc:
+        if not is_lock_contention_error(exc):
+            raise
+        _log.debug(
+            "record_audit: audit_log trim skipped due to lock contention "
+            "(row itself was already written): %s", exc,
+        )
+
+
+# ── Lock-contention loss counter (#2597) ────────────────────────────────────
+#
+# `retry_on_locked` above turns most lock collisions into a successful write
+# a few hundred milliseconds later, so a write reaching `record_audit`'s
+# `except` clause with `is_lock_contention_error(exc)` true means sustained
+# contention outlasted that whole retry budget — rare, but not impossible
+# under a genuinely busy DB. Logging each occurrence at WARNING is exactly
+# the flood #2597 measured (1,804 identical lines/day) — count them instead
+# and report ONE aggregated warning per process (`atexit`, so it fires for a
+# short-lived CLI invocation like `coord merge` and, if it ever shuts down
+# cleanly, the long-running daemon too) rather than leaving the loss
+# invisible or drowning the log in duplicates.
+
+_lock_contention_losses = 0
+_summary_flush_registered = False
+# #2597-review: `coord serve` calls `record_audit` from multiple
+# threads/greenlets (the tick loop's `run_in_threadpool` calls, concurrent
+# request handlers) — a bare `+= 1` on a module global can lose increments
+# under concurrent contention (read-modify-write race). The counter is
+# purely diagnostic (never gates a decision), but the whole point of it is
+# to make the loss rate trustworthy, so it should not itself be lossy.
+_lock_contention_lock = threading.Lock()
+
+
+def _record_lock_contention_loss() -> None:
+    """Bump the run-scoped lost-write counter and arrange for the aggregate
+    to be reported once this process exits (see
+    :func:`_flush_lock_contention_summary`)."""
+    global _lock_contention_losses, _summary_flush_registered
+    with _lock_contention_lock:
+        _lock_contention_losses += 1
+        register = not _summary_flush_registered
+        _summary_flush_registered = True
+    if register:
+        atexit.register(_flush_lock_contention_summary)
+
+
+def _flush_lock_contention_summary() -> None:
+    """Emit ONE aggregated warning for this run's audit writes lost to
+    sustained lock contention, if any, and reset the counter.
+
+    Registered via :func:`atexit.register` the first time a write is lost
+    (see :func:`_record_lock_contention_loss`) rather than called from a
+    fixed call site, so it fires for every entry point — `coord merge`,
+    `coord notify`, the daemon — without each one needing its own hook.
+    Also callable directly (tests; a caller that wants to flush mid-run
+    without waiting for process exit).
+    """
+    global _lock_contention_losses
+    with _lock_contention_lock:
+        pending = _lock_contention_losses
+        _lock_contention_losses = 0
+    if pending:
+        _log.warning("audit: %d writes lost to lock contention", pending)
+
+
+def audit_lock_contention_losses() -> int:
+    """Current count of audit writes lost to sustained lock contention this
+    run, not yet flushed by :func:`_flush_lock_contention_summary`.
+
+    Exposed for tests and diagnostics — advisory only, like the rest of the
+    audit log; nothing gates dispatch/review/merge decisions on this value.
+    """
+    with _lock_contention_lock:
+        return _lock_contention_losses
+
+
+def flush_lock_contention_summary() -> None:
+    """Public entry point for :func:`_flush_lock_contention_summary`.
+
+    #2597-review: the `atexit` registration above is the right default for
+    a short-lived CLI invocation (`coord merge`, `coord notify`) — it always
+    fires exactly once, at exit. It is a much weaker guarantee for a
+    long-running process (`coord serve`), which the code's own comment
+    already conceded only flushes "if it ever shuts down cleanly" — for a
+    daemon that runs for days between restarts, losses can accumulate for
+    its entire uptime with zero visibility. Call this on a periodic cadence
+    (e.g. once per daemon tick) to close that gap; it is a no-op cost-wise
+    when there is nothing pending (see :func:`_flush_lock_contention_summary`).
+    """
+    _flush_lock_contention_summary()
 
 
 def _maybe_trim(conn) -> None:

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/coord/auto_loop.py

@@ -42,6 +42,7 @@ from dataclasses import dataclass
 import httpx
 
 from coord.config import Config
+from coord.db import is_lock_contention_error
 from coord.dispatch import AGENT_PORT
 from coord import github_ops
 from coord.models import SEALED_PATH_AUTHOR_TYPES, Assignment, Board
@@ -1228,7 +1229,7 @@ def _dispatch_fix(
             repo_github=repo.github,
         )
     except sqlite3.OperationalError as exc:
-        if "database is locked" not in str(exc).lower():
+        if not is_lock_contention_error(exc):
             # Not the transient contention this guard exists for (#2538) —
             # a schema mismatch or malformed statement is a real bug and
             # must not be swallowed as an ordinary declined dispatch.

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/coord/commands/drive_queue.py

@@ -1111,6 +1111,79 @@ def drive_queue_status(output_json: bool, config_path: Path) -> None:
         click.echo("alert: (none)")
 
 
+@drive_queue_group.command("cancel-roll")
+@click.option("--json", "output_json", is_flag=True, default=False,
+              help="Emit the cancelled marker (and any cordons cleared with it) as JSON.")
+def drive_queue_cancel_roll(output_json: bool) -> None:
+    """Cancel a #2587 roll-pending marker so the queue resumes launching now.
+
+    #2607: every OTHER queue-blocking state has an operator override — a
+    held deploy gate has `coord drive-queue resume`, a release cordon has
+    `coord release cordon --clear` — but the roll-pending marker had none.
+    It is supposed to be self-bounding (a TTL / deferral ceiling —
+    `RollPending.expired`), but a marker that keeps getting RE-ARMED before
+    it ever expires never reaches that bound on its own (the defect #2607
+    fixed at the write sites); this command is the escape hatch for while
+    that marker is still live, and for any future case a re-arm loop is not
+    yet ruled out for. Also clears any release cordon still open for the
+    SAME target — the cordon a stalled `coord release propagate`/
+    `nightly-window` run can leave behind — so cancelling actually frees the
+    queue rather than trading one hold for another.
+
+    Does NOT change the target: the next `coord release propagate`/
+    `nightly-window` run re-arms a marker from scratch if the fleet is still
+    behind. This only stops the CURRENT wait.
+    """
+    from coord.machine_pause import clear_cordon, cordons as read_cordons  # noqa: PLC0415
+
+    pending = read_roll_pending()
+    cleared_cordons: list[str] = []
+    if pending is not None:
+        try:
+            active_cordons = read_cordons()
+        except Exception:  # noqa: BLE001 — best-effort; the marker clear below is load-bearing
+            active_cordons = {}
+        for name, cordon in active_cordons.items():
+            if getattr(cordon, "target_version", None) != pending.target_version:
+                continue
+            try:
+                if clear_cordon(name):
+                    cleared_cordons.append(name)
+            except Exception:  # noqa: BLE001 — best-effort; never block the marker clear on this
+                pass
+        clear_roll_pending()
+
+    if output_json:
+        click.echo(
+            _json.dumps(
+                {
+                    "cancelled": pending.to_dict() if pending is not None else None,
+                    "cleared_cordons": sorted(cleared_cordons),
+                },
+                sort_keys=True,
+            )
+        )
+        return
+
+    if pending is None:
+        click.echo("no roll-pending marker to cancel — the queue is not held for a roll")
+        return
+    click.echo(
+        f"cancelled {pending.describe()} — the drive-queue tick resumes "
+        "launching immediately"
+    )
+    if cleared_cordons:
+        click.echo(
+            "also cleared cordon(s) still open for this target: "
+            + ", ".join(sorted(cleared_cordons))
+        )
+    click.echo(
+        "note: this only stops the current wait — the next `coord release "
+        "propagate`/`nightly-window` run re-arms a fresh marker if the "
+        "fleet is still behind"
+    )
+
+
 # ── block-log (#2235 Phase 0) ────────────────────────────────────────────────
 
 

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/coord/commands/merge.py

@@ -4,6 +4,7 @@ coord/cli.py (#747)."""
 
 from __future__ import annotations
 
+import sqlite3
 import sys
 from pathlib import Path
 
@@ -11,6 +12,7 @@ import click
 
 
 from coord.commands._common import _CONFIG_OPTION, _load_config
+from coord.db import is_lock_contention_error, retry_on_locked
 from coord.models import WORK_LIKE_TYPES, effective_issue_number
 
 
@@ -2128,6 +2130,14 @@ def merge(
     open_by_repo, known_by_repo = _load_issue_states()
 
     auto_enqueued: list[str] = []
+    # #2597-review: assignments whose enqueue write is still locked after
+    # `retry_on_locked` exhausts its budget. Collected here instead of
+    # raised in place — one contended assignment must not abort the scan
+    # for every other assignment still waiting in this batch (that's the
+    # exact #1353 regression this file already fixed once, resurrected for
+    # lock contention specifically). The whole run still fails loudly: once
+    # every assignment has had its turn, a non-zero exit is raised below.
+    lock_contention_failures: list[tuple[str, sqlite3.OperationalError]] = []
     # Per-repo cache of branches that still exist on origin.  Lets us skip
     # re-enqueuing done-work whose branch was already merged-and-deleted — the
     # dominant merge-queue clog source.  A done assignment for a closed issue
@@ -2273,11 +2283,17 @@ def merge(
                 # longer matches.  Dedup by (repo_github, branch) is
                 # preserved — refresh_entry_assignment is a no-op when the
                 # entry is already correctly keyed.
-                if mq.refresh_entry_assignment(
+                # #2597: wrapped in retry_on_locked — this is a real DB write
+                # (merge_queue.save_queue), and unlike the gh-round-trip
+                # failures this try/except otherwise isolates (#1353), a
+                # `database is locked` collision here is pure transient
+                # contention with a concurrent writer, not a reason to give
+                # up on this assignment.
+                if retry_on_locked(lambda: mq.refresh_entry_assignment(
                     a,
                     repo_github=repo_cfg.github,
                     target_branch=target_branch,
-                ):
+                )):
                     auto_enqueued.append(
                         f"  auto-enqueued: {a.repo_name} #{a.issue_number} "
                         f"({a.branch} → {target_branch}){gate_note}"
@@ -2295,6 +2311,34 @@ def merge(
                         f"{target_branch}) — "
                         f"{mq.describe_merge_gate_failures(gate_failures)}"
                     )
+            except sqlite3.OperationalError as exc:
+                if is_lock_contention_error(exc):
+                    # #2597: `retry_on_locked` above already gave the write
+                    # several backed-off attempts — reaching here means the
+                    # DB is still locked after that budget, i.e. sustained
+                    # contention, not a momentary collision. Dropping a
+                    # mergeable assignment out of the scan on that basis is
+                    # the exact correctness bug #2597 reported (a skipped
+                    # entry here is silent — nothing re-scans it until the
+                    # next `coord merge` invocation). This is a hard
+                    # failure, not the soft "skipped" outcome below — but
+                    # (per review) it must not abort the rest of the batch
+                    # either. Record it and keep scanning; a clearly-flagged
+                    # line goes into the summary and the run still fails
+                    # loudly (non-zero exit, real exception) once every
+                    # other assignment has had its turn.
+                    lock_contention_failures.append((a.assignment_id, exc))
+                    auto_enqueued.append(
+                        f"  LOCK CONTENTION: {a.repo_name} #{a.issue_number} "
+                        f"(assignment {a.assignment_id}) — auto-enqueue write "
+                        f"still locked after exhausting retries: {exc!r}"
+                    )
+                    continue
+                auto_enqueued.append(
+                    f"  skipped: {a.repo_name} #{a.issue_number} "
+                    f"(assignment {a.assignment_id}) — auto-enqueue scan "
+                    f"failed, skipping this assignment: {exc!r}"
+                )
             except Exception as e:  # noqa: BLE001
                 auto_enqueued.append(
                     f"  skipped: {a.repo_name} #{a.issue_number} "
@@ -2304,6 +2348,20 @@ def merge(
     for line in auto_enqueued:
         click.echo(line)
 
+    if lock_contention_failures:
+        # #2597-review: every assignment in the batch has now had its turn
+        # — including the ones after the first contended write, and their
+        # results (if any) are already printed above. Surface the
+        # contention loudly now: a real `sqlite3.OperationalError`
+        # propagating out of this command (not a swallowed summary line)
+        # so `coord merge` exits non-zero and the failure is impossible to
+        # miss, same as any other genuinely unrecoverable write failure.
+        assignment_ids = ", ".join(aid for aid, _exc in lock_contention_failures)
+        raise sqlite3.OperationalError(
+            "database is locked — auto-enqueue write(s) still locked after "
+            f"exhausting retries for assignment(s): {assignment_ids}"
+        ) from lock_contention_failures[-1][1]
+
     # #1477: re-test any parked CONFLICT entry against GitHub's own
     # mergeability computation before the pending scan below — a branch
     # repaired by a conflict-fix worker (or by hand) since the last tick

{code_coordinator-0.5.238 → code_coordinator-0.5.240}/coord/commands/release.py

@@ -431,6 +431,24 @@ def _paused_machine_busy(config) -> list:
     return busy
 
 
+def _confirmation_running_busy() -> list:
+    """A #2464 out-of-band test confirmation in flight, as a `Busy` signal
+    (#2596).
+
+    Thin wiring around :func:`coord.release_propagate.confirmation_lock_busy`
+    — see that function's docstring for the incident and why a LOCAL,
+    non-networked lock probe is the right (and sufficient) check here. Feeds
+    the same `extra_busy` seam `_interactive_session_busy`/
+    `_paused_machine_busy` do, at both call sites that build one (the top of
+    `propagate` and `_drain`'s default `extra_busy_fetch`), so a confirmation
+    in flight defers a restart exactly like a live headless assignment does.
+    """
+    from coord import release_propagate as rp  # noqa: PLC0415
+    from coord.filelock import notify_lock_path  # noqa: PLC0415
+
+    return rp.confirmation_lock_busy(notify_lock_path())
+
+
 def _daemon_machine_name(
     config, override: str | None, machine_health: dict | None = None
 ) -> str | None:
@@ -595,14 +613,22 @@ def _ensure_roll_pending_marker(target_version: str, *, reason: str) -> None:
     :class:`coord.drive_queue.RollPending`'s docstring for the mechanism
     this feeds.
 
-    Never overwrites an existing marker for the SAME target — resetting
-    ``set_at``/``deferrals`` every few minutes while this timer keeps
-    finding the fleet busy would make the TTL bound
-    (:meth:`~coord.drive_queue.RollPending.expired`) unreachable in
-    practice, exactly the "never actually bounded" failure #2587's own
-    bound exists to prevent. A marker for a DIFFERENT (stale) target IS
-    replaced — the newly resolved version is the one that should roll.
+    Never overwrites an existing marker's ``set_at``/``deferrals`` — for the
+    SAME target this is a plain no-op (unchanged from before #2607); for a
+    DIFFERENT target the target/reason are updated in place but the clock
+    and deferral count carry over unchanged. #2607: PyPI's "latest" climbs on
+    every merge (`Auto-release on merge to main`), so a busy fleet's target
+    had almost always moved by the time this timer re-fired — treating that
+    as a "new" roll and resetting ``set_at``/``deferrals`` made the TTL/
+    deferral bound (:meth:`~coord.drive_queue.RollPending.expired`)
+    unreachable in practice, exactly the "never actually bounded" failure
+    #2587's own bound exists to prevent, and exactly how #2607's queue froze
+    for good. A re-arm — same target or not — is a continuation of the SAME
+    stuck roll, not a new operator request; only a marker set from scratch
+    (no existing one at all, e.g. right after `coord drive-queue
+    cancel-roll`) starts a new clock.
     """
+    import dataclasses as _dataclasses  # noqa: PLC0415
     import time as _time  # noqa: PLC0415
 
     from coord.commands.drive_queue import read_roll_pending, write_roll_pending  # noqa: PLC0415
@@ -611,6 +637,11 @@ def _ensure_roll_pending_marker(target_version: str, *, reason: str) -> None:
     existing = read_roll_pending()
     if existing is not None and existing.target_version == target_version:
         return
+    if existing is not None:
+        write_roll_pending(
+            _dataclasses.replace(existing, target_version=target_version, reason=reason)
+        )
+        return
     write_roll_pending(
         RollPending(target_version=target_version, set_at=_time.time(), reason=reason)
     )
@@ -808,6 +839,13 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
     # seam from the daemon-aware pause store, so an operator-paused machine
     # defers exactly like a live assignment does instead of reading idle.
     extra_busy.extend(_paused_machine_busy(config))
+    # #2596: nor can it see a #2464 confirmation running in the notify
+    # drain — a restart landing mid-confirmation is exactly what let a
+    # SIGTERM get read as a test failure on 2026-08-22 (the confirmation
+    # itself now refuses that read, #2527, but the restart still shouldn't
+    # have happened at all). Feed the same seam so it defers like a live
+    # assignment instead of getting reaped.
+    extra_busy.extend(_confirmation_running_busy())
     quiescence = rp.assess_quiescence(
         queue_entries=board.get("drive_queue") or [],
         assignments=board.get("assignments") or [],
@@ -2510,8 +2548,15 @@ def _drain(
             # #2174: paused/quiet-hours state, same as the top-level
             # `propagate` call site — a paused daemon host must keep the
             # drain from ever reading "clear" just because tmux is quiet.
+            # #2596: same for a confirmation running under `notify.lock` —
+            # gated on `config is not None` too, even though the check
+            # itself needs no config, purely to keep this function's
+            # documented "no *config* means no real filesystem I/O"
+            # contract intact for the `config=None` unit-test path above.
             extra_busy_fetch = lambda: (  # noqa: E731
-                _interactive_session_busy(config) + _paused_machine_busy(config)
+                _interactive_session_busy(config)
+                + _paused_machine_busy(config)
+                + _confirmation_running_busy()
             )
         else:
             extra_busy_fetch = lambda: []  # noqa: E731
@@ -2799,6 +2844,7 @@ def release_nightly_window(
     genuinely idle is a fine moment to roll immediately. A still-busy fleet
     just leaves the marker exactly as it was for the tick to keep watching.
     """
+    import dataclasses as _dataclasses  # noqa: PLC0415
     import json as _json  # noqa: PLC0415
     import time  # noqa: PLC0415
 
@@ -3002,18 +3048,35 @@ def release_nightly_window(
             _finish(rw.STATUS_PROPAGATE_FAILED, prop_exit or 1)
 
     # ── 3. set (or refresh) the marker and return immediately (#2587) ─────
+    # #2607: a marker already pending for a DIFFERENT target is updated in
+    # place — target_version/reason move, but set_at/deferrals carry over
+    # unchanged. This is the exact site the incident traced back to: PyPI's
+    # "latest" climbs on every merge, so by the time this nightly timer
+    # re-fired the target had almost always moved, and writing a brand new
+    # `RollPending` here reset the TTL/deferral escape hatch on every single
+    # re-arm — the queue froze because the bound that was supposed to save it
+    # could never be reached. Only a genuinely fresh marker (no existing one
+    # at all) gets a new clock.
     if existing is not None:
         click.echo(
             f"replacing stale roll-pending marker ({existing.describe()}) "
-            f"with v{record.target_version}"
+            f"with v{record.target_version} — preserving its original "
+            f"set_at/deferrals (#2607: a re-arm is a continuation of the "
+            f"same stuck roll, not a new request)"
         )
-    write_roll_pending(
-        RollPending(
-            target_version=record.target_version,
-            set_at=time.time(),
-            reason="nightly-window",
+        write_roll_pending(
+            _dataclasses.replace(
+                existing, target_version=record.target_version, reason="nightly-window",
+            )
+        )
+    else:
+        write_roll_pending(
+            RollPending(
+                target_version=record.target_version,
+                set_at=time.time(),
+                reason="nightly-window",
+            )
         )
-    )
     click.echo(
         f"roll pending: v{record.target_version} — the drive-queue tick will "
         f"fire `coord release propagate` at the next inter-drive gap "