code-coordinator 0.5.276__tar.gz → 0.5.277__tar.gz

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.276
+Version: 0.5.277
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.276
+Version: 0.5.277
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.276",
+  "tag": "0.5.277",
   "distance": 0,
-  "node": "gf933f3cbf3912a33e93783fcaece9a9d6aac054b",
+  "node": "ga1b2068ad0c1dc506226ba5fbe7adb8fd566702b",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-29"

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/coord/commands/drive_queue.py

@@ -53,6 +53,7 @@ from coord.drive_queue import (
     QUEUE_ALERT_REPO,
     QUEUE_ALERT_STAGE,
     RESUME_PROBE_TIMEOUT_SECONDS,
+    ROLL_LEDGER_CUMULATIVE_BOUND_SECONDS,
     ROLL_PENDING_DEFAULT_MAX_DEFERRALS,
     ROLL_PENDING_DEFAULT_TTL_SECONDS,
     STATE_BLOCKED,
@@ -66,6 +67,7 @@ from coord.drive_queue import (
     ProbeResult,
     QueueEntry,
     QueueError,
+    RollLedger,
     RollPending,
     TickPlan,
     add_preflight_notice,
@@ -1628,10 +1630,22 @@ def drive_queue_cancel_roll(output_json: bool) -> None:
     Does NOT change the target: the next `coord release propagate`/
     `nightly-window` run re-arms a marker from scratch if the fleet is still
     behind. This only stops the CURRENT wait.
+
+    #2889: also resets the roll LEDGER (`RollLedger`) — the cumulative
+    frozen-time bookkeeping a fresh marker's own arm now checks — so an
+    operator clearing a stuck marker also clears whatever run-up of
+    cumulative frozen time and fresh-arm history led to it, exactly the
+    "operator intervenes" escape hatch item 1's cumulative bound requires.
+    Reset unconditionally, even when there is no LIVE marker to cancel: a
+    ledger can be `escalated` (refusing every fresh arm) with nothing
+    currently pending — its last marker already expired and cleared on its
+    own — and that state needs the SAME override.
     """
     from coord.machine_pause import clear_cordon, cordons as read_cordons  # noqa: PLC0415
 
     pending = read_roll_pending()
+    ledger = read_roll_ledger()
+    ledger_had_history = ledger.marker_count > 0 or ledger.cumulative_frozen_seconds > 0
     cleared_cordons: list[str] = []
     if pending is not None:
         try:
@@ -1647,6 +1661,8 @@ def drive_queue_cancel_roll(output_json: bool) -> None:
             except Exception:  # noqa: BLE001 — best-effort; never block the marker clear on this
                 pass
         clear_roll_pending()
+    if ledger_had_history:
+        reset_roll_ledger()
 
     if output_json:
         click.echo(
@@ -1654,29 +1670,45 @@ def drive_queue_cancel_roll(output_json: bool) -> None:
                 {
                     "cancelled": pending.to_dict() if pending is not None else None,
                     "cleared_cordons": sorted(cleared_cordons),
+                    "ledger_reset": ledger.to_dict() if ledger_had_history else None,
                 },
                 sort_keys=True,
             )
         )
         return
 
-    if pending is None:
+    if pending is None and not ledger_had_history:
         click.echo("no roll-pending marker to cancel — the queue is not held for a roll")
         return
-    click.echo(
-        f"cancelled {pending.describe()} — the drive-queue tick resumes "
-        "launching immediately"
-    )
+    if pending is not None:
+        click.echo(
+            f"cancelled {pending.describe()} — the drive-queue tick resumes "
+            "launching immediately"
+        )
+    else:
+        click.echo(
+            "no roll-pending marker to cancel, but the roll ledger had "
+            f"{ledger.cumulative_frozen_seconds:.0f}s cumulative frozen time "
+            f"across {ledger.marker_count} marker generation(s)"
+        )
     if cleared_cordons:
         click.echo(
             "also cleared cordon(s) still open for this target: "
             + ", ".join(sorted(cleared_cordons))
         )
-    click.echo(
-        "note: this only stops the current wait — the next `coord release "
-        "propagate`/`nightly-window` run re-arms a fresh marker if the "
-        "fleet is still behind"
-    )
+    if ledger_had_history:
+        click.echo(
+            "also reset the #2889 roll ledger "
+            f"({ledger.cumulative_frozen_seconds:.0f}s cumulative frozen time / "
+            f"{ledger.marker_count} marker generation(s)) — a fresh arm is no "
+            "longer refused for having run up that history"
+        )
+    if pending is not None:
+        click.echo(
+            "note: this only stops the current wait — the next `coord release "
+            "propagate`/`nightly-window` run re-arms a fresh marker if the "
+            "fleet is still behind"
+        )
 
 
 # ── block-log (#2235 Phase 0) ────────────────────────────────────────────────
@@ -2587,6 +2619,136 @@ def clear_roll_pending() -> None:
         pass
 
 
+# ── #2889: the roll ledger — memory that survives a marker's own clear ──────
+
+_ROLL_LEDGER_FILENAME = "roll_pending_ledger.json"
+
+
+def roll_pending_ledger_path() -> Path:
+    """Absolute path to the #2889 roll ledger.
+
+    ``$COORD_ROLL_PENDING_LEDGER_STATE`` overrides it — same test-isolation
+    seam as :func:`roll_pending_path`; never let a test write the operator's
+    real ``~/.coord/roll_pending_ledger.json``.
+    """
+    import os  # noqa: PLC0415
+
+    from coord.platform_paths import default_coord_dir  # noqa: PLC0415
+
+    override = os.environ.get("COORD_ROLL_PENDING_LEDGER_STATE")
+    if override:
+        return Path(override).expanduser()
+    return default_coord_dir() / _ROLL_LEDGER_FILENAME
+
+
+def read_roll_ledger() -> RollLedger:
+    """The current ledger — never ``None``: an absent/corrupt/hand-edited
+    file reads as a fresh, empty ``RollLedger()``, the same fail-soft
+    posture :func:`read_roll_pending` takes on its own file, and for the
+    same reason — a ledger this cannot make sense of must never wedge future
+    arms forever.
+    """
+    path = roll_pending_ledger_path()
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except OSError:
+        return RollLedger()
+    if not raw.strip():
+        return RollLedger()
+    try:
+        data = _json.loads(raw)
+    except ValueError:
+        return RollLedger()
+    if not isinstance(data, dict):
+        return RollLedger()
+    return RollLedger.from_dict(data)
+
+
+def write_roll_ledger(ledger: RollLedger) -> None:
+    """Persist *ledger*, overwriting whatever was there before. Same atomic
+    tempfile-then-rename pattern as :func:`write_roll_pending`."""
+    import os  # noqa: PLC0415
+    import tempfile  # noqa: PLC0415
+
+    path = roll_pending_ledger_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = _json.dumps(ledger.to_dict(), sort_keys=True)
+    fd, tmp_name = tempfile.mkstemp(prefix=".roll-ledger.", suffix=".tmp", dir=path.parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(payload)
+        os.replace(tmp_name, path)
+    except Exception:
+        try:
+            os.unlink(tmp_name)
+        except OSError:
+            pass
+        raise
+
+
+def reset_roll_ledger() -> None:
+    """Drop the ledger — a CONFIRMED roll or explicit operator intervention
+    (`coord drive-queue cancel-roll`) both mean "clean slate": whatever ran
+    up the cumulative bound is over. A no-op when there was nothing to
+    reset."""
+    path = roll_pending_ledger_path()
+    try:
+        path.unlink()
+    except FileNotFoundError:
+        pass
+
+
+#: #2889's own escalation key — deliberately distinct from
+#: `ROLL_PENDING_ALERT_STAGE` (a single expired marker) so the two can
+#: coexist as separate rows: a marker expiring is routine (bounded by design,
+#: #2587's whole point); the LEDGER crossing its cumulative bound is the
+#: escalation that means "this has now failed to roll unattended several
+#: times in a row — an operator needs to look, not just wait for the next
+#: gap."
+ROLL_LEDGER_ALERT_REPO = "(roll-pending)"
+ROLL_LEDGER_ALERT_ISSUE = 0
+ROLL_LEDGER_ALERT_STAGE = "roll-ledger-escalated"
+
+
+def _escalate_roll_ledger(ledger: RollLedger, *, now: float) -> None:
+    """Surface a ledger that just crossed its cumulative bound (#2889 item
+    1's "escalate loudly") — mirrors `_escalate_roll_pending_expired` one
+    level up: that one is routine (a single marker hit ITS OWN bound, exactly
+    as designed); this one means the SAME target has now failed to roll
+    unattended across `ledger.marker_count` separate marker generations, and
+    arming a fresh one is refused until `coord drive-queue cancel-roll`
+    clears this ledger.
+    """
+    from coord.state import record_drive_escalation  # noqa: PLC0415
+
+    reason = (
+        f"roll-pending ledger escalated: {ledger.cumulative_frozen_seconds:.0f}s "
+        f"cumulative frozen time across {ledger.marker_count} marker "
+        "generation(s), none of them confirming a roll — refusing to arm a "
+        "fresh marker until `coord drive-queue cancel-roll` clears this "
+        "ledger (#2889: a per-marker TTL bounds one marker; this bounds the "
+        "RATE of fresh ones that follow)"
+    )
+    click.echo(f"warning: {reason}", err=True)
+    try:
+        record_drive_escalation(
+            ROLL_LEDGER_ALERT_REPO,
+            ROLL_LEDGER_ALERT_ISSUE,
+            stage=ROLL_LEDGER_ALERT_STAGE,
+            reason=reason,
+            gate_readings=(
+                f"cumulative_frozen_seconds={ledger.cumulative_frozen_seconds:.0f} | "
+                f"bound={ROLL_LEDGER_CUMULATIVE_BOUND_SECONDS:.0f} | "
+                f"marker_count={ledger.marker_count}"
+            ),
+            proposed_command="coord drive-queue cancel-roll",
+        )
+    except Exception as exc:  # noqa: BLE001 — the stderr line above is the
+        # floor; an escalation table that cannot be written must not take
+        # the message down with it.
+        click.echo(f"  (could not record the roll-ledger escalation: {exc})", err=True)
+
+
 def _roll_pending_may_fire(*, occupied: int, now: float, queue_rows) -> tuple[bool, str]:
     """Is it worth attempting to fire a pending roll THIS tick? (#2870 part 2)
 
@@ -2664,12 +2826,25 @@ def _fire_pending_roll(*, dry_run: bool = False) -> tuple[bool, str]:
     Returns ``(fired, detail)``, never raises — a `systemctl` this host
     cannot run (no systemd, PATH issue, unit not installed) must be a
     deferral the marker survives to retry next tick, not a crashed tick.
+
+    #2889 item 4: ``--setenv=COORD_ROLL_INVOKER=drive-queue-tick`` tags THIS
+    specific start with a caller identity `coord release nightly-window`
+    (the spawned unit's `ExecStart=`) reads back and journals as
+    `WindowRecord.invoked_by` — overriding the unit file's own static
+    `Environment=` default for just this one invocation (systemd layers a
+    transient `--setenv=` on top of a unit's static `Environment=`), so
+    "what invoked this run" is answerable from `coord release
+    window-history` instead of requiring a live reproduction.
     """
     if dry_run:
         return True, "--dry-run: would run `systemctl --user start --no-block coord-release-window.service`"
     try:
         proc = subprocess.run(  # noqa: S603, S607 — fixed argv, no shell
-            ["systemctl", "--user", "start", "--no-block", "coord-release-window.service"],
+            [
+                "systemctl", "--user", "start", "--no-block",
+                "--setenv=COORD_ROLL_INVOKER=drive-queue-tick",
+                "coord-release-window.service",
+            ],
             capture_output=True,
             text=True,
             timeout=30,
@@ -4483,10 +4658,22 @@ def drive_queue_tick(
         # is dropped right here, loudly (`_escalate_roll_pending_expired`),
         # and this tick proceeds exactly as if no marker had ever been set —
         # the #2587 "never hold the queue down indefinitely" requirement.
+        # #2889: before it is dropped, its lived duration is folded into the
+        # roll LEDGER (`RollLedger`, `roll_pending_ledger.json`) — memory
+        # that survives THIS clear, unlike the marker itself, so a fresh arm
+        # right after this one cannot dodge the cumulative bound the way a
+        # brand new `RollPending` dodges a single marker's own TTL. Crossing
+        # that bound is its own, separate, louder escalation.
         now = time.time()
         roll_pending = read_roll_pending()
         if roll_pending is not None and roll_pending.expired(now):
             _escalate_roll_pending_expired(roll_pending, now=now)
+            ledger = read_roll_ledger()
+            was_escalated = ledger.escalated
+            ledger = ledger.record_expiry(roll_pending, now=now)
+            write_roll_ledger(ledger)
+            if ledger.escalated and not was_escalated:
+                _escalate_roll_ledger(ledger, now=now)
             clear_roll_pending()
             roll_pending = None
         if roll_pending is not None:

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/coord/commands/release.py

@@ -653,13 +653,96 @@ def _lane_versions_by_host(report) -> dict[str, list[str | None]]:
     return out
 
 
+def _queue_provably_busy(quiescence, daemon_name: str | None) -> bool:
+    """#2889 item 2: does *quiescence* name a genuine `drive_queue` row (not
+    some OTHER busy reason — an interactive session, a paused machine, an
+    in-flight confirmation) actively occupying *daemon_name* right now?
+
+    Deliberately narrower than "the daemon is busy" (`daemon_name in
+    busy_hosts`, this module's existing arm TRIGGER — see
+    `_ensure_roll_pending_marker`'s call site): a genuine drive-queue entry
+    is the one busy reason a freshly-armed marker cannot do anything about
+    faster than `coord drive-queue tick`'s own reconciliation already will
+    on its normal ~3-minute cadence — arming one anyway just spends up to a
+    full TTL (#2587's own hour-long bound) learning that for free, exactly
+    the "10 fresh arms, 49 ticks refused to launch" pathology #2889 reports.
+    A busy signal of any OTHER kind (`Quiescence.busy`'s `kind` field —
+    `coord.release_propagate.assess_quiescence`'s own vocabulary:
+    "confirmation/notify drain running", "live RUNNING assignment", a paused
+    machine) is exactly the case a marker's capacity-0 freeze DOES help:
+    nothing NEW should launch onto the daemon host while it clears, and
+    nothing here can predict when it will on its own.
+
+    Unattributable signals (``b.host is None`` — `Quiescence.fleet_wide_busy`)
+    are included: an unreadable board or an unattributed drive-queue row
+    means "busy somewhere unknown", which must be read as busy everywhere,
+    the same rule `Quiescence.rollable_hosts` already applies.
+    """
+    return any(
+        b.kind == "drive-queue entry running" and (b.host is None or b.host == daemon_name)
+        for b in quiescence.busy
+    )
+
+
+def _fresh_arm_refusal_reason(ledger, *, now: float, queue_provably_busy: bool) -> str:
+    """Empty string when a FRESH `RollPending` arm (no existing marker at
+    all) may proceed; otherwise the reason it must not, in priority order —
+    escalated ledger, then the rate limit, then a provably busy queue.
+
+    Shared by `_ensure_roll_pending_marker` and `release_nightly_window`'s
+    own arm site (#2587 section 3) so the three #2889 checks can never drift
+    between the two places a fresh marker gets armed — #2096's "two
+    surfaces, one function" rule, one level up from `_drain`'s own reuse of
+    `release_propagate.assess_quiescence` for the identical reason.
+
+    Pure — takes a `coord.drive_queue.RollLedger` and the caller's own
+    already-computed *queue_provably_busy* rather than reading a clock or a
+    board itself, so it stays trivially testable and every call site decides
+    for itself how loud to be about the result (only the ESCALATED reason is
+    meant to also trigger a recorded escalation — see both callers).
+    """
+    if ledger.escalated:
+        return (
+            f"the #2889 roll ledger has escalated "
+            f"({ledger.cumulative_frozen_seconds:.0f}s cumulative frozen time across "
+            f"{ledger.marker_count} marker generation(s)) — run `coord drive-queue "
+            "cancel-roll` to clear it before arming again"
+        )
+    wait = ledger.seconds_until_next_arm(now)
+    if wait > 0:
+        return (
+            f"under the #2889 rate limit ({wait:.0f}s remaining since the last "
+            "fresh arm)"
+        )
+    if queue_provably_busy:
+        return (
+            "a genuine drive-queue entry is actively occupying the daemon host "
+            "right now (#2889 item 2) — a fresh marker cannot roll any faster "
+            "than the tick's own reconciliation already will"
+        )
+    return ""
+
+
 def _ensure_roll_pending_marker(
     target_version: str, *, reason: str, min_releases_behind: int | None = None,
-    dry_run: bool = False,
-) -> None:
+    dry_run: bool = False, queue_provably_busy: bool = False,
+) -> bool:
     """#2587: make sure a roll-pending marker exists for *target_version*,
     without resetting an already-live one's clock.
 
+    #2889: returns whether a marker is live as a result of this call
+    (existing/updated/freshly armed) — ``False`` only for a REFUSED fresh
+    arm (see the three checks below). Every existing call site predates this
+    return value and simply ignores it — a refused arm is already, on its
+    own, an ordinary deferral from the caller's point of view, exactly like
+    a still-busy fleet. *queue_provably_busy* (see
+    :func:`_queue_provably_busy`) only ever affects a FRESH arm (no existing
+    marker at all): a re-arm of an already-live marker (same OR different
+    target) is a continuation of a campaign already in progress and proceeds
+    regardless — refusing THAT would just reintroduce #2607's own bug by a
+    different door, since a marker that cannot be kept current would then
+    fire against a target it no longer names.
+
     #2870: *min_releases_behind* is the effective `propagation.
     min_releases_behind`/`--min-behind` threshold THIS run resolved and
     already gated on before reaching this call (see the #2583 gate just
@@ -710,12 +793,17 @@ def _ensure_roll_pending_marker(
     import dataclasses as _dataclasses  # noqa: PLC0415
     import time as _time  # noqa: PLC0415
 
-    from coord.commands.drive_queue import read_roll_pending, write_roll_pending  # noqa: PLC0415
+    from coord.commands.drive_queue import (  # noqa: PLC0415
+        _escalate_roll_ledger,
+        read_roll_ledger,
+        read_roll_pending,
+        write_roll_pending,
+    )
     from coord.drive_queue import RollPending  # noqa: PLC0415
 
     existing = read_roll_pending()
     if existing is not None and existing.target_version == target_version:
-        return
+        return True
     if existing is not None:
         if dry_run:
             click.echo(
@@ -723,27 +811,56 @@ def _ensure_roll_pending_marker(
                 f"({existing.describe()}) with v{target_version} (reason="
                 f"{reason!r}), preserving its original set_at/deferrals (#2607)"
             )
-            return
+            return True
         write_roll_pending(
             _dataclasses.replace(
                 existing, target_version=target_version, reason=reason,
                 min_releases_behind=min_releases_behind,
             )
         )
-        return
+        return True
+
+    # #2889: a genuinely FRESH arm — no existing marker at all — is the one
+    # case the RATE of markers can run away (#2889's own report: ten fresh
+    # arms in ~15 hours, each one individually well-behaved).
+    now = _time.time()
+    ledger = read_roll_ledger()
+    refusal = _fresh_arm_refusal_reason(
+        ledger, now=now, queue_provably_busy=queue_provably_busy,
+    )
+    if refusal:
+        message = f"declining to arm a roll-pending marker for v{target_version} — {refusal}"
+        if dry_run:
+            click.echo(f"--dry-run: would refuse to arm — {refusal}")
+            return False
+        click.echo(message, err=True)
+        if ledger.escalated:
+            # Re-recorded on every refused attempt, same "quiet re-record
+            # keeps the escalation fresh" posture the #2572 self-cordon
+            # escalation already documents — cheap (an upsert) and means an
+            # operator checking `coord drive-queue status` mid-day sees this
+            # is still live, not a one-time blip from hours ago.
+            _escalate_roll_ledger(ledger, now=now)
+        return False
+
     if dry_run:
         click.echo(
             f"--dry-run: would set a roll-pending marker for v{target_version} "
             f"(reason={reason!r}) — the drive-queue tick would then hold "
             "capacity at 0 until the queue drains"
         )
-        return
+        return True
     write_roll_pending(
         RollPending(
-            target_version=target_version, set_at=_time.time(), reason=reason,
+            target_version=target_version, set_at=now, reason=reason,
             min_releases_behind=min_releases_behind,
         )
     )
+    # Nothing to write to the ledger here — only an EXPIRY moves its
+    # rate-limit clock (`RollLedger.record_expiry`), never an arm. See that
+    # method's own docstring for why measuring from the arm, not the clear,
+    # would let the exact re-arm-right-after-TTL-expiry case through.
+    return True
 
 
 @release_group.command(
@@ -1146,9 +1263,15 @@ def release_propagate(  # noqa: PLR0912, PLR0915 — a pipeline; the decisions a
         # (resolved above, 3a) onto the marker, so whatever discharges it is
         # gated at the threshold that armed it, not a threshold re-resolved
         # from scratch later.
+        # #2889 item 2: `quiescence` was already computed above (3) — reuse
+        # it rather than re-fetching the board, to tell a genuine
+        # drive-queue occupancy apart from the OTHER busy reasons that put
+        # the daemon in `busy_hosts` (see `_queue_provably_busy`'s
+        # docstring). Only affects a genuinely FRESH arm.
         _ensure_roll_pending_marker(
             record.target_version, reason="propagate",
             min_releases_behind=effective_min_behind, dry_run=dry_run,
+            queue_provably_busy=_queue_provably_busy(quiescence, daemon_name),
         )
         _finish(rp.STATUS_DEFERRED, 0)
 
@@ -3088,7 +3211,9 @@ def release_nightly_window(
     from coord.commands._common import _load_config  # noqa: PLC0415
     from coord.commands.drive_queue import (  # noqa: PLC0415
         clear_roll_pending,
+        read_roll_ledger,
         read_roll_pending,
+        reset_roll_ledger,
         write_roll_pending,
     )
     from coord.drive_queue import RollPending  # noqa: PLC0415
@@ -3100,8 +3225,20 @@ def release_nightly_window(
 
     config = _load_config(config_path)
     state_dir = _state_dir()
+    # #2889 item 4: "what invoked this?" straight from the journal —
+    # `$COORD_ROLL_INVOKER` is set on the process environment by whichever
+    # trigger started `coord-release-window.service`: the unit's own static
+    # `Environment=` default (its timer, or a human running `systemctl
+    # --user start` by hand — systemd cannot tell those two apart, see the
+    # unit's own "INVOKER" section) or `_fire_pending_roll`'s explicit
+    # `--setenv=` override when the drive-queue tick is the one firing it.
+    # Empty for a bare CLI invocation outside the unit (a test, an
+    # operator's own shell) — nothing set the env var at all.
+    import os as _os  # noqa: PLC0415
+
     record = rw.WindowRecord(
-        started_at=time.time(), dry_run=dry_run, queue_timer=queue_timer
+        started_at=time.time(), dry_run=dry_run, queue_timer=queue_timer,
+        invoked_by=_os.environ.get("COORD_ROLL_INVOKER", ""),
     )
 
     def _finish(status: str, exit_code: int = 0) -> None:
@@ -3185,6 +3322,11 @@ def release_nightly_window(
     if not rw.needs_roll(record.daemon_version, record.target_version):
         if read_roll_pending() is not None:
             clear_roll_pending()
+            # #2889: the goal is reached without needing the marker at all —
+            # a clean slate, same as a CONFIRMED roll below. Whatever
+            # cumulative frozen time/fresh-arm history the ledger carried is
+            # over; a FUTURE delta starts counting fresh.
+            reset_roll_ledger()
         _finish(rw.STATUS_UP_TO_DATE, 0)
 
     # ── 2b. #2583 min-releases-behind gate ────────────────────────────────
@@ -3296,6 +3438,11 @@ def release_nightly_window(
             rp.STATUS_VERIFIED, rp.STATUS_UP_TO_DATE, rp.STATUS_ROLLED,
         ):
             clear_roll_pending()
+            # #2889: a CONFIRMED roll (or a race that found it already
+            # current) is the clean-slate outcome — reset the ledger so a
+            # FUTURE delta starts counting fresh rather than inheriting
+            # whatever run-up this campaign accumulated on its way here.
+            reset_roll_ledger()
             status = (
                 rw.STATUS_UP_TO_DATE if prop_status == rp.STATUS_UP_TO_DATE
                 else rw.STATUS_ROLLED
@@ -3381,14 +3528,56 @@ def release_nightly_window(
             )
         )
     else:
+        # #2889: a genuinely FRESH arm — no existing marker for this
+        # campaign at all — is the one case the RATE of markers can run
+        # away. Same three checks `_ensure_roll_pending_marker` applies to
+        # `coord release propagate`'s own arm site (#2096: one function, not
+        # two definitions that could drift) — `_queue_provably_busy` needs
+        # its own board read here since this command does not otherwise
+        # compute a `Quiescence` (unlike `release_propagate`, which already
+        # has one to reuse).
+        arm_now = time.time()
+        ledger = read_roll_ledger()
+        board, board_error = _fetch_board()
+        if board_error:
+            # Cannot prove the queue non-empty either way — the #2889 item 2
+            # check contributes nothing this run; the ledger's own
+            # escalation/rate-limit checks still apply regardless.
+            queue_provably_busy = False
+        else:
+            quiescence = rp.assess_quiescence(
+                queue_entries=board.get("drive_queue") or [],
+                assignments=board.get("assignments") or [],
+                issues=board.get("issues") or [],
+                now=arm_now,
+            )
+            queue_provably_busy = _queue_provably_busy(quiescence, daemon_name)
+        refusal = _fresh_arm_refusal_reason(
+            ledger, now=arm_now, queue_provably_busy=queue_provably_busy,
+        )
+        if refusal:
+            record.error = (
+                f"declined to arm a roll-pending marker for v{record.target_version} "
+                f"— {refusal}"
+            )
+            click.echo(f"note: {record.error}")
+            if ledger.escalated:
+                from coord.commands.drive_queue import _escalate_roll_ledger  # noqa: PLC0415
+
+                _escalate_roll_ledger(ledger, now=arm_now)
+                _escalate_window(record, reason=record.error)
+                _finish(rw.STATUS_LEDGER_ESCALATED, 1)
+            _finish(rw.STATUS_ARM_DEFERRED, 0)
         write_roll_pending(
             RollPending(
                 target_version=record.target_version,
-                set_at=time.time(),
+                set_at=arm_now,
                 reason="nightly-window",
                 min_releases_behind=effective_min_behind,
             )
         )
+        # Nothing to write to the ledger here — only an EXPIRY moves its
+        # rate-limit clock (`RollLedger.record_expiry`), never an arm.
     click.echo(
         f"roll pending: v{record.target_version} — the drive-queue tick will "
         f"fire `coord release propagate` at the next inter-drive gap "

{code_coordinator-0.5.276 → code_coordinator-0.5.277}/coord/deploy/coord-release-window.service

@@ -173,6 +173,36 @@
 # since it was added but would NOT have survived a from-scratch rebuild of
 # this host — the whole reason this is now the packaged default instead of
 # operator-local state.
+#
+# INVOKER (#2889 item 4) — ANSWERS "WHAT STARTED THIS?", HONESTLY BOUNDED
+# --------------------------------------------------------------------------
+# 2026-08-28's #2889 report found this unit starting ~10 times with BOTH
+# release timers masked, and could rule out every trigger this codebase
+# actually controls (`coord-release-window.timer` masked, the tick's own
+# `_fire_pending_roll` only fires when a marker already exists, the watchdog
+# only restarts `failed` units, no cron entry) without ever identifying the
+# real cause — because `systemd` does not record a `systemctl start`'s
+# CALLING process anywhere this unit (or its journal) can read.
+#
+# `$COORD_ROLL_INVOKER`, read by `coord release nightly-window` itself and
+# journalled as `WindowRecord.invoked_by` (`coord release window-history`),
+# is the closest honest answer available without deeper systemd
+# instrumentation:
+#   * `Environment=` below sets the STATIC default, "timer-or-manual" —
+#     covers BOTH this unit's own `.timer` firing it AND a human running
+#     `systemctl --user start coord-release-window.service` by hand; those
+#     two genuinely cannot be told apart from inside the unit.
+#   * `coord.commands.drive_queue._fire_pending_roll` (the tick's own
+#     trigger — see "TWO TRIGGERS" above) overrides it with
+#     `--setenv=COORD_ROLL_INVOKER=drive-queue-tick` on that ONE start
+#     request — a transient per-invocation override, layered on top of this
+#     unit's static default by systemd itself, never touching this file.
+# A future THIRD trigger (the open question #2889 could not close) will
+# show up as "timer-or-manual" here too, same as a human running it by
+# hand — this narrows the search space (it did NOT come from the tick) but
+# does not, on its own, distinguish the timer from a person. If that
+# distinction becomes necessary, `coord-release-window.timer`'s own unit can
+# set a THIRD value via its own `Environment=`/`--setenv=` the same way.
 
 [Unit]
 Description=Release window: set/fire the #2587 roll-pending marker on the daemon host
@@ -181,6 +211,11 @@ Wants=network-online.target
 
 [Service]
 Type=oneshot
+# #2889 item 4: the static default this run's `$COORD_ROLL_INVOKER` reads
+# back as `WindowRecord.invoked_by` unless `_fire_pending_roll`'s own
+# `--setenv=` overrides it for that one invocation — see the "INVOKER"
+# section above for the full mechanism and its honest limits.
+Environment=COORD_ROLL_INVOKER=timer-or-manual
 # No --force, ever (trap 1) — coord/commands/release.py's
 # release_nightly_window never passes it to the `coord release propagate`
 # it shells out to when firing an already-pending marker. A fleet that is