code-coordinator 0.5.758__tar.gz → 0.5.759__tar.gz

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.758
+Version: 0.5.759
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.758
+Version: 0.5.759
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

code_coordinator-0.5.759/code_coordinator.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.5.759",
+  "distance": 0,
+  "node": "g8877e507364adbe8c3051ef031849fd96e87e159",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-11-12"
+}

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/coord/commands/drive_queue.py

@@ -93,6 +93,8 @@ from coord.drive_queue import (
     find_cycle,
     fired_holds,
     flag_shadows_config_warning,
+    format_coarse_duration,
+    is_benign_queue_alert_command,
     is_dispatch_failure_reason,
     is_fix_round_ceiling_blocked,
     is_merge_gate_block_reason,
@@ -2069,15 +2071,12 @@ def _reason_age_suffix(entry: QueueEntry, now: float) -> str:
     return f" ({_age_str(max(0.0, now - entry.reason_at))} ago)"
 
 
-def _age_str(delta_seconds: float) -> str:
-    """``42s`` / ``3h`` / ``5d`` — coarse, single-unit, matches #2133's ask."""
-    if delta_seconds < 60:
-        return f"{int(delta_seconds)}s"
-    if delta_seconds < 3600:
-        return f"{int(delta_seconds // 60)}m"
-    if delta_seconds < 86400:
-        return f"{int(delta_seconds // 3600)}h"
-    return f"{int(delta_seconds // 86400)}d"
+# #3983: `coord.drive_queue.format_coarse_duration` — imported under this
+# module's own pre-existing name so every call site below (and the
+# `describe_unreachable_wait`-driven alert text) renders "how long has this
+# held" with the identical single-unit shape, never a second formatter that
+# could drift from it.
+_age_str = format_coarse_duration
 
 
 def _hold_lines(entry: QueueEntry) -> list[str]:
@@ -2566,7 +2565,7 @@ def _unreachable_wait_alert_dict(rows: list[Mapping[str, Any]]) -> dict | None:
     waits = detect_unreachable_waits(entries_from_rows(rows))
     if not waits:
         return None
-    synthetic = unreachable_wait_alert(waits)
+    synthetic = unreachable_wait_alert(waits, now=time.time())
     return {
         "reason": synthetic.reason,
         "gate_readings": " | ".join(synthetic.details),
@@ -2583,13 +2582,32 @@ def drive_queue_status(output_json: bool, config_path: Path) -> None:
 
     rows = list_drive_queue()
     counts = _counts(rows)
-    # #2944: a tick-raised alert (a HELD gate, "nothing eligible to launch")
+    # #2944: a tick-raised alert (a HELD gate, a real BLOCKED-entry remedy)
     # takes priority when one exists — it is fresher and, on a busy queue,
     # `_unreachable_wait_alert_dict` would silently win-and-hide it every
-    # time a real, actionable alert IS present. Only when the tick has
-    # nothing to say (the common, dangerous case a queue with free slots
-    # sits in for hours) does the guaranteed-false-wait check get to speak.
-    alert = _queue_alert() or _unreachable_wait_alert_dict(rows)
+    # time a real, actionable alert IS present.
+    #
+    # #3983: EXCEPT the one tick-raised shape that is itself a sentinel for
+    # "nothing is actually broken" — `QUEUE_ALERT_NO_REMEDY_COMMAND` (step
+    # 4's "nothing eligible to launch" write when every stalled entry is a
+    # benign deferral, never an actual block). The quadraui#2341 incident: a
+    # root parked `needs_decomposition` with 13 `waiting` dependents chained
+    # on it via `after=` never reaches `blocked` at all — `_resolve_prereqs`
+    # treats a `parked` pre-req as an ordinary unsatisfied deferral, not the
+    # unsatisfiable-and-escalate shape it gives a `blocked`/`failed` one — so
+    # EVERY tick re-wrote this exact no-remedy sentinel over the root's own
+    # #2944/#2978 finding, and `status` never got a chance to name it. Same
+    # benign-sentinel check `coord.notifier.collect.halted_drives` already
+    # uses to keep this shape out of the pager — one question ("is this
+    # tick-raised alert itself the thing, or just noise?"), one answer,
+    # shared by both callers.
+    tick_alert = _queue_alert()
+    if tick_alert is not None and not is_benign_queue_alert_command(
+        tick_alert.get("proposed_command")
+    ):
+        alert = tick_alert
+    else:
+        alert = _unreachable_wait_alert_dict(rows) or tick_alert
     # #3413: `alert["created_at"]` (present only for a tick-raised record —
     # `_unreachable_wait_alert_dict` is computed fresh on every call and
     # carries no timestamp of its own) now survives an unbroken same-reason

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/coord/drive_queue.py

@@ -1551,6 +1551,20 @@ QUEUE_ALERT_STAGE = "drive-queue"
 #: change — `coord drive-queue status` still reads the record unchanged.
 QUEUE_ALERT_NO_REMEDY_COMMAND = "coord drive-queue list"
 
+
+def is_benign_queue_alert_command(command: object) -> bool:
+    """Is *command* the :data:`QUEUE_ALERT_NO_REMEDY_COMMAND` sentinel?
+
+    One normalization (`str(...).strip()`), shared by `coord drive-queue
+    status` (`coord.commands.drive_queue._queue_alert`'s caller) and the
+    notifier (`coord.notifier.collect.halted_drives`) — before #3983 each
+    wrote its own comparison (one on the raw dict value, the other via
+    `str(...).strip()` plus its own repo/issue check), which happened to
+    agree only because neither had yet drifted; this is the one answer both
+    now ask the identical question of (#2096).
+    """
+    return str(command or "").strip() == QUEUE_ALERT_NO_REMEDY_COMMAND
+
 #: #3413: a DISPLAY hint only — production's `coord-drive-queue.timer` fires
 #: every 3 minutes (`OnUnitActiveSec=3min`, `deploy/coord-drive-queue.timer`),
 #: not a value this module reads from the deployed unit (there is no seam to
@@ -4196,6 +4210,16 @@ class UnreachableWait:
     is accurate even though those dependents themselves never appear in the
     result (see the predicate's own docstring for why). `0` for an entry
     nothing is chained behind.
+
+    `cause` (#3983) is :func:`classify_unreachable_wait_cause`'s verdict —
+    one of :data:`UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION`/
+    `_PREMISE_REFUSAL`/`_UPSTREAM_GAP_PARK` for a terminal park, or
+    `_DISPATCH_FAILURE`/`_NEVER_DISPATCHED` for the two shapes
+    :func:`is_unreachable_wait_shape` itself distinguishes. `reason_at` is
+    `entry.reason_at` carried straight through — the wall-clock stamp of
+    when `last_reason` (hence this wait) was written, so a renderer can
+    show how long the row has sat this way instead of a bare, undated
+    reason (#2133's rule, applied here).
     """
 
     key: str
@@ -4203,6 +4227,8 @@ class UnreachableWait:
     deferrals: int
     last_reason: str
     dependents: int = 0
+    cause: str = ""
+    reason_at: float | None = None
 
 
 def is_unreachable_wait_shape(entry: QueueEntry) -> bool:
@@ -4273,6 +4299,135 @@ def is_unreachable_wait_shape(entry: QueueEntry) -> bool:
     return entry.state == STATE_BLOCKED and _is_dispatch_failure_reason(entry.last_reason)
 
 
+# #3983: the cause label `classify_unreachable_wait_cause` assigns a row
+# matching `is_unreachable_wait_shape`. The three terminal-park values are
+# the SAME strings `coord.queue_diagnose` names
+# `CAUSE_NEEDS_DECOMPOSITION`/`CAUSE_PREMISE_REFUSAL`/`CAUSE_UPSTREAM_GAP_PARK`
+# — deliberately not imported from there: `coord.queue_diagnose` imports
+# `QueueEntry`/`is_unreachable_wait_shape` FROM this module, so importing its
+# constants back here would cycle. Keeping the literal text identical is
+# what lets a reader compare `coord drive-queue status`'s alert and `coord
+# drive-queue diagnose`'s verdict for the same row without translating
+# between two vocabularies for the same finding (#2096).
+UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION = "needs-decomposition"
+UNREACHABLE_WAIT_CAUSE_PREMISE_REFUSAL = "premise-refusal"
+UNREACHABLE_WAIT_CAUSE_UPSTREAM_GAP_PARK = "upstream-gap-park"
+#: `attempts > 0`, exhausted the retry budget with no #2273 dispatch-layer
+#: assignment ever created — the second `is_unreachable_wait_shape` branch.
+UNREACHABLE_WAIT_CAUSE_DISPATCH_FAILURE = "dispatch-failure"
+#: `attempts == 0`, no terminal-park marker — the original #2944 shape:
+#: nothing has ever tried to dispatch this row at all.
+UNREACHABLE_WAIT_CAUSE_NEVER_DISPATCHED = "never-dispatched"
+
+#: The three causes a worker/park-writer already asserted are terminal —
+#: "no fix-it leg or relaunch can resolve this" is the row's OWN last_reason
+#: text, not a guess this module is making. See
+#: :func:`unreachable_wait_remedy_command` for why that rules out the
+#: blanket `remove && add` requeue for exactly these three.
+_UNREACHABLE_WAIT_TERMINAL_PARK_CAUSES = frozenset(
+    {
+        UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION,
+        UNREACHABLE_WAIT_CAUSE_PREMISE_REFUSAL,
+        UNREACHABLE_WAIT_CAUSE_UPSTREAM_GAP_PARK,
+    }
+)
+
+_UNREACHABLE_WAIT_CAUSE_LABEL: dict[str, str] = {
+    UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION: "needs-decomposition park (#3931)",
+    UNREACHABLE_WAIT_CAUSE_PREMISE_REFUSAL: "premise-refusal park (#3164)",
+    UNREACHABLE_WAIT_CAUSE_UPSTREAM_GAP_PARK: "upstream-gap park (#3977)",
+    UNREACHABLE_WAIT_CAUSE_DISPATCH_FAILURE: "exhausted dispatch-layer death (#2273)",
+    UNREACHABLE_WAIT_CAUSE_NEVER_DISPATCHED: "never dispatched",
+}
+
+#: The manual next-step for a cause no blind command can fix — embedded
+#: straight into :func:`describe_unreachable_wait` so the reason text stays
+#: self-contained even where :func:`unreachable_wait_remedy_command` can only
+#: offer a read-only fallback (#3983).
+_UNREACHABLE_WAIT_CAUSE_MANUAL_NOTE: dict[str, str] = {
+    UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION: (
+        "not fixable by any fix-it leg or relaunch — decompose the issue "
+        "into the proposed sub-issues and clear the row by hand"
+    ),
+    UNREACHABLE_WAIT_CAUSE_PREMISE_REFUSAL: (
+        "not fixable by a requeue alone — run `coord drive-queue "
+        "clear-refusal` once the missing prerequisite is confirmed, THEN "
+        "`coord drive-queue remove` + `add`"
+    ),
+    UNREACHABLE_WAIT_CAUSE_UPSTREAM_GAP_PARK: (
+        "not fixable by any fix-it leg or relaunch — clear the row by hand "
+        "once the release lands or the gap is confirmed"
+    ),
+}
+
+
+def classify_unreachable_wait_cause(entry: QueueEntry) -> str:
+    """Which `is_unreachable_wait_shape` branch explains *entry*'s wedge.
+
+    Read straight off `entry.last_reason` via the same marker predicates
+    `coord.drive_queue`'s own parked pre-pass and
+    `coord.queue_diagnose._rule_terminal_park` already trust for this text
+    — never a live probe, so this is exactly as cheap as the shape check
+    itself (#2096: one question, one answer, asked the same way everywhere).
+    Callers are expected to gate on :func:`is_unreachable_wait_shape` first;
+    called on a row that fails that shape, this still returns a value
+    rather than raising (the two dispatch-shaped labels degrade safely for
+    an unmatched row — see :func:`unreachable_wait_remedy_command`, which
+    treats an unrecognised cause the same as "never dispatched").
+    """
+    reason = entry.last_reason
+    if is_premise_refusal_reason(reason):
+        return UNREACHABLE_WAIT_CAUSE_PREMISE_REFUSAL
+    if is_needs_decomposition_reason(reason):
+        return UNREACHABLE_WAIT_CAUSE_NEEDS_DECOMPOSITION
+    if is_upstream_gap_park_reason(reason):
+        return UNREACHABLE_WAIT_CAUSE_UPSTREAM_GAP_PARK
+    if entry.attempts and _is_dispatch_failure_reason(reason):
+        return UNREACHABLE_WAIT_CAUSE_DISPATCH_FAILURE
+    return UNREACHABLE_WAIT_CAUSE_NEVER_DISPATCHED
+
+
+def unreachable_wait_remedy_command(w: UnreachableWait) -> str:
+    """The one-line fix for *w*, picked by `.cause` (#3983) — never the
+    blanket `coord drive-queue remove && add` requeue for a cause the row's
+    OWN last_reason already asserts no relaunch can resolve.
+
+    Mirrors :func:`merge_gate_remedy_command`'s #3016 rule for the same
+    reason: a wrong "run this" is worse than no recommendation, so a cause
+    with no blind-safe command falls back to the identical read-only
+    inspect pattern (`coord drive-queue list`, scoped to the entry's repo)
+    rather than ever proposing `remove`+`add` for a needs-decomposition,
+    premise-refusal or upstream-gap park — relaunching one of those just
+    re-runs the drive straight back into the identical park (the #3922
+    needs-decomposition incident this guards against).
+    """
+    parsed = parse_key(w.key)
+    repo, issue = parsed if parsed is not None else ("", 0)
+    if w.cause in _UNREACHABLE_WAIT_TERMINAL_PARK_CAUSES:
+        return f"coord drive-queue list --repo {repo}"
+    return (
+        f"coord drive-queue remove {repo} {issue} && "
+        f"coord drive-queue add {repo} {issue}"
+    )
+
+
+def format_coarse_duration(delta_seconds: float) -> str:
+    """``42s`` / ``3h`` / ``5d`` — coarse, single-unit (#2133's ask).
+
+    Shared by :func:`describe_unreachable_wait` and
+    `coord.commands.drive_queue`'s own age renderings (imported from here
+    rather than re-implemented) so "how long has this held" always reads
+    the same shape everywhere it is shown.
+    """
+    if delta_seconds < 60:
+        return f"{int(delta_seconds)}s"
+    if delta_seconds < 3600:
+        return f"{int(delta_seconds // 60)}m"
+    if delta_seconds < 86400:
+        return f"{int(delta_seconds // 3600)}h"
+    return f"{int(delta_seconds // 86400)}d"
+
+
 def is_unreachable_wait(
     entry: QueueEntry,
     *,
@@ -4327,21 +4482,74 @@ def detect_unreachable_waits(
                 deferrals=e.deferrals,
                 last_reason=e.last_reason,
                 dependents=dependents_of.get(e.key, 0),
+                cause=classify_unreachable_wait_cause(e),
+                reason_at=e.reason_at,
             )
         )
     return waits
 
 
-def unreachable_wait_alert(waits: Sequence[UnreachableWait]) -> QueueAlert:
+def describe_unreachable_wait(w: UnreachableWait, *, now: float | None = None) -> str:
+    """Render one :class:`UnreachableWait` — the entry, its cause (#3983),
+    how long it has sat this way, and (#2978) its dependent count when
+    non-zero, naming WHY those dependents need no remedy of their own (they
+    self-heal on #2756 the moment the root clears, so a reader must not
+    read this as "fix these too").
+
+    *now* is the caller's wall clock: omitted (the default), the age clause
+    is left out rather than guessed — the same degrade `_reason_age_suffix`
+    uses for a row with no `reason_at` at all. Every live caller has a
+    clock available (`time.time()` for the CLI/notifier, `ctx.now` for the
+    health probe) and is expected to pass it; only a clockless test exercises
+    the bare form.
+
+    One rendering, shared by :func:`unreachable_wait_alert` (the `coord
+    drive-queue status` `alert:` line), `coord doctor`'s WARN sample
+    (:mod:`coord.health.checks.wedged_drive_queue`) and (#3983)
+    :mod:`coord.notifier.collect`'s own per-root ``HaltedDrive`` reason —
+    so the CLI, the health probe and the pager phrase the identical finding
+    identically rather than drifting apart (#2096).
+    """
+    age = ""
+    if now is not None and w.reason_at is not None:
+        age = f", stuck {format_coarse_duration(max(0.0, now - w.reason_at))}"
+    cause_label = _UNREACHABLE_WAIT_CAUSE_LABEL.get(w.cause, w.cause or "unknown cause")
+    base = f"{w.key} ({w.state}, {cause_label}, {w.deferrals} deferrals{age})"
+    notes: list[str] = []
+    manual_note = _UNREACHABLE_WAIT_CAUSE_MANUAL_NOTE.get(w.cause)
+    if manual_note:
+        notes.append(manual_note)
+    if w.dependents:
+        plural = "y" if w.dependents == 1 else "ies"
+        notes.append(
+            f"{w.dependents} dependent entr{plural} chained behind it will "
+            "self-heal automatically once this clears (#2756), no remedy "
+            "needed on them"
+        )
+    if notes:
+        base += " — " + "; ".join(notes)
+    return base
+
+
+def unreachable_wait_alert(
+    waits: Sequence[UnreachableWait], *, now: float | None = None
+) -> QueueAlert:
     """The queue-level record #2944's predicate raises, in :func:`_hold_alert`'s
     shape — a message naming the entry (or entries), the evidence that
-    qualified it, and the ONE remedy that actually clears it.
+    qualified it, and the remedy that actually clears it.
+
+    #3983: the remedy is picked PER ENTRY by :func:`unreachable_wait_remedy_command`
+    — never the blanket ``remove`` + ``add`` for a needs-decomposition/
+    premise-refusal/upstream-gap park, each of which the row's own
+    last_reason already asserts no relaunch can resolve (see that
+    function's docstring). The top-level ``command`` field mirrors the
+    single entry's remedy when there is exactly one; with several entries
+    of possibly different causes there is no one command that is safe for
+    all of them, so it falls back to the same read-only ``coord
+    drive-queue list`` #3016's ``merge_gate_remedy_command`` falls back to
+    — the per-entry remedy is still named in ``details`` and inline in
+    ``reason`` via :func:`describe_unreachable_wait`.
 
-    ``add`` alone does not clear ``blocked`` (the queue list already
-    documents this as a footgun elsewhere) — only ``remove`` then ``add``
-    does, so that is the command surfaced here rather than the generic
-    ``coord drive-queue list`` other alerts fall back on; naming the wrong
-    remedy on a state this non-obvious defeats the point of alerting at all.
     Named INLINE in ``reason`` (not only in ``command``, which
     ``drive-queue status``'s plain-text rendering never echoes — only its
     ``gate_readings``/``details`` lines are printed) so the remedy is visible
@@ -4358,42 +4566,40 @@ def unreachable_wait_alert(waits: Sequence[UnreachableWait]) -> QueueAlert:
     that were never named — but easy to mis-generalize to "the dependents
     too" by an operator who does not already know #2756 self-heals them.
     """
-    def _name(w: UnreachableWait) -> str:
-        base = f"{w.key} ({w.state}, {w.deferrals} deferrals)"
-        if w.dependents:
-            plural = "ies" if w.dependents != 1 else "y"
-            base += (
-                f" — {w.dependents} dependent entr{plural} chained behind it "
-                "will self-heal automatically once this clears (#2756), no "
-                "remedy needed on them"
-            )
-        return base
-
-    names = ", ".join(_name(w) for w in waits[:5])
+    names = ", ".join(describe_unreachable_wait(w, now=now) for w in waits[:5])
     more = ", ..." if len(waits) > 5 else ""
     plural = len(waits) != 1
+    if len(waits) == 1:
+        command = unreachable_wait_remedy_command(waits[0])
+    elif waits and all(
+        w.cause not in _UNREACHABLE_WAIT_TERMINAL_PARK_CAUSES for w in waits
+    ):
+        command = (
+            "coord drive-queue remove <repo> <issue> && "
+            "coord drive-queue add <repo> <issue> — for each ROOT entry "
+            "named above; any dependents chained behind it self-heal on "
+            "their own (#2756)"
+        )
+    else:
+        command = "coord drive-queue list  # remedy differs per entry — see above"
     return QueueAlert(
         reason=(
             f"{len(waits)} queue entr{'ies' if plural else 'y'} stuck in a "
             "guaranteed-false wait — blocked/parked with no branch/PR/"
             "merge-queue row any sweep could ever act on (never dispatched, "
             "or every dispatch attempt died before creating an assignment); "
-            f"no sweep can ever clear this on its own: {names}{more}. Fix: "
-            "coord drive-queue remove <repo> <issue> && "
-            "coord drive-queue add <repo> <issue> for each ROOT entry named "
-            "above — never on a dependent chained behind one."
+            f"no sweep can ever clear this on its own: {names}{more}. Fix "
+            "is named per entry above — a needs-decomposition/"
+            "premise-refusal/upstream-gap park is NEVER cleared by `remove`"
+            "+`add`, only an ordinary never-dispatched/dispatch-failure root is."
         ),
         details=tuple(
-            f"{w.key}: {w.state}, {w.deferrals} deferrals, "
-            f"dependents={w.dependents}, last_reason={w.last_reason!r}"
+            f"{w.key}: {w.state}, cause={w.cause}, {w.deferrals} deferrals, "
+            f"dependents={w.dependents}, last_reason={w.last_reason!r}, "
+            f"fix={unreachable_wait_remedy_command(w)}"
             for w in waits
         ),
-        command=(
-            "coord drive-queue remove <repo> <issue> && "
-            "coord drive-queue add <repo> <issue> — for each ROOT entry "
-            "named above; any dependents chained behind it self-heal on "
-            "their own (#2756)"
-        ),
+        command=command,
     )
 
 

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/coord/health/checks/wedged_drive_queue.py

@@ -33,9 +33,13 @@ that produced the #2900/#2907 incident) and #2230 (the sweep's origin): both
 are about *why* a row gets wedged this way; this check is about the row
 being wedged *at all*, whatever sweep — present or future — produced it.
 
-WARN, not CRIT: the fix costs an operator ~10 seconds (``coord drive-queue
-remove`` + ``add``) once they know, and this check exists to make sure they
-know — it is not itself an outage.
+WARN, not CRIT: for the never-dispatched/dispatch-failure shapes the fix
+costs an operator ~10 seconds (``coord drive-queue remove`` + ``add``) once
+they know; for the three terminal-park causes (#3983 —
+needs-decomposition/premise-refusal/upstream-gap-park) it costs a human
+decision instead (decompose the issue, confirm the prerequisite, or wait for
+the release), never that requeue — either way this check exists to make
+sure an operator knows, it is not itself an outage.
 
 ``cost=COST_NETWORK``, not ``COST_CHEAP``: ``coord.state.list_drive_queue``
 routes to the daemon over HTTP whenever ``board_service`` is set, which is
@@ -76,10 +80,11 @@ from coord.health.registry import COST_NETWORK, check
     ),
 )
 def probe_wedged_drive_queue(ctx: HealthContext) -> CheckResult:
-    del ctx  # unused: this check reads local queue state directly, not config
     from coord.drive_queue import (  # noqa: PLC0415
+        describe_unreachable_wait,
         detect_unreachable_waits,
         entries_from_rows,
+        unreachable_wait_remedy_command,
     )
     from coord.state import list_drive_queue  # noqa: PLC0415
 
@@ -103,11 +108,12 @@ def probe_wedged_drive_queue(ctx: HealthContext) -> CheckResult:
             headroom="0 wedged drive-queue entries",
         )
 
+    # #3983: `describe_unreachable_wait` — the SAME rendering `coord
+    # drive-queue status`'s `alert:` line and the notifier's pager entry
+    # use — rather than this probe building its own text for the identical
+    # finding (#2096: one question, one answer, three surfaces).
     sample = ", ".join(
-        f"{w.key} ({w.state}, {w.deferrals} deferrals"
-        + (f", {w.dependents} dependents self-heal (#2756)" if w.dependents else "")
-        + ")"
-        for w in waits[:5]
+        describe_unreachable_wait(w, now=ctx.now) for w in waits[:5]
     )
     return CheckResult(
         check_id="wedged_drive_queue",
@@ -119,11 +125,13 @@ def probe_wedged_drive_queue(ctx: HealthContext) -> CheckResult:
         ),
         detail=(
             f"e.g. {sample}" + (", ..." if len(waits) > 5 else "")
-            + " — no branch/PR/merge-queue row any sweep could ever act on "
-            "(never dispatched, or every dispatch attempt died before "
-            "creating an assignment); cannot clear on its own. fix (root "
-            "only — any dependents self-heal, #2756): coord drive-queue "
-            "remove <repo> <issue> && coord drive-queue add <repo> <issue>"
+            + " — fix per entry (never a blanket command — a "
+            "needs-decomposition/premise-refusal/upstream-gap park is NOT "
+            "cleared by `remove`+`add`): "
+            + "; ".join(
+                f"{w.key}: {unreachable_wait_remedy_command(w)}"
+                for w in waits[:5]
+            )
         ),
         threshold=(
             "warn when any entry is blocked/parked with no branch/PR a sweep "
@@ -136,6 +144,7 @@ def probe_wedged_drive_queue(ctx: HealthContext) -> CheckResult:
                 {
                     "key": w.key,
                     "state": w.state,
+                    "cause": w.cause,
                     "deferrals": w.deferrals,
                     "dependents": w.dependents,
                     "last_reason": w.last_reason,

{code_coordinator-0.5.758 → code_coordinator-0.5.759}/coord/notifier/collect.py

@@ -296,11 +296,37 @@ def halted_drives(*, notifier_state: NotifierState, now: float) -> list[HaltedDr
     that DOES carry a real ``proposed_command`` (because step 4 found at
     least one genuinely blocked entry) is never a sentinel match here, so it
     pages like any other halted drive.
+
+    #3983: the benign-sentinel skip above is exactly the gap that let
+    quadraui#2341 page nobody for ~13h. That row's own last_reason (a #3931
+    needs-decomposition park) qualifies `coord.drive_queue.
+    detect_unreachable_waits`'s #2944/#2978 guaranteed-false-wait shape, but
+    its 13 `waiting` dependents never reach `blocked` — `_resolve_prereqs`
+    treats a `parked` pre-req as an ordinary unsatisfied deferral, never the
+    unsatisfiable one a `blocked`/`failed` pre-req gets — so step 4 kept
+    re-writing the SAME benign sentinel over the root's own finding, every
+    3-minute tick, for as long as the dependents sat `waiting`. The
+    board-doctor alert tier (#3979's CAUSE_NEEDS_DECOMPOSITION) would also
+    page this root, but ships dry-run by default (#3758) and its own
+    hourly sweep is a separate, operator-armed opt-in — not something this
+    function may assume is on. So this directly re-derives the same live
+    finding `coord drive-queue status`'s `alert:` line and `coord doctor`'s
+    `wedged_drive_queue` WARN already show, straight from the queue, with
+    the REAL (repo, issue) of the root rather than the synthetic
+    `QUEUE_ALERT_REPO`/`QUEUE_ALERT_ISSUE` key step 4 writes to — one
+    detection (`detect_unreachable_waits`), three surfaces that can never
+    say different things (#2096).
+
+    A root already carrying its OWN real escalation record (any stage) is
+    skipped here — that record is richer (board-doctor's cause-specific
+    diagnosis, when it ran) and must not be shadowed by this generic
+    fallback finding the identical row a second time under a different
+    reason string.
     """
     from coord.drive_queue import (  # noqa: PLC0415
         QUEUE_ALERT_ISSUE,
-        QUEUE_ALERT_NO_REMEDY_COMMAND,
         QUEUE_ALERT_REPO,
+        is_benign_queue_alert_command,
     )
 
     try:
@@ -309,10 +335,11 @@ def halted_drives(*, notifier_state: NotifierState, now: float) -> list[HaltedDr
         records = list_drive_escalations()
     except Exception as exc:  # noqa: BLE001
         log.debug("notifier: drive escalations unavailable (%s)", exc)
-        return []
+        records = []
 
     urgent = urgent_keys(notifier_state, now=now)
     out: list[HaltedDrive] = []
+    escalated_keys: set[tuple[str, int]] = set()
     for record in records:
         repo = str(record.get("repo_name") or "")
         try:
@@ -323,9 +350,11 @@ def halted_drives(*, notifier_state: NotifierState, now: float) -> list[HaltedDr
         if (
             repo == QUEUE_ALERT_REPO
             and issue == QUEUE_ALERT_ISSUE
-            and command == QUEUE_ALERT_NO_REMEDY_COMMAND
+            and is_benign_queue_alert_command(command)
         ):
             continue
+        if (repo, issue) != (QUEUE_ALERT_REPO, QUEUE_ALERT_ISSUE):
+            escalated_keys.add((repo, issue))
         stage = str(record.get("stage") or "").strip()
         reason = str(record.get("reason") or "").strip()
         out.append(
@@ -337,6 +366,72 @@ def halted_drives(*, notifier_state: NotifierState, now: float) -> list[HaltedDr
                 proposed_command=command or None,
             )
         )
+
+    out.extend(
+        _unreachable_wait_halted_drives(escalated_keys, urgent=urgent, now=now)
+    )
+    return out
+
+
+def _unreachable_wait_halted_drives(
+    escalated_keys: set[tuple[str, int]], *, urgent: set[str], now: float
+) -> list[HaltedDrive]:
+    """#3983: live `detect_unreachable_waits` roots, as `HaltedDrive`s.
+
+    ``proposed_command`` is picked PER ROOT by
+    `coord.drive_queue.unreachable_wait_remedy_command` — never the blanket
+    `remove`+`add` requeue. For the three terminal-park causes (#3979's
+    `needs-decomposition`/`premise-refusal`/`upstream-gap-park`) that
+    requeue is actively wrong: the row's own `last_reason` already asserts
+    no relaunch can resolve it, and a requeue just relaunches the drive
+    straight back into re-emitting the identical park (the #3922
+    needs-decomposition incident `coord.drive_queue.
+    unreachable_wait_remedy_command`'s own docstring names). Those three
+    get the same read-only `coord drive-queue list` fallback #3016's
+    `merge_gate_remedy_command` uses for its own no-safe-command shapes
+    instead — the manual next step is still named inline in ``reason`` via
+    `describe_unreachable_wait`.
+
+    Fails open to ``[]`` like every other source in this module — a queue
+    read that errors must degrade silently, never take the whole notifier
+    tick down with it.
+    """
+    from coord.drive_queue import (  # noqa: PLC0415
+        describe_unreachable_wait,
+        detect_unreachable_waits,
+        entries_from_rows,
+        parse_key,
+        unreachable_wait_remedy_command,
+    )
+
+    try:
+        from coord.state import list_drive_queue  # noqa: PLC0415
+
+        waits = detect_unreachable_waits(entries_from_rows(list_drive_queue()))
+    except Exception as exc:  # noqa: BLE001
+        log.debug("notifier: drive queue unavailable for unreachable-wait check (%s)", exc)
+        return []
+
+    out: list[HaltedDrive] = []
+    for wait in waits:
+        parsed = parse_key(wait.key)
+        if parsed is None:
+            continue
+        repo, issue = parsed
+        if (repo, issue) in escalated_keys:
+            continue
+        out.append(
+            HaltedDrive(
+                repo=repo,
+                issue=issue,
+                reason=(
+                    "guaranteed-false wait (#2944/#2978): "
+                    + describe_unreachable_wait(wait, now=now)
+                ),
+                urgent=urgent_key(repo, issue) in urgent,
+                proposed_command=unreachable_wait_remedy_command(wait),
+            )
+        )
     return out