code-coordinator 0.5.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. code_coordinator-0.5.46.dist-info/METADATA +625 -0
  2. code_coordinator-0.5.46.dist-info/RECORD +295 -0
  3. code_coordinator-0.5.46.dist-info/WHEEL +5 -0
  4. code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
  5. code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
  6. code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
  7. coord/__init__.py +176 -0
  8. coord/_board_mapping.py +229 -0
  9. coord/acceptance.py +468 -0
  10. coord/acceptance_drivers.py +632 -0
  11. coord/agent.py +7517 -0
  12. coord/agent_app.py +1555 -0
  13. coord/agent_update.py +417 -0
  14. coord/agents/opencode/.gitignore +13 -0
  15. coord/agents/opencode/agents/work.md +129 -0
  16. coord/agents/opencode/routing.jsonc +49 -0
  17. coord/audit.py +301 -0
  18. coord/auto_loop.py +1440 -0
  19. coord/board_bool_guard.py +72 -0
  20. coord/board_service.py +141 -0
  21. coord/board_wire.py +309 -0
  22. coord/brain.py +581 -0
  23. coord/branch_model.py +214 -0
  24. coord/cargo_cache.py +258 -0
  25. coord/ci_github.py +386 -0
  26. coord/ci_store.py +560 -0
  27. coord/claim.py +353 -0
  28. coord/cli.py +454 -0
  29. coord/client.py +610 -0
  30. coord/commands/__init__.py +1 -0
  31. coord/commands/_common.py +329 -0
  32. coord/commands/acceptance.py +916 -0
  33. coord/commands/agent_ops.py +1339 -0
  34. coord/commands/audit.py +131 -0
  35. coord/commands/chat.py +320 -0
  36. coord/commands/dispatch.py +1780 -0
  37. coord/commands/dispatch_workers.py +4894 -0
  38. coord/commands/drive.py +616 -0
  39. coord/commands/drive_queue.py +1203 -0
  40. coord/commands/gate_a.py +217 -0
  41. coord/commands/gates.py +89 -0
  42. coord/commands/issues.py +681 -0
  43. coord/commands/lifecycle.py +513 -0
  44. coord/commands/merge.py +1900 -0
  45. coord/commands/milestone.py +2081 -0
  46. coord/commands/plan_followup.py +1243 -0
  47. coord/commands/plans.py +156 -0
  48. coord/commands/release.py +2232 -0
  49. coord/commands/report.py +341 -0
  50. coord/commands/review.py +1523 -0
  51. coord/commands/scorecard.py +252 -0
  52. coord/commands/sessions.py +1930 -0
  53. coord/commands/setup.py +576 -0
  54. coord/commands/status.py +2089 -0
  55. coord/commands/terminal.py +385 -0
  56. coord/commands/test_gate.py +775 -0
  57. coord/commands/tui.py +288 -0
  58. coord/comments.py +718 -0
  59. coord/config.py +3032 -0
  60. coord/conflict_fix.py +633 -0
  61. coord/dao.py +483 -0
  62. coord/dashboard/__init__.py +0 -0
  63. coord/dashboard/fixture.py +376 -0
  64. coord/dashboard/index.html +658 -0
  65. coord/dashboard/server.py +1894 -0
  66. coord/dashboard/terminal.py +382 -0
  67. coord/dashboard/webapp/.gitignore +9 -0
  68. coord/dashboard/webapp/components.json +17 -0
  69. coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
  70. coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
  71. coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
  72. coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
  73. coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
  74. coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
  75. coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
  76. coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
  77. coord/dashboard/webapp/dist/icons/icon.svg +5 -0
  78. coord/dashboard/webapp/dist/index.html +38 -0
  79. coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
  80. coord/dashboard/webapp/dist/sw.js +1 -0
  81. coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
  82. coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
  83. coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
  84. coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
  85. coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
  86. coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
  87. coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
  88. coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
  89. coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
  90. coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
  91. coord/dashboard/webapp/eslint.config.js +20 -0
  92. coord/dashboard/webapp/index.html +37 -0
  93. coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
  94. coord/dashboard/webapp/package-lock.json +10584 -0
  95. coord/dashboard/webapp/package.json +63 -0
  96. coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
  97. coord/dashboard/webapp/playwright.config.ts +93 -0
  98. coord/dashboard/webapp/postcss.config.js +6 -0
  99. coord/dashboard/webapp/public/icons/icon-192.png +0 -0
  100. coord/dashboard/webapp/public/icons/icon-512.png +0 -0
  101. coord/dashboard/webapp/public/icons/icon.svg +5 -0
  102. coord/dashboard/webapp/src/App.tsx +140 -0
  103. coord/dashboard/webapp/src/api/client.ts +199 -0
  104. coord/dashboard/webapp/src/api/generated.ts +176 -0
  105. coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
  106. coord/dashboard/webapp/src/components/Detail.tsx +800 -0
  107. coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
  108. coord/dashboard/webapp/src/components/Home.tsx +435 -0
  109. coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
  110. coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
  111. coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
  112. coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
  113. coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
  114. coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
  115. coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
  116. coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
  117. coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
  118. coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
  119. coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
  120. coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
  121. coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
  122. coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
  123. coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
  124. coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
  125. coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
  126. coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
  127. coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
  128. coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
  129. coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
  130. coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
  131. coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
  132. coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
  133. coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
  134. coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
  135. coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
  136. coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
  137. coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
  138. coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
  139. coord/dashboard/webapp/src/index.css +210 -0
  140. coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
  141. coord/dashboard/webapp/src/lib/utils.ts +6 -0
  142. coord/dashboard/webapp/src/main.tsx +46 -0
  143. coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
  144. coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
  145. coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
  146. coord/dashboard/webapp/src/realtime/connection.ts +227 -0
  147. coord/dashboard/webapp/src/realtime/events.ts +100 -0
  148. coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
  149. coord/dashboard/webapp/src/routes/paths.ts +92 -0
  150. coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
  151. coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
  152. coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
  153. coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
  154. coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
  155. coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
  156. coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
  157. coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
  158. coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
  159. coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
  160. coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
  161. coord/dashboard/webapp/src/shell/railItems.ts +105 -0
  162. coord/dashboard/webapp/src/shell/shellState.ts +174 -0
  163. coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
  164. coord/dashboard/webapp/src/test-setup.ts +41 -0
  165. coord/dashboard/webapp/src/vite-env.d.ts +2 -0
  166. coord/dashboard/webapp/tailwind.config.js +140 -0
  167. coord/dashboard/webapp/tsconfig.json +25 -0
  168. coord/dashboard/webapp/tsconfig.node.json +11 -0
  169. coord/dashboard/webapp/vite.config.ts +71 -0
  170. coord/db.py +1076 -0
  171. coord/dead_end.py +332 -0
  172. coord/deploy/README.md +33 -0
  173. coord/deploy/coord-agent.service +89 -0
  174. coord/deploy/coord-db-backup.service +60 -0
  175. coord/deploy/coord-db-backup.sh +74 -0
  176. coord/deploy/coord-db-backup.timer +18 -0
  177. coord/deploy/coord-drive-queue.service +117 -0
  178. coord/deploy/coord-drive-queue.timer +39 -0
  179. coord/deploy/coord-notify.service +48 -0
  180. coord/deploy/coord-notify.timer +24 -0
  181. coord/deploy/coord-release-propagate.service +83 -0
  182. coord/deploy/coord-release-propagate.timer +38 -0
  183. coord/deploy/coord-release-window.service +119 -0
  184. coord/deploy/coord-release-window.timer +36 -0
  185. coord/deploy/coord-serve.service +82 -0
  186. coord/deploy/coord-web-dist-build.service +43 -0
  187. coord/deploy/coord-web-dist-build.timer +36 -0
  188. coord/deploy/coord-web.service +125 -0
  189. coord/deploy_manifest.py +80 -0
  190. coord/deploy_units.py +384 -0
  191. coord/deps.py +115 -0
  192. coord/diagnose.py +1623 -0
  193. coord/dispatch.py +1009 -0
  194. coord/dist_name.py +123 -0
  195. coord/drive.py +3101 -0
  196. coord/drive_queue.py +2298 -0
  197. coord/drive_state.py +870 -0
  198. coord/events.py +381 -0
  199. coord/failure_class.py +914 -0
  200. coord/filelock.py +168 -0
  201. coord/fleet_config_health.py +300 -0
  202. coord/freshness.py +206 -0
  203. coord/gate_a.py +469 -0
  204. coord/gate_b.py +411 -0
  205. coord/gate_snapshot.py +385 -0
  206. coord/gates.py +582 -0
  207. coord/github_ops.py +1954 -0
  208. coord/goal.py +125 -0
  209. coord/graph_health.py +348 -0
  210. coord/health/__init__.py +69 -0
  211. coord/health/aggregate.py +129 -0
  212. coord/health/checks/__init__.py +13 -0
  213. coord/health/checks/agent_install.py +280 -0
  214. coord/health/checks/cargo_targets.py +171 -0
  215. coord/health/checks/claude_binary.py +65 -0
  216. coord/health/checks/deploy_lane_facts.py +458 -0
  217. coord/health/checks/disk.py +99 -0
  218. coord/health/checks/fleet_board.py +89 -0
  219. coord/health/checks/fleet_deploy_lanes.py +469 -0
  220. coord/health/checks/fleet_phantom.py +69 -0
  221. coord/health/checks/fleet_unit_drift.py +151 -0
  222. coord/health/checks/graph.py +192 -0
  223. coord/health/checks/plan_usage.py +88 -0
  224. coord/health/checks/repo_state.py +161 -0
  225. coord/health/checks/spawned_coord.py +465 -0
  226. coord/health/checks/timer_active.py +254 -0
  227. coord/health/checks/toolchain.py +547 -0
  228. coord/health/checks/unit_drift.py +648 -0
  229. coord/health/checks/unit_enablement.py +171 -0
  230. coord/health/checks/worktrees.py +96 -0
  231. coord/health/cli.py +121 -0
  232. coord/health/context.py +106 -0
  233. coord/health/fleet_snapshot.py +477 -0
  234. coord/health/models.py +250 -0
  235. coord/health/pypi.py +231 -0
  236. coord/health/registry.py +240 -0
  237. coord/health/render.py +82 -0
  238. coord/health/units.py +60 -0
  239. coord/hooks.py +106 -0
  240. coord/housekeeping.py +204 -0
  241. coord/interactive.py +4286 -0
  242. coord/issue_store.py +1496 -0
  243. coord/liveness_auditor.py +293 -0
  244. coord/machine_pause.py +755 -0
  245. coord/merge_queue.py +4681 -0
  246. coord/milestone_chat.py +600 -0
  247. coord/milestone_dispatch.py +943 -0
  248. coord/milestone_gate.py +709 -0
  249. coord/milestone_order.py +840 -0
  250. coord/mock_author.py +334 -0
  251. coord/models.py +891 -0
  252. coord/network.py +269 -0
  253. coord/new_issue_chat.py +229 -0
  254. coord/notify.py +3226 -0
  255. coord/openapi.py +404 -0
  256. coord/overlap_fence.py +133 -0
  257. coord/parentage.py +200 -0
  258. coord/parentage_github.py +58 -0
  259. coord/pipeline.py +481 -0
  260. coord/plan_parser.py +266 -0
  261. coord/plans.py +543 -0
  262. coord/platform_paths.py +43 -0
  263. coord/pr_body_lint.py +67 -0
  264. coord/prereqs.py +533 -0
  265. coord/progress.py +425 -0
  266. coord/providers/__init__.py +683 -0
  267. coord/providers/base.py +218 -0
  268. coord/providers/claude.py +284 -0
  269. coord/providers/claude_pty.py +610 -0
  270. coord/providers/opencode.py +896 -0
  271. coord/reconcile.py +2233 -0
  272. coord/refine_chat.py +485 -0
  273. coord/release_cordon.py +525 -0
  274. coord/release_propagate.py +1176 -0
  275. coord/release_verify.py +777 -0
  276. coord/release_window.py +322 -0
  277. coord/reports.py +1643 -0
  278. coord/revalidate.py +1101 -0
  279. coord/review.py +3317 -0
  280. coord/scorecard.py +484 -0
  281. coord/serve_app.py +7192 -0
  282. coord/skills/update-issue/SKILL.md +93 -0
  283. coord/smoke.py +1030 -0
  284. coord/split_work.py +210 -0
  285. coord/stage_projection.py +650 -0
  286. coord/state.py +5720 -0
  287. coord/test_author.py +1064 -0
  288. coord/test_chat.py +352 -0
  289. coord/test_orchestrator.py +494 -0
  290. coord/test_report.py +178 -0
  291. coord/tui_release.py +271 -0
  292. coord/usage.py +753 -0
  293. coord/usage_limits.py +358 -0
  294. coord/usage_rollup.py +709 -0
  295. coord/worker_events.py +954 -0
coord/drive_queue.py ADDED
@@ -0,0 +1,2298 @@
1
+ """Pure decision half of the ``coord drive`` queue (#1754, DQ-2).
2
+
3
+ Phase B of #1750. DQ-1 gave the queue a home (``drive_queue``, one row per
4
+ ``(repo, issue)``, dense 0-based ``position``, an ``after_json`` pre-req list
5
+ nothing interpreted yet). This module is what interprets it: given the queue
6
+ rows, a typed projection of the board, and a concurrency ceiling, it returns a
7
+ :class:`TickPlan` — what to reconcile, the ONE entry to launch, what to block,
8
+ what merely deferred and why, and the single queue-level alert.
9
+
10
+ STRUCTURE — the split is copied verbatim from ``coord/drive.py``, for the
11
+ reason that module's docstring gives: *"every bug the bash version shipped was
12
+ in the decision half, which is why that half is where the tests are."* Nothing
13
+ in this file runs a subprocess, opens a socket, touches the DB, or reads the
14
+ clock. ``coord/commands/drive_queue.py`` is the thin I/O shell that fetches,
15
+ calls :func:`plan_tick`, and executes what comes back. #1794 needs wall-clock
16
+ age, so the clock is *passed in* (``plan_tick(..., now=time.time())``) rather
17
+ than read here — the rule is "no ambient state", not "no time".
18
+
19
+ TWO RULES THIS FILE EXISTS TO ENFORCE, both learned the hard way:
20
+
21
+ 1. **Capacity comes from BOARD STATE, not from a session count.** ``coord
22
+ drive`` returns ``EXIT_DEADLINE`` (3) when the *observer* gives up; the
23
+ worker, test and review keep running on the fleet (#1660). Such a drive is
24
+ invisible to ``coord drive-sessions`` but is still occupying a machine. So
25
+ an entry occupies capacity when its tmux session is alive **or** it still
26
+ has a live work-like assignment on the board — see
27
+ :func:`_reconcile_running`. Getting this wrong reproduces the 2026-08-01
28
+ incident where a sequential batch became concurrent on the fleet.
29
+
30
+ 2. **Typed state, never CLI prose** (#1523 §2). Everything here reads dicts
31
+ that came off ``GET /board`` and ``coord drive-sessions --json``. Both bugs
32
+ in the ad-hoc overnight sequencer were prose-parsing and both failed
33
+ *silently*.
34
+
35
+ DELIBERATELY NOT HERE: auto-demotion. A deferral increments a counter and
36
+ records a reason; it never reorders the queue (see #1750's design note). The
37
+ head of the queue stays the head until an operator moves it.
38
+
39
+ #1757 (DEPLOY GATES) adds a third rule: **merged is not live.** An entry may
40
+ be marked ``--hold-after``, and when the tick transitions THAT entry to
41
+ ``done`` the queue stops launching — even with free capacity and a fully
42
+ eligible successor — until a human deploys and releases it. That is not a
43
+ niche case; it is the shape of every change here that crosses a deploy lane
44
+ (``docs/OPERATING_GOTCHAS.md`` opens with the matrix). A queue that models
45
+ merge but not deploy would confidently sequence work into that trap overnight.
46
+ The gate's decision half is :func:`plan_tick`'s hold resolution below; running
47
+ the optional ``resume_when`` probe is the shell's job, and its result comes
48
+ back in as data (:class:`ProbeResult`) so this file stays pure.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import json
54
+ from dataclasses import dataclass, field
55
+ from typing import Any, Iterable, Mapping, Sequence
56
+
57
+ from coord.drive_state import TERMINAL_STATUSES, WORK_LIKE
58
+ from coord.gate_a import is_gate_a_refusal_reason
59
+ from coord.merge_queue import is_ci_infra_reason, is_ci_pending_reason
60
+
61
+ # ── queue states ─────────────────────────────────────────────────────────────
62
+ #
63
+ # `waiting` and `running` are the live states; `done`/`blocked`/`failed` are
64
+ # terminal and stay in the table until an operator removes them, so
65
+ # `coord drive-queue list` doubles as a short run history (coord/db.py's
66
+ # drive_queue comment states that contract).
67
+
68
+ STATE_WAITING = "waiting"
69
+ STATE_RUNNING = "running"
70
+ STATE_DONE = "done"
71
+ STATE_BLOCKED = "blocked"
72
+ STATE_FAILED = "failed"
73
+ # #1891: a drive that died while ITS OWN issue's merge was refused for
74
+ # nothing stronger than "CI checks have not reported yet" (see
75
+ # `coord.merge_queue.is_ci_pending_reason`) — as opposed to genuinely dead,
76
+ # genuinely refused, or genuinely out of attempts. Deliberately NOT in
77
+ # `TERMINAL_QUEUE_STATES`: unlike `blocked`, this is not a state an operator
78
+ # must release — `plan_tick` re-checks it every tick (see the pre-pass in
79
+ # `plan_tick`) and flips it straight back to `waiting` — without spending an
80
+ # attempt — the moment the board shows the gate has cleared. The whole
81
+ # feature this state exists for is "one GitHub Actions outage costs zero
82
+ # interventions", so a queue read (`coord drive-queue list`/`status`) must
83
+ # render it distinctly from both `waiting` (nothing wrong) and `blocked`
84
+ # (needs a human) — see `_STATE_ORDER` in `coord/commands/drive_queue.py`.
85
+ #
86
+ # #1892 extends the SAME state to a second trigger:
87
+ # `coord.merge_queue.is_ci_infra_reason` — a CI verdict that DID arrive but
88
+ # said nothing about the code (never assigned a runner, or died before
89
+ # checkout). There the "more real time" that un-parks the entry is the
90
+ # in-flight auto-rerun (`MAX_CI_INFRA_RERUNS`) landing, not a verdict that
91
+ # simply hasn't shown up yet — but the queue-level treatment is identical:
92
+ # relaunching a fresh `coord drive` right now would just observe the same
93
+ # rerun-in-progress and wait again, so this parks instead of spending an
94
+ # attempt. See `build_board_view`'s population of `merge_ci_pending` below.
95
+ STATE_PARKED = "parked"
96
+
97
+ TERMINAL_QUEUE_STATES: frozenset[str] = frozenset(
98
+ {STATE_DONE, STATE_BLOCKED, STATE_FAILED}
99
+ )
100
+
101
+ # ── deploy-gate states (#1757) ───────────────────────────────────────────────
102
+ #
103
+ # `hold_state` is the gate's LIFECYCLE, orthogonal to the entry's queue
104
+ # `state`. A gate is `armed` from the moment the operator declares it
105
+ # (`coord drive-queue add --hold-after`, written by `enqueue_drive_queue`),
106
+ # `fired` the tick the entry reaches `done`, and `released` once a human ran
107
+ # `coord drive-queue resume` or the entry's `resume_when` probe exited 0.
108
+ # `''` means the entry carries no gate at all.
109
+ #
110
+ # The queue is held for exactly as long as SOME entry sits at `fired` — the
111
+ # release, not the entry leaving the queue, is what unblocks the successors.
112
+ HOLD_NONE = ""
113
+ HOLD_ARMED = "armed"
114
+ HOLD_FIRED = "fired"
115
+ HOLD_RELEASED = "released"
116
+
117
+ # Wall-clock ceiling for one `resume_when` run. The shell enforces it; it
118
+ # lives here so the CLI's help text, the alert prose and the test all quote one
119
+ # number. A wedged probe must never wedge the tick (a tick that stops running
120
+ # is indistinguishable from a queue with nothing to do — #1616's lesson).
121
+ RESUME_PROBE_TIMEOUT_SECONDS = 5.0
122
+
123
+ # Launch attempts a single entry gets before it is blocked and escalated. An
124
+ # attempt is only consumed when a launched drive DIED without landing the work
125
+ # — a deferral (pre-req not satisfied yet) never touches it, and neither does
126
+ # an unsatisfiable pre-req.
127
+ DEFAULT_MAX_ATTEMPTS = 2
128
+
129
+ # ── the per-repo ceiling (#1972) ─────────────────────────────────────────────
130
+ #
131
+ # `--max-parallel` is one GLOBAL counter, which makes the queue answer the
132
+ # wrong question. The hazard that forced serialisation in the first place is
133
+ # strictly INTRA-repo: a merge stales the Test verdict of every other queued
134
+ # branch in that repo, because #1479's freshness keys on the base of the
135
+ # branch's own repo. A vimcode merge cannot stale a quadraui branch. So repo
136
+ # is precisely the boundary along which parallelism is safe — within a repo is
137
+ # the risky case, across repos is nearly free.
138
+ #
139
+ # Counting one global slot conflates the two. With `--max-parallel 3` and a
140
+ # queue of 39 claude-coordinator entries followed by one quadraui entry, the
141
+ # tick launches claude-coordinator #2 and #3 — the two launches most likely to
142
+ # stale each other — and never reaches the quadraui entry that could have run
143
+ # alongside them for free. Getting the wanted behaviour meant hand-chaining
144
+ # `--after` across 38 entries: tedious, fragile, and wrong the moment the queue
145
+ # is reordered.
146
+ #
147
+ # So occupancy is counted per repo as well as globally, and an entry whose repo
148
+ # is already at this ceiling DEFERS (position unchanged, no attempt consumed,
149
+ # no escalation — a "not yet", exactly like an unsatisfied `after`). The walk
150
+ # then lands naturally on the first entry from a repo that still has headroom.
151
+ #
152
+ # The default is 1 — today's effective behaviour for the single-repo queues
153
+ # that are the common case, since `--max-parallel` itself defaults to 1. It is
154
+ # configurable rather than hardcoded because #1715 (batch revalidation) closed,
155
+ # which makes intra-repo parallelism materially less punishing than it was; 0
156
+ # disables the per-repo ceiling entirely and restores the pre-#1972 behaviour.
157
+ #
158
+ # CAVEAT worth stating where the constant lives: per-repo occupancy inherits
159
+ # rule 1 above — it is counted from BOARD state, not live sessions (#1660). A
160
+ # drive whose observer died still holds its repo's slot until something
161
+ # reconciles it. That is strictly better than before (a wedged drive now
162
+ # blocks one repo instead of the whole queue) but it is also quieter, which is
163
+ # why `render_plan` prints the per-repo breakdown and says where it came from.
164
+ DEFAULT_MAX_PARALLEL_PER_REPO = 1
165
+
166
+ # ── the startup grace window (#1794) ─────────────────────────────────────────
167
+ #
168
+ # A drive is NOT established the instant `coord drive --tmux` exits 0. #1606's
169
+ # verification proves a tmux session exists and its run log has been written
170
+ # to; it does NOT prove the drive has registered anywhere the tick can see it.
171
+ # Between the launch and the first dispatch there is a window in which the
172
+ # entry has:
173
+ #
174
+ # * no live session in `board.live_sessions` — that snapshot is a
175
+ # `tmux list-sessions` reading, and `list_drive_sessions()` returns `[]`
176
+ # for "tmux unavailable" / "no server running" / "the call timed out"
177
+ # exactly as it does for "no sessions", so one bad reading makes EVERY
178
+ # running entry look dead at once;
179
+ # * no `active_work` on the board — the drive has not dispatched yet.
180
+ #
181
+ # Before #1794 that fell straight through all three non-death branches of
182
+ # `_reconcile_running` into `retry`. On 2026-08-03 a tick 40s after a launch
183
+ # declared a healthy drive dead, spent an attempt, and launched a SECOND
184
+ # `coord drive` for the same issue. Left alone that walks the entry to
185
+ # `attempts=2/2` and `blocked`, i.e. an unattended queue parks healthy work and
186
+ # reports it as failed. The two ticks were 40s apart because DRIVE_QUEUE.md §2's
187
+ # install sequence is `systemctl --user enable --now …timer` immediately
188
+ # followed by a verification `systemctl --user start …service` — i.e. the
189
+ # documented install reliably produces the back-to-back ticks that trigger it.
190
+ #
191
+ # So an entry launched within this window is `starting`, not dead: it OCCUPIES
192
+ # capacity and is never a retry candidate. The measured startup on a loaded
193
+ # dellserver was ~2 minutes (19:13:09 launch → 19:15:22 `drive loop started`),
194
+ # and this is 5 — deliberately >2x that, and still well under the timer's
195
+ # 15-minute cadence so a genuinely dead drive is only ever delayed by ONE
196
+ # interval before the retry path sees it.
197
+ #
198
+ # The window is also applied to the LAUNCH decision (see `_startup_cooldown`),
199
+ # so no code path in the tick — not a retry, not a hand-edited row — can start
200
+ # a second `coord drive` for an issue whose last launch is this recent.
201
+ # `coord drive`'s per-issue flock stays the last line of defence; the queue no
202
+ # longer relies on it.
203
+ DRIVE_STARTUP_GRACE_SECONDS = 300.0
204
+
205
+ # ── the queue-level alert's synthetic escalation key ─────────────────────────
206
+ #
207
+ # #1754 asks for "one queue-level record per tick, written through the DQ-1
208
+ # seam OR `record_drive_escalation` with a synthetic issue key — pick one and
209
+ # state it in the code comment, don't leave both live".
210
+ #
211
+ # CHOSEN: `record_drive_escalation` under the synthetic key below. Reasons:
212
+ # the alert is exactly the shape `drive_escalations` already stores (stage +
213
+ # reason + gate readings + a proposed command), that table's UNIQUE(repo_name,
214
+ # issue_number) + ON CONFLICT DO UPDATE gives "exactly one record, replaced
215
+ # each tick" for free, and `coord escalate list` / the TUI's escalation
216
+ # plumbing pick it up with no new wire type. The alternative — a synthetic
217
+ # `drive_queue` row — would have to be filtered out of `list`, `move`,
218
+ # `plan_tick`, and the dense-position renumbering, i.e. a special case in
219
+ # every function in this file. The DQ-1 seam stays strictly "real entries".
220
+ #
221
+ # The repo name is deliberately not a valid coordinator.yml repo, so this row
222
+ # can never collide with a real issue's escalation or match a Pipeline row.
223
+ QUEUE_ALERT_REPO = "(drive-queue)"
224
+ QUEUE_ALERT_ISSUE = 0
225
+ QUEUE_ALERT_STAGE = "drive-queue"
226
+
227
+
228
+ class QueueError(ValueError):
229
+ """A queue mutation was refused before it was written.
230
+
231
+ Carries a message naming the offending issue and the violated constraint,
232
+ the same posture ``coord milestone write-order`` takes for ``## Work
233
+ order`` (``coord.milestone_order.WorkOrderError``): validate, then write —
234
+ never write and then discover.
235
+ """
236
+
237
+
238
+ # ── keys ─────────────────────────────────────────────────────────────────────
239
+
240
+
241
+ def entry_key(repo: str, issue: int) -> str:
242
+ """The fully-qualified queue/pre-req key for an issue: ``"repo#N"``.
243
+
244
+ This is the on-disk form DQ-1 stores in ``after_json`` — one column
245
+ carries a cross-repo queue with no second column.
246
+ """
247
+ return f"{repo}#{int(issue)}"
248
+
249
+
250
+ def parse_key(key: str) -> tuple[str, int] | None:
251
+ """Inverse of :func:`entry_key`; ``None`` when *key* isn't ``repo#N``.
252
+
253
+ Splits on the LAST ``#`` so a repo name containing one still parses, and
254
+ requires the tail to be a bare number.
255
+ """
256
+ repo, sep, num = str(key).rpartition("#")
257
+ if not sep or not repo or not num.isdigit():
258
+ return None
259
+ return repo, int(num)
260
+
261
+
262
+ def parse_after_spec(raw: str | Iterable[str], default_repo: str) -> list[str]:
263
+ """Normalise a ``--after`` spec into fully-qualified ``repo#N`` keys.
264
+
265
+ Accepts ``N`` or ``REPO#N``, comma-separated, and (for repeatable Click
266
+ options) an iterable of either. Bare numbers resolve against
267
+ *default_repo* — the queue is usually single-repo, and typing the repo
268
+ name twice is the kind of friction that gets a flag skipped.
269
+
270
+ Raises :class:`QueueError` on anything that isn't one of those two forms,
271
+ rather than silently dropping it (a dropped pre-req launches work early,
272
+ which is the whole failure this feature exists to prevent).
273
+ """
274
+ chunks: list[str] = []
275
+ items: Iterable[str] = [raw] if isinstance(raw, str) else raw
276
+ for item in items:
277
+ chunks.extend(str(item).split(","))
278
+
279
+ keys: list[str] = []
280
+ for chunk in chunks:
281
+ text = chunk.strip()
282
+ if not text:
283
+ continue
284
+ if text.isdigit():
285
+ keys.append(entry_key(default_repo, int(text)))
286
+ continue
287
+ parsed = parse_key(text.lstrip("#"))
288
+ if parsed is None:
289
+ raise QueueError(
290
+ f"malformed --after entry {text!r} (expected 'N' or 'REPO#N')"
291
+ )
292
+ keys.append(entry_key(*parsed))
293
+ # De-duplicate, preserving declaration order.
294
+ seen: set[str] = set()
295
+ out: list[str] = []
296
+ for key in keys:
297
+ if key not in seen:
298
+ seen.add(key)
299
+ out.append(key)
300
+ return out
301
+
302
+
303
+ # ── the queue row ────────────────────────────────────────────────────────────
304
+
305
+
306
+ @dataclass(frozen=True)
307
+ class QueueEntry:
308
+ """One ``drive_queue`` row, typed.
309
+
310
+ Built from the dicts DQ-1's ``list_drive_queue()`` returns — identical
311
+ whether they came off the local DB or the daemon's ``/drive-queue``, which
312
+ is what lets the whole tick run unchanged on a thin client.
313
+ """
314
+
315
+ repo: str
316
+ issue: int
317
+ position: int = 0
318
+ machine: str = ""
319
+ after: tuple[str, ...] = ()
320
+ state: str = STATE_WAITING
321
+ attempts: int = 0
322
+ deferrals: int = 0
323
+ last_reason: str = ""
324
+ session_name: str = ""
325
+ launched_at: float | None = None
326
+ # #1870: the short hostname of the machine whose tick launched THIS
327
+ # session — stamped alongside `session_name`/`launched_at` when the
328
+ # launch succeeds. '' for a row predating this column or hand-flipped to
329
+ # `running`, which degrades to the pre-#1870 behaviour exactly (see
330
+ # `_reconcile_running`). Liveness (`list_drive_sessions`) is always a
331
+ # LOCAL tmux read; this is what lets a tick tell "no session because it's
332
+ # dead" apart from "no session because it's not MY session to see".
333
+ launch_host: str = ""
334
+ # #1757 deploy gate. `hold_after`/`hold_reason`/`resume_when` are
335
+ # operator-declared (written by `enqueue`); `hold_state`/`hold_probes` are
336
+ # the tick's run state.
337
+ hold_after: bool = False
338
+ hold_reason: str = ""
339
+ resume_when: str = ""
340
+ hold_state: str = HOLD_NONE
341
+ hold_probes: int = 0
342
+
343
+ @property
344
+ def key(self) -> str:
345
+ return entry_key(self.repo, self.issue)
346
+
347
+ @property
348
+ def gate_reason(self) -> str:
349
+ """What to tell the operator when this entry's gate fires.
350
+
351
+ Never empty: an operator who used ``--hold-after`` without a reason
352
+ still gets a sentence naming the entry, because an alert that says
353
+ only "HELD" is one the operator has to go and reconstruct.
354
+ """
355
+ return self.hold_reason or f"deploy gate declared on {self.key}"
356
+
357
+ @classmethod
358
+ def from_row(cls, row: Mapping[str, Any]) -> "QueueEntry":
359
+ """Type one raw queue row.
360
+
361
+ ``after_json`` arrives as a real list over the wire (DQ-1 put it in
362
+ ``coord.dao._JSON_COLUMNS``) but as a JSON *string* when a caller
363
+ reads the table directly, so both are accepted; anything unparseable
364
+ degrades to ``()`` rather than blowing up the whole tick.
365
+ """
366
+ raw_after: Any = row.get("after_json")
367
+ if isinstance(raw_after, str):
368
+ try:
369
+ raw_after = json.loads(raw_after)
370
+ except (TypeError, ValueError):
371
+ raw_after = []
372
+ if not isinstance(raw_after, list):
373
+ raw_after = []
374
+ launched_at = row.get("launched_at")
375
+ return cls(
376
+ repo=str(row.get("repo_name") or ""),
377
+ issue=int(row.get("issue_number") or 0),
378
+ position=int(row.get("position") or 0),
379
+ machine=str(row.get("machine") or ""),
380
+ after=tuple(str(a) for a in raw_after),
381
+ state=str(row.get("state") or STATE_WAITING),
382
+ attempts=int(row.get("attempts") or 0),
383
+ deferrals=int(row.get("deferrals") or 0),
384
+ last_reason=str(row.get("last_reason") or ""),
385
+ session_name=str(row.get("session_name") or ""),
386
+ launched_at=None if launched_at is None else float(launched_at),
387
+ launch_host=str(row.get("launch_host") or ""),
388
+ # SQLite hands `hold_after` back as 0/1; a JSON client may send a
389
+ # real bool. `bool(...)` accepts both and, for a row written
390
+ # before #1757's migration ran, an absent key reads as False —
391
+ # i.e. no gate, which is the pre-#1757 behaviour exactly.
392
+ hold_after=bool(row.get("hold_after") or 0),
393
+ hold_reason=str(row.get("hold_reason") or ""),
394
+ resume_when=str(row.get("resume_when") or ""),
395
+ hold_state=str(row.get("hold_state") or HOLD_NONE),
396
+ hold_probes=int(row.get("hold_probes") or 0),
397
+ )
398
+
399
+
400
+ def entries_from_rows(rows: Iterable[Mapping[str, Any]]) -> list[QueueEntry]:
401
+ """Type a whole queue read, in ``position`` order."""
402
+ return sorted(
403
+ (QueueEntry.from_row(r) for r in rows), key=lambda e: (e.position, e.key)
404
+ )
405
+
406
+
407
+ # ── the board projection ─────────────────────────────────────────────────────
408
+
409
+
410
+ @dataclass(frozen=True)
411
+ class IssueFacts:
412
+ """Everything the tick needs to know about one issue, and nothing else.
413
+
414
+ All four fields come off ``GET /board`` — no ``gh`` call, no CLI prose.
415
+ ``known=False`` means the board has never heard of this issue at all
416
+ (unsynced, or a typo'd number), which is deliberately NOT the same as
417
+ "open": an unknown pre-req is unsatisfiable, an open one merely defers.
418
+ """
419
+
420
+ known: bool = False
421
+ issue_state: str = "" # "open" / "closed" / "" when the board has no row
422
+ merged: bool = False # a work-like assignment with status == 'merged'
423
+ active_work: bool = False # a NON-terminal work-like assignment
424
+ # #1891: this issue's CURRENT merge-queue entry is refused for nothing
425
+ # stronger than "CI checks have not reported yet" — see
426
+ # `build_board_view`'s population of this field for exactly which board
427
+ # sections it reads (and why it reads BOTH of them) and
428
+ # `_reconcile_running`'s `parked` outcome for the one place it changes a
429
+ # decision.
430
+ merge_ci_pending: bool = False
431
+ # The actual board/queue reason text `merge_ci_pending` was derived from
432
+ # (e.g. ``"CI running: build, lint"``) — carried alongside the bool
433
+ # purely for diagnostics, so a `parked` reconcile's `reason` can quote
434
+ # the SAME text an operator would see on `IssueState.merge_reason`
435
+ # instead of a generic synthesised sentence.
436
+ merge_ci_pending_reason: str = ""
437
+
438
+ @property
439
+ def open(self) -> bool:
440
+ return self.issue_state == "open"
441
+
442
+ @property
443
+ def closed(self) -> bool:
444
+ return self.issue_state == "closed"
445
+
446
+ @property
447
+ def landed(self) -> bool:
448
+ """The work is done, by either witness.
449
+
450
+ Both are checked because #611 leaves merged work with ``branch=None``
451
+ rows the merge projection can miss, and quadraui-style repos can merge
452
+ a PR into ``develop`` while the linked issue stays open — so neither
453
+ signal alone is reliable.
454
+ """
455
+ return self.merged or self.closed
456
+
457
+
458
+ @dataclass(frozen=True)
459
+ class BoardView:
460
+ """The whole board reduced to per-issue facts plus live drive sessions."""
461
+
462
+ issues: Mapping[str, IssueFacts] = field(default_factory=dict)
463
+ live_sessions: frozenset[str] = frozenset()
464
+
465
+ def facts(self, key: str) -> IssueFacts:
466
+ return self.issues.get(key, IssueFacts())
467
+
468
+
469
+ def build_board_view(
470
+ payload: Mapping[str, Any],
471
+ live_sessions: Iterable[Mapping[str, Any] | str] = (),
472
+ ) -> BoardView:
473
+ """Reduce a ``/board`` payload + ``drive-sessions --json`` to a :class:`BoardView`.
474
+
475
+ Pure: *payload* is whatever ``coord.drive_state.BoardFetcher.fetch()``
476
+ returned and *live_sessions* is whatever ``coord.drive.list_drive_sessions()``
477
+ returned (dicts with ``repo``/``issue``), or a plain iterable of
478
+ ``"repo#N"`` keys for tests.
479
+ """
480
+ facts: dict[str, dict[str, Any]] = {}
481
+
482
+ def slot(key: str) -> dict[str, Any]:
483
+ return facts.setdefault(
484
+ key,
485
+ {"known": True, "issue_state": "", "merged": False, "active_work": False},
486
+ )
487
+
488
+ for row in payload.get("assignments") or []:
489
+ if (row.get("type") or "") not in WORK_LIKE:
490
+ continue
491
+ repo = row.get("repo_name") or ""
492
+ number = row.get("issue_number")
493
+ if not repo or number is None:
494
+ continue
495
+ entry = slot(entry_key(repo, int(number)))
496
+ status = row.get("status") or ""
497
+ if status == "merged":
498
+ entry["merged"] = True
499
+ if status not in TERMINAL_STATUSES:
500
+ entry["active_work"] = True
501
+
502
+ for row in payload.get("issues") or []:
503
+ repo = row.get("repo_name") or ""
504
+ number = row.get("number")
505
+ if not repo or number is None:
506
+ continue
507
+ entry = slot(entry_key(repo, int(number)))
508
+ entry["issue_state"] = str(row.get("state") or "").lower()
509
+
510
+ # #1891: `merge_ci_pending` — mirrors `drive_state._merge_entry`'s OWN
511
+ # reason resolution exactly (live `merge_plan` reason, falling back to
512
+ # the raw `merge_queue` row's persisted `error` when the plan's
513
+ # re-evaluation comes back empty) rather than importing that per-issue
514
+ # function and calling it once per queue entry: this is a single O(N)
515
+ # pass over the SAME two board sections `_merge_entry` scans, building a
516
+ # dict up front the way every other fact in this function already does.
517
+ # See `coord.merge_queue.CI_PENDING_PREFIX`'s docstring for why the raw
518
+ # row is a required second read, not a belt-and-braces extra one.
519
+ plan_reasons: dict[str, str] = {}
520
+ for row in payload.get("merge_plan") or []:
521
+ repo = row.get("repo_name") or ""
522
+ number = row.get("issue_number")
523
+ if not repo or number is None:
524
+ continue
525
+ plan_reasons[entry_key(repo, int(number))] = str(row.get("reason") or "")
526
+
527
+ for row in payload.get("merge_queue") or []:
528
+ repo = row.get("repo_name") or ""
529
+ number = row.get("issue_number")
530
+ if not repo or number is None:
531
+ continue
532
+ key = entry_key(repo, int(number))
533
+ plan_reason = plan_reasons.get(key) or ""
534
+ raw_reason = str(row.get("error") or "")
535
+ reason = plan_reason or raw_reason
536
+ # #1892: same recovery `drive_state._merge_entry` applies — the
537
+ # plan's own reason is `_entry_gate_status`'s fresh re-derivation at
538
+ # board-build time, which never computes the CI_INFRA_PREFIX
539
+ # classification (it needs an extra `gh api .../jobs` call the
540
+ # board *read* path must never make — see `coord.gate_snapshot`'s
541
+ # Invariant 1). Only a LIVE `coord merge` attempt computes it and
542
+ # persists it onto the raw row. Prefer the raw reading whenever it
543
+ # carries the classification and the plan's fresher one doesn't —
544
+ # otherwise a verdictless failure would never park here at all.
545
+ if is_ci_infra_reason(raw_reason) and not is_ci_infra_reason(plan_reason):
546
+ reason = raw_reason
547
+ if is_ci_pending_reason(reason) or is_ci_infra_reason(reason):
548
+ got = slot(key)
549
+ got["merge_ci_pending"] = True
550
+ got["merge_ci_pending_reason"] = reason
551
+
552
+ sessions: set[str] = set()
553
+ for item in live_sessions:
554
+ if isinstance(item, str):
555
+ sessions.add(item)
556
+ continue
557
+ repo = item.get("repo") or ""
558
+ number = item.get("issue")
559
+ if repo and number is not None:
560
+ sessions.add(entry_key(repo, int(number)))
561
+
562
+ return BoardView(
563
+ issues={key: IssueFacts(**value) for key, value in facts.items()},
564
+ live_sessions=frozenset(sessions),
565
+ )
566
+
567
+
568
+ # ── the plan ─────────────────────────────────────────────────────────────────
569
+ #
570
+ # Every item carries an explicit `updates` mapping of DQ-1-whitelisted columns
571
+ # (see `_DRIVE_QUEUE_UPDATABLE` in coord/state.py). The shell's apply loop is
572
+ # therefore a single uniform `update_drive_queue_entry(repo, issue, **updates)`
573
+ # per item — it never re-derives a decision, and a plan with no updates is
574
+ # provably a no-op, which is what makes `--dry-run` trustworthy.
575
+
576
+
577
+ @dataclass(frozen=True)
578
+ class Reconcile:
579
+ """The resolved outcome for one ``running`` entry.
580
+
581
+ ``outcome`` is one of:
582
+
583
+ * ``alive`` — a live ``coord-drive-*`` tmux session. Occupies.
584
+ * ``starting`` — launched inside :data:`DRIVE_STARTUP_GRACE_SECONDS` and
585
+ not yet visible anywhere else (#1794). Occupies; never a death.
586
+ * ``held`` — session gone but work still ACTIVE on the board (the
587
+ #1660 observer-deadline case). Occupies; never a death.
588
+ * ``unknown`` — this entry's ``launch_host`` names a DIFFERENT machine
589
+ than the one running this tick (#1870). Liveness is always a LOCAL
590
+ tmux read, so a foreign host's session is invisible here — that is not
591
+ evidence of anything. Occupies; never a death, never a retry.
592
+ * ``done`` — merged, or the issue closed.
593
+ * ``refused`` — #1844: the drive's own exit was a PERMANENT pre-dispatch
594
+ guard refusal (``coord.drive.EXIT_DISPATCH_REFUSED``). Goes straight to
595
+ ``blocked``; costs NO attempt — pairs with a :class:`Blocked`.
596
+ * ``dead_end`` — #2019: the drive's own exit was ``coord.drive.
597
+ EXIT_DEAD_END`` — its dead-end predicate found the row terminal and
598
+ unactionable (nothing active, every stage terminal, no gate transition
599
+ available). Same disposition as ``refused`` (straight to ``blocked``,
600
+ NO attempt spent, pairs with a :class:`Blocked`); a distinct outcome
601
+ only so the journal line names the right cause.
602
+ * ``parked`` — #1891: no session, no active work, nothing landed — same
603
+ evidence as ``retry`` — but the board's OWN current read of this
604
+ entry's merge gate names nothing stronger than "CI checks have not
605
+ reported yet" (``IssueFacts.merge_ci_pending``, sourced independently
606
+ of whatever killed the drive). Goes straight to :data:`STATE_PARKED`;
607
+ costs NO attempt — a missing verdict is not a failed one, and no
608
+ number of relaunches changes it, only more real time. Re-checked
609
+ every tick by the pre-pass in :func:`plan_tick`, which flips it back
610
+ to ``waiting`` — no human, no escalation — the moment the board shows
611
+ the gate has cleared.
612
+ * ``retry`` — genuinely dead: no session, no active work, and past the
613
+ startup grace window. Costs one attempt.
614
+ * ``exhausted`` — as ``retry``, but out of attempts; pairs with a
615
+ :class:`Blocked`.
616
+ """
617
+
618
+ key: str
619
+ outcome: str # alive | starting | held | unknown | done | refused | parked | retry | exhausted
620
+ reason: str
621
+ occupies: bool = False
622
+ updates: Mapping[str, Any] = field(default_factory=dict)
623
+
624
+
625
+ @dataclass(frozen=True)
626
+ class Blocked:
627
+ """An entry to mark ``blocked`` and escalate."""
628
+
629
+ key: str
630
+ reason: str
631
+ updates: Mapping[str, Any] = field(default_factory=dict)
632
+
633
+
634
+ @dataclass(frozen=True)
635
+ class Deferral:
636
+ """An entry whose pre-reqs aren't satisfied YET. Position unchanged.
637
+
638
+ ``counted=False`` marks a REPORT-ONLY deferral: an entry the walk reached
639
+ only after a launch had already been chosen, so it never actually
640
+ competed for a free slot. Its ``updates`` are empty, so it mutates
641
+ nothing — it exists purely so ``--dry-run`` can answer "and why isn't the
642
+ rest of the queue going?" in the same breath. Counting it would ruin the
643
+ signal ``deferrals`` carries: *how many times this entry was passed over
644
+ while a slot was actually available*.
645
+
646
+ ``repo_limited=True`` marks the #1972 deferral — the entry was otherwise
647
+ fully eligible and lost its turn only because its REPO was already at
648
+ ``max_parallel_per_repo``. It is flagged rather than string-matched
649
+ because the tick has to tell that case apart from a genuine stall: a queue
650
+ whose remaining entries are all waiting on their own repo's in-flight work
651
+ is the queue working exactly as designed, so it raises no queue-level
652
+ alert — the same posture the global at-capacity return takes.
653
+
654
+ ``cordoned=True`` is #2101's twin of that flag: the entry is pinned to a
655
+ machine that is draining for a release right now. Same posture and same
656
+ reason — a drain is the fleet working, lasts minutes, and ends by itself,
657
+ so escalating it every tick is how an alert channel gets muted. A
658
+ separate flag rather than reusing ``repo_limited`` because the two produce
659
+ different prose and different remedies, and a render that blames the
660
+ repo limit for a cordon is a render that sends the operator to the wrong
661
+ knob.
662
+ """
663
+
664
+ key: str
665
+ reason: str
666
+ updates: Mapping[str, Any] = field(default_factory=dict)
667
+ counted: bool = True
668
+ repo_limited: bool = False
669
+ cordoned: bool = False
670
+
671
+ @property
672
+ def benign(self) -> bool:
673
+ """Is this a "the fleet is working" deferral rather than a stall?
674
+
675
+ The single predicate both the queue-level alert and `render_plan`
676
+ consult, so a future third benign cause cannot be added to one and
677
+ forgotten in the other.
678
+ """
679
+ return self.repo_limited or self.cordoned
680
+
681
+
682
+ @dataclass(frozen=True)
683
+ class ProbeResult:
684
+ """The outcome of ONE ``resume_when`` run, handed back in by the shell.
685
+
686
+ Exit 0 (``ok=True``) releases the gate; anything else — non-zero, a
687
+ timeout, or a command that could not be spawned at all — keeps it held.
688
+ Fail-CLOSED is the only safe default here: a gate that releases because
689
+ its probe blew up is a gate that did not exist.
690
+ """
691
+
692
+ key: str
693
+ ok: bool
694
+ detail: str = ""
695
+
696
+
697
+ @dataclass(frozen=True)
698
+ class Hold:
699
+ """One entry's deploy gate, as resolved by this tick (#1757).
700
+
701
+ ``outcome``:
702
+
703
+ * ``fired`` — the entry reached ``done`` on THIS tick and the gate has
704
+ just closed the queue. ``updates`` arms the run state.
705
+ * ``held`` — the gate was already ``fired`` and is still closed
706
+ (either no probe is declared, or the probe ran and failed).
707
+ * ``released`` — the probe exited 0; the walk continues in this same tick.
708
+
709
+ ``blocking`` is the single thing the tick acts on, so a future outcome
710
+ can be added without every caller re-deriving the rule.
711
+ """
712
+
713
+ key: str
714
+ outcome: str
715
+ reason: str
716
+ resume_when: str = ""
717
+ probes: int = 0
718
+ probe_detail: str = ""
719
+ updates: Mapping[str, Any] = field(default_factory=dict)
720
+
721
+ @property
722
+ def blocking(self) -> bool:
723
+ return self.outcome in ("fired", "held")
724
+
725
+
726
+ @dataclass(frozen=True)
727
+ class QueueAlert:
728
+ """The one queue-level record a tick may raise (see QUEUE_ALERT_REPO).
729
+
730
+ ``command`` is the proposed fix written into the escalation record. It is
731
+ carried here rather than derived in the shell so the alert's prose and its
732
+ one-key remedy are decided together — a "HELD" alert whose command says
733
+ ``coord drive-queue list`` teaches the operator to ignore the field.
734
+ """
735
+
736
+ reason: str
737
+ details: tuple[str, ...] = ()
738
+ command: str = "coord drive-queue list"
739
+
740
+
741
+ @dataclass(frozen=True)
742
+ class TickPlan:
743
+ """Everything one tick decided, and nothing it has done yet."""
744
+
745
+ reconciles: tuple[Reconcile, ...] = ()
746
+ launch: QueueEntry | None = None
747
+ blocked: tuple[Blocked, ...] = ()
748
+ deferrals: tuple[Deferral, ...] = ()
749
+ holds: tuple[Hold, ...] = ()
750
+ alert: QueueAlert | None = None
751
+ occupied: int = 0
752
+ capacity: int = 0
753
+ # #1972: the same occupancy, broken down by repo, plus the per-repo ceiling
754
+ # it is measured against. `repo_occupied` holds only repos that actually
755
+ # occupy something (a repo with no live drive is simply absent, not 0), and
756
+ # it is the PRE-launch reading — the same instant `occupied` is taken — so
757
+ # the two never disagree. `repo_capacity == 0` means no per-repo ceiling
758
+ # was applied at all, which is what a `TickPlan` built by hand (or by a
759
+ # pre-#1972 caller) gets, and what makes `render_plan` fall back to the
760
+ # original single-line capacity render.
761
+ repo_occupied: Mapping[str, int] = field(default_factory=dict)
762
+ repo_capacity: int = 0
763
+ # #2101: non-empty when THIS host is under a release cordon, in which case
764
+ # nothing launched this tick and `launch` is guaranteed None. Carried as
765
+ # the cordon's own sentence ("cordoned: draining for v0.5.31") rather than
766
+ # a bool, because a queue that stops with no stated reason is the failure
767
+ # the cordon mechanism exists to stop repeating.
768
+ cordon_reason: str = ""
769
+
770
+ @property
771
+ def free_slots(self) -> int:
772
+ return max(0, self.capacity - self.occupied)
773
+
774
+ @property
775
+ def held(self) -> Hold | None:
776
+ """The gate holding the queue shut, if any (lowest position wins)."""
777
+ for item in self.holds:
778
+ if item.blocking:
779
+ return item
780
+ return None
781
+
782
+ def writes(self) -> list[tuple[str, Mapping[str, Any]]]:
783
+ """``(key, updates)`` for every row this plan mutates, in apply order.
784
+
785
+ The launch is NOT here: its row is written by the shell only after
786
+ ``coord drive --tmux`` has confirmed a live session, so a launch that
787
+ dies immediately is recorded as a failed attempt rather than as a
788
+ running entry (#1606 makes that exit code trustworthy).
789
+
790
+ Holds come straight after reconciles: the reconcile that moved an
791
+ entry to ``done`` and the hold that fires off it touch the same row,
792
+ and the gate's run state must land after the state that triggered it.
793
+ """
794
+ out: list[tuple[str, Mapping[str, Any]]] = []
795
+ for item in (*self.reconciles, *self.holds, *self.blocked, *self.deferrals):
796
+ if item.updates:
797
+ out.append((item.key, dict(item.updates)))
798
+ return out
799
+
800
+
801
+ # ── cycle detection ──────────────────────────────────────────────────────────
802
+
803
+
804
+ def find_cycle(edges: Mapping[str, Sequence[str]]) -> list[str] | None:
805
+ """Return one cycle in *edges* (``key -> pre-req keys``), or ``None``.
806
+
807
+ Same three-colour DFS as ``coord.milestone_order._check_cycles`` — the
808
+ validation posture ``coord milestone write-order`` applies to ``## Work
809
+ order``, applied to the same shape of graph. Edges pointing outside
810
+ *edges* (a pre-req that isn't itself queued) are ignored: they cannot
811
+ close a loop.
812
+ """
813
+ white, gray, black = 0, 1, 2
814
+ color = {key: white for key in edges}
815
+
816
+ def visit(node: str, path: list[str]) -> list[str] | None:
817
+ color[node] = gray
818
+ path.append(node)
819
+ for dep in edges.get(node, ()): # noqa: SIM118 — Mapping, not dict
820
+ if dep not in color:
821
+ continue
822
+ if color[dep] == gray:
823
+ return path[path.index(dep):] + [dep]
824
+ if color[dep] == white:
825
+ found = visit(dep, path)
826
+ if found is not None:
827
+ return found
828
+ path.pop()
829
+ color[node] = black
830
+ return None
831
+
832
+ for key in edges:
833
+ if color[key] == white:
834
+ found = visit(key, [])
835
+ if found is not None:
836
+ return found
837
+ return None
838
+
839
+
840
+ def validate_enqueue(
841
+ entries: Sequence[QueueEntry],
842
+ repo: str,
843
+ issue: int,
844
+ after: Sequence[str],
845
+ ) -> None:
846
+ """Refuse an ``add`` that would be malformed, BEFORE anything is written.
847
+
848
+ Checks, in the order an operator is most likely to hit them: a self-edge,
849
+ then a cycle across the queue as it would look *after* this write (the
850
+ entry being added replaces its own current edges, because ``enqueue``
851
+ upserts). Raises :class:`QueueError`; the caller writes nothing.
852
+
853
+ A pre-req that isn't queued is NOT an error here — the point of `--after`
854
+ is often "run this after that other thing merges", and that thing may
855
+ never be queued at all. Whether such an edge is satisfiable is a *tick*
856
+ question (:func:`plan_tick`), answered against the board.
857
+ """
858
+ key = entry_key(repo, issue)
859
+ normalised = [str(a) for a in after]
860
+ if key in normalised:
861
+ raise QueueError(f"{key} cannot depend on itself")
862
+
863
+ edges: dict[str, list[str]] = {
864
+ e.key: list(e.after) for e in entries if e.key != key
865
+ }
866
+ edges[key] = normalised
867
+ cycle = find_cycle(edges)
868
+ if cycle is not None:
869
+ raise QueueError("dependency cycle: " + " -> ".join(cycle))
870
+
871
+
872
+ # ── pre-req resolution ───────────────────────────────────────────────────────
873
+
874
+
875
+ @dataclass(frozen=True)
876
+ class _Verdict:
877
+ satisfied: bool
878
+ unsatisfiable: bool = False
879
+ reason: str = ""
880
+
881
+
882
+ def _resolve_prereqs(
883
+ entry: QueueEntry,
884
+ board: BoardView,
885
+ states: Mapping[str, str],
886
+ cycle_keys: Mapping[str, str],
887
+ ) -> _Verdict:
888
+ """Decide whether *entry* may launch now.
889
+
890
+ Three outcomes, and the difference between the last two is the whole
891
+ point: an *unsatisfied* pre-req will plausibly clear on a later tick, so
892
+ the entry defers and keeps its position; an *unsatisfiable* one never
893
+ will, so waiting forever is the silent-stall failure mode this feature
894
+ exists to remove — it blocks and escalates instead.
895
+ """
896
+ if entry.key in cycle_keys:
897
+ return _Verdict(False, True, cycle_keys[entry.key])
898
+
899
+ for dep in entry.after:
900
+ facts = board.facts(dep)
901
+ if facts.landed:
902
+ continue
903
+ dep_state = states.get(dep)
904
+ if dep_state is not None:
905
+ if dep_state in (STATE_BLOCKED, STATE_FAILED):
906
+ return _Verdict(
907
+ False,
908
+ True,
909
+ f"pre-req {dep} is queued but {dep_state} — it will never satisfy",
910
+ )
911
+ return _Verdict(
912
+ False, False, f"waiting on {dep} (queued, {dep_state})"
913
+ )
914
+ if facts.open:
915
+ return _Verdict(
916
+ False, False, f"waiting on {dep} (open, not queued)"
917
+ )
918
+ if facts.active_work:
919
+ # No `issues` row (the standalone `serialize_board` payload ships
920
+ # assignments only) but live work-like assignment rows — the issue
921
+ # is demonstrably in flight, so this defers rather than blocking.
922
+ return _Verdict(
923
+ False, False, f"waiting on {dep} (work in flight, not queued)"
924
+ )
925
+ return _Verdict(
926
+ False,
927
+ True,
928
+ f"pre-req {dep} is not queued, not merged and not open on the board "
929
+ f"(unknown issue, or the board has not synced it — try `coord sync`)",
930
+ )
931
+ return _Verdict(True)
932
+
933
+
934
+ # ── reconciliation ───────────────────────────────────────────────────────────
935
+
936
+
937
+ def _startup_age(entry: QueueEntry, now: float | None) -> float | None:
938
+ """Seconds since *entry*'s drive was launched, or ``None`` when unknowable.
939
+
940
+ ``None`` — meaning "no startup grace applies" — for three distinct cases,
941
+ all of which must degrade to the pre-#1794 behaviour rather than to an
942
+ entry that can never be retried:
943
+
944
+ * the caller passed no clock (``now is None``): a pure-logic caller that
945
+ does not care about the window, e.g. a test pinning pre-req resolution;
946
+ * the row has no ``launched_at``: a row written before DQ-1 shipped the
947
+ column, or one a human flipped to ``running`` by hand;
948
+ * the stamp is in the FUTURE (negative age): a clock that jumped backwards
949
+ must not be able to pin an entry inside the grace window indefinitely.
950
+ """
951
+ if now is None or entry.launched_at is None:
952
+ return None
953
+ age = now - entry.launched_at
954
+ return age if age >= 0.0 else None
955
+
956
+
957
+ def _startup_cooldown(
958
+ entry: QueueEntry, now: float | None, grace_seconds: float
959
+ ) -> float | None:
960
+ """The entry's age when it is still inside the startup window, else ``None``.
961
+
962
+ The age is returned (rather than a bare bool) so every caller can put the
963
+ real number in its reason string — a journal line that says "launched 41s
964
+ ago" is diagnosable; one that says "still starting" is not.
965
+ """
966
+ age = _startup_age(entry, now)
967
+ if age is None or age >= grace_seconds:
968
+ return None
969
+ return age
970
+
971
+
972
+ def _reconcile_running(
973
+ entry: QueueEntry,
974
+ board: BoardView,
975
+ max_attempts: int,
976
+ *,
977
+ now: float | None = None,
978
+ grace_seconds: float = DRIVE_STARTUP_GRACE_SECONDS,
979
+ local_host: str | None = None,
980
+ exit_reasons: Mapping[str, str] | None = None,
981
+ exit_refused: Mapping[str, bool] | None = None,
982
+ exit_dead_end: Mapping[str, bool] | None = None,
983
+ ) -> tuple[Reconcile, Blocked | None]:
984
+ """Resolve one ``running`` entry against the board.
985
+
986
+ The branch ORDER is the contract; each non-death branch exists because a
987
+ real incident proved the fall-through to ``retry`` was wrong:
988
+
989
+ * ``held`` is rule 1 from this module's docstring and the reason capacity
990
+ is not a session count: ``coord drive`` exits ``EXIT_DEADLINE`` (3) when
991
+ the observer's budget runs out, but the worker/test/review it was
992
+ watching keep running on the fleet (#1660). Such an entry has no tmux
993
+ session and no merge yet — counting it as free is exactly the 2026-08-01
994
+ incident, where five expired drives were each stacked on top of.
995
+ * ``refused``/``dead_end`` are #1844/#2019: a drive that exited on a
996
+ PERMANENT pre-dispatch guard refusal — or, since #2019, on a
997
+ terminal-and-unactionable board row — is definitively finished for this
998
+ launch. Both share one branch below; only the wording differs.
999
+ Checked right after ``held``,
1000
+ BEFORE the #1870 cross-host guard and the #1794 startup grace window,
1001
+ because this evidence (the drive's own audit trail, scoped to this
1002
+ exact launch) is stronger than anything a local tmux read or the
1003
+ startup clock can offer; neither of those exists to protect a
1004
+ conclusion this certain. See the extended note below.
1005
+ * ``unknown`` is #1870: ``board.live_sessions`` is always a LOCAL tmux
1006
+ read, but the queue is fleet-global. When *entry* was launched on a
1007
+ DIFFERENT host than *local_host*, an absent local session proves
1008
+ nothing — the drive may be 47 minutes into Test on the machine that
1009
+ actually launched it. Checked AFTER ``held``/``refused`` (real
1010
+ evidence always wins) and BEFORE the grace window / death (neither of
1011
+ which may run on evidence this tick cannot see).
1012
+ * ``starting`` is #1794: a drive that has been launched but has not yet
1013
+ registered a session reading OR put work on the board is not dead, it is
1014
+ young. See :data:`DRIVE_STARTUP_GRACE_SECONDS`.
1015
+
1016
+ ``retry`` is therefore reachable only when the session is absent, no work
1017
+ is active, nothing landed, the drive's own exit was not a permanent
1018
+ refusal, the launch host is this host (or unrecorded), AND the launch is
1019
+ older than the grace window — i.e. when death is the only remaining
1020
+ explanation.
1021
+
1022
+ *local_host* is the shell's identity for the machine THIS tick is running
1023
+ on (``None`` disables the check entirely — the pre-#1870 behaviour, same
1024
+ posture as ``now=None`` disabling the grace window). An entry with no
1025
+ recorded ``launch_host`` (predates #1870, or a hand-edited row) is always
1026
+ treated as launched here, so it degrades to today's behaviour exactly.
1027
+
1028
+ #1845/#1844: "no session, no active work, nothing landed" is also exactly
1029
+ what a drive that exited *deliberately* — a clean, non-crash exit after it
1030
+ diagnosed its own blocker and gave up — looks like from here. The drive
1031
+ already wrote the true reason to the audit trail (``drive_exited``,
1032
+ ``coord.drive.Driver._drive_exit_summary``); nothing downstream of that
1033
+ write used to read it, so every one of those orderly exits was reported
1034
+ as "drive session died" — a crash where there was none. *exit_reasons*
1035
+ (keyed by :attr:`QueueEntry.key`, fetched by the shell from
1036
+ :func:`coord.audit.query_audit_log` for the current run only — never a
1037
+ stale reason from a prior attempt on the same entry) is that write,
1038
+ threaded through as data so this function stays pure.
1039
+
1040
+ *exit_refused* (same keying, same "this run only" scoping) is #1844's
1041
+ addition: ``True`` when that exit carried ``coord.drive.
1042
+ EXIT_DISPATCH_REFUSED`` rather than a generic non-zero code — i.e. a
1043
+ PERMANENT pre-dispatch guard refusal, not a transient death. That one
1044
+ boolean is the only thing that changes the state transition: an entry
1045
+ with ``exit_refused=True`` goes straight to ``blocked`` (the ``refused``
1046
+ branch above), attempts untouched, on the FIRST tick that observes it —
1047
+ never ``retry``, because nothing about waiting and relaunching can change
1048
+ a condition a retry cannot affect.
1049
+
1050
+ *exit_dead_end* (#2019) is the SAME contract for a second permanent cause:
1051
+ ``True`` when the exit carried ``coord.drive.EXIT_DEAD_END`` — the drive's
1052
+ own dead-end predicate (``coord.dead_end.detect_dead_end``) found the row
1053
+ terminal and unactionable, with nothing active on the fleet and no gate
1054
+ transition available. Relaunching a drive against an unchanged dead-end
1055
+ row reproduces the dead end exactly, so it too blocks without spending an
1056
+ attempt; only the reason wording differs from ``exit_refused``'s. Before
1057
+ #2019 this shape did not even reach here — the drive never exited, it
1058
+ counted ``no state change`` against a held tmux session, a held queue slot
1059
+ and (since #1972) a whole repo's capacity lane for 140 minutes.
1060
+
1061
+ Every other exit reason — present or
1062
+ absent, refused or not — only ever changes the WORDING below; whether the
1063
+ entry gets another attempt is otherwise unaffected by #1845/#1844 (still
1064
+ ``retry`` until ``max_attempts``, still ``exhausted`` → ``blocked`` after).
1065
+ """
1066
+ facts = board.facts(entry.key)
1067
+
1068
+ if entry.key in board.live_sessions:
1069
+ return (
1070
+ Reconcile(entry.key, "alive", "drive session is live", occupies=True),
1071
+ None,
1072
+ )
1073
+
1074
+ if facts.landed:
1075
+ witness = "merged" if facts.merged else "issue closed"
1076
+ return (
1077
+ Reconcile(
1078
+ entry.key,
1079
+ "done",
1080
+ f"drive finished ({witness})",
1081
+ occupies=False,
1082
+ updates={
1083
+ "state": STATE_DONE,
1084
+ "last_reason": f"done ({witness})",
1085
+ "session_name": None,
1086
+ },
1087
+ ),
1088
+ None,
1089
+ )
1090
+
1091
+ if facts.active_work:
1092
+ return (
1093
+ Reconcile(
1094
+ entry.key,
1095
+ "held",
1096
+ "drive session is gone but work is still ACTIVE on the board "
1097
+ "(observer deadline, #1660) — still occupying a machine",
1098
+ occupies=True,
1099
+ updates={
1100
+ "last_reason": "session gone, work still active on the board",
1101
+ },
1102
+ ),
1103
+ None,
1104
+ )
1105
+
1106
+ # #1844: a drive that exited on a PERMANENT pre-dispatch guard refusal
1107
+ # (`coord.dispatch.enforce_oracle_readiness`, `enforce_epic_dispatch_
1108
+ # guard`, or any other check `coord assign`/`coord approve-plan`/`coord
1109
+ # fix` raises a plain `ValueError` for — see `coord.drive.
1110
+ # EXIT_DISPATCH_REFUSED`'s docstring) is definitively FINISHED for this
1111
+ # launch, not merely absent from this tick's evidence. Checked before the
1112
+ # #1870 cross-host guard and the #1794 startup grace window below — both
1113
+ # of which exist only to withhold judgement on WEAK evidence (an absent
1114
+ # local tmux session proves nothing about a foreign host, or about a
1115
+ # drive that has not had time to start yet). This is the strongest
1116
+ # evidence available: the drive's own audit trail, scoped to THIS launch
1117
+ # by the shell (`since=entry.launched_at`), naming its own exit code.
1118
+ # Retrying a deterministic refusal costs a full tick cycle and changes
1119
+ # nothing — the #1817 overnight incident this issue is named for spent
1120
+ # both of its attempts on an identical, guaranteed-to-fail dispatch
1121
+ # before exhausting to `blocked` anyway. So this goes straight to
1122
+ # `blocked`, WITHOUT incrementing `attempts` — there was never anything
1123
+ # to retry.
1124
+ #
1125
+ # #2019 rides the SAME branch with a second cause: `exit_dead_end`. The
1126
+ # evidence is identically strong (the drive's own audit trail, this launch,
1127
+ # naming its own exit code) and the conclusion is identical (relaunching
1128
+ # against an unchanged row reproduces the outcome exactly), so only the
1129
+ # wording and the reported outcome differ. `exit_refused` is checked FIRST
1130
+ # purely for stability — the two codes are mutually exclusive by
1131
+ # construction (`_drive_exit_summary` records exactly one), so the order
1132
+ # is never actually load-bearing.
1133
+ own_reason = (exit_reasons or {}).get(entry.key)
1134
+
1135
+ # #2063 rides the SAME evidence as `refused` below but reaches the
1136
+ # OPPOSITE conclusion, so it is checked first. A Gate-A "no recorded
1137
+ # human sign-off" refusal is not permanent: it is an explicitly
1138
+ # operator-fixable condition with a one-command remedy (`coord gate-a
1139
+ # --approved`), and it self-clears the moment that verdict is recorded.
1140
+ # Landing it in terminal `blocked` — which nothing re-evaluates and
1141
+ # `coord drive-queue add` will not clear (#2040) — would leave the entry
1142
+ # dead AFTER the human approved, requiring an undocumented remove+add.
1143
+ # So it parks (#1891 semantics: re-checked every tick, no attempt spent),
1144
+ # and `plan_tick`'s pre-pass below un-parks it once the verdict exists.
1145
+ if own_reason and is_gate_a_refusal_reason(own_reason):
1146
+ reason = (
1147
+ f"{own_reason} — parking without spending an attempt; the queue "
1148
+ "resumes it automatically once a human records the verdict, no "
1149
+ "queue surgery needed (#2063)"
1150
+ )
1151
+ return (
1152
+ Reconcile(
1153
+ entry.key,
1154
+ "parked",
1155
+ reason,
1156
+ occupies=False,
1157
+ updates={
1158
+ "state": STATE_PARKED,
1159
+ "last_reason": reason,
1160
+ "session_name": None,
1161
+ },
1162
+ ),
1163
+ None,
1164
+ )
1165
+
1166
+ permanent: tuple[str, str] | None = None
1167
+ if own_reason and (exit_refused or {}).get(entry.key):
1168
+ permanent = (
1169
+ "refused",
1170
+ "refused by a pre-dispatch guard, which cannot change on retry "
1171
+ "(#1844); blocking without spending an attempt",
1172
+ )
1173
+ elif own_reason and (exit_dead_end or {}).get(entry.key):
1174
+ permanent = (
1175
+ "dead_end",
1176
+ "the board row is terminal and unactionable (nothing active, no "
1177
+ "gate transition available), which cannot change on retry "
1178
+ "(#2019); blocking without spending an attempt",
1179
+ )
1180
+ if permanent is not None:
1181
+ outcome, explanation = permanent
1182
+ reason = f"{own_reason} — {explanation}"
1183
+ # `Reconcile.updates` is deliberately EMPTY, same as `exhausted`
1184
+ # below — the paired `Blocked` carries every write, applied once by
1185
+ # `TickPlan.writes()`. `attempts` is absent from BOTH: there is
1186
+ # nothing to spend, unlike `exhausted`'s Blocked which stamps the
1187
+ # final attempt count.
1188
+ return (
1189
+ Reconcile(entry.key, outcome, reason, occupies=False),
1190
+ Blocked(
1191
+ entry.key,
1192
+ reason,
1193
+ updates={
1194
+ "state": STATE_BLOCKED,
1195
+ "last_reason": reason,
1196
+ "session_name": None,
1197
+ },
1198
+ ),
1199
+ )
1200
+
1201
+ if (
1202
+ local_host is not None
1203
+ and entry.launch_host
1204
+ and entry.launch_host.lower() != local_host.lower()
1205
+ ):
1206
+ # #1870. This tick's tmux read is LOCAL; it cannot see a session on
1207
+ # the host that actually launched this entry, so its absence here is
1208
+ # not evidence of anything. Fail-soft exactly like an unreachable
1209
+ # probe would: occupy the slot, touch neither `state` nor `attempts`,
1210
+ # and never relaunch — the same posture #1794 established for "tmux
1211
+ # unavailable" / "no server running" / "timed out".
1212
+ reason = (
1213
+ f"drive was launched on {entry.launch_host!r}, not this host "
1214
+ f"({local_host!r}) — liveness cannot be verified from here, so "
1215
+ f"this is UNKNOWN, not dead (#1870); still occupying a slot, no "
1216
+ f"attempt spent"
1217
+ )
1218
+ return (
1219
+ Reconcile(
1220
+ entry.key,
1221
+ "unknown",
1222
+ reason,
1223
+ occupies=True,
1224
+ updates={"last_reason": reason},
1225
+ ),
1226
+ None,
1227
+ )
1228
+
1229
+ age = _startup_cooldown(entry, now, grace_seconds)
1230
+ if age is not None:
1231
+ # #1794. Launched, but not yet visible as a session and not yet
1232
+ # visible as work. A tick that fires inside this window sees exactly
1233
+ # what a dead drive looks like, so it must not be allowed to conclude
1234
+ # anything: the entry keeps its state, keeps its attempts, and keeps
1235
+ # its slot.
1236
+ reason = (
1237
+ f"drive is still starting — launched {age:.0f}s ago, inside the "
1238
+ f"{grace_seconds:.0f}s startup grace window (#1794); "
1239
+ f"not a death, still occupying a machine"
1240
+ )
1241
+ return (
1242
+ Reconcile(
1243
+ entry.key,
1244
+ "starting",
1245
+ reason,
1246
+ occupies=True,
1247
+ updates={"last_reason": reason},
1248
+ ),
1249
+ None,
1250
+ )
1251
+
1252
+ # Past the grace window (or with no launch stamp to measure), with no
1253
+ # session, no active work and nothing landed: this entry did not land the
1254
+ # work by any board-visible path. #1845/#1844: that no longer means
1255
+ # "died" — the drive may have exited deliberately, with its own reason
1256
+ # already on the audit trail. Prefer that reason when one was recorded
1257
+ # for this run; fall back to the synthesised wording (with the launch age
1258
+ # quoted, so a journal reader can tell a genuine death from a grace
1259
+ # window that was set too short) when it wasn't — e.g. no audit row at
1260
+ # all, a crash that never reached the `drive_exited` write, or a shell
1261
+ # that failed to fetch it.
1262
+ since = _startup_age(entry, now)
1263
+ launched = f", launched {since:.0f}s ago" if since is not None else ""
1264
+ # `own_reason` was already resolved above (before the cross-host/startup
1265
+ # checks) so the `refused` branch could use it; reused here unchanged —
1266
+ # a non-refusal exit reason (a genuine death that still narrated why)
1267
+ # still wins over the synthesised wording, same as #1845.
1268
+
1269
+ # #1891: checked BEFORE the retry/exhausted computation below, and
1270
+ # deliberately independent of `own_reason`/`exit_refused` — it does not
1271
+ # matter WHY this drive is no longer visible (a deadline, a crash, a
1272
+ # machine reboot mid-wait); what matters is whether the board's OWN
1273
+ # current read of this entry's issue still shows nothing stronger than
1274
+ # "CI checks have not reported yet". Relaunching a fresh `coord drive`
1275
+ # right now would just observe the identical silence and wait again — so
1276
+ # this parks instead, without spending an attempt (mirrors `refused`
1277
+ # just above: `Reconcile.updates` carries the whole transition, no paired
1278
+ # `Blocked`, because unlike `refused` this is not a terminal condition —
1279
+ # see `plan_tick`'s pre-pass, which is what un-parks it).
1280
+ if facts.merge_ci_pending:
1281
+ reason = (
1282
+ f"{facts.merge_ci_pending_reason or 'CI checks have not reported yet'}"
1283
+ f"{launched} — parking without spending an attempt; the queue "
1284
+ "resumes it automatically once they do, no operator needed (#1891)"
1285
+ )
1286
+ return (
1287
+ Reconcile(
1288
+ entry.key,
1289
+ "parked",
1290
+ reason,
1291
+ occupies=False,
1292
+ updates={
1293
+ "state": STATE_PARKED,
1294
+ "last_reason": reason,
1295
+ "session_name": None,
1296
+ },
1297
+ ),
1298
+ None,
1299
+ )
1300
+
1301
+ attempts = entry.attempts + 1
1302
+ if attempts < max_attempts:
1303
+ if own_reason:
1304
+ reason = (
1305
+ f"{own_reason} (attempt {attempts}/{max_attempts}) — "
1306
+ f"requeued at position {entry.position}"
1307
+ )
1308
+ else:
1309
+ reason = (
1310
+ f"drive session died without landing the work"
1311
+ f"{launched} (attempt {attempts}/{max_attempts}) — requeued at "
1312
+ f"position {entry.position}"
1313
+ )
1314
+ return (
1315
+ Reconcile(
1316
+ entry.key,
1317
+ "retry",
1318
+ reason,
1319
+ occupies=False,
1320
+ updates={
1321
+ "state": STATE_WAITING,
1322
+ "attempts": attempts,
1323
+ "last_reason": reason,
1324
+ "session_name": None,
1325
+ },
1326
+ ),
1327
+ None,
1328
+ )
1329
+
1330
+ if own_reason:
1331
+ reason = (
1332
+ f"{own_reason} ({attempts}/{max_attempts} attempts) — giving up"
1333
+ )
1334
+ else:
1335
+ reason = (
1336
+ f"drive session died without landing the work"
1337
+ f"{launched} {attempts}/{max_attempts} times — giving up"
1338
+ )
1339
+ return (
1340
+ Reconcile(entry.key, "exhausted", reason, occupies=False),
1341
+ Blocked(
1342
+ entry.key,
1343
+ reason,
1344
+ updates={
1345
+ "state": STATE_BLOCKED,
1346
+ "attempts": attempts,
1347
+ "last_reason": reason,
1348
+ "session_name": None,
1349
+ },
1350
+ ),
1351
+ )
1352
+
1353
+
1354
+ # ── deploy gates (#1757) ─────────────────────────────────────────────────────
1355
+
1356
+
1357
+ def pending_probe_targets(entries: Sequence[QueueEntry]) -> list[QueueEntry]:
1358
+ """Entries whose ``resume_when`` the shell should run BEFORE this tick.
1359
+
1360
+ Only an ALREADY-``fired`` gate is probed: a gate that fires during this
1361
+ tick's own reconcile holds unconditionally for one interval, which is the
1362
+ issue's rule ("a ``fired`` hold makes each SUBSEQUENT tick run the
1363
+ command") and also the honest one — the deploy cannot have happened in the
1364
+ microseconds since the merge was observed.
1365
+
1366
+ Pure and position-ordered, so the shell has no decision left to make: it
1367
+ runs exactly this list, in this order, and hands the results back to
1368
+ :func:`plan_tick`.
1369
+ """
1370
+ return [
1371
+ e
1372
+ for e in sorted(entries, key=lambda e: (e.position, e.key))
1373
+ if e.hold_state == HOLD_FIRED and e.resume_when
1374
+ ]
1375
+
1376
+
1377
+ def fired_holds(entries: Sequence[QueueEntry]) -> list[QueueEntry]:
1378
+ """Entries whose gate has fired and is still holding the queue shut.
1379
+
1380
+ What ``coord drive-queue resume`` releases and what ``status`` reports.
1381
+ Position-ordered so "the hold" is always the same entry in both.
1382
+ """
1383
+ return [
1384
+ e
1385
+ for e in sorted(entries, key=lambda e: (e.position, e.key))
1386
+ if e.hold_state == HOLD_FIRED
1387
+ ]
1388
+
1389
+
1390
+ def _resolve_holds(
1391
+ ordered: Sequence[QueueEntry],
1392
+ reconciled_states: Mapping[str, str],
1393
+ probes: Mapping[str, ProbeResult],
1394
+ ) -> list[Hold]:
1395
+ """Fire / probe / release every gate, in position order.
1396
+
1397
+ *reconciled_states* is each entry's queue state AFTER step 1 of the tick,
1398
+ which is what makes "fires on ``done`` only" checkable here: a
1399
+ ``--hold-after`` entry that reconciled to ``blocked`` never reaches this
1400
+ branch, so it produces the existing escalation and NOT a second alert (the
1401
+ issue's explicit rule — two alerts for one condition is how an alert
1402
+ channel gets muted).
1403
+ """
1404
+ holds: list[Hold] = []
1405
+ for entry in ordered:
1406
+ if not entry.hold_after:
1407
+ continue
1408
+
1409
+ # ARMED → FIRED, the tick the entry lands. Nothing else fires a gate:
1410
+ # `blocked`/`failed` already stop the queue through the escalation
1411
+ # path, and `waiting`/`running` have not finished anything yet.
1412
+ if (
1413
+ entry.hold_state == HOLD_ARMED
1414
+ and reconciled_states.get(entry.key) == STATE_DONE
1415
+ ):
1416
+ holds.append(
1417
+ Hold(
1418
+ key=entry.key,
1419
+ outcome="fired",
1420
+ reason=entry.gate_reason,
1421
+ resume_when=entry.resume_when,
1422
+ probes=0,
1423
+ updates={"hold_state": HOLD_FIRED, "hold_probes": 0},
1424
+ )
1425
+ )
1426
+ continue
1427
+
1428
+ if entry.hold_state != HOLD_FIRED:
1429
+ # `''` (no gate yet armed), `armed` on an entry that has not
1430
+ # landed, or `released` — none of which hold anything.
1431
+ continue
1432
+
1433
+ probe = probes.get(entry.key)
1434
+ if probe is None:
1435
+ # No probe declared, or the shell did not run one. Manual resume
1436
+ # only; the count does not move, so a hold that nobody probes
1437
+ # never grows a fake attempt number.
1438
+ holds.append(
1439
+ Hold(
1440
+ key=entry.key,
1441
+ outcome="held",
1442
+ reason=entry.gate_reason,
1443
+ resume_when=entry.resume_when,
1444
+ probes=entry.hold_probes,
1445
+ )
1446
+ )
1447
+ continue
1448
+
1449
+ if probe.ok:
1450
+ holds.append(
1451
+ Hold(
1452
+ key=entry.key,
1453
+ outcome="released",
1454
+ reason=entry.gate_reason,
1455
+ resume_when=entry.resume_when,
1456
+ probes=entry.hold_probes,
1457
+ probe_detail=probe.detail,
1458
+ updates={"hold_state": HOLD_RELEASED, "hold_probes": 0},
1459
+ )
1460
+ )
1461
+ continue
1462
+
1463
+ attempts = entry.hold_probes + 1
1464
+ holds.append(
1465
+ Hold(
1466
+ key=entry.key,
1467
+ outcome="held",
1468
+ reason=entry.gate_reason,
1469
+ resume_when=entry.resume_when,
1470
+ probes=attempts,
1471
+ probe_detail=probe.detail,
1472
+ updates={"hold_probes": attempts},
1473
+ )
1474
+ )
1475
+ return holds
1476
+
1477
+
1478
+ def _norm_host(name: str | None) -> str:
1479
+ """#2101: the one host-identity normalisation this module compares by.
1480
+
1481
+ Machine names come from ``coordinator.yml`` (``dellserver``) and host
1482
+ identities from ``socket.gethostname()`` (``dellserver.local``), and a
1483
+ cordon that fails to match because of a domain suffix is a cordon that
1484
+ silently does nothing — the exact class of failure #1563 closed for
1485
+ pause. Same normalisation `coord/commands/drive_queue.py`'s
1486
+ ``_local_host_id`` already applies: short hostname, lowercased.
1487
+ """
1488
+ return str(name or "").split(".")[0].strip().lower()
1489
+
1490
+
1491
+ def _normalized_cordons(cordons: Mapping[str, str] | None) -> dict[str, str]:
1492
+ return {
1493
+ _norm_host(name): str(reason)
1494
+ for name, reason in (cordons or {}).items()
1495
+ if _norm_host(name) and reason
1496
+ }
1497
+
1498
+
1499
+ def _cordon_alert(host: str, reason: str) -> QueueAlert:
1500
+ """The queue-level record for "this host is cordoned" (#2101 trap E).
1501
+
1502
+ Mirrors :func:`_hold_alert`: a queue that stops must say why, in the same
1503
+ channel and with the same one-key remedy field, or "stopped" and "wedged"
1504
+ look identical from the outside — which is how a fleet sits eleven
1505
+ releases behind for a day with every readout silent.
1506
+ """
1507
+ return QueueAlert(
1508
+ reason=(
1509
+ f"no launch — {host} is {reason}. In-flight drives are draining; "
1510
+ "the queue resumes automatically the moment this host is rolled "
1511
+ "and uncordoned (#2101)."
1512
+ ),
1513
+ details=(
1514
+ "release cordons expire on their own if the propagate run that "
1515
+ "set one dies, so this can never wedge the queue permanently",
1516
+ ),
1517
+ command=f"coord release cordon --clear {host}",
1518
+ )
1519
+
1520
+
1521
+ def _hold_alert(hold: Hold) -> QueueAlert:
1522
+ """The one queue-level record a closed gate raises.
1523
+
1524
+ Carries the operator's own ``hold_reason`` verbatim in ``reason`` — that
1525
+ string is the entire point of the feature (it is the runbook line for the
1526
+ deploy the queue is waiting on), so it must survive into the alert without
1527
+ being summarised.
1528
+ """
1529
+ details = [f"held after {hold.key} — nothing will launch until this is released"]
1530
+ if hold.resume_when:
1531
+ outcome = (
1532
+ f"attempt {hold.probes} failed"
1533
+ if hold.probes
1534
+ else "not probed yet (fires on the next tick)"
1535
+ )
1536
+ if hold.probe_detail:
1537
+ outcome += f": {hold.probe_detail}"
1538
+ details.append(f"resume-when: {hold.resume_when} ({outcome})")
1539
+ else:
1540
+ details.append("no --resume-when probe: release manually")
1541
+ return QueueAlert(
1542
+ reason=f"QUEUE HELD — {hold.reason}",
1543
+ details=tuple(details),
1544
+ command="coord drive-queue resume",
1545
+ )
1546
+
1547
+
1548
+ # ── the tick ─────────────────────────────────────────────────────────────────
1549
+
1550
+
1551
+ def plan_tick(
1552
+ entries: Sequence[QueueEntry],
1553
+ board: BoardView,
1554
+ capacity: int,
1555
+ *,
1556
+ max_attempts: int = DEFAULT_MAX_ATTEMPTS,
1557
+ max_parallel_per_repo: int = DEFAULT_MAX_PARALLEL_PER_REPO,
1558
+ probes: Mapping[str, ProbeResult] | None = None,
1559
+ now: float | None = None,
1560
+ grace_seconds: float = DRIVE_STARTUP_GRACE_SECONDS,
1561
+ local_host: str | None = None,
1562
+ exit_reasons: Mapping[str, str] | None = None,
1563
+ exit_refused: Mapping[str, bool] | None = None,
1564
+ exit_dead_end: Mapping[str, bool] | None = None,
1565
+ gate_a_pending: Mapping[str, bool] | None = None,
1566
+ cordons: Mapping[str, str] | None = None,
1567
+ ) -> TickPlan:
1568
+ """Decide one tick. Pure; the caller executes the returned plan.
1569
+
1570
+ *capacity* is the CEILING (``--max-parallel``), not the number of free
1571
+ slots — how many slots are already occupied is a decision (rule 1 above),
1572
+ and decisions live in here, not in the shell.
1573
+
1574
+ *max_parallel_per_repo* is the SECOND ceiling (#1972), applied per repo
1575
+ after the global one: an entry whose repo already occupies this many slots
1576
+ defers, so the walk lands on the first entry from a repo with headroom.
1577
+ Both ceilings apply, global first. ``0`` disables it (pre-#1972
1578
+ behaviour); the default of 1 is per-repo serialisation, which for a
1579
+ single-repo queue at ``--max-parallel 1`` is exactly what the queue did
1580
+ before. See :data:`DEFAULT_MAX_PARALLEL_PER_REPO` for why repo is the
1581
+ right axis and what the board-derived counting means for a wedged drive.
1582
+
1583
+ *probes* maps an entry key to the :class:`ProbeResult` the shell got from
1584
+ running that entry's ``resume_when`` (see :func:`pending_probe_targets`);
1585
+ an absent key simply means no probe ran.
1586
+
1587
+ *exit_reasons* maps a ``running`` entry's key to the drive's own
1588
+ ``drive_exited`` audit summary for THIS launch (#1845/#1844) — see
1589
+ :func:`_reconcile_running` for what it changes (wording, and — when
1590
+ *exit_refused* also marks the entry — the ``retry``/``exhausted``
1591
+ decision itself) and why "for this launch" matters (a stale reason from a
1592
+ prior attempt on the same entry must never be replayed as if it explained
1593
+ the current one).
1594
+
1595
+ *exit_refused* maps the same keys to ``True`` when that exit was a
1596
+ PERMANENT pre-dispatch guard refusal (``coord.drive.
1597
+ EXIT_DISPATCH_REFUSED``) rather than a transient death (#1844). Unlike
1598
+ *exit_reasons*, this DOES change the decision: such an entry reconciles
1599
+ straight to ``blocked`` with ``attempts`` unchanged, never ``retry`` —
1600
+ see :func:`_reconcile_running`'s ``refused`` branch.
1601
+
1602
+ *exit_dead_end* is the #2019 twin: ``True`` when the exit was
1603
+ ``coord.drive.EXIT_DEAD_END`` (the row was terminal and unactionable).
1604
+ Same disposition, same branch, different wording — a relaunch against an
1605
+ unchanged dead-end row reproduces the dead end exactly, so it too costs no
1606
+ attempt.
1607
+
1608
+ *now* is the shell's ``time.time()``, passed in rather than read here (see
1609
+ the module docstring). It powers #1794's startup grace window on both
1610
+ sides of the tick: a recently-launched entry never reconciles to ``retry``
1611
+ (:func:`_reconcile_running`), and no entry is relaunched while its last
1612
+ launch is still that recent (step 4 below). ``None`` disables the window
1613
+ entirely, which is the pre-#1794 behaviour — the production shell always
1614
+ passes a real clock.
1615
+
1616
+ *local_host* is the shell's identity for the machine THIS tick is running
1617
+ on (#1870). It powers the cross-host guard in :func:`_reconcile_running`:
1618
+ an entry whose ``launch_host`` names a DIFFERENT machine reconciles to
1619
+ ``unknown`` rather than ``retry``, because this tick's tmux read cannot
1620
+ see that host at all. ``None`` disables the check entirely, the
1621
+ pre-#1870 behaviour — the production shell always passes its own
1622
+ hostname.
1623
+
1624
+ *cordons* maps a machine name to the reason it is under a #2101 release
1625
+ cordon ("cordoned: draining for v0.5.31"). Two distinct effects, because
1626
+ a drive occupies two machines:
1627
+
1628
+ * this tick's own host (*local_host*) cordoned ⇒ launch NOTHING at all,
1629
+ because ``coord drive --tmux`` starts its session HERE. This is the
1630
+ hole #2101 names outright: `coord/drive.py` checks pause only when
1631
+ routing a *worker*, so before this the main launcher walked straight
1632
+ through the cordon and the fleet could never drain.
1633
+ * an entry pinned (``--machine``) to a cordoned host ⇒ that ENTRY defers,
1634
+ position unchanged, no attempt spent; the walk moves on to the next
1635
+ entry, which may well be launchable on an uncordoned machine.
1636
+
1637
+ Reconciliation (steps 1/1b) still runs under a cordon, and that is the
1638
+ whole point: a cordon that also froze the queue's view of reality would
1639
+ leave a finished drive's `running` row pinning propagation forever — the
1640
+ #2110 deadlock, re-created by the very mechanism meant to end it. A
1641
+ cordoned tick is exactly `--reconcile-only`.
1642
+
1643
+ The algorithm, from #1754, plus #1757's step 2, #1891's step 1b, and
1644
+ #2055's extension of it:
1645
+
1646
+ 1. Reconcile every ``running`` entry (:func:`_reconcile_running`).
1647
+ 1b. Re-check every ``parked``/``blocked``/``failed`` entry against the
1648
+ CURRENT board: landed ⇒ ``done`` (#1891 for ``parked``, #2055 for
1649
+ ``blocked``/``failed``). For ``parked`` only, not-yet-landed then
1650
+ also checks the gate: cleared ⇒ ``waiting`` (falls into step 4 on
1651
+ this SAME tick); still shut ⇒ untouched, no write, nothing to
1652
+ report. ``blocked``/``failed`` that haven't landed are left alone
1653
+ entirely — this never resurrects them for dispatch, only lets a
1654
+ finished one stop claiming to be unfinished. Never spends an
1655
+ attempt either way — a missing CI verdict is not a failed one.
1656
+ 2. Resolve deploy gates (:func:`_resolve_holds`). ANY gate left closed
1657
+ returns immediately with no launch and a HELD alert — before the
1658
+ capacity check, and regardless of how eligible the rest of the queue
1659
+ is. That "even with free capacity and an eligible successor" clause is
1660
+ the entire feature: the successor is exactly the thing that must not
1661
+ run until the deploy lands.
1662
+ 3. ``free = capacity - occupied``; ``<= 0`` returns with no launch and no
1663
+ alert — being at capacity is the queue working, not a problem to
1664
+ report.
1665
+ 4. Walk ``waiting`` by ``position``, FIRST ELIGIBLE WINS: an entry still
1666
+ inside its startup grace window defers (#1794); an entry whose own
1667
+ issue is already landed (merged or closed) reconciles straight to
1668
+ ``done`` without ever launching (#1873) — checked before its `after=`
1669
+ graph, so a landed entry is never blocked or deferred on account of its
1670
+ own now-irrelevant pre-reqs; unsatisfiable blocks and escalates,
1671
+ unsatisfied defers (position unchanged); an entry whose REPO is already
1672
+ at *max_parallel_per_repo* defers too (#1972, checked LAST — a broken
1673
+ pre-req is a permanent fact and must still escalate, whatever the
1674
+ repo's occupancy is doing this tick); the first eligible entry is the
1675
+ launch. Everything after the launch is walked in REPORT-ONLY mode
1676
+ (``Deferral.counted=False``, no updates) so ``--dry-run`` can explain
1677
+ the rest of the queue — including against the launch's own repo, which
1678
+ the report-only pass counts as occupied.
1679
+ 5. No launch with at least one entry STILL genuinely waiting (deferred or
1680
+ blocked — #1873 reconciliations do not count, see below) ⇒ exactly ONE
1681
+ queue-level alert. #1972's repo-limit deferrals do not count either:
1682
+ a queue whose every remaining entry is waiting on its own repo's
1683
+ in-flight work is saturated, not stalled, and alerting on it every tick
1684
+ is how an alert channel gets muted (same reasoning as step 3).
1685
+
1686
+ An entry reconciled from ``running`` back to ``waiting`` in step 1 IS
1687
+ walked in step 4 — its attempt was already consumed, so a drive that died
1688
+ early relaunches on the same tick instead of idling a whole interval. A
1689
+ gate RELEASED in step 2 likewise falls straight through into step 4, so a
1690
+ probe that starts passing launches in the same tick rather than costing
1691
+ the queue a whole interval.
1692
+
1693
+ #1794 puts one bound on that same-tick relaunch, and it is the reason the
1694
+ grace window is checked TWICE. Step 1 can only produce a ``retry`` for an
1695
+ entry whose launch is older than *grace_seconds*, so the relaunch is only
1696
+ ever of a drive the tick is confident is gone; and step 4 refuses the
1697
+ launch outright for anything launched more recently, whatever put it back
1698
+ in ``waiting``. Between them, no single tick can start a second ``coord
1699
+ drive`` for an issue whose first one may still be coming up.
1700
+ """
1701
+ ordered = sorted(entries, key=lambda e: (e.position, e.key))
1702
+ states: dict[str, str] = {e.key: e.state for e in ordered}
1703
+ by_key = {e.key: e for e in ordered}
1704
+
1705
+ reconciles: list[Reconcile] = []
1706
+ blocked: list[Blocked] = []
1707
+ deferrals: list[Deferral] = []
1708
+ occupied = 0
1709
+ # #1972: the same count, keyed by repo. Populated from the SAME
1710
+ # `reconcile.occupies` verdict as `occupied` above — one source of truth, so
1711
+ # the per-repo view can never claim a slot the global view does not.
1712
+ repo_occupied: dict[str, int] = {}
1713
+ repo_capacity = max(0, int(max_parallel_per_repo))
1714
+
1715
+ for entry in ordered:
1716
+ if entry.state != STATE_RUNNING:
1717
+ continue
1718
+ reconcile, block = _reconcile_running(
1719
+ entry,
1720
+ board,
1721
+ max_attempts,
1722
+ now=now,
1723
+ grace_seconds=grace_seconds,
1724
+ local_host=local_host,
1725
+ exit_reasons=exit_reasons,
1726
+ exit_refused=exit_refused,
1727
+ exit_dead_end=exit_dead_end,
1728
+ )
1729
+ reconciles.append(reconcile)
1730
+ if reconcile.occupies:
1731
+ occupied += 1
1732
+ repo_occupied[entry.repo] = repo_occupied.get(entry.repo, 0) + 1
1733
+ new_state = reconcile.updates.get("state")
1734
+ if new_state:
1735
+ states[entry.key] = str(new_state)
1736
+ if block is not None:
1737
+ blocked.append(block)
1738
+ states[entry.key] = STATE_BLOCKED
1739
+
1740
+ # #1891 step 1b: re-check every `parked` entry against the CURRENT board,
1741
+ # independent of capacity/holds below — mirrors step 1's own `done` check
1742
+ # (an entry can land while parked exactly as it can while running) and,
1743
+ # like step 1, never spends an attempt either way. `entry.landed` wins
1744
+ # unconditionally over "still gated", same ordering `_reconcile_running`
1745
+ # uses for a `running` entry. A gate that CLEARED flips `states` straight
1746
+ # to `waiting` here — not `by_key`, which stays whatever DQ-1 loaded — so
1747
+ # it falls into the SAME step-4 walk below, on the SAME tick, exactly
1748
+ # like a deploy gate released in step 2 (see this function's docstring
1749
+ # for why that same-tick fall-through matters). A gate that is STILL
1750
+ # shut is left alone entirely: no reconcile, no write, nothing to
1751
+ # report — the parked row itself, rendered by `coord drive-queue list`/
1752
+ # `status`, already answers "why isn't this launching".
1753
+ #
1754
+ # #2055 extends the SAME `landed` check to `blocked` and `failed`
1755
+ # entries. `blocked`/`failed` are terminal for dispatch — the queue gave
1756
+ # up on them, and this loop must NOT resurrect them for a relaunch, the
1757
+ # way the `parked` branch below resumes to `waiting` on cleared CI. But
1758
+ # "the queue gave up" and "the work is done" are independent facts: a
1759
+ # human fixes a blocked/failed issue by hand and merges it out of band
1760
+ # exactly as often as a parked one lands while its gate is still shut.
1761
+ # Without this, that merge is invisible forever — `blocked`/`failed`
1762
+ # have no other re-check, so the board keeps reporting finished work as
1763
+ # outstanding until someone notices and runs
1764
+ # `coord drive-queue remove`. See #1956 for a live instance.
1765
+ for entry in ordered:
1766
+ if entry.state not in (STATE_PARKED, STATE_BLOCKED, STATE_FAILED):
1767
+ continue
1768
+ facts = board.facts(entry.key)
1769
+ if facts.landed:
1770
+ witness = "merged" if facts.merged else "closed"
1771
+ reason = f"done — issue already {witness} while {entry.state} (#2055)"
1772
+ reconciles.append(
1773
+ Reconcile(
1774
+ entry.key,
1775
+ "done",
1776
+ reason,
1777
+ occupies=False,
1778
+ updates={
1779
+ "state": STATE_DONE,
1780
+ "last_reason": reason,
1781
+ "session_name": None,
1782
+ },
1783
+ )
1784
+ )
1785
+ states[entry.key] = STATE_DONE
1786
+ continue
1787
+ if entry.state != STATE_PARKED:
1788
+ # `blocked`/`failed` entries are terminal for dispatch: the
1789
+ # landed check above is the only re-check they get. Never fall
1790
+ # through to the parked-only CI resume below — that would
1791
+ # relaunch a gave-up entry outside the `blocked`/`failed`
1792
+ # attempt-tracking this function's docstring describes (#2055).
1793
+ continue
1794
+ if facts.merge_ci_pending:
1795
+ continue
1796
+ # #2063: a Gate-A park is gated on a HUMAN, not on the board, so the
1797
+ # `merge_ci_pending` predicate above says nothing about it. Without
1798
+ # this branch such an entry would resume on the very next tick and
1799
+ # relaunch straight back into the identical refusal, forever — the
1800
+ # hot loop that "park, don't block" is supposed to avoid. The
1801
+ # shell resolves `gate_a_pending` by re-reading the recorded verdict
1802
+ # for the (repo, milestone) embedded in the park reason's marker (a
1803
+ # local board read, no `gh` call per entry); an entry it can't
1804
+ # resolve stays parked, which fails closed exactly like the guard.
1805
+ if is_gate_a_refusal_reason(entry.last_reason):
1806
+ if (gate_a_pending or {}).get(entry.key, True):
1807
+ continue
1808
+ reason = (
1809
+ f"Gate A sign-off recorded for {entry.key} — resuming from "
1810
+ "parked without spending an attempt (#2063)"
1811
+ )
1812
+ reconciles.append(
1813
+ Reconcile(
1814
+ entry.key,
1815
+ "resumed",
1816
+ reason,
1817
+ occupies=False,
1818
+ updates={"state": STATE_WAITING, "last_reason": reason},
1819
+ )
1820
+ )
1821
+ states[entry.key] = STATE_WAITING
1822
+ continue
1823
+ reason = (
1824
+ f"CI checks for {entry.key} have reported — resuming from "
1825
+ "parked without spending an attempt (#1891)"
1826
+ )
1827
+ reconciles.append(
1828
+ Reconcile(
1829
+ entry.key,
1830
+ "resumed",
1831
+ reason,
1832
+ occupies=False,
1833
+ updates={"state": STATE_WAITING, "last_reason": reason},
1834
+ )
1835
+ )
1836
+ states[entry.key] = STATE_WAITING
1837
+
1838
+ # #1757 step 2: deploy gates. Resolved from the POST-reconcile states, so
1839
+ # a `--hold-after` entry that reconciled to `blocked` cannot also fire a
1840
+ # gate, and `released` falls through to the walk below in this same tick.
1841
+ holds = _resolve_holds(ordered, states, probes or {})
1842
+
1843
+ # NOTE: "reconciles" is deliberately NOT in plan_base. The waiting-entry
1844
+ # walk below (#1873) can append to `reconciles` too — a `waiting` entry
1845
+ # whose own issue already landed reconciles to `done` there — so every
1846
+ # return site passes `reconciles=tuple(reconciles)` explicitly, taken at
1847
+ # the point of that return rather than frozen here before the walk runs.
1848
+ plan_base = {
1849
+ "holds": tuple(holds),
1850
+ "occupied": occupied,
1851
+ "capacity": capacity,
1852
+ # A copy, not the live dict: the walk below mutates its own projection
1853
+ # of these counts (it charges the launch to its repo) and the plan must
1854
+ # report the reading that `occupied` was taken from.
1855
+ "repo_occupied": dict(repo_occupied),
1856
+ "repo_capacity": repo_capacity,
1857
+ }
1858
+
1859
+ gate = next((h for h in holds if h.blocking), None)
1860
+ if gate is not None:
1861
+ # Launch NOTHING. Not "launch if there is spare capacity", not
1862
+ # "launch anything whose pre-reqs don't mention the held entry" — the
1863
+ # deploy this gate is waiting on is invisible to the dependency graph,
1864
+ # which is exactly why an explicit operator-declared gate exists.
1865
+ return TickPlan(
1866
+ **plan_base,
1867
+ reconciles=tuple(reconciles),
1868
+ blocked=tuple(blocked),
1869
+ deferrals=(),
1870
+ alert=_hold_alert(gate),
1871
+ launch=None,
1872
+ )
1873
+
1874
+ # #2101 step 2b: is THIS host cordoned? Checked after reconciliation (so
1875
+ # a cordoned host still drains its view of reality — see the docstring)
1876
+ # and after the deploy gate (which is the older, narrower stop and keeps
1877
+ # its own alert), but BEFORE capacity: a cordoned host launches nothing
1878
+ # however many slots are free, which is the entire mechanism.
1879
+ cordon_map = _normalized_cordons(cordons)
1880
+ local_cordon = cordon_map.get(_norm_host(local_host)) if local_host else None
1881
+ if local_cordon:
1882
+ return TickPlan(
1883
+ **plan_base,
1884
+ reconciles=tuple(reconciles),
1885
+ blocked=tuple(blocked),
1886
+ deferrals=(),
1887
+ alert=_cordon_alert(local_host or "this host", local_cordon),
1888
+ launch=None,
1889
+ cordon_reason=local_cordon,
1890
+ )
1891
+
1892
+ if capacity - occupied <= 0:
1893
+ return TickPlan(
1894
+ **plan_base,
1895
+ reconciles=tuple(reconciles),
1896
+ blocked=tuple(blocked),
1897
+ deferrals=(),
1898
+ alert=None,
1899
+ launch=None,
1900
+ )
1901
+
1902
+ # Cycles are re-checked here, not just at `add` time: `remove` can leave
1903
+ # the surviving edges in a shape `add` never validated, and a hand-edited
1904
+ # DB row is always possible. A cycle makes every member unsatisfiable.
1905
+ cycle_keys: dict[str, str] = {}
1906
+ cycle = find_cycle({e.key: list(e.after) for e in ordered})
1907
+ if cycle is not None:
1908
+ message = "dependency cycle: " + " -> ".join(cycle)
1909
+ for key in cycle:
1910
+ cycle_keys[key] = message
1911
+
1912
+ def _cooldown_reason(candidate: QueueEntry) -> str:
1913
+ """#1794's launch-side guard: '' unless this entry was just launched.
1914
+
1915
+ A `waiting` row carrying a recent `launched_at` means SOMETHING put a
1916
+ drive up for this issue moments ago — a retry decided on stale
1917
+ evidence, a launch subprocess whose exit code lied, an operator's hand
1918
+ edit. Whatever it was, starting a second `coord drive` now is the
1919
+ failure #1794 exists to prevent, so the entry defers and tries again
1920
+ on the next tick, by which point the reconcile branches above have
1921
+ real evidence to work with.
1922
+ """
1923
+ age = _startup_cooldown(candidate, now, grace_seconds)
1924
+ if age is None:
1925
+ return ""
1926
+ return (
1927
+ f"launched {age:.0f}s ago — inside the {grace_seconds:.0f}s startup "
1928
+ f"grace window, so a second `coord drive` is refused (#1794)"
1929
+ )
1930
+
1931
+ # #1972's projection of per-repo occupancy AS THE WALK SEES IT: the board
1932
+ # reading above, plus this tick's own launch once one is chosen. Kept
1933
+ # separate from `repo_occupied` (reported in the plan) so the launch's own
1934
+ # slot is charged to the report-only pass — otherwise `--dry-run` would
1935
+ # cheerfully explain that the next same-repo entry is eligible, one line
1936
+ # under the launch that just took its repo's last slot.
1937
+ repo_slots: dict[str, int] = dict(repo_occupied)
1938
+
1939
+ def _repo_limit_reason(candidate: QueueEntry) -> str:
1940
+ """#1972's per-repo ceiling: '' unless this entry's repo is full.
1941
+
1942
+ A DEFER, never a block: nothing is wrong with the entry, its position
1943
+ does not move, no attempt is spent and nothing escalates. It is the
1944
+ same "not yet" an unsatisfied `after` produces — the difference is only
1945
+ that what it is waiting on is its own repo's in-flight drive rather
1946
+ than a named pre-req.
1947
+ """
1948
+ if not repo_capacity:
1949
+ return ""
1950
+ used = repo_slots.get(candidate.repo, 0)
1951
+ if used < repo_capacity:
1952
+ return ""
1953
+ return (
1954
+ f"repo {candidate.repo} at its limit ({used}/{repo_capacity}) — "
1955
+ "deferring so a different repo can launch"
1956
+ )
1957
+
1958
+ def _cordon_reason(candidate: QueueEntry) -> str:
1959
+ """#2101: '' unless this entry is PINNED to a cordoned machine.
1960
+
1961
+ A DEFER, never a block: nothing is wrong with the entry, its position
1962
+ does not move and no attempt is spent — its destination is simply
1963
+ draining for a release right now and will take work again in minutes.
1964
+ Only an explicit ``--machine`` pin is checked here; an unpinned entry
1965
+ auto-picks its host at dispatch time, where `coord.drive_state`'s
1966
+ machine picker already skips paused machines (a cordon IS a routing
1967
+ pause — see `coord.machine_pause`), so guessing a destination here
1968
+ would be a second, weaker copy of that decision.
1969
+ """
1970
+ if not candidate.machine:
1971
+ return ""
1972
+ reason = cordon_map.get(_norm_host(candidate.machine))
1973
+ if not reason:
1974
+ return ""
1975
+ return (
1976
+ f"{candidate.machine} is {reason} — deferring rather than "
1977
+ "dispatching into a host that is draining for a release (#2101)"
1978
+ )
1979
+
1980
+ launch: QueueEntry | None = None
1981
+ # #1873: keys that reconciled straight to `done` in the walk below —
1982
+ # landed under someone else's branch/PR, closed by hand as obsolete, or
1983
+ # picked up by `coord reconcile-merges` — WITHOUT this queue ever
1984
+ # launching them. These must not count toward the queue-level alert
1985
+ # below: they were neither deferred nor blocked, so they have nothing to
1986
+ # show up in `details`, and counting them in "considered N" without a
1987
+ # matching detail line is exactly the "considered N, N-1 explained"
1988
+ # contradiction the "considered N" comment below warns about — see the
1989
+ # #1864 incident this branch exists to fix, where the ENTIRE queue was
1990
+ # this case and the tick has nothing to be stalled about.
1991
+ landed_keys: set[str] = set()
1992
+ waiting = [e for e in ordered if states.get(e.key) == STATE_WAITING]
1993
+ for entry in waiting:
1994
+ if launch is not None:
1995
+ # Report-only pass over the tail of the queue. The launch above
1996
+ # already won this tick, so nothing here is mutated (see
1997
+ # Deferral.counted) — this exists so `--dry-run` explains the rest
1998
+ # of the queue instead of going silent after the first line.
1999
+ cooldown = _cooldown_reason(entry)
2000
+ if cooldown:
2001
+ deferrals.append(Deferral(entry.key, cooldown, counted=False))
2002
+ continue
2003
+ verdict = _resolve_prereqs(entry, board, states, cycle_keys)
2004
+ if not verdict.satisfied:
2005
+ deferrals.append(
2006
+ Deferral(entry.key, verdict.reason, counted=False)
2007
+ )
2008
+ continue
2009
+ cordoned = _cordon_reason(entry)
2010
+ if cordoned:
2011
+ deferrals.append(
2012
+ Deferral(entry.key, cordoned, counted=False, cordoned=True)
2013
+ )
2014
+ continue
2015
+ repo_limit = _repo_limit_reason(entry)
2016
+ if repo_limit:
2017
+ deferrals.append(
2018
+ Deferral(
2019
+ entry.key, repo_limit, counted=False, repo_limited=True
2020
+ )
2021
+ )
2022
+ continue
2023
+ cooldown = _cooldown_reason(entry)
2024
+ if cooldown:
2025
+ deferrals.append(
2026
+ Deferral(
2027
+ entry.key,
2028
+ cooldown,
2029
+ updates={
2030
+ "deferrals": entry.deferrals + 1,
2031
+ "last_reason": cooldown,
2032
+ },
2033
+ )
2034
+ )
2035
+ continue
2036
+ # #1873: checked BEFORE `_resolve_prereqs`, not after. The entry's
2037
+ # own board state is unconditional — if this issue is already landed,
2038
+ # its `after=` graph is irrelevant, including when that graph is
2039
+ # itself unsatisfiable (unknown pre-req, cycle, a pre-req that is
2040
+ # `blocked`/`failed`). Checking prereqs first would route a landed
2041
+ # entry with a broken pre-req into the BLOCKED branch below, which
2042
+ # escalates and demands a manual `remove && add` for an entry that
2043
+ # needs neither — it is already done. `_reconcile_running` catches
2044
+ # this same fact for entries that WERE launched (:813); a `waiting`
2045
+ # entry never enters that function at all, so nothing had checked the
2046
+ # board against the entry's own issue until now.
2047
+ facts = board.facts(entry.key)
2048
+ if facts.landed:
2049
+ witness = "merged" if facts.merged else "closed"
2050
+ reason = (
2051
+ f"done — issue already {witness}, never launched by this queue"
2052
+ )
2053
+ reconciles.append(
2054
+ Reconcile(
2055
+ entry.key,
2056
+ "done",
2057
+ reason,
2058
+ occupies=False,
2059
+ # attempts is deliberately NOT incremented: nothing was
2060
+ # ever launched for this entry, so charging it a retry
2061
+ # would be charging it for work that landed elsewhere
2062
+ # (same reasoning as the BLOCKED branch's "operator's
2063
+ # typo" comment just below).
2064
+ updates={
2065
+ "state": STATE_DONE,
2066
+ "last_reason": reason,
2067
+ },
2068
+ )
2069
+ )
2070
+ states[entry.key] = STATE_DONE
2071
+ landed_keys.add(entry.key)
2072
+ continue
2073
+ verdict = _resolve_prereqs(entry, board, states, cycle_keys)
2074
+ if verdict.unsatisfiable:
2075
+ blocked.append(
2076
+ Blocked(
2077
+ entry.key,
2078
+ verdict.reason,
2079
+ # attempts is deliberately NOT incremented: nothing was
2080
+ # ever launched for this entry, so charging it a retry
2081
+ # would be charging it for the operator's typo.
2082
+ updates={
2083
+ "state": STATE_BLOCKED,
2084
+ "last_reason": verdict.reason,
2085
+ },
2086
+ )
2087
+ )
2088
+ states[entry.key] = STATE_BLOCKED
2089
+ continue
2090
+ if not verdict.satisfied:
2091
+ deferrals.append(
2092
+ Deferral(
2093
+ entry.key,
2094
+ verdict.reason,
2095
+ updates={
2096
+ "deferrals": entry.deferrals + 1,
2097
+ "last_reason": verdict.reason,
2098
+ },
2099
+ )
2100
+ )
2101
+ continue
2102
+ # #2101, checked with the same "facts about the ENTRY come first"
2103
+ # rule #1972 states below: a landed entry still reconciles to `done`
2104
+ # and a broken pre-req still blocks and escalates, whatever its
2105
+ # pinned machine's cordon is doing this tick. Only the LAUNCH is
2106
+ # withheld.
2107
+ cordoned = _cordon_reason(entry)
2108
+ if cordoned:
2109
+ deferrals.append(
2110
+ Deferral(
2111
+ entry.key,
2112
+ cordoned,
2113
+ updates={
2114
+ "deferrals": entry.deferrals + 1,
2115
+ "last_reason": cordoned,
2116
+ },
2117
+ # Same posture as #1972's repo limit: this is the fleet
2118
+ # working as designed (a host draining so it can be
2119
+ # rolled), not a stalled queue, so it must not raise the
2120
+ # queue-level alert every tick for the duration of a
2121
+ # drain. The cordon has its OWN alert when it is THIS
2122
+ # host that is stopped — see `_cordon_alert`.
2123
+ cordoned=True,
2124
+ )
2125
+ )
2126
+ continue
2127
+ # #1972, checked LAST: everything above is a fact about the ENTRY (is
2128
+ # it still starting, has it already landed, are its pre-reqs sound),
2129
+ # and those verdicts must not change because some unrelated drive in
2130
+ # the same repo happens to be up. In particular an unsatisfiable
2131
+ # pre-req still blocks and escalates here rather than hiding behind a
2132
+ # repo-limit deferral that would silently postpone it forever.
2133
+ repo_limit = _repo_limit_reason(entry)
2134
+ if repo_limit:
2135
+ deferrals.append(
2136
+ Deferral(
2137
+ entry.key,
2138
+ repo_limit,
2139
+ updates={
2140
+ "deferrals": entry.deferrals + 1,
2141
+ "last_reason": repo_limit,
2142
+ },
2143
+ repo_limited=True,
2144
+ )
2145
+ )
2146
+ continue
2147
+ launch = by_key[entry.key]
2148
+ # The launch takes its repo's slot for the rest of THIS walk, so the
2149
+ # report-only tail explains the remaining same-repo entries correctly.
2150
+ repo_slots[launch.repo] = repo_slots.get(launch.repo, 0) + 1
2151
+
2152
+ alert: QueueAlert | None = None
2153
+ # `waiting`, minus anything the walk above reconciled straight to `done`
2154
+ # (#1873) — those were never deferred or blocked, so they have no line in
2155
+ # `details` and must not be counted as "considered" either. What is left
2156
+ # is exactly the set of entries that are genuinely still waiting: deferred
2157
+ # or blocked, each with a matching `details` entry.
2158
+ still_waiting = [e for e in waiting if e.key not in landed_keys]
2159
+ # #1972: minus anything whose ONLY reason for standing still is that its
2160
+ # own repo is busy. That is the queue doing its job — the same condition
2161
+ # the global at-capacity return above answers with `alert=None` — and a
2162
+ # 39-entry single-repo queue would otherwise escalate on every tick for the
2163
+ # duration of the batch. A MIXED tick still alerts: if even one entry is
2164
+ # deferred on a pre-req or blocked outright, something really is stuck and
2165
+ # the alert names all of it, repo-limit lines included.
2166
+ # #2101 adds the release cordon to that same set: an entry pinned to a
2167
+ # host that is draining for a release is waiting on the fleet working, not
2168
+ # on something wedged. See `Deferral.benign`.
2169
+ benign_keys = {item.key for item in deferrals if item.benign}
2170
+ stalled = [e for e in still_waiting if e.key not in benign_keys]
2171
+ if launch is None and stalled:
2172
+ details = [f"{item.key}: {item.reason}" for item in deferrals]
2173
+ details += [f"{item.key}: BLOCKED — {item.reason}" for item in blocked]
2174
+ alert = QueueAlert(
2175
+ # "considered N" rather than "N waiting": some of those entries are
2176
+ # blocked by the time this line is written, and an alert that
2177
+ # contradicts `coord drive-queue status` two lines below it is an
2178
+ # alert operators learn to distrust.
2179
+ reason=(
2180
+ f"nothing eligible to launch: considered {len(still_waiting)} "
2181
+ f"waiting entr{'y' if len(still_waiting) == 1 else 'ies'}, "
2182
+ f"{capacity - occupied} free slot(s)"
2183
+ ),
2184
+ details=tuple(details),
2185
+ )
2186
+
2187
+ return TickPlan(
2188
+ **plan_base,
2189
+ reconciles=tuple(reconciles),
2190
+ blocked=tuple(blocked),
2191
+ deferrals=tuple(deferrals),
2192
+ alert=alert,
2193
+ launch=launch,
2194
+ )
2195
+
2196
+
2197
+ # ── rendering (pure, so `--dry-run` is testable without a CLI) ───────────────
2198
+
2199
+
2200
+ def render_plan(plan: TickPlan, *, dry_run: bool = False) -> list[str]:
2201
+ """The human-readable form of a :class:`TickPlan`, one line per element."""
2202
+ prefix = "would " if dry_run else ""
2203
+ lines = [
2204
+ f"capacity: {plan.occupied}/{plan.capacity} occupied, "
2205
+ f"{plan.free_slots} free"
2206
+ ]
2207
+ if plan.repo_capacity:
2208
+ # #1972: "1/3 occupied" alone cannot answer "so why didn't item 2 go?"
2209
+ # — the answer is per-repo, so print the breakdown rather than making
2210
+ # the operator read the code. The provenance is spelled out because
2211
+ # this counter inherits rule 1 (board state, not live sessions): a
2212
+ # drive whose observer died still holds its repo's slot, and after
2213
+ # #1972 that wedges ONE repo instead of the whole queue, which is
2214
+ # better but also much quieter.
2215
+ detail = ", ".join(
2216
+ f"{repo} {count}/{plan.repo_capacity}"
2217
+ for repo, count in sorted(plan.repo_occupied.items())
2218
+ )
2219
+ lines.append(
2220
+ f" per-repo: {detail or 'no repo occupied'} (limit "
2221
+ f"{plan.repo_capacity}/repo, counted from board state — a drive "
2222
+ "whose observer died still holds its repo's slot)"
2223
+ )
2224
+ for item in plan.reconciles:
2225
+ lines.append(f" reconcile {item.key}: {item.outcome} — {item.reason}")
2226
+ # #1757: the gate line goes directly under its reconcile, because "1753
2227
+ # done" immediately followed by "and therefore nothing launches" is the
2228
+ # sentence an operator reading a timer log needs to read as one thought.
2229
+ for item in plan.holds:
2230
+ probe = ""
2231
+ if item.resume_when:
2232
+ probe = f" [resume-when: {item.resume_when}"
2233
+ if item.probes:
2234
+ probe += f", {item.probes} failed attempt(s)"
2235
+ if item.probe_detail:
2236
+ probe += f" — {item.probe_detail}"
2237
+ probe += "]"
2238
+ lines.append(f" hold {item.key}: {item.outcome} — {item.reason}{probe}")
2239
+ for item in plan.blocked:
2240
+ lines.append(f" {prefix}block {item.key}: {item.reason}")
2241
+ # Counted deferrals come BEFORE the launch line and report-only ones after,
2242
+ # so the output reads in the order the walk actually happened: these lost
2243
+ # their turn while a slot was free; that one took it; the rest were never
2244
+ # reached.
2245
+ for item in plan.deferrals:
2246
+ if item.counted:
2247
+ lines.append(f" defer {item.key}: {item.reason}")
2248
+ if plan.launch is not None:
2249
+ target = plan.launch
2250
+ pinned = f" on {target.machine}" if target.machine else ""
2251
+ lines.append(f" {prefix}launch {target.key}{pinned}")
2252
+ elif plan.held is not None:
2253
+ lines.append(
2254
+ f" no launch — HELD by the deploy gate on {plan.held.key} "
2255
+ f"(release with `coord drive-queue resume`)"
2256
+ )
2257
+ elif plan.cordon_reason:
2258
+ # #2101 trap E: naming the cordon here is the difference between a
2259
+ # journal that reads "the fleet is upgrading itself" and one that
2260
+ # reads "the queue mysteriously stopped".
2261
+ lines.append(
2262
+ f" no launch — this host is {plan.cordon_reason}; in-flight "
2263
+ "drives are draining and the queue resumes once it is rolled"
2264
+ )
2265
+ elif plan.capacity and plan.free_slots == 0:
2266
+ # Naming the reason matters more here than anywhere else in this
2267
+ # render: #1794 was diagnosed entirely from a journal, and "no launch"
2268
+ # on its own is indistinguishable from a stalled queue.
2269
+ lines.append(
2270
+ f" no launch — at capacity ({plan.occupied}/{plan.capacity} occupied)"
2271
+ )
2272
+ elif plan.deferrals and all(item.benign for item in plan.deferrals):
2273
+ # Same reasoning as the at-capacity line above: with free GLOBAL slots
2274
+ # and no launch, a bare "no launch" reads as a stalled queue in a
2275
+ # journal. This one is saturated per repo (or draining for a release
2276
+ # — #2101), not stalled, and unlike the global case it raises no
2277
+ # alert, so this line is the only place it is ever said.
2278
+ if any(item.cordoned for item in plan.deferrals):
2279
+ lines.append(
2280
+ " no launch — every waiting entry is pinned to a machine "
2281
+ "under a release cordon (draining to be rolled)"
2282
+ )
2283
+ else:
2284
+ lines.append(
2285
+ f" no launch — every waiting entry's repo is at its per-repo "
2286
+ f"limit ({plan.repo_capacity}/repo)"
2287
+ )
2288
+ else:
2289
+ lines.append(" no launch")
2290
+ for item in plan.deferrals:
2291
+ if not item.counted:
2292
+ lines.append(
2293
+ f" defer {item.key}: {item.reason} (not reached this tick)"
2294
+ )
2295
+ if plan.alert is not None:
2296
+ lines.append(f" {prefix}alert: {plan.alert.reason}")
2297
+ lines.extend(f" {detail}" for detail in plan.alert.details)
2298
+ return lines