code-coordinator 0.5.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. code_coordinator-0.5.46.dist-info/METADATA +625 -0
  2. code_coordinator-0.5.46.dist-info/RECORD +295 -0
  3. code_coordinator-0.5.46.dist-info/WHEEL +5 -0
  4. code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
  5. code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
  6. code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
  7. coord/__init__.py +176 -0
  8. coord/_board_mapping.py +229 -0
  9. coord/acceptance.py +468 -0
  10. coord/acceptance_drivers.py +632 -0
  11. coord/agent.py +7517 -0
  12. coord/agent_app.py +1555 -0
  13. coord/agent_update.py +417 -0
  14. coord/agents/opencode/.gitignore +13 -0
  15. coord/agents/opencode/agents/work.md +129 -0
  16. coord/agents/opencode/routing.jsonc +49 -0
  17. coord/audit.py +301 -0
  18. coord/auto_loop.py +1440 -0
  19. coord/board_bool_guard.py +72 -0
  20. coord/board_service.py +141 -0
  21. coord/board_wire.py +309 -0
  22. coord/brain.py +581 -0
  23. coord/branch_model.py +214 -0
  24. coord/cargo_cache.py +258 -0
  25. coord/ci_github.py +386 -0
  26. coord/ci_store.py +560 -0
  27. coord/claim.py +353 -0
  28. coord/cli.py +454 -0
  29. coord/client.py +610 -0
  30. coord/commands/__init__.py +1 -0
  31. coord/commands/_common.py +329 -0
  32. coord/commands/acceptance.py +916 -0
  33. coord/commands/agent_ops.py +1339 -0
  34. coord/commands/audit.py +131 -0
  35. coord/commands/chat.py +320 -0
  36. coord/commands/dispatch.py +1780 -0
  37. coord/commands/dispatch_workers.py +4894 -0
  38. coord/commands/drive.py +616 -0
  39. coord/commands/drive_queue.py +1203 -0
  40. coord/commands/gate_a.py +217 -0
  41. coord/commands/gates.py +89 -0
  42. coord/commands/issues.py +681 -0
  43. coord/commands/lifecycle.py +513 -0
  44. coord/commands/merge.py +1900 -0
  45. coord/commands/milestone.py +2081 -0
  46. coord/commands/plan_followup.py +1243 -0
  47. coord/commands/plans.py +156 -0
  48. coord/commands/release.py +2232 -0
  49. coord/commands/report.py +341 -0
  50. coord/commands/review.py +1523 -0
  51. coord/commands/scorecard.py +252 -0
  52. coord/commands/sessions.py +1930 -0
  53. coord/commands/setup.py +576 -0
  54. coord/commands/status.py +2089 -0
  55. coord/commands/terminal.py +385 -0
  56. coord/commands/test_gate.py +775 -0
  57. coord/commands/tui.py +288 -0
  58. coord/comments.py +718 -0
  59. coord/config.py +3032 -0
  60. coord/conflict_fix.py +633 -0
  61. coord/dao.py +483 -0
  62. coord/dashboard/__init__.py +0 -0
  63. coord/dashboard/fixture.py +376 -0
  64. coord/dashboard/index.html +658 -0
  65. coord/dashboard/server.py +1894 -0
  66. coord/dashboard/terminal.py +382 -0
  67. coord/dashboard/webapp/.gitignore +9 -0
  68. coord/dashboard/webapp/components.json +17 -0
  69. coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
  70. coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
  71. coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
  72. coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
  73. coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
  74. coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
  75. coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
  76. coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
  77. coord/dashboard/webapp/dist/icons/icon.svg +5 -0
  78. coord/dashboard/webapp/dist/index.html +38 -0
  79. coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
  80. coord/dashboard/webapp/dist/sw.js +1 -0
  81. coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
  82. coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
  83. coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
  84. coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
  85. coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
  86. coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
  87. coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
  88. coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
  89. coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
  90. coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
  91. coord/dashboard/webapp/eslint.config.js +20 -0
  92. coord/dashboard/webapp/index.html +37 -0
  93. coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
  94. coord/dashboard/webapp/package-lock.json +10584 -0
  95. coord/dashboard/webapp/package.json +63 -0
  96. coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
  97. coord/dashboard/webapp/playwright.config.ts +93 -0
  98. coord/dashboard/webapp/postcss.config.js +6 -0
  99. coord/dashboard/webapp/public/icons/icon-192.png +0 -0
  100. coord/dashboard/webapp/public/icons/icon-512.png +0 -0
  101. coord/dashboard/webapp/public/icons/icon.svg +5 -0
  102. coord/dashboard/webapp/src/App.tsx +140 -0
  103. coord/dashboard/webapp/src/api/client.ts +199 -0
  104. coord/dashboard/webapp/src/api/generated.ts +176 -0
  105. coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
  106. coord/dashboard/webapp/src/components/Detail.tsx +800 -0
  107. coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
  108. coord/dashboard/webapp/src/components/Home.tsx +435 -0
  109. coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
  110. coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
  111. coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
  112. coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
  113. coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
  114. coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
  115. coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
  116. coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
  117. coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
  118. coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
  119. coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
  120. coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
  121. coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
  122. coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
  123. coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
  124. coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
  125. coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
  126. coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
  127. coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
  128. coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
  129. coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
  130. coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
  131. coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
  132. coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
  133. coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
  134. coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
  135. coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
  136. coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
  137. coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
  138. coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
  139. coord/dashboard/webapp/src/index.css +210 -0
  140. coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
  141. coord/dashboard/webapp/src/lib/utils.ts +6 -0
  142. coord/dashboard/webapp/src/main.tsx +46 -0
  143. coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
  144. coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
  145. coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
  146. coord/dashboard/webapp/src/realtime/connection.ts +227 -0
  147. coord/dashboard/webapp/src/realtime/events.ts +100 -0
  148. coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
  149. coord/dashboard/webapp/src/routes/paths.ts +92 -0
  150. coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
  151. coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
  152. coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
  153. coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
  154. coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
  155. coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
  156. coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
  157. coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
  158. coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
  159. coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
  160. coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
  161. coord/dashboard/webapp/src/shell/railItems.ts +105 -0
  162. coord/dashboard/webapp/src/shell/shellState.ts +174 -0
  163. coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
  164. coord/dashboard/webapp/src/test-setup.ts +41 -0
  165. coord/dashboard/webapp/src/vite-env.d.ts +2 -0
  166. coord/dashboard/webapp/tailwind.config.js +140 -0
  167. coord/dashboard/webapp/tsconfig.json +25 -0
  168. coord/dashboard/webapp/tsconfig.node.json +11 -0
  169. coord/dashboard/webapp/vite.config.ts +71 -0
  170. coord/db.py +1076 -0
  171. coord/dead_end.py +332 -0
  172. coord/deploy/README.md +33 -0
  173. coord/deploy/coord-agent.service +89 -0
  174. coord/deploy/coord-db-backup.service +60 -0
  175. coord/deploy/coord-db-backup.sh +74 -0
  176. coord/deploy/coord-db-backup.timer +18 -0
  177. coord/deploy/coord-drive-queue.service +117 -0
  178. coord/deploy/coord-drive-queue.timer +39 -0
  179. coord/deploy/coord-notify.service +48 -0
  180. coord/deploy/coord-notify.timer +24 -0
  181. coord/deploy/coord-release-propagate.service +83 -0
  182. coord/deploy/coord-release-propagate.timer +38 -0
  183. coord/deploy/coord-release-window.service +119 -0
  184. coord/deploy/coord-release-window.timer +36 -0
  185. coord/deploy/coord-serve.service +82 -0
  186. coord/deploy/coord-web-dist-build.service +43 -0
  187. coord/deploy/coord-web-dist-build.timer +36 -0
  188. coord/deploy/coord-web.service +125 -0
  189. coord/deploy_manifest.py +80 -0
  190. coord/deploy_units.py +384 -0
  191. coord/deps.py +115 -0
  192. coord/diagnose.py +1623 -0
  193. coord/dispatch.py +1009 -0
  194. coord/dist_name.py +123 -0
  195. coord/drive.py +3101 -0
  196. coord/drive_queue.py +2298 -0
  197. coord/drive_state.py +870 -0
  198. coord/events.py +381 -0
  199. coord/failure_class.py +914 -0
  200. coord/filelock.py +168 -0
  201. coord/fleet_config_health.py +300 -0
  202. coord/freshness.py +206 -0
  203. coord/gate_a.py +469 -0
  204. coord/gate_b.py +411 -0
  205. coord/gate_snapshot.py +385 -0
  206. coord/gates.py +582 -0
  207. coord/github_ops.py +1954 -0
  208. coord/goal.py +125 -0
  209. coord/graph_health.py +348 -0
  210. coord/health/__init__.py +69 -0
  211. coord/health/aggregate.py +129 -0
  212. coord/health/checks/__init__.py +13 -0
  213. coord/health/checks/agent_install.py +280 -0
  214. coord/health/checks/cargo_targets.py +171 -0
  215. coord/health/checks/claude_binary.py +65 -0
  216. coord/health/checks/deploy_lane_facts.py +458 -0
  217. coord/health/checks/disk.py +99 -0
  218. coord/health/checks/fleet_board.py +89 -0
  219. coord/health/checks/fleet_deploy_lanes.py +469 -0
  220. coord/health/checks/fleet_phantom.py +69 -0
  221. coord/health/checks/fleet_unit_drift.py +151 -0
  222. coord/health/checks/graph.py +192 -0
  223. coord/health/checks/plan_usage.py +88 -0
  224. coord/health/checks/repo_state.py +161 -0
  225. coord/health/checks/spawned_coord.py +465 -0
  226. coord/health/checks/timer_active.py +254 -0
  227. coord/health/checks/toolchain.py +547 -0
  228. coord/health/checks/unit_drift.py +648 -0
  229. coord/health/checks/unit_enablement.py +171 -0
  230. coord/health/checks/worktrees.py +96 -0
  231. coord/health/cli.py +121 -0
  232. coord/health/context.py +106 -0
  233. coord/health/fleet_snapshot.py +477 -0
  234. coord/health/models.py +250 -0
  235. coord/health/pypi.py +231 -0
  236. coord/health/registry.py +240 -0
  237. coord/health/render.py +82 -0
  238. coord/health/units.py +60 -0
  239. coord/hooks.py +106 -0
  240. coord/housekeeping.py +204 -0
  241. coord/interactive.py +4286 -0
  242. coord/issue_store.py +1496 -0
  243. coord/liveness_auditor.py +293 -0
  244. coord/machine_pause.py +755 -0
  245. coord/merge_queue.py +4681 -0
  246. coord/milestone_chat.py +600 -0
  247. coord/milestone_dispatch.py +943 -0
  248. coord/milestone_gate.py +709 -0
  249. coord/milestone_order.py +840 -0
  250. coord/mock_author.py +334 -0
  251. coord/models.py +891 -0
  252. coord/network.py +269 -0
  253. coord/new_issue_chat.py +229 -0
  254. coord/notify.py +3226 -0
  255. coord/openapi.py +404 -0
  256. coord/overlap_fence.py +133 -0
  257. coord/parentage.py +200 -0
  258. coord/parentage_github.py +58 -0
  259. coord/pipeline.py +481 -0
  260. coord/plan_parser.py +266 -0
  261. coord/plans.py +543 -0
  262. coord/platform_paths.py +43 -0
  263. coord/pr_body_lint.py +67 -0
  264. coord/prereqs.py +533 -0
  265. coord/progress.py +425 -0
  266. coord/providers/__init__.py +683 -0
  267. coord/providers/base.py +218 -0
  268. coord/providers/claude.py +284 -0
  269. coord/providers/claude_pty.py +610 -0
  270. coord/providers/opencode.py +896 -0
  271. coord/reconcile.py +2233 -0
  272. coord/refine_chat.py +485 -0
  273. coord/release_cordon.py +525 -0
  274. coord/release_propagate.py +1176 -0
  275. coord/release_verify.py +777 -0
  276. coord/release_window.py +322 -0
  277. coord/reports.py +1643 -0
  278. coord/revalidate.py +1101 -0
  279. coord/review.py +3317 -0
  280. coord/scorecard.py +484 -0
  281. coord/serve_app.py +7192 -0
  282. coord/skills/update-issue/SKILL.md +93 -0
  283. coord/smoke.py +1030 -0
  284. coord/split_work.py +210 -0
  285. coord/stage_projection.py +650 -0
  286. coord/state.py +5720 -0
  287. coord/test_author.py +1064 -0
  288. coord/test_chat.py +352 -0
  289. coord/test_orchestrator.py +494 -0
  290. coord/test_report.py +178 -0
  291. coord/tui_release.py +271 -0
  292. coord/usage.py +753 -0
  293. coord/usage_limits.py +358 -0
  294. coord/usage_rollup.py +709 -0
  295. coord/worker_events.py +954 -0
coord/notify.py ADDED
@@ -0,0 +1,3226 @@
1
+ """Poll agent servers and post completion/failure comments to GitHub."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import time
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import TYPE_CHECKING
10
+
11
+ import httpx
12
+
13
+ if TYPE_CHECKING:
14
+ from coord.models import Assignment, Board
15
+
16
+ log = logging.getLogger(__name__)
17
+
18
+ # Cache: machine_name → host. Populated by `run(config)` so post_transition →
19
+ # _try_parse_and_post_review can fetch a remote agent's log via /logs/<id>
20
+ # without threading the Config through every helper.
21
+ _AGENT_HOSTS: dict[str, str] = {}
22
+
23
+
24
+ def _agent_host(machine_name: str) -> str | None:
25
+ return _AGENT_HOSTS.get(machine_name)
26
+
27
+ from coord import github_ops
28
+ from coord.comments import (
29
+ EVENT_ADVISORY,
30
+ EVENT_COMPLETION,
31
+ EVENT_FAILURE,
32
+ EVENT_LIVENESS_STALL,
33
+ EVENT_NEEDS_ATTENTION,
34
+ EVENT_PLAN,
35
+ EVENT_STALLED,
36
+ EVENT_STUCK,
37
+ format_liveness_stall,
38
+ format_needs_attention,
39
+ format_plan,
40
+ format_stalled_pipeline,
41
+ format_stalled_pipeline_dispatch,
42
+ format_stuck,
43
+ )
44
+ from coord.config import Config
45
+ from coord.dispatch import AGENT_PORT, post_advisory, post_completion, post_failure
46
+ from coord.progress import parse_progress
47
+ from coord.state import (
48
+ load_dispatched,
49
+ load_done_reviews_needing_post,
50
+ load_liveness_audit_state,
51
+ load_notified,
52
+ mark_notified,
53
+ mark_review_posted,
54
+ save_liveness_audit_state,
55
+ save_plan,
56
+ )
57
+
58
+ # #1710 inventory: kept as a direct import — `is_usage_limit_reason` is a
59
+ # trivial string-prefix predicate over `Assignment.failure_reason` (a
60
+ # coordinator-authored value stamped by `format_usage_limit_reason`, itself
61
+ # only ever produced by the reap path's claude-specific kill detection), not
62
+ # a per-provider log-format parse. Any provider's `failure_reason` would be
63
+ # checked the same way.
64
+ from coord.worker_events import is_usage_limit_reason
65
+
66
+
67
+ @dataclass
68
+ class Transition:
69
+ assignment_id: str
70
+ machine_name: str
71
+ repo_name: str
72
+ issue_number: int
73
+ event: str # completion | failure
74
+ exit_code: int | None
75
+
76
+
77
+ @dataclass
78
+ class StuckDetection:
79
+ assignment_id: str
80
+ machine_name: str
81
+ repo_name: str
82
+ issue_number: int
83
+ stuck_message: str
84
+ log_path: str | None
85
+
86
+
87
+ def _stuck_notified_key(assignment_id: str) -> str:
88
+ """Notified ledger key for stuck events.
89
+
90
+ Uses a composite key so that a stuck notification does not block later
91
+ completion/failure notifications (which key on bare assignment_id).
92
+ """
93
+ return f"{assignment_id}:stuck"
94
+
95
+
96
+ @dataclass
97
+ class NeedsAttentionDetection:
98
+ assignment_id: str
99
+ machine_name: str
100
+ repo_name: str
101
+ issue_number: int
102
+ reason: str # "wall_clock" | "non_convergence"
103
+ detail: str
104
+
105
+
106
+ def _needs_attention_notified_key(assignment_id: str) -> str:
107
+ """Notified ledger key for needs-attention events (#846).
108
+
109
+ Composite key (mirrors :func:`_stuck_notified_key`) so a one-shot
110
+ needs-attention comment does not block later completion/failure/stuck
111
+ notifications, and vice versa.
112
+ """
113
+ return f"{assignment_id}:needs-attention"
114
+
115
+
116
+ @dataclass
117
+ class StalledDetection:
118
+ """#1441: a pipeline row whose auto-loop transition already fired once
119
+ but which is stuck on a precondition that landed too late for that
120
+ one-shot reaction to see. See :func:`detect_stalled_pipeline`."""
121
+
122
+ assignment_id: str
123
+ machine_name: str
124
+ repo_name: str
125
+ issue_number: int
126
+ reason: str # "review_request_changes_no_fix" | "review_done_no_verdict" |
127
+ # "done_no_review" | "approved_not_queued" | "merge_conflict_unresolved"
128
+ # (#1478, #1582) | "review_failed_no_verdict" (#1584)
129
+ detail: str
130
+
131
+
132
+ def _stalled_notified_key(assignment_id: str) -> str:
133
+ """Notified ledger key for stalled-pipeline events (#1441).
134
+
135
+ Composite key (mirrors :func:`_needs_attention_notified_key`) so a
136
+ one-shot stalled-pipeline comment does not block later completion/
137
+ failure/stuck/needs-attention notifications for the same assignment_id,
138
+ and vice versa.
139
+ """
140
+ return f"{assignment_id}:stalled"
141
+
142
+
143
+ @dataclass
144
+ class LivenessStallDetection:
145
+ """#2048: N consecutive ``blocked`` verdicts from the cheap per-turn
146
+ liveness auditor. See :func:`detect_liveness_stall`."""
147
+
148
+ assignment_id: str
149
+ machine_name: str
150
+ repo_name: str
151
+ issue_number: int
152
+ consecutive_blocked: int
153
+ last_verdict: str | None
154
+
155
+
156
+ def _liveness_notified_key(assignment_id: str) -> str:
157
+ """Notified ledger key for liveness-stall events (#2048).
158
+
159
+ Composite key (mirrors :func:`_needs_attention_notified_key` /
160
+ :func:`_stalled_notified_key`) so a one-shot liveness comment does not
161
+ block later completion/failure/stuck/needs-attention/stalled
162
+ notifications for the same assignment_id, and vice versa. This exact
163
+ shape is also what keeps ``mark_notified``'s bare-``else`` branch from
164
+ ever writing ``status='failed'`` onto a real assignment row for this
165
+ event — see the comment in ``coord.state._mark_notified_local``.
166
+ """
167
+ return f"{assignment_id}:liveness"
168
+
169
+
170
+ def _fmt_minutes(seconds: float) -> str:
171
+ minutes = seconds / 60.0
172
+ if minutes < 1:
173
+ return f"{seconds:.0f}s"
174
+ if minutes == int(minutes):
175
+ return f"{int(minutes)}m"
176
+ return f"{minutes:.1f}m"
177
+
178
+
179
+ def attention_signal(
180
+ *,
181
+ assignment_type: str,
182
+ status: str | None,
183
+ dispatched_at: float | None,
184
+ review_iteration: int,
185
+ config: Config,
186
+ now: float | None = None,
187
+ provider_name: str | None = None,
188
+ review_of_assignment_id: str | None = None,
189
+ ) -> tuple[str, str] | tuple[None, None]:
190
+ """Pure #846 detection core: the two "needs attention" signals, decoupled
191
+ from where the assignment's fields come from.
192
+
193
+ 1. **Non-convergence**: ``review_iteration >= config.pipeline.
194
+ convergence_rounds`` fix/review rounds without reaching a terminal
195
+ green test verdict + approved review. Checked first — a thrashing
196
+ assignment is worth flagging even if it hasn't yet cleared the
197
+ wall-clock threshold.
198
+ 2. **Wall-clock**: running longer than
199
+ ``config.pipeline.attention_threshold_for(assignment_type,
200
+ provider_name=..., review_of_assignment_id=...)``, computed from
201
+ *dispatched_at*. ``provider_name``/``review_of_assignment_id``
202
+ (#1137) let an interactive ``--fix-of``/``--rework-of`` session be
203
+ recognized despite sharing ``type="work"`` with headless coding
204
+ workers — see :meth:`Config.pipeline.attention_threshold_for`'s
205
+ docstring. Both default to ``None`` (no effect) for callers that
206
+ don't have the full assignment record.
207
+
208
+ Deliberately time/round-based rather than self-report-based (#448: the
209
+ failure mode that motivated this was a worker that never emitted a
210
+ ``STUCK:`` line — it just silently burned budget while looking
211
+ "productive").
212
+
213
+ Shared by :func:`detect_needs_attention` (the coordinator backstop,
214
+ dispatch-ledger-dict based), ``coord.pipeline.compute_pipeline`` (the
215
+ ``/api/pipeline`` field the web dashboard renders), and the dashboard's
216
+ background poller (``Assignment``-object based) — one signal, several
217
+ call sites, instead of three copies of the same threshold logic.
218
+
219
+ Returns ``(reason, detail)`` — ``reason`` is ``"wall_clock"`` or
220
+ ``"non_convergence"`` — or ``(None, None)`` when nothing is flagged.
221
+ """
222
+ if (status or "").lower() != "running":
223
+ return None, None
224
+ if now is None:
225
+ now = time.time()
226
+
227
+ if review_iteration >= config.pipeline.convergence_rounds:
228
+ return "non_convergence", (
229
+ f"{review_iteration} fix/review round(s) on this assignment "
230
+ f"without reaching a green test verdict + approved review "
231
+ f"(threshold: {config.pipeline.convergence_rounds})."
232
+ )
233
+
234
+ threshold = config.pipeline.attention_threshold_for(
235
+ assignment_type,
236
+ provider_name=provider_name,
237
+ review_of_assignment_id=review_of_assignment_id,
238
+ )
239
+ if dispatched_at is not None:
240
+ running_for = now - dispatched_at
241
+ if running_for > threshold:
242
+ return "wall_clock", (
243
+ f"Running {_fmt_minutes(running_for)}, past the "
244
+ f"{_fmt_minutes(threshold)} threshold for "
245
+ f"type={assignment_type!r}."
246
+ )
247
+
248
+ return None, None
249
+
250
+
251
+ def detect_needs_attention(
252
+ config: Config, *, now: float | None = None
253
+ ) -> list[tuple[NeedsAttentionDetection, dict]]:
254
+ """Scan dispatched assignments for the two #846 "needs attention" signals
255
+ (see :func:`attention_signal`). Detection only — no dispatch/kill/handoff
256
+ behaviour.
257
+
258
+ Returns ``(NeedsAttentionDetection, dispatch_record)`` pairs for
259
+ assignments that haven't already been notified as needing attention (or
260
+ reached a terminal notification), mirroring :func:`detect_stuck`'s shape
261
+ so callers can post + mark idempotently the same way.
262
+ """
263
+ dispatched = load_dispatched()
264
+ if not dispatched:
265
+ return []
266
+ notified = load_notified()
267
+
268
+ active_records = [
269
+ r for r in dispatched
270
+ if r["assignment_id"] not in notified
271
+ and _needs_attention_notified_key(r["assignment_id"]) not in notified
272
+ ]
273
+ if not active_records:
274
+ return []
275
+
276
+ results: list[tuple[NeedsAttentionDetection, dict]] = []
277
+ for record in active_records:
278
+ reason, detail = attention_signal(
279
+ assignment_type=record.get("type") or "work",
280
+ status=record.get("status"),
281
+ dispatched_at=record.get("dispatched_at"),
282
+ review_iteration=record.get("review_iteration") or 0,
283
+ config=config,
284
+ now=now,
285
+ provider_name=record.get("provider_name"),
286
+ review_of_assignment_id=record.get("review_of_assignment_id"),
287
+ )
288
+ if reason is None:
289
+ continue
290
+ results.append((
291
+ NeedsAttentionDetection(
292
+ assignment_id=record["assignment_id"],
293
+ machine_name=record["machine_name"],
294
+ repo_name=record["repo_name"],
295
+ issue_number=record["issue_number"],
296
+ reason=reason,
297
+ detail=detail,
298
+ ),
299
+ record,
300
+ ))
301
+
302
+ return results
303
+
304
+
305
+ def post_needs_attention(detection: NeedsAttentionDetection, record: dict) -> None:
306
+ """Post a needs-attention comment to GitHub and mark notified (#846)."""
307
+ body = format_needs_attention(
308
+ assignment_id=detection.assignment_id,
309
+ machine_name=detection.machine_name,
310
+ repo_name=detection.repo_name,
311
+ issue_number=detection.issue_number,
312
+ reason=detection.reason,
313
+ detail=detection.detail,
314
+ )
315
+ github_ops.post_issue_comment(
316
+ record["repo_github"], detection.issue_number, body
317
+ )
318
+ mark_notified(_needs_attention_notified_key(detection.assignment_id), EVENT_NEEDS_ATTENTION)
319
+
320
+
321
+ # ── Liveness auditor (#2048) ─────────────────────────────────────────────────
322
+ #
323
+ # Tier 2.5 in the stall-detection ladder (see coord/liveness_auditor.py's
324
+ # module docstring): a cheap, independent, per-turn judgment call, sitting
325
+ # between EVENT_NEEDS_ATTENTION (a clock, no judgment) and a metered
326
+ # adversarial review (judgment, but only at a stage boundary). Detection +
327
+ # a one-shot GitHub comment only, mirroring detect_needs_attention's/
328
+ # detect_stalled_pipeline's contract exactly: this function NEVER sets
329
+ # Assignment.status/review_state/test_state, never kills/reassigns a
330
+ # worker, and never influences a merge decision. It only ever (a) runs a
331
+ # `claude -p` subprocess against a fixed-size (objective, latest-turn)
332
+ # context and (b) records/reads the resulting strike streak.
333
+
334
+
335
+ def _latest_turn_text_for_liveness(
336
+ machine_name: str, log_path: str | None, assignment_id: str,
337
+ ) -> str | None:
338
+ """Best-effort latest-assistant-turn text for a RUNNING assignment, for
339
+ the liveness auditor.
340
+
341
+ Mirrors :func:`_fetch_raw_log_text`'s local-file-then-agent-fetch
342
+ fallback (that helper takes a completed :class:`Transition`; this one
343
+ is called against a still-running assignment, so it takes the bare
344
+ fields instead) — with one deliberate difference: the local-file
345
+ branch uses :func:`coord.worker_events.latest_assistant_turn_text`'s
346
+ seek-based tail read (``tail_bytes=65536``) instead of reading the
347
+ whole file into memory. This runs once per debounce interval for the
348
+ *entire lifetime* of a running assignment, so a full read here would
349
+ quietly turn "audit cost is flat" into "disk I/O scales with log
350
+ size" — worst in exactly the stuck-worker-with-a-growing-log scenario
351
+ the auditor exists to catch (#2048 review). The remote (agent-fetch)
352
+ branch has no tail-range support server-side, so it still fetches the
353
+ full response and slices in Python.
354
+
355
+ Returns ``None`` on any I/O failure or if the tail has no assistant
356
+ turn yet — best-effort, the auditor must never be the reason
357
+ ``coord notify`` raises.
358
+
359
+ The ``log_path`` recorded on an :class:`~coord.agent.AgentAssignment` is
360
+ a path on the *worker's own* machine, and ``coord notify`` normally runs
361
+ on the daemon host — so for the multi-machine fleet topology this repo
362
+ is built around, that path usually does NOT exist locally and the HTTP
363
+ fallback is the only branch that can ever return text. The local branch
364
+ is therefore taken only when the file actually exists: because
365
+ :func:`~coord.worker_events.latest_assistant_turn_text` swallows a
366
+ missing/unreadable file and returns ``None`` internally, committing to
367
+ it unconditionally would make the agent-fetch below dead code and the
368
+ auditor would silently never fire for any remote worker (#2048 review).
369
+ """
370
+ if log_path:
371
+ from coord.worker_events import latest_assistant_turn_text # noqa: PLC0415
372
+
373
+ try:
374
+ local_readable = Path(log_path).is_file()
375
+ except OSError:
376
+ local_readable = False
377
+ if local_readable:
378
+ return latest_assistant_turn_text(log_path, tail_bytes=65536)
379
+ host = _agent_host(machine_name)
380
+ if host:
381
+ from coord.worker_events import latest_assistant_turn_text_from_text # noqa: PLC0415
382
+
383
+ try:
384
+ resp = httpx.get(
385
+ f"http://{host}:{AGENT_PORT}/logs/{assignment_id}", timeout=15.0
386
+ )
387
+ resp.raise_for_status()
388
+ except (httpx.HTTPError, httpx.TimeoutException):
389
+ return None
390
+ return latest_assistant_turn_text_from_text(resp.text[-65536:])
391
+ return None
392
+
393
+
394
+ def detect_liveness_stall(
395
+ config: Config, *, now: float | None = None
396
+ ) -> list[tuple[LivenessStallDetection, dict]]:
397
+ """#2048: run the cheap per-turn liveness auditor against running
398
+ assignments and flag the ones whose latest turns earned
399
+ ``config.pipeline.liveness_auditor.strikes`` consecutive ``blocked``
400
+ verdicts in a row.
401
+
402
+ Returns ``(LivenessStallDetection, dispatch_record)`` pairs, mirroring
403
+ :func:`detect_needs_attention`'s shape. No-ops entirely (returns ``[]``
404
+ without touching the DB or spawning a subprocess) when
405
+ ``config.pipeline.liveness_auditor.enabled`` is ``False`` — the default.
406
+ """
407
+ cfg = config.pipeline.liveness_auditor
408
+ if not cfg.enabled:
409
+ return []
410
+
411
+ from coord.liveness_auditor import ( # noqa: PLC0415
412
+ apply_verdict,
413
+ run_audit,
414
+ should_audit,
415
+ strip_self_report_lines,
416
+ )
417
+
418
+ if now is None:
419
+ now = time.time()
420
+
421
+ dispatched = load_dispatched()
422
+ if not dispatched:
423
+ return []
424
+ notified = load_notified()
425
+
426
+ active_records = [
427
+ r for r in dispatched
428
+ if (r.get("status") or "").lower() == "running"
429
+ and r["assignment_id"] not in notified
430
+ and _liveness_notified_key(r["assignment_id"]) not in notified
431
+ ]
432
+ if not active_records:
433
+ return []
434
+
435
+ machines_by_name = {m.name: m for m in config.machines}
436
+ by_machine: dict[str, list[dict]] = {}
437
+ for r in active_records:
438
+ by_machine.setdefault(r["machine_name"], []).append(r)
439
+
440
+ # Deliberately serial: each `run_audit` call below is a subprocess
441
+ # spawn with up to a `timeout_seconds` (default 30s) ceiling, and
442
+ # audits for every not-yet-raised running assignment past its
443
+ # debounce window run one at a time in this loop. At today's typical
444
+ # concurrency this is a non-issue; if the audited fleet grows enough
445
+ # for a single `coord notify` pass to stack up meaningful wall-clock
446
+ # time here, parallelize (e.g. a bounded thread pool around
447
+ # `run_audit`) rather than accept unbounded serial latency (#2048
448
+ # review).
449
+ results: list[tuple[LivenessStallDetection, dict]] = []
450
+ for machine_name, records in by_machine.items():
451
+ machine = machines_by_name.get(machine_name)
452
+ if machine is None:
453
+ continue
454
+ status = _agent_status(machine.host)
455
+ if status is None:
456
+ continue
457
+ active_by_id: dict[str, dict] = {}
458
+ for entry in status.get("active", []):
459
+ eid = entry.get("id")
460
+ if eid:
461
+ active_by_id[eid] = entry
462
+
463
+ for record in records:
464
+ aid = record["assignment_id"]
465
+ entry = active_by_id.get(aid)
466
+ if entry is None:
467
+ continue
468
+
469
+ state = load_liveness_audit_state(aid)
470
+ if state.raised:
471
+ continue
472
+ if not should_audit(
473
+ last_audit_at=state.last_audit_at,
474
+ now=now,
475
+ debounce_seconds=cfg.debounce_seconds,
476
+ ):
477
+ continue
478
+
479
+ # Seek-based tail read for a local log (see
480
+ # _latest_turn_text_for_liveness's docstring) — the auditor
481
+ # only ever needs the single most recent turn, never the
482
+ # whole (potentially multi-MB) transcript, and this repeats
483
+ # every debounce interval for the assignment's whole runtime.
484
+ turn_text = _latest_turn_text_for_liveness(
485
+ record["machine_name"], entry.get("log_path"), aid
486
+ )
487
+ if turn_text is None:
488
+ continue # no assistant turn yet, or fetch failed
489
+
490
+ # #2048 context isolation: strip the worker's own STATUS:/
491
+ # STUCK: lines before the auditor ever sees this turn — see
492
+ # coord.liveness_auditor's module docstring.
493
+ turn_text = strip_self_report_lines(turn_text)
494
+
495
+ outcome = run_audit(
496
+ record.get("briefing") or "",
497
+ turn_text,
498
+ model=cfg.model,
499
+ claude_bin=cfg.claude_bin,
500
+ timeout=cfg.timeout_seconds,
501
+ )
502
+ new_state, just_raised = apply_verdict(
503
+ state, outcome.verdict, now=now, strikes=cfg.strikes
504
+ )
505
+ save_liveness_audit_state(aid, new_state)
506
+
507
+ if just_raised:
508
+ results.append((
509
+ LivenessStallDetection(
510
+ assignment_id=aid,
511
+ machine_name=record["machine_name"],
512
+ repo_name=record["repo_name"],
513
+ issue_number=record["issue_number"],
514
+ consecutive_blocked=new_state.consecutive_blocked,
515
+ last_verdict=new_state.last_verdict,
516
+ ),
517
+ record,
518
+ ))
519
+
520
+ return results
521
+
522
+
523
+ def post_liveness_stall(detection: LivenessStallDetection, record: dict) -> None:
524
+ """Post a liveness-stall comment to GitHub and mark notified (#2048)."""
525
+ body = format_liveness_stall(
526
+ assignment_id=detection.assignment_id,
527
+ machine_name=detection.machine_name,
528
+ repo_name=detection.repo_name,
529
+ issue_number=detection.issue_number,
530
+ consecutive_blocked=detection.consecutive_blocked,
531
+ )
532
+ github_ops.post_issue_comment(
533
+ record["repo_github"], detection.issue_number, body
534
+ )
535
+ mark_notified(_liveness_notified_key(detection.assignment_id), EVENT_LIVENESS_STALL)
536
+
537
+
538
+ # ── Stalled-pipeline sweeper (#1441) ────────────────────────────────────────
539
+ #
540
+ # The auto-loop (coord.auto_loop) only reacts to review/fix TRANSITIONS — the
541
+ # instant `coord notify` sees a review or fix flip to `done` during THAT
542
+ # pass. Once the transition is consumed nothing ever re-examines the row, so
543
+ # a precondition that lands late (a Test verdict backfilled two days after
544
+ # the review completed — vimcode #602) leaves it stranded: looks complete on
545
+ # the board, isn't. This sweeper re-scans every *done* work chain on the
546
+ # board each notify pass and flags the ones stuck on an unmet precondition
547
+ # a fresh transition would have already resolved. Detection only — no
548
+ # dispatch, mirroring detect_needs_attention's contract.
549
+
550
+
551
+ def _pipeline_heads(board: "Board") -> list["Assignment"]:
552
+ """Return the most-recent WORK_LIKE_TYPES assignment per (repo, issue).
553
+
554
+ A row can be bounced through 1+ auto-loop fix iterations, each a
555
+ separate ``Assignment`` sharing the same ``(repo_name, issue_number)``.
556
+ Only the most recent one reflects the pipeline's actual current
557
+ position — earlier rows in the chain are superseded, and evaluating them
558
+ too would re-flag a condition a later fix already addressed.
559
+ """
560
+ from coord.models import WORK_LIKE_TYPES # noqa: PLC0415
561
+
562
+ all_assignments = list(board.active) + list(board.completed)
563
+ heads: dict[tuple[str, int], "Assignment"] = {}
564
+ for a in all_assignments:
565
+ if a.type not in WORK_LIKE_TYPES:
566
+ continue
567
+ key = (a.repo_name, a.issue_number)
568
+ ts = a.dispatched_at or a.finished_at or 0.0
569
+ cur = heads.get(key)
570
+ cur_ts = (cur.dispatched_at or cur.finished_at or 0.0) if cur is not None else -1.0
571
+ if cur is None or ts >= cur_ts:
572
+ heads[key] = a
573
+ return list(heads.values())
574
+
575
+
576
+ def detect_stalled_pipeline(
577
+ config: Config,
578
+ *,
579
+ board: "Board | None" = None,
580
+ merge_queue_items: list | None = None,
581
+ terminal_cache: dict | None = None,
582
+ ) -> list[tuple[StalledDetection, "Assignment"]]:
583
+ """Scan the board for *done* work chains stuck on an unmet precondition
584
+ that a fresh review/fix transition would already have resolved (#1441).
585
+
586
+ Five candidate stall states, checked per pipeline "head" (the most
587
+ recent work-like assignment for a given (repo, issue) — see
588
+ :func:`_pipeline_heads`):
589
+
590
+ 1. ``review_request_changes_no_fix`` — the head's linked review
591
+ completed with verdict ``request-changes`` and no fix assignment was
592
+ ever dispatched in response (the vimcode #602 reference case: the
593
+ review's transition fired and was consumed while some other
594
+ precondition was outstanding, and nothing has re-examined it since).
595
+ 2. ``review_done_no_verdict`` (#1582) — the head's linked review is
596
+ ``status="done"`` but ``review_verdict IS NULL``: the reviewing
597
+ session finalised without ever capturing a verdict (elitebook's
598
+ documented ~14% review-verdict drop rate, #873).
599
+
600
+ #2019: the detail text used to blame this on "the session likely
601
+ failed to start or exited before recording one (#812)" for EVERY
602
+ review. That was wrong twice over on the #1956 reference case — the
603
+ session ran 392s, produced a complete 6.5KB review and exited 0, and
604
+ #812 is CLOSED and was about *interactive* reviews, while that one was
605
+ ``interactive=False``. An operator following it looked at a closed
606
+ issue and a false cause. The wording is now provider-aware: a headless
607
+ review that reached ``done`` is reported as the
608
+ END_REVIEW-without-verdict class (#1956), and only a ``claude-pty``
609
+ review still cites #812's never-started shape.
610
+
611
+ This matches NONE of the other three arms — it
612
+ isn't ``request-changes`` (no verdict at all), a review WAS
613
+ dispatched (so not ``done_no_review``), and there is no approval (so
614
+ not ``approved_not_queued``) — so before this arm existed it fell
615
+ through every check and parked the drive forever (#1582's own
616
+ observed case, #1563).
617
+ 3. ``done_no_review`` — the head carries a terminal Test verdict
618
+ (``passed``/``skipped``), the "review" gate is required, the
619
+ completion is not an interactive (``provider_name="claude-pty"``)
620
+ session (interactive completions are deliberately excluded from
621
+ automatic review dispatch — #555), and yet no review assignment was
622
+ ever dispatched for it.
623
+ 4. ``approved_not_queued`` — the head satisfies every merge gate
624
+ (:func:`coord.merge_queue.passes_merge_gates` — reused rather than
625
+ re-derived, per #1441's own request) but has no merge-queue entry.
626
+ 5. ``merge_conflict_unresolved`` (#1478) — the head already HAS a
627
+ merge-queue entry, but that entry is parked ``CONFLICT`` with an
628
+ error :func:`coord.merge_queue.classify_conflict` calls
629
+ ``"rebaseable"`` and no conflict-fix attempt is active or already
630
+ failed (:func:`coord.conflict_fix.has_prior_conflict_fix`). This is
631
+ exactly the gap :mod:`coord.commands.merge`'s
632
+ ``_dispatch_conflict_fixes`` docstring calls out for the ``--only``
633
+ path pre-#1474 — a bare ``CONFLICT`` row that never got a second
634
+ classify-and-dispatch pass, except here for *any* path (not only
635
+ ``--only``): a ``coord merge`` invocation that dispatched a
636
+ conflict-fix which then failed to actually attempt (no idle
637
+ machine) leaves the entry parked with nothing watching it.
638
+ 5. ``review_failed_no_verdict`` (#1584) — the head's linked review
639
+ WORKER died (transient API error, network drop, ...) before ever
640
+ producing a verdict — ``status="failed"`` with no
641
+ ``review_verdict``. Before #1584 this could not happen (a dying
642
+ review was mislabelled ``done``, silently masquerading as a real
643
+ completion); now that it is correctly ``failed``, it needs its own
644
+ arm here so it is not silently skipped (``reason`` staying ``None``)
645
+ the way an unrecognized status would be.
646
+
647
+ Every candidate is checked against the shared #522 terminal-state guard
648
+ (:func:`coord.github_ops.work_is_terminal`, via *terminal_cache* — the
649
+ same cache :func:`coord.notify.run` threads through the review/fix
650
+ auto-loop calls) so a closed issue or merged PR never surfaces, and
651
+ against the ``notified`` ledger (composite key, :func:`_stalled_notified_key`)
652
+ so a flagged row is not re-flagged every pass.
653
+
654
+ Detection only, mirroring :func:`detect_needs_attention`'s contract — no
655
+ dispatch, no kill, no handoff (that lives in
656
+ :func:`dispatch_stalled_pipeline_action`, #1478, gated behind
657
+ ``config.pipeline.auto_dispatch_stalled``). *board* / *merge_queue_items*
658
+ / *terminal_cache* are all optional so callers (tests, or a future
659
+ ``reconcile()`` caller) can supply their own instead of hitting the
660
+ board service / DB / GitHub.
661
+ """
662
+ # `github_ops` is already imported at module level (used by every other
663
+ # post_* helper in this file) — no local re-import here, so a caller
664
+ # that mocks `coord.notify.github_ops.post_issue_comment` for the
665
+ # posting side doesn't also have to reason about a separately-imported
666
+ # local name for the terminal-state check below.
667
+ from coord.auto_loop import FIX_DISPATCH_TYPES # noqa: PLC0415
668
+ from coord.conflict_fix import has_prior_conflict_fix # noqa: PLC0415
669
+ from coord.merge_queue import ( # noqa: PLC0415
670
+ CONFLICT,
671
+ classify_conflict,
672
+ live_gate_entry,
673
+ load_queue,
674
+ passes_merge_gates,
675
+ )
676
+
677
+ if board is None:
678
+ from coord.board_service import read_board # noqa: PLC0415
679
+ board = read_board()
680
+ if merge_queue_items is None:
681
+ merge_queue_items = load_queue()
682
+ if terminal_cache is None:
683
+ terminal_cache = {}
684
+
685
+ notified = load_notified()
686
+ all_assignments = list(board.active) + list(board.completed)
687
+
688
+ results: list[tuple[StalledDetection, "Assignment"]] = []
689
+ for work in _pipeline_heads(board):
690
+ if work.status != "done" or not work.assignment_id:
691
+ continue
692
+ if _stalled_notified_key(work.assignment_id) in notified:
693
+ continue
694
+
695
+ repo = config.repo(work.repo_name)
696
+ repo_github = repo.github if repo is not None else None
697
+ if repo_github and github_ops.work_is_terminal(
698
+ repo_github, work.issue_number, work.branch, cache=terminal_cache
699
+ ):
700
+ continue
701
+
702
+ required_gates = work.required_gates or list(config.pipeline.default_gates)
703
+
704
+ review = next(
705
+ (
706
+ a for a in all_assignments
707
+ if a.review_of_assignment_id == work.assignment_id and a.type == "review"
708
+ ),
709
+ None,
710
+ )
711
+
712
+ # #1566: a review that just finished lands on status="finalizing"
713
+ # (not "done") until `coord notify`'s own _try_parse_and_post_review
714
+ # promotes it — i.e. THIS function is what closes that window. None
715
+ # of the `review.status == "done"` checks below match "finalizing",
716
+ # so a still-finalizing review falls through this whole if/elif
717
+ # chain with `reason` left unset (no stall reported), which is
718
+ # correct as long as the finalizing window stays short. That relies
719
+ # on `coord notify` actually running again soon — nothing here
720
+ # guards against `coord notify` itself never running (e.g. daemon
721
+ # down), which would leave the row on "finalizing" forever without
722
+ # ever tripping this stall detector.
723
+ reason: str | None = None
724
+ detail = ""
725
+
726
+ if (
727
+ review is not None
728
+ and review.status == "done"
729
+ and review.review_verdict == "request-changes"
730
+ ):
731
+ fix = next(
732
+ (
733
+ a for a in all_assignments
734
+ if a.review_of_assignment_id == work.assignment_id
735
+ and a.type in FIX_DISPATCH_TYPES
736
+ ),
737
+ None,
738
+ )
739
+ if fix is None:
740
+ reason = "review_request_changes_no_fix"
741
+ detail = (
742
+ f"Review {review.assignment_id} completed with "
743
+ "request-changes and no fix worker was ever dispatched "
744
+ "for it."
745
+ )
746
+ elif (
747
+ review is not None
748
+ and review.status == "done"
749
+ and review.review_verdict is None
750
+ ):
751
+ # #1582: a review that finalised `done` with NO verdict ever
752
+ # captured. Checked BEFORE the `review is None or review.status
753
+ # == "done"` catch-all below — that branch's merge-gate check
754
+ # (`passes_merge_gates`) never fires for a `None` verdict (no
755
+ # approval), so this row would otherwise fall all the way
756
+ # through with `reason` left unset.
757
+ reason = "review_done_no_verdict"
758
+ # #2019: provider-aware, because the pre-#2019 single sentence
759
+ # ("the session likely failed to start or exited before recording
760
+ # one (#812)") was demonstrably false for the headless case and
761
+ # pointed at a CLOSED issue about interactive reviews. See this
762
+ # function's docstring, arm 2.
763
+ if review.provider_name == "claude-pty":
764
+ detail = (
765
+ f"Review {review.assignment_id} finalised as done but no "
766
+ "verdict was ever captured — an interactive review that "
767
+ "failed to start, or exited before `coord report-result` "
768
+ "ran (#812)."
769
+ )
770
+ else:
771
+ detail = (
772
+ f"Review {review.assignment_id} finalised as done but no "
773
+ "verdict was ever captured. This is a HEADLESS review "
774
+ "that ran to completion (a session that died lands "
775
+ "status='failed', not 'done'), so the reviewer's "
776
+ "REVIEW_VERDICT header was omitted or unparsed — the "
777
+ "END_REVIEW-without-verdict class (#1956), not a "
778
+ "never-started session. The verdict is very likely "
779
+ "already in the transcript; relay it rather than "
780
+ "re-dispatching: coord report-result --assignment "
781
+ f"{review.assignment_id} --status done --verdict "
782
+ "<approve|request-changes> --verdict-source recovered "
783
+ "--verdict-reason '...' --body-file <extracted-review.md>"
784
+ )
785
+ elif (
786
+ review is None
787
+ and "review" in required_gates
788
+ and work.provider_name != "claude-pty"
789
+ and work.test_state in ("passed", "skipped")
790
+ ):
791
+ reason = "done_no_review"
792
+ detail = (
793
+ f"Work is done with test_state={work.test_state!r} but no "
794
+ "review assignment was ever dispatched for it."
795
+ )
796
+ elif review is not None and review.status == "failed":
797
+ # #1584: the review worker died (transient API error, network
798
+ # drop, ...) before producing a verdict. Checked before the
799
+ # `review is None or review.status == "done"` catch-all below so
800
+ # a failed review is never mistaken for "no review dispatched"
801
+ # or "review approved" — neither of which is true here.
802
+ #
803
+ # ...UNLESS it was killed by the account's usage limit. That is
804
+ # an account-wide exhausted budget, not a per-review defect:
805
+ # `AgentServer._reap` lands a usage-limit kill on FAILED exactly
806
+ # like an api_error kill, so without this guard the sweep would
807
+ # spend this work row's ONE auto-recovery action (the
808
+ # `_stalled_notified_key` ledger is one-shot per work row) on a
809
+ # `dispatch_review` that is guaranteed to die the same way until
810
+ # the reset — the precise anti-pattern `reconcile.py`'s
811
+ # `auto_reassign` block was hardened against in #1461, and the
812
+ # one `coord/drive.py`'s `_decide_review` already guards with
813
+ # this same predicate. Skipped at CLASSIFICATION rather than
814
+ # declined at dispatch so the row is never marked notified: a
815
+ # later review attempt that fails for a *different* (genuinely
816
+ # recoverable) reason can still be picked up by a future tick.
817
+ if is_usage_limit_reason(review.failure_reason):
818
+ continue
819
+ reason = "review_failed_no_verdict"
820
+ detail = (
821
+ f"Review {review.assignment_id} failed "
822
+ f"({review.failure_reason or 'no reason recorded'}) before "
823
+ "producing a verdict, and no retry was dispatched."
824
+ )
825
+ elif review is None or review.status == "done":
826
+ # Either the review gate doesn't apply, or a review already
827
+ # completed without leaving a request-changes verdict blocking
828
+ # it (approved, or advanced past advisory-only nits) — the only
829
+ # remaining question is whether it made it into the merge queue,
830
+ # and if it did, whether that entry is stuck.
831
+ matching_entry = next(
832
+ (m for m in merge_queue_items if m.assignment_id == work.assignment_id),
833
+ None,
834
+ )
835
+ if matching_entry is None:
836
+ # #2085: `work` is a raw board Assignment — no
837
+ # `branch_head_sha`/`repo_github`/`target_branch` attribute,
838
+ # so handing it straight to `passes_merge_gates` made the
839
+ # #821 SHA-freshness check inside `has_approved_review`
840
+ # permanently unconfirmable (fails closed on every review
841
+ # carrying a real `review_head_sha`, i.e. virtually every
842
+ # modern approval). Build the same live-anchored synthetic
843
+ # entry `coord.gates.build_gate_report` uses so a genuinely
844
+ # fresh approval can still be confirmed via `github_ops`
845
+ # (already imported at module level). Falls back to the raw
846
+ # `work` row (still gh_ops-backed, just missing target_branch/
847
+ # repo_github) when the repo isn't configured — the gate
848
+ # then fails closed exactly as before, never open.
849
+ gate_entry = work
850
+ if repo is not None and repo_github:
851
+ from coord.branch_model import ( # noqa: PLC0415
852
+ resolve_base_branch_for_issue_number,
853
+ )
854
+ target_branch = resolve_base_branch_for_issue_number(
855
+ repo, repo_github, work.issue_number,
856
+ )
857
+ gate_entry = live_gate_entry(
858
+ work, repo_github, target_branch, github_ops
859
+ )
860
+ if passes_merge_gates(gate_entry, config, board, gh_ops=github_ops):
861
+ reason = "approved_not_queued"
862
+ detail = (
863
+ "Work passes every merge gate (review + test) but has "
864
+ "no merge-queue entry."
865
+ )
866
+ elif (
867
+ matching_entry.state == CONFLICT
868
+ and classify_conflict(matching_entry.error) == "rebaseable"
869
+ and not has_prior_conflict_fix(board, matching_entry.assignment_id)
870
+ ):
871
+ # #1478: a rebaseable CONFLICT with no active/failed
872
+ # conflict-fix attempt — the #1474 classify-and-dispatch step
873
+ # never got (or never got a second) chance at this entry.
874
+ reason = "merge_conflict_unresolved"
875
+ detail = (
876
+ f"Merge queue entry for branch {matching_entry.branch!r} is "
877
+ f"stuck in CONFLICT ({matching_entry.error or 'no error recorded'}) "
878
+ "with no active or previously-failed conflict-fix attempt."
879
+ )
880
+
881
+ if reason is None:
882
+ continue
883
+
884
+ results.append((
885
+ StalledDetection(
886
+ assignment_id=work.assignment_id,
887
+ machine_name=work.machine_name,
888
+ repo_name=work.repo_name,
889
+ issue_number=work.issue_number,
890
+ reason=reason,
891
+ detail=detail,
892
+ ),
893
+ work,
894
+ ))
895
+
896
+ return results
897
+
898
+
899
+ def post_stalled_pipeline(detection: StalledDetection, config: Config) -> None:
900
+ """Post a stalled-pipeline comment to GitHub and mark notified (#1441)."""
901
+ repo = config.repo(detection.repo_name)
902
+ repo_github = repo.github if repo is not None else None
903
+ if not repo_github:
904
+ return
905
+ body = format_stalled_pipeline(
906
+ assignment_id=detection.assignment_id,
907
+ machine_name=detection.machine_name,
908
+ repo_name=detection.repo_name,
909
+ issue_number=detection.issue_number,
910
+ reason=detection.reason,
911
+ detail=detection.detail,
912
+ )
913
+ github_ops.post_issue_comment(repo_github, detection.issue_number, body)
914
+ mark_notified(_stalled_notified_key(detection.assignment_id), EVENT_STALLED)
915
+
916
+
917
+ # ── #1478: dispatch arm ──────────────────────────────────────────────────────
918
+
919
+
920
+ @dataclass
921
+ class StalledDispatchAction:
922
+ """The outcome of :func:`dispatch_stalled_pipeline_action` for one
923
+ :class:`StalledDetection`."""
924
+
925
+ kind: str
926
+ """One of:
927
+ - ``"fix_dispatch_attempted"`` — re-ran the review-completion transition
928
+ (:func:`coord.auto_loop.process_review_completion`) for
929
+ ``review_request_changes_no_fix`` (or, #1582, for a
930
+ ``review_done_no_verdict`` whose verdict was just recovered from the
931
+ transcript) and it dispatched a fix worker; see *detail* for what it
932
+ did.
933
+ - ``"review_transition_applied"`` — re-ran
934
+ :func:`coord.auto_loop.process_review_completion` for
935
+ ``review_request_changes_no_fix`` (or a transcript-recovered
936
+ ``review_done_no_verdict``, #1582) and it resolved as ``approved``,
937
+ ``approved_with_nits`` (the #476 advisory-only gate), or
938
+ ``terminal_skip`` — no fix worker was dispatched, but the call still
939
+ mutated *board* in place (``review.review_verdict``,
940
+ ``work.review_state = "done"``, a merge-queue ``refresh_entry_assignment``)
941
+ per that function's own "the caller is responsible for persisting the
942
+ board after this returns" contract. Must be persisted exactly like a
943
+ real dispatch even though no agent was launched.
944
+ - ``"review_verdict_recovered"`` — ``review_done_no_verdict``: a verdict
945
+ was recovered from the reviewing session's own transcript (#617's
946
+ ``_review_findings_from_transcript``, the same recovery
947
+ ``coord diagnose --stage review`` runs) and durably persisted, but
948
+ ``process_review_completion`` made no further board mutation from it
949
+ (e.g. ``pipeline.auto_loop`` is off). See *detail* for the recovered
950
+ verdict.
951
+ - ``"review_reset_redispatched"`` — ``review_done_no_verdict``: nothing
952
+ was recoverable from the transcript, so the review stage was reset
953
+ (the review rows deleted, ``work.review_state`` cleared — #1180's
954
+ ``_reset_review_stage``, branch/commits always kept) and a fresh
955
+ review dispatched for the same work.
956
+ - ``"review_dispatched"`` — a review was dispatched for
957
+ ``done_no_review``.
958
+ - ``"enqueued"`` — the work was enqueued for merge for
959
+ ``approved_not_queued`` (including when a *different* row's
960
+ ``enqueue_approved_work`` call already enqueued this one earlier in
961
+ the same sweep tick — see the queue-membership check below).
962
+ - ``"conflict_fix_dispatched"`` — a conflict-fix worker was dispatched
963
+ for ``merge_conflict_unresolved``.
964
+ - ``"no_action"`` — the reused dispatcher declined (no
965
+ capable machine, already in flight, gate not actually satisfied,
966
+ entry vanished from the board/queue between detection and dispatch).
967
+ - ``"skipped_live_session"`` — a running/pending assignment already
968
+ exists for this (repo, issue); never act underneath a live session
969
+ (#602).
970
+ - ``"skipped_human_required"`` — the conflict-fix retry cap was already
971
+ hit; surfacing to a human, not auto-retrying.
972
+ - ``"disabled"`` — ``pipeline.auto_dispatch_stalled`` is
973
+ off; detection/narration still happened, dispatch did not.
974
+ """
975
+ detail: str = ""
976
+
977
+
978
+ # Action kinds that represent a REAL dispatch OR a board mutation that must
979
+ # be persisted (mutate the board / merge queue / fire an agent request) —
980
+ # used to decide (a) whether the board needs writing back, (b) which GitHub
981
+ # comment to post, and (c) whether the audit row is business-tier (a real
982
+ # transition) or operational-tier (a no-op/skip, informational only).
983
+ #
984
+ # ``review_transition_applied`` belongs here even though it does not launch
985
+ # an agent: an approved/approved-with-nits/terminal-skip resolution from
986
+ # ``process_review_completion`` still flips ``work.review_state``/
987
+ # ``review.review_verdict`` in place, and losing that mutation while the
988
+ # one-shot ledger marks the row notified anyway is exactly the #1478 review
989
+ # bug this set exists to prevent.
990
+ _STALLED_DISPATCH_KINDS = frozenset({
991
+ "fix_dispatch_attempted", "review_transition_applied", "review_dispatched",
992
+ "enqueued", "conflict_fix_dispatched",
993
+ # #1582
994
+ "review_verdict_recovered", "review_reset_redispatched",
995
+ })
996
+
997
+ # process_review_completion (and the _dispatch_fix_for_review it may call)
998
+ # kinds that mutate `board` in place per its own documented contract, even
999
+ # when they don't dispatch a fix worker. `disabled`/`no_findings` return
1000
+ # before any mutation; `no_work_found`/`max_iterations` return without
1001
+ # touching `board` (only a GitHub notice for the latter).
1002
+ _MUTATING_REVIEW_COMPLETION_KINDS = frozenset({
1003
+ "fix_dispatched", "approved", "approved_with_nits", "terminal_skip",
1004
+ })
1005
+
1006
+
1007
+ def _stalled_row_has_live_session(board: "Board", work: "Assignment") -> bool:
1008
+ """#602 guardrail: true when a running/pending assignment already exists
1009
+ for *work*'s (repo, issue) — e.g. an interactive ``--fix-of``/
1010
+ ``--review-of``/``--merge-of`` session a human is actively driving.
1011
+ :func:`dispatch_stalled_pipeline_action` must never act underneath one:
1012
+ racing an auto-dispatch against a live session can duplicate or clobber
1013
+ it. Broader than :func:`coord.claim.has_active_work_followup` (which
1014
+ only checks ``work``/``conflict-fix``) — any live assignment type
1015
+ (review, smoke, chat, ...) for the same issue counts here.
1016
+ """
1017
+ for a in board.active:
1018
+ if a.status not in ("running", "pending"):
1019
+ continue
1020
+ if a.repo_name == work.repo_name and a.issue_number == work.issue_number:
1021
+ return True
1022
+ return False
1023
+
1024
+
1025
+ def dispatch_stalled_pipeline_action(
1026
+ detection: StalledDetection,
1027
+ work: "Assignment",
1028
+ board: "Board",
1029
+ config: Config,
1030
+ *,
1031
+ terminal_cache: dict | None = None,
1032
+ ) -> StalledDispatchAction:
1033
+ """#1478: act on a #1441 stalled-pipeline detection instead of only
1034
+ narrating it.
1035
+
1036
+ Gated by ``config.pipeline.auto_dispatch_stalled`` (default ``False`` —
1037
+ detection/narration via :func:`post_stalled_pipeline` is unconditional;
1038
+ this is the opt-in action half). Mutates *board* in place exactly like
1039
+ the auto-loop / review-dispatch helpers it delegates to — the caller is
1040
+ responsible for persisting it.
1041
+
1042
+ Reuses the SAME dispatch machinery the original, on-time transition
1043
+ would have used for each reason, rather than re-deriving new logic:
1044
+
1045
+ - ``review_request_changes_no_fix`` → re-locates the ``request-changes``
1046
+ review and re-runs :func:`coord.auto_loop.process_review_completion`
1047
+ on it — the exact function the auto-loop calls the instant a review
1048
+ transitions to done, complete with its iteration cap and terminal
1049
+ guard.
1050
+ - ``review_done_no_verdict`` (#1582) → :func:`coord.diagnose._recover_review`
1051
+ (the exact recovery ``coord diagnose --stage review`` runs: try the
1052
+ session transcript first). A recovered verdict is then run through
1053
+ :func:`coord.auto_loop.process_review_completion` like a normal
1054
+ transition; nothing recoverable falls through to
1055
+ :func:`coord.diagnose._reset_review_stage` (the exact reset
1056
+ ``coord diagnose --stage review --reset`` runs — keeps the branch,
1057
+ wipes the review rows + review_state) followed by a fresh
1058
+ :func:`coord.review.dispatch_review` call.
1059
+ - ``done_no_review`` → :func:`coord.review.dispatch_review`, the same
1060
+ call ``detect_transitions``/``dispatch_pending_reviews`` make on a
1061
+ fresh work completion.
1062
+ - ``approved_not_queued`` → :func:`coord.merge_queue.enqueue_approved_work`,
1063
+ the same bulk gate-checked enqueue the daemon passive tick already
1064
+ runs on every interval.
1065
+ - ``merge_conflict_unresolved`` → :func:`coord.conflict_fix.dispatch_conflict_fix`,
1066
+ the #1474 ``_dispatch_conflict_fixes`` path.
1067
+ - ``review_failed_no_verdict`` (#1584) → :func:`coord.review.dispatch_review`
1068
+ again, the SAME call as ``done_no_review`` — the failed review left no
1069
+ verdict behind, so recovery is identical to "no review was ever
1070
+ dispatched": open a fresh one against the still-``done`` work row.
1071
+
1072
+ Never re-entrant across ticks: the caller only reaches this after
1073
+ :func:`detect_stalled_pipeline` has already filtered out any row whose
1074
+ ``_stalled_notified_key`` is in the ``notified`` ledger, and the caller
1075
+ marks that key notified right after this returns (via
1076
+ :func:`post_stalled_pipeline` or :func:`post_stalled_pipeline_dispatch`)
1077
+ — so a given assignment_id gets exactly one dispatch attempt per stall,
1078
+ mirroring the one-shot comment (#1441's own guardrail, reused rather
1079
+ than re-derived per #1478's own request).
1080
+ """
1081
+ if not config.pipeline.auto_dispatch_stalled:
1082
+ return StalledDispatchAction(
1083
+ kind="disabled", detail="pipeline.auto_dispatch_stalled is False",
1084
+ )
1085
+
1086
+ if _stalled_row_has_live_session(board, work):
1087
+ return StalledDispatchAction(
1088
+ kind="skipped_live_session",
1089
+ detail=(
1090
+ f"a running/pending assignment already exists for "
1091
+ f"{work.repo_name}#{work.issue_number} — not acting "
1092
+ "underneath a live session (#602)"
1093
+ ),
1094
+ )
1095
+
1096
+ if detection.reason == "review_request_changes_no_fix":
1097
+ from coord.auto_loop import process_review_completion # noqa: PLC0415
1098
+
1099
+ all_assignments = list(board.active) + list(board.completed)
1100
+ review = next(
1101
+ (
1102
+ a for a in all_assignments
1103
+ if a.review_of_assignment_id == work.assignment_id and a.type == "review"
1104
+ ),
1105
+ None,
1106
+ )
1107
+ if review is None:
1108
+ return StalledDispatchAction(
1109
+ kind="no_action", detail="review no longer found on board",
1110
+ )
1111
+ machine_host = next(
1112
+ (m.host for m in config.machines if m.name == review.machine_name), None,
1113
+ )
1114
+ actions = process_review_completion(
1115
+ review, board, config,
1116
+ machine_host=machine_host, terminal_cache=terminal_cache,
1117
+ )
1118
+ kind_set = {a.kind for a in actions}
1119
+ kinds = ", ".join(a.kind for a in actions) or "no_action"
1120
+ details = "; ".join(a.detail for a in actions if a.detail)
1121
+ detail_msg = f"process_review_completion → {kinds}" + (f" ({details})" if details else "")
1122
+ # #1478 review fix: `process_review_completion` mutates `board` in
1123
+ # place for several outcomes besides `fix_dispatched` — an
1124
+ # `approved`/`approved_with_nits`/`terminal_skip` resolution still
1125
+ # flips `review.review_verdict`/`work.review_state` and refreshes the
1126
+ # merge-queue entry (see that function's own "caller is responsible
1127
+ # for persisting the board" contract). Classifying those as
1128
+ # `no_action` silently dropped the mutation (the sweep's `board_dirty`
1129
+ # never got set) while the one-shot ledger still marked the row
1130
+ # notified — permanently losing the transition. Any kind in
1131
+ # `_MUTATING_REVIEW_COMPLETION_KINDS` must therefore map to a
1132
+ # `_STALLED_DISPATCH_KINDS` member so `_sweep_stalled_pipeline`
1133
+ # persists it.
1134
+ if "fix_dispatched" in kind_set:
1135
+ return StalledDispatchAction(kind="fix_dispatch_attempted", detail=detail_msg)
1136
+ if kind_set & _MUTATING_REVIEW_COMPLETION_KINDS:
1137
+ return StalledDispatchAction(kind="review_transition_applied", detail=detail_msg)
1138
+ return StalledDispatchAction(kind="no_action", detail=detail_msg)
1139
+
1140
+ if detection.reason == "review_done_no_verdict":
1141
+ # #1582: a review finalised `done` with no verdict ever captured
1142
+ # (#812). Reuse the SAME two steps `coord diagnose --stage review
1143
+ # [--reset]` runs for this exact shape, rather than re-deriving new
1144
+ # recovery/reset logic — `_recover_review`/`_reset_review_stage` are
1145
+ # the private functions behind that command for this branch. Called
1146
+ # directly (not through the full `diagnose_stage` orchestration),
1147
+ # which skips that command's tmux session-state probe and
1148
+ # issue-wide phantom-row cleanup — the review here is already
1149
+ # terminal, so neither applies, and both would add real
1150
+ # subprocess/ssh cost to every notify sweep tick.
1151
+ from coord.diagnose import ( # noqa: PLC0415
1152
+ DiagnoseResult,
1153
+ _recover_review,
1154
+ _reset_review_stage,
1155
+ )
1156
+
1157
+ all_assignments = list(board.active) + list(board.completed)
1158
+ review = next(
1159
+ (
1160
+ a for a in all_assignments
1161
+ if a.review_of_assignment_id == work.assignment_id and a.type == "review"
1162
+ ),
1163
+ None,
1164
+ )
1165
+ if review is None:
1166
+ return StalledDispatchAction(
1167
+ kind="no_action", detail="review no longer found on board",
1168
+ )
1169
+
1170
+ diag = DiagnoseResult(
1171
+ repo_name=work.repo_name, issue_number=work.issue_number, stage="review",
1172
+ )
1173
+ # `state="unknown"` is safe: `_recover_review`'s live/dead-session
1174
+ # branches are only reached when `latest.status != "done"`, which
1175
+ # can't happen here (`detect_stalled_pipeline` only flags this
1176
+ # reason for a `status="done"` review).
1177
+ _recover_review(board, config, review, "unknown", diag, dry_run=False)
1178
+
1179
+ if diag.recovered:
1180
+ # A verdict was recovered from the session transcript and
1181
+ # durably persisted (#617's `_review_findings_from_transcript` →
1182
+ # `issue_store.post_result`). Run it through the SAME auto-loop
1183
+ # chokepoint a live review completion would have used — mirrors
1184
+ # `review_request_changes_no_fix` just above — so a recovered
1185
+ # `request-changes` still gets its fix worker and a recovered
1186
+ # `approve` still advances the pipeline.
1187
+ from coord.auto_loop import process_review_completion # noqa: PLC0415
1188
+
1189
+ machine_host = next(
1190
+ (m.host for m in config.machines if m.name == review.machine_name), None,
1191
+ )
1192
+ actions = process_review_completion(
1193
+ review, board, config,
1194
+ machine_host=machine_host, terminal_cache=terminal_cache,
1195
+ )
1196
+ kind_set = {a.kind for a in actions}
1197
+ kinds = ", ".join(a.kind for a in actions) or "no_action"
1198
+ details = "; ".join(a.detail for a in actions if a.detail)
1199
+ detail_msg = (
1200
+ "recovered verdict from the session transcript → "
1201
+ f"process_review_completion → {kinds}" + (f" ({details})" if details else "")
1202
+ )
1203
+ if "fix_dispatched" in kind_set:
1204
+ return StalledDispatchAction(kind="fix_dispatch_attempted", detail=detail_msg)
1205
+ if kind_set & _MUTATING_REVIEW_COMPLETION_KINDS:
1206
+ return StalledDispatchAction(kind="review_transition_applied", detail=detail_msg)
1207
+ return StalledDispatchAction(kind="review_verdict_recovered", detail=detail_msg)
1208
+
1209
+ if not diag.needs_reset:
1210
+ return StalledDispatchAction(
1211
+ kind="no_action", detail="; ".join(diag.findings) or "nothing to do",
1212
+ )
1213
+
1214
+ # Nothing recoverable — reset the review stage (delete the review
1215
+ # rows, clear review_state — #1180's `_reset_review_stage`, KEEPS
1216
+ # the branch/commits) and re-dispatch a fresh review.
1217
+ reset_res = DiagnoseResult(
1218
+ repo_name=work.repo_name, issue_number=work.issue_number, stage="review",
1219
+ )
1220
+ _reset_review_stage(
1221
+ config, work.repo_name, work.issue_number, reset_res,
1222
+ dry_run=False, assignment_id=work.assignment_id,
1223
+ )
1224
+ if not reset_res.reset_performed:
1225
+ return StalledDispatchAction(
1226
+ kind="no_action",
1227
+ detail="reset did not complete: " + "; ".join(reset_res.findings),
1228
+ )
1229
+
1230
+ # `_reset_review_stage` writes the canonical DB directly (the same
1231
+ # seam `coord diagnose --reset` uses — see commands/status.py's
1232
+ # "NOTE: deliberately NO save_board" comment for why) WITHOUT
1233
+ # touching `board`. Mirror the same two writes on `board` in place
1234
+ # so a later `write_board` upsert of the now-stale `review`/`work`
1235
+ # objects doesn't resurrect the just-deleted review row or clobber
1236
+ # the just-cleared review_state back to its wedged value.
1237
+ board.active[:] = [
1238
+ a for a in board.active
1239
+ if not (a.type == "review" and a.review_of_assignment_id == work.assignment_id)
1240
+ ]
1241
+ board.completed[:] = [
1242
+ a for a in board.completed
1243
+ if not (a.type == "review" and a.review_of_assignment_id == work.assignment_id)
1244
+ ]
1245
+ work.review_state = "pending"
1246
+ work.review_verdict = None
1247
+ work.review_posted_at = None
1248
+
1249
+ from coord.review import dispatch_review # noqa: PLC0415
1250
+
1251
+ new_review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
1252
+ if new_review is None:
1253
+ return StalledDispatchAction(
1254
+ kind="no_action",
1255
+ detail=(
1256
+ "review stage reset (no verdict recoverable) but "
1257
+ "re-dispatch declined (no machine / already in flight / gate)"
1258
+ ),
1259
+ )
1260
+ return StalledDispatchAction(
1261
+ kind="review_reset_redispatched",
1262
+ detail=(
1263
+ "no verdict recoverable from transcript — reset the review "
1264
+ f"stage and re-dispatched as {new_review.assignment_id} to "
1265
+ f"{new_review.machine_name}"
1266
+ ),
1267
+ )
1268
+
1269
+ if detection.reason == "done_no_review":
1270
+ from coord.review import dispatch_review # noqa: PLC0415
1271
+
1272
+ review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
1273
+ if review is None:
1274
+ return StalledDispatchAction(
1275
+ kind="no_action",
1276
+ detail="dispatch_review declined (no machine / already in flight / gate)",
1277
+ )
1278
+ return StalledDispatchAction(
1279
+ kind="review_dispatched",
1280
+ detail=f"review {review.assignment_id} dispatched to {review.machine_name}",
1281
+ )
1282
+
1283
+ if detection.reason == "review_failed_no_verdict":
1284
+ # #1584: the previous review died with no verdict — recovery is
1285
+ # identical to `done_no_review` above: `work` itself is still
1286
+ # `status="done"` (only the review it spawned failed), so a fresh
1287
+ # `dispatch_review` call is a normal, ungated re-dispatch. Reusing
1288
+ # the same call (rather than e.g. `coord retry` against the dead
1289
+ # review row) also picks up any board state that changed since —
1290
+ # same reasoning `done_no_review` already relies on.
1291
+ from coord.review import dispatch_review # noqa: PLC0415
1292
+
1293
+ # Belt-and-braces against the usage-limit kill (#1461/#1584):
1294
+ # `detect_stalled_pipeline` already skips those rows at
1295
+ # classification, but this function is public and is also reachable
1296
+ # with a caller-built detection, or after a race in which the
1297
+ # usage-limit `failure_reason` was stamped onto the review row
1298
+ # between detection and dispatch. Re-dispatching into an
1299
+ # account-wide exhausted budget only produces another corpse, so
1300
+ # decline — mirroring `_decide_review`'s WAIT in `coord/drive.py`.
1301
+ all_assignments = list(board.active) + list(board.completed)
1302
+ dead_review = next(
1303
+ (
1304
+ a for a in all_assignments
1305
+ if a.review_of_assignment_id == work.assignment_id
1306
+ and a.type == "review"
1307
+ and a.status == "failed"
1308
+ ),
1309
+ None,
1310
+ )
1311
+ if dead_review is not None and is_usage_limit_reason(dead_review.failure_reason):
1312
+ return StalledDispatchAction(
1313
+ kind="no_action",
1314
+ detail=(
1315
+ f"review {dead_review.assignment_id} was killed by the "
1316
+ f"usage limit ({dead_review.failure_reason}) — waiting "
1317
+ "for the reset instead of re-dispatching"
1318
+ ),
1319
+ )
1320
+
1321
+ review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
1322
+ if review is None:
1323
+ return StalledDispatchAction(
1324
+ kind="no_action",
1325
+ detail="dispatch_review declined (no machine / already in flight / gate)",
1326
+ )
1327
+ return StalledDispatchAction(
1328
+ kind="review_dispatched",
1329
+ detail=f"review {review.assignment_id} dispatched to {review.machine_name}",
1330
+ )
1331
+
1332
+ if detection.reason == "approved_not_queued":
1333
+ from coord.merge_queue import enqueue_approved_work, load_queue # noqa: PLC0415
1334
+
1335
+ changed = enqueue_approved_work(config, board)
1336
+ if work.assignment_id in changed:
1337
+ return StalledDispatchAction(
1338
+ kind="enqueued", detail=f"{work.assignment_id} enqueued for merge",
1339
+ )
1340
+ # #1478 review non-blocking finding: `enqueue_approved_work` bulk-
1341
+ # enqueues EVERY eligible row on `board.completed`, not just this one.
1342
+ # If an earlier row in the same sweep tick already triggered the
1343
+ # enqueue for this assignment, this call's `changed` list comes back
1344
+ # without it (nothing new to do) even though it genuinely is queued —
1345
+ # checking `changed` alone would misreport a real outcome as
1346
+ # `no_action`. Check queue membership directly instead of relying
1347
+ # solely on `changed`.
1348
+ if any(m.assignment_id == work.assignment_id for m in load_queue()):
1349
+ return StalledDispatchAction(
1350
+ kind="enqueued",
1351
+ detail=(
1352
+ f"{work.assignment_id} already enqueued for merge (queued "
1353
+ "earlier in this sweep tick)"
1354
+ ),
1355
+ )
1356
+ return StalledDispatchAction(
1357
+ kind="no_action",
1358
+ detail="enqueue_approved_work made no change for this assignment",
1359
+ )
1360
+
1361
+ if detection.reason == "merge_conflict_unresolved":
1362
+ from coord.conflict_fix import ( # noqa: PLC0415
1363
+ dispatch_conflict_fix,
1364
+ has_prior_conflict_fix,
1365
+ )
1366
+ from coord.merge_queue import load_queue # noqa: PLC0415
1367
+
1368
+ entry = next(
1369
+ (m for m in load_queue() if m.assignment_id == work.assignment_id), None,
1370
+ )
1371
+ if entry is None:
1372
+ return StalledDispatchAction(
1373
+ kind="no_action", detail="merge queue entry no longer found",
1374
+ )
1375
+ if has_prior_conflict_fix(board, entry.assignment_id):
1376
+ return StalledDispatchAction(
1377
+ kind="skipped_human_required",
1378
+ detail="conflict-fix already active or its retry cap was already hit",
1379
+ )
1380
+ fix = dispatch_conflict_fix(entry, board, config, prefer_machine=work.machine_name)
1381
+ if fix is None:
1382
+ return StalledDispatchAction(
1383
+ kind="no_action",
1384
+ detail="dispatch_conflict_fix declined (no machine / no repo_path)",
1385
+ )
1386
+ return StalledDispatchAction(
1387
+ kind="conflict_fix_dispatched",
1388
+ detail=f"conflict-fix {fix.assignment_id} dispatched to {fix.machine_name}",
1389
+ )
1390
+
1391
+ return StalledDispatchAction(
1392
+ kind="no_action", detail=f"no dispatch arm for reason={detection.reason!r}",
1393
+ )
1394
+
1395
+
1396
+ def post_stalled_pipeline_dispatch(
1397
+ detection: StalledDetection, action: StalledDispatchAction, config: Config,
1398
+ ) -> None:
1399
+ """Post the #1478 auto-dispatch outcome comment and mark notified.
1400
+
1401
+ Posted INSTEAD OF :func:`post_stalled_pipeline` when
1402
+ :func:`dispatch_stalled_pipeline_action` actually dispatched something
1403
+ for this row (see that function's *kind* values) — the two write to the
1404
+ same GitHub thread, so posting both would leave a directly
1405
+ contradictory "nothing was dispatched automatically" comment sitting
1406
+ right above this one.
1407
+ """
1408
+ repo = config.repo(detection.repo_name)
1409
+ repo_github = repo.github if repo is not None else None
1410
+ if not repo_github:
1411
+ return
1412
+ body = format_stalled_pipeline_dispatch(
1413
+ assignment_id=detection.assignment_id,
1414
+ repo_name=detection.repo_name,
1415
+ issue_number=detection.issue_number,
1416
+ reason=detection.reason,
1417
+ action_kind=action.kind,
1418
+ action_detail=action.detail,
1419
+ )
1420
+ github_ops.post_issue_comment(repo_github, detection.issue_number, body)
1421
+ mark_notified(_stalled_notified_key(detection.assignment_id), EVENT_STALLED)
1422
+
1423
+
1424
+ def _agent_status(host: str, port: int = AGENT_PORT, timeout: float = 5.0) -> dict | None:
1425
+ try:
1426
+ resp = httpx.get(f"http://{host}:{port}/status", timeout=timeout)
1427
+ resp.raise_for_status()
1428
+ return resp.json()
1429
+ except (httpx.HTTPError, httpx.TimeoutException):
1430
+ return None
1431
+
1432
+
1433
+ def detect_transitions(config: Config) -> list[tuple[Transition, dict, dict]]:
1434
+ """Return (transition, dispatch_record, agent_assignment) for each
1435
+ assignment whose terminal state has not yet been notified.
1436
+
1437
+ Splitting detection from posting makes the loop testable without
1438
+ mocking GitHub.
1439
+ """
1440
+ dispatched = load_dispatched()
1441
+ if not dispatched:
1442
+ return []
1443
+ notified = load_notified()
1444
+ by_id = {r["assignment_id"]: r for r in dispatched}
1445
+
1446
+ # Collect machine hostnames we care about
1447
+ machines_by_name = {m.name: m for m in config.machines}
1448
+ needed = {r["machine_name"] for r in dispatched if r["assignment_id"] not in notified}
1449
+
1450
+ transitions: list[tuple[Transition, dict, dict]] = []
1451
+ for machine_name in needed:
1452
+ machine = machines_by_name.get(machine_name)
1453
+ if machine is None:
1454
+ continue
1455
+ status = _agent_status(machine.host)
1456
+ if status is None:
1457
+ continue
1458
+ for entry in status.get("completed", []):
1459
+ aid = entry.get("id")
1460
+ record = by_id.get(aid)
1461
+ if record is None or aid in notified:
1462
+ continue
1463
+ entry_status = entry.get("status")
1464
+ # Cancelled-on-agent for an assignment the DB already marks done
1465
+ # is cleanup noise (e.g. operator ran POST /cancel to unstick a
1466
+ # hung reap). Don't post a false failure for it.
1467
+ db_status = (record.get("status") or "").lower()
1468
+ if entry_status == "cancelled" and db_status == "done":
1469
+ continue
1470
+ if entry_status == "done":
1471
+ event = EVENT_COMPLETION
1472
+ elif entry_status in ("failed", "cancelled"):
1473
+ event = EVENT_FAILURE
1474
+ elif entry_status == "advisory":
1475
+ # #448: advisory (0-commit clean exit) — post a distinctive
1476
+ # GitHub comment so operators who rely on GitHub (not just
1477
+ # coord status) know the worker finished with no code change
1478
+ # and that human review is needed.
1479
+ event = EVENT_ADVISORY
1480
+ else:
1481
+ continue
1482
+ transitions.append(
1483
+ (
1484
+ Transition(
1485
+ assignment_id=aid,
1486
+ machine_name=record["machine_name"],
1487
+ repo_name=record["repo_name"],
1488
+ issue_number=record["issue_number"],
1489
+ event=event,
1490
+ exit_code=entry.get("exit_code"),
1491
+ ),
1492
+ record,
1493
+ entry,
1494
+ )
1495
+ )
1496
+ return transitions
1497
+
1498
+
1499
+ def detect_stuck(config: Config) -> list[tuple[StuckDetection, dict]]:
1500
+ """Scan active worker logs for STUCK signals.
1501
+
1502
+ Returns (StuckDetection, dispatch_record) for each stuck worker that
1503
+ hasn't already been notified as stuck.
1504
+ """
1505
+ dispatched = load_dispatched()
1506
+ if not dispatched:
1507
+ return []
1508
+ notified = load_notified()
1509
+ by_id = {r["assignment_id"]: r for r in dispatched}
1510
+
1511
+ machines_by_name = {m.name: m for m in config.machines}
1512
+
1513
+ # Only look at assignments that haven't been notified at all (still active)
1514
+ # and haven't already been notified as stuck.
1515
+ active_records = [
1516
+ r for r in dispatched
1517
+ if r["assignment_id"] not in notified
1518
+ and _stuck_notified_key(r["assignment_id"]) not in notified
1519
+ ]
1520
+ if not active_records:
1521
+ return []
1522
+
1523
+ # Group by machine
1524
+ by_machine: dict[str, list[dict]] = {}
1525
+ for r in active_records:
1526
+ by_machine.setdefault(r["machine_name"], []).append(r)
1527
+
1528
+ results: list[tuple[StuckDetection, dict]] = []
1529
+ for machine_name, records in by_machine.items():
1530
+ machine = machines_by_name.get(machine_name)
1531
+ if machine is None:
1532
+ continue
1533
+ status = _agent_status(machine.host)
1534
+ if status is None:
1535
+ continue
1536
+
1537
+ # Build lookup of active entries by id
1538
+ active_by_id: dict[str, dict] = {}
1539
+ for entry in status.get("active", []):
1540
+ eid = entry.get("id")
1541
+ if eid:
1542
+ active_by_id[eid] = entry
1543
+
1544
+ for record in records:
1545
+ aid = record["assignment_id"]
1546
+ entry = active_by_id.get(aid)
1547
+ if entry is None:
1548
+ continue
1549
+
1550
+ stuck_message: str | None = None
1551
+ log_path: str | None = None
1552
+
1553
+ # Check progress data from agent status
1554
+ progress = entry.get("progress")
1555
+ if progress and progress.get("stuck"):
1556
+ stuck_message = progress["stuck"]
1557
+ log_path = entry.get("log_path")
1558
+
1559
+ # Also try parsing the log file directly
1560
+ entry_log = entry.get("log_path")
1561
+ if entry_log and not stuck_message:
1562
+ try:
1563
+ # #1710: thread the dispatch record's resolved provider
1564
+ # name through so a non-claude worker's log parses via
1565
+ # its own provider rather than always assuming claude.
1566
+ parsed = parse_progress(
1567
+ entry_log, provider_name=record.get("provider_name"),
1568
+ )
1569
+ if parsed.stuck:
1570
+ stuck_message = parsed.stuck
1571
+ log_path = entry_log
1572
+ except Exception: # noqa: BLE001
1573
+ pass
1574
+
1575
+ if stuck_message:
1576
+ results.append(
1577
+ (
1578
+ StuckDetection(
1579
+ assignment_id=aid,
1580
+ machine_name=record["machine_name"],
1581
+ repo_name=record["repo_name"],
1582
+ issue_number=record["issue_number"],
1583
+ stuck_message=stuck_message,
1584
+ log_path=log_path,
1585
+ ),
1586
+ record,
1587
+ )
1588
+ )
1589
+
1590
+ return results
1591
+
1592
+
1593
+ def post_stuck(detection: StuckDetection, record: dict) -> None:
1594
+ """Post a stuck comment to GitHub and mark notified."""
1595
+ body = format_stuck(
1596
+ assignment_id=detection.assignment_id,
1597
+ machine_name=detection.machine_name,
1598
+ repo_name=detection.repo_name,
1599
+ issue_number=detection.issue_number,
1600
+ stuck_message=detection.stuck_message,
1601
+ )
1602
+ github_ops.post_issue_comment(
1603
+ record["repo_github"], detection.issue_number, body
1604
+ )
1605
+ mark_notified(_stuck_notified_key(detection.assignment_id), EVENT_STUCK)
1606
+
1607
+
1608
+ def _capture_completion_summary(transition: Transition, entry: dict) -> None:
1609
+ """#874: parse the worker's ### Summary block and persist it on the row.
1610
+
1611
+ Tries the local log first, then falls back to the agent's /logs/<id>
1612
+ endpoint for remote-agent assignments. Silent on failure — a worker
1613
+ that emits no summary leaves the field NULL without error.
1614
+ """
1615
+ from coord.progress import ( # noqa: PLC0415
1616
+ parse_completion_summary_from_agent,
1617
+ parse_completion_summary_from_log,
1618
+ )
1619
+ from coord.state import update_assignment_completion_summary # noqa: PLC0415
1620
+
1621
+ prose: str | None = None
1622
+ log_path = entry.get("log_path")
1623
+ if log_path:
1624
+ try:
1625
+ prose = parse_completion_summary_from_log(Path(log_path))
1626
+ except Exception as exc: # noqa: BLE001
1627
+ log.debug(
1628
+ "_capture_completion_summary: failed to parse local log for %s: %s",
1629
+ transition.assignment_id, exc,
1630
+ )
1631
+
1632
+ if prose is None:
1633
+ # Local log unavailable (remote-agent assignment) — fetch via the
1634
+ # agent's /logs/<id> endpoint. Same fallback used by smoke tests.
1635
+ host = _agent_host(transition.machine_name)
1636
+ if host:
1637
+ try:
1638
+ prose = parse_completion_summary_from_agent(host, transition.assignment_id)
1639
+ except Exception as exc: # noqa: BLE001
1640
+ log.debug(
1641
+ "_capture_completion_summary: failed to fetch from agent %s for %s: %s",
1642
+ host, transition.assignment_id, exc,
1643
+ )
1644
+
1645
+ if prose is None:
1646
+ # No ### Summary block anywhere — leave completion_summary NULL.
1647
+ return
1648
+ try:
1649
+ update_assignment_completion_summary(transition.assignment_id, prose)
1650
+ except Exception as exc: # noqa: BLE001
1651
+ log.warning(
1652
+ "_capture_completion_summary: failed to persist summary for %s: %s",
1653
+ transition.assignment_id, exc,
1654
+ )
1655
+
1656
+
1657
+ def _capture_smoke_tests(transition: Transition, entry: dict) -> None:
1658
+ """#252: parse the worker's SMOKE_TESTS block and persist it on the row.
1659
+
1660
+ Tries the local log first, then falls back to the agent's /logs/<id>
1661
+ endpoint for remote-agent assignments (mirrors the plan and review
1662
+ capture paths). Silent on failure.
1663
+ """
1664
+ from coord.progress import ( # noqa: PLC0415
1665
+ parse_smoke_tests_from_agent,
1666
+ parse_smoke_tests_from_log,
1667
+ )
1668
+ from coord.state import update_assignment_smoke_tests # noqa: PLC0415
1669
+
1670
+ parsed: list[str] | None = None
1671
+ log_path = entry.get("log_path")
1672
+ if log_path:
1673
+ try:
1674
+ parsed = parse_smoke_tests_from_log(Path(log_path))
1675
+ except Exception as exc: # noqa: BLE001
1676
+ log.debug(
1677
+ "_capture_smoke_tests: failed to parse local log for %s: %s",
1678
+ transition.assignment_id, exc,
1679
+ )
1680
+
1681
+ if parsed is None:
1682
+ # Local log unavailable (remote-agent assignment) — fetch via the
1683
+ # agent's /logs/<id> endpoint. Same fallback the plan and review
1684
+ # paths use.
1685
+ host = _agent_host(transition.machine_name)
1686
+ if host:
1687
+ try:
1688
+ parsed = parse_smoke_tests_from_agent(host, transition.assignment_id)
1689
+ except Exception as exc: # noqa: BLE001
1690
+ log.debug(
1691
+ "_capture_smoke_tests: failed to fetch from agent %s for %s: %s",
1692
+ host, transition.assignment_id, exc,
1693
+ )
1694
+
1695
+ if parsed is None:
1696
+ # No SMOKE_TESTS block anywhere — leave smoke_tests NULL so the
1697
+ # TUI shows the graceful-degradation placeholder.
1698
+ return
1699
+ try:
1700
+ update_assignment_smoke_tests(transition.assignment_id, parsed)
1701
+ except Exception as exc: # noqa: BLE001
1702
+ log.warning(
1703
+ "_capture_smoke_tests: failed to persist list for %s: %s",
1704
+ transition.assignment_id, exc,
1705
+ )
1706
+
1707
+
1708
+ def _capture_cost(transition: Transition, entry: dict, record: dict | None = None) -> None:
1709
+ """#208/#546: parse the worker's final cost+tokens and persist them.
1710
+
1711
+ Preferred source is the local stream-json log (cheap, no network).
1712
+ Falls back to the agent's status entry, which carries ``cost_so_far``
1713
+ / ``total_cost_usd`` reported live by the worker. Tokens are only
1714
+ available from the log (not from the agent status dict), so they are
1715
+ captured when the local log exists. Either path is best-effort —
1716
+ failure is silent so it can't block the comment post.
1717
+
1718
+ #1710: *record* (the dispatch record from ``load_dispatched()``) carries
1719
+ ``provider_name`` — threaded into :func:`coord.usage.parse_usage_from_log`
1720
+ so cost/token parsing uses the assignment's actual provider instead of
1721
+ always assuming claude. ``None`` (no record, or predates #324) falls back
1722
+ to the claude default, unchanged from before #1710.
1723
+ """
1724
+ from coord.state import update_assignment_cost, update_assignment_tokens # noqa: PLC0415
1725
+ from coord.usage import parse_usage_from_log # noqa: PLC0415
1726
+
1727
+ cost: float | None = None
1728
+ input_tokens: int = 0
1729
+ output_tokens: int = 0
1730
+ cache_creation_tokens: int = 0
1731
+ cache_read_tokens: int = 0
1732
+ provider_name = (record or {}).get("provider_name")
1733
+
1734
+ log_path = entry.get("log_path")
1735
+ if log_path:
1736
+ try:
1737
+ parsed = parse_usage_from_log(Path(log_path), provider_name=provider_name)
1738
+ if parsed is not None:
1739
+ if parsed.total_cost_usd > 0:
1740
+ cost = parsed.total_cost_usd
1741
+ # #546: also capture token counts from the same parse.
1742
+ input_tokens = parsed.input_tokens
1743
+ output_tokens = parsed.output_tokens
1744
+ cache_creation_tokens = parsed.cache_creation_tokens
1745
+ cache_read_tokens = parsed.cache_read_tokens
1746
+ except Exception as exc: # noqa: BLE001
1747
+ log.debug(
1748
+ "_capture_cost: failed to parse log for %s: %s",
1749
+ transition.assignment_id, exc,
1750
+ )
1751
+
1752
+ if cost is None:
1753
+ # Fall back to the live value the agent had at reap time.
1754
+ remote_cost = entry.get("total_cost_usd") or entry.get("cost_so_far")
1755
+ if remote_cost is not None:
1756
+ try:
1757
+ cost = float(remote_cost)
1758
+ except (TypeError, ValueError):
1759
+ cost = None
1760
+
1761
+ # #667: token fallback — when the local log was absent/unreadable the
1762
+ # token counts are still 0. The agent now includes them in the /status
1763
+ # completed entry, so read them from there.
1764
+ if input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens == 0:
1765
+ try:
1766
+ input_tokens = int(entry.get("input_tokens") or 0)
1767
+ output_tokens = int(entry.get("output_tokens") or 0)
1768
+ cache_creation_tokens = int(entry.get("cache_creation_tokens") or 0)
1769
+ cache_read_tokens = int(entry.get("cache_read_tokens") or 0)
1770
+ except (TypeError, ValueError):
1771
+ pass
1772
+
1773
+ if cost is not None and cost > 0:
1774
+ try:
1775
+ update_assignment_cost(transition.assignment_id, cost)
1776
+ except Exception as exc: # noqa: BLE001
1777
+ log.warning(
1778
+ "_capture_cost: failed to persist cost for %s: %s",
1779
+ transition.assignment_id, exc,
1780
+ )
1781
+
1782
+ # #546: persist token counts (best-effort; silent on missing columns).
1783
+ if input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens > 0:
1784
+ try:
1785
+ update_assignment_tokens(
1786
+ transition.assignment_id,
1787
+ input_tokens=input_tokens,
1788
+ output_tokens=output_tokens,
1789
+ cache_creation_tokens=cache_creation_tokens,
1790
+ cache_read_tokens=cache_read_tokens,
1791
+ )
1792
+ except Exception as exc: # noqa: BLE001
1793
+ log.warning(
1794
+ "_capture_cost: failed to persist tokens for %s: %s",
1795
+ transition.assignment_id, exc,
1796
+ )
1797
+
1798
+
1799
+ def _persist_review_verdict(assignment_id: str, verdict: str) -> None:
1800
+ """Store the parsed reviewer verdict on the review assignment row.
1801
+
1802
+ #253: consumed by ``coord.merge_queue.has_approved_review`` so the merge
1803
+ gate can refuse to merge work whose review hasn't approved. Best-effort;
1804
+ a DB error is logged and swallowed (the merge gate falls back to "no
1805
+ approval found" which is the safe answer).
1806
+ """
1807
+ if verdict not in ("approve", "request-changes"):
1808
+ return
1809
+ try:
1810
+ from coord.db import get_connection # noqa: PLC0415
1811
+
1812
+ conn = get_connection()
1813
+ with conn:
1814
+ conn.execute(
1815
+ "UPDATE assignments SET review_verdict = ? WHERE assignment_id = ?",
1816
+ (verdict, assignment_id),
1817
+ )
1818
+ except Exception as exc: # noqa: BLE001
1819
+ log.warning(
1820
+ "Failed to persist review_verdict for %s: %s", assignment_id, exc
1821
+ )
1822
+
1823
+
1824
+ def _persist_review_findings(assignment_id: str, verdict: str, body: str) -> None:
1825
+ """#bounce: persist both verdict + findings body in one shot.
1826
+
1827
+ Mirrors `_persist_review_verdict` (which we keep for callers that
1828
+ only have the verdict) but also caches the body so `coord bounce`
1829
+ can skip the slow HTTP log fetch. Best-effort; a DB error is
1830
+ logged and swallowed.
1831
+ """
1832
+ if verdict not in ("approve", "request-changes"):
1833
+ return
1834
+ try:
1835
+ from coord.state import update_assignment_review_findings # noqa: PLC0415
1836
+
1837
+ update_assignment_review_findings(
1838
+ assignment_id, verdict=verdict, body=body,
1839
+ )
1840
+ except Exception as exc: # noqa: BLE001
1841
+ log.warning(
1842
+ "Failed to persist review_findings for %s: %s", assignment_id, exc
1843
+ )
1844
+
1845
+
1846
+ def _fetch_raw_log_text(transition: Transition, entry: dict) -> str | None:
1847
+ """Best-effort raw log text for #1956/#1348 diagnostics.
1848
+
1849
+ Mirrors the local-file-then-agent-fetch fallback :func:`_try_parse_and_post_review`
1850
+ itself uses to PARSE the log, but returns the raw text instead — the
1851
+ diagnostic detectors (:func:`coord.review.detect_end_review_without_verdict`,
1852
+ :func:`coord.review.detect_unparsed_review_marker`) need the text the
1853
+ strict parser already rejected, not another parse attempt. Returns
1854
+ ``None`` on any I/O failure — diagnostics are best-effort by design and
1855
+ must never be the reason ``coord notify`` raises.
1856
+ """
1857
+ log_path = entry.get("log_path")
1858
+ if log_path:
1859
+ try:
1860
+ return Path(log_path).read_text(encoding="utf-8", errors="replace")
1861
+ except OSError:
1862
+ pass
1863
+ host = _agent_host(transition.machine_name)
1864
+ if host:
1865
+ try:
1866
+ resp = httpx.get(
1867
+ f"http://{host}:{AGENT_PORT}/logs/{transition.assignment_id}",
1868
+ timeout=15.0,
1869
+ )
1870
+ resp.raise_for_status()
1871
+ return resp.text
1872
+ except (httpx.HTTPError, httpx.TimeoutException):
1873
+ return None
1874
+ return None
1875
+
1876
+
1877
+ def _warn_missing_review_verdict(
1878
+ transition: Transition, entry: dict, diagnostic: list,
1879
+ ) -> None:
1880
+ """#1956: when a review's structured verdict could not be parsed, make it
1881
+ LOUD instead of silent — run the #1348/#1956 diagnostics against the raw
1882
+ log text and ``log.warning`` a recovery command.
1883
+
1884
+ Before this, a review that reached ``END_REVIEW`` with a full body but
1885
+ no ``REVIEW_VERDICT:`` header (quadraui#533's live incident — grepping
1886
+ the raw log found the string exactly once, inside the briefing's own
1887
+ instructions, never in an assistant message) landed ``status="done"``
1888
+ with ``review_verdict IS NULL`` and nothing anywhere said so; the merge
1889
+ gate just read ``review_required`` forever. Appends whichever
1890
+ diagnostic fired (if any) to *diagnostic* so :func:`post_transition` can
1891
+ tailor the GitHub-visible completion comment too — the operator should
1892
+ not have to go spelunking in ``coord notify``'s own log to learn this.
1893
+ Best-effort throughout: a failure to even fetch the raw text is
1894
+ swallowed, matching this module's "never crash notify" contract.
1895
+ """
1896
+ from coord.review import ( # noqa: PLC0415
1897
+ detect_end_review_without_verdict,
1898
+ detect_unparsed_review_marker,
1899
+ )
1900
+
1901
+ text = _fetch_raw_log_text(transition, entry)
1902
+ if not text:
1903
+ return
1904
+ aid = transition.assignment_id
1905
+ log_path = entry.get("log_path")
1906
+ recover_hint = (
1907
+ f"coord report-result --assignment {aid} "
1908
+ "--verdict <approve|request-changes> --verdict-source recovered "
1909
+ '--verdict-reason "<why>" --body-file <extracted-review.md>'
1910
+ )
1911
+
1912
+ end_marker = detect_end_review_without_verdict(text, transcript_path=log_path)
1913
+ if end_marker is not None:
1914
+ log.warning(
1915
+ "review %s: reviewer wrote END_REVIEW but never emitted "
1916
+ "REVIEW_VERDICT: anywhere (#1956) — this is NOT a crashed/"
1917
+ "truncated session, the verdict is very likely recoverable "
1918
+ "from the transcript. Recover with:\n %s\nExcerpt before "
1919
+ "END_REVIEW:\n%s",
1920
+ aid, recover_hint, end_marker.excerpt,
1921
+ )
1922
+ diagnostic.append(end_marker)
1923
+ return
1924
+
1925
+ marker = detect_unparsed_review_marker(text, transcript_path=log_path)
1926
+ if marker is not None:
1927
+ log.warning(
1928
+ "review %s: a REVIEW_VERDICT: marker is present but malformed "
1929
+ "(#1348, detected word=%r) — the strict parser rejected it. "
1930
+ "Recover with:\n %s",
1931
+ aid, marker.verdict_word, recover_hint,
1932
+ )
1933
+ diagnostic.append(marker)
1934
+ return
1935
+
1936
+ log.debug(
1937
+ "review %s: no REVIEW_VERDICT:/END_REVIEW markers found at all — "
1938
+ "likely a crashed or truncated session, not a #1956/#1348 "
1939
+ "recoverable case",
1940
+ aid,
1941
+ )
1942
+
1943
+
1944
+ def _try_parse_and_post_review(
1945
+ transition: Transition,
1946
+ record: dict,
1947
+ entry: dict,
1948
+ duration: float | None,
1949
+ *,
1950
+ _diagnostic: list | None = None,
1951
+ ) -> bool:
1952
+ """Parse reviewer findings from the log and post as a PR review or issue comment.
1953
+
1954
+ Returns True if a review was successfully posted (either as a ``gh pr review``
1955
+ or as an issue comment when no PR number is available), False on any failure.
1956
+ Silently swallows all errors so callers can fall back gracefully.
1957
+
1958
+ *_diagnostic* (#1956): optional out-parameter, mirroring
1959
+ ``coord.interactive``'s identically-shaped convention for #1348. When a
1960
+ list is supplied and the structured verdict cannot be parsed, whichever
1961
+ of :func:`coord.review.detect_end_review_without_verdict` /
1962
+ :func:`coord.review.detect_unparsed_review_marker` fires is appended to
1963
+ it, so the caller can tailor the fallback GitHub comment instead of a
1964
+ generic "could not be extracted" message every single time.
1965
+ """
1966
+ from coord.review import parse_review_from_log, parse_review_from_agent # noqa: PLC0415
1967
+
1968
+ log_path = entry.get("log_path")
1969
+ findings = None
1970
+ if log_path:
1971
+ try:
1972
+ findings = parse_review_from_log(log_path)
1973
+ except Exception as exc: # noqa: BLE001
1974
+ log.warning("Failed to parse review log for %s: %s", transition.assignment_id, exc)
1975
+
1976
+ # Local file unavailable (worker ran on a remote agent whose log isn't on
1977
+ # this filesystem) — fetch via the agent's /logs endpoint and parse the
1978
+ # same way. Agents never use gh; the coordinator pulls + posts.
1979
+ if findings is None:
1980
+ host = _agent_host(transition.machine_name)
1981
+ if host:
1982
+ try:
1983
+ findings = parse_review_from_agent(host, transition.assignment_id)
1984
+ except Exception as exc: # noqa: BLE001
1985
+ log.warning(
1986
+ "Failed to fetch review log from agent %s for %s: %s",
1987
+ host, transition.assignment_id, exc,
1988
+ )
1989
+
1990
+ if findings is None:
1991
+ if _diagnostic is not None:
1992
+ try:
1993
+ _warn_missing_review_verdict(transition, entry, _diagnostic)
1994
+ except Exception as exc: # noqa: BLE001 — diagnostics must never crash notify
1995
+ log.debug(
1996
+ "review %s: #1956 diagnostic itself failed: %s",
1997
+ transition.assignment_id, exc,
1998
+ )
1999
+ return False
2000
+
2001
+ # #253: persist the parsed verdict on the review assignment so the merge
2002
+ # gate can refuse to merge work whose review hasn't approved. Independent
2003
+ # of auto_loop (which may be disabled in config).
2004
+ # #bounce: also persist the findings.body so `coord bounce` (and the
2005
+ # future per-stage display) can read it from the DB without re-fetching
2006
+ # the worker's full log.
2007
+ _persist_review_findings(
2008
+ transition.assignment_id, findings.verdict, findings.body
2009
+ )
2010
+
2011
+ review_target = record.get("review_target")
2012
+ repo_github = record["repo_github"]
2013
+
2014
+ # Determine whether review_target is a PR number (integer string) or a branch.
2015
+ pr_number: int | None = None
2016
+ if review_target:
2017
+ try:
2018
+ pr_number = int(review_target)
2019
+ except (ValueError, TypeError):
2020
+ pr_number = None
2021
+
2022
+ # #248: prepend a machine-readable header so the TUI / coordinator can
2023
+ # surface the verdict + counts without re-ingesting the prose body.
2024
+ body_with_header = _attach_review_header(
2025
+ findings.body,
2026
+ verdict=findings.verdict,
2027
+ reviewer_machine=transition.machine_name,
2028
+ assignment_id=transition.assignment_id,
2029
+ )
2030
+
2031
+ if pr_number is not None:
2032
+ try:
2033
+ github_ops.post_pr_review(repo_github, pr_number, findings.verdict, body_with_header)
2034
+ mark_review_posted(transition.assignment_id)
2035
+ return True
2036
+ except Exception as exc: # noqa: BLE001
2037
+ # GitHub rejects self-reviews (same user who opened the PR can't
2038
+ # review it via the API). Log the actual error and fall through to
2039
+ # post the findings as an issue comment instead of silently failing.
2040
+ log.warning(
2041
+ "Failed to post PR review for %s PR#%s via gh: %s — "
2042
+ "falling back to issue comment",
2043
+ transition.assignment_id, pr_number, exc,
2044
+ )
2045
+ # Fall through to the issue-comment path below.
2046
+
2047
+ # No PR number available, or gh pr review was rejected — post findings as
2048
+ # an issue comment so they are never silently lost.
2049
+ verdict_label = "✅ Approved" if findings.verdict == "approve" else "⚠️ Changes Requested"
2050
+ if pr_number is not None:
2051
+ preamble = (
2052
+ f"*Reviewer findings could not be posted directly to PR #{pr_number} "
2053
+ f"(gh pr review was rejected — likely a self-review restriction). "
2054
+ f"Findings are reproduced here.*"
2055
+ )
2056
+ else:
2057
+ preamble = (
2058
+ "*Reviewer could not post directly to a PR (no PR number available). "
2059
+ "Findings are reproduced here.*"
2060
+ )
2061
+ body = (
2062
+ f"## Review Complete — {verdict_label}\n\n"
2063
+ f"{preamble}\n\n"
2064
+ f"{body_with_header}"
2065
+ )
2066
+ try:
2067
+ github_ops.post_issue_comment(repo_github, transition.issue_number, body)
2068
+ mark_review_posted(transition.assignment_id)
2069
+ return True
2070
+ except Exception as exc: # noqa: BLE001
2071
+ log.warning(
2072
+ "Failed to post review comment for %s: %s", transition.assignment_id, exc
2073
+ )
2074
+ return False
2075
+
2076
+
2077
+ def _attach_review_header(
2078
+ body: str,
2079
+ *,
2080
+ verdict: str,
2081
+ reviewer_machine: str | None = None,
2082
+ assignment_id: str | None = None,
2083
+ ) -> str:
2084
+ """#248: prepend the machine-readable header line to a review *body*.
2085
+
2086
+ Counts are derived heuristically from the body's markdown sections.
2087
+ The header always carries the verdict; counts/identity fields are
2088
+ omitted when unavailable.
2089
+ """
2090
+ from coord.review import ( # noqa: PLC0415 — local import keeps import graph clean
2091
+ estimate_review_counts, format_review_header,
2092
+ )
2093
+ blocking, nonblocking, nits = estimate_review_counts(body)
2094
+ header = format_review_header(
2095
+ verdict=verdict,
2096
+ reviewer_machine=reviewer_machine,
2097
+ assignment_id=assignment_id,
2098
+ blocking=blocking,
2099
+ nonblocking=nonblocking,
2100
+ nits=nits,
2101
+ )
2102
+ return f"{header}\n\n{body}"
2103
+
2104
+
2105
+ def _try_parse_and_post_plan(
2106
+ transition: Transition,
2107
+ record: dict,
2108
+ entry: dict,
2109
+ duration: float | None,
2110
+ ) -> bool:
2111
+ """Try to parse a WorkerPlan from the worker log and post it to GitHub.
2112
+
2113
+ Returns True if a plan comment was successfully posted, False otherwise.
2114
+ Silently swallows all errors so callers can fall back gracefully.
2115
+ """
2116
+ from coord.plan_parser import parse_plan_from_log, parse_plan_from_agent # noqa: PLC0415
2117
+
2118
+ log_path = entry.get("log_path")
2119
+ worker_plan = None
2120
+ if log_path:
2121
+ try:
2122
+ worker_plan = parse_plan_from_log(log_path)
2123
+ except Exception as exc: # noqa: BLE001
2124
+ log.warning("Failed to parse plan log for %s: %s", transition.assignment_id, exc)
2125
+
2126
+ # Local log unavailable (worker ran on a remote agent — entry.log_path
2127
+ # is the agent's filesystem path, not the coordinator's). Mirror the
2128
+ # review path: fall back to the agent's /logs/<id> endpoint. Without
2129
+ # this, every remote-agent plan got posted as a generic "completion"
2130
+ # comment and the structured plan was lost (we hit this on quadraui#264).
2131
+ if worker_plan is None or worker_plan.is_empty():
2132
+ host = _agent_host(transition.machine_name)
2133
+ if host:
2134
+ try:
2135
+ worker_plan = parse_plan_from_agent(host, transition.assignment_id)
2136
+ except Exception as exc: # noqa: BLE001
2137
+ log.warning(
2138
+ "Failed to fetch plan log from agent %s for %s: %s",
2139
+ host, transition.assignment_id, exc,
2140
+ )
2141
+
2142
+ if worker_plan is None or worker_plan.is_empty():
2143
+ return False
2144
+
2145
+ try:
2146
+ body = format_plan(
2147
+ assignment_id=transition.assignment_id,
2148
+ machine_name=transition.machine_name,
2149
+ repo_name=transition.repo_name,
2150
+ issue_number=transition.issue_number,
2151
+ plan=worker_plan,
2152
+ duration_seconds=duration,
2153
+ )
2154
+ github_ops.post_issue_comment(
2155
+ record["repo_github"], transition.issue_number, body
2156
+ )
2157
+ # Cache the parsed plan in the state directory.
2158
+ save_plan(transition.assignment_id, worker_plan.to_dict())
2159
+ except Exception as exc: # noqa: BLE001
2160
+ log.warning("Failed to post plan comment for %s: %s", transition.assignment_id, exc)
2161
+ return False
2162
+
2163
+ return True
2164
+
2165
+
2166
+ def _capture_claude_session_id(transition: Transition, entry: dict) -> None:
2167
+ """#315: persist the worker's claude session ID to the coordinator DB.
2168
+
2169
+ The agent captures this from the ``system.init`` event in the worker log
2170
+ and includes it in the ``/status`` response. Once stored in the DB,
2171
+ ``coord chat-continue`` can read it and pass ``--resume <id>`` to the
2172
+ next worker so it loads the prior conversation. Best-effort; a missing
2173
+ ID just means chat-continue will refuse with a clear error.
2174
+ """
2175
+ session_id = entry.get("claude_session_id")
2176
+ if not isinstance(session_id, str) or not session_id:
2177
+ return
2178
+ try:
2179
+ from coord.state import update_assignment_claude_session_id # noqa: PLC0415
2180
+ update_assignment_claude_session_id(transition.assignment_id, session_id)
2181
+ except Exception as exc: # noqa: BLE001
2182
+ log.warning(
2183
+ "_capture_claude_session_id: failed for %s: %s",
2184
+ transition.assignment_id, exc,
2185
+ )
2186
+
2187
+
2188
+ def post_transition(transition: Transition, record: dict, entry: dict) -> None:
2189
+ """Post the GitHub comment for one transition and mark it notified."""
2190
+ started = entry.get("started_at")
2191
+ finished = entry.get("finished_at")
2192
+ duration = (finished - started) if (started and finished) else None
2193
+ # #208: capture worker cost as soon as the assignment completes — the
2194
+ # value is in the worker's final stream-json result event and would
2195
+ # otherwise be lost when the agent prunes the log. Best-effort:
2196
+ # local log → remote agent entry → skip.
2197
+ _capture_cost(transition, entry, record)
2198
+ # #252: capture the worker-emitted SMOKE_TESTS block at the same
2199
+ # moment so the TUI can render it under the Test stage. Same
2200
+ # best-effort discipline — failure is silent.
2201
+ _capture_smoke_tests(transition, entry)
2202
+ # #874: capture the worker's ### Summary prose block at the same moment
2203
+ # so the board has a durable, queryable summary field. Best-effort.
2204
+ _capture_completion_summary(transition, entry)
2205
+ # #315: persist the worker's claude session ID so chat-continue can
2206
+ # pass --resume to the next worker. Best-effort; silent on failure.
2207
+ _capture_claude_session_id(transition, entry)
2208
+ common = dict(
2209
+ assignment_id=transition.assignment_id,
2210
+ machine_name=transition.machine_name,
2211
+ repo_github=record["repo_github"],
2212
+ repo_name=transition.repo_name,
2213
+ issue_number=transition.issue_number,
2214
+ duration_seconds=duration,
2215
+ log_path=entry.get("log_path"),
2216
+ )
2217
+ assignment_type = record.get("type", "work")
2218
+ if transition.event == EVENT_COMPLETION and assignment_type in (
2219
+ "refinement",
2220
+ "milestone-chat",
2221
+ ):
2222
+ # #315: refinement chat turns are developer-side conversation — do NOT
2223
+ # post completion comments to GitHub. Each turn would spam the issue
2224
+ # with identical "assignment completed" noise. We still capture cost,
2225
+ # smoke tests, and session ID above; just skip the GitHub post.
2226
+ # #770: milestone-chat is dispatched AGAINST the tracking issue
2227
+ # itself (unlike refinement's target issue, this one is the live
2228
+ # planning document a human reads) — a generic completion comment on
2229
+ # every conversational turn would be even noisier here. The
2230
+ # meaningful GitHub-visible effect is the tracking issue's body
2231
+ # update via `coord milestone write-order`, not a completion comment.
2232
+ mark_notified(
2233
+ transition.assignment_id,
2234
+ transition.event,
2235
+ branch=entry.get("branch"),
2236
+ )
2237
+ elif transition.event == EVENT_COMPLETION and assignment_type == "plan":
2238
+ # For plan assignments, post the structured plan comment. Fall back
2239
+ # to a standard completion comment if the log can't be parsed.
2240
+ posted = _try_parse_and_post_plan(transition, record, entry, duration)
2241
+ if not posted:
2242
+ post_completion(exit_code=transition.exit_code or 0, **common)
2243
+ mark_notified(
2244
+ transition.assignment_id,
2245
+ EVENT_PLAN if posted else EVENT_COMPLETION,
2246
+ branch=entry.get("branch"),
2247
+ )
2248
+ elif transition.event == EVENT_COMPLETION and assignment_type == "review":
2249
+ # For review assignments, parse the structured findings and post as a
2250
+ # PR review (or issue comment when no PR number is available). Fall
2251
+ # back to a plain completion comment noting the parse failure — #1956:
2252
+ # tailored per-diagnostic instead of one generic message, so an
2253
+ # operator reading GitHub (not `coord notify`'s own log) can ALSO see
2254
+ # that a verdict is recoverable, not just that parsing failed.
2255
+ _diag: list = []
2256
+ posted = _try_parse_and_post_review(
2257
+ transition, record, entry, duration, _diagnostic=_diag,
2258
+ )
2259
+ if not posted:
2260
+ from coord.review import EndReviewWithoutVerdict # noqa: PLC0415
2261
+
2262
+ if _diag and isinstance(_diag[0], EndReviewWithoutVerdict):
2263
+ fallback_summary = (
2264
+ "Review assignment completed and the reviewer wrote END_REVIEW, "
2265
+ "but never emitted the machine-readable REVIEW_VERDICT: header "
2266
+ "(#1956) — this is NOT a crashed/truncated session, the verdict "
2267
+ "is very likely recoverable from the transcript. Recover with: "
2268
+ f"`coord report-result --assignment {transition.assignment_id} "
2269
+ "--verdict <approve|request-changes> --verdict-source recovered "
2270
+ '--verdict-reason "<why>" --body-file <extracted-review.md>`.'
2271
+ )
2272
+ elif _diag:
2273
+ fallback_summary = (
2274
+ "Review assignment completed but a REVIEW_VERDICT: marker in "
2275
+ "the worker log was malformed and could not be parsed (#1348) "
2276
+ "— the verdict is likely still recoverable from the transcript. "
2277
+ "Recover with: "
2278
+ f"`coord report-result --assignment {transition.assignment_id} "
2279
+ "--verdict <approve|request-changes> --verdict-source recovered "
2280
+ '--verdict-reason "<why>" --body-file <extracted-review.md>`.'
2281
+ )
2282
+ else:
2283
+ fallback_summary = (
2284
+ "Review assignment completed but findings could not be extracted "
2285
+ "from the worker log. The reviewer may not have produced the "
2286
+ "expected structured output (REVIEW_VERDICT / REVIEW_BODY / END_REVIEW)."
2287
+ )
2288
+ post_completion(
2289
+ exit_code=transition.exit_code or 0,
2290
+ summary=fallback_summary,
2291
+ **common,
2292
+ )
2293
+ mark_notified(
2294
+ transition.assignment_id,
2295
+ transition.event,
2296
+ branch=entry.get("branch"),
2297
+ )
2298
+ elif transition.event == EVENT_COMPLETION and assignment_type == "conflict-fix":
2299
+ post_completion(exit_code=transition.exit_code or 0, **common)
2300
+ mark_notified(
2301
+ transition.assignment_id,
2302
+ transition.event,
2303
+ branch=entry.get("branch"),
2304
+ )
2305
+ # Re-enqueue the parent merge entry so the next `coord merge` retries.
2306
+ # This mirrors the reconcile() path — whichever runs first wins.
2307
+ parent_id = record.get("review_of_assignment_id")
2308
+ if parent_id:
2309
+ from coord.reconcile import on_conflict_fix_done # noqa: PLC0415
2310
+ on_conflict_fix_done(
2311
+ parent_assignment_id=parent_id,
2312
+ fix_assignment_id=transition.assignment_id,
2313
+ machine_name=transition.machine_name,
2314
+ succeeded=True,
2315
+ )
2316
+ elif transition.event == EVENT_COMPLETION and assignment_type == "smoke":
2317
+ # #1021: propagate the headless smoke exit code to the parent work
2318
+ # row's Test verdict so the merge gate is satisfied automatically.
2319
+ post_completion(exit_code=transition.exit_code or 0, **common)
2320
+ mark_notified(
2321
+ transition.assignment_id,
2322
+ transition.event,
2323
+ branch=entry.get("branch"),
2324
+ )
2325
+ parent_id = record.get("review_of_assignment_id")
2326
+ if parent_id:
2327
+ # Guard: only auto-certify when the issue's test-mode is "auto"
2328
+ # or unset (no label). A "smoke" label means the TUI offers an
2329
+ # interactive smoke agent — do NOT auto-certify here.
2330
+ from coord.state import get_issue_test_mode, record_test_verdict # noqa: PLC0415
2331
+ test_mode = get_issue_test_mode(
2332
+ transition.repo_name, transition.issue_number
2333
+ )
2334
+ if test_mode != "smoke":
2335
+ succeeded = (transition.exit_code or 0) == 0
2336
+ # #1384: no `smoke_test=` argument needed — the writer
2337
+ # (`state._record_test_verdict_local`) derives the legacy
2338
+ # mirror from `test_state`, so a headless smoke FAILURE lands
2339
+ # as test_state='failed' AND smoke_test='fail' and stays
2340
+ # reachable from `coord fix`.
2341
+ record_test_verdict(
2342
+ assignment_id=parent_id,
2343
+ test_state="passed" if succeeded else "failed",
2344
+ test_reason="headless smoke",
2345
+ )
2346
+ elif transition.event == EVENT_FAILURE and assignment_type == "smoke":
2347
+ # #1605: the Test-stage WORKER itself died (a dead agent, a killed
2348
+ # process group, a terminal API error — anything short of the
2349
+ # worker actually printing `SMOKE: pass`/`SMOKE: fail`) without ever
2350
+ # producing a verdict. Mirrors the EVENT_COMPLETION branch above
2351
+ # (#1021) but for the terminal-FAILED case that branch never
2352
+ # covered: before this, a failed smoke row left the parent's
2353
+ # `test_state` at whatever `dispatch_smoke` set it to (almost always
2354
+ # `"running"`, #1426) — forever, since no gate ever resolves
2355
+ # `"running"` on its own. That is the #1598 incident: a smoke worker
2356
+ # died on a terminal API error and the issue was permanently
2357
+ # stranded with the board reporting a plausible in-progress state.
2358
+ # #1797: `push_failure_reason` is the same column too — see the
2359
+ # identical `or` chain in `coord.reconcile.reconcile_completed_assignments`.
2360
+ _failure_reason = (
2361
+ entry.get("usage_limit_reason")
2362
+ or entry.get("api_error_reason")
2363
+ or entry.get("push_failure_reason")
2364
+ )
2365
+ post_failure(
2366
+ exit_code=transition.exit_code,
2367
+ error=entry.get("error") or _failure_reason or "",
2368
+ **common,
2369
+ )
2370
+ mark_notified(
2371
+ transition.assignment_id,
2372
+ transition.event,
2373
+ branch=entry.get("branch"),
2374
+ failure_reason=_failure_reason,
2375
+ exit_code=transition.exit_code,
2376
+ )
2377
+ parent_id = record.get("review_of_assignment_id")
2378
+ if parent_id:
2379
+ from coord.reconcile import ( # noqa: PLC0415
2380
+ propagate_smoke_terminal_failure,
2381
+ )
2382
+ propagate_smoke_terminal_failure(
2383
+ parent_assignment_id=parent_id,
2384
+ failure_reason=_failure_reason,
2385
+ )
2386
+ elif transition.event == EVENT_COMPLETION:
2387
+ post_completion(exit_code=transition.exit_code or 0, **common)
2388
+ mark_notified(
2389
+ transition.assignment_id,
2390
+ transition.event,
2391
+ branch=entry.get("branch"),
2392
+ )
2393
+ elif transition.event == EVENT_ADVISORY:
2394
+ # #448: 0-commit clean exit — post a distinctive advisory comment.
2395
+ # No ❌ emoji, no re-dispatch suggestion; just surfaces the advisory
2396
+ # state on GitHub so operators not watching coord status are informed.
2397
+ post_advisory(
2398
+ reason=entry.get("zero_commit_reason") or "",
2399
+ **common,
2400
+ )
2401
+ mark_notified(
2402
+ transition.assignment_id,
2403
+ transition.event,
2404
+ branch=entry.get("branch"),
2405
+ )
2406
+ else:
2407
+ # #1605/#1797: carry the agent's own diagnostic (a usage-limit kill,
2408
+ # a terminal API-error classification, or an auth-shaped push
2409
+ # failure — all stamped by `AgentServer._reap` onto this same
2410
+ # `/status` completed entry — see
2411
+ # `coord.reconcile.reconcile_completed_assignments`'s identical
2412
+ # `or`) through to `mark_notified` so a `status='failed'` row is
2413
+ # never left with both `failure_reason` and `exit_code` null. This
2414
+ # is the branch a `type="work"` push-auth failure actually hits
2415
+ # (none of the type-specific `elif`s above match "work"), so
2416
+ # `_failure_reason` also feeds `error=` below — otherwise the
2417
+ # posted GitHub failure comment's `error` field is blank for
2418
+ # exactly the failure #1797 exists to surface.
2419
+ _failure_reason = (
2420
+ entry.get("usage_limit_reason")
2421
+ or entry.get("api_error_reason")
2422
+ or entry.get("push_failure_reason")
2423
+ )
2424
+ post_failure(
2425
+ exit_code=transition.exit_code,
2426
+ error=entry.get("error") or _failure_reason or "",
2427
+ **common,
2428
+ )
2429
+ mark_notified(
2430
+ transition.assignment_id,
2431
+ transition.event,
2432
+ branch=entry.get("branch"),
2433
+ failure_reason=_failure_reason,
2434
+ exit_code=transition.exit_code,
2435
+ )
2436
+
2437
+
2438
+ def post_orphaned_review_findings(
2439
+ config: Config,
2440
+ repo_name: str | None = None,
2441
+ ) -> list[str]:
2442
+ """Walk done-review assignments with unposted findings and attempt to post.
2443
+
2444
+ Handles two scenarios that cause findings to be lost:
2445
+
2446
+ 1. The agent reported the assignment as 'done' but notify never ran (or
2447
+ ran at the wrong time) — no notification record in the DB at all.
2448
+ 2. Notify ran and posted a fallback completion comment (because the log
2449
+ couldn't be parsed at that time), but findings were never extracted.
2450
+
2451
+ In both cases ``review_posted_at`` is NULL on the assignment row.
2452
+
2453
+ The function queries each relevant agent server to discover the log path,
2454
+ then re-parses and re-posts. If the agent is offline or its completed
2455
+ list no longer contains the assignment, the entry is silently skipped
2456
+ so ``coord notify`` stays non-fatal.
2457
+
2458
+ Returns a list of assignment_ids for which findings were successfully posted.
2459
+ Optionally filter to a single *repo_name*.
2460
+ """
2461
+ from coord.review import parse_review_from_log # noqa: PLC0415
2462
+
2463
+ candidates = load_done_reviews_needing_post(repo_name=repo_name)
2464
+ if not candidates:
2465
+ return []
2466
+
2467
+ notified = load_notified()
2468
+ machines_by_name = {m.name: m for m in config.machines}
2469
+
2470
+ # Group by machine so we query each agent server once.
2471
+ by_machine: dict[str, list[dict]] = {}
2472
+ for row in candidates:
2473
+ by_machine.setdefault(row["machine_name"], []).append(row)
2474
+
2475
+ posted_ids: list[str] = []
2476
+ for machine_name, rows in by_machine.items():
2477
+ machine = machines_by_name.get(machine_name)
2478
+ if machine is None:
2479
+ log.debug("post_orphaned: unknown machine %r — skipping %d assignment(s)", machine_name, len(rows))
2480
+ continue
2481
+
2482
+ status = _agent_status(machine.host)
2483
+ log_by_id: dict[str, str] = {}
2484
+ if status:
2485
+ for entry in status.get("completed", []):
2486
+ eid = entry.get("id")
2487
+ lp = entry.get("log_path")
2488
+ if eid and lp:
2489
+ log_by_id[eid] = lp
2490
+
2491
+ for row in rows:
2492
+ aid = row["assignment_id"]
2493
+ log_path = log_by_id.get(aid)
2494
+ findings = None
2495
+ # Try local file first (cheap) — works when notify runs on the
2496
+ # same host as the agent. Falls back to fetching via HTTP so the
2497
+ # coordinator can post reviews from any machine.
2498
+ if log_path:
2499
+ try:
2500
+ findings = parse_review_from_log(log_path)
2501
+ except Exception as exc: # noqa: BLE001
2502
+ log.warning("post_orphaned: failed to parse local log for %s: %s", aid, exc)
2503
+ if findings is None and machine.host:
2504
+ from coord.review import parse_review_from_agent # noqa: PLC0415
2505
+ try:
2506
+ findings = parse_review_from_agent(machine.host, aid)
2507
+ except Exception as exc: # noqa: BLE001
2508
+ log.warning(
2509
+ "post_orphaned: failed to fetch log from agent %s for %s: %s",
2510
+ machine.host, aid, exc,
2511
+ )
2512
+ if findings is None:
2513
+ log.debug("post_orphaned: no findings (local + agent both missed) for %s", aid)
2514
+ continue
2515
+
2516
+ # #bounce: cache the parsed findings so coord bounce + the
2517
+ # per-stage display can skip the HTTP fetch on later runs.
2518
+ _persist_review_findings(aid, findings.verdict, findings.body)
2519
+
2520
+ review_target = row.get("review_target")
2521
+ repo_github = row.get("repo_github") or ""
2522
+ issue_number = row.get("issue_number", 0)
2523
+
2524
+ pr_number: int | None = None
2525
+ if review_target:
2526
+ try:
2527
+ pr_number = int(review_target)
2528
+ except (ValueError, TypeError):
2529
+ pr_number = None
2530
+
2531
+ # Build a preamble that distinguishes retroactive posts from fresh ones.
2532
+ already_notified = aid in notified
2533
+ if already_notified:
2534
+ retro_note = (
2535
+ "\n\n*Note: a completion comment was posted earlier but findings "
2536
+ "could not be extracted at that time. These are the retroactive findings.*"
2537
+ )
2538
+ else:
2539
+ retro_note = ""
2540
+
2541
+ # #248: same header injection as the live path.
2542
+ body_with_header = _attach_review_header(
2543
+ findings.body,
2544
+ verdict=findings.verdict,
2545
+ reviewer_machine=machine.name,
2546
+ assignment_id=aid,
2547
+ )
2548
+
2549
+ posted = False
2550
+ if pr_number is not None:
2551
+ try:
2552
+ github_ops.post_pr_review(repo_github, pr_number, findings.verdict, body_with_header + retro_note)
2553
+ posted = True
2554
+ except Exception as exc: # noqa: BLE001
2555
+ log.warning(
2556
+ "post_orphaned: failed gh pr review for %s PR#%s: %s — "
2557
+ "falling back to issue comment",
2558
+ aid, pr_number, exc,
2559
+ )
2560
+
2561
+ if not posted:
2562
+ verdict_label = "✅ Approved" if findings.verdict == "approve" else "⚠️ Changes Requested"
2563
+ if pr_number is not None:
2564
+ preamble = (
2565
+ f"*Reviewer findings could not be posted directly to PR #{pr_number} "
2566
+ f"(gh pr review was rejected — likely a self-review restriction). "
2567
+ f"Findings are reproduced here.*"
2568
+ )
2569
+ else:
2570
+ preamble = (
2571
+ "*Reviewer could not post directly to a PR (no PR number available). "
2572
+ "Findings are reproduced here.*"
2573
+ )
2574
+ body = (
2575
+ f"## Review Complete — {verdict_label}\n\n"
2576
+ f"{preamble}{retro_note}\n\n"
2577
+ f"{body_with_header}"
2578
+ )
2579
+ try:
2580
+ github_ops.post_issue_comment(repo_github, issue_number, body)
2581
+ posted = True
2582
+ except Exception as exc: # noqa: BLE001
2583
+ log.warning("post_orphaned: failed to post comment for %s: %s", aid, exc)
2584
+
2585
+ if posted:
2586
+ mark_review_posted(aid)
2587
+ if not already_notified:
2588
+ mark_notified(aid, EVENT_COMPLETION)
2589
+ posted_ids.append(aid)
2590
+ log.info("post_orphaned: posted findings for review %s", aid)
2591
+
2592
+ return posted_ids
2593
+
2594
+
2595
+ def _dispatch_board_pending_smoke(config: Config) -> None:
2596
+ """Load the board, dispatch any pending Test-stage smoke, and save.
2597
+
2598
+ #1426: `dispatch_pending_smoke` (:mod:`coord.smoke`) was previously only
2599
+ ever called from `reconcile()`'s per-item loop, and the ONLY sanctioned
2600
+ caller of the full `reconcile()` is `coord resume`, a human-invoked
2601
+ command. A thin-client setup driven purely by `coord-notify.timer` (which
2602
+ calls `notify.run()`, not `reconcile()`) never dispatched the Test stage
2603
+ at all — the exact gap `scripts/drive-issue.sh` had to paper over with a
2604
+ local `scripts/coord-test-runner.sh` subprocess (#1395). Mirrors
2605
+ :func:`_dispatch_board_pending_reviews` exactly, and is safe to call even
2606
+ when the board file doesn't exist.
2607
+ """
2608
+ from coord.board_service import read_board, write_board
2609
+ from coord.smoke import dispatch_pending_smoke
2610
+
2611
+ board = read_board()
2612
+ dispatched = dispatch_pending_smoke(board, config)
2613
+ if dispatched:
2614
+ write_board(board)
2615
+
2616
+
2617
+ def _dispatch_board_pending_reviews(config: Config) -> None:
2618
+ """Load the board, dispatch any pending reviews, and save.
2619
+
2620
+ Mirrors the review-dispatch loop in reconcile() so that ``coord notify``
2621
+ also triggers review dispatch — not just ``coord status --reconcile``.
2622
+ Safe to call even when the board file doesn't exist.
2623
+ """
2624
+ from coord.board_service import read_board, write_board
2625
+ from coord.review import dispatch_pending_reviews, dispatch_scoped_reviews_for_queue
2626
+
2627
+ # #749: read_board()/write_board() route through the daemon when
2628
+ # board_service is configured, so this no longer silently no-ops on a
2629
+ # thin client's empty local DB — read_board() falls back to an
2630
+ # effectively-empty board when nothing has been saved yet, which is
2631
+ # exactly as harmless as the old "return early" guard.
2632
+ board = read_board()
2633
+
2634
+ # #465: review fires immediately on work completion — no manual smoke
2635
+ # prerequisite. Mirrors reconcile(). dispatch_pending_reviews() enforces
2636
+ # the bulk-dispatch flood guard (per-pass cap + surge gate, incident
2637
+ # 2026-06-08) and the #459 active-fix dedupe, so notify can't flood either.
2638
+ dispatched = dispatch_pending_reviews(board, config)
2639
+
2640
+ # #1476: same scoped-re-review dispatch reconcile() runs, so a conflict-fix
2641
+ # that voids an approval by changing content gets a delta-scoped re-review
2642
+ # from `coord notify` too, not just `coord status --reconcile`.
2643
+ dispatched = dispatched + dispatch_scoped_reviews_for_queue(board, config)
2644
+
2645
+ if dispatched:
2646
+ write_board(board)
2647
+
2648
+
2649
+ def _sweep_stalled_pipeline(
2650
+ config: Config, *, terminal_cache: dict | None = None,
2651
+ ) -> list[StalledDetection]:
2652
+ """Detect #1441 stalled-pipeline rows, post one comment per row, and —
2653
+ when ``config.pipeline.auto_dispatch_stalled`` is on — dispatch the
2654
+ action the original transition would have taken (#1478).
2655
+
2656
+ Loads its own board (rather than accepting one) so mutations from a
2657
+ dispatched action (a freshly-enqueued merge entry, a newly dispatched
2658
+ review/fix/conflict-fix, ``board.review_state`` flips) can be persisted
2659
+ back via ``write_board`` — mirrors ``_dispatch_board_pending_reviews``/
2660
+ ``_dispatch_board_pending_smoke`` above. A comment-posting failure for
2661
+ one row must not stop the sweep from reaching the rest (matches every
2662
+ other best-effort loop in this module) — the ``continue`` on failure
2663
+ means that row's ``notified`` key is never set, so it is picked back up
2664
+ on the next tick rather than silently dropped.
2665
+
2666
+ An unexpected exception *from* ``dispatch_stalled_pipeline_action``
2667
+ itself (e.g. a momentarily-unreachable agent during ``dispatch_review``/
2668
+ ``dispatch_conflict_fix``) gets the same treatment, not the "declined"
2669
+ treatment: no comment is posted and the row is NOT marked notified, so
2670
+ it is retried on the next tick rather than permanently foreclosed. A
2671
+ considered decline (``no_action`` returned normally — no capable
2672
+ machine, gate not satisfied, entry vanished) still posts the diagnostic
2673
+ comment and marks notified per the one-shot "act once" guardrail; only a
2674
+ genuine raised exception gets the retry treatment.
2675
+ """
2676
+ from coord.board_service import read_board, write_board
2677
+
2678
+ board = read_board()
2679
+ detections = detect_stalled_pipeline(config, board=board, terminal_cache=terminal_cache)
2680
+
2681
+ posted: list[StalledDetection] = []
2682
+ board_dirty = False
2683
+ for detection, work in detections:
2684
+ try:
2685
+ action = dispatch_stalled_pipeline_action(
2686
+ detection, work, board, config, terminal_cache=terminal_cache,
2687
+ )
2688
+ except Exception: # noqa: BLE001
2689
+ log.exception(
2690
+ "dispatch_stalled_pipeline_action: unexpected error for %s — "
2691
+ "not marking notified so this row is retried next tick",
2692
+ detection.assignment_id,
2693
+ )
2694
+ continue
2695
+
2696
+ dispatched = action.kind in _STALLED_DISPATCH_KINDS
2697
+ try:
2698
+ if dispatched:
2699
+ post_stalled_pipeline_dispatch(detection, action, config)
2700
+ else:
2701
+ post_stalled_pipeline(detection, config)
2702
+ except Exception: # noqa: BLE001
2703
+ continue
2704
+ posted.append(detection)
2705
+ if dispatched:
2706
+ board_dirty = True
2707
+
2708
+ # #1478 guardrail: "log every auto-dispatch to the audit trail with
2709
+ # the detection that triggered it" — business-tier (never dropped
2710
+ # by the operational/business audit-level gate) for an actual
2711
+ # dispatch; operational-tier for a no-op/skip, so the "nothing
2712
+ # happened" rows don't inflate the business audit stream but are
2713
+ # still reconstructable when `audit.level` includes operational.
2714
+ try:
2715
+ from coord.audit import record_audit # noqa: PLC0415
2716
+
2717
+ record_audit(
2718
+ tier="business" if dispatched else "operational",
2719
+ category="pipeline",
2720
+ event_type="stalled_pipeline_auto_dispatch",
2721
+ actor="coordinator",
2722
+ summary=(
2723
+ f"stalled-pipeline sweep ({detection.reason}) -> {action.kind} "
2724
+ f"for {detection.repo_name}#{detection.issue_number}"
2725
+ ),
2726
+ repo=detection.repo_name,
2727
+ issue=detection.issue_number,
2728
+ assignment_id=detection.assignment_id,
2729
+ machine=detection.machine_name,
2730
+ details={
2731
+ "stalled_reason": detection.reason,
2732
+ "stalled_detail": detection.detail,
2733
+ "action_kind": action.kind,
2734
+ "action_detail": action.detail,
2735
+ },
2736
+ )
2737
+ except Exception: # noqa: BLE001
2738
+ log.exception(
2739
+ "record_audit failed for stalled dispatch %s", detection.assignment_id,
2740
+ )
2741
+
2742
+ if board_dirty:
2743
+ try:
2744
+ write_board(board)
2745
+ except Exception: # noqa: BLE001
2746
+ log.exception("write_board failed after stalled-pipeline dispatch")
2747
+
2748
+ return posted
2749
+
2750
+
2751
+ @dataclass(frozen=True)
2752
+ class DrainResult:
2753
+ """What one :func:`run_drain` pass actually did.
2754
+
2755
+ ``skipped_locked`` is the "someone else is draining" outcome, which is a
2756
+ success, not an error — the next tick picks the work up.
2757
+
2758
+ ``propagated_verdicts`` (#1663) lists the review assignment IDs whose
2759
+ verdict this pass wrote through onto the parent **work** row. Never
2760
+ implies a fix worker was dispatched — the drain cannot dispatch one.
2761
+ """
2762
+
2763
+ transitions: list[Transition] = field(default_factory=list)
2764
+ orphaned_findings: list[str] = field(default_factory=list)
2765
+ propagated_verdicts: list[str] = field(default_factory=list)
2766
+ skipped_locked: bool = False
2767
+
2768
+ def __bool__(self) -> bool:
2769
+ """Truthy when this pass advanced something (for terse log guards)."""
2770
+ return bool(
2771
+ self.transitions or self.orphaned_findings or self.propagated_verdicts
2772
+ )
2773
+
2774
+
2775
+ def run_drain(
2776
+ config: Config,
2777
+ *,
2778
+ lock_path: "Path | None" = None,
2779
+ lock_timeout: float = 0.0,
2780
+ ) -> DrainResult:
2781
+ """The pipeline's **clock** (#1616) — advance terminal rows' side effects.
2782
+
2783
+ ``reconcile_completed_assignments`` (the daemon's passive tick) writes
2784
+ ``status='done'`` and stops there, by contract. Everything downstream —
2785
+ ``finished_at``, the completion comment, the #1076/#1152 test-gate
2786
+ backfill, the Test-stage smoke dispatch, the review dispatch, the #1610
2787
+ ``finalizing`` → verdict capture — is a side effect of ``coord notify``.
2788
+ On this fleet ``coord-notify.timer`` is deliberately disabled and the only
2789
+ caller of ``coord notify`` is a live ``coord drive``'s **stall nudge**, so
2790
+ a completed stage sat until the stall detector gave up (9 min on #1123,
2791
+ 47 min on #1122) — and rows with no drive at all (vimcode#611/#613) sat
2792
+ until a human poked the daemon. This function is what the daemon tick
2793
+ calls so the pipeline advances on a clock instead of on an accident.
2794
+
2795
+ **Scope is the whole point — this is deliberately NOT ``run()``.**
2796
+ ``coord notify`` triggers five side effects; four are bookkeeping with no
2797
+ race and no cost if repeated, and one spawns a metered worker. The line
2798
+ sits at exactly one place:
2799
+
2800
+ ========================================== ======= ===================================
2801
+ side effect here? why
2802
+ ========================================== ======= ===================================
2803
+ ``finished_at`` stamped yes no race, no cost
2804
+ completion comment posted yes ``coord:`` markers make it idempotent
2805
+ test-gate backfill (#1076/#1152) yes no race, no cost
2806
+ Test-stage smoke dispatch (#1426) yes the gate review waits on; see below
2807
+ orphaned review findings posted yes comment + verdict capture only
2808
+ review dispatch yes guarded; see below
2809
+ verdict → parent work row (#1663) yes no race, no cost; see below
2810
+ merge enqueue n/a the daemon tick already runs
2811
+ ``enqueue_approved_work`` right after
2812
+ **work dispatch** **no** stays with a drive or a human
2813
+ **fix-round dispatch** (``auto_loop``) **no** this is where #476/#477 lives
2814
+ **stalled-pipeline sweep/dispatch** **no** can dispatch work (#1478)
2815
+ ========================================== ======= ===================================
2816
+
2817
+ #1663 is what the "verdict → parent work row" row costs to learn. That
2818
+ write — ``work.review_state='done'``, ``work.review_verdict=<verdict>``,
2819
+ ``record_work_review_verdict``, the merge-queue refresh — is bookkeeping by
2820
+ every criterion in this table, but it lived *inside*
2821
+ ``auto_loop.process_review_completion`` alongside the fix dispatch, and
2822
+ excluding the function excluded both. So every verdict the daemon consumed
2823
+ instead of a human's ``coord notify`` was captured on the review row and
2824
+ dropped on the way to the work row, for **both** verdicts — the approve
2825
+ case stayed invisible only because ``merge_queue.has_approved_review``
2826
+ reads the *review* row. ``coord drive``, the TUI's Review stage and the
2827
+ auto-loop all read the *work* row, so an approved issue simply stopped:
2828
+ 2026-08-01's overnight batch reviewed five issues clean and merged none of
2829
+ them in 4h02m. The propagation half is now separately callable
2830
+ (``auto_loop.propagate_review_verdict_for_transition``) and step 5 calls
2831
+ only that; fix dispatch is as unreachable from here as it ever was.
2832
+
2833
+ Why review dispatch is in and fix dispatch is out — the asymmetry is the
2834
+ whole argument. #476/#477, the incident that got ``coord-notify.timer``
2835
+ disabled, was duplicate **fix-workers**: they create conflicting branches
2836
+ on the same issue and cost real recovery work. A duplicate *review* costs
2837
+ a few dollars and a redundant comment. Withholding reviews inherits a
2838
+ mitigation for a risk that does not apply to them. And bookkeeping-only
2839
+ is not sufficient: work→review is the most frequent boundary in the
2840
+ pipeline and the one that stalled #1122, so a drain that stamps state but
2841
+ will not dispatch reviews fixes the *watched* half and leaves the
2842
+ unwatched half exactly as broken as before.
2843
+
2844
+ Smoke dispatch rides along because ``dispatch_pending_reviews`` holds
2845
+ review dispatch until ``test_state`` is passed/skipped when
2846
+ ``pipeline.test_precedes_review()`` (#1612). Draining reviews without
2847
+ ever dispatching the Test stage would just move the stall one box left —
2848
+ that is #1605. It is a Test-stage worker on the work's own branch, not a
2849
+ second author on a fresh branch, so it carries none of the #476/#477
2850
+ shape.
2851
+
2852
+ Stuck / needs-attention detection is deliberately absent: those are
2853
+ *notifications*, not pipeline advancement, and giving the daemon a
2854
+ periodic detector is #1632's job (which is blocked on this).
2855
+
2856
+ **Concurrency.** The whole pass runs under ``~/.coord/notify.lock`` —
2857
+ literally :class:`coord.filelock.FileLock`, the same class on the same
2858
+ path ``coord drive``'s ``run_notify()`` takes — so a drive's nudge and the
2859
+ daemon's clock can never both be inside ``dispatch_pending_reviews``,
2860
+ which reads ``review_state == 'pending'`` and writes ``'dispatched'``
2861
+ non-atomically (two concurrent passes would both see ``pending`` and
2862
+ dispatch two reviews). ``lock_timeout`` defaults to **0.0**
2863
+ (non-blocking): if another drain holds it, return ``skipped_locked`` and
2864
+ let the next tick retry rather than pinning a threadpool worker.
2865
+
2866
+ Every step is independently try/except'd — one failing side effect must
2867
+ never sink the rest of the pass, and a drain must never crash the daemon.
2868
+ """
2869
+ from coord.filelock import FileLock, LockBusy, notify_lock_path # noqa: PLC0415
2870
+
2871
+ lock = FileLock(lock_path if lock_path is not None else notify_lock_path())
2872
+ try:
2873
+ lock.acquire(timeout=lock_timeout)
2874
+ except LockBusy:
2875
+ log.debug("notify drain: %s held elsewhere — skipping this pass", lock.path)
2876
+ return DrainResult(skipped_locked=True)
2877
+ try:
2878
+ return _run_drain_locked(config)
2879
+ finally:
2880
+ lock.release()
2881
+
2882
+
2883
+ def _run_drain_locked(config: Config) -> DrainResult:
2884
+ """:func:`run_drain`'s body, with the lock already held.
2885
+
2886
+ Split out so tests can exercise the side effects without the lock and the
2887
+ lock without the side effects.
2888
+ """
2889
+ # Refresh the agent-host cache so _try_parse_and_post_review (and any other
2890
+ # helper using _agent_host) can resolve hostnames without threading config
2891
+ # through every call. Mirrors run().
2892
+ global _AGENT_HOSTS
2893
+ _AGENT_HOSTS = {m.name: m.host for m in config.machines}
2894
+
2895
+ # Step 1: post completion/failure/advisory/plan/review comments for rows
2896
+ # the agent reports terminal. This is what stamps `finished_at` (via
2897
+ # mark_notified) and captures cost / SMOKE_TESTS / summary / session id /
2898
+ # the review verdict + findings. Idempotent: detect_transitions skips any
2899
+ # assignment already in the `notifications` table, so a second drain over
2900
+ # the same board posts nothing.
2901
+ posted: list[Transition] = []
2902
+ # #1663: (transition, record, entry) for every review that completed in
2903
+ # THIS pass, so step 5 can propagate its verdict onto the parent work row.
2904
+ review_completions: list[tuple[Transition, dict, dict]] = []
2905
+ try:
2906
+ from coord.comments import EVENT_COMPLETION # noqa: PLC0415
2907
+
2908
+ for transition, record, entry in detect_transitions(config):
2909
+ try:
2910
+ post_transition(transition, record, entry)
2911
+ except Exception: # noqa: BLE001 — one bad row must not sink the pass
2912
+ log.exception(
2913
+ "notify drain: post_transition failed for %s",
2914
+ transition.assignment_id,
2915
+ )
2916
+ continue
2917
+ posted.append(transition)
2918
+ if (
2919
+ record.get("type") == "review"
2920
+ and transition.event == EVENT_COMPLETION
2921
+ ):
2922
+ review_completions.append((transition, record, entry))
2923
+ except Exception: # noqa: BLE001
2924
+ log.exception("notify drain: detect_transitions failed")
2925
+
2926
+ # Step 2: dispatch pending Test-stage smoke (#1426). Runs BEFORE review
2927
+ # dispatch to mirror the pipeline's Work -> Test -> Review order.
2928
+ try:
2929
+ _dispatch_board_pending_smoke(config)
2930
+ except Exception: # noqa: BLE001
2931
+ log.exception("notify drain: smoke dispatch failed")
2932
+
2933
+ # Step 3: dispatch pending reviews. Carries the #1612 test-precedes-review
2934
+ # gate, the #1076/#1152 test-gate backfill, the #946 enqueue gate, the
2935
+ # 2026-06-08 flood guard (per-pass cap + surge gate) and the #459 active-fix
2936
+ # dedupe — this is calling existing machinery from a clock, not new
2937
+ # machinery.
2938
+ try:
2939
+ _dispatch_board_pending_reviews(config)
2940
+ except Exception: # noqa: BLE001
2941
+ log.exception("notify drain: review dispatch failed")
2942
+
2943
+ # Step 4: post findings for done-review assignments that were never
2944
+ # processed (agent reported 'cancelled', a human marked the row done, or
2945
+ # notify ran at the wrong time). Comment + verdict capture only.
2946
+ orphaned: list[str] = []
2947
+ try:
2948
+ orphaned = post_orphaned_review_findings(config) or []
2949
+ except Exception: # noqa: BLE001
2950
+ log.exception("notify drain: post_orphaned_review_findings failed")
2951
+
2952
+ # Step 5 (#1663): propagate each captured verdict onto its parent WORK row.
2953
+ #
2954
+ # Steps 1 and 4 both stamp the verdict on the *review* row and stop there.
2955
+ # Everything that reads the *work* row — `coord drive`, the TUI's Review
2956
+ # stage, `_stalled_pipeline`, any state-derived recovery — therefore saw
2957
+ # `review_state='dispatched'` / `review_verdict=NULL` for every verdict the
2958
+ # daemon consumed instead of a human's `coord notify`. The 2026-08-01
2959
+ # overnight batch is the receipt: five issues reviewed, four clean approves,
2960
+ # not one reached its work row, 4h02m of wall clock and zero merges.
2961
+ #
2962
+ # This is the bookkeeping half ONLY — `propagate_review_verdict_for_
2963
+ # transition` cannot reach `_dispatch_fix_for_review`, so the #476/#477
2964
+ # line (no metered fix worker from a clock) is exactly where it was. The
2965
+ # exclusion used to sit at function granularity and took the parent-row
2966
+ # write down with the dispatch; it now sits at side-effect granularity,
2967
+ # which is where the table above always said it belonged.
2968
+ _propagated: list[str] = []
2969
+ if review_completions or orphaned:
2970
+ try:
2971
+ from coord.auto_loop import ( # noqa: PLC0415
2972
+ propagate_review_verdict_for_transition,
2973
+ )
2974
+
2975
+ seen: set[str] = set()
2976
+ # Orphaned rows have no transition tuple (their comment was posted
2977
+ # on an earlier pass, or never). `_load_review_findings` reads the
2978
+ # DB findings cache first — which step 4 just populated — so an
2979
+ # empty record/entry still resolves the verdict without any I/O.
2980
+ pending: list[tuple[str, dict, dict]] = [
2981
+ (t.assignment_id, record, entry)
2982
+ for t, record, entry in review_completions
2983
+ ] + [(aid, {"type": "review"}, {}) for aid in orphaned]
2984
+
2985
+ for aid, record, entry in pending:
2986
+ if not aid or aid in seen:
2987
+ continue
2988
+ seen.add(aid)
2989
+ try:
2990
+ actions = propagate_review_verdict_for_transition(
2991
+ aid, record, entry, config,
2992
+ )
2993
+ except Exception: # noqa: BLE001 — never sink the pass
2994
+ log.exception(
2995
+ "notify drain: verdict propagation failed for %s", aid,
2996
+ )
2997
+ continue
2998
+ for action in actions:
2999
+ log.info(
3000
+ "notify drain: verdict propagation %s: %s (assignment=%s)",
3001
+ action.kind, action.detail, action.assignment_id,
3002
+ )
3003
+ if action.kind in (
3004
+ "approved", "approved_with_nits", "verdict_propagated",
3005
+ "terminal_skip",
3006
+ ):
3007
+ _propagated.append(aid)
3008
+ except Exception: # noqa: BLE001
3009
+ log.exception("notify drain: verdict propagation loop failed")
3010
+
3011
+ return DrainResult(
3012
+ transitions=posted,
3013
+ orphaned_findings=orphaned,
3014
+ propagated_verdicts=_propagated,
3015
+ )
3016
+
3017
+
3018
+ def run(
3019
+ config: Config,
3020
+ ) -> tuple[
3021
+ list[Transition],
3022
+ list[StuckDetection],
3023
+ list[NeedsAttentionDetection],
3024
+ list[StalledDetection],
3025
+ list[LivenessStallDetection],
3026
+ ]:
3027
+ """Detect and post all pending transitions, stuck signals, #846
3028
+ needs-attention detections, #1441 stalled-pipeline detections, and
3029
+ #2048 liveness-auditor stalls.
3030
+
3031
+ Also dispatches any pending reviews found on the saved board so that
3032
+ ``coord notify`` acts as a reliable review-dispatch trigger in addition
3033
+ to ``coord status --reconcile``.
3034
+
3035
+ Returns (posted_transitions, posted_stuck, posted_needs_attention,
3036
+ posted_stalled, posted_liveness). The liveness entry is new in #2048 —
3037
+ appended rather than inserted, following #1441's own precedent: any
3038
+ existing caller unpacking a 4-tuple positionally breaks loudly, which
3039
+ is a good thing (it means the CLI/board/TUI surfacing was actually
3040
+ wired up, not silently skipped). ``posted_liveness`` is always ``[]``
3041
+ when ``config.pipeline.liveness_auditor.enabled`` is ``False`` (the
3042
+ default).
3043
+ """
3044
+ # Refresh the agent-host cache so _try_parse_and_post_review (and any
3045
+ # other helper using _agent_host) can resolve hostnames without
3046
+ # threading config through every call.
3047
+ global _AGENT_HOSTS
3048
+ _AGENT_HOSTS = {m.name: m.host for m in config.machines}
3049
+
3050
+ # #522: one terminal-state cache shared across every gh-hitting check in
3051
+ # this notify run (the auto-loop review/fix dispatches below, and the
3052
+ # #1441 stalled-pipeline sweep at the end), so a burst of activity for
3053
+ # the same merged/closed issue (the #349 ×4 case) costs a single `gh`
3054
+ # round-trip, not one per caller.
3055
+ terminal_cache: dict = {}
3056
+
3057
+ # Collect (transition, record, entry) tuples for review completions so we
3058
+ # can feed them to the auto-loop after all notifications are posted.
3059
+ review_completions: list[tuple[Transition, dict, dict]] = []
3060
+ # Collect (transition, record) tuples for completed fix workers so we can
3061
+ # dispatch a fresh review against each one after notifications are posted.
3062
+ fix_completions: list[tuple[Transition, dict]] = []
3063
+
3064
+ posted: list[Transition] = []
3065
+ for transition, record, entry in detect_transitions(config):
3066
+ try:
3067
+ post_transition(transition, record, entry)
3068
+ except Exception: # noqa: BLE001 — surface to caller; continue with rest
3069
+ continue
3070
+ posted.append(transition)
3071
+ # Track completed reviews for auto-loop processing below.
3072
+ from coord.comments import EVENT_COMPLETION # noqa: PLC0415
3073
+ from coord.auto_loop import FIX_DISPATCH_TYPES # noqa: PLC0415
3074
+ if (
3075
+ record.get("type") == "review"
3076
+ and transition.event == EVENT_COMPLETION
3077
+ ):
3078
+ review_completions.append((transition, record, entry))
3079
+ # Track completed fix workers (type in FIX_DISPATCH_TYPES,
3080
+ # review_of_assignment_id set, title starts with "[fix-") for
3081
+ # auto-loop re-review dispatch. #1176 review: this used to hardcode
3082
+ # type == "work", which meant a completed type="test-author" fix
3083
+ # (added by #1176 itself) never reached run_for_fix_transition —
3084
+ # the same class of bug as #1141 ("test-author was never added to
3085
+ # WORK_LIKE_TYPES"). FIX_DISPATCH_TYPES is the single source of
3086
+ # truth for what _dispatch_fix can emit, so a future fix-dispatch
3087
+ # type can't reintroduce this gap silently.
3088
+ elif (
3089
+ record.get("type") in FIX_DISPATCH_TYPES
3090
+ and transition.event == EVENT_COMPLETION
3091
+ and record.get("review_of_assignment_id")
3092
+ and (record.get("issue_title") or "").startswith("[fix-")
3093
+ ):
3094
+ fix_completions.append((transition, record))
3095
+
3096
+ # Also detect and post stuck signals
3097
+ stuck_posted: list[StuckDetection] = []
3098
+ for detection, record in detect_stuck(config):
3099
+ try:
3100
+ post_stuck(detection, record)
3101
+ except Exception: # noqa: BLE001
3102
+ continue
3103
+ stuck_posted.append(detection)
3104
+
3105
+ # #846: coordinator backstop for long-running / non-converging
3106
+ # assignments. Best-effort, non-fatal — one bad record must not sink the
3107
+ # rest of the notify run.
3108
+ needs_attention_posted: list[NeedsAttentionDetection] = []
3109
+ try:
3110
+ for detection, record in detect_needs_attention(config):
3111
+ try:
3112
+ post_needs_attention(detection, record)
3113
+ except Exception: # noqa: BLE001
3114
+ continue
3115
+ needs_attention_posted.append(detection)
3116
+ except Exception: # noqa: BLE001
3117
+ log.exception("detect_needs_attention: unexpected error")
3118
+
3119
+ # Dispatch pending Test-stage smoke from the saved board (#1426;
3120
+ # best-effort, non-fatal). Runs BEFORE review dispatch to mirror the
3121
+ # pipeline's Work -> Test -> Review order, though ordering isn't load-
3122
+ # bearing here: dispatch_pending_reviews already holds review dispatch
3123
+ # until test_state is passed/skipped regardless of which runs first in
3124
+ # a given pass.
3125
+ try:
3126
+ _dispatch_board_pending_smoke(config)
3127
+ except Exception: # noqa: BLE001
3128
+ pass
3129
+
3130
+ # Dispatch pending reviews from the saved board (best-effort, non-fatal).
3131
+ try:
3132
+ _dispatch_board_pending_reviews(config)
3133
+ except Exception: # noqa: BLE001
3134
+ pass
3135
+
3136
+ # Post findings for done-review assignments that were never processed
3137
+ # (e.g. agent reported 'cancelled', user manually marked done, or notify
3138
+ # ran at the wrong time). Best-effort, non-fatal.
3139
+ try:
3140
+ post_orphaned_review_findings(config)
3141
+ except Exception: # noqa: BLE001
3142
+ log.exception("post_orphaned_review_findings: unexpected error")
3143
+
3144
+ # Auto-loop: for each completed review, optionally dispatch a fix worker.
3145
+ # Runs after notify posts the completion comment so GitHub has the full
3146
+ # review body before any fix briefing references "previous findings".
3147
+ if review_completions:
3148
+ try:
3149
+ from coord.auto_loop import run_for_review_transition # noqa: PLC0415
3150
+ for transition, record, entry in review_completions:
3151
+ try:
3152
+ actions = run_for_review_transition(
3153
+ transition.assignment_id, record, entry, config,
3154
+ terminal_cache=terminal_cache,
3155
+ )
3156
+ for action in actions:
3157
+ log.info(
3158
+ "auto_loop %s: %s (assignment=%s)",
3159
+ action.kind, action.detail, action.assignment_id,
3160
+ )
3161
+ except Exception: # noqa: BLE001
3162
+ log.exception(
3163
+ "auto_loop: error processing review %s",
3164
+ transition.assignment_id,
3165
+ )
3166
+ except Exception: # noqa: BLE001
3167
+ log.exception("auto_loop: unexpected error in review completion loop")
3168
+
3169
+ # Auto-loop: for each completed fix worker, dispatch a fresh review so
3170
+ # the review → fix → re-review cycle closes without manual coord pr invocations.
3171
+ # Runs after review_completions so a simultaneous review + fix completion
3172
+ # in the same notify run is handled review-first.
3173
+ if fix_completions:
3174
+ try:
3175
+ from coord.auto_loop import run_for_fix_transition # noqa: PLC0415
3176
+ for transition, _record in fix_completions:
3177
+ try:
3178
+ actions = run_for_fix_transition(
3179
+ transition.assignment_id, config,
3180
+ terminal_cache=terminal_cache,
3181
+ )
3182
+ for action in actions:
3183
+ log.info(
3184
+ "auto_loop fix_transition %s: %s (assignment=%s)",
3185
+ action.kind, action.detail, action.assignment_id,
3186
+ )
3187
+ except Exception: # noqa: BLE001
3188
+ log.exception(
3189
+ "auto_loop: error processing fix completion %s",
3190
+ transition.assignment_id,
3191
+ )
3192
+ except Exception: # noqa: BLE001
3193
+ log.exception("auto_loop: unexpected error in fix completion loop")
3194
+
3195
+ # #1441/#1478: sweep for pipeline rows whose auto-loop transition
3196
+ # already fired once but which are now stuck on a precondition that
3197
+ # landed too late (the vimcode #602 reference case), post a diagnostic
3198
+ # (or, when `pipeline.auto_dispatch_stalled` is on, act). Runs last,
3199
+ # after the review/fix auto-loop above has had a chance to act on THIS
3200
+ # pass's transitions, so a row that just got a fresh fix/review
3201
+ # dispatched above is not also flagged as stalled in the same pass.
3202
+ # Best-effort, non-fatal — mirrors the #846 needs-attention block above;
3203
+ # the crucial difference from `reconcile()`-only sweepers (see
3204
+ # docs/OPERATING_GOTCHAS.md §7) is that this runs from `coord notify`.
3205
+ stalled_posted: list[StalledDetection] = []
3206
+ try:
3207
+ stalled_posted = _sweep_stalled_pipeline(config, terminal_cache=terminal_cache)
3208
+ except Exception: # noqa: BLE001
3209
+ log.exception("detect_stalled_pipeline: unexpected error")
3210
+
3211
+ # #2048: cheap per-turn liveness auditor. Best-effort, non-fatal —
3212
+ # mirrors the #846 needs-attention block above. Entirely a no-op
3213
+ # (returns [] immediately, no subprocess, no DB write) unless
3214
+ # config.pipeline.liveness_auditor.enabled is set.
3215
+ liveness_posted: list[LivenessStallDetection] = []
3216
+ try:
3217
+ for detection, record in detect_liveness_stall(config):
3218
+ try:
3219
+ post_liveness_stall(detection, record)
3220
+ except Exception: # noqa: BLE001
3221
+ continue
3222
+ liveness_posted.append(detection)
3223
+ except Exception: # noqa: BLE001
3224
+ log.exception("detect_liveness_stall: unexpected error")
3225
+
3226
+ return posted, stuck_posted, needs_attention_posted, stalled_posted, liveness_posted