code-coordinator 0.5.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. code_coordinator-0.5.46.dist-info/METADATA +625 -0
  2. code_coordinator-0.5.46.dist-info/RECORD +295 -0
  3. code_coordinator-0.5.46.dist-info/WHEEL +5 -0
  4. code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
  5. code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
  6. code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
  7. coord/__init__.py +176 -0
  8. coord/_board_mapping.py +229 -0
  9. coord/acceptance.py +468 -0
  10. coord/acceptance_drivers.py +632 -0
  11. coord/agent.py +7517 -0
  12. coord/agent_app.py +1555 -0
  13. coord/agent_update.py +417 -0
  14. coord/agents/opencode/.gitignore +13 -0
  15. coord/agents/opencode/agents/work.md +129 -0
  16. coord/agents/opencode/routing.jsonc +49 -0
  17. coord/audit.py +301 -0
  18. coord/auto_loop.py +1440 -0
  19. coord/board_bool_guard.py +72 -0
  20. coord/board_service.py +141 -0
  21. coord/board_wire.py +309 -0
  22. coord/brain.py +581 -0
  23. coord/branch_model.py +214 -0
  24. coord/cargo_cache.py +258 -0
  25. coord/ci_github.py +386 -0
  26. coord/ci_store.py +560 -0
  27. coord/claim.py +353 -0
  28. coord/cli.py +454 -0
  29. coord/client.py +610 -0
  30. coord/commands/__init__.py +1 -0
  31. coord/commands/_common.py +329 -0
  32. coord/commands/acceptance.py +916 -0
  33. coord/commands/agent_ops.py +1339 -0
  34. coord/commands/audit.py +131 -0
  35. coord/commands/chat.py +320 -0
  36. coord/commands/dispatch.py +1780 -0
  37. coord/commands/dispatch_workers.py +4894 -0
  38. coord/commands/drive.py +616 -0
  39. coord/commands/drive_queue.py +1203 -0
  40. coord/commands/gate_a.py +217 -0
  41. coord/commands/gates.py +89 -0
  42. coord/commands/issues.py +681 -0
  43. coord/commands/lifecycle.py +513 -0
  44. coord/commands/merge.py +1900 -0
  45. coord/commands/milestone.py +2081 -0
  46. coord/commands/plan_followup.py +1243 -0
  47. coord/commands/plans.py +156 -0
  48. coord/commands/release.py +2232 -0
  49. coord/commands/report.py +341 -0
  50. coord/commands/review.py +1523 -0
  51. coord/commands/scorecard.py +252 -0
  52. coord/commands/sessions.py +1930 -0
  53. coord/commands/setup.py +576 -0
  54. coord/commands/status.py +2089 -0
  55. coord/commands/terminal.py +385 -0
  56. coord/commands/test_gate.py +775 -0
  57. coord/commands/tui.py +288 -0
  58. coord/comments.py +718 -0
  59. coord/config.py +3032 -0
  60. coord/conflict_fix.py +633 -0
  61. coord/dao.py +483 -0
  62. coord/dashboard/__init__.py +0 -0
  63. coord/dashboard/fixture.py +376 -0
  64. coord/dashboard/index.html +658 -0
  65. coord/dashboard/server.py +1894 -0
  66. coord/dashboard/terminal.py +382 -0
  67. coord/dashboard/webapp/.gitignore +9 -0
  68. coord/dashboard/webapp/components.json +17 -0
  69. coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
  70. coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
  71. coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
  72. coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
  73. coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
  74. coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
  75. coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
  76. coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
  77. coord/dashboard/webapp/dist/icons/icon.svg +5 -0
  78. coord/dashboard/webapp/dist/index.html +38 -0
  79. coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
  80. coord/dashboard/webapp/dist/sw.js +1 -0
  81. coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
  82. coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
  83. coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
  84. coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
  85. coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
  86. coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
  87. coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
  88. coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
  89. coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
  90. coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
  91. coord/dashboard/webapp/eslint.config.js +20 -0
  92. coord/dashboard/webapp/index.html +37 -0
  93. coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
  94. coord/dashboard/webapp/package-lock.json +10584 -0
  95. coord/dashboard/webapp/package.json +63 -0
  96. coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
  97. coord/dashboard/webapp/playwright.config.ts +93 -0
  98. coord/dashboard/webapp/postcss.config.js +6 -0
  99. coord/dashboard/webapp/public/icons/icon-192.png +0 -0
  100. coord/dashboard/webapp/public/icons/icon-512.png +0 -0
  101. coord/dashboard/webapp/public/icons/icon.svg +5 -0
  102. coord/dashboard/webapp/src/App.tsx +140 -0
  103. coord/dashboard/webapp/src/api/client.ts +199 -0
  104. coord/dashboard/webapp/src/api/generated.ts +176 -0
  105. coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
  106. coord/dashboard/webapp/src/components/Detail.tsx +800 -0
  107. coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
  108. coord/dashboard/webapp/src/components/Home.tsx +435 -0
  109. coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
  110. coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
  111. coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
  112. coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
  113. coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
  114. coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
  115. coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
  116. coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
  117. coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
  118. coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
  119. coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
  120. coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
  121. coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
  122. coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
  123. coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
  124. coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
  125. coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
  126. coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
  127. coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
  128. coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
  129. coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
  130. coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
  131. coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
  132. coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
  133. coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
  134. coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
  135. coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
  136. coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
  137. coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
  138. coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
  139. coord/dashboard/webapp/src/index.css +210 -0
  140. coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
  141. coord/dashboard/webapp/src/lib/utils.ts +6 -0
  142. coord/dashboard/webapp/src/main.tsx +46 -0
  143. coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
  144. coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
  145. coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
  146. coord/dashboard/webapp/src/realtime/connection.ts +227 -0
  147. coord/dashboard/webapp/src/realtime/events.ts +100 -0
  148. coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
  149. coord/dashboard/webapp/src/routes/paths.ts +92 -0
  150. coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
  151. coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
  152. coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
  153. coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
  154. coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
  155. coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
  156. coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
  157. coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
  158. coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
  159. coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
  160. coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
  161. coord/dashboard/webapp/src/shell/railItems.ts +105 -0
  162. coord/dashboard/webapp/src/shell/shellState.ts +174 -0
  163. coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
  164. coord/dashboard/webapp/src/test-setup.ts +41 -0
  165. coord/dashboard/webapp/src/vite-env.d.ts +2 -0
  166. coord/dashboard/webapp/tailwind.config.js +140 -0
  167. coord/dashboard/webapp/tsconfig.json +25 -0
  168. coord/dashboard/webapp/tsconfig.node.json +11 -0
  169. coord/dashboard/webapp/vite.config.ts +71 -0
  170. coord/db.py +1076 -0
  171. coord/dead_end.py +332 -0
  172. coord/deploy/README.md +33 -0
  173. coord/deploy/coord-agent.service +89 -0
  174. coord/deploy/coord-db-backup.service +60 -0
  175. coord/deploy/coord-db-backup.sh +74 -0
  176. coord/deploy/coord-db-backup.timer +18 -0
  177. coord/deploy/coord-drive-queue.service +117 -0
  178. coord/deploy/coord-drive-queue.timer +39 -0
  179. coord/deploy/coord-notify.service +48 -0
  180. coord/deploy/coord-notify.timer +24 -0
  181. coord/deploy/coord-release-propagate.service +83 -0
  182. coord/deploy/coord-release-propagate.timer +38 -0
  183. coord/deploy/coord-release-window.service +119 -0
  184. coord/deploy/coord-release-window.timer +36 -0
  185. coord/deploy/coord-serve.service +82 -0
  186. coord/deploy/coord-web-dist-build.service +43 -0
  187. coord/deploy/coord-web-dist-build.timer +36 -0
  188. coord/deploy/coord-web.service +125 -0
  189. coord/deploy_manifest.py +80 -0
  190. coord/deploy_units.py +384 -0
  191. coord/deps.py +115 -0
  192. coord/diagnose.py +1623 -0
  193. coord/dispatch.py +1009 -0
  194. coord/dist_name.py +123 -0
  195. coord/drive.py +3101 -0
  196. coord/drive_queue.py +2298 -0
  197. coord/drive_state.py +870 -0
  198. coord/events.py +381 -0
  199. coord/failure_class.py +914 -0
  200. coord/filelock.py +168 -0
  201. coord/fleet_config_health.py +300 -0
  202. coord/freshness.py +206 -0
  203. coord/gate_a.py +469 -0
  204. coord/gate_b.py +411 -0
  205. coord/gate_snapshot.py +385 -0
  206. coord/gates.py +582 -0
  207. coord/github_ops.py +1954 -0
  208. coord/goal.py +125 -0
  209. coord/graph_health.py +348 -0
  210. coord/health/__init__.py +69 -0
  211. coord/health/aggregate.py +129 -0
  212. coord/health/checks/__init__.py +13 -0
  213. coord/health/checks/agent_install.py +280 -0
  214. coord/health/checks/cargo_targets.py +171 -0
  215. coord/health/checks/claude_binary.py +65 -0
  216. coord/health/checks/deploy_lane_facts.py +458 -0
  217. coord/health/checks/disk.py +99 -0
  218. coord/health/checks/fleet_board.py +89 -0
  219. coord/health/checks/fleet_deploy_lanes.py +469 -0
  220. coord/health/checks/fleet_phantom.py +69 -0
  221. coord/health/checks/fleet_unit_drift.py +151 -0
  222. coord/health/checks/graph.py +192 -0
  223. coord/health/checks/plan_usage.py +88 -0
  224. coord/health/checks/repo_state.py +161 -0
  225. coord/health/checks/spawned_coord.py +465 -0
  226. coord/health/checks/timer_active.py +254 -0
  227. coord/health/checks/toolchain.py +547 -0
  228. coord/health/checks/unit_drift.py +648 -0
  229. coord/health/checks/unit_enablement.py +171 -0
  230. coord/health/checks/worktrees.py +96 -0
  231. coord/health/cli.py +121 -0
  232. coord/health/context.py +106 -0
  233. coord/health/fleet_snapshot.py +477 -0
  234. coord/health/models.py +250 -0
  235. coord/health/pypi.py +231 -0
  236. coord/health/registry.py +240 -0
  237. coord/health/render.py +82 -0
  238. coord/health/units.py +60 -0
  239. coord/hooks.py +106 -0
  240. coord/housekeeping.py +204 -0
  241. coord/interactive.py +4286 -0
  242. coord/issue_store.py +1496 -0
  243. coord/liveness_auditor.py +293 -0
  244. coord/machine_pause.py +755 -0
  245. coord/merge_queue.py +4681 -0
  246. coord/milestone_chat.py +600 -0
  247. coord/milestone_dispatch.py +943 -0
  248. coord/milestone_gate.py +709 -0
  249. coord/milestone_order.py +840 -0
  250. coord/mock_author.py +334 -0
  251. coord/models.py +891 -0
  252. coord/network.py +269 -0
  253. coord/new_issue_chat.py +229 -0
  254. coord/notify.py +3226 -0
  255. coord/openapi.py +404 -0
  256. coord/overlap_fence.py +133 -0
  257. coord/parentage.py +200 -0
  258. coord/parentage_github.py +58 -0
  259. coord/pipeline.py +481 -0
  260. coord/plan_parser.py +266 -0
  261. coord/plans.py +543 -0
  262. coord/platform_paths.py +43 -0
  263. coord/pr_body_lint.py +67 -0
  264. coord/prereqs.py +533 -0
  265. coord/progress.py +425 -0
  266. coord/providers/__init__.py +683 -0
  267. coord/providers/base.py +218 -0
  268. coord/providers/claude.py +284 -0
  269. coord/providers/claude_pty.py +610 -0
  270. coord/providers/opencode.py +896 -0
  271. coord/reconcile.py +2233 -0
  272. coord/refine_chat.py +485 -0
  273. coord/release_cordon.py +525 -0
  274. coord/release_propagate.py +1176 -0
  275. coord/release_verify.py +777 -0
  276. coord/release_window.py +322 -0
  277. coord/reports.py +1643 -0
  278. coord/revalidate.py +1101 -0
  279. coord/review.py +3317 -0
  280. coord/scorecard.py +484 -0
  281. coord/serve_app.py +7192 -0
  282. coord/skills/update-issue/SKILL.md +93 -0
  283. coord/smoke.py +1030 -0
  284. coord/split_work.py +210 -0
  285. coord/stage_projection.py +650 -0
  286. coord/state.py +5720 -0
  287. coord/test_author.py +1064 -0
  288. coord/test_chat.py +352 -0
  289. coord/test_orchestrator.py +494 -0
  290. coord/test_report.py +178 -0
  291. coord/tui_release.py +271 -0
  292. coord/usage.py +753 -0
  293. coord/usage_limits.py +358 -0
  294. coord/usage_rollup.py +709 -0
  295. coord/worker_events.py +954 -0
coord/diagnose.py ADDED
@@ -0,0 +1,1623 @@
1
+ """Per-stage "doctor": diagnose a pipeline stage, best-effort recover, and —
2
+ when recovery isn't possible — offer a non-destructive reset.
3
+
4
+ Pipeline stages routinely get into bad DB states with no clean UI recovery:
5
+ phantom ``running`` rows (board says running, no live session — #366), reviews
6
+ whose findings were silently dropped (#607), stale-but-live detached sessions
7
+ days old (#494/#370/#546), merged-but-grey boxes, orphaned worktrees. This
8
+ module is the orchestration the TUI's "Diagnose & fix stage" action and the
9
+ ``coord diagnose`` command call; it *composes* existing primitives rather than
10
+ reinventing them:
11
+
12
+ * :func:`coord.interactive.finalize_interactive_exit` — record a terminal state
13
+ for a dead/phantom session (pushes commits, releases claim, prunes worktree).
14
+ * :func:`coord.interactive._review_findings_from_transcript` — the #617
15
+ remote-aware transcript-floor that recovers a review's verdict + findings from
16
+ the session's own host.
17
+ * :func:`coord.reconcile.reconcile_board_merges` — flip merged-but-grey work and
18
+ backfill missing branches.
19
+
20
+ Design decisions (locked with the operator):
21
+
22
+ * **Reset is non-destructive**: it clears the stage's board rows, releases the
23
+ claim, removes the orphaned worktree, and stops a live session — but NEVER
24
+ deletes the feature branch. ``origin/issue-<N>-*`` and its commits are
25
+ preserved, so the stage re-dispatches fresh with the work intact. (There is
26
+ deliberately no branch-deletion code path in this module.)
27
+ * **Cleanup is scoped to the one issue**, not a fleet-wide sweep.
28
+ * **The issue-wide phantom-row scan never writes without ``--reset``** (#1658):
29
+ ``diagnose_stage``'s targeted best-effort recovery of the STAGE the operator
30
+ asked about may still write without ``--reset`` (that's the whole point of
31
+ "best-effort recover"), but :func:`_cleanup_issue`'s sweep over the issue's
32
+ OTHER rows only ever reports a finding + ``needs_reset=True`` unless
33
+ ``--reset`` was passed. That sweep touches rows the operator did not ask
34
+ about and did not get a tailored diagnosis for, so a wrong liveness read on
35
+ it is pure collateral damage — see :func:`_session_state`'s note on why
36
+ tmux-only liveness was itself wrong for headless workers.
37
+
38
+ The side-effecting steps are factored into small module-level helpers so the
39
+ orchestration in :func:`diagnose_stage` is unit-testable by monkeypatching them.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import logging
45
+ import subprocess
46
+ import time
47
+ from dataclasses import dataclass, field
48
+ from pathlib import Path
49
+ from typing import TYPE_CHECKING
50
+
51
+ if TYPE_CHECKING: # avoid import cycles / heavy imports at module load
52
+ from coord.config import Config
53
+ from coord.models import Assignment, Board, Machine
54
+
55
+ # Stages the doctor understands. Each maps to the assignment ``type`` that
56
+ # carries its state; ``test`` and ``merge`` are tracked on the *work* row
57
+ # (``test_state`` / ``status='merged'`` + the merge queue) rather than a
58
+ # dedicated assignment type.
59
+ STAGE_ASSIGNMENT_TYPES: dict[str, tuple[str, ...]] = {
60
+ "plan": ("plan",),
61
+ "work": ("work", "plan"),
62
+ # #1180: a `type="test-author"`/`"mock-author"` completion carries its own
63
+ # `review_state`/`review_verdict` (they're in WORK_LIKE_TYPES and go
64
+ # through the same review chokepoint as `work`) but never spawns a
65
+ # dedicated `type="review"` row when it's wedged — before this fix,
66
+ # `coord diagnose --stage review` looked at `type="review"` rows only, so
67
+ # a test-author row stuck at `review_state="done"` with no verdict and no
68
+ # review assignment was invisible: the tool would report on whatever
69
+ # unrelated `type="review"` row happened to share the tracking issue
70
+ # number (false "stage looks healthy"/wrong-row confidence) instead of
71
+ # flagging the real wedge.
72
+ "review": ("review", "test-author", "mock-author"),
73
+ "test": ("work", "plan"),
74
+ "merge": ("work", "plan"),
75
+ # #2087: previously absent entirely — `--stage smoke` (or an implicit
76
+ # `current_stage()` pick landing on a `type="smoke"` row, e.g. a Test
77
+ # stage dispatched more recently than its parent `work` row) fell
78
+ # straight into the "no diagnosis available" dead end below with no
79
+ # recovery and no `--reset` path. Routed through the same work-like
80
+ # recovery as `work`/`plan` (`diagnose_stage`'s per-stage dispatch and
81
+ # `_do_reset` both fall back to that branch for any type not `review` or
82
+ # `test`) — a smoke row is a one-shot session exactly like a work row.
83
+ "smoke": ("smoke",),
84
+ }
85
+
86
+
87
+ @dataclass
88
+ class DiagnoseResult:
89
+ """Outcome of a diagnose/recover/reset run for one stage of one issue."""
90
+
91
+ repo_name: str
92
+ issue_number: int
93
+ stage: str
94
+ findings: list[str] = field(default_factory=list)
95
+ actions_taken: list[str] = field(default_factory=list)
96
+ # True when the stage is healthy after this run (nothing was wrong, or the
97
+ # problem was recovered). False + needs_reset=True means "still wedged".
98
+ recovered: bool = False
99
+ # True when best-effort recovery could not clear the problem and the only
100
+ # remaining option is a reset.
101
+ needs_reset: bool = False
102
+ # Always True for this module — reset keeps the branch. Surfaced so the TUI
103
+ # can promise "keeps branch + commits" in the confirm dialog.
104
+ branch_preserved: bool = True
105
+ # Whether a reset was actually performed this run.
106
+ reset_performed: bool = False
107
+
108
+ def to_json_dict(self) -> dict:
109
+ """Return a JSON-serialisable dict of all DiagnoseResult fields.
110
+ Used by ``coord diagnose --json`` and the daemon ``post_diagnose``
111
+ handler (#935 Part C) so the TUI can parse findings/actions without
112
+ scraping the human-readable output lines."""
113
+ import dataclasses # noqa: PLC0415 — lazy to avoid circular import risk
114
+ return dataclasses.asdict(self)
115
+
116
+ def summary_line(self) -> str:
117
+ """The machine-readable trailer the TUI greps for (mirrors the
118
+ ``coord:`` marker convention)."""
119
+ return (
120
+ f"DIAGNOSE_RESULT: stage={self.stage} "
121
+ f"recovered={str(self.recovered).lower()} "
122
+ f"needs_reset={str(self.needs_reset).lower()} "
123
+ f"reset_performed={str(self.reset_performed).lower()} "
124
+ f"actions={len(self.actions_taken)}"
125
+ )
126
+
127
+
128
+ # ── stage / assignment resolution ───────────────────────────────────────────
129
+
130
+
131
+ def stage_assignments(
132
+ board: "Board", repo_name: str, issue_number: int, stage: str
133
+ ) -> list["Assignment"]:
134
+ """All assignments for *issue_number* in *repo_name* matching *stage*,
135
+ newest-dispatched first. Mirrors the TUI's ``assignments_for_stage``."""
136
+ types = STAGE_ASSIGNMENT_TYPES.get(stage, (stage,))
137
+ rows = [
138
+ a
139
+ for a in (board.active + board.completed)
140
+ if a.issue_number == issue_number
141
+ and a.repo_name == repo_name
142
+ and (a.type or "work") in types
143
+ ]
144
+ rows.sort(key=lambda a: (a.dispatched_at or 0.0), reverse=True)
145
+ return rows
146
+
147
+
148
+ def _latest(assignments: list["Assignment"]) -> "Assignment | None":
149
+ return assignments[0] if assignments else None
150
+
151
+
152
+ def _flag_contradictory_failed(latest: "Assignment", res: DiagnoseResult) -> None:
153
+ """#1451: flag a ``status='failed'`` work row whose own fields already
154
+ prove it isn't — a passing test verdict and/or an approved review on a
155
+ row that pushed a real branch is self-evidently not a failure.
156
+
157
+ This is deliberately NOT based on ``exit_code``/``failure_reason`` being
158
+ empty — those are empty on the overwhelming majority of legitimate
159
+ ``failed`` rows too (a launch-failure ``failure_reason`` is the rare
160
+ exception, and no current write path persists ``exit_code`` to the DB at
161
+ all), so that pair is not a usable signal on its own. The reliable
162
+ signal is a genuine contradiction: evidence of *success* recorded on a
163
+ row the board calls failed.
164
+
165
+ Best-effort and read-only — appends a finding only, no write. Detection,
166
+ not correction: the fix is either ``coord report-result --assignment
167
+ <id> --status done`` (interactive) or re-running the completing worker.
168
+ """
169
+ if latest.status != "failed":
170
+ return
171
+ contradictions: list[str] = []
172
+ if latest.test_state == "passed":
173
+ contradictions.append("test_state=passed")
174
+ if latest.review_verdict == "approve":
175
+ contradictions.append("review_verdict=approve")
176
+ if not contradictions:
177
+ return
178
+ if not latest.branch:
179
+ # No pushed branch to review/test at all — the "passed"/"approve"
180
+ # values must be stale carry-over from a prior assignment row, not
181
+ # evidence about *this* failed row. Don't flag without a branch.
182
+ return
183
+ res.findings.append(
184
+ f"⚠ status='failed' contradicts its own fields ({', '.join(contradictions)}, "
185
+ f"branch={latest.branch}) — looks like a phantom failure (#1451), not a "
186
+ "real one. If the work is actually done, recover it with "
187
+ f"`coord report-result --assignment {latest.assignment_id} --status done "
188
+ '--summary "..."` (or re-run the completing worker if unsure).'
189
+ )
190
+
191
+
192
+ def current_stage(board: "Board", repo_name: str, issue_number: int) -> str:
193
+ """The stage of the most-recently-dispatched assignment for the issue
194
+ (what ``coord diagnose <repo> <issue>`` targets when ``--stage`` is
195
+ omitted). Falls back to ``work`` when the issue has no assignments.
196
+
197
+ #1083: previously coerced any assignment ``type`` this module doesn't
198
+ recognize (e.g. ``test-author``, ``mock-author``, ``smoke``) to
199
+ ``"work"`` — which then had ``diagnose_stage`` recover/report on
200
+ whatever unrelated ``work``/``plan`` row happened to exist for the issue,
201
+ *silently* presenting it as if it were a diagnosis of the real (ignored)
202
+ assignment. Now the actual type is returned verbatim; ``diagnose_stage``
203
+ explicitly reports "no diagnosis available" for types outside
204
+ :data:`STAGE_ASSIGNMENT_TYPES` instead of guessing.
205
+ """
206
+ rows = [
207
+ a
208
+ for a in (board.active + board.completed)
209
+ if a.issue_number == issue_number and a.repo_name == repo_name
210
+ ]
211
+ if not rows:
212
+ return "work"
213
+ newest = max(rows, key=lambda a: (a.dispatched_at or 0.0))
214
+ return newest.type or "work"
215
+
216
+
217
+ # ── monkeypatchable side-effecting wrappers ─────────────────────────────────
218
+ #
219
+ # Each wraps an existing primitive and is replaced in unit tests so the
220
+ # orchestration can be exercised without touching git/tmux/the network.
221
+
222
+
223
+ def _resolve_machine(config: "Config", machine_name: str | None):
224
+ if not machine_name:
225
+ return None
226
+ return next((m for m in config.machines if m.name == machine_name), None)
227
+
228
+
229
+ def _session_state(assignment: "Assignment", config: "Config") -> str:
230
+ """``"live"`` | ``"dead"`` | ``"unknown"`` for *assignment*.
231
+
232
+ Probes the assignment's machine (local tmux, or the remote host's tmux over
233
+ ssh — same mechanism as ``coord reattach`` / the stale-session reaper).
234
+ ``"unknown"`` when the machine can't be resolved or the probe errors, so the
235
+ caller never finalizes on a false negative.
236
+
237
+ #1658: tmux liveness alone is blind to HEADLESS workers. A headless
238
+ assignment (the normal shape for a daemon-dispatched review/work — see
239
+ ``AgentServer.assign``) runs as a plain subprocess tracked by the agent's
240
+ own ``_assignments`` dict; it never has a tmux session at all, so
241
+ ``tmux_session_alive`` reads "dead" for it unconditionally, regardless of
242
+ whether the worker is still running. Before this fix, that false "dead"
243
+ made every live headless assignment look like a phantom the instant
244
+ ``coord diagnose`` looked at it — the incident this closes: a live review
245
+ worker's row was finalized to ``failed`` mid-review. Now, when tmux says
246
+ dead, the assignment's own agent ``/status`` is consulted before trusting
247
+ that — the same seam :func:`coord.reconcile.reconcile_completed_assignments`
248
+ uses to tell "still running" from "actually finished" — and a match in its
249
+ ``active`` list is authoritative: the agent is the ground truth for its
250
+ own subprocesses. An unreachable agent still returns "unknown" rather than
251
+ "dead", preserving the never-finalize-on-a-probe-failure guarantee.
252
+ """
253
+ import socket # noqa: PLC0415
254
+
255
+ from coord.interactive import ( # noqa: PLC0415
256
+ TmuxHost,
257
+ tmux_session_alive,
258
+ tmux_session_name,
259
+ )
260
+
261
+ if not assignment.assignment_id:
262
+ return "unknown"
263
+ machine = _resolve_machine(config, assignment.machine_name)
264
+ ssh_target = None
265
+ if machine is not None:
266
+ local_hn = socket.gethostname().split(".")[0].lower()
267
+ is_local = (
268
+ machine.name.lower() == local_hn
269
+ or machine.host.split(".")[0].lower() == local_hn
270
+ )
271
+ if not is_local:
272
+ ssh_target = machine.host
273
+ elif assignment.machine_name:
274
+ # machine_name set but unknown in config — can't probe safely.
275
+ return "unknown"
276
+ host = TmuxHost(ssh_target=ssh_target)
277
+ sname = tmux_session_name(assignment.assignment_id)
278
+ try:
279
+ if tmux_session_alive(sname, host=host):
280
+ return "live"
281
+ except Exception: # noqa: BLE001 — never let a probe error finalize a session
282
+ return "unknown"
283
+
284
+ # tmux says dead (or the assignment never had a tmux session at all — the
285
+ # headless case). Consult the agent before trusting that.
286
+ return _agent_liveness(assignment, machine)
287
+
288
+
289
+ def _agent_liveness(assignment: "Assignment", machine: "Machine | None") -> str:
290
+ """``"dead"`` | ``"live"`` | ``"unknown"`` per the assignment's own agent
291
+ ``/status`` — see :func:`_session_state`'s #1658 note for why this exists.
292
+ ``machine`` is ``None`` when the assignment has no ``machine_name`` at all
293
+ (rare) — treated as genuinely dead since there's nothing to probe."""
294
+ if machine is None:
295
+ return "dead"
296
+ from coord.network import fetch_status # noqa: PLC0415
297
+
298
+ result = fetch_status(machine)
299
+ if not result.ok or result.data is None:
300
+ # Agent unreachable — don't trust tmux-dead alone, but don't claim
301
+ # "live" either. Matches the "never finalize on a probe failure"
302
+ # contract the rest of this module relies on.
303
+ return "unknown"
304
+ active = result.data.get("active") or []
305
+ if any(isinstance(e, dict) and e.get("id") == assignment.assignment_id for e in active):
306
+ return "live"
307
+ return "dead"
308
+
309
+
310
+ def _ssh_target_for(assignment: "Assignment", config: "Config") -> str | None:
311
+ """The ssh host for *assignment*'s machine, or ``None`` when it's local."""
312
+ import socket # noqa: PLC0415
313
+
314
+ machine = _resolve_machine(config, assignment.machine_name)
315
+ if machine is None:
316
+ return None
317
+ local_hn = socket.gethostname().split(".")[0].lower()
318
+ if machine.name.lower() == local_hn or machine.host.split(".")[0].lower() == local_hn:
319
+ return None
320
+ return machine.host
321
+
322
+
323
+ def _recover_review_findings(assignment: "Assignment", config: "Config") -> str | None:
324
+ """Recover a review's verdict + findings from its session transcript and
325
+ persist them through the durable seam (#617). Returns the verdict on
326
+ success, ``None`` when nothing was recoverable. Read-only w.r.t. the
327
+ session (safe to run even while it's live)."""
328
+ from coord import issue_store # noqa: PLC0415
329
+ from coord.interactive import _review_findings_from_transcript # noqa: PLC0415
330
+
331
+ if not assignment.assignment_id:
332
+ return None
333
+ assignment_id: str = assignment.assignment_id
334
+ ssh_target = _ssh_target_for(assignment, config)
335
+ started_at = assignment.dispatched_at
336
+ findings = _review_findings_from_transcript(
337
+ assignment.issue_number,
338
+ started_at,
339
+ assignment_id=assignment_id,
340
+ ssh_target=ssh_target,
341
+ )
342
+ if findings is None:
343
+ return None
344
+ repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
345
+ try:
346
+ issue_store.post_result(
347
+ issue_store.ResultRecord(
348
+ assignment_id=assignment_id,
349
+ machine_name=assignment.machine_name or "unknown",
350
+ repo_name=assignment.repo_name,
351
+ repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
352
+ issue_number=assignment.issue_number,
353
+ status="done",
354
+ verdict=findings.verdict, # type: ignore[arg-type]
355
+ summary="Findings recovered from the session transcript by coord diagnose.",
356
+ findings_body=findings.body,
357
+ branch=None,
358
+ )
359
+ )
360
+ except RuntimeError as exc:
361
+ # #990: the verdict was recovered from the transcript but couldn't be
362
+ # durably persisted (retries exhausted / readback mismatch). Surface
363
+ # this instead of letting it crash `coord diagnose` — the caller
364
+ # treats a ``None`` return as "not recoverable" and reports
365
+ # "re-review needed", which is the safe outcome here too since the
366
+ # write did not actually land.
367
+ import click # noqa: PLC0415
368
+
369
+ click.echo(
370
+ f" ⚠ recovered verdict {findings.verdict!r} from transcript for "
371
+ f"{assignment.assignment_id} but failed to persist it: {exc}",
372
+ err=True,
373
+ )
374
+ return None
375
+ return findings.verdict
376
+
377
+
378
+ def _finalize_dead(assignment: "Assignment", config: "Config") -> str:
379
+ """Finalize a dead/phantom session: record a terminal state, push any
380
+ commits, release the claim, prune the worktree. Returns a short status."""
381
+ from coord.interactive import finalize_interactive_exit # noqa: PLC0415
382
+ from coord.state import COORD_DIR # noqa: PLC0415
383
+
384
+ machine = _resolve_machine(config, assignment.machine_name)
385
+ repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
386
+ base = (repo_cfg.default_branch if repo_cfg else None) or "main"
387
+ repo_github = repo_cfg.github if repo_cfg else assignment.repo_name
388
+ repo_path = None
389
+ if machine is not None and assignment.repo_name:
390
+ from pathlib import Path # noqa: PLC0415
391
+
392
+ rp = machine.repo_path(assignment.repo_name)
393
+ if rp:
394
+ repo_path = str(Path(rp).expanduser())
395
+ worktree = str(COORD_DIR / "worktrees" / (assignment.assignment_id or ""))
396
+ fr = finalize_interactive_exit(
397
+ assignment_id=assignment.assignment_id or "",
398
+ repo_name=assignment.repo_name,
399
+ repo_github=repo_github,
400
+ issue_number=assignment.issue_number,
401
+ machine_name=assignment.machine_name or "unknown",
402
+ worktree_path=worktree if assignment.type in ("work", "plan") else None,
403
+ base_branch=base,
404
+ exit_code=0,
405
+ started_at=assignment.dispatched_at,
406
+ repo_path=repo_path,
407
+ ssh_target=_ssh_target_for(assignment, config),
408
+ # #1256: unconditional — restore_live_checkout_from_smoke_snapshot()
409
+ # is a documented no-op unless a snapshot marker for this
410
+ # assignment_id exists in repo_path's .git/ dir, so this is safe for
411
+ # every assignment type, not just "smoke". `coord diagnose`'s
412
+ # unfiltered dead-session sweep (_cleanup_issue) is one of the two
413
+ # automated/operator-recovery paths (the other is
414
+ # reap_stale_interactive_sessions) that exist specifically to handle
415
+ # a session that died without a clean exit — exactly when a
416
+ # --smoke-of session's live-checkout mutation needs reverting.
417
+ smoke_repo_path=repo_path,
418
+ )
419
+ return fr.terminal_status or "finalized"
420
+
421
+
422
+ def _kill_session(assignment: "Assignment", config: "Config") -> bool:
423
+ """``tmux kill-session`` for *assignment* (local or remote). Used by reset
424
+ to stop a live session before finalizing. Returns True when the kill ran."""
425
+ import subprocess # noqa: PLC0415
426
+
427
+ from coord.interactive import ( # noqa: PLC0415
428
+ TmuxHost,
429
+ tmux_session_name,
430
+ )
431
+
432
+ if not assignment.assignment_id:
433
+ return False
434
+ host = TmuxHost(ssh_target=_ssh_target_for(assignment, config))
435
+ sname = tmux_session_name(assignment.assignment_id)
436
+ try:
437
+ subprocess.run(
438
+ host.cmd(["kill-session", "-t", sname]),
439
+ capture_output=True,
440
+ timeout=20,
441
+ )
442
+ return True
443
+ except Exception: # noqa: BLE001 — best-effort
444
+ return False
445
+
446
+
447
+ def _reconcile_issue_merges(
448
+ board: "Board", config: "Config", repo_name: str, issue_number: int, *, dry_run: bool
449
+ ) -> list[str]:
450
+ """Run the merge reconcile sweep scoped to one issue (branch backfill +
451
+ out-of-band-merge detection)."""
452
+ from coord.reconcile import reconcile_board_merges # noqa: PLC0415
453
+
454
+ return reconcile_board_merges(
455
+ board, config, repo=repo_name, issue=issue_number, dry_run=dry_run
456
+ )
457
+
458
+
459
+ def _mark_terminal(assignment: "Assignment", config: "Config") -> None:
460
+ """Best-effort terminal write via the issue_store seam — the fallback used
461
+ only when :func:`_finalize_dead` itself raised. Records a failed completion
462
+ so the phantom row leaves ``running`` and persists to the canonical DB
463
+ WITHOUT relying on ``save_board`` (which the diagnose path deliberately does
464
+ not call — it would clobber the seam writes with a stale snapshot)."""
465
+ from coord import issue_store # noqa: PLC0415
466
+
467
+ if not assignment.assignment_id:
468
+ return
469
+ repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
470
+ try:
471
+ issue_store.post_completion(
472
+ issue_store.CompletionRecord(
473
+ assignment_id=assignment.assignment_id,
474
+ machine_name=assignment.machine_name or "unknown",
475
+ repo_name=assignment.repo_name,
476
+ repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
477
+ issue_number=assignment.issue_number,
478
+ exit_code=1, # → failed terminal state (out of 'running')
479
+ commits_ahead=0,
480
+ branch=assignment.branch,
481
+ )
482
+ )
483
+ except Exception: # noqa: BLE001 — fallback of a fallback; leave the phantom
484
+ pass
485
+
486
+
487
+ def _downgrade_empty_branch_done(assignment: "Assignment", config: "Config") -> str:
488
+ """#1155: flip a wedged ``done``-with-empty-branch work row to ``advisory``
489
+ via the issue_store seam (same seam :func:`_mark_terminal` uses for its
490
+ own fallback write). A ``done`` row with no branch has nothing to review
491
+ — it must not sit in the Pipeline masquerading as reviewable work.
492
+ Best-effort: on failure the row is left as-is and the caller's finding
493
+ still surfaces the problem to the operator."""
494
+ from coord import issue_store # noqa: PLC0415
495
+
496
+ if not assignment.assignment_id:
497
+ return "skipped (no assignment_id)"
498
+ repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
499
+ try:
500
+ outcome = issue_store.post_completion(
501
+ issue_store.CompletionRecord(
502
+ assignment_id=assignment.assignment_id,
503
+ machine_name=assignment.machine_name or "unknown",
504
+ repo_name=assignment.repo_name,
505
+ repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
506
+ issue_number=assignment.issue_number,
507
+ exit_code=0,
508
+ commits_ahead=0, # → advisory terminal state (the #448 shape)
509
+ branch=assignment.branch,
510
+ )
511
+ )
512
+ return outcome.status
513
+ except Exception as exc: # noqa: BLE001 — best-effort recovery
514
+ return f"failed ({exc})"
515
+
516
+
517
+ # ── orchestration ───────────────────────────────────────────────────────────
518
+
519
+
520
+ def diagnose_stage(
521
+ board: "Board",
522
+ config: "Config",
523
+ repo_name: str,
524
+ issue_number: int,
525
+ stage: str,
526
+ *,
527
+ reset: bool = False,
528
+ dry_run: bool = False,
529
+ ) -> DiagnoseResult:
530
+ """Diagnose *stage* of *repo_name* #*issue_number*; best-effort recover;
531
+ always reconcile this issue's DB; optionally reset (non-destructive).
532
+
533
+ Returns a :class:`DiagnoseResult`. Board mutations happen on the board
534
+ passed in; the caller is responsible for persisting it (the CLI/daemon do
535
+ so after this returns) — consistent with ``reconcile_board_merges``.
536
+ """
537
+ res = DiagnoseResult(repo_name=repo_name, issue_number=issue_number, stage=stage)
538
+
539
+ # #1083: `stage` came either from an explicit `--stage` or from
540
+ # `current_stage()`'s newest-assignment lookup. `current_stage()` now
541
+ # surfaces a non-standard assignment type (e.g. "test-author",
542
+ # "mock-author", "smoke") verbatim instead of silently mapping it to
543
+ # "work" — so a type this module has no recovery logic for lands here as
544
+ # `stage` rather than being guessed at. Report that plainly instead of
545
+ # running `_recover_work_like` against it (which was never validated for
546
+ # these types) or, worse, silently returning an unrelated `work`/`plan`
547
+ # row's status as if it were this stage's diagnosis (the bug reported in
548
+ # #1083: `coord diagnose` picked an unrelated, already-merged assignment
549
+ # instead of flagging the real problem).
550
+ if stage not in STAGE_ASSIGNMENT_TYPES:
551
+ assignments = stage_assignments(board, repo_name, issue_number, stage)
552
+ latest = _latest(assignments)
553
+ known = ", ".join(sorted(STAGE_ASSIGNMENT_TYPES))
554
+ if latest is None:
555
+ res.findings.append(
556
+ f"no diagnosis available for assignment type {stage!r} — "
557
+ f"coord diagnose only understands: {known} (and no {stage!r} "
558
+ f"assignment exists for #{issue_number} either)"
559
+ )
560
+ else:
561
+ res.findings.append(
562
+ f"no diagnosis available for assignment type {stage!r} — "
563
+ f"coord diagnose only understands: {known}. Latest {stage!r} "
564
+ f"assignment: {latest.assignment_id} status={latest.status} "
565
+ f"branch={latest.branch or '(none)'} machine={latest.machine_name}"
566
+ )
567
+ res.recovered = False
568
+ res.needs_reset = False
569
+ return res
570
+
571
+ assignments = stage_assignments(board, repo_name, issue_number, stage)
572
+ latest = _latest(assignments)
573
+
574
+ if latest is None:
575
+ res.findings.append(f"no {stage} assignment on the board for #{issue_number}")
576
+ res.recovered = True # nothing wedged
577
+ # Still run the issue-wide cleanup below.
578
+ _cleanup_issue(board, config, repo_name, issue_number, res, dry_run=dry_run, reset=reset)
579
+ return res
580
+
581
+ # The stage step owns *latest*; record it so the issue-wide cleanup pass
582
+ # doesn't re-finalize the same row (finalize writes the DB, not this
583
+ # in-memory board row, so its status would still read "running" here).
584
+ handled = {latest.assignment_id} if latest.assignment_id else set()
585
+
586
+ state = _session_state(latest, config)
587
+ res.findings.append(
588
+ f"{stage}: latest={latest.assignment_id} status={latest.status} "
589
+ f"session={state} machine={latest.machine_name}"
590
+ )
591
+ if stage in ("work", "test", "merge"):
592
+ _flag_contradictory_failed(latest, res)
593
+
594
+ # #2087: a `running`/`pending` row naming a machine that isn't a
595
+ # configured machine can never be probed — `_session_state` above
596
+ # already returns "unknown" for exactly that reason ("machine_name set
597
+ # but unknown in config — can't probe safely"). Before this,
598
+ # every stage's best-effort recovery below had no branch for
599
+ # state=="unknown", so it fell through to that recovery function's own
600
+ # "stage looks healthy" catch-all — the textbook phantom `coord
601
+ # diagnose` exists to find, reported as fine (the exact `work-repro`
602
+ # shape: `machine=laptop`, `status=running` for 9h, "stage looks
603
+ # healthy"). Flag it here, uniformly across every stage, BEFORE any
604
+ # stage-specific recovery gets a chance to fall through to its healthy
605
+ # catch-all — and reuse the existing non-destructive `--reset` path
606
+ # (branch/commits always preserved) as the supported way to clear it.
607
+ machine_unconfigured = (
608
+ bool(latest.machine_name) and _resolve_machine(config, latest.machine_name) is None
609
+ )
610
+ if latest.status in ("running", "pending") and machine_unconfigured:
611
+ res.findings.append(
612
+ f"{stage}: machine {latest.machine_name!r} is not a configured "
613
+ "machine (not in coordinator.yml) — a 'running' row with no "
614
+ "host to poll is a phantom, not healthy; re-run with --reset to "
615
+ "clear it (the branch and any commits are preserved)"
616
+ )
617
+ res.recovered = False
618
+ res.needs_reset = True
619
+
620
+ if reset:
621
+ _do_reset(
622
+ board, config, assignments, res, stage=stage,
623
+ repo_name=repo_name, issue_number=issue_number, dry_run=dry_run,
624
+ )
625
+ _cleanup_issue(
626
+ board, config, repo_name, issue_number, res,
627
+ dry_run=dry_run, reset=reset, skip_ids=handled,
628
+ )
629
+ return res
630
+
631
+ if latest.status in ("running", "pending") and machine_unconfigured:
632
+ # Reported above; nothing safe to attempt without --reset (no host
633
+ # to probe/finalize against) and no stage-specific recovery below
634
+ # would do better than mis-file it as healthy.
635
+ _cleanup_issue(
636
+ board, config, repo_name, issue_number, res,
637
+ dry_run=dry_run, reset=reset, skip_ids=handled,
638
+ )
639
+ return res
640
+
641
+ # ── Best-effort recovery, per stage ─────────────────────────────────────
642
+ if stage in ("review",):
643
+ _recover_review(board, config, latest, state, res, dry_run=dry_run)
644
+ elif stage in ("merge",):
645
+ _recover_merge(board, config, repo_name, issue_number, latest, res, dry_run=dry_run)
646
+ elif stage == "test":
647
+ _recover_test(board, latest, state, res, config=config, dry_run=dry_run)
648
+ else: # work / plan
649
+ _recover_work_like(board, config, latest, state, res, dry_run=dry_run)
650
+
651
+ _cleanup_issue(
652
+ board, config, repo_name, issue_number, res,
653
+ dry_run=dry_run, reset=reset, skip_ids=handled,
654
+ )
655
+ return res
656
+
657
+
658
+ def _recover_review(
659
+ board, config, latest, state, res: DiagnoseResult, *, dry_run: bool
660
+ ) -> None:
661
+ from coord.state import load_assignment_review_findings # noqa: PLC0415
662
+
663
+ has_findings = False
664
+ if latest.assignment_id:
665
+ cached = load_assignment_review_findings(latest.assignment_id)
666
+ has_findings = bool(cached and (cached[1] or "").strip())
667
+
668
+ verdict = latest.review_verdict
669
+ if verdict == "request-changes" and not has_findings:
670
+ res.findings.append("review verdict is request-changes but findings are EMPTY (#607 class)")
671
+ if dry_run:
672
+ res.findings.append("(dry-run) would recover findings from the session transcript")
673
+ res.needs_reset = True
674
+ return
675
+ recovered_verdict = _recover_review_findings(latest, config)
676
+ if recovered_verdict:
677
+ res.actions_taken.append("recovered review findings from the session transcript → #603 store")
678
+ res.recovered = True
679
+ else:
680
+ res.findings.append("findings NOT recoverable from transcript — re-review needed")
681
+ res.needs_reset = True
682
+ elif latest.status == "done" and verdict is None:
683
+ # #812: review finalised as done but no verdict was ever captured.
684
+ # The session likely failed to start (no session_id, no exit_code) or
685
+ # exited before the reviewer ran coord report-result / the transcript-floor.
686
+ # This is a permanent stuck state: nothing is running, TUI rendered it
687
+ # blue/Active (now Fixed → red/Failed), Diagnose & Reset must handle it.
688
+ res.findings.append(
689
+ "review finalised as done but has no verdict — "
690
+ "session likely failed to start or exited before verdict capture (#812)"
691
+ )
692
+ if dry_run:
693
+ res.findings.append(
694
+ "(dry-run) would try transcript recovery; if verdict not found, reset"
695
+ )
696
+ res.needs_reset = True
697
+ return
698
+ recovered_verdict = _recover_review_findings(latest, config)
699
+ if recovered_verdict:
700
+ res.actions_taken.append(
701
+ "recovered review verdict/findings from session transcript"
702
+ )
703
+ res.recovered = True
704
+ else:
705
+ res.findings.append(
706
+ "no verdict recoverable from transcript — "
707
+ "reset to re-dispatch a fresh review"
708
+ )
709
+ res.needs_reset = True
710
+ elif state == "dead" and latest.status == "running":
711
+ res.findings.append("review session is dead but board still says running (phantom)")
712
+ if not dry_run:
713
+ # Try a transcript recovery first (captures the verdict if present),
714
+ # then finalize to clear the phantom.
715
+ if _recover_review_findings(latest, config):
716
+ res.actions_taken.append("recovered review verdict/findings from transcript")
717
+ res.actions_taken.append(f"finalized phantom review session ({_finalize_dead(latest, config)})")
718
+ res.recovered = True
719
+ elif state == "live" and _is_stale(latest):
720
+ res.findings.append("review session is LIVE but stale (idle days) — capturing read-only, reset to clear")
721
+ if not dry_run and _recover_review_findings(latest, config):
722
+ res.actions_taken.append("captured current review findings from transcript (session left running)")
723
+ res.needs_reset = True
724
+ else:
725
+ res.findings.append("review stage looks healthy")
726
+ res.recovered = True
727
+
728
+
729
+ def _recover_merge(
730
+ board, config, repo_name, issue_number, latest, res: DiagnoseResult, *, dry_run: bool
731
+ ) -> None:
732
+ actions = _reconcile_issue_merges(board, config, repo_name, issue_number, dry_run=dry_run)
733
+ if actions:
734
+ res.actions_taken.extend(actions)
735
+ res.recovered = True
736
+ return
737
+
738
+ # #1601: reconcile_board_merges (above) only ever does two things —
739
+ # backfill a missing `branch` and detect an out-of-band GitHub merge. It
740
+ # has never asked the one question this issue is about: "is this done,
741
+ # approved work even IN the merge queue at all?" A fix round's terminal
742
+ # review verdict routinely lands on a *different* board row than
743
+ # `latest`/the parent work row's own `review_state` (which can be stuck
744
+ # at "dispatched" forever once a later round supersedes it — see the
745
+ # #1566 incident this fixed), and when the periodic enqueue sweep
746
+ # (`merge_queue.enqueue_approved_work`, run from the daemon passive tick)
747
+ # misses its window, the branch is left approved-and-done with an EMPTY
748
+ # merge_queue. Before this, that state was indistinguishable here from
749
+ # actually healthy ("nothing to reconcile"). Ask the same question the
750
+ # merge gate itself asks (`passes_merge_gates`) so this can never
751
+ # disagree with what `coord merge --plan`/`--only` decide.
752
+ _diagnose_unqueued_merge(
753
+ board, config, repo_name, issue_number, latest, res, dry_run=dry_run
754
+ )
755
+
756
+
757
+ def _diagnose_unqueued_merge(
758
+ board, config, repo_name, issue_number, latest, res: DiagnoseResult, *, dry_run: bool
759
+ ) -> None:
760
+ """#1601: detect (and, when possible, fix) a done+approved branch with
761
+ no merge_queue entry — the "nothing ever enqueued the merge" failure
762
+ mode. Reuses `merge_queue`'s own winner-resolution
763
+ (`group_branch_candidates`) and gate predicate (`passes_merge_gates`) so
764
+ this reports the exact same verdict `coord merge --plan`/`--only` would,
765
+ never a re-derived one."""
766
+ from coord import merge_queue as mq # noqa: PLC0415
767
+
768
+ branch = getattr(latest, "branch", None)
769
+ if not branch:
770
+ res.findings.append("merge stage: nothing to reconcile")
771
+ res.recovered = True
772
+ return
773
+
774
+ existing_queue = mq.load_queue()
775
+ if any(getattr(e, "branch", None) == branch for e in existing_queue):
776
+ res.findings.append("merge stage: nothing to reconcile")
777
+ res.recovered = True
778
+ return
779
+
780
+ scoped_completed = [
781
+ a for a in board.completed
782
+ if a.repo_name == repo_name and getattr(a, "branch", None) == branch
783
+ ]
784
+ winners = mq.group_branch_candidates(scoped_completed)
785
+ if not winners:
786
+ res.findings.append("merge stage: nothing to reconcile")
787
+ res.recovered = True
788
+ return
789
+ winner, _superseded = winners[0]
790
+
791
+ # #2085: `winner` is a raw board Assignment — no `branch_head_sha`/
792
+ # `repo_github`/`target_branch` attribute, so handing it straight to
793
+ # `passes_merge_gates` made the #821 SHA-freshness check inside
794
+ # `has_approved_review` permanently unconfirmable (fails closed on every
795
+ # review carrying a real `review_head_sha`, i.e. virtually every modern
796
+ # approval — this diagnostic would report "waiting on the pipeline" for
797
+ # branches that actually pass every real gate). Build the same
798
+ # live-anchored synthetic entry `coord.gates.build_gate_report` uses so
799
+ # a genuinely fresh approval can still be confirmed. Falls back to the
800
+ # raw `winner` row (gate then fails closed, never open) when the repo
801
+ # isn't configured.
802
+ from coord import github_ops # noqa: PLC0415
803
+
804
+ gate_entry = winner
805
+ repo_cfg = config.repo(repo_name)
806
+ if repo_cfg is not None:
807
+ from coord.branch_model import resolve_base_branch_for_issue_number # noqa: PLC0415
808
+
809
+ target_branch = resolve_base_branch_for_issue_number(
810
+ repo_cfg, repo_cfg.github, issue_number,
811
+ )
812
+ gate_entry = mq.live_gate_entry(winner, repo_cfg.github, target_branch, github_ops)
813
+
814
+ if not mq.passes_merge_gates(gate_entry, config, board, gh_ops=github_ops):
815
+ res.findings.append(
816
+ f"merge stage: {branch} is done but not queued for merge, and does "
817
+ f"not (yet) pass the review/smoke gates — winning row "
818
+ f"{winner.assignment_id}: review_state={getattr(winner, 'review_state', None)!r} "
819
+ f"review_verdict={getattr(winner, 'review_verdict', None)!r} "
820
+ f"test_state={getattr(winner, 'test_state', None)!r}. Waiting on the "
821
+ "pipeline, not wedged."
822
+ )
823
+ res.recovered = True
824
+ return
825
+
826
+ if dry_run:
827
+ res.findings.append(
828
+ f"merge stage: {branch} passes every merge gate but has NO "
829
+ f"merge_queue entry — (dry-run) would enqueue {winner.assignment_id} now"
830
+ )
831
+ res.recovered = True
832
+ return
833
+
834
+ changed = mq.enqueue_approved_work(config, board)
835
+ if changed:
836
+ res.actions_taken.append(
837
+ f"merge stage: {branch} passed every merge gate but had no "
838
+ f"merge_queue entry (#1601) — enqueued {', '.join(changed)}"
839
+ )
840
+ res.recovered = True
841
+ else:
842
+ res.findings.append(
843
+ f"merge stage: {branch} appears to pass every merge gate but "
844
+ "enqueue_approved_work made no change for it — inspect by hand "
845
+ "(e.g. already merged/closed on GitHub, or repo not in config)"
846
+ )
847
+ res.recovered = False
848
+
849
+
850
+ def _recover_test(
851
+ board, latest, state, res: DiagnoseResult, *, config, dry_run: bool
852
+ ) -> None:
853
+ """#1605: the Test-gate check.
854
+
855
+ ``latest`` here is the WORK row (``STAGE_ASSIGNMENT_TYPES["test"] ==
856
+ ("work", "plan")`` — ``test_state`` lives on it, not on the
857
+ ``type="smoke"`` child that actually ran the suite). Before this,
858
+ nothing here ever looked past the work row itself: a work row wedged at
859
+ ``test_state="running"`` with its smoke child already dead/failed fell
860
+ straight through to :func:`_recover_work_like`'s catch-all
861
+ ("stage looks healthy", since ``latest.status`` is already ``"done"``)
862
+ — exactly the #1598 incident's "nothing to reconcile" symptom, and
863
+ exactly why the daemon restart mentioned in that report didn't clear it
864
+ either: nothing was ever looking at the CHILD row.
865
+ """
866
+ if latest.test_state == "running":
867
+ smoke = next(
868
+ (
869
+ a
870
+ for a in (board.active + board.completed)
871
+ if a.type == "smoke"
872
+ and a.review_of_assignment_id == latest.assignment_id
873
+ ),
874
+ None,
875
+ )
876
+ if smoke is None:
877
+ res.findings.append(
878
+ "⚠ test_state='running' but no Test-stage (smoke) assignment "
879
+ "exists for this work row at all — the 'running' marker is "
880
+ "set at dispatch (#1426) so the child row should exist "
881
+ "(#1605 class)."
882
+ )
883
+ res.needs_reset = True
884
+ return
885
+ if (smoke.status or "") in ("failed", "cancelled"):
886
+ res.findings.append(
887
+ f"⚠ test_state='running' but the Test-stage worker "
888
+ f"{smoke.assignment_id} already finished "
889
+ f"(status={smoke.status!r}, failure_reason="
890
+ f"{smoke.failure_reason or 'none recorded'!r}) — the parent "
891
+ "verdict was never resolved (#1605)."
892
+ )
893
+ if dry_run:
894
+ res.findings.append(
895
+ "(dry-run) would resolve test_state from the smoke "
896
+ "child's terminal status — passed, failed, or cleared "
897
+ "for re-dispatch depending on the #1590 environmental "
898
+ "classification"
899
+ )
900
+ res.needs_reset = True
901
+ return
902
+ from coord.reconcile import ( # noqa: PLC0415
903
+ propagate_smoke_terminal_failure,
904
+ )
905
+
906
+ propagate_smoke_terminal_failure(
907
+ parent_assignment_id=latest.assignment_id,
908
+ failure_reason=smoke.failure_reason,
909
+ )
910
+ res.actions_taken.append(
911
+ f"resolved stuck test_state='running' from smoke child "
912
+ f"{smoke.assignment_id}'s terminal status={smoke.status!r} "
913
+ "(#1605)"
914
+ )
915
+ res.recovered = True
916
+ return
917
+ _recover_work_like(board, config, latest, state, res, dry_run=dry_run)
918
+
919
+
920
+ def _recover_work_like(
921
+ board, config, latest, state, res: DiagnoseResult, *, dry_run: bool
922
+ ) -> None:
923
+ if state == "dead" and latest.status in ("running", "pending"):
924
+ res.findings.append("session is dead but board still says running (phantom)")
925
+ if not dry_run:
926
+ res.actions_taken.append(f"finalized phantom session ({_finalize_dead(latest, config)})")
927
+ res.recovered = True
928
+ elif latest.status == "failed" and latest.failure_reason:
929
+ # #618: assignment failed at launch (worktree-add or similar). The
930
+ # failure_reason tells us what happened; if it's a "branch already checked
931
+ # out" error we can detect and prune the blocking orphaned worktree.
932
+ res.findings.append(
933
+ f"launch-failed: {latest.failure_reason}"
934
+ )
935
+ if latest.branch:
936
+ _prune_orphan_for_failed(board, config, latest, res, dry_run=dry_run)
937
+ # Only mark recovered when _prune_orphan_for_failed did NOT set needs_reset
938
+ # (dirty worktrees that couldn't be pruned mean the block is still present).
939
+ if not res.needs_reset:
940
+ res.recovered = True # stage row is already terminal — nothing more needed
941
+ elif latest.status == "failed":
942
+ # #814: remote interactive sessions finalize as "failed" without setting
943
+ # failure_reason (the local-launch code path sets it; the remote backstop
944
+ # in finalize_remote_interactive_exit does not). The stage row is already
945
+ # terminal, but there may be a blocking branch lock on the remote machine
946
+ # that will cause the next retry to fail identically — detect and fix it.
947
+ res.findings.append("work stage failed (no captured failure reason)")
948
+ if latest.branch:
949
+ _prune_orphan_for_failed(board, config, latest, res, dry_run=dry_run)
950
+ if not res.needs_reset:
951
+ res.recovered = True
952
+ elif state == "live" and _is_stale(latest):
953
+ res.findings.append("session is LIVE but stale (idle days) — reset to clear it")
954
+ res.needs_reset = True
955
+ elif state == "live":
956
+ res.findings.append("session is live and recent — left running")
957
+ res.recovered = True
958
+ elif (
959
+ latest.type == "work"
960
+ and latest.status == "done"
961
+ and not (latest.branch or "").strip()
962
+ ):
963
+ # #1155: a `done` work row with no branch is never legitimately
964
+ # reviewable — there is no branch to open a PR against. This is
965
+ # exactly the shape the #448 zero-commit / unresolved-worktree guard
966
+ # is supposed to catch before it ever reaches `done`; if one slipped
967
+ # through anyway, downgrade it here rather than leaving it sitting in
968
+ # the Pipeline indistinguishable from real reviewable work.
969
+ res.findings.append(
970
+ "work stage is 'done' but has no branch — not reviewable (#1155)"
971
+ )
972
+ if not dry_run:
973
+ res.actions_taken.append(
974
+ f"downgraded empty-branch done row to advisory "
975
+ f"({_downgrade_empty_branch_done(latest, config)})"
976
+ )
977
+ res.recovered = True
978
+ elif latest.type in ("work", "plan") and latest.status == "advisory":
979
+ # #1606: an ADVISORY row is TERMINAL — no session is running and
980
+ # nothing else on the board will ever move it forward, so this must
981
+ # NOT fall through to "stage looks healthy" (that false-healthy read
982
+ # is exactly what made the state invisible: `coord retry` refused it,
983
+ # `--accept-advisory` adopted it, and this diagnose call said
984
+ # everything was fine). Ask GitHub the same zero-commit question
985
+ # #1534's review gate asks, so a genuine zero-commit exit (nothing
986
+ # pushed — `coord retry` now handles this) is reported distinctly
987
+ # from the #1357 false-positive shape (real commits present — that
988
+ # one needs `coord drive --accept-advisory`, not a diagnose fix).
989
+ #
990
+ # Deliberately `latest.type in ("work", "plan")`, not just "work":
991
+ # `_recover_work_like` also runs for `stage in ("plan", "test",
992
+ # "merge")` (STAGE_ASSIGNMENT_TYPES["work"] itself is `("work",
993
+ # "plan")` — a plan row can be `latest` for `--stage work` too), and
994
+ # `reconcile.py`'s advisory transition sets `done.status =
995
+ # "advisory"` unconditionally before its type-specific branches, so
996
+ # a zero-commit `type="plan"` row CAN land here. Without this it
997
+ # would silently fall through to the "stage looks healthy" catch-all
998
+ # below — the exact false-healthy read this fix exists to close.
999
+ stage_label = "work" if latest.type == "work" else "plan"
1000
+ ahead = _work_advisory_commits_ahead(latest, config)
1001
+ if ahead == 0:
1002
+ res.findings.append(
1003
+ f"{stage_label} stage is 'advisory' with 0 commits on its "
1004
+ "branch — nothing was pushed, so there is nothing to test, "
1005
+ f"review, or merge; re-dispatch with `coord retry "
1006
+ f"{latest.assignment_id}`"
1007
+ )
1008
+ res.recovered = False
1009
+ elif ahead is None:
1010
+ res.findings.append(
1011
+ f"{stage_label} stage is 'advisory' but its commit count "
1012
+ "against the base branch could not be confirmed (gh lookup "
1013
+ f"failed) — not reporting healthy; inspect by hand: coord "
1014
+ f"log {latest.assignment_id}"
1015
+ )
1016
+ res.recovered = False
1017
+ else:
1018
+ res.findings.append(
1019
+ f"{stage_label} stage is 'advisory' with {ahead} commit(s) "
1020
+ "on its branch — the #1357 false-positive signature, not a "
1021
+ "genuine zero-commit exit; `coord retry` refuses to touch "
1022
+ "it on purpose, use `coord drive --accept-advisory` to "
1023
+ "proceed"
1024
+ )
1025
+ res.recovered = True
1026
+ else:
1027
+ res.findings.append("stage looks healthy")
1028
+ res.recovered = True
1029
+
1030
+
1031
+ def _work_advisory_commits_ahead(assignment: "Assignment", config: "Config") -> int | None:
1032
+ """#1606: commits *assignment*'s branch carries over the repo's base
1033
+ branch, or ``None`` when it cannot be confirmed.
1034
+
1035
+ Thin wrapper (kept as its own name so tests can monkeypatch it without
1036
+ reaching into ``github_ops``) around
1037
+ :func:`coord.github_ops.branch_commits_ahead_for_assignment` — the one
1038
+ shared implementation `coord retry`'s advisory gate
1039
+ (``coord/commands/dispatch.py``) also calls, rather than each keeping
1040
+ its own copy of "branch empty -> 0, repo missing -> None, else ask
1041
+ GitHub". Reuses the ``gh api compare`` call (the same one #1534's review
1042
+ zero-commit gate uses) rather than a local git checkout — `coord
1043
+ diagnose` runs on the daemon host, which has no guarantee of a local
1044
+ clone of every worker's branch.
1045
+ """
1046
+ from coord import github_ops # noqa: PLC0415
1047
+
1048
+ return github_ops.branch_commits_ahead_for_assignment(assignment, config)
1049
+
1050
+
1051
+ def _prune_orphan_for_failed(
1052
+ board, config, latest: "Assignment", res: DiagnoseResult, *, dry_run: bool
1053
+ ) -> None:
1054
+ """#618/#814: if *latest* is a failed launch, detect and prune the orphaned
1055
+ worktree that caused the "branch already checked out" collision.
1056
+
1057
+ Also checks (#814) whether the blocking holder is the repo BASE checkout
1058
+ (~/src/<repo>) on the assignment's machine. Coord-managed worktrees are
1059
+ under ``~/.coord/worktrees/`` and can be force-removed; the base checkout
1060
+ must NEVER be removed — instead, ``git checkout <default_branch>`` frees the
1061
+ branch. This second check is performed remotely via SSH when the assignment
1062
+ ran on a different machine.
1063
+ """
1064
+ branch = latest.branch
1065
+ if not branch:
1066
+ return
1067
+ repo_name = latest.repo_name
1068
+ repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
1069
+ if repo_cfg is None:
1070
+ return
1071
+
1072
+ # Find the repo path on the local machine.
1073
+ repo_path: Path | None = None
1074
+ for machine in config.machines:
1075
+ rp = machine.repo_path(repo_name)
1076
+ if rp:
1077
+ candidate = Path(rp).expanduser()
1078
+ if candidate.exists():
1079
+ repo_path = candidate
1080
+ break
1081
+ if repo_path is None:
1082
+ # #814: even without a local path, attempt the remote base-checkout check.
1083
+ _maybe_fix_base_checkout_lock(latest, config, branch, res, dry_run=dry_run)
1084
+ return
1085
+
1086
+ active_ids = _active_assignment_ids_for_repo(board, repo_name)
1087
+ orphans = _find_orphaned_worktrees(repo_path, branch, active_assignment_ids=active_ids)
1088
+ if not orphans:
1089
+ # #814: no local coord worktree holding the branch — check whether the
1090
+ # BASE checkout on the assignment's machine is the blocker.
1091
+ _maybe_fix_base_checkout_lock(latest, config, branch, res, dry_run=dry_run)
1092
+ return
1093
+
1094
+ res.findings.append(
1095
+ f"found {len(orphans)} orphaned worktree(s) holding branch {branch!r}: "
1096
+ + ", ".join(str(p) for p in orphans)
1097
+ )
1098
+ if dry_run:
1099
+ res.findings.append(
1100
+ f"(dry-run) would prune {len(orphans)} orphaned worktree(s) "
1101
+ "(re-run without --dry-run to remove)"
1102
+ )
1103
+ return
1104
+
1105
+ removed, skipped = _prune_orphaned_worktrees(repo_path, orphans)
1106
+ if removed:
1107
+ res.actions_taken.append(
1108
+ f"pruned {len(removed)} orphaned worktree(s): "
1109
+ + ", ".join(str(p) for p in removed)
1110
+ )
1111
+ if skipped:
1112
+ res.findings.append(
1113
+ f"{len(skipped)} worktree(s) skipped (uncommitted work — inspect manually): "
1114
+ + ", ".join(str(p) for p in skipped)
1115
+ )
1116
+ res.needs_reset = True
1117
+
1118
+
1119
+ def _maybe_fix_base_checkout_lock(
1120
+ latest: "Assignment",
1121
+ config: "Config",
1122
+ branch: str,
1123
+ res: DiagnoseResult,
1124
+ *,
1125
+ dry_run: bool,
1126
+ ) -> None:
1127
+ """#814: detect and optionally fix a base-checkout branch lock on the
1128
+ assignment's machine (local or remote).
1129
+
1130
+ When ``~/src/<repo>`` on the target machine is checked out on *branch*,
1131
+ ``git worktree add`` refuses to create a worktree for that branch, causing
1132
+ launch failures that loop uselessly. The fix is ``git checkout
1133
+ <default_branch>`` in the base checkout — NEVER pruning or deleting it
1134
+ (invariant #561).
1135
+
1136
+ Works for both local assignments (SSH to ``localhost``) and remote ones.
1137
+ SSH failures are silently ignored — conservative: if we can't check we
1138
+ don't report a false "healthy".
1139
+ """
1140
+ machine = next(
1141
+ (m for m in config.machines if m.name == latest.machine_name), None
1142
+ )
1143
+ if machine is None:
1144
+ return
1145
+ repo_name = latest.repo_name
1146
+ repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
1147
+ if repo_cfg is None:
1148
+ return
1149
+
1150
+ rp_str = machine.repo_path(repo_name)
1151
+ if not rp_str:
1152
+ return
1153
+ # Build the $HOME-form path for the remote shell.
1154
+ if rp_str.startswith("~/"):
1155
+ remote_repo_sh = "$HOME/" + rp_str[2:]
1156
+ elif rp_str == "~":
1157
+ remote_repo_sh = "$HOME"
1158
+ else:
1159
+ remote_repo_sh = rp_str
1160
+
1161
+ default_branch = repo_cfg.default_branch or "main"
1162
+
1163
+ try:
1164
+ from coord.interactive import ( # noqa: PLC0415
1165
+ _holder_is_base_checkout,
1166
+ _remote_base_checkout_free_branch,
1167
+ find_remote_branch_holder,
1168
+ )
1169
+ except ImportError:
1170
+ return # interactive module unavailable — skip gracefully
1171
+
1172
+ holder = find_remote_branch_holder(machine.host, remote_repo_sh, branch)
1173
+ if holder is None or not _holder_is_base_checkout(holder):
1174
+ return # not the base-checkout case
1175
+
1176
+ res.findings.append(
1177
+ f"base checkout {holder!r} on {machine.host} is on branch {branch!r}"
1178
+ f" — this blocks worktree creation for {branch!r}"
1179
+ )
1180
+ if dry_run:
1181
+ res.findings.append(
1182
+ f"(dry-run) would checkout {default_branch!r} in {holder!r}"
1183
+ f" on {machine.host} to free the branch"
1184
+ )
1185
+ return
1186
+
1187
+ freed = _remote_base_checkout_free_branch(
1188
+ machine.host, remote_repo_sh, default_branch,
1189
+ )
1190
+ if freed:
1191
+ res.actions_taken.append(
1192
+ f"freed base checkout {holder!r} on {machine.host}:"
1193
+ f" checked out {default_branch!r} (was on {branch!r})"
1194
+ )
1195
+ else:
1196
+ res.findings.append(
1197
+ f"could not auto-free base checkout on {machine.host} —"
1198
+ f" run manually: ssh {machine.host}"
1199
+ f" 'git -C {remote_repo_sh} checkout {default_branch}'"
1200
+ )
1201
+ res.needs_reset = True
1202
+
1203
+
1204
+ def _do_reset(
1205
+ board, config, assignments, res: DiagnoseResult, *, stage: str,
1206
+ repo_name: str, issue_number: int, dry_run: bool,
1207
+ ) -> None:
1208
+ """Stage-aware, non-destructive reset (KEEP the branch + commits always).
1209
+
1210
+ The shape of "reset" depends on the stage's state, not just on a live
1211
+ session: a completed REVIEW has no session to kill — its data lives in the
1212
+ board rows + #603 store — so resetting it means wiping that data so the
1213
+ stage goes back to grey/unrun and re-reviewable.
1214
+ """
1215
+ latest = _latest(assignments)
1216
+ if latest is None:
1217
+ res.findings.append(f"no {stage} stage to reset")
1218
+ res.recovered = True
1219
+ return
1220
+
1221
+ if stage == "review":
1222
+ # #1180: `_reset_review_stage`'s `assignment_id` means "the id of the
1223
+ # assignment BEING reviewed" — that's the FK the review rows carry and
1224
+ # the id the test-author/mock-author review_state reset keys on. But
1225
+ # STAGE_ASSIGNMENT_TYPES["review"] matches ('review','test-author',
1226
+ # 'mock-author'), so `latest` is EITHER:
1227
+ # - the reviewed row itself (test-author/mock-author wedged before a
1228
+ # review was ever dispatched — the JIT-slice case), or
1229
+ # - a type='review' row, whose OWN id is meaningless here; the
1230
+ # reviewed assignment is its `review_of_assignment_id` FK
1231
+ # (set at review.py: review_of_assignment_id=completed.assignment_id).
1232
+ # Passing the review row's own id would match no FK, silently resetting
1233
+ # nothing — resolve the reviewed id explicitly.
1234
+ target_id = (
1235
+ latest.review_of_assignment_id
1236
+ if latest.type == "review" and latest.review_of_assignment_id
1237
+ else latest.assignment_id
1238
+ )
1239
+ _reset_review_stage(
1240
+ config, repo_name, issue_number, res,
1241
+ dry_run=dry_run, assignment_id=target_id,
1242
+ )
1243
+ return
1244
+ if stage == "test":
1245
+ _reset_test_stage(repo_name, issue_number, res, dry_run=dry_run)
1246
+ return
1247
+
1248
+ # work / plan / merge — clear a live/phantom session, KEEP the branch.
1249
+ # (Merge reset deliberately does NOT un-merge; it only clears a stuck
1250
+ # session/row, so a clean re-attempt is possible without rewriting history.)
1251
+ if dry_run:
1252
+ res.findings.append("(dry-run) would reset: stop session, finalize, clear row — branch kept")
1253
+ res.needs_reset = True
1254
+ return
1255
+ if _session_state(latest, config) == "live" and _kill_session(latest, config):
1256
+ res.actions_taken.append("stopped the live session (tmux kill-session)")
1257
+ try:
1258
+ res.actions_taken.append(f"finalized session ({_finalize_dead(latest, config)})")
1259
+ except Exception as exc: # noqa: BLE001 — fall back to a direct terminal mark
1260
+ res.findings.append(f"finalize failed ({exc}); marking row terminal directly")
1261
+ _mark_terminal(latest, config)
1262
+ res.actions_taken.append("marked stage row terminal")
1263
+ res.reset_performed = True
1264
+ res.recovered = True
1265
+ res.branch_preserved = True
1266
+ res.actions_taken.append("branch preserved — stage is re-dispatchable")
1267
+
1268
+
1269
+ def _reset_review_stage(
1270
+ config, repo_name: str, issue_number: int, res: DiagnoseResult, *,
1271
+ dry_run: bool, assignment_id: str,
1272
+ ) -> None:
1273
+ """Wipe a completed review so the stage returns to grey + re-reviewable:
1274
+ delete the ``type='review'`` rows, reset the work's ``review_state``, and
1275
+ purge the #603 ``source='review'`` context entries (the operator's
1276
+ 'completely cleared out' choice). No branch/commits touched.
1277
+
1278
+ #1180: ``assignment_id`` is **the id of the assignment being reviewed** —
1279
+ NOT the id of a ``type='review'`` row. It is threaded through to both the
1280
+ delete and the reset so a milestone tracking issue with multiple
1281
+ ``test-author``/``mock-author`` slices only has the *targeted* slice's
1282
+ review data touched — see ``state.delete_assignments_for_issue`` and
1283
+ ``state.reset_work_review_state`` docstrings for the aliasing hazard this
1284
+ guards against. ``work``/``plan`` behavior is unchanged (still issue-wide,
1285
+ which is safe for those types).
1286
+
1287
+ Callers must resolve this themselves: the review stage's ``latest`` row can
1288
+ be either the reviewed assignment (test-author/mock-author, no review
1289
+ dispatched yet) or a ``type='review'`` row pointing at it via
1290
+ ``review_of_assignment_id`` — the two cases need different resolution. See
1291
+ ``_do_reset``.
1292
+ """
1293
+ from coord import state # noqa: PLC0415
1294
+
1295
+ if dry_run:
1296
+ res.findings.append(
1297
+ "(dry-run) would DELETE the review rows, reset work review_state → "
1298
+ "pending, and purge #603 review notes (box → grey, re-reviewable)"
1299
+ )
1300
+ res.needs_reset = True
1301
+ return
1302
+ deleted = state.delete_assignments_for_issue(
1303
+ repo_name, issue_number, types=("review",),
1304
+ review_of_assignment_id=assignment_id,
1305
+ )
1306
+ res.actions_taken.append(f"deleted {deleted} review row(s) → stage grey")
1307
+ updated = state.reset_work_review_state(
1308
+ repo_name, issue_number, assignment_id=assignment_id
1309
+ )
1310
+ res.actions_taken.append(f"reset review_state→pending on {updated} work row(s) (re-reviewable)")
1311
+ purged = state.clear_issue_context_by_source(repo_name, issue_number, "review")
1312
+ res.actions_taken.append(f"purged {purged} #603 review note(s)")
1313
+ res.reset_performed = True
1314
+ res.recovered = True
1315
+ res.branch_preserved = True
1316
+
1317
+
1318
+ def _reset_test_stage(
1319
+ repo_name: str, issue_number: int, res: DiagnoseResult, *, dry_run: bool
1320
+ ) -> None:
1321
+ """Clear the Test-gate verdict so the issue is re-testable. No code touched."""
1322
+ from coord import state # noqa: PLC0415
1323
+
1324
+ if dry_run:
1325
+ res.findings.append("(dry-run) would clear test_state → re-testable")
1326
+ res.needs_reset = True
1327
+ return
1328
+ updated = state.reset_work_test_state(repo_name, issue_number)
1329
+ res.actions_taken.append(f"cleared Test verdict on {updated} work row(s) (re-testable)")
1330
+ res.reset_performed = True
1331
+ res.recovered = True
1332
+ res.branch_preserved = True
1333
+
1334
+
1335
+ def _cleanup_issue(
1336
+ board,
1337
+ config,
1338
+ repo_name,
1339
+ issue_number,
1340
+ res: DiagnoseResult,
1341
+ *,
1342
+ dry_run: bool,
1343
+ reset: bool,
1344
+ skip_ids: set | None = None,
1345
+ ) -> None:
1346
+ """Always-on, issue-scoped DB *scan*: any OTHER phantom ``running`` rows for
1347
+ this issue whose session is dead are reported. They are only FINALIZED
1348
+ (a write) when *reset* is set.
1349
+
1350
+ #1658: this sweep looks past the one row the operator explicitly asked to
1351
+ diagnose — at every other row for the issue — so a false "dead" verdict
1352
+ here (or a genuinely-live row this scan wasn't asked about) has no
1353
+ operator-reviewed finding backing it the way the targeted stage's own
1354
+ best-effort recovery does. That over-reach is exactly what turned a
1355
+ plain, no-flags ``coord diagnose --stage test`` into a write against a
1356
+ live headless review worker's row (finalized to ``failed`` mid-review):
1357
+ ``reset`` was ``False`` and it wrote anyway. Now a phantom found here is
1358
+ only a *recommendation* — ``needs_reset=True`` plus a finding telling the
1359
+ operator to re-run with ``--reset`` — unless ``--reset`` was already
1360
+ passed, in which case the existing finalize behaviour is unchanged.
1361
+
1362
+ #2087 (fix-review nit): a sibling row on an unconfigured machine is a
1363
+ phantom too, by the same reasoning ``diagnose_stage`` already applies to
1364
+ the row it was explicitly asked about — but ``_session_state`` reports
1365
+ "unknown" (not "dead") for it, since a machine that isn't in
1366
+ ``coordinator.yml`` can't be probed at all. Before this, that "unknown"
1367
+ made this sweep silently skip it: a milestone tracking issue with a
1368
+ ``work`` row *and* a sibling ``smoke`` row, both on the same
1369
+ unconfigured machine, diagnosed with ``--stage work``, reported the
1370
+ ``work`` row correctly but said nothing about the ``smoke`` sibling.
1371
+ Flag it the same way regardless of session probe result — reuses
1372
+ ``_finalize_dead``, already safe for an unconfigured machine (no host to
1373
+ probe/ssh into; see ``_do_reset``'s identical use for the targeted row).
1374
+ """
1375
+ skip = skip_ids or set()
1376
+ for a in (board.active + board.completed):
1377
+ if a.issue_number != issue_number or a.repo_name != repo_name:
1378
+ continue
1379
+ if a.assignment_id in skip:
1380
+ continue
1381
+ if a.status not in ("running", "pending"):
1382
+ continue
1383
+ machine_unconfigured = (
1384
+ bool(a.machine_name) and _resolve_machine(config, a.machine_name) is None
1385
+ )
1386
+ if not machine_unconfigured and _session_state(a, config) != "dead":
1387
+ continue
1388
+ if machine_unconfigured:
1389
+ res.findings.append(
1390
+ f"cleanup: phantom {a.type} row {a.assignment_id} — machine "
1391
+ f"{a.machine_name!r} is not a configured machine (not in "
1392
+ "coordinator.yml)"
1393
+ )
1394
+ else:
1395
+ res.findings.append(f"cleanup: phantom {a.type} row {a.assignment_id} (session dead)")
1396
+ if not reset:
1397
+ res.findings.append(
1398
+ f"cleanup: would finalize phantom {a.type} row {a.assignment_id} "
1399
+ "— re-run with --reset to clear it"
1400
+ )
1401
+ res.needs_reset = True
1402
+ continue
1403
+ if dry_run:
1404
+ res.findings.append(f"(dry-run) would finalize phantom {a.type} row {a.assignment_id}")
1405
+ res.needs_reset = True
1406
+ continue
1407
+ try:
1408
+ _finalize_dead(a, config)
1409
+ res.actions_taken.append(f"cleanup: finalized phantom {a.type} row {a.assignment_id}")
1410
+ except Exception as exc: # noqa: BLE001
1411
+ _mark_terminal(a, config)
1412
+ res.actions_taken.append(f"cleanup: marked phantom row {a.assignment_id} terminal ({exc})")
1413
+
1414
+
1415
+ def _is_stale(assignment: "Assignment", *, max_age_hours: float = 12.0) -> bool:
1416
+ """A still-running session whose dispatch is older than *max_age_hours* is
1417
+ treated as stale (abandoned/idle) — recovery can't safely finalize a live
1418
+ session, so these escalate to a reset offer."""
1419
+ if not assignment.dispatched_at:
1420
+ return False
1421
+ return (time.time() - assignment.dispatched_at) > max_age_hours * 3600.0
1422
+
1423
+
1424
+ # ── #618: orphaned worktree detection + pruning ──────────────────────────────
1425
+
1426
+
1427
+ def _find_orphaned_worktrees(
1428
+ repo_path: Path,
1429
+ branch: str | None,
1430
+ *,
1431
+ active_assignment_ids: set[str],
1432
+ worktrees_dir: Path | None = None,
1433
+ ) -> list[Path]:
1434
+ """Return worktree paths under *worktrees_dir* that hold *branch* but belong
1435
+ to no active (live-tmux OR running-DB) assignment.
1436
+
1437
+ A worktree is "orphaned" when ALL of:
1438
+ * Its directory is under ``~/.coord/worktrees/`` (coordinator-managed).
1439
+ * Its git checkout has *branch* checked out (or *branch* is ``None``,
1440
+ meaning any branch — used for fleet sweeps).
1441
+ * Its assignment_id (derived from the directory name) is NOT in
1442
+ *active_assignment_ids* — i.e. no live tmux session and no running DB row.
1443
+
1444
+ Dirty worktrees (uncommitted changes) are listed but callers must skip
1445
+ force-remove — they'd lose uncommitted work. Use ``_prune_orphaned_worktrees``
1446
+ to prune them with an uncommitted-work guard.
1447
+ """
1448
+ if worktrees_dir is None:
1449
+ from coord.state import COORD_DIR # noqa: PLC0415
1450
+ worktrees_dir = COORD_DIR / "worktrees"
1451
+
1452
+ orphans: list[Path] = []
1453
+ try:
1454
+ result = subprocess.run(
1455
+ ["git", "worktree", "list", "--porcelain"],
1456
+ cwd=str(repo_path),
1457
+ capture_output=True,
1458
+ text=True,
1459
+ timeout=10.0,
1460
+ )
1461
+ except (subprocess.SubprocessError, OSError):
1462
+ return []
1463
+ if result.returncode != 0:
1464
+ return []
1465
+
1466
+ # Parse the porcelain output into blocks.
1467
+ current: dict[str, str] = {}
1468
+ for raw_line in result.stdout.splitlines():
1469
+ line = raw_line.strip()
1470
+ if not line:
1471
+ if current:
1472
+ _maybe_orphan(current, branch, worktrees_dir, active_assignment_ids, orphans)
1473
+ current = {}
1474
+ elif line.startswith("worktree "):
1475
+ current["worktree"] = line[len("worktree "):]
1476
+ elif line.startswith("branch "):
1477
+ raw_branch = line[len("branch "):]
1478
+ current["branch"] = (
1479
+ raw_branch[len("refs/heads/"):] if raw_branch.startswith("refs/heads/") else raw_branch
1480
+ )
1481
+ if current:
1482
+ _maybe_orphan(current, branch, worktrees_dir, active_assignment_ids, orphans)
1483
+
1484
+ return orphans
1485
+
1486
+
1487
+ def _maybe_orphan(
1488
+ entry: dict[str, str],
1489
+ branch: str | None,
1490
+ worktrees_dir: Path,
1491
+ active_assignment_ids: set[str],
1492
+ out: list[Path],
1493
+ ) -> None:
1494
+ """Append to *out* if *entry* is an orphaned worktree for *branch*.
1495
+
1496
+ When *branch* is ``None`` any branch matches (fleet sweep).
1497
+ """
1498
+ wt_str = entry.get("worktree", "")
1499
+ if not wt_str:
1500
+ return
1501
+ if branch is not None and entry.get("branch", "") != branch:
1502
+ return
1503
+ wt_path = Path(wt_str)
1504
+ # Only consider coordinator-managed worktrees (under ~/.coord/worktrees/).
1505
+ try:
1506
+ wt_path.relative_to(worktrees_dir)
1507
+ except ValueError:
1508
+ return
1509
+ # The assignment_id is the directory name component immediately under worktrees_dir.
1510
+ aid = wt_path.relative_to(worktrees_dir).parts[0]
1511
+ if aid in active_assignment_ids:
1512
+ return
1513
+ out.append(wt_path)
1514
+
1515
+
1516
+ def _prune_orphaned_worktrees(
1517
+ repo_path: Path,
1518
+ orphans: list[Path],
1519
+ *,
1520
+ force: bool = False,
1521
+ ) -> tuple[list[Path], list[Path]]:
1522
+ """Remove *orphans* from *repo_path* via ``git worktree remove``.
1523
+
1524
+ Returns ``(removed, skipped)``. Worktrees with uncommitted changes are
1525
+ skipped when *force* is ``False`` (default) so no uncommitted work is lost.
1526
+ After removal, runs ``git worktree prune`` to clean admin entries.
1527
+ """
1528
+ removed: list[Path] = []
1529
+ skipped: list[Path] = []
1530
+ for wt in orphans:
1531
+ if not wt.exists():
1532
+ removed.append(wt)
1533
+ continue
1534
+ if not force:
1535
+ # Check for uncommitted changes — skip dirty worktrees.
1536
+ try:
1537
+ dirty = subprocess.run(
1538
+ ["git", "status", "--porcelain"],
1539
+ cwd=str(wt),
1540
+ capture_output=True,
1541
+ text=True,
1542
+ timeout=10.0,
1543
+ )
1544
+ if dirty.returncode == 0 and dirty.stdout.strip():
1545
+ skipped.append(wt)
1546
+ continue
1547
+ except (subprocess.SubprocessError, OSError):
1548
+ skipped.append(wt)
1549
+ continue
1550
+ try:
1551
+ r = subprocess.run(
1552
+ ["git", "worktree", "remove", str(wt), "--force"],
1553
+ cwd=str(repo_path),
1554
+ capture_output=True,
1555
+ timeout=15.0,
1556
+ )
1557
+ if r.returncode == 0:
1558
+ removed.append(wt)
1559
+ else:
1560
+ skipped.append(wt)
1561
+ except (subprocess.SubprocessError, OSError):
1562
+ skipped.append(wt)
1563
+ # Prune stale git admin entries regardless of what was removed.
1564
+ try:
1565
+ subprocess.run(
1566
+ ["git", "worktree", "prune"],
1567
+ cwd=str(repo_path),
1568
+ capture_output=True,
1569
+ timeout=10.0,
1570
+ )
1571
+ except (subprocess.SubprocessError, OSError):
1572
+ pass
1573
+ return removed, skipped
1574
+
1575
+
1576
+ def _active_assignment_ids_for_repo(
1577
+ board: "Board", repo_name: str
1578
+ ) -> set[str]:
1579
+ """Return assignment IDs for *repo_name* that are still running/pending."""
1580
+ return {
1581
+ a.assignment_id
1582
+ for a in board.active
1583
+ if a.repo_name == repo_name and a.assignment_id
1584
+ }
1585
+
1586
+
1587
+ def find_and_prune_orphaned_worktrees(
1588
+ board: "Board",
1589
+ config: "Config",
1590
+ repo_name: str,
1591
+ branch: str,
1592
+ ) -> tuple[list[Path], list[Path]]:
1593
+ """Detect and prune orphaned coordinator worktrees holding *branch*.
1594
+
1595
+ Public entry point used by :func:`diagnose_stage` (Gap 2 of #618) and
1596
+ by the ``coord diagnose --orphan-worktrees`` fleet sweep.
1597
+
1598
+ Returns ``(removed, skipped)`` path lists. The *skipped* list contains
1599
+ worktrees that have uncommitted changes — the operator must inspect and
1600
+ clean them manually.
1601
+ """
1602
+ repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
1603
+ if repo_cfg is None:
1604
+ return [], []
1605
+
1606
+ # Find the local checkout path for this repo. We need it to run git commands.
1607
+ # On a thin client the local checkout may not exist; fall back gracefully.
1608
+ repo_path: Path | None = None
1609
+ for machine in config.machines:
1610
+ rp = machine.repo_path(repo_name)
1611
+ if rp:
1612
+ candidate = Path(rp).expanduser()
1613
+ if candidate.exists():
1614
+ repo_path = candidate
1615
+ break
1616
+ if repo_path is None:
1617
+ return [], []
1618
+
1619
+ active_ids = _active_assignment_ids_for_repo(board, repo_name)
1620
+ orphans = _find_orphaned_worktrees(repo_path, branch, active_assignment_ids=active_ids)
1621
+ if not orphans:
1622
+ return [], []
1623
+ return _prune_orphaned_worktrees(repo_path, orphans)