code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/diagnose.py
ADDED
|
@@ -0,0 +1,1623 @@
|
|
|
1
|
+
"""Per-stage "doctor": diagnose a pipeline stage, best-effort recover, and —
|
|
2
|
+
when recovery isn't possible — offer a non-destructive reset.
|
|
3
|
+
|
|
4
|
+
Pipeline stages routinely get into bad DB states with no clean UI recovery:
|
|
5
|
+
phantom ``running`` rows (board says running, no live session — #366), reviews
|
|
6
|
+
whose findings were silently dropped (#607), stale-but-live detached sessions
|
|
7
|
+
days old (#494/#370/#546), merged-but-grey boxes, orphaned worktrees. This
|
|
8
|
+
module is the orchestration the TUI's "Diagnose & fix stage" action and the
|
|
9
|
+
``coord diagnose`` command call; it *composes* existing primitives rather than
|
|
10
|
+
reinventing them:
|
|
11
|
+
|
|
12
|
+
* :func:`coord.interactive.finalize_interactive_exit` — record a terminal state
|
|
13
|
+
for a dead/phantom session (pushes commits, releases claim, prunes worktree).
|
|
14
|
+
* :func:`coord.interactive._review_findings_from_transcript` — the #617
|
|
15
|
+
remote-aware transcript-floor that recovers a review's verdict + findings from
|
|
16
|
+
the session's own host.
|
|
17
|
+
* :func:`coord.reconcile.reconcile_board_merges` — flip merged-but-grey work and
|
|
18
|
+
backfill missing branches.
|
|
19
|
+
|
|
20
|
+
Design decisions (locked with the operator):
|
|
21
|
+
|
|
22
|
+
* **Reset is non-destructive**: it clears the stage's board rows, releases the
|
|
23
|
+
claim, removes the orphaned worktree, and stops a live session — but NEVER
|
|
24
|
+
deletes the feature branch. ``origin/issue-<N>-*`` and its commits are
|
|
25
|
+
preserved, so the stage re-dispatches fresh with the work intact. (There is
|
|
26
|
+
deliberately no branch-deletion code path in this module.)
|
|
27
|
+
* **Cleanup is scoped to the one issue**, not a fleet-wide sweep.
|
|
28
|
+
* **The issue-wide phantom-row scan never writes without ``--reset``** (#1658):
|
|
29
|
+
``diagnose_stage``'s targeted best-effort recovery of the STAGE the operator
|
|
30
|
+
asked about may still write without ``--reset`` (that's the whole point of
|
|
31
|
+
"best-effort recover"), but :func:`_cleanup_issue`'s sweep over the issue's
|
|
32
|
+
OTHER rows only ever reports a finding + ``needs_reset=True`` unless
|
|
33
|
+
``--reset`` was passed. That sweep touches rows the operator did not ask
|
|
34
|
+
about and did not get a tailored diagnosis for, so a wrong liveness read on
|
|
35
|
+
it is pure collateral damage — see :func:`_session_state`'s note on why
|
|
36
|
+
tmux-only liveness was itself wrong for headless workers.
|
|
37
|
+
|
|
38
|
+
The side-effecting steps are factored into small module-level helpers so the
|
|
39
|
+
orchestration in :func:`diagnose_stage` is unit-testable by monkeypatching them.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import logging
|
|
45
|
+
import subprocess
|
|
46
|
+
import time
|
|
47
|
+
from dataclasses import dataclass, field
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
from typing import TYPE_CHECKING
|
|
50
|
+
|
|
51
|
+
if TYPE_CHECKING: # avoid import cycles / heavy imports at module load
|
|
52
|
+
from coord.config import Config
|
|
53
|
+
from coord.models import Assignment, Board, Machine
|
|
54
|
+
|
|
55
|
+
# Stages the doctor understands. Each maps to the assignment ``type`` that
|
|
56
|
+
# carries its state; ``test`` and ``merge`` are tracked on the *work* row
|
|
57
|
+
# (``test_state`` / ``status='merged'`` + the merge queue) rather than a
|
|
58
|
+
# dedicated assignment type.
|
|
59
|
+
STAGE_ASSIGNMENT_TYPES: dict[str, tuple[str, ...]] = {
|
|
60
|
+
"plan": ("plan",),
|
|
61
|
+
"work": ("work", "plan"),
|
|
62
|
+
# #1180: a `type="test-author"`/`"mock-author"` completion carries its own
|
|
63
|
+
# `review_state`/`review_verdict` (they're in WORK_LIKE_TYPES and go
|
|
64
|
+
# through the same review chokepoint as `work`) but never spawns a
|
|
65
|
+
# dedicated `type="review"` row when it's wedged — before this fix,
|
|
66
|
+
# `coord diagnose --stage review` looked at `type="review"` rows only, so
|
|
67
|
+
# a test-author row stuck at `review_state="done"` with no verdict and no
|
|
68
|
+
# review assignment was invisible: the tool would report on whatever
|
|
69
|
+
# unrelated `type="review"` row happened to share the tracking issue
|
|
70
|
+
# number (false "stage looks healthy"/wrong-row confidence) instead of
|
|
71
|
+
# flagging the real wedge.
|
|
72
|
+
"review": ("review", "test-author", "mock-author"),
|
|
73
|
+
"test": ("work", "plan"),
|
|
74
|
+
"merge": ("work", "plan"),
|
|
75
|
+
# #2087: previously absent entirely — `--stage smoke` (or an implicit
|
|
76
|
+
# `current_stage()` pick landing on a `type="smoke"` row, e.g. a Test
|
|
77
|
+
# stage dispatched more recently than its parent `work` row) fell
|
|
78
|
+
# straight into the "no diagnosis available" dead end below with no
|
|
79
|
+
# recovery and no `--reset` path. Routed through the same work-like
|
|
80
|
+
# recovery as `work`/`plan` (`diagnose_stage`'s per-stage dispatch and
|
|
81
|
+
# `_do_reset` both fall back to that branch for any type not `review` or
|
|
82
|
+
# `test`) — a smoke row is a one-shot session exactly like a work row.
|
|
83
|
+
"smoke": ("smoke",),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class DiagnoseResult:
|
|
89
|
+
"""Outcome of a diagnose/recover/reset run for one stage of one issue."""
|
|
90
|
+
|
|
91
|
+
repo_name: str
|
|
92
|
+
issue_number: int
|
|
93
|
+
stage: str
|
|
94
|
+
findings: list[str] = field(default_factory=list)
|
|
95
|
+
actions_taken: list[str] = field(default_factory=list)
|
|
96
|
+
# True when the stage is healthy after this run (nothing was wrong, or the
|
|
97
|
+
# problem was recovered). False + needs_reset=True means "still wedged".
|
|
98
|
+
recovered: bool = False
|
|
99
|
+
# True when best-effort recovery could not clear the problem and the only
|
|
100
|
+
# remaining option is a reset.
|
|
101
|
+
needs_reset: bool = False
|
|
102
|
+
# Always True for this module — reset keeps the branch. Surfaced so the TUI
|
|
103
|
+
# can promise "keeps branch + commits" in the confirm dialog.
|
|
104
|
+
branch_preserved: bool = True
|
|
105
|
+
# Whether a reset was actually performed this run.
|
|
106
|
+
reset_performed: bool = False
|
|
107
|
+
|
|
108
|
+
def to_json_dict(self) -> dict:
|
|
109
|
+
"""Return a JSON-serialisable dict of all DiagnoseResult fields.
|
|
110
|
+
Used by ``coord diagnose --json`` and the daemon ``post_diagnose``
|
|
111
|
+
handler (#935 Part C) so the TUI can parse findings/actions without
|
|
112
|
+
scraping the human-readable output lines."""
|
|
113
|
+
import dataclasses # noqa: PLC0415 — lazy to avoid circular import risk
|
|
114
|
+
return dataclasses.asdict(self)
|
|
115
|
+
|
|
116
|
+
def summary_line(self) -> str:
|
|
117
|
+
"""The machine-readable trailer the TUI greps for (mirrors the
|
|
118
|
+
``coord:`` marker convention)."""
|
|
119
|
+
return (
|
|
120
|
+
f"DIAGNOSE_RESULT: stage={self.stage} "
|
|
121
|
+
f"recovered={str(self.recovered).lower()} "
|
|
122
|
+
f"needs_reset={str(self.needs_reset).lower()} "
|
|
123
|
+
f"reset_performed={str(self.reset_performed).lower()} "
|
|
124
|
+
f"actions={len(self.actions_taken)}"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ── stage / assignment resolution ───────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def stage_assignments(
|
|
132
|
+
board: "Board", repo_name: str, issue_number: int, stage: str
|
|
133
|
+
) -> list["Assignment"]:
|
|
134
|
+
"""All assignments for *issue_number* in *repo_name* matching *stage*,
|
|
135
|
+
newest-dispatched first. Mirrors the TUI's ``assignments_for_stage``."""
|
|
136
|
+
types = STAGE_ASSIGNMENT_TYPES.get(stage, (stage,))
|
|
137
|
+
rows = [
|
|
138
|
+
a
|
|
139
|
+
for a in (board.active + board.completed)
|
|
140
|
+
if a.issue_number == issue_number
|
|
141
|
+
and a.repo_name == repo_name
|
|
142
|
+
and (a.type or "work") in types
|
|
143
|
+
]
|
|
144
|
+
rows.sort(key=lambda a: (a.dispatched_at or 0.0), reverse=True)
|
|
145
|
+
return rows
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _latest(assignments: list["Assignment"]) -> "Assignment | None":
|
|
149
|
+
return assignments[0] if assignments else None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _flag_contradictory_failed(latest: "Assignment", res: DiagnoseResult) -> None:
|
|
153
|
+
"""#1451: flag a ``status='failed'`` work row whose own fields already
|
|
154
|
+
prove it isn't — a passing test verdict and/or an approved review on a
|
|
155
|
+
row that pushed a real branch is self-evidently not a failure.
|
|
156
|
+
|
|
157
|
+
This is deliberately NOT based on ``exit_code``/``failure_reason`` being
|
|
158
|
+
empty — those are empty on the overwhelming majority of legitimate
|
|
159
|
+
``failed`` rows too (a launch-failure ``failure_reason`` is the rare
|
|
160
|
+
exception, and no current write path persists ``exit_code`` to the DB at
|
|
161
|
+
all), so that pair is not a usable signal on its own. The reliable
|
|
162
|
+
signal is a genuine contradiction: evidence of *success* recorded on a
|
|
163
|
+
row the board calls failed.
|
|
164
|
+
|
|
165
|
+
Best-effort and read-only — appends a finding only, no write. Detection,
|
|
166
|
+
not correction: the fix is either ``coord report-result --assignment
|
|
167
|
+
<id> --status done`` (interactive) or re-running the completing worker.
|
|
168
|
+
"""
|
|
169
|
+
if latest.status != "failed":
|
|
170
|
+
return
|
|
171
|
+
contradictions: list[str] = []
|
|
172
|
+
if latest.test_state == "passed":
|
|
173
|
+
contradictions.append("test_state=passed")
|
|
174
|
+
if latest.review_verdict == "approve":
|
|
175
|
+
contradictions.append("review_verdict=approve")
|
|
176
|
+
if not contradictions:
|
|
177
|
+
return
|
|
178
|
+
if not latest.branch:
|
|
179
|
+
# No pushed branch to review/test at all — the "passed"/"approve"
|
|
180
|
+
# values must be stale carry-over from a prior assignment row, not
|
|
181
|
+
# evidence about *this* failed row. Don't flag without a branch.
|
|
182
|
+
return
|
|
183
|
+
res.findings.append(
|
|
184
|
+
f"⚠ status='failed' contradicts its own fields ({', '.join(contradictions)}, "
|
|
185
|
+
f"branch={latest.branch}) — looks like a phantom failure (#1451), not a "
|
|
186
|
+
"real one. If the work is actually done, recover it with "
|
|
187
|
+
f"`coord report-result --assignment {latest.assignment_id} --status done "
|
|
188
|
+
'--summary "..."` (or re-run the completing worker if unsure).'
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def current_stage(board: "Board", repo_name: str, issue_number: int) -> str:
|
|
193
|
+
"""The stage of the most-recently-dispatched assignment for the issue
|
|
194
|
+
(what ``coord diagnose <repo> <issue>`` targets when ``--stage`` is
|
|
195
|
+
omitted). Falls back to ``work`` when the issue has no assignments.
|
|
196
|
+
|
|
197
|
+
#1083: previously coerced any assignment ``type`` this module doesn't
|
|
198
|
+
recognize (e.g. ``test-author``, ``mock-author``, ``smoke``) to
|
|
199
|
+
``"work"`` — which then had ``diagnose_stage`` recover/report on
|
|
200
|
+
whatever unrelated ``work``/``plan`` row happened to exist for the issue,
|
|
201
|
+
*silently* presenting it as if it were a diagnosis of the real (ignored)
|
|
202
|
+
assignment. Now the actual type is returned verbatim; ``diagnose_stage``
|
|
203
|
+
explicitly reports "no diagnosis available" for types outside
|
|
204
|
+
:data:`STAGE_ASSIGNMENT_TYPES` instead of guessing.
|
|
205
|
+
"""
|
|
206
|
+
rows = [
|
|
207
|
+
a
|
|
208
|
+
for a in (board.active + board.completed)
|
|
209
|
+
if a.issue_number == issue_number and a.repo_name == repo_name
|
|
210
|
+
]
|
|
211
|
+
if not rows:
|
|
212
|
+
return "work"
|
|
213
|
+
newest = max(rows, key=lambda a: (a.dispatched_at or 0.0))
|
|
214
|
+
return newest.type or "work"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ── monkeypatchable side-effecting wrappers ─────────────────────────────────
|
|
218
|
+
#
|
|
219
|
+
# Each wraps an existing primitive and is replaced in unit tests so the
|
|
220
|
+
# orchestration can be exercised without touching git/tmux/the network.
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _resolve_machine(config: "Config", machine_name: str | None):
|
|
224
|
+
if not machine_name:
|
|
225
|
+
return None
|
|
226
|
+
return next((m for m in config.machines if m.name == machine_name), None)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _session_state(assignment: "Assignment", config: "Config") -> str:
|
|
230
|
+
"""``"live"`` | ``"dead"`` | ``"unknown"`` for *assignment*.
|
|
231
|
+
|
|
232
|
+
Probes the assignment's machine (local tmux, or the remote host's tmux over
|
|
233
|
+
ssh — same mechanism as ``coord reattach`` / the stale-session reaper).
|
|
234
|
+
``"unknown"`` when the machine can't be resolved or the probe errors, so the
|
|
235
|
+
caller never finalizes on a false negative.
|
|
236
|
+
|
|
237
|
+
#1658: tmux liveness alone is blind to HEADLESS workers. A headless
|
|
238
|
+
assignment (the normal shape for a daemon-dispatched review/work — see
|
|
239
|
+
``AgentServer.assign``) runs as a plain subprocess tracked by the agent's
|
|
240
|
+
own ``_assignments`` dict; it never has a tmux session at all, so
|
|
241
|
+
``tmux_session_alive`` reads "dead" for it unconditionally, regardless of
|
|
242
|
+
whether the worker is still running. Before this fix, that false "dead"
|
|
243
|
+
made every live headless assignment look like a phantom the instant
|
|
244
|
+
``coord diagnose`` looked at it — the incident this closes: a live review
|
|
245
|
+
worker's row was finalized to ``failed`` mid-review. Now, when tmux says
|
|
246
|
+
dead, the assignment's own agent ``/status`` is consulted before trusting
|
|
247
|
+
that — the same seam :func:`coord.reconcile.reconcile_completed_assignments`
|
|
248
|
+
uses to tell "still running" from "actually finished" — and a match in its
|
|
249
|
+
``active`` list is authoritative: the agent is the ground truth for its
|
|
250
|
+
own subprocesses. An unreachable agent still returns "unknown" rather than
|
|
251
|
+
"dead", preserving the never-finalize-on-a-probe-failure guarantee.
|
|
252
|
+
"""
|
|
253
|
+
import socket # noqa: PLC0415
|
|
254
|
+
|
|
255
|
+
from coord.interactive import ( # noqa: PLC0415
|
|
256
|
+
TmuxHost,
|
|
257
|
+
tmux_session_alive,
|
|
258
|
+
tmux_session_name,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
if not assignment.assignment_id:
|
|
262
|
+
return "unknown"
|
|
263
|
+
machine = _resolve_machine(config, assignment.machine_name)
|
|
264
|
+
ssh_target = None
|
|
265
|
+
if machine is not None:
|
|
266
|
+
local_hn = socket.gethostname().split(".")[0].lower()
|
|
267
|
+
is_local = (
|
|
268
|
+
machine.name.lower() == local_hn
|
|
269
|
+
or machine.host.split(".")[0].lower() == local_hn
|
|
270
|
+
)
|
|
271
|
+
if not is_local:
|
|
272
|
+
ssh_target = machine.host
|
|
273
|
+
elif assignment.machine_name:
|
|
274
|
+
# machine_name set but unknown in config — can't probe safely.
|
|
275
|
+
return "unknown"
|
|
276
|
+
host = TmuxHost(ssh_target=ssh_target)
|
|
277
|
+
sname = tmux_session_name(assignment.assignment_id)
|
|
278
|
+
try:
|
|
279
|
+
if tmux_session_alive(sname, host=host):
|
|
280
|
+
return "live"
|
|
281
|
+
except Exception: # noqa: BLE001 — never let a probe error finalize a session
|
|
282
|
+
return "unknown"
|
|
283
|
+
|
|
284
|
+
# tmux says dead (or the assignment never had a tmux session at all — the
|
|
285
|
+
# headless case). Consult the agent before trusting that.
|
|
286
|
+
return _agent_liveness(assignment, machine)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _agent_liveness(assignment: "Assignment", machine: "Machine | None") -> str:
|
|
290
|
+
"""``"dead"`` | ``"live"`` | ``"unknown"`` per the assignment's own agent
|
|
291
|
+
``/status`` — see :func:`_session_state`'s #1658 note for why this exists.
|
|
292
|
+
``machine`` is ``None`` when the assignment has no ``machine_name`` at all
|
|
293
|
+
(rare) — treated as genuinely dead since there's nothing to probe."""
|
|
294
|
+
if machine is None:
|
|
295
|
+
return "dead"
|
|
296
|
+
from coord.network import fetch_status # noqa: PLC0415
|
|
297
|
+
|
|
298
|
+
result = fetch_status(machine)
|
|
299
|
+
if not result.ok or result.data is None:
|
|
300
|
+
# Agent unreachable — don't trust tmux-dead alone, but don't claim
|
|
301
|
+
# "live" either. Matches the "never finalize on a probe failure"
|
|
302
|
+
# contract the rest of this module relies on.
|
|
303
|
+
return "unknown"
|
|
304
|
+
active = result.data.get("active") or []
|
|
305
|
+
if any(isinstance(e, dict) and e.get("id") == assignment.assignment_id for e in active):
|
|
306
|
+
return "live"
|
|
307
|
+
return "dead"
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _ssh_target_for(assignment: "Assignment", config: "Config") -> str | None:
|
|
311
|
+
"""The ssh host for *assignment*'s machine, or ``None`` when it's local."""
|
|
312
|
+
import socket # noqa: PLC0415
|
|
313
|
+
|
|
314
|
+
machine = _resolve_machine(config, assignment.machine_name)
|
|
315
|
+
if machine is None:
|
|
316
|
+
return None
|
|
317
|
+
local_hn = socket.gethostname().split(".")[0].lower()
|
|
318
|
+
if machine.name.lower() == local_hn or machine.host.split(".")[0].lower() == local_hn:
|
|
319
|
+
return None
|
|
320
|
+
return machine.host
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _recover_review_findings(assignment: "Assignment", config: "Config") -> str | None:
|
|
324
|
+
"""Recover a review's verdict + findings from its session transcript and
|
|
325
|
+
persist them through the durable seam (#617). Returns the verdict on
|
|
326
|
+
success, ``None`` when nothing was recoverable. Read-only w.r.t. the
|
|
327
|
+
session (safe to run even while it's live)."""
|
|
328
|
+
from coord import issue_store # noqa: PLC0415
|
|
329
|
+
from coord.interactive import _review_findings_from_transcript # noqa: PLC0415
|
|
330
|
+
|
|
331
|
+
if not assignment.assignment_id:
|
|
332
|
+
return None
|
|
333
|
+
assignment_id: str = assignment.assignment_id
|
|
334
|
+
ssh_target = _ssh_target_for(assignment, config)
|
|
335
|
+
started_at = assignment.dispatched_at
|
|
336
|
+
findings = _review_findings_from_transcript(
|
|
337
|
+
assignment.issue_number,
|
|
338
|
+
started_at,
|
|
339
|
+
assignment_id=assignment_id,
|
|
340
|
+
ssh_target=ssh_target,
|
|
341
|
+
)
|
|
342
|
+
if findings is None:
|
|
343
|
+
return None
|
|
344
|
+
repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
|
|
345
|
+
try:
|
|
346
|
+
issue_store.post_result(
|
|
347
|
+
issue_store.ResultRecord(
|
|
348
|
+
assignment_id=assignment_id,
|
|
349
|
+
machine_name=assignment.machine_name or "unknown",
|
|
350
|
+
repo_name=assignment.repo_name,
|
|
351
|
+
repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
|
|
352
|
+
issue_number=assignment.issue_number,
|
|
353
|
+
status="done",
|
|
354
|
+
verdict=findings.verdict, # type: ignore[arg-type]
|
|
355
|
+
summary="Findings recovered from the session transcript by coord diagnose.",
|
|
356
|
+
findings_body=findings.body,
|
|
357
|
+
branch=None,
|
|
358
|
+
)
|
|
359
|
+
)
|
|
360
|
+
except RuntimeError as exc:
|
|
361
|
+
# #990: the verdict was recovered from the transcript but couldn't be
|
|
362
|
+
# durably persisted (retries exhausted / readback mismatch). Surface
|
|
363
|
+
# this instead of letting it crash `coord diagnose` — the caller
|
|
364
|
+
# treats a ``None`` return as "not recoverable" and reports
|
|
365
|
+
# "re-review needed", which is the safe outcome here too since the
|
|
366
|
+
# write did not actually land.
|
|
367
|
+
import click # noqa: PLC0415
|
|
368
|
+
|
|
369
|
+
click.echo(
|
|
370
|
+
f" ⚠ recovered verdict {findings.verdict!r} from transcript for "
|
|
371
|
+
f"{assignment.assignment_id} but failed to persist it: {exc}",
|
|
372
|
+
err=True,
|
|
373
|
+
)
|
|
374
|
+
return None
|
|
375
|
+
return findings.verdict
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _finalize_dead(assignment: "Assignment", config: "Config") -> str:
|
|
379
|
+
"""Finalize a dead/phantom session: record a terminal state, push any
|
|
380
|
+
commits, release the claim, prune the worktree. Returns a short status."""
|
|
381
|
+
from coord.interactive import finalize_interactive_exit # noqa: PLC0415
|
|
382
|
+
from coord.state import COORD_DIR # noqa: PLC0415
|
|
383
|
+
|
|
384
|
+
machine = _resolve_machine(config, assignment.machine_name)
|
|
385
|
+
repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
|
|
386
|
+
base = (repo_cfg.default_branch if repo_cfg else None) or "main"
|
|
387
|
+
repo_github = repo_cfg.github if repo_cfg else assignment.repo_name
|
|
388
|
+
repo_path = None
|
|
389
|
+
if machine is not None and assignment.repo_name:
|
|
390
|
+
from pathlib import Path # noqa: PLC0415
|
|
391
|
+
|
|
392
|
+
rp = machine.repo_path(assignment.repo_name)
|
|
393
|
+
if rp:
|
|
394
|
+
repo_path = str(Path(rp).expanduser())
|
|
395
|
+
worktree = str(COORD_DIR / "worktrees" / (assignment.assignment_id or ""))
|
|
396
|
+
fr = finalize_interactive_exit(
|
|
397
|
+
assignment_id=assignment.assignment_id or "",
|
|
398
|
+
repo_name=assignment.repo_name,
|
|
399
|
+
repo_github=repo_github,
|
|
400
|
+
issue_number=assignment.issue_number,
|
|
401
|
+
machine_name=assignment.machine_name or "unknown",
|
|
402
|
+
worktree_path=worktree if assignment.type in ("work", "plan") else None,
|
|
403
|
+
base_branch=base,
|
|
404
|
+
exit_code=0,
|
|
405
|
+
started_at=assignment.dispatched_at,
|
|
406
|
+
repo_path=repo_path,
|
|
407
|
+
ssh_target=_ssh_target_for(assignment, config),
|
|
408
|
+
# #1256: unconditional — restore_live_checkout_from_smoke_snapshot()
|
|
409
|
+
# is a documented no-op unless a snapshot marker for this
|
|
410
|
+
# assignment_id exists in repo_path's .git/ dir, so this is safe for
|
|
411
|
+
# every assignment type, not just "smoke". `coord diagnose`'s
|
|
412
|
+
# unfiltered dead-session sweep (_cleanup_issue) is one of the two
|
|
413
|
+
# automated/operator-recovery paths (the other is
|
|
414
|
+
# reap_stale_interactive_sessions) that exist specifically to handle
|
|
415
|
+
# a session that died without a clean exit — exactly when a
|
|
416
|
+
# --smoke-of session's live-checkout mutation needs reverting.
|
|
417
|
+
smoke_repo_path=repo_path,
|
|
418
|
+
)
|
|
419
|
+
return fr.terminal_status or "finalized"
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _kill_session(assignment: "Assignment", config: "Config") -> bool:
|
|
423
|
+
"""``tmux kill-session`` for *assignment* (local or remote). Used by reset
|
|
424
|
+
to stop a live session before finalizing. Returns True when the kill ran."""
|
|
425
|
+
import subprocess # noqa: PLC0415
|
|
426
|
+
|
|
427
|
+
from coord.interactive import ( # noqa: PLC0415
|
|
428
|
+
TmuxHost,
|
|
429
|
+
tmux_session_name,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
if not assignment.assignment_id:
|
|
433
|
+
return False
|
|
434
|
+
host = TmuxHost(ssh_target=_ssh_target_for(assignment, config))
|
|
435
|
+
sname = tmux_session_name(assignment.assignment_id)
|
|
436
|
+
try:
|
|
437
|
+
subprocess.run(
|
|
438
|
+
host.cmd(["kill-session", "-t", sname]),
|
|
439
|
+
capture_output=True,
|
|
440
|
+
timeout=20,
|
|
441
|
+
)
|
|
442
|
+
return True
|
|
443
|
+
except Exception: # noqa: BLE001 — best-effort
|
|
444
|
+
return False
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _reconcile_issue_merges(
|
|
448
|
+
board: "Board", config: "Config", repo_name: str, issue_number: int, *, dry_run: bool
|
|
449
|
+
) -> list[str]:
|
|
450
|
+
"""Run the merge reconcile sweep scoped to one issue (branch backfill +
|
|
451
|
+
out-of-band-merge detection)."""
|
|
452
|
+
from coord.reconcile import reconcile_board_merges # noqa: PLC0415
|
|
453
|
+
|
|
454
|
+
return reconcile_board_merges(
|
|
455
|
+
board, config, repo=repo_name, issue=issue_number, dry_run=dry_run
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _mark_terminal(assignment: "Assignment", config: "Config") -> None:
|
|
460
|
+
"""Best-effort terminal write via the issue_store seam — the fallback used
|
|
461
|
+
only when :func:`_finalize_dead` itself raised. Records a failed completion
|
|
462
|
+
so the phantom row leaves ``running`` and persists to the canonical DB
|
|
463
|
+
WITHOUT relying on ``save_board`` (which the diagnose path deliberately does
|
|
464
|
+
not call — it would clobber the seam writes with a stale snapshot)."""
|
|
465
|
+
from coord import issue_store # noqa: PLC0415
|
|
466
|
+
|
|
467
|
+
if not assignment.assignment_id:
|
|
468
|
+
return
|
|
469
|
+
repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
|
|
470
|
+
try:
|
|
471
|
+
issue_store.post_completion(
|
|
472
|
+
issue_store.CompletionRecord(
|
|
473
|
+
assignment_id=assignment.assignment_id,
|
|
474
|
+
machine_name=assignment.machine_name or "unknown",
|
|
475
|
+
repo_name=assignment.repo_name,
|
|
476
|
+
repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
|
|
477
|
+
issue_number=assignment.issue_number,
|
|
478
|
+
exit_code=1, # → failed terminal state (out of 'running')
|
|
479
|
+
commits_ahead=0,
|
|
480
|
+
branch=assignment.branch,
|
|
481
|
+
)
|
|
482
|
+
)
|
|
483
|
+
except Exception: # noqa: BLE001 — fallback of a fallback; leave the phantom
|
|
484
|
+
pass
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _downgrade_empty_branch_done(assignment: "Assignment", config: "Config") -> str:
|
|
488
|
+
"""#1155: flip a wedged ``done``-with-empty-branch work row to ``advisory``
|
|
489
|
+
via the issue_store seam (same seam :func:`_mark_terminal` uses for its
|
|
490
|
+
own fallback write). A ``done`` row with no branch has nothing to review
|
|
491
|
+
— it must not sit in the Pipeline masquerading as reviewable work.
|
|
492
|
+
Best-effort: on failure the row is left as-is and the caller's finding
|
|
493
|
+
still surfaces the problem to the operator."""
|
|
494
|
+
from coord import issue_store # noqa: PLC0415
|
|
495
|
+
|
|
496
|
+
if not assignment.assignment_id:
|
|
497
|
+
return "skipped (no assignment_id)"
|
|
498
|
+
repo_cfg = next((r for r in config.repos if r.name == assignment.repo_name), None)
|
|
499
|
+
try:
|
|
500
|
+
outcome = issue_store.post_completion(
|
|
501
|
+
issue_store.CompletionRecord(
|
|
502
|
+
assignment_id=assignment.assignment_id,
|
|
503
|
+
machine_name=assignment.machine_name or "unknown",
|
|
504
|
+
repo_name=assignment.repo_name,
|
|
505
|
+
repo_github=(repo_cfg.github if repo_cfg else assignment.repo_name),
|
|
506
|
+
issue_number=assignment.issue_number,
|
|
507
|
+
exit_code=0,
|
|
508
|
+
commits_ahead=0, # → advisory terminal state (the #448 shape)
|
|
509
|
+
branch=assignment.branch,
|
|
510
|
+
)
|
|
511
|
+
)
|
|
512
|
+
return outcome.status
|
|
513
|
+
except Exception as exc: # noqa: BLE001 — best-effort recovery
|
|
514
|
+
return f"failed ({exc})"
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
# ── orchestration ───────────────────────────────────────────────────────────
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def diagnose_stage(
|
|
521
|
+
board: "Board",
|
|
522
|
+
config: "Config",
|
|
523
|
+
repo_name: str,
|
|
524
|
+
issue_number: int,
|
|
525
|
+
stage: str,
|
|
526
|
+
*,
|
|
527
|
+
reset: bool = False,
|
|
528
|
+
dry_run: bool = False,
|
|
529
|
+
) -> DiagnoseResult:
|
|
530
|
+
"""Diagnose *stage* of *repo_name* #*issue_number*; best-effort recover;
|
|
531
|
+
always reconcile this issue's DB; optionally reset (non-destructive).
|
|
532
|
+
|
|
533
|
+
Returns a :class:`DiagnoseResult`. Board mutations happen on the board
|
|
534
|
+
passed in; the caller is responsible for persisting it (the CLI/daemon do
|
|
535
|
+
so after this returns) — consistent with ``reconcile_board_merges``.
|
|
536
|
+
"""
|
|
537
|
+
res = DiagnoseResult(repo_name=repo_name, issue_number=issue_number, stage=stage)
|
|
538
|
+
|
|
539
|
+
# #1083: `stage` came either from an explicit `--stage` or from
|
|
540
|
+
# `current_stage()`'s newest-assignment lookup. `current_stage()` now
|
|
541
|
+
# surfaces a non-standard assignment type (e.g. "test-author",
|
|
542
|
+
# "mock-author", "smoke") verbatim instead of silently mapping it to
|
|
543
|
+
# "work" — so a type this module has no recovery logic for lands here as
|
|
544
|
+
# `stage` rather than being guessed at. Report that plainly instead of
|
|
545
|
+
# running `_recover_work_like` against it (which was never validated for
|
|
546
|
+
# these types) or, worse, silently returning an unrelated `work`/`plan`
|
|
547
|
+
# row's status as if it were this stage's diagnosis (the bug reported in
|
|
548
|
+
# #1083: `coord diagnose` picked an unrelated, already-merged assignment
|
|
549
|
+
# instead of flagging the real problem).
|
|
550
|
+
if stage not in STAGE_ASSIGNMENT_TYPES:
|
|
551
|
+
assignments = stage_assignments(board, repo_name, issue_number, stage)
|
|
552
|
+
latest = _latest(assignments)
|
|
553
|
+
known = ", ".join(sorted(STAGE_ASSIGNMENT_TYPES))
|
|
554
|
+
if latest is None:
|
|
555
|
+
res.findings.append(
|
|
556
|
+
f"no diagnosis available for assignment type {stage!r} — "
|
|
557
|
+
f"coord diagnose only understands: {known} (and no {stage!r} "
|
|
558
|
+
f"assignment exists for #{issue_number} either)"
|
|
559
|
+
)
|
|
560
|
+
else:
|
|
561
|
+
res.findings.append(
|
|
562
|
+
f"no diagnosis available for assignment type {stage!r} — "
|
|
563
|
+
f"coord diagnose only understands: {known}. Latest {stage!r} "
|
|
564
|
+
f"assignment: {latest.assignment_id} status={latest.status} "
|
|
565
|
+
f"branch={latest.branch or '(none)'} machine={latest.machine_name}"
|
|
566
|
+
)
|
|
567
|
+
res.recovered = False
|
|
568
|
+
res.needs_reset = False
|
|
569
|
+
return res
|
|
570
|
+
|
|
571
|
+
assignments = stage_assignments(board, repo_name, issue_number, stage)
|
|
572
|
+
latest = _latest(assignments)
|
|
573
|
+
|
|
574
|
+
if latest is None:
|
|
575
|
+
res.findings.append(f"no {stage} assignment on the board for #{issue_number}")
|
|
576
|
+
res.recovered = True # nothing wedged
|
|
577
|
+
# Still run the issue-wide cleanup below.
|
|
578
|
+
_cleanup_issue(board, config, repo_name, issue_number, res, dry_run=dry_run, reset=reset)
|
|
579
|
+
return res
|
|
580
|
+
|
|
581
|
+
# The stage step owns *latest*; record it so the issue-wide cleanup pass
|
|
582
|
+
# doesn't re-finalize the same row (finalize writes the DB, not this
|
|
583
|
+
# in-memory board row, so its status would still read "running" here).
|
|
584
|
+
handled = {latest.assignment_id} if latest.assignment_id else set()
|
|
585
|
+
|
|
586
|
+
state = _session_state(latest, config)
|
|
587
|
+
res.findings.append(
|
|
588
|
+
f"{stage}: latest={latest.assignment_id} status={latest.status} "
|
|
589
|
+
f"session={state} machine={latest.machine_name}"
|
|
590
|
+
)
|
|
591
|
+
if stage in ("work", "test", "merge"):
|
|
592
|
+
_flag_contradictory_failed(latest, res)
|
|
593
|
+
|
|
594
|
+
# #2087: a `running`/`pending` row naming a machine that isn't a
|
|
595
|
+
# configured machine can never be probed — `_session_state` above
|
|
596
|
+
# already returns "unknown" for exactly that reason ("machine_name set
|
|
597
|
+
# but unknown in config — can't probe safely"). Before this,
|
|
598
|
+
# every stage's best-effort recovery below had no branch for
|
|
599
|
+
# state=="unknown", so it fell through to that recovery function's own
|
|
600
|
+
# "stage looks healthy" catch-all — the textbook phantom `coord
|
|
601
|
+
# diagnose` exists to find, reported as fine (the exact `work-repro`
|
|
602
|
+
# shape: `machine=laptop`, `status=running` for 9h, "stage looks
|
|
603
|
+
# healthy"). Flag it here, uniformly across every stage, BEFORE any
|
|
604
|
+
# stage-specific recovery gets a chance to fall through to its healthy
|
|
605
|
+
# catch-all — and reuse the existing non-destructive `--reset` path
|
|
606
|
+
# (branch/commits always preserved) as the supported way to clear it.
|
|
607
|
+
machine_unconfigured = (
|
|
608
|
+
bool(latest.machine_name) and _resolve_machine(config, latest.machine_name) is None
|
|
609
|
+
)
|
|
610
|
+
if latest.status in ("running", "pending") and machine_unconfigured:
|
|
611
|
+
res.findings.append(
|
|
612
|
+
f"{stage}: machine {latest.machine_name!r} is not a configured "
|
|
613
|
+
"machine (not in coordinator.yml) — a 'running' row with no "
|
|
614
|
+
"host to poll is a phantom, not healthy; re-run with --reset to "
|
|
615
|
+
"clear it (the branch and any commits are preserved)"
|
|
616
|
+
)
|
|
617
|
+
res.recovered = False
|
|
618
|
+
res.needs_reset = True
|
|
619
|
+
|
|
620
|
+
if reset:
|
|
621
|
+
_do_reset(
|
|
622
|
+
board, config, assignments, res, stage=stage,
|
|
623
|
+
repo_name=repo_name, issue_number=issue_number, dry_run=dry_run,
|
|
624
|
+
)
|
|
625
|
+
_cleanup_issue(
|
|
626
|
+
board, config, repo_name, issue_number, res,
|
|
627
|
+
dry_run=dry_run, reset=reset, skip_ids=handled,
|
|
628
|
+
)
|
|
629
|
+
return res
|
|
630
|
+
|
|
631
|
+
if latest.status in ("running", "pending") and machine_unconfigured:
|
|
632
|
+
# Reported above; nothing safe to attempt without --reset (no host
|
|
633
|
+
# to probe/finalize against) and no stage-specific recovery below
|
|
634
|
+
# would do better than mis-file it as healthy.
|
|
635
|
+
_cleanup_issue(
|
|
636
|
+
board, config, repo_name, issue_number, res,
|
|
637
|
+
dry_run=dry_run, reset=reset, skip_ids=handled,
|
|
638
|
+
)
|
|
639
|
+
return res
|
|
640
|
+
|
|
641
|
+
# ── Best-effort recovery, per stage ─────────────────────────────────────
|
|
642
|
+
if stage in ("review",):
|
|
643
|
+
_recover_review(board, config, latest, state, res, dry_run=dry_run)
|
|
644
|
+
elif stage in ("merge",):
|
|
645
|
+
_recover_merge(board, config, repo_name, issue_number, latest, res, dry_run=dry_run)
|
|
646
|
+
elif stage == "test":
|
|
647
|
+
_recover_test(board, latest, state, res, config=config, dry_run=dry_run)
|
|
648
|
+
else: # work / plan
|
|
649
|
+
_recover_work_like(board, config, latest, state, res, dry_run=dry_run)
|
|
650
|
+
|
|
651
|
+
_cleanup_issue(
|
|
652
|
+
board, config, repo_name, issue_number, res,
|
|
653
|
+
dry_run=dry_run, reset=reset, skip_ids=handled,
|
|
654
|
+
)
|
|
655
|
+
return res
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _recover_review(
|
|
659
|
+
board, config, latest, state, res: DiagnoseResult, *, dry_run: bool
|
|
660
|
+
) -> None:
|
|
661
|
+
from coord.state import load_assignment_review_findings # noqa: PLC0415
|
|
662
|
+
|
|
663
|
+
has_findings = False
|
|
664
|
+
if latest.assignment_id:
|
|
665
|
+
cached = load_assignment_review_findings(latest.assignment_id)
|
|
666
|
+
has_findings = bool(cached and (cached[1] or "").strip())
|
|
667
|
+
|
|
668
|
+
verdict = latest.review_verdict
|
|
669
|
+
if verdict == "request-changes" and not has_findings:
|
|
670
|
+
res.findings.append("review verdict is request-changes but findings are EMPTY (#607 class)")
|
|
671
|
+
if dry_run:
|
|
672
|
+
res.findings.append("(dry-run) would recover findings from the session transcript")
|
|
673
|
+
res.needs_reset = True
|
|
674
|
+
return
|
|
675
|
+
recovered_verdict = _recover_review_findings(latest, config)
|
|
676
|
+
if recovered_verdict:
|
|
677
|
+
res.actions_taken.append("recovered review findings from the session transcript → #603 store")
|
|
678
|
+
res.recovered = True
|
|
679
|
+
else:
|
|
680
|
+
res.findings.append("findings NOT recoverable from transcript — re-review needed")
|
|
681
|
+
res.needs_reset = True
|
|
682
|
+
elif latest.status == "done" and verdict is None:
|
|
683
|
+
# #812: review finalised as done but no verdict was ever captured.
|
|
684
|
+
# The session likely failed to start (no session_id, no exit_code) or
|
|
685
|
+
# exited before the reviewer ran coord report-result / the transcript-floor.
|
|
686
|
+
# This is a permanent stuck state: nothing is running, TUI rendered it
|
|
687
|
+
# blue/Active (now Fixed → red/Failed), Diagnose & Reset must handle it.
|
|
688
|
+
res.findings.append(
|
|
689
|
+
"review finalised as done but has no verdict — "
|
|
690
|
+
"session likely failed to start or exited before verdict capture (#812)"
|
|
691
|
+
)
|
|
692
|
+
if dry_run:
|
|
693
|
+
res.findings.append(
|
|
694
|
+
"(dry-run) would try transcript recovery; if verdict not found, reset"
|
|
695
|
+
)
|
|
696
|
+
res.needs_reset = True
|
|
697
|
+
return
|
|
698
|
+
recovered_verdict = _recover_review_findings(latest, config)
|
|
699
|
+
if recovered_verdict:
|
|
700
|
+
res.actions_taken.append(
|
|
701
|
+
"recovered review verdict/findings from session transcript"
|
|
702
|
+
)
|
|
703
|
+
res.recovered = True
|
|
704
|
+
else:
|
|
705
|
+
res.findings.append(
|
|
706
|
+
"no verdict recoverable from transcript — "
|
|
707
|
+
"reset to re-dispatch a fresh review"
|
|
708
|
+
)
|
|
709
|
+
res.needs_reset = True
|
|
710
|
+
elif state == "dead" and latest.status == "running":
|
|
711
|
+
res.findings.append("review session is dead but board still says running (phantom)")
|
|
712
|
+
if not dry_run:
|
|
713
|
+
# Try a transcript recovery first (captures the verdict if present),
|
|
714
|
+
# then finalize to clear the phantom.
|
|
715
|
+
if _recover_review_findings(latest, config):
|
|
716
|
+
res.actions_taken.append("recovered review verdict/findings from transcript")
|
|
717
|
+
res.actions_taken.append(f"finalized phantom review session ({_finalize_dead(latest, config)})")
|
|
718
|
+
res.recovered = True
|
|
719
|
+
elif state == "live" and _is_stale(latest):
|
|
720
|
+
res.findings.append("review session is LIVE but stale (idle days) — capturing read-only, reset to clear")
|
|
721
|
+
if not dry_run and _recover_review_findings(latest, config):
|
|
722
|
+
res.actions_taken.append("captured current review findings from transcript (session left running)")
|
|
723
|
+
res.needs_reset = True
|
|
724
|
+
else:
|
|
725
|
+
res.findings.append("review stage looks healthy")
|
|
726
|
+
res.recovered = True
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _recover_merge(
|
|
730
|
+
board, config, repo_name, issue_number, latest, res: DiagnoseResult, *, dry_run: bool
|
|
731
|
+
) -> None:
|
|
732
|
+
actions = _reconcile_issue_merges(board, config, repo_name, issue_number, dry_run=dry_run)
|
|
733
|
+
if actions:
|
|
734
|
+
res.actions_taken.extend(actions)
|
|
735
|
+
res.recovered = True
|
|
736
|
+
return
|
|
737
|
+
|
|
738
|
+
# #1601: reconcile_board_merges (above) only ever does two things —
|
|
739
|
+
# backfill a missing `branch` and detect an out-of-band GitHub merge. It
|
|
740
|
+
# has never asked the one question this issue is about: "is this done,
|
|
741
|
+
# approved work even IN the merge queue at all?" A fix round's terminal
|
|
742
|
+
# review verdict routinely lands on a *different* board row than
|
|
743
|
+
# `latest`/the parent work row's own `review_state` (which can be stuck
|
|
744
|
+
# at "dispatched" forever once a later round supersedes it — see the
|
|
745
|
+
# #1566 incident this fixed), and when the periodic enqueue sweep
|
|
746
|
+
# (`merge_queue.enqueue_approved_work`, run from the daemon passive tick)
|
|
747
|
+
# misses its window, the branch is left approved-and-done with an EMPTY
|
|
748
|
+
# merge_queue. Before this, that state was indistinguishable here from
|
|
749
|
+
# actually healthy ("nothing to reconcile"). Ask the same question the
|
|
750
|
+
# merge gate itself asks (`passes_merge_gates`) so this can never
|
|
751
|
+
# disagree with what `coord merge --plan`/`--only` decide.
|
|
752
|
+
_diagnose_unqueued_merge(
|
|
753
|
+
board, config, repo_name, issue_number, latest, res, dry_run=dry_run
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def _diagnose_unqueued_merge(
|
|
758
|
+
board, config, repo_name, issue_number, latest, res: DiagnoseResult, *, dry_run: bool
|
|
759
|
+
) -> None:
|
|
760
|
+
"""#1601: detect (and, when possible, fix) a done+approved branch with
|
|
761
|
+
no merge_queue entry — the "nothing ever enqueued the merge" failure
|
|
762
|
+
mode. Reuses `merge_queue`'s own winner-resolution
|
|
763
|
+
(`group_branch_candidates`) and gate predicate (`passes_merge_gates`) so
|
|
764
|
+
this reports the exact same verdict `coord merge --plan`/`--only` would,
|
|
765
|
+
never a re-derived one."""
|
|
766
|
+
from coord import merge_queue as mq # noqa: PLC0415
|
|
767
|
+
|
|
768
|
+
branch = getattr(latest, "branch", None)
|
|
769
|
+
if not branch:
|
|
770
|
+
res.findings.append("merge stage: nothing to reconcile")
|
|
771
|
+
res.recovered = True
|
|
772
|
+
return
|
|
773
|
+
|
|
774
|
+
existing_queue = mq.load_queue()
|
|
775
|
+
if any(getattr(e, "branch", None) == branch for e in existing_queue):
|
|
776
|
+
res.findings.append("merge stage: nothing to reconcile")
|
|
777
|
+
res.recovered = True
|
|
778
|
+
return
|
|
779
|
+
|
|
780
|
+
scoped_completed = [
|
|
781
|
+
a for a in board.completed
|
|
782
|
+
if a.repo_name == repo_name and getattr(a, "branch", None) == branch
|
|
783
|
+
]
|
|
784
|
+
winners = mq.group_branch_candidates(scoped_completed)
|
|
785
|
+
if not winners:
|
|
786
|
+
res.findings.append("merge stage: nothing to reconcile")
|
|
787
|
+
res.recovered = True
|
|
788
|
+
return
|
|
789
|
+
winner, _superseded = winners[0]
|
|
790
|
+
|
|
791
|
+
# #2085: `winner` is a raw board Assignment — no `branch_head_sha`/
|
|
792
|
+
# `repo_github`/`target_branch` attribute, so handing it straight to
|
|
793
|
+
# `passes_merge_gates` made the #821 SHA-freshness check inside
|
|
794
|
+
# `has_approved_review` permanently unconfirmable (fails closed on every
|
|
795
|
+
# review carrying a real `review_head_sha`, i.e. virtually every modern
|
|
796
|
+
# approval — this diagnostic would report "waiting on the pipeline" for
|
|
797
|
+
# branches that actually pass every real gate). Build the same
|
|
798
|
+
# live-anchored synthetic entry `coord.gates.build_gate_report` uses so
|
|
799
|
+
# a genuinely fresh approval can still be confirmed. Falls back to the
|
|
800
|
+
# raw `winner` row (gate then fails closed, never open) when the repo
|
|
801
|
+
# isn't configured.
|
|
802
|
+
from coord import github_ops # noqa: PLC0415
|
|
803
|
+
|
|
804
|
+
gate_entry = winner
|
|
805
|
+
repo_cfg = config.repo(repo_name)
|
|
806
|
+
if repo_cfg is not None:
|
|
807
|
+
from coord.branch_model import resolve_base_branch_for_issue_number # noqa: PLC0415
|
|
808
|
+
|
|
809
|
+
target_branch = resolve_base_branch_for_issue_number(
|
|
810
|
+
repo_cfg, repo_cfg.github, issue_number,
|
|
811
|
+
)
|
|
812
|
+
gate_entry = mq.live_gate_entry(winner, repo_cfg.github, target_branch, github_ops)
|
|
813
|
+
|
|
814
|
+
if not mq.passes_merge_gates(gate_entry, config, board, gh_ops=github_ops):
|
|
815
|
+
res.findings.append(
|
|
816
|
+
f"merge stage: {branch} is done but not queued for merge, and does "
|
|
817
|
+
f"not (yet) pass the review/smoke gates — winning row "
|
|
818
|
+
f"{winner.assignment_id}: review_state={getattr(winner, 'review_state', None)!r} "
|
|
819
|
+
f"review_verdict={getattr(winner, 'review_verdict', None)!r} "
|
|
820
|
+
f"test_state={getattr(winner, 'test_state', None)!r}. Waiting on the "
|
|
821
|
+
"pipeline, not wedged."
|
|
822
|
+
)
|
|
823
|
+
res.recovered = True
|
|
824
|
+
return
|
|
825
|
+
|
|
826
|
+
if dry_run:
|
|
827
|
+
res.findings.append(
|
|
828
|
+
f"merge stage: {branch} passes every merge gate but has NO "
|
|
829
|
+
f"merge_queue entry — (dry-run) would enqueue {winner.assignment_id} now"
|
|
830
|
+
)
|
|
831
|
+
res.recovered = True
|
|
832
|
+
return
|
|
833
|
+
|
|
834
|
+
changed = mq.enqueue_approved_work(config, board)
|
|
835
|
+
if changed:
|
|
836
|
+
res.actions_taken.append(
|
|
837
|
+
f"merge stage: {branch} passed every merge gate but had no "
|
|
838
|
+
f"merge_queue entry (#1601) — enqueued {', '.join(changed)}"
|
|
839
|
+
)
|
|
840
|
+
res.recovered = True
|
|
841
|
+
else:
|
|
842
|
+
res.findings.append(
|
|
843
|
+
f"merge stage: {branch} appears to pass every merge gate but "
|
|
844
|
+
"enqueue_approved_work made no change for it — inspect by hand "
|
|
845
|
+
"(e.g. already merged/closed on GitHub, or repo not in config)"
|
|
846
|
+
)
|
|
847
|
+
res.recovered = False
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _recover_test(
|
|
851
|
+
board, latest, state, res: DiagnoseResult, *, config, dry_run: bool
|
|
852
|
+
) -> None:
|
|
853
|
+
"""#1605: the Test-gate check.
|
|
854
|
+
|
|
855
|
+
``latest`` here is the WORK row (``STAGE_ASSIGNMENT_TYPES["test"] ==
|
|
856
|
+
("work", "plan")`` — ``test_state`` lives on it, not on the
|
|
857
|
+
``type="smoke"`` child that actually ran the suite). Before this,
|
|
858
|
+
nothing here ever looked past the work row itself: a work row wedged at
|
|
859
|
+
``test_state="running"`` with its smoke child already dead/failed fell
|
|
860
|
+
straight through to :func:`_recover_work_like`'s catch-all
|
|
861
|
+
("stage looks healthy", since ``latest.status`` is already ``"done"``)
|
|
862
|
+
— exactly the #1598 incident's "nothing to reconcile" symptom, and
|
|
863
|
+
exactly why the daemon restart mentioned in that report didn't clear it
|
|
864
|
+
either: nothing was ever looking at the CHILD row.
|
|
865
|
+
"""
|
|
866
|
+
if latest.test_state == "running":
|
|
867
|
+
smoke = next(
|
|
868
|
+
(
|
|
869
|
+
a
|
|
870
|
+
for a in (board.active + board.completed)
|
|
871
|
+
if a.type == "smoke"
|
|
872
|
+
and a.review_of_assignment_id == latest.assignment_id
|
|
873
|
+
),
|
|
874
|
+
None,
|
|
875
|
+
)
|
|
876
|
+
if smoke is None:
|
|
877
|
+
res.findings.append(
|
|
878
|
+
"⚠ test_state='running' but no Test-stage (smoke) assignment "
|
|
879
|
+
"exists for this work row at all — the 'running' marker is "
|
|
880
|
+
"set at dispatch (#1426) so the child row should exist "
|
|
881
|
+
"(#1605 class)."
|
|
882
|
+
)
|
|
883
|
+
res.needs_reset = True
|
|
884
|
+
return
|
|
885
|
+
if (smoke.status or "") in ("failed", "cancelled"):
|
|
886
|
+
res.findings.append(
|
|
887
|
+
f"⚠ test_state='running' but the Test-stage worker "
|
|
888
|
+
f"{smoke.assignment_id} already finished "
|
|
889
|
+
f"(status={smoke.status!r}, failure_reason="
|
|
890
|
+
f"{smoke.failure_reason or 'none recorded'!r}) — the parent "
|
|
891
|
+
"verdict was never resolved (#1605)."
|
|
892
|
+
)
|
|
893
|
+
if dry_run:
|
|
894
|
+
res.findings.append(
|
|
895
|
+
"(dry-run) would resolve test_state from the smoke "
|
|
896
|
+
"child's terminal status — passed, failed, or cleared "
|
|
897
|
+
"for re-dispatch depending on the #1590 environmental "
|
|
898
|
+
"classification"
|
|
899
|
+
)
|
|
900
|
+
res.needs_reset = True
|
|
901
|
+
return
|
|
902
|
+
from coord.reconcile import ( # noqa: PLC0415
|
|
903
|
+
propagate_smoke_terminal_failure,
|
|
904
|
+
)
|
|
905
|
+
|
|
906
|
+
propagate_smoke_terminal_failure(
|
|
907
|
+
parent_assignment_id=latest.assignment_id,
|
|
908
|
+
failure_reason=smoke.failure_reason,
|
|
909
|
+
)
|
|
910
|
+
res.actions_taken.append(
|
|
911
|
+
f"resolved stuck test_state='running' from smoke child "
|
|
912
|
+
f"{smoke.assignment_id}'s terminal status={smoke.status!r} "
|
|
913
|
+
"(#1605)"
|
|
914
|
+
)
|
|
915
|
+
res.recovered = True
|
|
916
|
+
return
|
|
917
|
+
_recover_work_like(board, config, latest, state, res, dry_run=dry_run)
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def _recover_work_like(
|
|
921
|
+
board, config, latest, state, res: DiagnoseResult, *, dry_run: bool
|
|
922
|
+
) -> None:
|
|
923
|
+
if state == "dead" and latest.status in ("running", "pending"):
|
|
924
|
+
res.findings.append("session is dead but board still says running (phantom)")
|
|
925
|
+
if not dry_run:
|
|
926
|
+
res.actions_taken.append(f"finalized phantom session ({_finalize_dead(latest, config)})")
|
|
927
|
+
res.recovered = True
|
|
928
|
+
elif latest.status == "failed" and latest.failure_reason:
|
|
929
|
+
# #618: assignment failed at launch (worktree-add or similar). The
|
|
930
|
+
# failure_reason tells us what happened; if it's a "branch already checked
|
|
931
|
+
# out" error we can detect and prune the blocking orphaned worktree.
|
|
932
|
+
res.findings.append(
|
|
933
|
+
f"launch-failed: {latest.failure_reason}"
|
|
934
|
+
)
|
|
935
|
+
if latest.branch:
|
|
936
|
+
_prune_orphan_for_failed(board, config, latest, res, dry_run=dry_run)
|
|
937
|
+
# Only mark recovered when _prune_orphan_for_failed did NOT set needs_reset
|
|
938
|
+
# (dirty worktrees that couldn't be pruned mean the block is still present).
|
|
939
|
+
if not res.needs_reset:
|
|
940
|
+
res.recovered = True # stage row is already terminal — nothing more needed
|
|
941
|
+
elif latest.status == "failed":
|
|
942
|
+
# #814: remote interactive sessions finalize as "failed" without setting
|
|
943
|
+
# failure_reason (the local-launch code path sets it; the remote backstop
|
|
944
|
+
# in finalize_remote_interactive_exit does not). The stage row is already
|
|
945
|
+
# terminal, but there may be a blocking branch lock on the remote machine
|
|
946
|
+
# that will cause the next retry to fail identically — detect and fix it.
|
|
947
|
+
res.findings.append("work stage failed (no captured failure reason)")
|
|
948
|
+
if latest.branch:
|
|
949
|
+
_prune_orphan_for_failed(board, config, latest, res, dry_run=dry_run)
|
|
950
|
+
if not res.needs_reset:
|
|
951
|
+
res.recovered = True
|
|
952
|
+
elif state == "live" and _is_stale(latest):
|
|
953
|
+
res.findings.append("session is LIVE but stale (idle days) — reset to clear it")
|
|
954
|
+
res.needs_reset = True
|
|
955
|
+
elif state == "live":
|
|
956
|
+
res.findings.append("session is live and recent — left running")
|
|
957
|
+
res.recovered = True
|
|
958
|
+
elif (
|
|
959
|
+
latest.type == "work"
|
|
960
|
+
and latest.status == "done"
|
|
961
|
+
and not (latest.branch or "").strip()
|
|
962
|
+
):
|
|
963
|
+
# #1155: a `done` work row with no branch is never legitimately
|
|
964
|
+
# reviewable — there is no branch to open a PR against. This is
|
|
965
|
+
# exactly the shape the #448 zero-commit / unresolved-worktree guard
|
|
966
|
+
# is supposed to catch before it ever reaches `done`; if one slipped
|
|
967
|
+
# through anyway, downgrade it here rather than leaving it sitting in
|
|
968
|
+
# the Pipeline indistinguishable from real reviewable work.
|
|
969
|
+
res.findings.append(
|
|
970
|
+
"work stage is 'done' but has no branch — not reviewable (#1155)"
|
|
971
|
+
)
|
|
972
|
+
if not dry_run:
|
|
973
|
+
res.actions_taken.append(
|
|
974
|
+
f"downgraded empty-branch done row to advisory "
|
|
975
|
+
f"({_downgrade_empty_branch_done(latest, config)})"
|
|
976
|
+
)
|
|
977
|
+
res.recovered = True
|
|
978
|
+
elif latest.type in ("work", "plan") and latest.status == "advisory":
|
|
979
|
+
# #1606: an ADVISORY row is TERMINAL — no session is running and
|
|
980
|
+
# nothing else on the board will ever move it forward, so this must
|
|
981
|
+
# NOT fall through to "stage looks healthy" (that false-healthy read
|
|
982
|
+
# is exactly what made the state invisible: `coord retry` refused it,
|
|
983
|
+
# `--accept-advisory` adopted it, and this diagnose call said
|
|
984
|
+
# everything was fine). Ask GitHub the same zero-commit question
|
|
985
|
+
# #1534's review gate asks, so a genuine zero-commit exit (nothing
|
|
986
|
+
# pushed — `coord retry` now handles this) is reported distinctly
|
|
987
|
+
# from the #1357 false-positive shape (real commits present — that
|
|
988
|
+
# one needs `coord drive --accept-advisory`, not a diagnose fix).
|
|
989
|
+
#
|
|
990
|
+
# Deliberately `latest.type in ("work", "plan")`, not just "work":
|
|
991
|
+
# `_recover_work_like` also runs for `stage in ("plan", "test",
|
|
992
|
+
# "merge")` (STAGE_ASSIGNMENT_TYPES["work"] itself is `("work",
|
|
993
|
+
# "plan")` — a plan row can be `latest` for `--stage work` too), and
|
|
994
|
+
# `reconcile.py`'s advisory transition sets `done.status =
|
|
995
|
+
# "advisory"` unconditionally before its type-specific branches, so
|
|
996
|
+
# a zero-commit `type="plan"` row CAN land here. Without this it
|
|
997
|
+
# would silently fall through to the "stage looks healthy" catch-all
|
|
998
|
+
# below — the exact false-healthy read this fix exists to close.
|
|
999
|
+
stage_label = "work" if latest.type == "work" else "plan"
|
|
1000
|
+
ahead = _work_advisory_commits_ahead(latest, config)
|
|
1001
|
+
if ahead == 0:
|
|
1002
|
+
res.findings.append(
|
|
1003
|
+
f"{stage_label} stage is 'advisory' with 0 commits on its "
|
|
1004
|
+
"branch — nothing was pushed, so there is nothing to test, "
|
|
1005
|
+
f"review, or merge; re-dispatch with `coord retry "
|
|
1006
|
+
f"{latest.assignment_id}`"
|
|
1007
|
+
)
|
|
1008
|
+
res.recovered = False
|
|
1009
|
+
elif ahead is None:
|
|
1010
|
+
res.findings.append(
|
|
1011
|
+
f"{stage_label} stage is 'advisory' but its commit count "
|
|
1012
|
+
"against the base branch could not be confirmed (gh lookup "
|
|
1013
|
+
f"failed) — not reporting healthy; inspect by hand: coord "
|
|
1014
|
+
f"log {latest.assignment_id}"
|
|
1015
|
+
)
|
|
1016
|
+
res.recovered = False
|
|
1017
|
+
else:
|
|
1018
|
+
res.findings.append(
|
|
1019
|
+
f"{stage_label} stage is 'advisory' with {ahead} commit(s) "
|
|
1020
|
+
"on its branch — the #1357 false-positive signature, not a "
|
|
1021
|
+
"genuine zero-commit exit; `coord retry` refuses to touch "
|
|
1022
|
+
"it on purpose, use `coord drive --accept-advisory` to "
|
|
1023
|
+
"proceed"
|
|
1024
|
+
)
|
|
1025
|
+
res.recovered = True
|
|
1026
|
+
else:
|
|
1027
|
+
res.findings.append("stage looks healthy")
|
|
1028
|
+
res.recovered = True
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _work_advisory_commits_ahead(assignment: "Assignment", config: "Config") -> int | None:
|
|
1032
|
+
"""#1606: commits *assignment*'s branch carries over the repo's base
|
|
1033
|
+
branch, or ``None`` when it cannot be confirmed.
|
|
1034
|
+
|
|
1035
|
+
Thin wrapper (kept as its own name so tests can monkeypatch it without
|
|
1036
|
+
reaching into ``github_ops``) around
|
|
1037
|
+
:func:`coord.github_ops.branch_commits_ahead_for_assignment` — the one
|
|
1038
|
+
shared implementation `coord retry`'s advisory gate
|
|
1039
|
+
(``coord/commands/dispatch.py``) also calls, rather than each keeping
|
|
1040
|
+
its own copy of "branch empty -> 0, repo missing -> None, else ask
|
|
1041
|
+
GitHub". Reuses the ``gh api compare`` call (the same one #1534's review
|
|
1042
|
+
zero-commit gate uses) rather than a local git checkout — `coord
|
|
1043
|
+
diagnose` runs on the daemon host, which has no guarantee of a local
|
|
1044
|
+
clone of every worker's branch.
|
|
1045
|
+
"""
|
|
1046
|
+
from coord import github_ops # noqa: PLC0415
|
|
1047
|
+
|
|
1048
|
+
return github_ops.branch_commits_ahead_for_assignment(assignment, config)
|
|
1049
|
+
|
|
1050
|
+
|
|
1051
|
+
def _prune_orphan_for_failed(
|
|
1052
|
+
board, config, latest: "Assignment", res: DiagnoseResult, *, dry_run: bool
|
|
1053
|
+
) -> None:
|
|
1054
|
+
"""#618/#814: if *latest* is a failed launch, detect and prune the orphaned
|
|
1055
|
+
worktree that caused the "branch already checked out" collision.
|
|
1056
|
+
|
|
1057
|
+
Also checks (#814) whether the blocking holder is the repo BASE checkout
|
|
1058
|
+
(~/src/<repo>) on the assignment's machine. Coord-managed worktrees are
|
|
1059
|
+
under ``~/.coord/worktrees/`` and can be force-removed; the base checkout
|
|
1060
|
+
must NEVER be removed — instead, ``git checkout <default_branch>`` frees the
|
|
1061
|
+
branch. This second check is performed remotely via SSH when the assignment
|
|
1062
|
+
ran on a different machine.
|
|
1063
|
+
"""
|
|
1064
|
+
branch = latest.branch
|
|
1065
|
+
if not branch:
|
|
1066
|
+
return
|
|
1067
|
+
repo_name = latest.repo_name
|
|
1068
|
+
repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
|
|
1069
|
+
if repo_cfg is None:
|
|
1070
|
+
return
|
|
1071
|
+
|
|
1072
|
+
# Find the repo path on the local machine.
|
|
1073
|
+
repo_path: Path | None = None
|
|
1074
|
+
for machine in config.machines:
|
|
1075
|
+
rp = machine.repo_path(repo_name)
|
|
1076
|
+
if rp:
|
|
1077
|
+
candidate = Path(rp).expanduser()
|
|
1078
|
+
if candidate.exists():
|
|
1079
|
+
repo_path = candidate
|
|
1080
|
+
break
|
|
1081
|
+
if repo_path is None:
|
|
1082
|
+
# #814: even without a local path, attempt the remote base-checkout check.
|
|
1083
|
+
_maybe_fix_base_checkout_lock(latest, config, branch, res, dry_run=dry_run)
|
|
1084
|
+
return
|
|
1085
|
+
|
|
1086
|
+
active_ids = _active_assignment_ids_for_repo(board, repo_name)
|
|
1087
|
+
orphans = _find_orphaned_worktrees(repo_path, branch, active_assignment_ids=active_ids)
|
|
1088
|
+
if not orphans:
|
|
1089
|
+
# #814: no local coord worktree holding the branch — check whether the
|
|
1090
|
+
# BASE checkout on the assignment's machine is the blocker.
|
|
1091
|
+
_maybe_fix_base_checkout_lock(latest, config, branch, res, dry_run=dry_run)
|
|
1092
|
+
return
|
|
1093
|
+
|
|
1094
|
+
res.findings.append(
|
|
1095
|
+
f"found {len(orphans)} orphaned worktree(s) holding branch {branch!r}: "
|
|
1096
|
+
+ ", ".join(str(p) for p in orphans)
|
|
1097
|
+
)
|
|
1098
|
+
if dry_run:
|
|
1099
|
+
res.findings.append(
|
|
1100
|
+
f"(dry-run) would prune {len(orphans)} orphaned worktree(s) "
|
|
1101
|
+
"(re-run without --dry-run to remove)"
|
|
1102
|
+
)
|
|
1103
|
+
return
|
|
1104
|
+
|
|
1105
|
+
removed, skipped = _prune_orphaned_worktrees(repo_path, orphans)
|
|
1106
|
+
if removed:
|
|
1107
|
+
res.actions_taken.append(
|
|
1108
|
+
f"pruned {len(removed)} orphaned worktree(s): "
|
|
1109
|
+
+ ", ".join(str(p) for p in removed)
|
|
1110
|
+
)
|
|
1111
|
+
if skipped:
|
|
1112
|
+
res.findings.append(
|
|
1113
|
+
f"{len(skipped)} worktree(s) skipped (uncommitted work — inspect manually): "
|
|
1114
|
+
+ ", ".join(str(p) for p in skipped)
|
|
1115
|
+
)
|
|
1116
|
+
res.needs_reset = True
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _maybe_fix_base_checkout_lock(
|
|
1120
|
+
latest: "Assignment",
|
|
1121
|
+
config: "Config",
|
|
1122
|
+
branch: str,
|
|
1123
|
+
res: DiagnoseResult,
|
|
1124
|
+
*,
|
|
1125
|
+
dry_run: bool,
|
|
1126
|
+
) -> None:
|
|
1127
|
+
"""#814: detect and optionally fix a base-checkout branch lock on the
|
|
1128
|
+
assignment's machine (local or remote).
|
|
1129
|
+
|
|
1130
|
+
When ``~/src/<repo>`` on the target machine is checked out on *branch*,
|
|
1131
|
+
``git worktree add`` refuses to create a worktree for that branch, causing
|
|
1132
|
+
launch failures that loop uselessly. The fix is ``git checkout
|
|
1133
|
+
<default_branch>`` in the base checkout — NEVER pruning or deleting it
|
|
1134
|
+
(invariant #561).
|
|
1135
|
+
|
|
1136
|
+
Works for both local assignments (SSH to ``localhost``) and remote ones.
|
|
1137
|
+
SSH failures are silently ignored — conservative: if we can't check we
|
|
1138
|
+
don't report a false "healthy".
|
|
1139
|
+
"""
|
|
1140
|
+
machine = next(
|
|
1141
|
+
(m for m in config.machines if m.name == latest.machine_name), None
|
|
1142
|
+
)
|
|
1143
|
+
if machine is None:
|
|
1144
|
+
return
|
|
1145
|
+
repo_name = latest.repo_name
|
|
1146
|
+
repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
|
|
1147
|
+
if repo_cfg is None:
|
|
1148
|
+
return
|
|
1149
|
+
|
|
1150
|
+
rp_str = machine.repo_path(repo_name)
|
|
1151
|
+
if not rp_str:
|
|
1152
|
+
return
|
|
1153
|
+
# Build the $HOME-form path for the remote shell.
|
|
1154
|
+
if rp_str.startswith("~/"):
|
|
1155
|
+
remote_repo_sh = "$HOME/" + rp_str[2:]
|
|
1156
|
+
elif rp_str == "~":
|
|
1157
|
+
remote_repo_sh = "$HOME"
|
|
1158
|
+
else:
|
|
1159
|
+
remote_repo_sh = rp_str
|
|
1160
|
+
|
|
1161
|
+
default_branch = repo_cfg.default_branch or "main"
|
|
1162
|
+
|
|
1163
|
+
try:
|
|
1164
|
+
from coord.interactive import ( # noqa: PLC0415
|
|
1165
|
+
_holder_is_base_checkout,
|
|
1166
|
+
_remote_base_checkout_free_branch,
|
|
1167
|
+
find_remote_branch_holder,
|
|
1168
|
+
)
|
|
1169
|
+
except ImportError:
|
|
1170
|
+
return # interactive module unavailable — skip gracefully
|
|
1171
|
+
|
|
1172
|
+
holder = find_remote_branch_holder(machine.host, remote_repo_sh, branch)
|
|
1173
|
+
if holder is None or not _holder_is_base_checkout(holder):
|
|
1174
|
+
return # not the base-checkout case
|
|
1175
|
+
|
|
1176
|
+
res.findings.append(
|
|
1177
|
+
f"base checkout {holder!r} on {machine.host} is on branch {branch!r}"
|
|
1178
|
+
f" — this blocks worktree creation for {branch!r}"
|
|
1179
|
+
)
|
|
1180
|
+
if dry_run:
|
|
1181
|
+
res.findings.append(
|
|
1182
|
+
f"(dry-run) would checkout {default_branch!r} in {holder!r}"
|
|
1183
|
+
f" on {machine.host} to free the branch"
|
|
1184
|
+
)
|
|
1185
|
+
return
|
|
1186
|
+
|
|
1187
|
+
freed = _remote_base_checkout_free_branch(
|
|
1188
|
+
machine.host, remote_repo_sh, default_branch,
|
|
1189
|
+
)
|
|
1190
|
+
if freed:
|
|
1191
|
+
res.actions_taken.append(
|
|
1192
|
+
f"freed base checkout {holder!r} on {machine.host}:"
|
|
1193
|
+
f" checked out {default_branch!r} (was on {branch!r})"
|
|
1194
|
+
)
|
|
1195
|
+
else:
|
|
1196
|
+
res.findings.append(
|
|
1197
|
+
f"could not auto-free base checkout on {machine.host} —"
|
|
1198
|
+
f" run manually: ssh {machine.host}"
|
|
1199
|
+
f" 'git -C {remote_repo_sh} checkout {default_branch}'"
|
|
1200
|
+
)
|
|
1201
|
+
res.needs_reset = True
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _do_reset(
|
|
1205
|
+
board, config, assignments, res: DiagnoseResult, *, stage: str,
|
|
1206
|
+
repo_name: str, issue_number: int, dry_run: bool,
|
|
1207
|
+
) -> None:
|
|
1208
|
+
"""Stage-aware, non-destructive reset (KEEP the branch + commits always).
|
|
1209
|
+
|
|
1210
|
+
The shape of "reset" depends on the stage's state, not just on a live
|
|
1211
|
+
session: a completed REVIEW has no session to kill — its data lives in the
|
|
1212
|
+
board rows + #603 store — so resetting it means wiping that data so the
|
|
1213
|
+
stage goes back to grey/unrun and re-reviewable.
|
|
1214
|
+
"""
|
|
1215
|
+
latest = _latest(assignments)
|
|
1216
|
+
if latest is None:
|
|
1217
|
+
res.findings.append(f"no {stage} stage to reset")
|
|
1218
|
+
res.recovered = True
|
|
1219
|
+
return
|
|
1220
|
+
|
|
1221
|
+
if stage == "review":
|
|
1222
|
+
# #1180: `_reset_review_stage`'s `assignment_id` means "the id of the
|
|
1223
|
+
# assignment BEING reviewed" — that's the FK the review rows carry and
|
|
1224
|
+
# the id the test-author/mock-author review_state reset keys on. But
|
|
1225
|
+
# STAGE_ASSIGNMENT_TYPES["review"] matches ('review','test-author',
|
|
1226
|
+
# 'mock-author'), so `latest` is EITHER:
|
|
1227
|
+
# - the reviewed row itself (test-author/mock-author wedged before a
|
|
1228
|
+
# review was ever dispatched — the JIT-slice case), or
|
|
1229
|
+
# - a type='review' row, whose OWN id is meaningless here; the
|
|
1230
|
+
# reviewed assignment is its `review_of_assignment_id` FK
|
|
1231
|
+
# (set at review.py: review_of_assignment_id=completed.assignment_id).
|
|
1232
|
+
# Passing the review row's own id would match no FK, silently resetting
|
|
1233
|
+
# nothing — resolve the reviewed id explicitly.
|
|
1234
|
+
target_id = (
|
|
1235
|
+
latest.review_of_assignment_id
|
|
1236
|
+
if latest.type == "review" and latest.review_of_assignment_id
|
|
1237
|
+
else latest.assignment_id
|
|
1238
|
+
)
|
|
1239
|
+
_reset_review_stage(
|
|
1240
|
+
config, repo_name, issue_number, res,
|
|
1241
|
+
dry_run=dry_run, assignment_id=target_id,
|
|
1242
|
+
)
|
|
1243
|
+
return
|
|
1244
|
+
if stage == "test":
|
|
1245
|
+
_reset_test_stage(repo_name, issue_number, res, dry_run=dry_run)
|
|
1246
|
+
return
|
|
1247
|
+
|
|
1248
|
+
# work / plan / merge — clear a live/phantom session, KEEP the branch.
|
|
1249
|
+
# (Merge reset deliberately does NOT un-merge; it only clears a stuck
|
|
1250
|
+
# session/row, so a clean re-attempt is possible without rewriting history.)
|
|
1251
|
+
if dry_run:
|
|
1252
|
+
res.findings.append("(dry-run) would reset: stop session, finalize, clear row — branch kept")
|
|
1253
|
+
res.needs_reset = True
|
|
1254
|
+
return
|
|
1255
|
+
if _session_state(latest, config) == "live" and _kill_session(latest, config):
|
|
1256
|
+
res.actions_taken.append("stopped the live session (tmux kill-session)")
|
|
1257
|
+
try:
|
|
1258
|
+
res.actions_taken.append(f"finalized session ({_finalize_dead(latest, config)})")
|
|
1259
|
+
except Exception as exc: # noqa: BLE001 — fall back to a direct terminal mark
|
|
1260
|
+
res.findings.append(f"finalize failed ({exc}); marking row terminal directly")
|
|
1261
|
+
_mark_terminal(latest, config)
|
|
1262
|
+
res.actions_taken.append("marked stage row terminal")
|
|
1263
|
+
res.reset_performed = True
|
|
1264
|
+
res.recovered = True
|
|
1265
|
+
res.branch_preserved = True
|
|
1266
|
+
res.actions_taken.append("branch preserved — stage is re-dispatchable")
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
def _reset_review_stage(
|
|
1270
|
+
config, repo_name: str, issue_number: int, res: DiagnoseResult, *,
|
|
1271
|
+
dry_run: bool, assignment_id: str,
|
|
1272
|
+
) -> None:
|
|
1273
|
+
"""Wipe a completed review so the stage returns to grey + re-reviewable:
|
|
1274
|
+
delete the ``type='review'`` rows, reset the work's ``review_state``, and
|
|
1275
|
+
purge the #603 ``source='review'`` context entries (the operator's
|
|
1276
|
+
'completely cleared out' choice). No branch/commits touched.
|
|
1277
|
+
|
|
1278
|
+
#1180: ``assignment_id`` is **the id of the assignment being reviewed** —
|
|
1279
|
+
NOT the id of a ``type='review'`` row. It is threaded through to both the
|
|
1280
|
+
delete and the reset so a milestone tracking issue with multiple
|
|
1281
|
+
``test-author``/``mock-author`` slices only has the *targeted* slice's
|
|
1282
|
+
review data touched — see ``state.delete_assignments_for_issue`` and
|
|
1283
|
+
``state.reset_work_review_state`` docstrings for the aliasing hazard this
|
|
1284
|
+
guards against. ``work``/``plan`` behavior is unchanged (still issue-wide,
|
|
1285
|
+
which is safe for those types).
|
|
1286
|
+
|
|
1287
|
+
Callers must resolve this themselves: the review stage's ``latest`` row can
|
|
1288
|
+
be either the reviewed assignment (test-author/mock-author, no review
|
|
1289
|
+
dispatched yet) or a ``type='review'`` row pointing at it via
|
|
1290
|
+
``review_of_assignment_id`` — the two cases need different resolution. See
|
|
1291
|
+
``_do_reset``.
|
|
1292
|
+
"""
|
|
1293
|
+
from coord import state # noqa: PLC0415
|
|
1294
|
+
|
|
1295
|
+
if dry_run:
|
|
1296
|
+
res.findings.append(
|
|
1297
|
+
"(dry-run) would DELETE the review rows, reset work review_state → "
|
|
1298
|
+
"pending, and purge #603 review notes (box → grey, re-reviewable)"
|
|
1299
|
+
)
|
|
1300
|
+
res.needs_reset = True
|
|
1301
|
+
return
|
|
1302
|
+
deleted = state.delete_assignments_for_issue(
|
|
1303
|
+
repo_name, issue_number, types=("review",),
|
|
1304
|
+
review_of_assignment_id=assignment_id,
|
|
1305
|
+
)
|
|
1306
|
+
res.actions_taken.append(f"deleted {deleted} review row(s) → stage grey")
|
|
1307
|
+
updated = state.reset_work_review_state(
|
|
1308
|
+
repo_name, issue_number, assignment_id=assignment_id
|
|
1309
|
+
)
|
|
1310
|
+
res.actions_taken.append(f"reset review_state→pending on {updated} work row(s) (re-reviewable)")
|
|
1311
|
+
purged = state.clear_issue_context_by_source(repo_name, issue_number, "review")
|
|
1312
|
+
res.actions_taken.append(f"purged {purged} #603 review note(s)")
|
|
1313
|
+
res.reset_performed = True
|
|
1314
|
+
res.recovered = True
|
|
1315
|
+
res.branch_preserved = True
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _reset_test_stage(
|
|
1319
|
+
repo_name: str, issue_number: int, res: DiagnoseResult, *, dry_run: bool
|
|
1320
|
+
) -> None:
|
|
1321
|
+
"""Clear the Test-gate verdict so the issue is re-testable. No code touched."""
|
|
1322
|
+
from coord import state # noqa: PLC0415
|
|
1323
|
+
|
|
1324
|
+
if dry_run:
|
|
1325
|
+
res.findings.append("(dry-run) would clear test_state → re-testable")
|
|
1326
|
+
res.needs_reset = True
|
|
1327
|
+
return
|
|
1328
|
+
updated = state.reset_work_test_state(repo_name, issue_number)
|
|
1329
|
+
res.actions_taken.append(f"cleared Test verdict on {updated} work row(s) (re-testable)")
|
|
1330
|
+
res.reset_performed = True
|
|
1331
|
+
res.recovered = True
|
|
1332
|
+
res.branch_preserved = True
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def _cleanup_issue(
|
|
1336
|
+
board,
|
|
1337
|
+
config,
|
|
1338
|
+
repo_name,
|
|
1339
|
+
issue_number,
|
|
1340
|
+
res: DiagnoseResult,
|
|
1341
|
+
*,
|
|
1342
|
+
dry_run: bool,
|
|
1343
|
+
reset: bool,
|
|
1344
|
+
skip_ids: set | None = None,
|
|
1345
|
+
) -> None:
|
|
1346
|
+
"""Always-on, issue-scoped DB *scan*: any OTHER phantom ``running`` rows for
|
|
1347
|
+
this issue whose session is dead are reported. They are only FINALIZED
|
|
1348
|
+
(a write) when *reset* is set.
|
|
1349
|
+
|
|
1350
|
+
#1658: this sweep looks past the one row the operator explicitly asked to
|
|
1351
|
+
diagnose — at every other row for the issue — so a false "dead" verdict
|
|
1352
|
+
here (or a genuinely-live row this scan wasn't asked about) has no
|
|
1353
|
+
operator-reviewed finding backing it the way the targeted stage's own
|
|
1354
|
+
best-effort recovery does. That over-reach is exactly what turned a
|
|
1355
|
+
plain, no-flags ``coord diagnose --stage test`` into a write against a
|
|
1356
|
+
live headless review worker's row (finalized to ``failed`` mid-review):
|
|
1357
|
+
``reset`` was ``False`` and it wrote anyway. Now a phantom found here is
|
|
1358
|
+
only a *recommendation* — ``needs_reset=True`` plus a finding telling the
|
|
1359
|
+
operator to re-run with ``--reset`` — unless ``--reset`` was already
|
|
1360
|
+
passed, in which case the existing finalize behaviour is unchanged.
|
|
1361
|
+
|
|
1362
|
+
#2087 (fix-review nit): a sibling row on an unconfigured machine is a
|
|
1363
|
+
phantom too, by the same reasoning ``diagnose_stage`` already applies to
|
|
1364
|
+
the row it was explicitly asked about — but ``_session_state`` reports
|
|
1365
|
+
"unknown" (not "dead") for it, since a machine that isn't in
|
|
1366
|
+
``coordinator.yml`` can't be probed at all. Before this, that "unknown"
|
|
1367
|
+
made this sweep silently skip it: a milestone tracking issue with a
|
|
1368
|
+
``work`` row *and* a sibling ``smoke`` row, both on the same
|
|
1369
|
+
unconfigured machine, diagnosed with ``--stage work``, reported the
|
|
1370
|
+
``work`` row correctly but said nothing about the ``smoke`` sibling.
|
|
1371
|
+
Flag it the same way regardless of session probe result — reuses
|
|
1372
|
+
``_finalize_dead``, already safe for an unconfigured machine (no host to
|
|
1373
|
+
probe/ssh into; see ``_do_reset``'s identical use for the targeted row).
|
|
1374
|
+
"""
|
|
1375
|
+
skip = skip_ids or set()
|
|
1376
|
+
for a in (board.active + board.completed):
|
|
1377
|
+
if a.issue_number != issue_number or a.repo_name != repo_name:
|
|
1378
|
+
continue
|
|
1379
|
+
if a.assignment_id in skip:
|
|
1380
|
+
continue
|
|
1381
|
+
if a.status not in ("running", "pending"):
|
|
1382
|
+
continue
|
|
1383
|
+
machine_unconfigured = (
|
|
1384
|
+
bool(a.machine_name) and _resolve_machine(config, a.machine_name) is None
|
|
1385
|
+
)
|
|
1386
|
+
if not machine_unconfigured and _session_state(a, config) != "dead":
|
|
1387
|
+
continue
|
|
1388
|
+
if machine_unconfigured:
|
|
1389
|
+
res.findings.append(
|
|
1390
|
+
f"cleanup: phantom {a.type} row {a.assignment_id} — machine "
|
|
1391
|
+
f"{a.machine_name!r} is not a configured machine (not in "
|
|
1392
|
+
"coordinator.yml)"
|
|
1393
|
+
)
|
|
1394
|
+
else:
|
|
1395
|
+
res.findings.append(f"cleanup: phantom {a.type} row {a.assignment_id} (session dead)")
|
|
1396
|
+
if not reset:
|
|
1397
|
+
res.findings.append(
|
|
1398
|
+
f"cleanup: would finalize phantom {a.type} row {a.assignment_id} "
|
|
1399
|
+
"— re-run with --reset to clear it"
|
|
1400
|
+
)
|
|
1401
|
+
res.needs_reset = True
|
|
1402
|
+
continue
|
|
1403
|
+
if dry_run:
|
|
1404
|
+
res.findings.append(f"(dry-run) would finalize phantom {a.type} row {a.assignment_id}")
|
|
1405
|
+
res.needs_reset = True
|
|
1406
|
+
continue
|
|
1407
|
+
try:
|
|
1408
|
+
_finalize_dead(a, config)
|
|
1409
|
+
res.actions_taken.append(f"cleanup: finalized phantom {a.type} row {a.assignment_id}")
|
|
1410
|
+
except Exception as exc: # noqa: BLE001
|
|
1411
|
+
_mark_terminal(a, config)
|
|
1412
|
+
res.actions_taken.append(f"cleanup: marked phantom row {a.assignment_id} terminal ({exc})")
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def _is_stale(assignment: "Assignment", *, max_age_hours: float = 12.0) -> bool:
|
|
1416
|
+
"""A still-running session whose dispatch is older than *max_age_hours* is
|
|
1417
|
+
treated as stale (abandoned/idle) — recovery can't safely finalize a live
|
|
1418
|
+
session, so these escalate to a reset offer."""
|
|
1419
|
+
if not assignment.dispatched_at:
|
|
1420
|
+
return False
|
|
1421
|
+
return (time.time() - assignment.dispatched_at) > max_age_hours * 3600.0
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
# ── #618: orphaned worktree detection + pruning ──────────────────────────────
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _find_orphaned_worktrees(
|
|
1428
|
+
repo_path: Path,
|
|
1429
|
+
branch: str | None,
|
|
1430
|
+
*,
|
|
1431
|
+
active_assignment_ids: set[str],
|
|
1432
|
+
worktrees_dir: Path | None = None,
|
|
1433
|
+
) -> list[Path]:
|
|
1434
|
+
"""Return worktree paths under *worktrees_dir* that hold *branch* but belong
|
|
1435
|
+
to no active (live-tmux OR running-DB) assignment.
|
|
1436
|
+
|
|
1437
|
+
A worktree is "orphaned" when ALL of:
|
|
1438
|
+
* Its directory is under ``~/.coord/worktrees/`` (coordinator-managed).
|
|
1439
|
+
* Its git checkout has *branch* checked out (or *branch* is ``None``,
|
|
1440
|
+
meaning any branch — used for fleet sweeps).
|
|
1441
|
+
* Its assignment_id (derived from the directory name) is NOT in
|
|
1442
|
+
*active_assignment_ids* — i.e. no live tmux session and no running DB row.
|
|
1443
|
+
|
|
1444
|
+
Dirty worktrees (uncommitted changes) are listed but callers must skip
|
|
1445
|
+
force-remove — they'd lose uncommitted work. Use ``_prune_orphaned_worktrees``
|
|
1446
|
+
to prune them with an uncommitted-work guard.
|
|
1447
|
+
"""
|
|
1448
|
+
if worktrees_dir is None:
|
|
1449
|
+
from coord.state import COORD_DIR # noqa: PLC0415
|
|
1450
|
+
worktrees_dir = COORD_DIR / "worktrees"
|
|
1451
|
+
|
|
1452
|
+
orphans: list[Path] = []
|
|
1453
|
+
try:
|
|
1454
|
+
result = subprocess.run(
|
|
1455
|
+
["git", "worktree", "list", "--porcelain"],
|
|
1456
|
+
cwd=str(repo_path),
|
|
1457
|
+
capture_output=True,
|
|
1458
|
+
text=True,
|
|
1459
|
+
timeout=10.0,
|
|
1460
|
+
)
|
|
1461
|
+
except (subprocess.SubprocessError, OSError):
|
|
1462
|
+
return []
|
|
1463
|
+
if result.returncode != 0:
|
|
1464
|
+
return []
|
|
1465
|
+
|
|
1466
|
+
# Parse the porcelain output into blocks.
|
|
1467
|
+
current: dict[str, str] = {}
|
|
1468
|
+
for raw_line in result.stdout.splitlines():
|
|
1469
|
+
line = raw_line.strip()
|
|
1470
|
+
if not line:
|
|
1471
|
+
if current:
|
|
1472
|
+
_maybe_orphan(current, branch, worktrees_dir, active_assignment_ids, orphans)
|
|
1473
|
+
current = {}
|
|
1474
|
+
elif line.startswith("worktree "):
|
|
1475
|
+
current["worktree"] = line[len("worktree "):]
|
|
1476
|
+
elif line.startswith("branch "):
|
|
1477
|
+
raw_branch = line[len("branch "):]
|
|
1478
|
+
current["branch"] = (
|
|
1479
|
+
raw_branch[len("refs/heads/"):] if raw_branch.startswith("refs/heads/") else raw_branch
|
|
1480
|
+
)
|
|
1481
|
+
if current:
|
|
1482
|
+
_maybe_orphan(current, branch, worktrees_dir, active_assignment_ids, orphans)
|
|
1483
|
+
|
|
1484
|
+
return orphans
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def _maybe_orphan(
|
|
1488
|
+
entry: dict[str, str],
|
|
1489
|
+
branch: str | None,
|
|
1490
|
+
worktrees_dir: Path,
|
|
1491
|
+
active_assignment_ids: set[str],
|
|
1492
|
+
out: list[Path],
|
|
1493
|
+
) -> None:
|
|
1494
|
+
"""Append to *out* if *entry* is an orphaned worktree for *branch*.
|
|
1495
|
+
|
|
1496
|
+
When *branch* is ``None`` any branch matches (fleet sweep).
|
|
1497
|
+
"""
|
|
1498
|
+
wt_str = entry.get("worktree", "")
|
|
1499
|
+
if not wt_str:
|
|
1500
|
+
return
|
|
1501
|
+
if branch is not None and entry.get("branch", "") != branch:
|
|
1502
|
+
return
|
|
1503
|
+
wt_path = Path(wt_str)
|
|
1504
|
+
# Only consider coordinator-managed worktrees (under ~/.coord/worktrees/).
|
|
1505
|
+
try:
|
|
1506
|
+
wt_path.relative_to(worktrees_dir)
|
|
1507
|
+
except ValueError:
|
|
1508
|
+
return
|
|
1509
|
+
# The assignment_id is the directory name component immediately under worktrees_dir.
|
|
1510
|
+
aid = wt_path.relative_to(worktrees_dir).parts[0]
|
|
1511
|
+
if aid in active_assignment_ids:
|
|
1512
|
+
return
|
|
1513
|
+
out.append(wt_path)
|
|
1514
|
+
|
|
1515
|
+
|
|
1516
|
+
def _prune_orphaned_worktrees(
|
|
1517
|
+
repo_path: Path,
|
|
1518
|
+
orphans: list[Path],
|
|
1519
|
+
*,
|
|
1520
|
+
force: bool = False,
|
|
1521
|
+
) -> tuple[list[Path], list[Path]]:
|
|
1522
|
+
"""Remove *orphans* from *repo_path* via ``git worktree remove``.
|
|
1523
|
+
|
|
1524
|
+
Returns ``(removed, skipped)``. Worktrees with uncommitted changes are
|
|
1525
|
+
skipped when *force* is ``False`` (default) so no uncommitted work is lost.
|
|
1526
|
+
After removal, runs ``git worktree prune`` to clean admin entries.
|
|
1527
|
+
"""
|
|
1528
|
+
removed: list[Path] = []
|
|
1529
|
+
skipped: list[Path] = []
|
|
1530
|
+
for wt in orphans:
|
|
1531
|
+
if not wt.exists():
|
|
1532
|
+
removed.append(wt)
|
|
1533
|
+
continue
|
|
1534
|
+
if not force:
|
|
1535
|
+
# Check for uncommitted changes — skip dirty worktrees.
|
|
1536
|
+
try:
|
|
1537
|
+
dirty = subprocess.run(
|
|
1538
|
+
["git", "status", "--porcelain"],
|
|
1539
|
+
cwd=str(wt),
|
|
1540
|
+
capture_output=True,
|
|
1541
|
+
text=True,
|
|
1542
|
+
timeout=10.0,
|
|
1543
|
+
)
|
|
1544
|
+
if dirty.returncode == 0 and dirty.stdout.strip():
|
|
1545
|
+
skipped.append(wt)
|
|
1546
|
+
continue
|
|
1547
|
+
except (subprocess.SubprocessError, OSError):
|
|
1548
|
+
skipped.append(wt)
|
|
1549
|
+
continue
|
|
1550
|
+
try:
|
|
1551
|
+
r = subprocess.run(
|
|
1552
|
+
["git", "worktree", "remove", str(wt), "--force"],
|
|
1553
|
+
cwd=str(repo_path),
|
|
1554
|
+
capture_output=True,
|
|
1555
|
+
timeout=15.0,
|
|
1556
|
+
)
|
|
1557
|
+
if r.returncode == 0:
|
|
1558
|
+
removed.append(wt)
|
|
1559
|
+
else:
|
|
1560
|
+
skipped.append(wt)
|
|
1561
|
+
except (subprocess.SubprocessError, OSError):
|
|
1562
|
+
skipped.append(wt)
|
|
1563
|
+
# Prune stale git admin entries regardless of what was removed.
|
|
1564
|
+
try:
|
|
1565
|
+
subprocess.run(
|
|
1566
|
+
["git", "worktree", "prune"],
|
|
1567
|
+
cwd=str(repo_path),
|
|
1568
|
+
capture_output=True,
|
|
1569
|
+
timeout=10.0,
|
|
1570
|
+
)
|
|
1571
|
+
except (subprocess.SubprocessError, OSError):
|
|
1572
|
+
pass
|
|
1573
|
+
return removed, skipped
|
|
1574
|
+
|
|
1575
|
+
|
|
1576
|
+
def _active_assignment_ids_for_repo(
|
|
1577
|
+
board: "Board", repo_name: str
|
|
1578
|
+
) -> set[str]:
|
|
1579
|
+
"""Return assignment IDs for *repo_name* that are still running/pending."""
|
|
1580
|
+
return {
|
|
1581
|
+
a.assignment_id
|
|
1582
|
+
for a in board.active
|
|
1583
|
+
if a.repo_name == repo_name and a.assignment_id
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
def find_and_prune_orphaned_worktrees(
|
|
1588
|
+
board: "Board",
|
|
1589
|
+
config: "Config",
|
|
1590
|
+
repo_name: str,
|
|
1591
|
+
branch: str,
|
|
1592
|
+
) -> tuple[list[Path], list[Path]]:
|
|
1593
|
+
"""Detect and prune orphaned coordinator worktrees holding *branch*.
|
|
1594
|
+
|
|
1595
|
+
Public entry point used by :func:`diagnose_stage` (Gap 2 of #618) and
|
|
1596
|
+
by the ``coord diagnose --orphan-worktrees`` fleet sweep.
|
|
1597
|
+
|
|
1598
|
+
Returns ``(removed, skipped)`` path lists. The *skipped* list contains
|
|
1599
|
+
worktrees that have uncommitted changes — the operator must inspect and
|
|
1600
|
+
clean them manually.
|
|
1601
|
+
"""
|
|
1602
|
+
repo_cfg = next((r for r in config.repos if r.name == repo_name), None)
|
|
1603
|
+
if repo_cfg is None:
|
|
1604
|
+
return [], []
|
|
1605
|
+
|
|
1606
|
+
# Find the local checkout path for this repo. We need it to run git commands.
|
|
1607
|
+
# On a thin client the local checkout may not exist; fall back gracefully.
|
|
1608
|
+
repo_path: Path | None = None
|
|
1609
|
+
for machine in config.machines:
|
|
1610
|
+
rp = machine.repo_path(repo_name)
|
|
1611
|
+
if rp:
|
|
1612
|
+
candidate = Path(rp).expanduser()
|
|
1613
|
+
if candidate.exists():
|
|
1614
|
+
repo_path = candidate
|
|
1615
|
+
break
|
|
1616
|
+
if repo_path is None:
|
|
1617
|
+
return [], []
|
|
1618
|
+
|
|
1619
|
+
active_ids = _active_assignment_ids_for_repo(board, repo_name)
|
|
1620
|
+
orphans = _find_orphaned_worktrees(repo_path, branch, active_assignment_ids=active_ids)
|
|
1621
|
+
if not orphans:
|
|
1622
|
+
return [], []
|
|
1623
|
+
return _prune_orphaned_worktrees(repo_path, orphans)
|