code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/reconcile.py
ADDED
|
@@ -0,0 +1,2233 @@
|
|
|
1
|
+
"""Reconcile the coordinator's board with live agent server state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
import uuid
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from coord.config import Config
|
|
14
|
+
from coord.dispatch import AGENT_PORT
|
|
15
|
+
from coord.models import WORK_LIKE_TYPES, Assignment, Board, Machine
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from coord.merge_queue import QueuedMerge
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _query_agent(host: str, port: int = AGENT_PORT, timeout: float = 5.0) -> dict | None:
|
|
22
|
+
try:
|
|
23
|
+
resp = httpx.get(f"http://{host}:{port}/status", timeout=timeout)
|
|
24
|
+
resp.raise_for_status()
|
|
25
|
+
return resp.json()
|
|
26
|
+
except (httpx.HTTPError, httpx.TimeoutException):
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Terminal statuses an agent reports in its /status `completed` history,
|
|
31
|
+
# mapped to the board terminal status we persist. (#625)
|
|
32
|
+
_AGENT_TERMINAL_STATUS = {
|
|
33
|
+
"done": "done",
|
|
34
|
+
"advisory": "advisory",
|
|
35
|
+
"failed": "failed",
|
|
36
|
+
"cancelled": "failed",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def effective_agent_status(entry: dict) -> str:
|
|
41
|
+
"""The agent-reported status for *entry*, with #1534's ``done`` refusal.
|
|
42
|
+
|
|
43
|
+
An agent ``completed`` entry that reports ``status="done"`` while ALSO
|
|
44
|
+
carrying a ``usage_limit_reason`` is self-contradictory: the worker was
|
|
45
|
+
killed mid-task by the account's Claude session/weekly usage limit (its
|
|
46
|
+
transcript ends on "You've hit your session limit · resets <time>"), so
|
|
47
|
+
whatever it did or didn't push, it did not *finish*. Recording that as
|
|
48
|
+
``done`` is the silent corruption #1534 was filed for — it burns money and
|
|
49
|
+
reports success, and every downstream gate (review dispatch, the
|
|
50
|
+
acceptance gate, the Pipeline view) then behaves as if the slice exists.
|
|
51
|
+
|
|
52
|
+
The agent-side reap (``AgentServer._reap``) already refuses this at the
|
|
53
|
+
source as of #1534, but that fix only reaches the fleet after a PyPI
|
|
54
|
+
release and ``coord agent update``. This is the coordinator-side backstop
|
|
55
|
+
for the interim — and permanently, for agents pinned to an older build.
|
|
56
|
+
|
|
57
|
+
Downgraded to ``failed`` rather than ``advisory`` for the same reason
|
|
58
|
+
``_record_usage_limit_reason`` normalises to ``failed``: a usage-limit kill
|
|
59
|
+
is the one terminal state known safe to re-dispatch unchanged once the
|
|
60
|
+
window resets, whereas ``advisory`` means "a human needs to look".
|
|
61
|
+
|
|
62
|
+
Any truthy ``usage_limit_reason`` counts — not just one matching
|
|
63
|
+
:func:`coord.worker_events.is_usage_limit_reason`'s prefix — because
|
|
64
|
+
refusing a ``done`` is the fail-safe direction and the field is only ever
|
|
65
|
+
written by the kill detector.
|
|
66
|
+
"""
|
|
67
|
+
raw = (entry.get("status") or "").lower()
|
|
68
|
+
if raw == "done" and entry.get("usage_limit_reason"):
|
|
69
|
+
return "failed"
|
|
70
|
+
return raw
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def reconcile_completed_assignments(
|
|
74
|
+
config: Config,
|
|
75
|
+
*,
|
|
76
|
+
board: Board | None = None,
|
|
77
|
+
agent_status_fn=_query_agent,
|
|
78
|
+
update_state_fn=None,
|
|
79
|
+
capture_plan: bool = True,
|
|
80
|
+
) -> list[dict]:
|
|
81
|
+
"""Dispatch-free passive completion reconcile (#625).
|
|
82
|
+
|
|
83
|
+
Poll the agent of every RUNNING board assignment; for any the agent
|
|
84
|
+
reports terminal in its ``/status`` ``completed`` history, write the
|
|
85
|
+
terminal status + ``finished_at`` to the board via the issue_store seam
|
|
86
|
+
and (best-effort) capture a plan's structured output. This reflects a
|
|
87
|
+
headless worker's already-finished state so the board — and the TUI box
|
|
88
|
+
colour — stops lying when the auto-loop (the only other thing that polled
|
|
89
|
+
agents) is turned off.
|
|
90
|
+
|
|
91
|
+
Deliberately minimal — it is the WHOLE point of #625 that reflecting a
|
|
92
|
+
termination is *passive* state, decoupled from auto-dispatch so it can
|
|
93
|
+
never re-introduce the dispatch flood:
|
|
94
|
+
|
|
95
|
+
* NEVER dispatches work/review.
|
|
96
|
+
* NEVER posts a GitHub comment.
|
|
97
|
+
* Only acts on ``status == "running"`` rows, so it is idempotent — once a
|
|
98
|
+
row is flipped terminal a later tick skips it.
|
|
99
|
+
|
|
100
|
+
**#1616 — what runs the side effects this function refuses to (CONTRACT
|
|
101
|
+
CHANGE, read this before adding anything here).** This function's scope is
|
|
102
|
+
unchanged and must stay unchanged; what changed is *who runs the rest*.
|
|
103
|
+
Until #1616 the parenthetical here read "the single completion/plan comment
|
|
104
|
+
is left to an explicit ``coord notify``" — and on a fleet where
|
|
105
|
+
``coord-notify.timer`` is deliberately disabled, "an explicit ``coord
|
|
106
|
+
notify``" meant *a live* ``coord drive``'s **stall nudge**, and nothing
|
|
107
|
+
else. So this function advancing a row to ``done`` was the LAST thing that
|
|
108
|
+
happened to it for as long as 47 minutes (#1122), or forever when no drive
|
|
109
|
+
was running at all (vimcode#611/#613). ``status`` said done; ``finished_at``
|
|
110
|
+
was NULL, no comment was posted, no review was dispatched, and every surface
|
|
111
|
+
rendered the stage as complete.
|
|
112
|
+
|
|
113
|
+
The fix was **not** to widen this function. The daemon ``_tick_loop`` now
|
|
114
|
+
calls :func:`coord.serve_app._notify_drain_tick` →
|
|
115
|
+
:func:`coord.notify.run_drain` as a SIBLING step immediately after this one,
|
|
116
|
+
which posts the comment, stamps ``finished_at``, backfills the #1076/#1152
|
|
117
|
+
test gate, and dispatches Test + Review under ``~/.coord/notify.lock``.
|
|
118
|
+
That is where pipeline-advancing behaviour belongs; this stays passive.
|
|
119
|
+
|
|
120
|
+
This is the failure shape ``docs/OPERATING_GOTCHAS.md`` §7 already names —
|
|
121
|
+
"``reconcile()`` accretes behaviour the automatic drivers never invoke".
|
|
122
|
+
If you are about to add a side effect here because "nothing else runs it",
|
|
123
|
+
that is the bug, not the fix: add it to ``run_drain`` instead, and decide
|
|
124
|
+
deliberately whether it belongs on the daemon's clock at all (work dispatch
|
|
125
|
+
and fix-round dispatch explicitly do NOT — see that docstring).
|
|
126
|
+
|
|
127
|
+
One consequence worth knowing when reading a board: a ``type="review"``
|
|
128
|
+
``done`` is downgraded to ``"finalizing"`` below and only leaves that state
|
|
129
|
+
when the drain captures the verdict. Pre-#1616 that window was unbounded
|
|
130
|
+
(#1610); it is now bounded by ``COORD_NOTIFY_DRAIN_INTERVAL`` (default 60s).
|
|
131
|
+
|
|
132
|
+
Interactive sessions are tmux launches, not agent subprocesses, so they
|
|
133
|
+
never appear in the agent's ``completed`` list — a live attended session
|
|
134
|
+
can't be reaped by this path.
|
|
135
|
+
|
|
136
|
+
Returns one dict per reconciled assignment (empty when nothing changed).
|
|
137
|
+
"""
|
|
138
|
+
if update_state_fn is None:
|
|
139
|
+
from coord.issue_store import _update_local_state # noqa: PLC0415
|
|
140
|
+
|
|
141
|
+
update_state_fn = _update_local_state
|
|
142
|
+
|
|
143
|
+
if board is None:
|
|
144
|
+
from coord.state import build_board # noqa: PLC0415
|
|
145
|
+
|
|
146
|
+
board = build_board()
|
|
147
|
+
|
|
148
|
+
running = [a for a in board.active if a.status == "running"]
|
|
149
|
+
if not running:
|
|
150
|
+
return []
|
|
151
|
+
|
|
152
|
+
hosts = {m.name: m.host for m in config.machines}
|
|
153
|
+
status_by_host: dict[str, dict | None] = {} # poll each agent at most once
|
|
154
|
+
reconciled: list[dict] = []
|
|
155
|
+
|
|
156
|
+
for a in running:
|
|
157
|
+
aid = a.assignment_id
|
|
158
|
+
if not aid:
|
|
159
|
+
continue
|
|
160
|
+
host = hosts.get(a.machine_name)
|
|
161
|
+
if not host:
|
|
162
|
+
continue
|
|
163
|
+
if host not in status_by_host:
|
|
164
|
+
status_by_host[host] = agent_status_fn(host)
|
|
165
|
+
status = status_by_host[host]
|
|
166
|
+
if not status:
|
|
167
|
+
continue # agent unreachable → leave the row, retry next tick
|
|
168
|
+
entry = next(
|
|
169
|
+
(e for e in status.get("completed", []) if e.get("id") == aid),
|
|
170
|
+
None,
|
|
171
|
+
)
|
|
172
|
+
if entry is None:
|
|
173
|
+
continue # still active on the agent (or rolled off history) → leave it
|
|
174
|
+
# #1534: `effective_agent_status` refuses an agent-reported `done`
|
|
175
|
+
# that also carries a usage-limit-kill reason — the daemon's passive
|
|
176
|
+
# tick is the FIRST place most completions are observed, so without
|
|
177
|
+
# this the corrupt `done` is persisted here before any other path
|
|
178
|
+
# gets a chance to look at it.
|
|
179
|
+
terminal = _AGENT_TERMINAL_STATUS.get(effective_agent_status(entry))
|
|
180
|
+
if terminal is None:
|
|
181
|
+
continue
|
|
182
|
+
# #1566: a review agent reporting `done` has only finished the LLM
|
|
183
|
+
# session — the verdict itself is parsed + persisted by `coord
|
|
184
|
+
# notify` (`_try_parse_and_post_review`), a separate, slower step
|
|
185
|
+
# that can run minutes after this tick observes the completion (this
|
|
186
|
+
# passive tick runs on a short ~30s cadence; `coord notify` runs on
|
|
187
|
+
# whatever cadence its caller configures). Writing `status="done"`
|
|
188
|
+
# straight away leaves a window where the board shows a finished
|
|
189
|
+
# review with `review_verdict IS NULL` — indistinguishable from the
|
|
190
|
+
# verdict having been dropped (the #1346/#1348/#1563 failure mode).
|
|
191
|
+
# `finalizing` closes that window: it reads as "still wrapping up"
|
|
192
|
+
# (not in `drive_state.TERMINAL_STATUSES`, so `coord drive` correctly
|
|
193
|
+
# waits rather than declaring a dead end) until `coord notify`'s own
|
|
194
|
+
# `mark_notified` advances it to the real `done` alongside the
|
|
195
|
+
# verdict. Other terminal outcomes (`failed`, `advisory`) never go
|
|
196
|
+
# through that verdict-capture step, so they are unaffected.
|
|
197
|
+
if terminal == "done" and a.type == "review":
|
|
198
|
+
terminal = "finalizing"
|
|
199
|
+
|
|
200
|
+
# #1083: prefer the board's already-known branch, but fall back to
|
|
201
|
+
# the agent's live ``completed`` entry (populated by AgentServer._reap
|
|
202
|
+
# from the worktree's checked-out HEAD — see agent.py) when the board
|
|
203
|
+
# doesn't have one yet. This is almost always the FIRST place a
|
|
204
|
+
# freshly-completed assignment is observed (the daemon runs this tick
|
|
205
|
+
# on a short interval, well ahead of any human-triggered `coord
|
|
206
|
+
# notify`), so passing the board's stale (usually still-None) branch
|
|
207
|
+
# here — as this used to do unconditionally — let status flip to
|
|
208
|
+
# "done" with branch left NULL. For `type="work"` that NULL branch is
|
|
209
|
+
# later patched by the #611 remote-branch-listing backfill sweep in
|
|
210
|
+
# `reconcile()`, but that sweep is scoped to `type="work"` only, so
|
|
211
|
+
# every other write-capable type (mock-author, test-author, ...) had
|
|
212
|
+
# no path back to a correct branch once this tick got there first.
|
|
213
|
+
# #1461: stamp a usage-limit-kill diagnostic onto the board row when
|
|
214
|
+
# the agent's own reap flagged one (AgentServer._reap, agent.py) —
|
|
215
|
+
# regardless of whether the agent landed on FAILED or ADVISORY, both
|
|
216
|
+
# observed for a real kill. This is the primary production path
|
|
217
|
+
# (the daemon's passive tick) for getting the reason out of the
|
|
218
|
+
# ephemeral agent-side JSON and into the persisted, drive.py-visible
|
|
219
|
+
# `failure_reason` column.
|
|
220
|
+
# #1584: `api_error_reason` (a terminal `is_error: true` result event
|
|
221
|
+
# — e.g. "529 Overloaded") is the SAME `failure_reason` column,
|
|
222
|
+
# stamped by `AgentServer._reap` exactly like `usage_limit_reason`
|
|
223
|
+
# (see `coord.agent.AgentAssignment`, both surfaced on the same
|
|
224
|
+
# `/status` `completed` entry via `to_status_dict`'s `asdict`). The
|
|
225
|
+
# two are mutually exclusive by construction — a usage-limit kill is
|
|
226
|
+
# detected from a TRUNCATED log with no terminal `result` event,
|
|
227
|
+
# while an API-error is read OFF that terminal `result` event — so
|
|
228
|
+
# this `or` never picks the wrong one.
|
|
229
|
+
# #1797: `push_failure_reason` is the SAME column too — stamped by
|
|
230
|
+
# `AgentServer._reap` when the reap-time safety-net push hits an
|
|
231
|
+
# auth-shaped rejection (see `_is_auth_push_failure`). It never
|
|
232
|
+
# coexists with the other two either: it is only ever set on a
|
|
233
|
+
# clean `exit_code == 0` reap, which both `usage_limit_reason` and
|
|
234
|
+
# `api_error_reason` preempt before the push-failure branch even
|
|
235
|
+
# runs (see the `elif` chain in `AgentServer._reap`). Without this,
|
|
236
|
+
# a `type="work"` auth-push failure lands FAILED with no
|
|
237
|
+
# `failure_reason` at all — invisible to `coord status`, the TUI,
|
|
238
|
+
# and drive.py, which is the exact visibility gap #1797 exists to
|
|
239
|
+
# close.
|
|
240
|
+
_failure_reason = (
|
|
241
|
+
entry.get("usage_limit_reason")
|
|
242
|
+
or entry.get("api_error_reason")
|
|
243
|
+
or entry.get("push_failure_reason")
|
|
244
|
+
)
|
|
245
|
+
update_state_fn(
|
|
246
|
+
assignment_id=aid,
|
|
247
|
+
terminal_status=terminal,
|
|
248
|
+
branch=a.branch or entry.get("branch"),
|
|
249
|
+
review_state=None,
|
|
250
|
+
failure_reason=_failure_reason,
|
|
251
|
+
# #1605: the reap already computed the exit code (AgentServer._reap,
|
|
252
|
+
# agent.py) and it rides on this same `/status` `completed` entry —
|
|
253
|
+
# nothing downstream of THIS write path ever persisted it, so a
|
|
254
|
+
# failed Test-stage row was undiagnosable from the board (both
|
|
255
|
+
# `failure_reason` AND `exit_code` null) even when the reap knew
|
|
256
|
+
# exactly why it died.
|
|
257
|
+
exit_code=entry.get("exit_code"),
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
# #1605: a `type="smoke"` (Test-stage) assignment reaching a terminal
|
|
261
|
+
# FAILED status must resolve the PARENT work row's `test_state` —
|
|
262
|
+
# never leave it `running` forever. `running` is a documented
|
|
263
|
+
# transient non-verdict marker (#1395) that every gate treats as "no
|
|
264
|
+
# verdict yet", so a stranded child leaves the issue permanently
|
|
265
|
+
# unresolvable and invisible to every instrument except a worker
|
|
266
|
+
# transcript. See `propagate_smoke_terminal_failure` for the
|
|
267
|
+
# environmental-vs-work classification (#1590) that decides whether
|
|
268
|
+
# this clears the verdict for a fresh auto-dispatch or records a real
|
|
269
|
+
# test failure.
|
|
270
|
+
if a.type == "smoke" and terminal == "failed":
|
|
271
|
+
propagate_smoke_terminal_failure(
|
|
272
|
+
parent_assignment_id=a.review_of_assignment_id,
|
|
273
|
+
failure_reason=_failure_reason,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
# #666 Gap A: best-effort cost/token capture from the agent completed
|
|
277
|
+
# entry. Must never raise — a tick crash breaks the daemon.
|
|
278
|
+
_capture_cost_from_entry_best_effort(aid, entry)
|
|
279
|
+
|
|
280
|
+
plan_captured = (
|
|
281
|
+
_capture_plan_best_effort(host, aid)
|
|
282
|
+
if capture_plan and a.type == "plan"
|
|
283
|
+
else False
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# #667: capture token counts from the /status entry (the agent now
|
|
287
|
+
# includes them there after parsing its own log). Best-effort — any
|
|
288
|
+
# failure is swallowed so it can't break the reconcile.
|
|
289
|
+
_capture_tokens_best_effort(aid, entry)
|
|
290
|
+
|
|
291
|
+
reconciled.append(
|
|
292
|
+
{
|
|
293
|
+
"assignment_id": aid,
|
|
294
|
+
"issue_number": a.issue_number,
|
|
295
|
+
"repo": a.repo_name,
|
|
296
|
+
"type": a.type,
|
|
297
|
+
"to_status": terminal,
|
|
298
|
+
"plan_captured": plan_captured,
|
|
299
|
+
}
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
return reconciled
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def propagate_smoke_terminal_failure(
|
|
306
|
+
*, parent_assignment_id: str | None, failure_reason: str | None,
|
|
307
|
+
) -> None:
|
|
308
|
+
"""#1605: resolve a work row's ``test_state`` when its Test-stage
|
|
309
|
+
(``type="smoke"``) child dies without ever reporting pass/fail.
|
|
310
|
+
|
|
311
|
+
Before this, a smoke assignment landing on ``status="failed"`` (a dead
|
|
312
|
+
agent, a killed process group, a terminal API error — anything short of
|
|
313
|
+
the worker itself printing ``SMOKE: pass``/``SMOKE: fail``) left the
|
|
314
|
+
parent's ``test_state`` at whatever it was — almost always ``"running"``,
|
|
315
|
+
the marker `dispatch_smoke` stamps the instant it dispatches (#1426).
|
|
316
|
+
Every downstream gate treats ``"running"`` as "no verdict yet" (#1395),
|
|
317
|
+
so the work sits in a state nothing will ever resolve: `coord drive`
|
|
318
|
+
polls it forever, the merge gate never sees a verdict, and `coord
|
|
319
|
+
diagnose --stage test` had nothing to say because it never looked past
|
|
320
|
+
the (terminal, `status="done"`) work row itself.
|
|
321
|
+
|
|
322
|
+
Classified through :func:`coord.failure_class.classify_failure` — the
|
|
323
|
+
same #1590 environmental-vs-work split already used for the work/review
|
|
324
|
+
stages, applied here for the first time to the Test stage:
|
|
325
|
+
|
|
326
|
+
* **environmental** (usage limit, an API 5xx, a network drop) — the
|
|
327
|
+
provider's fault, not the work's. Clears ``test_state`` back to
|
|
328
|
+
``NULL`` (not ``"failed"``) so the daemon's normal
|
|
329
|
+
:func:`coord.smoke.dispatch_pending_smoke` auto-queue picks the work
|
|
330
|
+
row back up on its next tick and re-dispatches a fresh Test stage —
|
|
331
|
+
never spending the bounded ``coord fix`` retry budget on a code defect
|
|
332
|
+
that never existed.
|
|
333
|
+
* **work** (an unclassifiable crash, a real defect) — records
|
|
334
|
+
``test_state="failed"`` exactly like a normal non-zero-exit smoke
|
|
335
|
+
completion already does (`coord/notify.py`'s completion handler), so
|
|
336
|
+
the existing bounded `coord fix` loop picks it up from there.
|
|
337
|
+
|
|
338
|
+
A no-op when *parent_assignment_id* is falsy (a smoke row somehow
|
|
339
|
+
missing its ``review_of_assignment_id`` — should not happen in practice,
|
|
340
|
+
but this must never raise on it).
|
|
341
|
+
"""
|
|
342
|
+
if not parent_assignment_id:
|
|
343
|
+
return
|
|
344
|
+
from coord.failure_class import classify_failure # noqa: PLC0415
|
|
345
|
+
from coord.state import record_test_verdict # noqa: PLC0415
|
|
346
|
+
|
|
347
|
+
classification = classify_failure(failure_reason=failure_reason)
|
|
348
|
+
if classification.is_environmental:
|
|
349
|
+
record_test_verdict(
|
|
350
|
+
assignment_id=parent_assignment_id,
|
|
351
|
+
test_state=None,
|
|
352
|
+
test_reason=(
|
|
353
|
+
"Test stage worker died environmentally "
|
|
354
|
+
f"({classification.reason}) — cleared for automatic "
|
|
355
|
+
"re-dispatch, not recorded as a work failure (#1605)"
|
|
356
|
+
),
|
|
357
|
+
)
|
|
358
|
+
else:
|
|
359
|
+
record_test_verdict(
|
|
360
|
+
assignment_id=parent_assignment_id,
|
|
361
|
+
test_state="failed",
|
|
362
|
+
test_reason=(
|
|
363
|
+
failure_reason
|
|
364
|
+
or "Test stage worker failed with no reason recorded (#1605)"
|
|
365
|
+
),
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _capture_plan_best_effort(host: str, assignment_id: str) -> bool:
|
|
370
|
+
"""Fetch + persist a plan's structured output from the agent log so the
|
|
371
|
+
TUI's plan detail panel isn't empty after a passive reconcile. Best
|
|
372
|
+
effort: any failure is swallowed — the terminal-status write already
|
|
373
|
+
landed and is what fixes the stuck box."""
|
|
374
|
+
try:
|
|
375
|
+
from coord.plan_parser import parse_plan_from_agent # noqa: PLC0415
|
|
376
|
+
from coord.state import save_plan # noqa: PLC0415
|
|
377
|
+
|
|
378
|
+
plan = parse_plan_from_agent(host, assignment_id)
|
|
379
|
+
if plan is None or plan.is_empty():
|
|
380
|
+
return False
|
|
381
|
+
save_plan(assignment_id, plan.to_dict())
|
|
382
|
+
return True
|
|
383
|
+
except Exception: # noqa: BLE001 — never let plan capture break the reconcile
|
|
384
|
+
return False
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _capture_cost_from_entry_best_effort(assignment_id: str, entry: dict) -> None:
|
|
388
|
+
"""#666 Gap A: capture cost from an agent ``completed`` entry when flipping
|
|
389
|
+
a row terminal.
|
|
390
|
+
|
|
391
|
+
Best-effort and silent — any exception is swallowed so a cost-capture
|
|
392
|
+
failure never crashes the daemon's reconcile tick.
|
|
393
|
+
|
|
394
|
+
Cost source: ``total_cost_usd`` (full-log parse, available when the agent
|
|
395
|
+
serves terminal entries) with ``cost_so_far`` as a fallback. Either is
|
|
396
|
+
used only when present and > 0 so an un-measured session isn't written as 0.
|
|
397
|
+
|
|
398
|
+
Token counts are captured separately by ``_capture_tokens_best_effort``
|
|
399
|
+
(#667 Gap B), which is called at the same call site.
|
|
400
|
+
"""
|
|
401
|
+
try:
|
|
402
|
+
from coord.state import update_assignment_cost # noqa: PLC0415
|
|
403
|
+
|
|
404
|
+
raw_cost = entry.get("total_cost_usd") or entry.get("cost_so_far")
|
|
405
|
+
if raw_cost is not None:
|
|
406
|
+
try:
|
|
407
|
+
cost = float(raw_cost)
|
|
408
|
+
except (TypeError, ValueError):
|
|
409
|
+
cost = None
|
|
410
|
+
else:
|
|
411
|
+
if cost > 0:
|
|
412
|
+
update_assignment_cost(assignment_id, cost)
|
|
413
|
+
except Exception: # noqa: BLE001 — never let cost capture break the reconcile
|
|
414
|
+
pass
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _capture_tokens_best_effort(assignment_id: str, entry: dict) -> None:
|
|
418
|
+
"""#667: persist token counts from a /status completed entry.
|
|
419
|
+
|
|
420
|
+
The agent now parses its own log and includes
|
|
421
|
+
``input_tokens`` / ``output_tokens`` / ``cache_creation_tokens`` /
|
|
422
|
+
``cache_read_tokens`` in the completed entry. We write them to the DB
|
|
423
|
+
here so a passive reconcile also captures tokens (not just cost).
|
|
424
|
+
Best-effort — any failure is swallowed.
|
|
425
|
+
"""
|
|
426
|
+
try:
|
|
427
|
+
input_tokens = int(entry.get("input_tokens") or 0)
|
|
428
|
+
output_tokens = int(entry.get("output_tokens") or 0)
|
|
429
|
+
cache_creation_tokens = int(entry.get("cache_creation_tokens") or 0)
|
|
430
|
+
cache_read_tokens = int(entry.get("cache_read_tokens") or 0)
|
|
431
|
+
if input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens == 0:
|
|
432
|
+
return
|
|
433
|
+
from coord.state import update_assignment_tokens # noqa: PLC0415
|
|
434
|
+
|
|
435
|
+
update_assignment_tokens(
|
|
436
|
+
assignment_id,
|
|
437
|
+
input_tokens=input_tokens,
|
|
438
|
+
output_tokens=output_tokens,
|
|
439
|
+
cache_creation_tokens=cache_creation_tokens,
|
|
440
|
+
cache_read_tokens=cache_read_tokens,
|
|
441
|
+
)
|
|
442
|
+
except Exception: # noqa: BLE001 — never let token capture break the reconcile
|
|
443
|
+
pass
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _build_fix_round_retry_briefing(
|
|
447
|
+
failed: Assignment, board: Board, repo_cfg, max_review_iterations: int,
|
|
448
|
+
) -> str | None:
|
|
449
|
+
"""#1411: rebuild the FIX briefing (reviewer findings included) for a
|
|
450
|
+
retried fix-round assignment.
|
|
451
|
+
|
|
452
|
+
``failed`` is itself a fix worker (``review_iteration > 0``) dispatched
|
|
453
|
+
by the auto-loop — its branch already carries the code the reviewer
|
|
454
|
+
rejected. The generic retry briefing built by :func:`_build_retry_briefing`
|
|
455
|
+
has no notion of *why*, so a plain continuation retry reliably repeats
|
|
456
|
+
the same request-changes verdict (a wasted work+review round) and, worse,
|
|
457
|
+
used to reset ``review_iteration`` to 0 — silently disabling the
|
|
458
|
+
``max_review_iterations`` flood guard.
|
|
459
|
+
|
|
460
|
+
Reuses ``auto_loop._build_fix_briefing`` — the exact function the
|
|
461
|
+
original fix dispatch used — fed with the reviewer's findings recovered
|
|
462
|
+
via ``auto_loop._load_review_findings`` (DB cache → local log → agent
|
|
463
|
+
HTTP → GitHub message bus), the same resolution chain the auto-loop
|
|
464
|
+
itself relies on. This is deliberately a reuse, not a new capability.
|
|
465
|
+
|
|
466
|
+
Returns ``None`` when the review chain can't be reconstructed (the
|
|
467
|
+
reviewed work assignment or its review is missing from the board, or the
|
|
468
|
+
findings can't be recovered from any source) — the caller falls back to
|
|
469
|
+
the generic continuation briefing rather than blocking the retry.
|
|
470
|
+
"""
|
|
471
|
+
work = board.find_by_id(failed.review_of_assignment_id)
|
|
472
|
+
if work is None:
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
review = next(
|
|
476
|
+
(
|
|
477
|
+
a for a in (*board.active, *board.completed)
|
|
478
|
+
if a.type == "review"
|
|
479
|
+
and a.review_of_assignment_id == failed.review_of_assignment_id
|
|
480
|
+
),
|
|
481
|
+
None,
|
|
482
|
+
)
|
|
483
|
+
if review is None:
|
|
484
|
+
return None
|
|
485
|
+
|
|
486
|
+
from coord.auto_loop import _build_fix_briefing, _load_review_findings # noqa: PLC0415
|
|
487
|
+
|
|
488
|
+
findings = _load_review_findings(
|
|
489
|
+
review, None, None,
|
|
490
|
+
repo_github=repo_cfg.github if repo_cfg is not None else None,
|
|
491
|
+
)
|
|
492
|
+
if findings is None:
|
|
493
|
+
return None
|
|
494
|
+
|
|
495
|
+
return _build_fix_briefing(
|
|
496
|
+
work, findings, failed.review_iteration, max_review_iterations,
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _build_retry_briefing(
|
|
501
|
+
failed: Assignment, repo_cfg, *, default_branch: str | None = None,
|
|
502
|
+
board: Board | None = None, max_review_iterations: int = 3,
|
|
503
|
+
) -> str:
|
|
504
|
+
"""#1101: reconstruct a real briefing for a retried assignment.
|
|
505
|
+
|
|
506
|
+
``failed.briefing`` is frequently empty or unhelpful by the time a
|
|
507
|
+
failed assignment is retried — a `work` assignment's fully-assembled
|
|
508
|
+
briefing (issue body + board context) is built at initial-dispatch
|
|
509
|
+
time and not always persisted back onto the stored ``Assignment``.
|
|
510
|
+
Replaying it verbatim can hand the retried worker nothing at all,
|
|
511
|
+
which reproduced as the worker exiting in one turn with 0 commits
|
|
512
|
+
(silently reclassified as "advisory" instead of a broken dispatch).
|
|
513
|
+
|
|
514
|
+
This rebuilds something the worker can act on:
|
|
515
|
+
- the original briefing text when present, else a fresh fetch of the
|
|
516
|
+
issue body from GitHub (mirrors ``coord assign``'s own
|
|
517
|
+
auto-generate-from-issue-body fallback) so the briefing is never
|
|
518
|
+
blank;
|
|
519
|
+
- continuation instructions when the failed assignment already has a
|
|
520
|
+
branch (mirrors the equivalent ``coord fix`` briefing in
|
|
521
|
+
``plan_followup.py``): don't start over, inspect what's already
|
|
522
|
+
committed;
|
|
523
|
+
- the recorded failure reason, so the worker knows why the previous
|
|
524
|
+
attempt stopped instead of re-discovering it from scratch.
|
|
525
|
+
|
|
526
|
+
#1411: when *failed* is itself a fix round (``review_iteration > 0``),
|
|
527
|
+
the ``## Task`` section is instead the rebuilt FIX briefing — reviewer
|
|
528
|
+
findings included — via :func:`_build_fix_round_retry_briefing`, so the
|
|
529
|
+
retry knows what the reviewer objected to instead of blindly redoing
|
|
530
|
+
work the branch already contains.
|
|
531
|
+
"""
|
|
532
|
+
fix_task: str | None = None
|
|
533
|
+
if failed.review_iteration and failed.review_iteration > 0 and board is not None:
|
|
534
|
+
fix_task = _build_fix_round_retry_briefing(
|
|
535
|
+
failed, board, repo_cfg, max_review_iterations,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
base = (failed.briefing or "").strip()
|
|
539
|
+
if not base and fix_task is None and repo_cfg is not None:
|
|
540
|
+
try:
|
|
541
|
+
from coord import github_ops # noqa: PLC0415
|
|
542
|
+
|
|
543
|
+
issue_data = github_ops.get_issue(repo_cfg.github, failed.issue_number)
|
|
544
|
+
issue_body = issue_data.get("body", "")
|
|
545
|
+
if issue_body:
|
|
546
|
+
base = f"Issue #{failed.issue_number}: {failed.issue_title}\n\n{issue_body}"
|
|
547
|
+
except RuntimeError:
|
|
548
|
+
pass # best-effort — fall through with whatever we have
|
|
549
|
+
|
|
550
|
+
sections: list[str] = []
|
|
551
|
+
if failed.branch:
|
|
552
|
+
# #934: point the retry's diff/log instructions at `feature/ms-NN`
|
|
553
|
+
# when this issue belongs to a milestone and the repo opted into the
|
|
554
|
+
# git model — falls back to `default_branch` (today's behavior)
|
|
555
|
+
# otherwise. Callers that already resolved this (``_reassign``) pass
|
|
556
|
+
# it in via *default_branch*; otherwise resolve it here, but only
|
|
557
|
+
# perform the milestone lookup (a `gh` call) when the repo opted in.
|
|
558
|
+
if default_branch is None:
|
|
559
|
+
default_branch = (repo_cfg.default_branch if repo_cfg is not None else None) or "main"
|
|
560
|
+
if repo_cfg is not None and getattr(repo_cfg, "develop_branch", None):
|
|
561
|
+
from coord.branch_model import ( # noqa: PLC0415
|
|
562
|
+
fetch_issue_milestone_number,
|
|
563
|
+
resolve_base_branch,
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
milestone_number = fetch_issue_milestone_number(
|
|
567
|
+
repo_cfg.github, failed.issue_number,
|
|
568
|
+
)
|
|
569
|
+
default_branch = resolve_base_branch(repo_cfg, milestone_number)
|
|
570
|
+
sections.append(
|
|
571
|
+
"## Retry — continuing existing work\n"
|
|
572
|
+
f"This is a retry of a previously failed assignment "
|
|
573
|
+
f"({failed.assignment_id}). The previous worker's branch "
|
|
574
|
+
f"`{failed.branch}` already exists and may carry real, "
|
|
575
|
+
f"committed work — you are continuing it, NOT starting over.\n"
|
|
576
|
+
f"Run `git fetch origin && git log --oneline "
|
|
577
|
+
f"origin/{default_branch}..HEAD` to see what's already done, "
|
|
578
|
+
f"and `git diff origin/{default_branch}...HEAD` for the full "
|
|
579
|
+
f"diff, before writing any new code."
|
|
580
|
+
)
|
|
581
|
+
if failed.failure_reason:
|
|
582
|
+
sections.append(f"## Why the previous attempt failed\n{failed.failure_reason}")
|
|
583
|
+
if fix_task is not None:
|
|
584
|
+
sections.append(
|
|
585
|
+
f"## Task — fix round {failed.review_iteration} "
|
|
586
|
+
f"(reviewer findings included)\n{fix_task}"
|
|
587
|
+
)
|
|
588
|
+
elif base:
|
|
589
|
+
sections.append(f"## Task\n{base}")
|
|
590
|
+
if not sections:
|
|
591
|
+
# Nothing stored, nothing fetched, no branch context either — this
|
|
592
|
+
# is exactly the silent-empty-briefing failure mode from #1101.
|
|
593
|
+
sections.append(
|
|
594
|
+
f"Issue #{failed.issue_number}: {failed.issue_title}\n\n"
|
|
595
|
+
f"(No stored briefing or issue body was available to "
|
|
596
|
+
f"reconstruct this retry — investigate issue "
|
|
597
|
+
f"#{failed.issue_number} directly.)"
|
|
598
|
+
)
|
|
599
|
+
return "\n\n".join(sections)
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _running_by_machine(board: Board) -> dict[str, list[Assignment]]:
|
|
603
|
+
"""Group ``board.active`` running assignments by machine name (#1417).
|
|
604
|
+
|
|
605
|
+
Shared by :func:`_reassign` and :func:`describe_no_candidate_machines` so
|
|
606
|
+
the two paths can never drift on what counts as "running".
|
|
607
|
+
"""
|
|
608
|
+
running: dict[str, list[Assignment]] = {}
|
|
609
|
+
for a in board.active:
|
|
610
|
+
if a.status == "running":
|
|
611
|
+
running.setdefault(a.machine_name, []).append(a)
|
|
612
|
+
return running
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _machine_capacity(machine: Machine, config: Config) -> int:
|
|
616
|
+
"""Effective concurrent-assignment cap for *machine* (#1417).
|
|
617
|
+
|
|
618
|
+
``machines[].max_workers`` in coordinator.yml overrides the fleet-wide
|
|
619
|
+
``concurrency.max_workers`` default — set it lower on hardware that
|
|
620
|
+
can't keep up with the fleet norm (e.g. a 4-core box among 20-core
|
|
621
|
+
desktops). Unset (``None``) means "use the fleet-wide default", so a
|
|
622
|
+
single running assignment no longer reads as "full" the way a bare
|
|
623
|
+
``machine in busy`` membership check used to (#1417).
|
|
624
|
+
"""
|
|
625
|
+
return machine.max_workers if machine.max_workers is not None else config.concurrency.max_workers
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
class UnsupportedRetryType(ValueError):
|
|
629
|
+
"""Raised by :func:`_reassign` when *failed.type* cannot be safely
|
|
630
|
+
re-dispatched through the work-retry path (#1636).
|
|
631
|
+
|
|
632
|
+
``_reassign`` used to hardcode ``type="work"`` on every retry regardless
|
|
633
|
+
of the failed assignment's actual type — a retried ``smoke``/``review``
|
|
634
|
+
row silently came back as a fresh WORK worker (model escalated) pointed
|
|
635
|
+
at the already-complete branch, instead of re-running the Test/Review
|
|
636
|
+
stage. Raising here — instead of silently downgrading to work — lets
|
|
637
|
+
every caller (``coord retry``, ``auto_reassign``) surface the command
|
|
638
|
+
that actually re-runs the right stage rather than quietly doing the
|
|
639
|
+
wrong thing.
|
|
640
|
+
"""
|
|
641
|
+
|
|
642
|
+
def __init__(self, assignment_type: str, work_assignment_id: str | None):
|
|
643
|
+
self.assignment_type = assignment_type
|
|
644
|
+
self.work_assignment_id = work_assignment_id
|
|
645
|
+
super().__init__(
|
|
646
|
+
f"assignment type {assignment_type!r} cannot be retried "
|
|
647
|
+
"through the work-retry path"
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
# #1636: types whose failed row can be re-dispatched with the exact command
|
|
652
|
+
# that re-runs their stage — `review_of_assignment_id` on a smoke/review
|
|
653
|
+
# assignment is the work assignment it targets, so the hint is always
|
|
654
|
+
# actionable when set. Extend this map, not the work-retry path, when a new
|
|
655
|
+
# non-WORK_LIKE type grows its own retry story.
|
|
656
|
+
_RETRY_REDIRECT_FLAGS: dict[str, str] = {
|
|
657
|
+
"smoke": "--smoke-of",
|
|
658
|
+
"review": "--review-of",
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def describe_unsupported_retry_type(exc: UnsupportedRetryType) -> str:
|
|
663
|
+
"""Human-readable refusal message for :class:`UnsupportedRetryType` (#1636).
|
|
664
|
+
|
|
665
|
+
Mirrors :func:`describe_no_candidate_machines` — a caller shouldn't have
|
|
666
|
+
to hand-craft the "this can't be retried" message.
|
|
667
|
+
"""
|
|
668
|
+
flag = _RETRY_REDIRECT_FLAGS.get(exc.assignment_type)
|
|
669
|
+
if flag is not None and exc.work_assignment_id:
|
|
670
|
+
return (
|
|
671
|
+
f"assignment type {exc.assignment_type!r} cannot be retried "
|
|
672
|
+
"with `coord retry` — that would silently re-dispatch it as a "
|
|
673
|
+
"fresh work worker on the already-complete branch. Re-run its "
|
|
674
|
+
f"stage instead: `coord assign --interactive {flag} "
|
|
675
|
+
f"{exc.work_assignment_id}`."
|
|
676
|
+
)
|
|
677
|
+
return (
|
|
678
|
+
f"assignment type {exc.assignment_type!r} cannot be retried with "
|
|
679
|
+
"`coord retry` — that would silently re-dispatch it as a fresh "
|
|
680
|
+
"work worker. Re-dispatch it through its own path instead."
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _reassign(
|
|
685
|
+
failed: Assignment, board: Board, config: Config,
|
|
686
|
+
*,
|
|
687
|
+
model: str | None = None,
|
|
688
|
+
) -> Assignment | None:
|
|
689
|
+
"""Re-dispatch a failed assignment to a machine with spare capacity.
|
|
690
|
+
|
|
691
|
+
*model* overrides the model tier on the retry. When None, the
|
|
692
|
+
original assignment's model is reused (escalation happens at the call
|
|
693
|
+
site).
|
|
694
|
+
|
|
695
|
+
Raises :class:`UnsupportedRetryType` when ``failed.type`` is not in
|
|
696
|
+
:data:`coord.models.WORK_LIKE_TYPES` — a ``smoke``/``review``/other
|
|
697
|
+
non-work row must not be silently re-dispatched as a fresh
|
|
698
|
+
``type="work"`` worker (#1636).
|
|
699
|
+
"""
|
|
700
|
+
if failed.type not in WORK_LIKE_TYPES:
|
|
701
|
+
raise UnsupportedRetryType(failed.type, failed.review_of_assignment_id)
|
|
702
|
+
|
|
703
|
+
from coord.machine_pause import paused_set
|
|
704
|
+
paused = paused_set(config.machines)
|
|
705
|
+
running = _running_by_machine(board)
|
|
706
|
+
|
|
707
|
+
# #1417: fleet-wide cap first — respected regardless of per-machine
|
|
708
|
+
# headroom, mirroring `concurrency.max_workers`'s documented meaning as
|
|
709
|
+
# the total concurrent-worker budget across the whole fleet.
|
|
710
|
+
fleet_running = sum(len(v) for v in running.values())
|
|
711
|
+
if fleet_running >= config.concurrency.max_workers:
|
|
712
|
+
return None
|
|
713
|
+
|
|
714
|
+
def has_room(m: Machine) -> bool:
|
|
715
|
+
return len(running.get(m.name, [])) < _machine_capacity(m, config)
|
|
716
|
+
|
|
717
|
+
candidates = [
|
|
718
|
+
m for m in config.machines
|
|
719
|
+
if m.can_work_on(failed.repo_name)
|
|
720
|
+
and m.repo_path(failed.repo_name) is not None
|
|
721
|
+
and has_room(m)
|
|
722
|
+
and m.name != failed.machine_name
|
|
723
|
+
and m.name not in paused
|
|
724
|
+
]
|
|
725
|
+
if not candidates:
|
|
726
|
+
# Fall back to including the same machine that failed last time —
|
|
727
|
+
# paused machines stay excluded even from the fallback.
|
|
728
|
+
candidates = [
|
|
729
|
+
m for m in config.machines
|
|
730
|
+
if m.can_work_on(failed.repo_name)
|
|
731
|
+
and m.repo_path(failed.repo_name) is not None
|
|
732
|
+
and has_room(m)
|
|
733
|
+
and m.name not in paused
|
|
734
|
+
]
|
|
735
|
+
if not candidates:
|
|
736
|
+
return None
|
|
737
|
+
|
|
738
|
+
machine = candidates[0]
|
|
739
|
+
repo_path = machine.repo_path(failed.repo_name)
|
|
740
|
+
|
|
741
|
+
# #437: STRUCTURAL TOS-COMPLIANCE GATE — auto-reassign is an
|
|
742
|
+
# unattended dispatch path; refuse to retry through a provider that
|
|
743
|
+
# opts out of unattended use. Resolve precedence with per-repo
|
|
744
|
+
# override and the global default (the failed assignment doesn't
|
|
745
|
+
# carry a spec-level provider into this path). On refusal: skip the
|
|
746
|
+
# reassignment — the failed assignment stays failed for human
|
|
747
|
+
# attention rather than getting silently re-tried on the wrong
|
|
748
|
+
# provider.
|
|
749
|
+
from coord.providers import guard_unattended_dispatch # noqa: PLC0415
|
|
750
|
+
repo_for_provider = config.repo(failed.repo_name)
|
|
751
|
+
try:
|
|
752
|
+
guard_unattended_dispatch(
|
|
753
|
+
spec_provider=None,
|
|
754
|
+
repo_provider=(
|
|
755
|
+
repo_for_provider.provider
|
|
756
|
+
if repo_for_provider is not None
|
|
757
|
+
else None
|
|
758
|
+
),
|
|
759
|
+
providers_cfg=config.providers,
|
|
760
|
+
models_cfg=config.models,
|
|
761
|
+
where="auto-reassign (reconcile)",
|
|
762
|
+
)
|
|
763
|
+
except ValueError:
|
|
764
|
+
return None
|
|
765
|
+
|
|
766
|
+
retry_model = model if model is not None else failed.model
|
|
767
|
+
# The Assignment keeps the alias for legibility; the wire payload is
|
|
768
|
+
# resolved through models.versions when an exact id is pinned.
|
|
769
|
+
retry_model_wire = config.models.resolve(retry_model)
|
|
770
|
+
|
|
771
|
+
repo_cfg = config.repo(failed.repo_name)
|
|
772
|
+
# #934: retry inherits `feature/ms-NN` as its integration base when this
|
|
773
|
+
# issue belongs to a milestone and the repo opted into the git model —
|
|
774
|
+
# falls back to `default_branch` (today's behavior) otherwise. Resolved
|
|
775
|
+
# once and reused for both the briefing's diff instructions and the
|
|
776
|
+
# `branch` payload field below, so they never disagree. The milestone
|
|
777
|
+
# lookup itself is skipped (no `gh` call) when the repo hasn't opted in.
|
|
778
|
+
retry_default_branch = (repo_cfg.default_branch if repo_cfg is not None else None) or "main"
|
|
779
|
+
if repo_cfg is not None and getattr(repo_cfg, "develop_branch", None):
|
|
780
|
+
from coord.branch_model import ( # noqa: PLC0415
|
|
781
|
+
fetch_issue_milestone_number,
|
|
782
|
+
resolve_base_branch,
|
|
783
|
+
)
|
|
784
|
+
|
|
785
|
+
milestone_number = fetch_issue_milestone_number(
|
|
786
|
+
repo_cfg.github, failed.issue_number,
|
|
787
|
+
)
|
|
788
|
+
retry_default_branch = resolve_base_branch(repo_cfg, milestone_number)
|
|
789
|
+
retry_briefing = _build_retry_briefing(
|
|
790
|
+
failed, repo_cfg, default_branch=retry_default_branch,
|
|
791
|
+
board=board, max_review_iterations=config.pipeline.max_review_iterations,
|
|
792
|
+
)
|
|
793
|
+
payload = {
|
|
794
|
+
"repo_name": failed.repo_name,
|
|
795
|
+
"repo_path": repo_path,
|
|
796
|
+
"issue_number": failed.issue_number,
|
|
797
|
+
"issue_title": f"[retry] {failed.issue_title}",
|
|
798
|
+
"briefing": retry_briefing,
|
|
799
|
+
"files_allowed": failed.files_allowed,
|
|
800
|
+
"files_forbidden": failed.files_forbidden,
|
|
801
|
+
"pull_repos": [],
|
|
802
|
+
# #1636: carries the failed assignment's own type (guaranteed to be
|
|
803
|
+
# in WORK_LIKE_TYPES by the guard above) instead of hardcoding
|
|
804
|
+
# "work" — a "mock-author"/"test-author" retry must not silently
|
|
805
|
+
# relabel itself as plain work.
|
|
806
|
+
"type": failed.type,
|
|
807
|
+
"model": retry_model_wire,
|
|
808
|
+
# #255: retry inherits the repo's configured default branch as the
|
|
809
|
+
# worker's integration base (the start point / rebase target).
|
|
810
|
+
"branch": retry_default_branch,
|
|
811
|
+
}
|
|
812
|
+
# #1101: continue the failed assignment's actual branch instead of
|
|
813
|
+
# silently forking a fresh one off the repo default — any real work it
|
|
814
|
+
# already committed and pushed must not be orphaned by a retry. Mirrors
|
|
815
|
+
# the `target_branch` wire field `--fix-of`/`--rework-of`/
|
|
816
|
+
# `_dispatch_followup` already use; the agent checks out this exact
|
|
817
|
+
# branch (hard-reset to the remote tip) when it exists on origin, and
|
|
818
|
+
# falls back to a fresh branch off `branch` above when it doesn't.
|
|
819
|
+
if failed.branch:
|
|
820
|
+
payload["target_branch"] = failed.branch
|
|
821
|
+
|
|
822
|
+
url = f"http://{machine.host}:{AGENT_PORT}/assign"
|
|
823
|
+
try:
|
|
824
|
+
resp = httpx.post(url, json=payload, timeout=15)
|
|
825
|
+
resp.raise_for_status()
|
|
826
|
+
agent_response = resp.json()
|
|
827
|
+
except (httpx.HTTPError, httpx.TimeoutException):
|
|
828
|
+
return None
|
|
829
|
+
|
|
830
|
+
retry_assignment = Assignment(
|
|
831
|
+
machine_name=machine.name,
|
|
832
|
+
repo_name=failed.repo_name,
|
|
833
|
+
issue_number=failed.issue_number,
|
|
834
|
+
issue_title=f"[retry] {failed.issue_title}",
|
|
835
|
+
files_allowed=failed.files_allowed,
|
|
836
|
+
files_forbidden=failed.files_forbidden,
|
|
837
|
+
briefing=retry_briefing,
|
|
838
|
+
assignment_id=agent_response.get("id") or uuid.uuid4().hex[:12],
|
|
839
|
+
status="running",
|
|
840
|
+
dispatched_at=time.time(),
|
|
841
|
+
type=failed.type,
|
|
842
|
+
model=retry_model,
|
|
843
|
+
# #1101: record the continued branch on the board immediately
|
|
844
|
+
# instead of waiting for a later reconcile backfill from agent
|
|
845
|
+
# /status — the retry payload above already told the agent to
|
|
846
|
+
# check out `failed.branch` via target_branch.
|
|
847
|
+
branch=failed.branch,
|
|
848
|
+
# #1411: carry the fix-loop bookkeeping across the retry. Without
|
|
849
|
+
# this the retry's review_iteration silently resets to 0, so
|
|
850
|
+
# `pipeline.max_review_iterations` loses its accounting — a story
|
|
851
|
+
# that already burned fix rounds looks fresh again. Preserving
|
|
852
|
+
# review_of_assignment_id too keeps the work/review chain intact
|
|
853
|
+
# for `_build_fix_round_retry_briefing` if THIS retry also fails
|
|
854
|
+
# and gets retried again.
|
|
855
|
+
review_iteration=failed.review_iteration,
|
|
856
|
+
review_of_assignment_id=failed.review_of_assignment_id,
|
|
857
|
+
# #1553: carry the oracle-loop slice attribution across the retry
|
|
858
|
+
# too — otherwise a retried acceptance slice silently falls back to
|
|
859
|
+
# being booked against the milestone's tracking issue and the
|
|
860
|
+
# child's row goes quiet again mid-run. None for ordinary work.
|
|
861
|
+
for_issue_number=failed.for_issue_number,
|
|
862
|
+
)
|
|
863
|
+
board.active.append(retry_assignment)
|
|
864
|
+
|
|
865
|
+
from coord.state import record_dispatched_assignment
|
|
866
|
+
repo = config.repo(failed.repo_name)
|
|
867
|
+
if repo is not None:
|
|
868
|
+
record_dispatched_assignment(
|
|
869
|
+
assignment=retry_assignment,
|
|
870
|
+
repo_github=repo.github,
|
|
871
|
+
)
|
|
872
|
+
|
|
873
|
+
return retry_assignment
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def describe_no_candidate_machines(
|
|
877
|
+
failed: Assignment, board: Board, config: Config,
|
|
878
|
+
) -> str:
|
|
879
|
+
"""Explain why :func:`_reassign` found no candidate machine (#1396).
|
|
880
|
+
|
|
881
|
+
``_reassign`` silently returns ``None`` on any of: no machine with spare
|
|
882
|
+
capacity can work on the repo, a TOS-gate refusal, or a dispatch POST
|
|
883
|
+
failure — so a caller (``coord retry``) can only ever say "no available
|
|
884
|
+
machine to retry on", which is true from the code's point of view and
|
|
885
|
+
useless to an operator when ``coord status`` shows every machine well
|
|
886
|
+
under capacity. The real cause is almost always a phantom ``running``
|
|
887
|
+
board row: a dead interactive (``claude-pty``) session that nothing
|
|
888
|
+
reaped, still counted against the machine's capacity.
|
|
889
|
+
|
|
890
|
+
Mirrors ``_reassign``'s exact candidate filter (repo capability, repo
|
|
891
|
+
path, pause set, capacity check, same-machine exclusion — #1417 replaced
|
|
892
|
+
the old binary "any running assignment = busy" rule with a per-machine
|
|
893
|
+
capacity count against ``machines[].max_workers``/``concurrency.
|
|
894
|
+
max_workers``) but keeps a reason per excluded machine instead of
|
|
895
|
+
discarding it, so the message names the blocking machines and what
|
|
896
|
+
they're apparently running — including the age, which makes a
|
|
897
|
+
400-hour-old phantom obvious at a glance.
|
|
898
|
+
"""
|
|
899
|
+
from coord.machine_pause import paused_set # noqa: PLC0415
|
|
900
|
+
|
|
901
|
+
paused = paused_set(config.machines)
|
|
902
|
+
now = time.time()
|
|
903
|
+
|
|
904
|
+
running_by_machine = _running_by_machine(board)
|
|
905
|
+
|
|
906
|
+
relevant_machines = [
|
|
907
|
+
m for m in config.machines
|
|
908
|
+
if m.can_work_on(failed.repo_name) and m.repo_path(failed.repo_name) is not None
|
|
909
|
+
]
|
|
910
|
+
if not relevant_machines:
|
|
911
|
+
return f"no machine in coordinator.yml can work on repo {failed.repo_name!r}"
|
|
912
|
+
|
|
913
|
+
# #1417: the fleet-wide cap blocks every machine regardless of
|
|
914
|
+
# individual headroom — computed once so a machine that's personally
|
|
915
|
+
# under its own cap can still be correctly labeled as blocked by the
|
|
916
|
+
# fleet-wide budget instead of silently reading as "free".
|
|
917
|
+
fleet_running = sum(len(v) for v in running_by_machine.values())
|
|
918
|
+
fleet_cap = config.concurrency.max_workers
|
|
919
|
+
fleet_full = fleet_running >= fleet_cap
|
|
920
|
+
|
|
921
|
+
lines: list[str] = []
|
|
922
|
+
has_free_candidate = False
|
|
923
|
+
for m in relevant_machines:
|
|
924
|
+
if m.name in paused:
|
|
925
|
+
lines.append(f" {m.name}: paused")
|
|
926
|
+
continue
|
|
927
|
+
running = running_by_machine.get(m.name, [])
|
|
928
|
+
cap = _machine_capacity(m, config)
|
|
929
|
+
own_full = len(running) >= cap
|
|
930
|
+
if own_full or fleet_full:
|
|
931
|
+
if running:
|
|
932
|
+
parts = []
|
|
933
|
+
for a in running:
|
|
934
|
+
age_h = (now - a.dispatched_at) / 3600 if a.dispatched_at else None
|
|
935
|
+
age_str = f"{age_h:.1f}h" if age_h is not None else "?h"
|
|
936
|
+
parts.append(
|
|
937
|
+
f"{a.repo_name}#{a.issue_number} type={a.type} age={age_str}"
|
|
938
|
+
)
|
|
939
|
+
load_desc = f"{len(running)}/{cap} running: {'; '.join(parts)}"
|
|
940
|
+
else:
|
|
941
|
+
load_desc = f"0/{cap} running"
|
|
942
|
+
if own_full:
|
|
943
|
+
reason = f"busy — at capacity ({load_desc})"
|
|
944
|
+
else:
|
|
945
|
+
# This machine has its own headroom, but the fleet-wide
|
|
946
|
+
# budget (concurrency.max_workers) is exhausted — name the
|
|
947
|
+
# actual binding constraint rather than implying the
|
|
948
|
+
# machine itself is the problem.
|
|
949
|
+
reason = (
|
|
950
|
+
f"fleet at capacity ({fleet_running}/{fleet_cap} running "
|
|
951
|
+
f"fleet-wide; this machine {load_desc})"
|
|
952
|
+
)
|
|
953
|
+
lines.append(f" {m.name}: {reason}")
|
|
954
|
+
continue
|
|
955
|
+
if m.name == failed.machine_name:
|
|
956
|
+
# `_reassign`'s fallback pass drops only the "different machine"
|
|
957
|
+
# constraint — it still honors capacity/paused — so a
|
|
958
|
+
# under-capacity machine that just failed IS a real fallback
|
|
959
|
+
# candidate (#1396 review finding 1). Categorize it as such; the
|
|
960
|
+
# "(fallback-only)" label stays in `lines` for readers of the
|
|
961
|
+
# capacity/paused branch below, but this path never reaches that
|
|
962
|
+
# branch once a candidate is found.
|
|
963
|
+
lines.append(f" {m.name}: same machine that just failed (fallback-only)")
|
|
964
|
+
has_free_candidate = True
|
|
965
|
+
continue
|
|
966
|
+
has_free_candidate = True
|
|
967
|
+
|
|
968
|
+
if has_free_candidate:
|
|
969
|
+
# A machine WAS free per this filter — _reassign must have failed for
|
|
970
|
+
# a different reason: a TOS-gate refusal or a dispatch POST error.
|
|
971
|
+
# Re-run the same config-only gate check (no network call) so the
|
|
972
|
+
# message states the *actual* reason instead of a guess that may be
|
|
973
|
+
# a dead end (#1396 review finding 2 — "check daemon logs" pointed
|
|
974
|
+
# nowhere, since neither failure path logs anything today).
|
|
975
|
+
from coord.providers import guard_unattended_dispatch # noqa: PLC0415
|
|
976
|
+
|
|
977
|
+
repo_for_provider = config.repo(failed.repo_name)
|
|
978
|
+
try:
|
|
979
|
+
guard_unattended_dispatch(
|
|
980
|
+
spec_provider=None,
|
|
981
|
+
repo_provider=(
|
|
982
|
+
repo_for_provider.provider
|
|
983
|
+
if repo_for_provider is not None
|
|
984
|
+
else None
|
|
985
|
+
),
|
|
986
|
+
providers_cfg=config.providers,
|
|
987
|
+
models_cfg=config.models,
|
|
988
|
+
where="describe_no_candidate_machines (diagnostic re-check)",
|
|
989
|
+
)
|
|
990
|
+
except ValueError as exc:
|
|
991
|
+
return (
|
|
992
|
+
"a candidate machine was available, but the retry was "
|
|
993
|
+
f"refused by the provider TOS gate: {exc}"
|
|
994
|
+
)
|
|
995
|
+
return (
|
|
996
|
+
"a candidate machine was available but the retry dispatch "
|
|
997
|
+
"request failed (network error or the agent was unreachable) "
|
|
998
|
+
"— re-run `coord retry` to try again"
|
|
999
|
+
)
|
|
1000
|
+
|
|
1001
|
+
return "no available machine to retry on:\n" + "\n".join(lines)
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def _record_usage_limit_reason(assignment_id: str | None, entry: dict) -> None:
|
|
1005
|
+
"""#1461/#1584/#1797: stamp a usage-limit-kill, terminal-API-error, or
|
|
1006
|
+
auth-shaped-push-failure diagnostic (whichever the agent flagged on
|
|
1007
|
+
*entry*) onto *assignment_id*'s persisted ``failure_reason``.
|
|
1008
|
+
|
|
1009
|
+
Used by :func:`reconcile`'s (``coord resume``) FAILED/ADVISORY branches.
|
|
1010
|
+
``reconcile_completed_assignments`` — the daemon's own passive tick and
|
|
1011
|
+
the primary production path — does the equivalent inline via
|
|
1012
|
+
``update_state_fn`` (a raw local write is safe there: that function is
|
|
1013
|
+
daemon-tick-only, never thin-client-reachable). ``reconcile()`` is
|
|
1014
|
+
different — it is called from ``coord resume``, which IS reachable from a
|
|
1015
|
+
thin client (same #906 audit gap `get_issue_test_mode` was fixed for) —
|
|
1016
|
+
so this goes through :func:`coord.state.set_assignment_failure_reason`,
|
|
1017
|
+
which is already daemon-aware (routes to ``POST
|
|
1018
|
+
/assignment-failure-reason`` when a board service is configured), rather
|
|
1019
|
+
than a raw ``get_connection()`` write that would silently land on a thin
|
|
1020
|
+
client's empty local DB instead.
|
|
1021
|
+
|
|
1022
|
+
``usage_limit_reason`` is tried first, then ``api_error_reason`` (#1584 —
|
|
1023
|
+
a terminal `is_error: true` result event, e.g. "529 Overloaded"; see
|
|
1024
|
+
`coord.agent.AgentAssignment.api_error_reason`), then
|
|
1025
|
+
``push_failure_reason`` (#1797 — an auth-shaped rejection from the
|
|
1026
|
+
reap-time safety-net push; see `coord.agent._is_auth_push_failure`). The
|
|
1027
|
+
three never coexist on the same entry — a usage-limit kill is detected
|
|
1028
|
+
from a truncated log with no terminal `result` event, an API error is
|
|
1029
|
+
read OFF that terminal `result` event, and a push failure only ever
|
|
1030
|
+
surfaces on an otherwise-clean `exit_code == 0` reap that neither of the
|
|
1031
|
+
other two preempted (see the `elif` chain in `AgentServer._reap`) — so
|
|
1032
|
+
trying them in order never picks the wrong reason.
|
|
1033
|
+
|
|
1034
|
+
This also normalises the row's status to ``'failed'`` (that helper's own
|
|
1035
|
+
behaviour) even when the agent's reap landed on ADVISORY — a usage-limit
|
|
1036
|
+
kill is, per #1461, the ONE terminal state known safe to re-dispatch
|
|
1037
|
+
unchanged, which is what `coord/drive.py`'s FAILED bucket already means;
|
|
1038
|
+
ADVISORY otherwise implies "needs a human look", which a kill does not.
|
|
1039
|
+
(An `api_error_reason` or `push_failure_reason` entry is never ADVISORY —
|
|
1040
|
+
`AgentServer._reap` always lands both directly on FAILED — so this
|
|
1041
|
+
normalisation is a no-op for those cases, not a behaviour change.)
|
|
1042
|
+
|
|
1043
|
+
Best-effort: never raises — a diagnostic write must not break a real
|
|
1044
|
+
status transition.
|
|
1045
|
+
"""
|
|
1046
|
+
reason = (
|
|
1047
|
+
entry.get("usage_limit_reason")
|
|
1048
|
+
or entry.get("api_error_reason")
|
|
1049
|
+
or entry.get("push_failure_reason")
|
|
1050
|
+
)
|
|
1051
|
+
if not reason or not assignment_id:
|
|
1052
|
+
return
|
|
1053
|
+
try:
|
|
1054
|
+
from coord.state import set_assignment_failure_reason # noqa: PLC0415
|
|
1055
|
+
|
|
1056
|
+
set_assignment_failure_reason(assignment_id, reason)
|
|
1057
|
+
except Exception: # noqa: BLE001
|
|
1058
|
+
pass
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def reconcile(board: Board, config: Config) -> list[str]:
|
|
1062
|
+
"""Poll agent servers and update board assignments that have finished.
|
|
1063
|
+
|
|
1064
|
+
Returns assignment_ids whose status changed or were backfilled.
|
|
1065
|
+
"""
|
|
1066
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
1067
|
+
|
|
1068
|
+
# Collect all machines we need to query: those with active assignments
|
|
1069
|
+
# OR completed assignments missing branch info.
|
|
1070
|
+
machines_to_query: set[str] = set()
|
|
1071
|
+
for a in board.active:
|
|
1072
|
+
machines_to_query.add(a.machine_name)
|
|
1073
|
+
for a in board.completed:
|
|
1074
|
+
if a.branch is None and a.assignment_id is not None:
|
|
1075
|
+
machines_to_query.add(a.machine_name)
|
|
1076
|
+
|
|
1077
|
+
# Query each machine once and cache the result.
|
|
1078
|
+
agent_completed: dict[str, dict] = {}
|
|
1079
|
+
reachable_machines: set[str] = set()
|
|
1080
|
+
for machine_name in machines_to_query:
|
|
1081
|
+
machine = machines_by_name.get(machine_name)
|
|
1082
|
+
if machine is None:
|
|
1083
|
+
continue
|
|
1084
|
+
status = _query_agent(machine.host)
|
|
1085
|
+
if status is None:
|
|
1086
|
+
continue
|
|
1087
|
+
reachable_machines.add(machine_name)
|
|
1088
|
+
for e in status.get("completed", []):
|
|
1089
|
+
agent_completed[e["id"]] = e
|
|
1090
|
+
|
|
1091
|
+
changed: list[str] = []
|
|
1092
|
+
newly_failed: list = [] # assignments that just transitioned to failed
|
|
1093
|
+
|
|
1094
|
+
# Sweep for dead interactive (--interactive / claude-pty) sessions before
|
|
1095
|
+
# processing agent-based assignments. A killed tmux session leaves a
|
|
1096
|
+
# stale "running" board row + orphaned worktree that blocks relaunch.
|
|
1097
|
+
# Reaping here ensures ``coord resume`` / ``coord notify`` clean up
|
|
1098
|
+
# without requiring the user to first run ``coord reattach``.
|
|
1099
|
+
from coord.interactive import ( # noqa: PLC0415
|
|
1100
|
+
reap_stale_interactive_sessions,
|
|
1101
|
+
reap_stale_remote_interactive_sessions,
|
|
1102
|
+
)
|
|
1103
|
+
|
|
1104
|
+
reaped = reap_stale_interactive_sessions(board, config)
|
|
1105
|
+
changed.extend(reaped)
|
|
1106
|
+
|
|
1107
|
+
# #588: probe remote claude-pty sessions older than the configured timeout
|
|
1108
|
+
# threshold. The local reaper above skips these; this sweep SSHes to the
|
|
1109
|
+
# remote host and finalizes sessions whose tmux has exited.
|
|
1110
|
+
remote_reaped = reap_stale_remote_interactive_sessions(board, config)
|
|
1111
|
+
changed.extend(remote_reaped)
|
|
1112
|
+
|
|
1113
|
+
# Pass 1: transition active assignments that have finished.
|
|
1114
|
+
for a in board.active[:]:
|
|
1115
|
+
if a.assignment_id is None:
|
|
1116
|
+
continue
|
|
1117
|
+
|
|
1118
|
+
# Track unreachable agents for stale detection
|
|
1119
|
+
if a.machine_name in machines_to_query and a.machine_name not in reachable_machines:
|
|
1120
|
+
a.unreachable_count = getattr(a, "unreachable_count", 0) + 1
|
|
1121
|
+
stale_threshold = getattr(config.concurrency, "stale_threshold", 3)
|
|
1122
|
+
if a.unreachable_count >= stale_threshold:
|
|
1123
|
+
board.mark_failed_by_id(a.assignment_id)
|
|
1124
|
+
newly_failed.append(a)
|
|
1125
|
+
changed.append(a.assignment_id)
|
|
1126
|
+
continue
|
|
1127
|
+
elif a.machine_name in reachable_machines:
|
|
1128
|
+
a.unreachable_count = 0
|
|
1129
|
+
|
|
1130
|
+
entry = agent_completed.get(a.assignment_id)
|
|
1131
|
+
if entry is None:
|
|
1132
|
+
continue
|
|
1133
|
+
branch = entry.get("branch")
|
|
1134
|
+
# #1534: read the status through the `done`-refusal helper so a
|
|
1135
|
+
# usage-limit kill the agent mislabelled `done` lands in the `failed`
|
|
1136
|
+
# branch below (which stamps `failure_reason` via
|
|
1137
|
+
# `_record_usage_limit_reason`) instead of being recorded as a clean,
|
|
1138
|
+
# unmarked completion that auto-dispatches a review.
|
|
1139
|
+
agent_status = effective_agent_status(entry)
|
|
1140
|
+
if agent_status == "done":
|
|
1141
|
+
done = board.mark_done_by_id(
|
|
1142
|
+
a.assignment_id,
|
|
1143
|
+
finished_at=entry.get("finished_at"),
|
|
1144
|
+
branch=branch,
|
|
1145
|
+
)
|
|
1146
|
+
if done is not None:
|
|
1147
|
+
if done.type in WORK_LIKE_TYPES:
|
|
1148
|
+
# Always mark work(-like) completions as pending review so
|
|
1149
|
+
# the dispatch loop below (and future reconcile passes)
|
|
1150
|
+
# can pick them up reliably. #930: "mock-author" is
|
|
1151
|
+
# work-like too — see WORK_LIKE_TYPES. (#1426: Test-stage
|
|
1152
|
+
# dispatch no longer needs its own "just transitioned"
|
|
1153
|
+
# list — dispatch_pending_smoke scans the full completed
|
|
1154
|
+
# backlog, the same shape as dispatch_pending_reviews.)
|
|
1155
|
+
done.review_state = "pending"
|
|
1156
|
+
elif done.type == "review":
|
|
1157
|
+
# #1566: `mark_done_by_id` just stamped `done.status =
|
|
1158
|
+
# "done"` — correct it to "finalizing". The review
|
|
1159
|
+
# AGENT finished, but the verdict is parsed + persisted
|
|
1160
|
+
# by `coord notify` (a separate, slower step — see the
|
|
1161
|
+
# matching comment in `reconcile_completed_assignments`
|
|
1162
|
+
# above), so calling this row "done" before that lands
|
|
1163
|
+
# would show a finished review with no verdict, which is
|
|
1164
|
+
# indistinguishable from a dropped one. `orig.review_state`
|
|
1165
|
+
# below is intentionally left "done" immediately (that
|
|
1166
|
+
# field means "the review PROCESS is over", not "verdict
|
|
1167
|
+
# known" — see #1584 — and `drive_state.TERMINAL_STATUSES`
|
|
1168
|
+
# not listing "finalizing" is what keeps `coord drive`
|
|
1169
|
+
# from misreading this row as a dead end in the meantime).
|
|
1170
|
+
done.status = "finalizing"
|
|
1171
|
+
# A review finished — update the original work assignment.
|
|
1172
|
+
orig_id = done.review_of_assignment_id
|
|
1173
|
+
if orig_id:
|
|
1174
|
+
orig = board.find_by_id(orig_id)
|
|
1175
|
+
if orig is not None:
|
|
1176
|
+
orig.review_state = "done"
|
|
1177
|
+
elif done.type == "conflict-fix":
|
|
1178
|
+
# #241: re-enqueue the parent merge entry for retry.
|
|
1179
|
+
_on_conflict_fix_done(done, succeeded=True)
|
|
1180
|
+
elif agent_status == "advisory":
|
|
1181
|
+
# #448: worker exited cleanly but pushed 0 commits. Move to
|
|
1182
|
+
# completed with status "advisory" — NOT "failed" — so that
|
|
1183
|
+
# auto_reassign does not loop on it. Review is also skipped
|
|
1184
|
+
# because there is no code to review on the branch.
|
|
1185
|
+
done = board.mark_done_by_id(
|
|
1186
|
+
a.assignment_id,
|
|
1187
|
+
finished_at=entry.get("finished_at"),
|
|
1188
|
+
branch=branch,
|
|
1189
|
+
)
|
|
1190
|
+
if done is not None:
|
|
1191
|
+
# mark_done_by_id sets status="done"; correct it to "advisory".
|
|
1192
|
+
done.status = "advisory"
|
|
1193
|
+
if done.type in WORK_LIKE_TYPES:
|
|
1194
|
+
# No code pushed → nothing to review. Set review_state to
|
|
1195
|
+
# "advisory" so the review-dispatch loop skips this entry.
|
|
1196
|
+
done.review_state = "advisory"
|
|
1197
|
+
elif done.type == "review":
|
|
1198
|
+
# Defensive (should not occur after Bug 2 fix): review
|
|
1199
|
+
# workers that somehow hit advisory still advance the
|
|
1200
|
+
# original work assignment's review_state.
|
|
1201
|
+
orig_id = done.review_of_assignment_id
|
|
1202
|
+
if orig_id:
|
|
1203
|
+
orig = board.find_by_id(orig_id)
|
|
1204
|
+
if orig is not None:
|
|
1205
|
+
orig.review_state = "done"
|
|
1206
|
+
elif done.type == "conflict-fix":
|
|
1207
|
+
# A conflict-fix with 0 commits didn't resolve anything.
|
|
1208
|
+
_on_conflict_fix_done(
|
|
1209
|
+
done, succeeded=False,
|
|
1210
|
+
agent_entry=entry, board=board, config=config,
|
|
1211
|
+
)
|
|
1212
|
+
_record_usage_limit_reason(a.assignment_id, entry)
|
|
1213
|
+
# NOTE: do NOT add to newly_failed — prevents auto_reassign loop.
|
|
1214
|
+
else:
|
|
1215
|
+
# Defensive: don't downgrade a DB-done assignment to failed when
|
|
1216
|
+
# the agent reports cancelled (e.g. after POST /cancel cleanup
|
|
1217
|
+
# of a hung reap). The work succeeded; cancellation here is
|
|
1218
|
+
# bookkeeping noise.
|
|
1219
|
+
if (agent_status == "cancelled"
|
|
1220
|
+
and (a.status or "").lower() == "done"):
|
|
1221
|
+
continue
|
|
1222
|
+
failed = board.mark_failed_by_id(
|
|
1223
|
+
a.assignment_id,
|
|
1224
|
+
finished_at=entry.get("finished_at"),
|
|
1225
|
+
)
|
|
1226
|
+
if failed is not None:
|
|
1227
|
+
newly_failed.append(failed)
|
|
1228
|
+
if failed.type == "conflict-fix":
|
|
1229
|
+
# #241: the auto-fix didn't work — escalate. #1291: a
|
|
1230
|
+
# SEMANTIC give-up buys one stronger attempt first.
|
|
1231
|
+
_on_conflict_fix_done(
|
|
1232
|
+
failed, succeeded=False,
|
|
1233
|
+
agent_entry=entry, board=board, config=config,
|
|
1234
|
+
)
|
|
1235
|
+
elif failed.type == "review":
|
|
1236
|
+
# #1584: a review worker that died (transient API error,
|
|
1237
|
+
# network drop, ...) before producing a verdict is now
|
|
1238
|
+
# correctly recorded FAILED (not the pre-#1584 silent
|
|
1239
|
+
# `done`) — but without this, the ORIGINAL work row's
|
|
1240
|
+
# `review_state` is left at "dispatched" forever, exactly
|
|
1241
|
+
# like the `done` (above) and `advisory` branches this
|
|
1242
|
+
# mirrors would leave it if THEY skipped this update.
|
|
1243
|
+
# "done" (not "failed") to match the existing
|
|
1244
|
+
# `coord._board_mapping.infer_review_state` convention,
|
|
1245
|
+
# which already treats a review row's `status in ("done",
|
|
1246
|
+
# "failed")` identically when inferring this same field —
|
|
1247
|
+
# the review PROCESS is over either way; `review_verdict`
|
|
1248
|
+
# (left empty here) is what actually distinguishes "no
|
|
1249
|
+
# verdict" from an approval, and `coord/drive.py`'s
|
|
1250
|
+
# `_decide_review` reads `review_status`/`review_verdict`
|
|
1251
|
+
# directly rather than this field for that distinction.
|
|
1252
|
+
orig_id = failed.review_of_assignment_id
|
|
1253
|
+
if orig_id:
|
|
1254
|
+
orig = board.find_by_id(orig_id)
|
|
1255
|
+
if orig is not None:
|
|
1256
|
+
orig.review_state = "done"
|
|
1257
|
+
_record_usage_limit_reason(a.assignment_id, entry)
|
|
1258
|
+
changed.append(a.assignment_id)
|
|
1259
|
+
|
|
1260
|
+
# Dispatch pending reviews for all completed work assignments.
|
|
1261
|
+
# We iterate board.completed (not just newly-done) so that a failed
|
|
1262
|
+
# dispatch on a previous reconcile pass is retried here automatically.
|
|
1263
|
+
#
|
|
1264
|
+
# #465: review fires immediately on work completion — no manual smoke
|
|
1265
|
+
# prerequisite (the interactive smoke gate now lives on merge).
|
|
1266
|
+
# dispatch_pending_reviews() bounds this with a per-pass cap + surge gate
|
|
1267
|
+
# (flood guard, incident 2026-06-08) and applies the #459 active-fix
|
|
1268
|
+
# dedupe, so a backlog unmasking can't flood metered reviews.
|
|
1269
|
+
from coord.review import dispatch_pending_reviews, dispatch_scoped_reviews_for_queue
|
|
1270
|
+
|
|
1271
|
+
for review in dispatch_pending_reviews(board, config):
|
|
1272
|
+
if review.assignment_id is not None:
|
|
1273
|
+
changed.append(review.assignment_id)
|
|
1274
|
+
|
|
1275
|
+
# #1476: a conflict-fix rebase can void an already-approved review by
|
|
1276
|
+
# changing content (patch-id mismatch) without any other new commit —
|
|
1277
|
+
# dispatch a re-review SCOPED to just the resolution delta instead of
|
|
1278
|
+
# leaving the merge entry blocked until a human notices and forces a
|
|
1279
|
+
# full re-review. Independent of dispatch_pending_reviews above (that
|
|
1280
|
+
# one looks at completed WORK rows; this one looks at PENDING merge
|
|
1281
|
+
# queue entries whose approval a rebase just voided).
|
|
1282
|
+
for review in dispatch_scoped_reviews_for_queue(board, config):
|
|
1283
|
+
if review.assignment_id is not None:
|
|
1284
|
+
changed.append(review.assignment_id)
|
|
1285
|
+
|
|
1286
|
+
# Auto-queue smoke tests for any completed work-like assignment still
|
|
1287
|
+
# missing a test verdict. Independent of review dispatch — both can fire
|
|
1288
|
+
# for the same completion.
|
|
1289
|
+
#
|
|
1290
|
+
# #1426: routed through `dispatch_pending_smoke`, the single choke point
|
|
1291
|
+
# `reconcile()` and `coord notify` both call (mirroring
|
|
1292
|
+
# `dispatch_pending_reviews` for the Review stage) — it scans the FULL
|
|
1293
|
+
# completed backlog, not just this pass's `newly_done_work`, so a row
|
|
1294
|
+
# that was missed on an earlier pass (e.g. no capable machine existed
|
|
1295
|
+
# yet) is retried here automatically instead of staying stuck forever.
|
|
1296
|
+
# `dispatch_pending_smoke` itself enforces `smoke_tests.auto_queue`, the
|
|
1297
|
+
# #685 per-issue test-mode gate (test-mode:smoke skips auto-dispatch —
|
|
1298
|
+
# the TUI offers the interactive smoke agent instead), and the
|
|
1299
|
+
# has_active_followup dedupe.
|
|
1300
|
+
from coord.smoke import dispatch_pending_smoke
|
|
1301
|
+
|
|
1302
|
+
for smoke in dispatch_pending_smoke(board, config):
|
|
1303
|
+
if smoke.assignment_id is not None:
|
|
1304
|
+
changed.append(smoke.assignment_id)
|
|
1305
|
+
|
|
1306
|
+
# Auto-reassign failed work assignments to a different machine.
|
|
1307
|
+
if newly_failed and getattr(config.concurrency, "auto_reassign", False):
|
|
1308
|
+
# #1590: one decision point for "was this the weather or the work".
|
|
1309
|
+
from coord.failure_class import classify_failure # noqa: PLC0415
|
|
1310
|
+
|
|
1311
|
+
for failed_a in newly_failed:
|
|
1312
|
+
if getattr(failed_a, "type", "work") != "work":
|
|
1313
|
+
continue
|
|
1314
|
+
# #1461 review finding 1: a usage-limit kill is an account-wide
|
|
1315
|
+
# exhausted budget, not a per-machine defect — re-dispatching it
|
|
1316
|
+
# onto a *different* machine still burns the same subscription
|
|
1317
|
+
# limit and is guaranteed to die the same way until the reset.
|
|
1318
|
+
# Check the just-seen agent entry (this pass; `_record_usage_
|
|
1319
|
+
# limit_reason` below only writes through to the DB / board
|
|
1320
|
+
# service, it does not mutate this in-memory `failed_a`) AND the
|
|
1321
|
+
# already-persisted `failure_reason` (a prior pass already
|
|
1322
|
+
# stamped it, e.g. after a race with `reconcile_completed_
|
|
1323
|
+
# assignments`'s own tick).
|
|
1324
|
+
#
|
|
1325
|
+
# #1590 deliberately does NOT widen this skip to every
|
|
1326
|
+
# `environmental` class: an API 5xx/network failure genuinely can
|
|
1327
|
+
# be machine-local (one agent host's DNS, one flaky link), so
|
|
1328
|
+
# moving it to another machine is a reasonable first move and the
|
|
1329
|
+
# bounded `auto_reassign` retry still terminates. Only the
|
|
1330
|
+
# usage limit is provably account-wide, and only it is skipped.
|
|
1331
|
+
entry = agent_completed.get(failed_a.assignment_id)
|
|
1332
|
+
classification = classify_failure(
|
|
1333
|
+
usage_limit_reason=(entry or {}).get("usage_limit_reason"),
|
|
1334
|
+
failure_reason=getattr(failed_a, "failure_reason", None),
|
|
1335
|
+
)
|
|
1336
|
+
if classification.is_usage_limit:
|
|
1337
|
+
continue
|
|
1338
|
+
reassigned = _reassign(failed_a, board, config)
|
|
1339
|
+
if reassigned is not None and reassigned.assignment_id is not None:
|
|
1340
|
+
changed.append(reassigned.assignment_id)
|
|
1341
|
+
|
|
1342
|
+
# Pass 2: backfill branch on completed assignments that are missing it.
|
|
1343
|
+
for a in board.completed:
|
|
1344
|
+
if a.branch is not None or a.assignment_id is None:
|
|
1345
|
+
continue
|
|
1346
|
+
entry = agent_completed.get(a.assignment_id)
|
|
1347
|
+
if entry is None:
|
|
1348
|
+
continue
|
|
1349
|
+
branch = entry.get("branch")
|
|
1350
|
+
if branch:
|
|
1351
|
+
a.branch = branch
|
|
1352
|
+
changed.append(a.assignment_id)
|
|
1353
|
+
|
|
1354
|
+
return changed
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def _post_human_required_comment_raw(
|
|
1358
|
+
entry: QueuedMerge,
|
|
1359
|
+
fix_assignment_id: str,
|
|
1360
|
+
machine_name: str,
|
|
1361
|
+
) -> None:
|
|
1362
|
+
"""Notify the user on GitHub that a conflict-fix worker gave up."""
|
|
1363
|
+
from coord import github_ops # noqa: PLC0415
|
|
1364
|
+
|
|
1365
|
+
body = (
|
|
1366
|
+
"## Conflict-fix worker could not auto-resolve\n\n"
|
|
1367
|
+
f"Worker `{fix_assignment_id}` on "
|
|
1368
|
+
f"`{machine_name}` attempted to rebase "
|
|
1369
|
+
f"`{entry.branch}` onto `{entry.target_branch}` and exited "
|
|
1370
|
+
"non-zero. The merge queue entry is now `HUMAN_REQUIRED`.\n\n"
|
|
1371
|
+
f"**Last error:** `{entry.error or 'unknown'}`\n\n"
|
|
1372
|
+
"Manual resolution required: rebase the branch locally and "
|
|
1373
|
+
"`git push --force-with-lease`, then re-run `coord merge`. The "
|
|
1374
|
+
"coordinator will not re-dispatch a conflict-fix for this entry "
|
|
1375
|
+
"in the current session."
|
|
1376
|
+
)
|
|
1377
|
+
try:
|
|
1378
|
+
github_ops.post_issue_comment(entry.repo_github, entry.issue_number, body)
|
|
1379
|
+
except Exception as exc: # noqa: BLE001 — best-effort notification
|
|
1380
|
+
import logging # noqa: PLC0415
|
|
1381
|
+
logging.warning(
|
|
1382
|
+
"could not post HUMAN_REQUIRED comment on %s#%d: %s",
|
|
1383
|
+
entry.repo_github, entry.issue_number, exc,
|
|
1384
|
+
)
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _post_semantic_escalation_comment(
|
|
1388
|
+
entry: QueuedMerge,
|
|
1389
|
+
*,
|
|
1390
|
+
model: str,
|
|
1391
|
+
escalated_assignment_id: str,
|
|
1392
|
+
machine_name: str,
|
|
1393
|
+
) -> None:
|
|
1394
|
+
"""#1291: tell the operator a SEMANTIC merge is being attempted.
|
|
1395
|
+
|
|
1396
|
+
A semantic auto-resolution is higher-trust than a mechanical rebase, so
|
|
1397
|
+
it is announced up front — the point is that the human reviews the diff
|
|
1398
|
+
rather than discovering it after the merge.
|
|
1399
|
+
"""
|
|
1400
|
+
from coord import github_ops # noqa: PLC0415
|
|
1401
|
+
|
|
1402
|
+
body = (
|
|
1403
|
+
"## Semantic conflict — escalated for one stronger attempt\n\n"
|
|
1404
|
+
f"The conflict-fix worker judged the conflict on `{entry.branch}` → "
|
|
1405
|
+
f"`{entry.target_branch}` **semantic** and stopped rather than "
|
|
1406
|
+
f"guess. The coordinator has dispatched ONE escalated attempt with "
|
|
1407
|
+
f"model `{model}` (assignment `{escalated_assignment_id}` on "
|
|
1408
|
+
f"`{machine_name}`).\n\n"
|
|
1409
|
+
f"**Last error:** `{entry.error or 'unknown'}`\n\n"
|
|
1410
|
+
"⚠️ **Review this diff before it merges.** A semantic resolution "
|
|
1411
|
+
"reconciles two different intents — it is a judgement call, not a "
|
|
1412
|
+
"mechanical rebase. Every gate still applies (tests, CI, "
|
|
1413
|
+
"`verify-merge`, review); nothing is force-merged. If this attempt "
|
|
1414
|
+
"fails, the merge entry goes to `HUMAN_REQUIRED` — there is no "
|
|
1415
|
+
"second escalation."
|
|
1416
|
+
)
|
|
1417
|
+
try:
|
|
1418
|
+
github_ops.post_issue_comment(entry.repo_github, entry.issue_number, body)
|
|
1419
|
+
except Exception as exc: # noqa: BLE001 — best-effort notification
|
|
1420
|
+
import logging # noqa: PLC0415
|
|
1421
|
+
logging.warning(
|
|
1422
|
+
"could not post semantic-escalation comment on %s#%d: %s",
|
|
1423
|
+
entry.repo_github, entry.issue_number, exc,
|
|
1424
|
+
)
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _try_semantic_escalation(
|
|
1428
|
+
entry: QueuedMerge,
|
|
1429
|
+
*,
|
|
1430
|
+
board: Board | None,
|
|
1431
|
+
config: Config | None,
|
|
1432
|
+
machine_name: str,
|
|
1433
|
+
stuck_summary: str | None,
|
|
1434
|
+
) -> "Assignment | None":
|
|
1435
|
+
"""Dispatch the one escalated (semantic) conflict-fix attempt, if allowed.
|
|
1436
|
+
|
|
1437
|
+
Returns the escalated assignment, or ``None`` when the feature is off,
|
|
1438
|
+
the plumbing isn't available, this entry already had its one escalation,
|
|
1439
|
+
or dispatch failed — in every ``None`` case the caller falls through to
|
|
1440
|
+
today's HUMAN_REQUIRED behaviour.
|
|
1441
|
+
"""
|
|
1442
|
+
if board is None or config is None:
|
|
1443
|
+
return None
|
|
1444
|
+
pipeline = getattr(config, "pipeline", None)
|
|
1445
|
+
if pipeline is None or not getattr(pipeline, "escalate_semantic_conflicts", False):
|
|
1446
|
+
return None
|
|
1447
|
+
|
|
1448
|
+
from coord.conflict_fix import dispatch_conflict_fix # noqa: PLC0415
|
|
1449
|
+
|
|
1450
|
+
model = getattr(pipeline, "semantic_conflict_model", None) or "fable"
|
|
1451
|
+
try:
|
|
1452
|
+
return dispatch_conflict_fix(
|
|
1453
|
+
entry,
|
|
1454
|
+
board,
|
|
1455
|
+
config,
|
|
1456
|
+
prefer_machine=machine_name or None,
|
|
1457
|
+
semantic=True,
|
|
1458
|
+
model=model,
|
|
1459
|
+
stuck_summary=stuck_summary,
|
|
1460
|
+
)
|
|
1461
|
+
except Exception as exc: # noqa: BLE001 — never break reconcile on this
|
|
1462
|
+
import logging # noqa: PLC0415
|
|
1463
|
+
logging.warning("semantic escalation dispatch failed: %s", exc)
|
|
1464
|
+
return None
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def on_conflict_fix_done(
|
|
1468
|
+
*,
|
|
1469
|
+
parent_assignment_id: str,
|
|
1470
|
+
fix_assignment_id: str,
|
|
1471
|
+
machine_name: str,
|
|
1472
|
+
succeeded: bool,
|
|
1473
|
+
semantic: bool = False,
|
|
1474
|
+
board: Board | None = None,
|
|
1475
|
+
config: Config | None = None,
|
|
1476
|
+
stuck_summary: str | None = None,
|
|
1477
|
+
usage_limit_reason: str | None = None,
|
|
1478
|
+
) -> None:
|
|
1479
|
+
"""Update the parent merge entry after a conflict-fix worker finishes.
|
|
1480
|
+
|
|
1481
|
+
On *succeeded*: the merge entry is reset to PENDING so the next
|
|
1482
|
+
``coord merge`` retries. On failure: marked HUMAN_REQUIRED so the TUI
|
|
1483
|
+
can surface "manual resolution required", and a comment is posted on
|
|
1484
|
+
the underlying issue so the user is notified outside the TUI too.
|
|
1485
|
+
|
|
1486
|
+
*usage_limit_reason* (#1461 review finding 2): when the conflict-fix
|
|
1487
|
+
worker was killed by the account's usage limit mid-fix, it did not
|
|
1488
|
+
actually fail to resolve anything — frame the parked entry as "wait for
|
|
1489
|
+
the reset", not "manual rebase required", so the operator isn't sent
|
|
1490
|
+
chasing a defect that doesn't exist. Still lands in HUMAN_REQUIRED
|
|
1491
|
+
(rather than auto-retrying, which would just burn more of the same
|
|
1492
|
+
exhausted budget — the #1461 "do not auto-retry immediately" rule
|
|
1493
|
+
applies here too), just with an accurate message.
|
|
1494
|
+
|
|
1495
|
+
Called from both ``reconcile()`` (via mark_done/failed) and
|
|
1496
|
+
``coord notify`` (via post_transition) — both paths must trigger this
|
|
1497
|
+
so the re-enqueue fires regardless of which polling command runs first.
|
|
1498
|
+
"""
|
|
1499
|
+
from coord import merge_queue as mq # noqa: PLC0415
|
|
1500
|
+
|
|
1501
|
+
items = mq.load_queue()
|
|
1502
|
+
changed = False
|
|
1503
|
+
failed_entry: mq.QueuedMerge | None = None
|
|
1504
|
+
escalated: tuple[mq.QueuedMerge, str, str, str] | None = None
|
|
1505
|
+
for entry in items:
|
|
1506
|
+
if entry.assignment_id != parent_assignment_id:
|
|
1507
|
+
continue
|
|
1508
|
+
if succeeded:
|
|
1509
|
+
entry.state = mq.PENDING
|
|
1510
|
+
entry.error = None
|
|
1511
|
+
entry.last_attempt = None
|
|
1512
|
+
else:
|
|
1513
|
+
existing_error = entry.error or "conflict-fix failed"
|
|
1514
|
+
if usage_limit_reason:
|
|
1515
|
+
entry.state = mq.HUMAN_REQUIRED
|
|
1516
|
+
entry.error = (
|
|
1517
|
+
f"{existing_error}; conflict-fix worker was killed by "
|
|
1518
|
+
f"the account's {usage_limit_reason} — not a real "
|
|
1519
|
+
"conflict. Wait for the reset, then re-run `coord "
|
|
1520
|
+
"merge` to retry unchanged."
|
|
1521
|
+
)
|
|
1522
|
+
failed_entry = entry
|
|
1523
|
+
else:
|
|
1524
|
+
# #1291: a SEMANTIC give-up gets ONE escalated attempt from a
|
|
1525
|
+
# stronger model before the entry is parked. Everything
|
|
1526
|
+
# else — and a second semantic failure (the escalated
|
|
1527
|
+
# attempt is itself a conflict-fix row, so
|
|
1528
|
+
# `has_prior_semantic_escalation` blocks it) — behaves
|
|
1529
|
+
# exactly as before.
|
|
1530
|
+
fix = (
|
|
1531
|
+
_try_semantic_escalation(
|
|
1532
|
+
entry,
|
|
1533
|
+
board=board,
|
|
1534
|
+
config=config,
|
|
1535
|
+
machine_name=machine_name,
|
|
1536
|
+
stuck_summary=stuck_summary,
|
|
1537
|
+
)
|
|
1538
|
+
if semantic
|
|
1539
|
+
else None
|
|
1540
|
+
)
|
|
1541
|
+
if fix is not None:
|
|
1542
|
+
# Stay in CONFLICT, not HUMAN_REQUIRED — the escalated
|
|
1543
|
+
# worker is in flight. If it fails, this hook runs
|
|
1544
|
+
# again and the escalation guard sends the entry to
|
|
1545
|
+
# HUMAN_REQUIRED.
|
|
1546
|
+
entry.state = mq.CONFLICT
|
|
1547
|
+
model = fix.model or "escalated model"
|
|
1548
|
+
entry.error = (
|
|
1549
|
+
f"{existing_error}; semantic conflict escalated to "
|
|
1550
|
+
f"{model} (assignment {fix.assignment_id}) — review "
|
|
1551
|
+
"the resolution diff before merge."
|
|
1552
|
+
)
|
|
1553
|
+
escalated = (
|
|
1554
|
+
entry, model, fix.assignment_id or "",
|
|
1555
|
+
fix.machine_name or "",
|
|
1556
|
+
)
|
|
1557
|
+
else:
|
|
1558
|
+
entry.state = mq.HUMAN_REQUIRED
|
|
1559
|
+
entry.error = (
|
|
1560
|
+
f"{existing_error}; conflict-fix worker did not "
|
|
1561
|
+
"resolve. Manual rebase required."
|
|
1562
|
+
)
|
|
1563
|
+
failed_entry = entry
|
|
1564
|
+
changed = True
|
|
1565
|
+
if changed:
|
|
1566
|
+
mq.save_queue(items)
|
|
1567
|
+
|
|
1568
|
+
if escalated is not None:
|
|
1569
|
+
esc_entry, esc_model, esc_id, esc_machine = escalated
|
|
1570
|
+
_post_semantic_escalation_comment(
|
|
1571
|
+
esc_entry,
|
|
1572
|
+
model=esc_model,
|
|
1573
|
+
escalated_assignment_id=esc_id,
|
|
1574
|
+
machine_name=esc_machine,
|
|
1575
|
+
)
|
|
1576
|
+
|
|
1577
|
+
if failed_entry is not None:
|
|
1578
|
+
_post_human_required_comment_raw(
|
|
1579
|
+
entry=failed_entry,
|
|
1580
|
+
fix_assignment_id=fix_assignment_id,
|
|
1581
|
+
machine_name=machine_name,
|
|
1582
|
+
)
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def _on_conflict_fix_done(
|
|
1586
|
+
fix_assignment: Assignment,
|
|
1587
|
+
*,
|
|
1588
|
+
succeeded: bool,
|
|
1589
|
+
agent_entry: dict | None = None,
|
|
1590
|
+
board: Board | None = None,
|
|
1591
|
+
config: Config | None = None,
|
|
1592
|
+
) -> None:
|
|
1593
|
+
"""Thin wrapper used by the reconcile() loop.
|
|
1594
|
+
|
|
1595
|
+
On failure it also asks the worker's log whether the give-up was a
|
|
1596
|
+
SEMANTIC conflict (the ``coord:conflict=semantic`` marker), which — with
|
|
1597
|
+
``pipeline.escalate_semantic_conflicts`` on — buys one escalated attempt
|
|
1598
|
+
instead of an immediate HUMAN_REQUIRED (#1291).
|
|
1599
|
+
|
|
1600
|
+
#1461 review finding 2: when the conflict-fix worker was itself killed
|
|
1601
|
+
by the account's usage limit (flagged on *agent_entry* by
|
|
1602
|
+
``AgentServer._reap`` — the same signal ``_record_usage_limit_reason``
|
|
1603
|
+
stamps onto ordinary work assignments), it didn't actually fail to
|
|
1604
|
+
resolve anything. Skip the SEMANTIC-conflict check (there is nothing to
|
|
1605
|
+
diagnose in the transcript — it was cut off, not concluded) and pass the
|
|
1606
|
+
reason through so the parked entry gets an accurate message instead of
|
|
1607
|
+
"manual rebase required".
|
|
1608
|
+
"""
|
|
1609
|
+
parent_id = fix_assignment.review_of_assignment_id
|
|
1610
|
+
if not parent_id:
|
|
1611
|
+
return
|
|
1612
|
+
|
|
1613
|
+
usage_limit_reason = (agent_entry or {}).get("usage_limit_reason")
|
|
1614
|
+
|
|
1615
|
+
semantic = False
|
|
1616
|
+
stuck_summary: str | None = None
|
|
1617
|
+
if (
|
|
1618
|
+
not succeeded
|
|
1619
|
+
and not usage_limit_reason
|
|
1620
|
+
and board is not None
|
|
1621
|
+
and config is not None
|
|
1622
|
+
):
|
|
1623
|
+
semantic, stuck_summary = _semantic_verdict(
|
|
1624
|
+
fix_assignment, agent_entry, config,
|
|
1625
|
+
)
|
|
1626
|
+
|
|
1627
|
+
on_conflict_fix_done(
|
|
1628
|
+
parent_assignment_id=parent_id,
|
|
1629
|
+
fix_assignment_id=fix_assignment.assignment_id or "",
|
|
1630
|
+
machine_name=fix_assignment.machine_name or "",
|
|
1631
|
+
succeeded=succeeded,
|
|
1632
|
+
semantic=semantic,
|
|
1633
|
+
board=board,
|
|
1634
|
+
config=config,
|
|
1635
|
+
stuck_summary=stuck_summary,
|
|
1636
|
+
usage_limit_reason=usage_limit_reason,
|
|
1637
|
+
)
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
def _semantic_verdict(
|
|
1641
|
+
fix_assignment: Assignment,
|
|
1642
|
+
agent_entry: dict | None,
|
|
1643
|
+
config: Config,
|
|
1644
|
+
) -> tuple[bool, str | None]:
|
|
1645
|
+
"""(is_semantic, stuck line) for a finished conflict-fix worker.
|
|
1646
|
+
|
|
1647
|
+
Best-effort — any failure to read the log means "not semantic", which
|
|
1648
|
+
preserves the pre-#1291 HUMAN_REQUIRED path.
|
|
1649
|
+
"""
|
|
1650
|
+
from coord.conflict_fix import detect_semantic_conflict # noqa: PLC0415
|
|
1651
|
+
|
|
1652
|
+
log_path = (agent_entry or {}).get("log_path")
|
|
1653
|
+
machine = next(
|
|
1654
|
+
(m for m in config.machines if m.name == fix_assignment.machine_name), None,
|
|
1655
|
+
)
|
|
1656
|
+
try:
|
|
1657
|
+
semantic = detect_semantic_conflict(
|
|
1658
|
+
log_path=log_path,
|
|
1659
|
+
host=machine.host if machine is not None else None,
|
|
1660
|
+
assignment_id=fix_assignment.assignment_id,
|
|
1661
|
+
)
|
|
1662
|
+
except Exception: # noqa: BLE001 — never break reconcile on a log read
|
|
1663
|
+
return False, None
|
|
1664
|
+
|
|
1665
|
+
stuck_summary: str | None = None
|
|
1666
|
+
if semantic:
|
|
1667
|
+
progress = (agent_entry or {}).get("progress") or {}
|
|
1668
|
+
stuck_summary = progress.get("stuck")
|
|
1669
|
+
if not stuck_summary and log_path:
|
|
1670
|
+
try:
|
|
1671
|
+
from coord.progress import parse_progress # noqa: PLC0415
|
|
1672
|
+
stuck_summary = parse_progress(log_path).stuck
|
|
1673
|
+
except Exception: # noqa: BLE001
|
|
1674
|
+
stuck_summary = None
|
|
1675
|
+
return semantic, stuck_summary
|
|
1676
|
+
|
|
1677
|
+
|
|
1678
|
+
def _extract_issue_number(branch: str) -> int | None:
|
|
1679
|
+
"""Extract N from ``issue-{N}-*`` branch names; returns None if no match."""
|
|
1680
|
+
m = re.match(r"issue-(\d+)-", branch)
|
|
1681
|
+
return int(m.group(1)) if m else None
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
def close_stale_prs(
|
|
1685
|
+
config: Config,
|
|
1686
|
+
*,
|
|
1687
|
+
repo: str | None = None,
|
|
1688
|
+
issue: int | None = None,
|
|
1689
|
+
dry_run: bool = False,
|
|
1690
|
+
) -> list[str]:
|
|
1691
|
+
"""Close open PRs whose work is already on main or whose issue is closed.
|
|
1692
|
+
|
|
1693
|
+
Sweeps every coord-tracked repo (filtered by *repo* / *issue* when given)
|
|
1694
|
+
for OPEN PRs with ``issue-{N}-*`` head branches. Each PR is classified as
|
|
1695
|
+
stale when either condition holds:
|
|
1696
|
+
|
|
1697
|
+
1. The linked issue N is CLOSED on GitHub.
|
|
1698
|
+
2. The branch has 0 commits ahead of the repo's default branch (catches
|
|
1699
|
+
fast-forward merges; squash/rebase cases are caught by condition 1
|
|
1700
|
+
because coord closes the issue when squash-merging).
|
|
1701
|
+
|
|
1702
|
+
Stale PRs are closed with an explanatory comment. Non-stale PRs are left
|
|
1703
|
+
untouched. *dry_run* lists what would change without writing. Idempotent.
|
|
1704
|
+
"""
|
|
1705
|
+
from coord import github_ops # noqa: PLC0415
|
|
1706
|
+
|
|
1707
|
+
actions: list[str] = []
|
|
1708
|
+
|
|
1709
|
+
for repo_cfg in config.repos:
|
|
1710
|
+
if repo is not None and repo_cfg.name != repo:
|
|
1711
|
+
continue
|
|
1712
|
+
|
|
1713
|
+
try:
|
|
1714
|
+
open_prs = github_ops.list_open_prs(repo_cfg.github)
|
|
1715
|
+
except Exception as exc: # noqa: BLE001
|
|
1716
|
+
actions.append(
|
|
1717
|
+
f"skip stale-PR sweep for {repo_cfg.name}: could not list PRs ({exc})"
|
|
1718
|
+
)
|
|
1719
|
+
continue
|
|
1720
|
+
|
|
1721
|
+
default_branch = repo_cfg.default_branch or "main"
|
|
1722
|
+
# #934: per-run cache for the issue -> milestone-number lookup, since
|
|
1723
|
+
# this loop re-derives the base branch per-PR below (a milestone
|
|
1724
|
+
# issue's stale-PR base is `feature/ms-NN`, not the repo's flat
|
|
1725
|
+
# `default_branch`). Only populated when the repo opted in.
|
|
1726
|
+
milestone_cache: dict = {}
|
|
1727
|
+
|
|
1728
|
+
for pr in open_prs:
|
|
1729
|
+
branch = pr.get("headRefName") or ""
|
|
1730
|
+
pr_number = pr.get("number")
|
|
1731
|
+
if not branch or pr_number is None:
|
|
1732
|
+
continue
|
|
1733
|
+
|
|
1734
|
+
issue_number = _extract_issue_number(branch)
|
|
1735
|
+
if issue_number is None:
|
|
1736
|
+
continue # not a coord-managed branch — skip
|
|
1737
|
+
if issue is not None and issue_number != issue:
|
|
1738
|
+
continue
|
|
1739
|
+
|
|
1740
|
+
# Fail-safe classification: when uncertain, leave the PR open.
|
|
1741
|
+
stale_reason: str | None = None
|
|
1742
|
+
|
|
1743
|
+
# #934: this issue's actual base — `feature/ms-NN` when it
|
|
1744
|
+
# belongs to a milestone and the repo opted into the git model,
|
|
1745
|
+
# `repo_cfg.default_branch` (today's behavior) otherwise. The
|
|
1746
|
+
# milestone lookup itself is skipped (no `gh` call) when the
|
|
1747
|
+
# repo hasn't opted in.
|
|
1748
|
+
pr_base = default_branch
|
|
1749
|
+
if getattr(repo_cfg, "develop_branch", None):
|
|
1750
|
+
from coord.branch_model import ( # noqa: PLC0415
|
|
1751
|
+
fetch_issue_milestone_number,
|
|
1752
|
+
resolve_base_branch,
|
|
1753
|
+
)
|
|
1754
|
+
|
|
1755
|
+
milestone_number = fetch_issue_milestone_number(
|
|
1756
|
+
repo_cfg.github, issue_number, cache=milestone_cache,
|
|
1757
|
+
)
|
|
1758
|
+
pr_base = resolve_base_branch(repo_cfg, milestone_number)
|
|
1759
|
+
|
|
1760
|
+
if github_ops.issue_is_closed(repo_cfg.github, issue_number):
|
|
1761
|
+
stale_reason = f"issue #{issue_number} is closed"
|
|
1762
|
+
elif github_ops.branch_is_fully_merged(
|
|
1763
|
+
repo_cfg.github, branch, pr_base
|
|
1764
|
+
):
|
|
1765
|
+
stale_reason = f"all commits already on {pr_base}"
|
|
1766
|
+
|
|
1767
|
+
if stale_reason is None:
|
|
1768
|
+
continue # live PR — leave it alone
|
|
1769
|
+
|
|
1770
|
+
actions.append(
|
|
1771
|
+
f"close PR #{pr_number} "
|
|
1772
|
+
f"({repo_cfg.name} #{issue_number}, {branch}): {stale_reason}"
|
|
1773
|
+
+ (" [dry-run]" if dry_run else "")
|
|
1774
|
+
)
|
|
1775
|
+
|
|
1776
|
+
if not dry_run:
|
|
1777
|
+
comment = (
|
|
1778
|
+
f"Closing stale PR — {stale_reason}. "
|
|
1779
|
+
f"The work for issue #{issue_number} has already landed.\n\n"
|
|
1780
|
+
f"<!-- coord:stale-close issue={issue_number} -->"
|
|
1781
|
+
)
|
|
1782
|
+
try:
|
|
1783
|
+
github_ops.close_pr(repo_cfg.github, pr_number, comment=comment)
|
|
1784
|
+
except Exception as exc: # noqa: BLE001
|
|
1785
|
+
actions.append(f" ↳ error closing PR #{pr_number}: {exc}")
|
|
1786
|
+
|
|
1787
|
+
return actions
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def is_interactive_merge_session(a: object) -> bool:
|
|
1791
|
+
"""True when *a* is an interactive ``--merge-of`` session (#1110).
|
|
1792
|
+
|
|
1793
|
+
Interactive merge-prep sessions are dispatched with ``type="conflict-fix"``
|
|
1794
|
+
— the same type the automated #241 conflict-fix worker uses — so a bare
|
|
1795
|
+
``type`` check can't tell them apart. What distinguishes them:
|
|
1796
|
+
|
|
1797
|
+
* ``provider_name == "claude-pty"`` — the automated #241 worker runs
|
|
1798
|
+
headless ``claude -p`` and never sets this.
|
|
1799
|
+
* ``review_of_assignment_id`` is set — both share this, but combined with
|
|
1800
|
+
the provider check above it's unambiguous.
|
|
1801
|
+
|
|
1802
|
+
Used by :func:`reconcile_board_merges` (sweep b) and
|
|
1803
|
+
:func:`coord.serve_app._reap_merged_sessions_tick` to scope terminal-state
|
|
1804
|
+
detection / reaping to interactive merge sessions only, without touching
|
|
1805
|
+
automated conflict-fix workers or ordinary work/review/smoke rows.
|
|
1806
|
+
"""
|
|
1807
|
+
return (
|
|
1808
|
+
getattr(a, "type", None) == "conflict-fix"
|
|
1809
|
+
and getattr(a, "provider_name", None) == "claude-pty"
|
|
1810
|
+
and getattr(a, "review_of_assignment_id", None) is not None
|
|
1811
|
+
)
|
|
1812
|
+
|
|
1813
|
+
|
|
1814
|
+
def reconcile_board_merges(
|
|
1815
|
+
board: Board,
|
|
1816
|
+
config: Config,
|
|
1817
|
+
*,
|
|
1818
|
+
repo: str | None = None,
|
|
1819
|
+
issue: int | None = None,
|
|
1820
|
+
dry_run: bool = False,
|
|
1821
|
+
) -> list[str]:
|
|
1822
|
+
"""Reconcile done work assignments against git/GitHub reality.
|
|
1823
|
+
|
|
1824
|
+
Two conservative sweeps, returning a list of human-readable action (and
|
|
1825
|
+
skip) strings:
|
|
1826
|
+
|
|
1827
|
+
(a) #611/#1083 branch backfill — runs over ``status='done'`` rows whose
|
|
1828
|
+
``type`` is in :data:`coord.models.WORK_LIKE_TYPES` (``work``,
|
|
1829
|
+
``mock-author``, ``test-author``). A remote interactive work session
|
|
1830
|
+
(or a headless ``test-author``/``mock-author`` session finalized by
|
|
1831
|
+
the #625 passive reconcile tick before its branch was known — #1083)
|
|
1832
|
+
can finish ``status=done`` with ``branch=None`` even though it pushed
|
|
1833
|
+
``issue-{N}-*`` to origin, which greys the TUI Start review/test/merge
|
|
1834
|
+
buttons (they require a done work assignment WITH a branch) and makes
|
|
1835
|
+
``coord pr <aid>`` refuse outright. When exactly one remote branch
|
|
1836
|
+
matches ``issue-{N}-*`` for the issue, the branch is backfilled via
|
|
1837
|
+
:func:`state.update_assignment_branch`. More than one candidate (or
|
|
1838
|
+
none) is left untouched and logged. #1574: sweep (b) below now shares
|
|
1839
|
+
this same ``WORK_LIKE_TYPES`` scope (plus interactive merge sessions,
|
|
1840
|
+
#1110) — a landed branch is a landed branch regardless of which
|
|
1841
|
+
work-like type authored it; only ``type='review'`` (and other
|
|
1842
|
+
non-work-like types) stay out of scope for the terminal-merge check.
|
|
1843
|
+
|
|
1844
|
+
(b) #609/#951 record out-of-band merges — work merged directly on GitHub,
|
|
1845
|
+
or a ``merge_queue`` row that drained without flipping the board, is
|
|
1846
|
+
never recorded as ``status='merged'`` so the TUI shows a grey merge
|
|
1847
|
+
box forever. When :func:`github_ops.work_is_terminal` reports the
|
|
1848
|
+
issue closed OR the PR merged (fail-open), the row is flipped via
|
|
1849
|
+
:func:`state.mark_assignment_merged`. ``work_is_terminal``'s
|
|
1850
|
+
issue-closed check needs **no branch**, so this still fires even when
|
|
1851
|
+
sweep (a)'s backfill couldn't resolve one (#951) — an unresolved
|
|
1852
|
+
branch must not block the issue-closed fast path. Because every
|
|
1853
|
+
finished work assignment defaults to ``review_state='pending'``
|
|
1854
|
+
(reconcile's own Pass 1 sets it unconditionally so the review-dispatch
|
|
1855
|
+
loop can pick it up), flipping ``status`` alone leaves that ghost
|
|
1856
|
+
behind — the row keeps surfacing as "[awaiting review]" forever even
|
|
1857
|
+
though it's merged. So this sweep also clears a lingering
|
|
1858
|
+
``review_state='pending'`` via :func:`state.mark_work_review_settled`
|
|
1859
|
+
(#951), mirroring how sweep (e) below settles the sibling
|
|
1860
|
+
review/smoke/conflict-fix rows. This only reaches rows still carrying
|
|
1861
|
+
``status='done'`` — a row whose ``status`` already flipped to
|
|
1862
|
+
``'merged'`` in a *prior* reconcile run permanently drops out of this
|
|
1863
|
+
sweep's candidate list, so sweep (e) below also matches
|
|
1864
|
+
``type='work' status='merged' review_state='pending'`` to catch those
|
|
1865
|
+
(#951 round 2).
|
|
1866
|
+
|
|
1867
|
+
Both sweeps are **conservative**: they never act when uncertain and append a
|
|
1868
|
+
skip reason instead. *repo* filters to a single local repo name. When
|
|
1869
|
+
*dry_run* is True no writes happen (no ``state.update_*`` calls) — the
|
|
1870
|
+
actions list still describes what *would* change. The board objects are
|
|
1871
|
+
mutated in place on a real run so a subsequent ``save_board`` agrees with
|
|
1872
|
+
the targeted DB writes.
|
|
1873
|
+
"""
|
|
1874
|
+
from coord import github_ops, state # noqa: PLC0415
|
|
1875
|
+
|
|
1876
|
+
actions: list[str] = []
|
|
1877
|
+
terminal_cache: dict = {}
|
|
1878
|
+
# One remote-branch listing per repo, fetched lazily and reused.
|
|
1879
|
+
branches_by_repo: dict[str, set[str]] = {}
|
|
1880
|
+
|
|
1881
|
+
candidates = [
|
|
1882
|
+
a
|
|
1883
|
+
for a in board.active + board.completed
|
|
1884
|
+
if (a.type in WORK_LIKE_TYPES or is_interactive_merge_session(a))
|
|
1885
|
+
and a.status == "done"
|
|
1886
|
+
and (repo is None or a.repo_name == repo)
|
|
1887
|
+
and (issue is None or a.issue_number == issue)
|
|
1888
|
+
]
|
|
1889
|
+
|
|
1890
|
+
for a in candidates:
|
|
1891
|
+
repo_cfg = config.repo(a.repo_name)
|
|
1892
|
+
if repo_cfg is None:
|
|
1893
|
+
actions.append(
|
|
1894
|
+
f"skip {a.assignment_id} ({a.repo_name} #{a.issue_number}): "
|
|
1895
|
+
"repo not in config"
|
|
1896
|
+
)
|
|
1897
|
+
continue
|
|
1898
|
+
|
|
1899
|
+
# (a) #611/#1083 — backfill a missing branch from origin.
|
|
1900
|
+
if not a.branch:
|
|
1901
|
+
if repo_cfg.github not in branches_by_repo:
|
|
1902
|
+
branches_by_repo[repo_cfg.github] = (
|
|
1903
|
+
github_ops.list_remote_branch_names(repo_cfg.github)
|
|
1904
|
+
)
|
|
1905
|
+
prefix = f"issue-{a.issue_number}-"
|
|
1906
|
+
matches = sorted(
|
|
1907
|
+
name
|
|
1908
|
+
for name in branches_by_repo[repo_cfg.github]
|
|
1909
|
+
if name.startswith(prefix)
|
|
1910
|
+
)
|
|
1911
|
+
if len(matches) == 1:
|
|
1912
|
+
branch = matches[0]
|
|
1913
|
+
actions.append(
|
|
1914
|
+
f"backfill branch {a.assignment_id} "
|
|
1915
|
+
f"({a.repo_name} #{a.issue_number}) -> {branch}"
|
|
1916
|
+
+ (" [dry-run]" if dry_run else "")
|
|
1917
|
+
)
|
|
1918
|
+
if not dry_run:
|
|
1919
|
+
a.branch = branch
|
|
1920
|
+
state.update_assignment_branch(a.assignment_id or "", branch)
|
|
1921
|
+
elif len(matches) > 1:
|
|
1922
|
+
actions.append(
|
|
1923
|
+
f"skip backfill {a.assignment_id} "
|
|
1924
|
+
f"({a.repo_name} #{a.issue_number}): "
|
|
1925
|
+
f"{len(matches)} ambiguous branch candidates {matches}"
|
|
1926
|
+
)
|
|
1927
|
+
# #951: do NOT bail out here — a.branch is still None, but the
|
|
1928
|
+
# issue-closed fast path below needs no branch, so give it a
|
|
1929
|
+
# chance instead of stranding the row forever.
|
|
1930
|
+
else:
|
|
1931
|
+
actions.append(
|
|
1932
|
+
f"skip backfill {a.assignment_id} "
|
|
1933
|
+
f"({a.repo_name} #{a.issue_number}): "
|
|
1934
|
+
f"no remote branch matching {prefix}*"
|
|
1935
|
+
)
|
|
1936
|
+
# #951: same — fall through rather than `continue`.
|
|
1937
|
+
|
|
1938
|
+
# (b) #609/#951 — flip done work whose branch is merged on GitHub, OR
|
|
1939
|
+
# whose issue is closed even when no branch could be resolved above
|
|
1940
|
+
# (work_is_terminal's issue-closed check needs no branch). #1083
|
|
1941
|
+
# originally scoped this to type='work' only — test-author rows were
|
|
1942
|
+
# added to `candidates` above for sweep (a)'s branch backfill alone,
|
|
1943
|
+
# with the merged/review-settled semantics here deliberately left
|
|
1944
|
+
# out of scope. #1574: that scope limit meant a `type='test-author'`
|
|
1945
|
+
# row (every oracle-loop acceptance slice, and by the same token
|
|
1946
|
+
# `type='mock-author'`, #930 Gate A) could never reach `status=
|
|
1947
|
+
# 'merged'` no matter how completely its branch landed, since
|
|
1948
|
+
# `work_is_terminal` — branch/commit-scoped since #1150 — already
|
|
1949
|
+
# answers correctly for these rows too. There's nothing pipeline-
|
|
1950
|
+
# specific about "this branch merged"; widened to the same
|
|
1951
|
+
# :data:`coord.models.WORK_LIKE_TYPES` set sweep (a) uses. #1110:
|
|
1952
|
+
# interactive merge sessions (type='conflict-fix',
|
|
1953
|
+
# provider_name='claude-pty', review_of_assignment_id set — see
|
|
1954
|
+
# :func:`is_interactive_merge_session`) reach 'done' the same way work
|
|
1955
|
+
# sessions do, so they get the same terminal-detection sweep so the
|
|
1956
|
+
# auto-reaper can pick them up. Automated #241 conflict-fix workers
|
|
1957
|
+
# are deliberately excluded (they never set provider_name='claude-pty').
|
|
1958
|
+
# `type='review'` rows never reach this point at all — they aren't in
|
|
1959
|
+
# `candidates` (sweep (a) above is also WORK_LIKE_TYPES-scoped).
|
|
1960
|
+
if (
|
|
1961
|
+
a.type in WORK_LIKE_TYPES or is_interactive_merge_session(a)
|
|
1962
|
+
) and github_ops.work_is_terminal(
|
|
1963
|
+
repo_cfg.github, a.issue_number, a.branch, cache=terminal_cache
|
|
1964
|
+
):
|
|
1965
|
+
actions.append(
|
|
1966
|
+
f"mark merged {a.assignment_id} "
|
|
1967
|
+
f"({a.repo_name} #{a.issue_number}, {a.branch or 'no branch'})"
|
|
1968
|
+
+ (" [dry-run]" if dry_run else "")
|
|
1969
|
+
)
|
|
1970
|
+
if not dry_run:
|
|
1971
|
+
a.status = "merged"
|
|
1972
|
+
state.mark_assignment_merged(a.assignment_id or "")
|
|
1973
|
+
# #951: mark_assignment_merged only flips status — clear a
|
|
1974
|
+
# lingering review_state='pending' ghost too, or the row keeps
|
|
1975
|
+
# showing "[awaiting review]" forever despite being merged.
|
|
1976
|
+
# #1574: kept ``type == "work"``-only (not widened to
|
|
1977
|
+
# WORK_LIKE_TYPES like the status flip above) — a
|
|
1978
|
+
# test-author/mock-author row's review_state is exactly what
|
|
1979
|
+
# sweep (f)'s #1180 wedged-review repair polices (a stray
|
|
1980
|
+
# review_state='done' with no real review behind it), and
|
|
1981
|
+
# settling it to 'done' here would immediately be flagged as
|
|
1982
|
+
# wedged and reset back to 'pending' by that sweep, an
|
|
1983
|
+
# unhelpful churn this fix doesn't need to introduce. Only
|
|
1984
|
+
# `is_interactive_merge_session` rows share sweep (b)'s
|
|
1985
|
+
# type='work' review-settle path, same as before #1574.
|
|
1986
|
+
if a.review_state == "pending" and (
|
|
1987
|
+
a.type == "work" or is_interactive_merge_session(a)
|
|
1988
|
+
):
|
|
1989
|
+
a.review_state = "done"
|
|
1990
|
+
state.mark_work_review_settled(a.assignment_id or "")
|
|
1991
|
+
|
|
1992
|
+
# (c) #721 — close open PRs whose work has already landed.
|
|
1993
|
+
actions.extend(close_stale_prs(config, repo=repo, issue=issue, dry_run=dry_run))
|
|
1994
|
+
|
|
1995
|
+
# (d) #732 — prune stale merge_queue entries for closed issues / merged PRs.
|
|
1996
|
+
# Runs after the board sweeps so a just-marked-merged assignment doesn't
|
|
1997
|
+
# also appear as a pruned queue entry in the same reconcile run.
|
|
1998
|
+
# repo/issue filters don't apply here — we always scan the full queue, since
|
|
1999
|
+
# a stale entry affects every `coord merge` run regardless of --repo.
|
|
2000
|
+
from coord import merge_queue as mq # noqa: PLC0415
|
|
2001
|
+
|
|
2002
|
+
pruned = mq.prune_stale_queue_entries(dry_run=dry_run)
|
|
2003
|
+
for entry in pruned:
|
|
2004
|
+
actions.append(
|
|
2005
|
+
f"prune queue entry {entry.assignment_id} "
|
|
2006
|
+
f"({entry.repo_name} #{entry.issue_number}, state={entry.state})"
|
|
2007
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2008
|
+
)
|
|
2009
|
+
|
|
2010
|
+
# (e) #894/#951 — settle sibling ghost rows for terminal issues.
|
|
2011
|
+
#
|
|
2012
|
+
# The #609 sweep (b) only processes type='work' status='done' rows, so it
|
|
2013
|
+
# misses three classes of lingering ghost rows for already-merged/closed issues:
|
|
2014
|
+
#
|
|
2015
|
+
# * type=review/smoke/conflict-fix rows whose status='done' but
|
|
2016
|
+
# review_state='pending' — the interactive-completion path
|
|
2017
|
+
# (issue_store._update_local_state) sets review_state='pending' on ALL
|
|
2018
|
+
# completed assignments so reconcile picks them up like claude -p workers.
|
|
2019
|
+
# When the parent issue closes before that handoff fires, these rows
|
|
2020
|
+
# surface as "awaiting review" in coord status / the TUI forever.
|
|
2021
|
+
#
|
|
2022
|
+
# * status='advisory' rows (any type) — the #609 candidates filter requires
|
|
2023
|
+
# status='done', so advisory rows are never reached. They linger in the
|
|
2024
|
+
# TUI's advisory view after the issue is terminal.
|
|
2025
|
+
#
|
|
2026
|
+
# * type='work' rows whose status is ALREADY 'merged' but review_state is
|
|
2027
|
+
# still 'pending' (#951) — once `mark_assignment_merged` (#609) flips a
|
|
2028
|
+
# row's status to 'merged' (in this run's sweep (b) above, or in a prior
|
|
2029
|
+
# reconcile run), it permanently drops out of sweep (b)'s
|
|
2030
|
+
# status=='done' candidates list on every future pass, so a
|
|
2031
|
+
# review_state='pending' ghost left on it (from #609 predating the
|
|
2032
|
+
# review_state clear added above, or any other stale write) is never
|
|
2033
|
+
# revisited. `state.mark_work_review_settled` already handles
|
|
2034
|
+
# status='merged' rows fine (no status gate) — the gap was purely that
|
|
2035
|
+
# reconcile() never called it for a row outside sweep (b)'s candidate
|
|
2036
|
+
# set. This class fixes that: it directly matches the bug report's
|
|
2037
|
+
# scenario of already-merged+closed issues stuck "awaiting review".
|
|
2038
|
+
#
|
|
2039
|
+
# This sweep is conservative and fail-open:
|
|
2040
|
+
# - Only acts when work_is_terminal(...) is confirmed true.
|
|
2041
|
+
# - Uses the terminal_cache populated by sweep (b) to avoid extra GH calls;
|
|
2042
|
+
# falls back to a fresh check (still fail-open) for ghost rows whose issue
|
|
2043
|
+
# wasn't processed in sweep (b) (e.g. work already merged in a prior run).
|
|
2044
|
+
# - Respects the repo/issue filter so --repo/--issue scopes apply.
|
|
2045
|
+
# - Terminality is keyed on issue_is_closed OR pr_is_merged — NOT branch
|
|
2046
|
+
# ancestry, so rebase/squash merges with new SHAs are correctly handled.
|
|
2047
|
+
|
|
2048
|
+
# Build a (repo_name, issue_number) → branch lookup from all work rows so
|
|
2049
|
+
# that sibling rows lacking a branch can still pass a branch to work_is_terminal
|
|
2050
|
+
# (enabling the pr_is_merged fast-path in addition to issue_is_closed).
|
|
2051
|
+
work_branch_for: dict[tuple[str, int], str | None] = {}
|
|
2052
|
+
for _a in board.active + board.completed:
|
|
2053
|
+
if _a.type == "work" and _a.issue_number is not None:
|
|
2054
|
+
key = (_a.repo_name, _a.issue_number)
|
|
2055
|
+
# Prefer a non-None branch; first seen wins (done rows come before
|
|
2056
|
+
# merged rows in board.completed, but any non-None branch is fine).
|
|
2057
|
+
if key not in work_branch_for or work_branch_for[key] is None:
|
|
2058
|
+
work_branch_for[key] = _a.branch
|
|
2059
|
+
|
|
2060
|
+
# Identify ghost sibling rows subject to this sweep.
|
|
2061
|
+
ghost_candidates = [
|
|
2062
|
+
a
|
|
2063
|
+
for a in board.active + board.completed
|
|
2064
|
+
if (
|
|
2065
|
+
(
|
|
2066
|
+
a.type in ("review", "smoke", "conflict-fix")
|
|
2067
|
+
and a.status == "done"
|
|
2068
|
+
and a.review_state == "pending"
|
|
2069
|
+
)
|
|
2070
|
+
or a.status == "advisory"
|
|
2071
|
+
or (
|
|
2072
|
+
a.type == "work"
|
|
2073
|
+
and a.status == "merged"
|
|
2074
|
+
and a.review_state == "pending"
|
|
2075
|
+
)
|
|
2076
|
+
)
|
|
2077
|
+
and (repo is None or a.repo_name == repo)
|
|
2078
|
+
and (issue is None or a.issue_number == issue)
|
|
2079
|
+
]
|
|
2080
|
+
|
|
2081
|
+
for a in ghost_candidates:
|
|
2082
|
+
repo_cfg = config.repo(a.repo_name)
|
|
2083
|
+
if repo_cfg is None:
|
|
2084
|
+
actions.append(
|
|
2085
|
+
f"skip settle {a.assignment_id} "
|
|
2086
|
+
f"({a.repo_name} #{a.issue_number}): repo not in config"
|
|
2087
|
+
)
|
|
2088
|
+
continue
|
|
2089
|
+
|
|
2090
|
+
# Resolve the best available branch for the terminality probe. The
|
|
2091
|
+
# sibling row itself may carry a branch; fall back to the work row's
|
|
2092
|
+
# branch so the pr_is_merged check fires even when the sibling has none.
|
|
2093
|
+
branch = a.branch or work_branch_for.get((a.repo_name, a.issue_number))
|
|
2094
|
+
|
|
2095
|
+
if not github_ops.work_is_terminal(
|
|
2096
|
+
repo_cfg.github, a.issue_number, branch, cache=terminal_cache
|
|
2097
|
+
):
|
|
2098
|
+
continue # Issue still live — leave this row alone.
|
|
2099
|
+
|
|
2100
|
+
if a.status == "advisory":
|
|
2101
|
+
actions.append(
|
|
2102
|
+
f"settle advisory {a.assignment_id} "
|
|
2103
|
+
f"({a.repo_name} #{a.issue_number})"
|
|
2104
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2105
|
+
)
|
|
2106
|
+
if not dry_run:
|
|
2107
|
+
a.status = "merged"
|
|
2108
|
+
state.mark_advisory_settled(a.assignment_id or "")
|
|
2109
|
+
elif a.type == "work":
|
|
2110
|
+
# #951: type=work, status=merged, review_state=pending — a row
|
|
2111
|
+
# that already fell out of sweep (b)'s status=='done' candidates
|
|
2112
|
+
# in a prior run (or earlier in this run) but still carries a
|
|
2113
|
+
# review_state ghost. status is already 'merged', so only the
|
|
2114
|
+
# review_state needs settling.
|
|
2115
|
+
actions.append(
|
|
2116
|
+
f"settle work review_state {a.assignment_id} "
|
|
2117
|
+
f"({a.repo_name} #{a.issue_number})"
|
|
2118
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2119
|
+
)
|
|
2120
|
+
if not dry_run:
|
|
2121
|
+
a.review_state = "done"
|
|
2122
|
+
state.mark_work_review_settled(a.assignment_id or "")
|
|
2123
|
+
else:
|
|
2124
|
+
# type=review/smoke/conflict-fix, status=done, review_state=pending
|
|
2125
|
+
actions.append(
|
|
2126
|
+
f"settle sibling {a.assignment_id} "
|
|
2127
|
+
f"({a.repo_name} #{a.issue_number}, type={a.type})"
|
|
2128
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2129
|
+
)
|
|
2130
|
+
if not dry_run:
|
|
2131
|
+
a.review_state = "done"
|
|
2132
|
+
state.mark_sibling_review_done(a.assignment_id or "")
|
|
2133
|
+
|
|
2134
|
+
# (f) #1180 — un-wedge a test-author/mock-author row whose review_state
|
|
2135
|
+
# was stamped 'done' by a `work_is_terminal` false positive (pre-#1150:
|
|
2136
|
+
# test-author assignments carry issue_number = the milestone's *tracking*
|
|
2137
|
+
# issue — the JIT-slice aliasing convention, #1142/#1150 — so a tracking
|
|
2138
|
+
# issue with ANY historical merged PR could satisfy the then-issue-only
|
|
2139
|
+
# terminal check for an unrelated, still-open slice sharing that number).
|
|
2140
|
+
# #1150 fixed the check going forward (branch/commit-scoped) but did not
|
|
2141
|
+
# repair rows it had already corrupted: a row stuck at
|
|
2142
|
+
# review_state='done' with no verdict and no type='review' assignment
|
|
2143
|
+
# ever dispatched against its branch is invisible to
|
|
2144
|
+
# dispatch_pending_reviews (only review_state in (None, 'pending') is
|
|
2145
|
+
# eligible) AND to the merge gate (requires a real approved type='review'
|
|
2146
|
+
# row) — a permanent deadlock between the two subsystems. Reset
|
|
2147
|
+
# review_state -> 'pending' so the (now-fixed) auto-loop retries a real
|
|
2148
|
+
# review. This is safe even if the branch genuinely IS terminal by now:
|
|
2149
|
+
# the very next dispatch_pending_reviews pass re-checks
|
|
2150
|
+
# work_is_terminal (correctly branch-scoped post-#1150) and re-settles
|
|
2151
|
+
# the row.
|
|
2152
|
+
wedged_review_candidates = [
|
|
2153
|
+
a
|
|
2154
|
+
for a in board.active + board.completed
|
|
2155
|
+
if a.type in ("test-author", "mock-author")
|
|
2156
|
+
and a.review_state == "done"
|
|
2157
|
+
and a.review_verdict is None
|
|
2158
|
+
and a.branch
|
|
2159
|
+
and (repo is None or a.repo_name == repo)
|
|
2160
|
+
and (issue is None or a.issue_number == issue)
|
|
2161
|
+
]
|
|
2162
|
+
for a in wedged_review_candidates:
|
|
2163
|
+
has_review = any(
|
|
2164
|
+
r.type == "review"
|
|
2165
|
+
and r.repo_name == a.repo_name
|
|
2166
|
+
and r.branch == a.branch
|
|
2167
|
+
# #1566: "finalizing" is a review row whose agent already
|
|
2168
|
+
# finished but whose verdict hasn't been parsed/posted by
|
|
2169
|
+
# `coord notify` yet — it must count as "has a review" here too,
|
|
2170
|
+
# or a review that lands on 'finalizing' the instant its
|
|
2171
|
+
# candidate check above resolves triggers a spurious "repair
|
|
2172
|
+
# wedged review_state ... done -> pending" and a duplicate
|
|
2173
|
+
# dispatch_pending_reviews pass while the first review is still
|
|
2174
|
+
# wrapping up.
|
|
2175
|
+
and r.status in ("done", "finalizing")
|
|
2176
|
+
for r in board.active + board.completed
|
|
2177
|
+
)
|
|
2178
|
+
if has_review:
|
|
2179
|
+
continue
|
|
2180
|
+
actions.append(
|
|
2181
|
+
f"repair wedged review_state {a.assignment_id} "
|
|
2182
|
+
f"({a.repo_name} #{a.issue_number}, branch={a.branch}): "
|
|
2183
|
+
"done -> pending (#1180)"
|
|
2184
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2185
|
+
)
|
|
2186
|
+
if not dry_run:
|
|
2187
|
+
a.review_state = "pending"
|
|
2188
|
+
state.reset_wedged_test_author_review(a.assignment_id or "")
|
|
2189
|
+
|
|
2190
|
+
# (g) #1767 — drop drive escalations whose issue resolved out of band.
|
|
2191
|
+
#
|
|
2192
|
+
# `coord merge`'s success path (merge_queue.process()) dismisses the
|
|
2193
|
+
# escalation for the issue it just merged, but that only covers work
|
|
2194
|
+
# that landed *through* `coord merge`. Work merged directly on GitHub,
|
|
2195
|
+
# or closed without merging, never goes through that path — its
|
|
2196
|
+
# escalation (if any) would otherwise linger forever, since nothing
|
|
2197
|
+
# else ever clears one short of `coord escalate dismiss`. Measured on
|
|
2198
|
+
# the live board (#1767): four open escalations, three already stale —
|
|
2199
|
+
# PRs merged and issues closed days earlier, with the escalation the
|
|
2200
|
+
# only record that hadn't caught up.
|
|
2201
|
+
#
|
|
2202
|
+
# Conservative like every sweep above: only acts when `work_is_terminal`
|
|
2203
|
+
# confirms the issue closed or its PR merged, reusing `terminal_cache`
|
|
2204
|
+
# and the `work_branch_for` lookup built for sweep (e) so this costs no
|
|
2205
|
+
# extra `gh` calls for issues already resolved elsewhere in this run.
|
|
2206
|
+
# An escalation on a still-open, still-blocked issue is never touched,
|
|
2207
|
+
# no matter how old — age is not the signal, resolved state is.
|
|
2208
|
+
for _entry in state.list_drive_escalations(repo):
|
|
2209
|
+
_esc_repo = _entry.get("repo_name")
|
|
2210
|
+
_esc_issue = _entry.get("issue_number")
|
|
2211
|
+
if _esc_repo is None or _esc_issue is None:
|
|
2212
|
+
continue
|
|
2213
|
+
if issue is not None and _esc_issue != issue:
|
|
2214
|
+
continue
|
|
2215
|
+
_repo_cfg = config.repo(_esc_repo)
|
|
2216
|
+
if _repo_cfg is None:
|
|
2217
|
+
# Also filters out the drive-queue's own synthetic alert entry
|
|
2218
|
+
# (repo_name="(drive-queue)"), which isn't a real GitHub issue.
|
|
2219
|
+
continue
|
|
2220
|
+
_branch = work_branch_for.get((_esc_repo, _esc_issue))
|
|
2221
|
+
if not github_ops.work_is_terminal(
|
|
2222
|
+
_repo_cfg.github, _esc_issue, _branch, cache=terminal_cache
|
|
2223
|
+
):
|
|
2224
|
+
continue
|
|
2225
|
+
actions.append(
|
|
2226
|
+
f"dismiss escalation {_esc_repo} #{_esc_issue}: "
|
|
2227
|
+
"issue resolved out of band (#1767)"
|
|
2228
|
+
+ (" [dry-run]" if dry_run else "")
|
|
2229
|
+
)
|
|
2230
|
+
if not dry_run:
|
|
2231
|
+
state.dismiss_drive_escalation(_esc_repo, _esc_issue)
|
|
2232
|
+
|
|
2233
|
+
return actions
|