code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/notify.py
ADDED
|
@@ -0,0 +1,3226 @@
|
|
|
1
|
+
"""Poll agent servers and post completion/failure comments to GitHub."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from coord.models import Assignment, Board
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# Cache: machine_name → host. Populated by `run(config)` so post_transition →
|
|
19
|
+
# _try_parse_and_post_review can fetch a remote agent's log via /logs/<id>
|
|
20
|
+
# without threading the Config through every helper.
|
|
21
|
+
_AGENT_HOSTS: dict[str, str] = {}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _agent_host(machine_name: str) -> str | None:
|
|
25
|
+
return _AGENT_HOSTS.get(machine_name)
|
|
26
|
+
|
|
27
|
+
from coord import github_ops
|
|
28
|
+
from coord.comments import (
|
|
29
|
+
EVENT_ADVISORY,
|
|
30
|
+
EVENT_COMPLETION,
|
|
31
|
+
EVENT_FAILURE,
|
|
32
|
+
EVENT_LIVENESS_STALL,
|
|
33
|
+
EVENT_NEEDS_ATTENTION,
|
|
34
|
+
EVENT_PLAN,
|
|
35
|
+
EVENT_STALLED,
|
|
36
|
+
EVENT_STUCK,
|
|
37
|
+
format_liveness_stall,
|
|
38
|
+
format_needs_attention,
|
|
39
|
+
format_plan,
|
|
40
|
+
format_stalled_pipeline,
|
|
41
|
+
format_stalled_pipeline_dispatch,
|
|
42
|
+
format_stuck,
|
|
43
|
+
)
|
|
44
|
+
from coord.config import Config
|
|
45
|
+
from coord.dispatch import AGENT_PORT, post_advisory, post_completion, post_failure
|
|
46
|
+
from coord.progress import parse_progress
|
|
47
|
+
from coord.state import (
|
|
48
|
+
load_dispatched,
|
|
49
|
+
load_done_reviews_needing_post,
|
|
50
|
+
load_liveness_audit_state,
|
|
51
|
+
load_notified,
|
|
52
|
+
mark_notified,
|
|
53
|
+
mark_review_posted,
|
|
54
|
+
save_liveness_audit_state,
|
|
55
|
+
save_plan,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# #1710 inventory: kept as a direct import — `is_usage_limit_reason` is a
|
|
59
|
+
# trivial string-prefix predicate over `Assignment.failure_reason` (a
|
|
60
|
+
# coordinator-authored value stamped by `format_usage_limit_reason`, itself
|
|
61
|
+
# only ever produced by the reap path's claude-specific kill detection), not
|
|
62
|
+
# a per-provider log-format parse. Any provider's `failure_reason` would be
|
|
63
|
+
# checked the same way.
|
|
64
|
+
from coord.worker_events import is_usage_limit_reason
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class Transition:
|
|
69
|
+
assignment_id: str
|
|
70
|
+
machine_name: str
|
|
71
|
+
repo_name: str
|
|
72
|
+
issue_number: int
|
|
73
|
+
event: str # completion | failure
|
|
74
|
+
exit_code: int | None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class StuckDetection:
|
|
79
|
+
assignment_id: str
|
|
80
|
+
machine_name: str
|
|
81
|
+
repo_name: str
|
|
82
|
+
issue_number: int
|
|
83
|
+
stuck_message: str
|
|
84
|
+
log_path: str | None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _stuck_notified_key(assignment_id: str) -> str:
|
|
88
|
+
"""Notified ledger key for stuck events.
|
|
89
|
+
|
|
90
|
+
Uses a composite key so that a stuck notification does not block later
|
|
91
|
+
completion/failure notifications (which key on bare assignment_id).
|
|
92
|
+
"""
|
|
93
|
+
return f"{assignment_id}:stuck"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class NeedsAttentionDetection:
|
|
98
|
+
assignment_id: str
|
|
99
|
+
machine_name: str
|
|
100
|
+
repo_name: str
|
|
101
|
+
issue_number: int
|
|
102
|
+
reason: str # "wall_clock" | "non_convergence"
|
|
103
|
+
detail: str
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _needs_attention_notified_key(assignment_id: str) -> str:
|
|
107
|
+
"""Notified ledger key for needs-attention events (#846).
|
|
108
|
+
|
|
109
|
+
Composite key (mirrors :func:`_stuck_notified_key`) so a one-shot
|
|
110
|
+
needs-attention comment does not block later completion/failure/stuck
|
|
111
|
+
notifications, and vice versa.
|
|
112
|
+
"""
|
|
113
|
+
return f"{assignment_id}:needs-attention"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class StalledDetection:
|
|
118
|
+
"""#1441: a pipeline row whose auto-loop transition already fired once
|
|
119
|
+
but which is stuck on a precondition that landed too late for that
|
|
120
|
+
one-shot reaction to see. See :func:`detect_stalled_pipeline`."""
|
|
121
|
+
|
|
122
|
+
assignment_id: str
|
|
123
|
+
machine_name: str
|
|
124
|
+
repo_name: str
|
|
125
|
+
issue_number: int
|
|
126
|
+
reason: str # "review_request_changes_no_fix" | "review_done_no_verdict" |
|
|
127
|
+
# "done_no_review" | "approved_not_queued" | "merge_conflict_unresolved"
|
|
128
|
+
# (#1478, #1582) | "review_failed_no_verdict" (#1584)
|
|
129
|
+
detail: str
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _stalled_notified_key(assignment_id: str) -> str:
|
|
133
|
+
"""Notified ledger key for stalled-pipeline events (#1441).
|
|
134
|
+
|
|
135
|
+
Composite key (mirrors :func:`_needs_attention_notified_key`) so a
|
|
136
|
+
one-shot stalled-pipeline comment does not block later completion/
|
|
137
|
+
failure/stuck/needs-attention notifications for the same assignment_id,
|
|
138
|
+
and vice versa.
|
|
139
|
+
"""
|
|
140
|
+
return f"{assignment_id}:stalled"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass
|
|
144
|
+
class LivenessStallDetection:
|
|
145
|
+
"""#2048: N consecutive ``blocked`` verdicts from the cheap per-turn
|
|
146
|
+
liveness auditor. See :func:`detect_liveness_stall`."""
|
|
147
|
+
|
|
148
|
+
assignment_id: str
|
|
149
|
+
machine_name: str
|
|
150
|
+
repo_name: str
|
|
151
|
+
issue_number: int
|
|
152
|
+
consecutive_blocked: int
|
|
153
|
+
last_verdict: str | None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _liveness_notified_key(assignment_id: str) -> str:
|
|
157
|
+
"""Notified ledger key for liveness-stall events (#2048).
|
|
158
|
+
|
|
159
|
+
Composite key (mirrors :func:`_needs_attention_notified_key` /
|
|
160
|
+
:func:`_stalled_notified_key`) so a one-shot liveness comment does not
|
|
161
|
+
block later completion/failure/stuck/needs-attention/stalled
|
|
162
|
+
notifications for the same assignment_id, and vice versa. This exact
|
|
163
|
+
shape is also what keeps ``mark_notified``'s bare-``else`` branch from
|
|
164
|
+
ever writing ``status='failed'`` onto a real assignment row for this
|
|
165
|
+
event — see the comment in ``coord.state._mark_notified_local``.
|
|
166
|
+
"""
|
|
167
|
+
return f"{assignment_id}:liveness"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _fmt_minutes(seconds: float) -> str:
|
|
171
|
+
minutes = seconds / 60.0
|
|
172
|
+
if minutes < 1:
|
|
173
|
+
return f"{seconds:.0f}s"
|
|
174
|
+
if minutes == int(minutes):
|
|
175
|
+
return f"{int(minutes)}m"
|
|
176
|
+
return f"{minutes:.1f}m"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def attention_signal(
|
|
180
|
+
*,
|
|
181
|
+
assignment_type: str,
|
|
182
|
+
status: str | None,
|
|
183
|
+
dispatched_at: float | None,
|
|
184
|
+
review_iteration: int,
|
|
185
|
+
config: Config,
|
|
186
|
+
now: float | None = None,
|
|
187
|
+
provider_name: str | None = None,
|
|
188
|
+
review_of_assignment_id: str | None = None,
|
|
189
|
+
) -> tuple[str, str] | tuple[None, None]:
|
|
190
|
+
"""Pure #846 detection core: the two "needs attention" signals, decoupled
|
|
191
|
+
from where the assignment's fields come from.
|
|
192
|
+
|
|
193
|
+
1. **Non-convergence**: ``review_iteration >= config.pipeline.
|
|
194
|
+
convergence_rounds`` fix/review rounds without reaching a terminal
|
|
195
|
+
green test verdict + approved review. Checked first — a thrashing
|
|
196
|
+
assignment is worth flagging even if it hasn't yet cleared the
|
|
197
|
+
wall-clock threshold.
|
|
198
|
+
2. **Wall-clock**: running longer than
|
|
199
|
+
``config.pipeline.attention_threshold_for(assignment_type,
|
|
200
|
+
provider_name=..., review_of_assignment_id=...)``, computed from
|
|
201
|
+
*dispatched_at*. ``provider_name``/``review_of_assignment_id``
|
|
202
|
+
(#1137) let an interactive ``--fix-of``/``--rework-of`` session be
|
|
203
|
+
recognized despite sharing ``type="work"`` with headless coding
|
|
204
|
+
workers — see :meth:`Config.pipeline.attention_threshold_for`'s
|
|
205
|
+
docstring. Both default to ``None`` (no effect) for callers that
|
|
206
|
+
don't have the full assignment record.
|
|
207
|
+
|
|
208
|
+
Deliberately time/round-based rather than self-report-based (#448: the
|
|
209
|
+
failure mode that motivated this was a worker that never emitted a
|
|
210
|
+
``STUCK:`` line — it just silently burned budget while looking
|
|
211
|
+
"productive").
|
|
212
|
+
|
|
213
|
+
Shared by :func:`detect_needs_attention` (the coordinator backstop,
|
|
214
|
+
dispatch-ledger-dict based), ``coord.pipeline.compute_pipeline`` (the
|
|
215
|
+
``/api/pipeline`` field the web dashboard renders), and the dashboard's
|
|
216
|
+
background poller (``Assignment``-object based) — one signal, several
|
|
217
|
+
call sites, instead of three copies of the same threshold logic.
|
|
218
|
+
|
|
219
|
+
Returns ``(reason, detail)`` — ``reason`` is ``"wall_clock"`` or
|
|
220
|
+
``"non_convergence"`` — or ``(None, None)`` when nothing is flagged.
|
|
221
|
+
"""
|
|
222
|
+
if (status or "").lower() != "running":
|
|
223
|
+
return None, None
|
|
224
|
+
if now is None:
|
|
225
|
+
now = time.time()
|
|
226
|
+
|
|
227
|
+
if review_iteration >= config.pipeline.convergence_rounds:
|
|
228
|
+
return "non_convergence", (
|
|
229
|
+
f"{review_iteration} fix/review round(s) on this assignment "
|
|
230
|
+
f"without reaching a green test verdict + approved review "
|
|
231
|
+
f"(threshold: {config.pipeline.convergence_rounds})."
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
threshold = config.pipeline.attention_threshold_for(
|
|
235
|
+
assignment_type,
|
|
236
|
+
provider_name=provider_name,
|
|
237
|
+
review_of_assignment_id=review_of_assignment_id,
|
|
238
|
+
)
|
|
239
|
+
if dispatched_at is not None:
|
|
240
|
+
running_for = now - dispatched_at
|
|
241
|
+
if running_for > threshold:
|
|
242
|
+
return "wall_clock", (
|
|
243
|
+
f"Running {_fmt_minutes(running_for)}, past the "
|
|
244
|
+
f"{_fmt_minutes(threshold)} threshold for "
|
|
245
|
+
f"type={assignment_type!r}."
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
return None, None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def detect_needs_attention(
|
|
252
|
+
config: Config, *, now: float | None = None
|
|
253
|
+
) -> list[tuple[NeedsAttentionDetection, dict]]:
|
|
254
|
+
"""Scan dispatched assignments for the two #846 "needs attention" signals
|
|
255
|
+
(see :func:`attention_signal`). Detection only — no dispatch/kill/handoff
|
|
256
|
+
behaviour.
|
|
257
|
+
|
|
258
|
+
Returns ``(NeedsAttentionDetection, dispatch_record)`` pairs for
|
|
259
|
+
assignments that haven't already been notified as needing attention (or
|
|
260
|
+
reached a terminal notification), mirroring :func:`detect_stuck`'s shape
|
|
261
|
+
so callers can post + mark idempotently the same way.
|
|
262
|
+
"""
|
|
263
|
+
dispatched = load_dispatched()
|
|
264
|
+
if not dispatched:
|
|
265
|
+
return []
|
|
266
|
+
notified = load_notified()
|
|
267
|
+
|
|
268
|
+
active_records = [
|
|
269
|
+
r for r in dispatched
|
|
270
|
+
if r["assignment_id"] not in notified
|
|
271
|
+
and _needs_attention_notified_key(r["assignment_id"]) not in notified
|
|
272
|
+
]
|
|
273
|
+
if not active_records:
|
|
274
|
+
return []
|
|
275
|
+
|
|
276
|
+
results: list[tuple[NeedsAttentionDetection, dict]] = []
|
|
277
|
+
for record in active_records:
|
|
278
|
+
reason, detail = attention_signal(
|
|
279
|
+
assignment_type=record.get("type") or "work",
|
|
280
|
+
status=record.get("status"),
|
|
281
|
+
dispatched_at=record.get("dispatched_at"),
|
|
282
|
+
review_iteration=record.get("review_iteration") or 0,
|
|
283
|
+
config=config,
|
|
284
|
+
now=now,
|
|
285
|
+
provider_name=record.get("provider_name"),
|
|
286
|
+
review_of_assignment_id=record.get("review_of_assignment_id"),
|
|
287
|
+
)
|
|
288
|
+
if reason is None:
|
|
289
|
+
continue
|
|
290
|
+
results.append((
|
|
291
|
+
NeedsAttentionDetection(
|
|
292
|
+
assignment_id=record["assignment_id"],
|
|
293
|
+
machine_name=record["machine_name"],
|
|
294
|
+
repo_name=record["repo_name"],
|
|
295
|
+
issue_number=record["issue_number"],
|
|
296
|
+
reason=reason,
|
|
297
|
+
detail=detail,
|
|
298
|
+
),
|
|
299
|
+
record,
|
|
300
|
+
))
|
|
301
|
+
|
|
302
|
+
return results
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def post_needs_attention(detection: NeedsAttentionDetection, record: dict) -> None:
|
|
306
|
+
"""Post a needs-attention comment to GitHub and mark notified (#846)."""
|
|
307
|
+
body = format_needs_attention(
|
|
308
|
+
assignment_id=detection.assignment_id,
|
|
309
|
+
machine_name=detection.machine_name,
|
|
310
|
+
repo_name=detection.repo_name,
|
|
311
|
+
issue_number=detection.issue_number,
|
|
312
|
+
reason=detection.reason,
|
|
313
|
+
detail=detection.detail,
|
|
314
|
+
)
|
|
315
|
+
github_ops.post_issue_comment(
|
|
316
|
+
record["repo_github"], detection.issue_number, body
|
|
317
|
+
)
|
|
318
|
+
mark_notified(_needs_attention_notified_key(detection.assignment_id), EVENT_NEEDS_ATTENTION)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# ── Liveness auditor (#2048) ─────────────────────────────────────────────────
|
|
322
|
+
#
|
|
323
|
+
# Tier 2.5 in the stall-detection ladder (see coord/liveness_auditor.py's
|
|
324
|
+
# module docstring): a cheap, independent, per-turn judgment call, sitting
|
|
325
|
+
# between EVENT_NEEDS_ATTENTION (a clock, no judgment) and a metered
|
|
326
|
+
# adversarial review (judgment, but only at a stage boundary). Detection +
|
|
327
|
+
# a one-shot GitHub comment only, mirroring detect_needs_attention's/
|
|
328
|
+
# detect_stalled_pipeline's contract exactly: this function NEVER sets
|
|
329
|
+
# Assignment.status/review_state/test_state, never kills/reassigns a
|
|
330
|
+
# worker, and never influences a merge decision. It only ever (a) runs a
|
|
331
|
+
# `claude -p` subprocess against a fixed-size (objective, latest-turn)
|
|
332
|
+
# context and (b) records/reads the resulting strike streak.
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _latest_turn_text_for_liveness(
|
|
336
|
+
machine_name: str, log_path: str | None, assignment_id: str,
|
|
337
|
+
) -> str | None:
|
|
338
|
+
"""Best-effort latest-assistant-turn text for a RUNNING assignment, for
|
|
339
|
+
the liveness auditor.
|
|
340
|
+
|
|
341
|
+
Mirrors :func:`_fetch_raw_log_text`'s local-file-then-agent-fetch
|
|
342
|
+
fallback (that helper takes a completed :class:`Transition`; this one
|
|
343
|
+
is called against a still-running assignment, so it takes the bare
|
|
344
|
+
fields instead) — with one deliberate difference: the local-file
|
|
345
|
+
branch uses :func:`coord.worker_events.latest_assistant_turn_text`'s
|
|
346
|
+
seek-based tail read (``tail_bytes=65536``) instead of reading the
|
|
347
|
+
whole file into memory. This runs once per debounce interval for the
|
|
348
|
+
*entire lifetime* of a running assignment, so a full read here would
|
|
349
|
+
quietly turn "audit cost is flat" into "disk I/O scales with log
|
|
350
|
+
size" — worst in exactly the stuck-worker-with-a-growing-log scenario
|
|
351
|
+
the auditor exists to catch (#2048 review). The remote (agent-fetch)
|
|
352
|
+
branch has no tail-range support server-side, so it still fetches the
|
|
353
|
+
full response and slices in Python.
|
|
354
|
+
|
|
355
|
+
Returns ``None`` on any I/O failure or if the tail has no assistant
|
|
356
|
+
turn yet — best-effort, the auditor must never be the reason
|
|
357
|
+
``coord notify`` raises.
|
|
358
|
+
|
|
359
|
+
The ``log_path`` recorded on an :class:`~coord.agent.AgentAssignment` is
|
|
360
|
+
a path on the *worker's own* machine, and ``coord notify`` normally runs
|
|
361
|
+
on the daemon host — so for the multi-machine fleet topology this repo
|
|
362
|
+
is built around, that path usually does NOT exist locally and the HTTP
|
|
363
|
+
fallback is the only branch that can ever return text. The local branch
|
|
364
|
+
is therefore taken only when the file actually exists: because
|
|
365
|
+
:func:`~coord.worker_events.latest_assistant_turn_text` swallows a
|
|
366
|
+
missing/unreadable file and returns ``None`` internally, committing to
|
|
367
|
+
it unconditionally would make the agent-fetch below dead code and the
|
|
368
|
+
auditor would silently never fire for any remote worker (#2048 review).
|
|
369
|
+
"""
|
|
370
|
+
if log_path:
|
|
371
|
+
from coord.worker_events import latest_assistant_turn_text # noqa: PLC0415
|
|
372
|
+
|
|
373
|
+
try:
|
|
374
|
+
local_readable = Path(log_path).is_file()
|
|
375
|
+
except OSError:
|
|
376
|
+
local_readable = False
|
|
377
|
+
if local_readable:
|
|
378
|
+
return latest_assistant_turn_text(log_path, tail_bytes=65536)
|
|
379
|
+
host = _agent_host(machine_name)
|
|
380
|
+
if host:
|
|
381
|
+
from coord.worker_events import latest_assistant_turn_text_from_text # noqa: PLC0415
|
|
382
|
+
|
|
383
|
+
try:
|
|
384
|
+
resp = httpx.get(
|
|
385
|
+
f"http://{host}:{AGENT_PORT}/logs/{assignment_id}", timeout=15.0
|
|
386
|
+
)
|
|
387
|
+
resp.raise_for_status()
|
|
388
|
+
except (httpx.HTTPError, httpx.TimeoutException):
|
|
389
|
+
return None
|
|
390
|
+
return latest_assistant_turn_text_from_text(resp.text[-65536:])
|
|
391
|
+
return None
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def detect_liveness_stall(
|
|
395
|
+
config: Config, *, now: float | None = None
|
|
396
|
+
) -> list[tuple[LivenessStallDetection, dict]]:
|
|
397
|
+
"""#2048: run the cheap per-turn liveness auditor against running
|
|
398
|
+
assignments and flag the ones whose latest turns earned
|
|
399
|
+
``config.pipeline.liveness_auditor.strikes`` consecutive ``blocked``
|
|
400
|
+
verdicts in a row.
|
|
401
|
+
|
|
402
|
+
Returns ``(LivenessStallDetection, dispatch_record)`` pairs, mirroring
|
|
403
|
+
:func:`detect_needs_attention`'s shape. No-ops entirely (returns ``[]``
|
|
404
|
+
without touching the DB or spawning a subprocess) when
|
|
405
|
+
``config.pipeline.liveness_auditor.enabled`` is ``False`` — the default.
|
|
406
|
+
"""
|
|
407
|
+
cfg = config.pipeline.liveness_auditor
|
|
408
|
+
if not cfg.enabled:
|
|
409
|
+
return []
|
|
410
|
+
|
|
411
|
+
from coord.liveness_auditor import ( # noqa: PLC0415
|
|
412
|
+
apply_verdict,
|
|
413
|
+
run_audit,
|
|
414
|
+
should_audit,
|
|
415
|
+
strip_self_report_lines,
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
if now is None:
|
|
419
|
+
now = time.time()
|
|
420
|
+
|
|
421
|
+
dispatched = load_dispatched()
|
|
422
|
+
if not dispatched:
|
|
423
|
+
return []
|
|
424
|
+
notified = load_notified()
|
|
425
|
+
|
|
426
|
+
active_records = [
|
|
427
|
+
r for r in dispatched
|
|
428
|
+
if (r.get("status") or "").lower() == "running"
|
|
429
|
+
and r["assignment_id"] not in notified
|
|
430
|
+
and _liveness_notified_key(r["assignment_id"]) not in notified
|
|
431
|
+
]
|
|
432
|
+
if not active_records:
|
|
433
|
+
return []
|
|
434
|
+
|
|
435
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
436
|
+
by_machine: dict[str, list[dict]] = {}
|
|
437
|
+
for r in active_records:
|
|
438
|
+
by_machine.setdefault(r["machine_name"], []).append(r)
|
|
439
|
+
|
|
440
|
+
# Deliberately serial: each `run_audit` call below is a subprocess
|
|
441
|
+
# spawn with up to a `timeout_seconds` (default 30s) ceiling, and
|
|
442
|
+
# audits for every not-yet-raised running assignment past its
|
|
443
|
+
# debounce window run one at a time in this loop. At today's typical
|
|
444
|
+
# concurrency this is a non-issue; if the audited fleet grows enough
|
|
445
|
+
# for a single `coord notify` pass to stack up meaningful wall-clock
|
|
446
|
+
# time here, parallelize (e.g. a bounded thread pool around
|
|
447
|
+
# `run_audit`) rather than accept unbounded serial latency (#2048
|
|
448
|
+
# review).
|
|
449
|
+
results: list[tuple[LivenessStallDetection, dict]] = []
|
|
450
|
+
for machine_name, records in by_machine.items():
|
|
451
|
+
machine = machines_by_name.get(machine_name)
|
|
452
|
+
if machine is None:
|
|
453
|
+
continue
|
|
454
|
+
status = _agent_status(machine.host)
|
|
455
|
+
if status is None:
|
|
456
|
+
continue
|
|
457
|
+
active_by_id: dict[str, dict] = {}
|
|
458
|
+
for entry in status.get("active", []):
|
|
459
|
+
eid = entry.get("id")
|
|
460
|
+
if eid:
|
|
461
|
+
active_by_id[eid] = entry
|
|
462
|
+
|
|
463
|
+
for record in records:
|
|
464
|
+
aid = record["assignment_id"]
|
|
465
|
+
entry = active_by_id.get(aid)
|
|
466
|
+
if entry is None:
|
|
467
|
+
continue
|
|
468
|
+
|
|
469
|
+
state = load_liveness_audit_state(aid)
|
|
470
|
+
if state.raised:
|
|
471
|
+
continue
|
|
472
|
+
if not should_audit(
|
|
473
|
+
last_audit_at=state.last_audit_at,
|
|
474
|
+
now=now,
|
|
475
|
+
debounce_seconds=cfg.debounce_seconds,
|
|
476
|
+
):
|
|
477
|
+
continue
|
|
478
|
+
|
|
479
|
+
# Seek-based tail read for a local log (see
|
|
480
|
+
# _latest_turn_text_for_liveness's docstring) — the auditor
|
|
481
|
+
# only ever needs the single most recent turn, never the
|
|
482
|
+
# whole (potentially multi-MB) transcript, and this repeats
|
|
483
|
+
# every debounce interval for the assignment's whole runtime.
|
|
484
|
+
turn_text = _latest_turn_text_for_liveness(
|
|
485
|
+
record["machine_name"], entry.get("log_path"), aid
|
|
486
|
+
)
|
|
487
|
+
if turn_text is None:
|
|
488
|
+
continue # no assistant turn yet, or fetch failed
|
|
489
|
+
|
|
490
|
+
# #2048 context isolation: strip the worker's own STATUS:/
|
|
491
|
+
# STUCK: lines before the auditor ever sees this turn — see
|
|
492
|
+
# coord.liveness_auditor's module docstring.
|
|
493
|
+
turn_text = strip_self_report_lines(turn_text)
|
|
494
|
+
|
|
495
|
+
outcome = run_audit(
|
|
496
|
+
record.get("briefing") or "",
|
|
497
|
+
turn_text,
|
|
498
|
+
model=cfg.model,
|
|
499
|
+
claude_bin=cfg.claude_bin,
|
|
500
|
+
timeout=cfg.timeout_seconds,
|
|
501
|
+
)
|
|
502
|
+
new_state, just_raised = apply_verdict(
|
|
503
|
+
state, outcome.verdict, now=now, strikes=cfg.strikes
|
|
504
|
+
)
|
|
505
|
+
save_liveness_audit_state(aid, new_state)
|
|
506
|
+
|
|
507
|
+
if just_raised:
|
|
508
|
+
results.append((
|
|
509
|
+
LivenessStallDetection(
|
|
510
|
+
assignment_id=aid,
|
|
511
|
+
machine_name=record["machine_name"],
|
|
512
|
+
repo_name=record["repo_name"],
|
|
513
|
+
issue_number=record["issue_number"],
|
|
514
|
+
consecutive_blocked=new_state.consecutive_blocked,
|
|
515
|
+
last_verdict=new_state.last_verdict,
|
|
516
|
+
),
|
|
517
|
+
record,
|
|
518
|
+
))
|
|
519
|
+
|
|
520
|
+
return results
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def post_liveness_stall(detection: LivenessStallDetection, record: dict) -> None:
|
|
524
|
+
"""Post a liveness-stall comment to GitHub and mark notified (#2048)."""
|
|
525
|
+
body = format_liveness_stall(
|
|
526
|
+
assignment_id=detection.assignment_id,
|
|
527
|
+
machine_name=detection.machine_name,
|
|
528
|
+
repo_name=detection.repo_name,
|
|
529
|
+
issue_number=detection.issue_number,
|
|
530
|
+
consecutive_blocked=detection.consecutive_blocked,
|
|
531
|
+
)
|
|
532
|
+
github_ops.post_issue_comment(
|
|
533
|
+
record["repo_github"], detection.issue_number, body
|
|
534
|
+
)
|
|
535
|
+
mark_notified(_liveness_notified_key(detection.assignment_id), EVENT_LIVENESS_STALL)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
# ── Stalled-pipeline sweeper (#1441) ────────────────────────────────────────
|
|
539
|
+
#
|
|
540
|
+
# The auto-loop (coord.auto_loop) only reacts to review/fix TRANSITIONS — the
|
|
541
|
+
# instant `coord notify` sees a review or fix flip to `done` during THAT
|
|
542
|
+
# pass. Once the transition is consumed nothing ever re-examines the row, so
|
|
543
|
+
# a precondition that lands late (a Test verdict backfilled two days after
|
|
544
|
+
# the review completed — vimcode #602) leaves it stranded: looks complete on
|
|
545
|
+
# the board, isn't. This sweeper re-scans every *done* work chain on the
|
|
546
|
+
# board each notify pass and flags the ones stuck on an unmet precondition
|
|
547
|
+
# a fresh transition would have already resolved. Detection only — no
|
|
548
|
+
# dispatch, mirroring detect_needs_attention's contract.
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _pipeline_heads(board: "Board") -> list["Assignment"]:
|
|
552
|
+
"""Return the most-recent WORK_LIKE_TYPES assignment per (repo, issue).
|
|
553
|
+
|
|
554
|
+
A row can be bounced through 1+ auto-loop fix iterations, each a
|
|
555
|
+
separate ``Assignment`` sharing the same ``(repo_name, issue_number)``.
|
|
556
|
+
Only the most recent one reflects the pipeline's actual current
|
|
557
|
+
position — earlier rows in the chain are superseded, and evaluating them
|
|
558
|
+
too would re-flag a condition a later fix already addressed.
|
|
559
|
+
"""
|
|
560
|
+
from coord.models import WORK_LIKE_TYPES # noqa: PLC0415
|
|
561
|
+
|
|
562
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
563
|
+
heads: dict[tuple[str, int], "Assignment"] = {}
|
|
564
|
+
for a in all_assignments:
|
|
565
|
+
if a.type not in WORK_LIKE_TYPES:
|
|
566
|
+
continue
|
|
567
|
+
key = (a.repo_name, a.issue_number)
|
|
568
|
+
ts = a.dispatched_at or a.finished_at or 0.0
|
|
569
|
+
cur = heads.get(key)
|
|
570
|
+
cur_ts = (cur.dispatched_at or cur.finished_at or 0.0) if cur is not None else -1.0
|
|
571
|
+
if cur is None or ts >= cur_ts:
|
|
572
|
+
heads[key] = a
|
|
573
|
+
return list(heads.values())
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def detect_stalled_pipeline(
|
|
577
|
+
config: Config,
|
|
578
|
+
*,
|
|
579
|
+
board: "Board | None" = None,
|
|
580
|
+
merge_queue_items: list | None = None,
|
|
581
|
+
terminal_cache: dict | None = None,
|
|
582
|
+
) -> list[tuple[StalledDetection, "Assignment"]]:
|
|
583
|
+
"""Scan the board for *done* work chains stuck on an unmet precondition
|
|
584
|
+
that a fresh review/fix transition would already have resolved (#1441).
|
|
585
|
+
|
|
586
|
+
Five candidate stall states, checked per pipeline "head" (the most
|
|
587
|
+
recent work-like assignment for a given (repo, issue) — see
|
|
588
|
+
:func:`_pipeline_heads`):
|
|
589
|
+
|
|
590
|
+
1. ``review_request_changes_no_fix`` — the head's linked review
|
|
591
|
+
completed with verdict ``request-changes`` and no fix assignment was
|
|
592
|
+
ever dispatched in response (the vimcode #602 reference case: the
|
|
593
|
+
review's transition fired and was consumed while some other
|
|
594
|
+
precondition was outstanding, and nothing has re-examined it since).
|
|
595
|
+
2. ``review_done_no_verdict`` (#1582) — the head's linked review is
|
|
596
|
+
``status="done"`` but ``review_verdict IS NULL``: the reviewing
|
|
597
|
+
session finalised without ever capturing a verdict (elitebook's
|
|
598
|
+
documented ~14% review-verdict drop rate, #873).
|
|
599
|
+
|
|
600
|
+
#2019: the detail text used to blame this on "the session likely
|
|
601
|
+
failed to start or exited before recording one (#812)" for EVERY
|
|
602
|
+
review. That was wrong twice over on the #1956 reference case — the
|
|
603
|
+
session ran 392s, produced a complete 6.5KB review and exited 0, and
|
|
604
|
+
#812 is CLOSED and was about *interactive* reviews, while that one was
|
|
605
|
+
``interactive=False``. An operator following it looked at a closed
|
|
606
|
+
issue and a false cause. The wording is now provider-aware: a headless
|
|
607
|
+
review that reached ``done`` is reported as the
|
|
608
|
+
END_REVIEW-without-verdict class (#1956), and only a ``claude-pty``
|
|
609
|
+
review still cites #812's never-started shape.
|
|
610
|
+
|
|
611
|
+
This matches NONE of the other three arms — it
|
|
612
|
+
isn't ``request-changes`` (no verdict at all), a review WAS
|
|
613
|
+
dispatched (so not ``done_no_review``), and there is no approval (so
|
|
614
|
+
not ``approved_not_queued``) — so before this arm existed it fell
|
|
615
|
+
through every check and parked the drive forever (#1582's own
|
|
616
|
+
observed case, #1563).
|
|
617
|
+
3. ``done_no_review`` — the head carries a terminal Test verdict
|
|
618
|
+
(``passed``/``skipped``), the "review" gate is required, the
|
|
619
|
+
completion is not an interactive (``provider_name="claude-pty"``)
|
|
620
|
+
session (interactive completions are deliberately excluded from
|
|
621
|
+
automatic review dispatch — #555), and yet no review assignment was
|
|
622
|
+
ever dispatched for it.
|
|
623
|
+
4. ``approved_not_queued`` — the head satisfies every merge gate
|
|
624
|
+
(:func:`coord.merge_queue.passes_merge_gates` — reused rather than
|
|
625
|
+
re-derived, per #1441's own request) but has no merge-queue entry.
|
|
626
|
+
5. ``merge_conflict_unresolved`` (#1478) — the head already HAS a
|
|
627
|
+
merge-queue entry, but that entry is parked ``CONFLICT`` with an
|
|
628
|
+
error :func:`coord.merge_queue.classify_conflict` calls
|
|
629
|
+
``"rebaseable"`` and no conflict-fix attempt is active or already
|
|
630
|
+
failed (:func:`coord.conflict_fix.has_prior_conflict_fix`). This is
|
|
631
|
+
exactly the gap :mod:`coord.commands.merge`'s
|
|
632
|
+
``_dispatch_conflict_fixes`` docstring calls out for the ``--only``
|
|
633
|
+
path pre-#1474 — a bare ``CONFLICT`` row that never got a second
|
|
634
|
+
classify-and-dispatch pass, except here for *any* path (not only
|
|
635
|
+
``--only``): a ``coord merge`` invocation that dispatched a
|
|
636
|
+
conflict-fix which then failed to actually attempt (no idle
|
|
637
|
+
machine) leaves the entry parked with nothing watching it.
|
|
638
|
+
5. ``review_failed_no_verdict`` (#1584) — the head's linked review
|
|
639
|
+
WORKER died (transient API error, network drop, ...) before ever
|
|
640
|
+
producing a verdict — ``status="failed"`` with no
|
|
641
|
+
``review_verdict``. Before #1584 this could not happen (a dying
|
|
642
|
+
review was mislabelled ``done``, silently masquerading as a real
|
|
643
|
+
completion); now that it is correctly ``failed``, it needs its own
|
|
644
|
+
arm here so it is not silently skipped (``reason`` staying ``None``)
|
|
645
|
+
the way an unrecognized status would be.
|
|
646
|
+
|
|
647
|
+
Every candidate is checked against the shared #522 terminal-state guard
|
|
648
|
+
(:func:`coord.github_ops.work_is_terminal`, via *terminal_cache* — the
|
|
649
|
+
same cache :func:`coord.notify.run` threads through the review/fix
|
|
650
|
+
auto-loop calls) so a closed issue or merged PR never surfaces, and
|
|
651
|
+
against the ``notified`` ledger (composite key, :func:`_stalled_notified_key`)
|
|
652
|
+
so a flagged row is not re-flagged every pass.
|
|
653
|
+
|
|
654
|
+
Detection only, mirroring :func:`detect_needs_attention`'s contract — no
|
|
655
|
+
dispatch, no kill, no handoff (that lives in
|
|
656
|
+
:func:`dispatch_stalled_pipeline_action`, #1478, gated behind
|
|
657
|
+
``config.pipeline.auto_dispatch_stalled``). *board* / *merge_queue_items*
|
|
658
|
+
/ *terminal_cache* are all optional so callers (tests, or a future
|
|
659
|
+
``reconcile()`` caller) can supply their own instead of hitting the
|
|
660
|
+
board service / DB / GitHub.
|
|
661
|
+
"""
|
|
662
|
+
# `github_ops` is already imported at module level (used by every other
|
|
663
|
+
# post_* helper in this file) — no local re-import here, so a caller
|
|
664
|
+
# that mocks `coord.notify.github_ops.post_issue_comment` for the
|
|
665
|
+
# posting side doesn't also have to reason about a separately-imported
|
|
666
|
+
# local name for the terminal-state check below.
|
|
667
|
+
from coord.auto_loop import FIX_DISPATCH_TYPES # noqa: PLC0415
|
|
668
|
+
from coord.conflict_fix import has_prior_conflict_fix # noqa: PLC0415
|
|
669
|
+
from coord.merge_queue import ( # noqa: PLC0415
|
|
670
|
+
CONFLICT,
|
|
671
|
+
classify_conflict,
|
|
672
|
+
live_gate_entry,
|
|
673
|
+
load_queue,
|
|
674
|
+
passes_merge_gates,
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
if board is None:
|
|
678
|
+
from coord.board_service import read_board # noqa: PLC0415
|
|
679
|
+
board = read_board()
|
|
680
|
+
if merge_queue_items is None:
|
|
681
|
+
merge_queue_items = load_queue()
|
|
682
|
+
if terminal_cache is None:
|
|
683
|
+
terminal_cache = {}
|
|
684
|
+
|
|
685
|
+
notified = load_notified()
|
|
686
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
687
|
+
|
|
688
|
+
results: list[tuple[StalledDetection, "Assignment"]] = []
|
|
689
|
+
for work in _pipeline_heads(board):
|
|
690
|
+
if work.status != "done" or not work.assignment_id:
|
|
691
|
+
continue
|
|
692
|
+
if _stalled_notified_key(work.assignment_id) in notified:
|
|
693
|
+
continue
|
|
694
|
+
|
|
695
|
+
repo = config.repo(work.repo_name)
|
|
696
|
+
repo_github = repo.github if repo is not None else None
|
|
697
|
+
if repo_github and github_ops.work_is_terminal(
|
|
698
|
+
repo_github, work.issue_number, work.branch, cache=terminal_cache
|
|
699
|
+
):
|
|
700
|
+
continue
|
|
701
|
+
|
|
702
|
+
required_gates = work.required_gates or list(config.pipeline.default_gates)
|
|
703
|
+
|
|
704
|
+
review = next(
|
|
705
|
+
(
|
|
706
|
+
a for a in all_assignments
|
|
707
|
+
if a.review_of_assignment_id == work.assignment_id and a.type == "review"
|
|
708
|
+
),
|
|
709
|
+
None,
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
# #1566: a review that just finished lands on status="finalizing"
|
|
713
|
+
# (not "done") until `coord notify`'s own _try_parse_and_post_review
|
|
714
|
+
# promotes it — i.e. THIS function is what closes that window. None
|
|
715
|
+
# of the `review.status == "done"` checks below match "finalizing",
|
|
716
|
+
# so a still-finalizing review falls through this whole if/elif
|
|
717
|
+
# chain with `reason` left unset (no stall reported), which is
|
|
718
|
+
# correct as long as the finalizing window stays short. That relies
|
|
719
|
+
# on `coord notify` actually running again soon — nothing here
|
|
720
|
+
# guards against `coord notify` itself never running (e.g. daemon
|
|
721
|
+
# down), which would leave the row on "finalizing" forever without
|
|
722
|
+
# ever tripping this stall detector.
|
|
723
|
+
reason: str | None = None
|
|
724
|
+
detail = ""
|
|
725
|
+
|
|
726
|
+
if (
|
|
727
|
+
review is not None
|
|
728
|
+
and review.status == "done"
|
|
729
|
+
and review.review_verdict == "request-changes"
|
|
730
|
+
):
|
|
731
|
+
fix = next(
|
|
732
|
+
(
|
|
733
|
+
a for a in all_assignments
|
|
734
|
+
if a.review_of_assignment_id == work.assignment_id
|
|
735
|
+
and a.type in FIX_DISPATCH_TYPES
|
|
736
|
+
),
|
|
737
|
+
None,
|
|
738
|
+
)
|
|
739
|
+
if fix is None:
|
|
740
|
+
reason = "review_request_changes_no_fix"
|
|
741
|
+
detail = (
|
|
742
|
+
f"Review {review.assignment_id} completed with "
|
|
743
|
+
"request-changes and no fix worker was ever dispatched "
|
|
744
|
+
"for it."
|
|
745
|
+
)
|
|
746
|
+
elif (
|
|
747
|
+
review is not None
|
|
748
|
+
and review.status == "done"
|
|
749
|
+
and review.review_verdict is None
|
|
750
|
+
):
|
|
751
|
+
# #1582: a review that finalised `done` with NO verdict ever
|
|
752
|
+
# captured. Checked BEFORE the `review is None or review.status
|
|
753
|
+
# == "done"` catch-all below — that branch's merge-gate check
|
|
754
|
+
# (`passes_merge_gates`) never fires for a `None` verdict (no
|
|
755
|
+
# approval), so this row would otherwise fall all the way
|
|
756
|
+
# through with `reason` left unset.
|
|
757
|
+
reason = "review_done_no_verdict"
|
|
758
|
+
# #2019: provider-aware, because the pre-#2019 single sentence
|
|
759
|
+
# ("the session likely failed to start or exited before recording
|
|
760
|
+
# one (#812)") was demonstrably false for the headless case and
|
|
761
|
+
# pointed at a CLOSED issue about interactive reviews. See this
|
|
762
|
+
# function's docstring, arm 2.
|
|
763
|
+
if review.provider_name == "claude-pty":
|
|
764
|
+
detail = (
|
|
765
|
+
f"Review {review.assignment_id} finalised as done but no "
|
|
766
|
+
"verdict was ever captured — an interactive review that "
|
|
767
|
+
"failed to start, or exited before `coord report-result` "
|
|
768
|
+
"ran (#812)."
|
|
769
|
+
)
|
|
770
|
+
else:
|
|
771
|
+
detail = (
|
|
772
|
+
f"Review {review.assignment_id} finalised as done but no "
|
|
773
|
+
"verdict was ever captured. This is a HEADLESS review "
|
|
774
|
+
"that ran to completion (a session that died lands "
|
|
775
|
+
"status='failed', not 'done'), so the reviewer's "
|
|
776
|
+
"REVIEW_VERDICT header was omitted or unparsed — the "
|
|
777
|
+
"END_REVIEW-without-verdict class (#1956), not a "
|
|
778
|
+
"never-started session. The verdict is very likely "
|
|
779
|
+
"already in the transcript; relay it rather than "
|
|
780
|
+
"re-dispatching: coord report-result --assignment "
|
|
781
|
+
f"{review.assignment_id} --status done --verdict "
|
|
782
|
+
"<approve|request-changes> --verdict-source recovered "
|
|
783
|
+
"--verdict-reason '...' --body-file <extracted-review.md>"
|
|
784
|
+
)
|
|
785
|
+
elif (
|
|
786
|
+
review is None
|
|
787
|
+
and "review" in required_gates
|
|
788
|
+
and work.provider_name != "claude-pty"
|
|
789
|
+
and work.test_state in ("passed", "skipped")
|
|
790
|
+
):
|
|
791
|
+
reason = "done_no_review"
|
|
792
|
+
detail = (
|
|
793
|
+
f"Work is done with test_state={work.test_state!r} but no "
|
|
794
|
+
"review assignment was ever dispatched for it."
|
|
795
|
+
)
|
|
796
|
+
elif review is not None and review.status == "failed":
|
|
797
|
+
# #1584: the review worker died (transient API error, network
|
|
798
|
+
# drop, ...) before producing a verdict. Checked before the
|
|
799
|
+
# `review is None or review.status == "done"` catch-all below so
|
|
800
|
+
# a failed review is never mistaken for "no review dispatched"
|
|
801
|
+
# or "review approved" — neither of which is true here.
|
|
802
|
+
#
|
|
803
|
+
# ...UNLESS it was killed by the account's usage limit. That is
|
|
804
|
+
# an account-wide exhausted budget, not a per-review defect:
|
|
805
|
+
# `AgentServer._reap` lands a usage-limit kill on FAILED exactly
|
|
806
|
+
# like an api_error kill, so without this guard the sweep would
|
|
807
|
+
# spend this work row's ONE auto-recovery action (the
|
|
808
|
+
# `_stalled_notified_key` ledger is one-shot per work row) on a
|
|
809
|
+
# `dispatch_review` that is guaranteed to die the same way until
|
|
810
|
+
# the reset — the precise anti-pattern `reconcile.py`'s
|
|
811
|
+
# `auto_reassign` block was hardened against in #1461, and the
|
|
812
|
+
# one `coord/drive.py`'s `_decide_review` already guards with
|
|
813
|
+
# this same predicate. Skipped at CLASSIFICATION rather than
|
|
814
|
+
# declined at dispatch so the row is never marked notified: a
|
|
815
|
+
# later review attempt that fails for a *different* (genuinely
|
|
816
|
+
# recoverable) reason can still be picked up by a future tick.
|
|
817
|
+
if is_usage_limit_reason(review.failure_reason):
|
|
818
|
+
continue
|
|
819
|
+
reason = "review_failed_no_verdict"
|
|
820
|
+
detail = (
|
|
821
|
+
f"Review {review.assignment_id} failed "
|
|
822
|
+
f"({review.failure_reason or 'no reason recorded'}) before "
|
|
823
|
+
"producing a verdict, and no retry was dispatched."
|
|
824
|
+
)
|
|
825
|
+
elif review is None or review.status == "done":
|
|
826
|
+
# Either the review gate doesn't apply, or a review already
|
|
827
|
+
# completed without leaving a request-changes verdict blocking
|
|
828
|
+
# it (approved, or advanced past advisory-only nits) — the only
|
|
829
|
+
# remaining question is whether it made it into the merge queue,
|
|
830
|
+
# and if it did, whether that entry is stuck.
|
|
831
|
+
matching_entry = next(
|
|
832
|
+
(m for m in merge_queue_items if m.assignment_id == work.assignment_id),
|
|
833
|
+
None,
|
|
834
|
+
)
|
|
835
|
+
if matching_entry is None:
|
|
836
|
+
# #2085: `work` is a raw board Assignment — no
|
|
837
|
+
# `branch_head_sha`/`repo_github`/`target_branch` attribute,
|
|
838
|
+
# so handing it straight to `passes_merge_gates` made the
|
|
839
|
+
# #821 SHA-freshness check inside `has_approved_review`
|
|
840
|
+
# permanently unconfirmable (fails closed on every review
|
|
841
|
+
# carrying a real `review_head_sha`, i.e. virtually every
|
|
842
|
+
# modern approval). Build the same live-anchored synthetic
|
|
843
|
+
# entry `coord.gates.build_gate_report` uses so a genuinely
|
|
844
|
+
# fresh approval can still be confirmed via `github_ops`
|
|
845
|
+
# (already imported at module level). Falls back to the raw
|
|
846
|
+
# `work` row (still gh_ops-backed, just missing target_branch/
|
|
847
|
+
# repo_github) when the repo isn't configured — the gate
|
|
848
|
+
# then fails closed exactly as before, never open.
|
|
849
|
+
gate_entry = work
|
|
850
|
+
if repo is not None and repo_github:
|
|
851
|
+
from coord.branch_model import ( # noqa: PLC0415
|
|
852
|
+
resolve_base_branch_for_issue_number,
|
|
853
|
+
)
|
|
854
|
+
target_branch = resolve_base_branch_for_issue_number(
|
|
855
|
+
repo, repo_github, work.issue_number,
|
|
856
|
+
)
|
|
857
|
+
gate_entry = live_gate_entry(
|
|
858
|
+
work, repo_github, target_branch, github_ops
|
|
859
|
+
)
|
|
860
|
+
if passes_merge_gates(gate_entry, config, board, gh_ops=github_ops):
|
|
861
|
+
reason = "approved_not_queued"
|
|
862
|
+
detail = (
|
|
863
|
+
"Work passes every merge gate (review + test) but has "
|
|
864
|
+
"no merge-queue entry."
|
|
865
|
+
)
|
|
866
|
+
elif (
|
|
867
|
+
matching_entry.state == CONFLICT
|
|
868
|
+
and classify_conflict(matching_entry.error) == "rebaseable"
|
|
869
|
+
and not has_prior_conflict_fix(board, matching_entry.assignment_id)
|
|
870
|
+
):
|
|
871
|
+
# #1478: a rebaseable CONFLICT with no active/failed
|
|
872
|
+
# conflict-fix attempt — the #1474 classify-and-dispatch step
|
|
873
|
+
# never got (or never got a second) chance at this entry.
|
|
874
|
+
reason = "merge_conflict_unresolved"
|
|
875
|
+
detail = (
|
|
876
|
+
f"Merge queue entry for branch {matching_entry.branch!r} is "
|
|
877
|
+
f"stuck in CONFLICT ({matching_entry.error or 'no error recorded'}) "
|
|
878
|
+
"with no active or previously-failed conflict-fix attempt."
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
if reason is None:
|
|
882
|
+
continue
|
|
883
|
+
|
|
884
|
+
results.append((
|
|
885
|
+
StalledDetection(
|
|
886
|
+
assignment_id=work.assignment_id,
|
|
887
|
+
machine_name=work.machine_name,
|
|
888
|
+
repo_name=work.repo_name,
|
|
889
|
+
issue_number=work.issue_number,
|
|
890
|
+
reason=reason,
|
|
891
|
+
detail=detail,
|
|
892
|
+
),
|
|
893
|
+
work,
|
|
894
|
+
))
|
|
895
|
+
|
|
896
|
+
return results
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def post_stalled_pipeline(detection: StalledDetection, config: Config) -> None:
|
|
900
|
+
"""Post a stalled-pipeline comment to GitHub and mark notified (#1441)."""
|
|
901
|
+
repo = config.repo(detection.repo_name)
|
|
902
|
+
repo_github = repo.github if repo is not None else None
|
|
903
|
+
if not repo_github:
|
|
904
|
+
return
|
|
905
|
+
body = format_stalled_pipeline(
|
|
906
|
+
assignment_id=detection.assignment_id,
|
|
907
|
+
machine_name=detection.machine_name,
|
|
908
|
+
repo_name=detection.repo_name,
|
|
909
|
+
issue_number=detection.issue_number,
|
|
910
|
+
reason=detection.reason,
|
|
911
|
+
detail=detection.detail,
|
|
912
|
+
)
|
|
913
|
+
github_ops.post_issue_comment(repo_github, detection.issue_number, body)
|
|
914
|
+
mark_notified(_stalled_notified_key(detection.assignment_id), EVENT_STALLED)
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
# ── #1478: dispatch arm ──────────────────────────────────────────────────────
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
@dataclass
|
|
921
|
+
class StalledDispatchAction:
|
|
922
|
+
"""The outcome of :func:`dispatch_stalled_pipeline_action` for one
|
|
923
|
+
:class:`StalledDetection`."""
|
|
924
|
+
|
|
925
|
+
kind: str
|
|
926
|
+
"""One of:
|
|
927
|
+
- ``"fix_dispatch_attempted"`` — re-ran the review-completion transition
|
|
928
|
+
(:func:`coord.auto_loop.process_review_completion`) for
|
|
929
|
+
``review_request_changes_no_fix`` (or, #1582, for a
|
|
930
|
+
``review_done_no_verdict`` whose verdict was just recovered from the
|
|
931
|
+
transcript) and it dispatched a fix worker; see *detail* for what it
|
|
932
|
+
did.
|
|
933
|
+
- ``"review_transition_applied"`` — re-ran
|
|
934
|
+
:func:`coord.auto_loop.process_review_completion` for
|
|
935
|
+
``review_request_changes_no_fix`` (or a transcript-recovered
|
|
936
|
+
``review_done_no_verdict``, #1582) and it resolved as ``approved``,
|
|
937
|
+
``approved_with_nits`` (the #476 advisory-only gate), or
|
|
938
|
+
``terminal_skip`` — no fix worker was dispatched, but the call still
|
|
939
|
+
mutated *board* in place (``review.review_verdict``,
|
|
940
|
+
``work.review_state = "done"``, a merge-queue ``refresh_entry_assignment``)
|
|
941
|
+
per that function's own "the caller is responsible for persisting the
|
|
942
|
+
board after this returns" contract. Must be persisted exactly like a
|
|
943
|
+
real dispatch even though no agent was launched.
|
|
944
|
+
- ``"review_verdict_recovered"`` — ``review_done_no_verdict``: a verdict
|
|
945
|
+
was recovered from the reviewing session's own transcript (#617's
|
|
946
|
+
``_review_findings_from_transcript``, the same recovery
|
|
947
|
+
``coord diagnose --stage review`` runs) and durably persisted, but
|
|
948
|
+
``process_review_completion`` made no further board mutation from it
|
|
949
|
+
(e.g. ``pipeline.auto_loop`` is off). See *detail* for the recovered
|
|
950
|
+
verdict.
|
|
951
|
+
- ``"review_reset_redispatched"`` — ``review_done_no_verdict``: nothing
|
|
952
|
+
was recoverable from the transcript, so the review stage was reset
|
|
953
|
+
(the review rows deleted, ``work.review_state`` cleared — #1180's
|
|
954
|
+
``_reset_review_stage``, branch/commits always kept) and a fresh
|
|
955
|
+
review dispatched for the same work.
|
|
956
|
+
- ``"review_dispatched"`` — a review was dispatched for
|
|
957
|
+
``done_no_review``.
|
|
958
|
+
- ``"enqueued"`` — the work was enqueued for merge for
|
|
959
|
+
``approved_not_queued`` (including when a *different* row's
|
|
960
|
+
``enqueue_approved_work`` call already enqueued this one earlier in
|
|
961
|
+
the same sweep tick — see the queue-membership check below).
|
|
962
|
+
- ``"conflict_fix_dispatched"`` — a conflict-fix worker was dispatched
|
|
963
|
+
for ``merge_conflict_unresolved``.
|
|
964
|
+
- ``"no_action"`` — the reused dispatcher declined (no
|
|
965
|
+
capable machine, already in flight, gate not actually satisfied,
|
|
966
|
+
entry vanished from the board/queue between detection and dispatch).
|
|
967
|
+
- ``"skipped_live_session"`` — a running/pending assignment already
|
|
968
|
+
exists for this (repo, issue); never act underneath a live session
|
|
969
|
+
(#602).
|
|
970
|
+
- ``"skipped_human_required"`` — the conflict-fix retry cap was already
|
|
971
|
+
hit; surfacing to a human, not auto-retrying.
|
|
972
|
+
- ``"disabled"`` — ``pipeline.auto_dispatch_stalled`` is
|
|
973
|
+
off; detection/narration still happened, dispatch did not.
|
|
974
|
+
"""
|
|
975
|
+
detail: str = ""
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
# Action kinds that represent a REAL dispatch OR a board mutation that must
|
|
979
|
+
# be persisted (mutate the board / merge queue / fire an agent request) —
|
|
980
|
+
# used to decide (a) whether the board needs writing back, (b) which GitHub
|
|
981
|
+
# comment to post, and (c) whether the audit row is business-tier (a real
|
|
982
|
+
# transition) or operational-tier (a no-op/skip, informational only).
|
|
983
|
+
#
|
|
984
|
+
# ``review_transition_applied`` belongs here even though it does not launch
|
|
985
|
+
# an agent: an approved/approved-with-nits/terminal-skip resolution from
|
|
986
|
+
# ``process_review_completion`` still flips ``work.review_state``/
|
|
987
|
+
# ``review.review_verdict`` in place, and losing that mutation while the
|
|
988
|
+
# one-shot ledger marks the row notified anyway is exactly the #1478 review
|
|
989
|
+
# bug this set exists to prevent.
|
|
990
|
+
_STALLED_DISPATCH_KINDS = frozenset({
|
|
991
|
+
"fix_dispatch_attempted", "review_transition_applied", "review_dispatched",
|
|
992
|
+
"enqueued", "conflict_fix_dispatched",
|
|
993
|
+
# #1582
|
|
994
|
+
"review_verdict_recovered", "review_reset_redispatched",
|
|
995
|
+
})
|
|
996
|
+
|
|
997
|
+
# process_review_completion (and the _dispatch_fix_for_review it may call)
|
|
998
|
+
# kinds that mutate `board` in place per its own documented contract, even
|
|
999
|
+
# when they don't dispatch a fix worker. `disabled`/`no_findings` return
|
|
1000
|
+
# before any mutation; `no_work_found`/`max_iterations` return without
|
|
1001
|
+
# touching `board` (only a GitHub notice for the latter).
|
|
1002
|
+
_MUTATING_REVIEW_COMPLETION_KINDS = frozenset({
|
|
1003
|
+
"fix_dispatched", "approved", "approved_with_nits", "terminal_skip",
|
|
1004
|
+
})
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def _stalled_row_has_live_session(board: "Board", work: "Assignment") -> bool:
|
|
1008
|
+
"""#602 guardrail: true when a running/pending assignment already exists
|
|
1009
|
+
for *work*'s (repo, issue) — e.g. an interactive ``--fix-of``/
|
|
1010
|
+
``--review-of``/``--merge-of`` session a human is actively driving.
|
|
1011
|
+
:func:`dispatch_stalled_pipeline_action` must never act underneath one:
|
|
1012
|
+
racing an auto-dispatch against a live session can duplicate or clobber
|
|
1013
|
+
it. Broader than :func:`coord.claim.has_active_work_followup` (which
|
|
1014
|
+
only checks ``work``/``conflict-fix``) — any live assignment type
|
|
1015
|
+
(review, smoke, chat, ...) for the same issue counts here.
|
|
1016
|
+
"""
|
|
1017
|
+
for a in board.active:
|
|
1018
|
+
if a.status not in ("running", "pending"):
|
|
1019
|
+
continue
|
|
1020
|
+
if a.repo_name == work.repo_name and a.issue_number == work.issue_number:
|
|
1021
|
+
return True
|
|
1022
|
+
return False
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def dispatch_stalled_pipeline_action(
|
|
1026
|
+
detection: StalledDetection,
|
|
1027
|
+
work: "Assignment",
|
|
1028
|
+
board: "Board",
|
|
1029
|
+
config: Config,
|
|
1030
|
+
*,
|
|
1031
|
+
terminal_cache: dict | None = None,
|
|
1032
|
+
) -> StalledDispatchAction:
|
|
1033
|
+
"""#1478: act on a #1441 stalled-pipeline detection instead of only
|
|
1034
|
+
narrating it.
|
|
1035
|
+
|
|
1036
|
+
Gated by ``config.pipeline.auto_dispatch_stalled`` (default ``False`` —
|
|
1037
|
+
detection/narration via :func:`post_stalled_pipeline` is unconditional;
|
|
1038
|
+
this is the opt-in action half). Mutates *board* in place exactly like
|
|
1039
|
+
the auto-loop / review-dispatch helpers it delegates to — the caller is
|
|
1040
|
+
responsible for persisting it.
|
|
1041
|
+
|
|
1042
|
+
Reuses the SAME dispatch machinery the original, on-time transition
|
|
1043
|
+
would have used for each reason, rather than re-deriving new logic:
|
|
1044
|
+
|
|
1045
|
+
- ``review_request_changes_no_fix`` → re-locates the ``request-changes``
|
|
1046
|
+
review and re-runs :func:`coord.auto_loop.process_review_completion`
|
|
1047
|
+
on it — the exact function the auto-loop calls the instant a review
|
|
1048
|
+
transitions to done, complete with its iteration cap and terminal
|
|
1049
|
+
guard.
|
|
1050
|
+
- ``review_done_no_verdict`` (#1582) → :func:`coord.diagnose._recover_review`
|
|
1051
|
+
(the exact recovery ``coord diagnose --stage review`` runs: try the
|
|
1052
|
+
session transcript first). A recovered verdict is then run through
|
|
1053
|
+
:func:`coord.auto_loop.process_review_completion` like a normal
|
|
1054
|
+
transition; nothing recoverable falls through to
|
|
1055
|
+
:func:`coord.diagnose._reset_review_stage` (the exact reset
|
|
1056
|
+
``coord diagnose --stage review --reset`` runs — keeps the branch,
|
|
1057
|
+
wipes the review rows + review_state) followed by a fresh
|
|
1058
|
+
:func:`coord.review.dispatch_review` call.
|
|
1059
|
+
- ``done_no_review`` → :func:`coord.review.dispatch_review`, the same
|
|
1060
|
+
call ``detect_transitions``/``dispatch_pending_reviews`` make on a
|
|
1061
|
+
fresh work completion.
|
|
1062
|
+
- ``approved_not_queued`` → :func:`coord.merge_queue.enqueue_approved_work`,
|
|
1063
|
+
the same bulk gate-checked enqueue the daemon passive tick already
|
|
1064
|
+
runs on every interval.
|
|
1065
|
+
- ``merge_conflict_unresolved`` → :func:`coord.conflict_fix.dispatch_conflict_fix`,
|
|
1066
|
+
the #1474 ``_dispatch_conflict_fixes`` path.
|
|
1067
|
+
- ``review_failed_no_verdict`` (#1584) → :func:`coord.review.dispatch_review`
|
|
1068
|
+
again, the SAME call as ``done_no_review`` — the failed review left no
|
|
1069
|
+
verdict behind, so recovery is identical to "no review was ever
|
|
1070
|
+
dispatched": open a fresh one against the still-``done`` work row.
|
|
1071
|
+
|
|
1072
|
+
Never re-entrant across ticks: the caller only reaches this after
|
|
1073
|
+
:func:`detect_stalled_pipeline` has already filtered out any row whose
|
|
1074
|
+
``_stalled_notified_key`` is in the ``notified`` ledger, and the caller
|
|
1075
|
+
marks that key notified right after this returns (via
|
|
1076
|
+
:func:`post_stalled_pipeline` or :func:`post_stalled_pipeline_dispatch`)
|
|
1077
|
+
— so a given assignment_id gets exactly one dispatch attempt per stall,
|
|
1078
|
+
mirroring the one-shot comment (#1441's own guardrail, reused rather
|
|
1079
|
+
than re-derived per #1478's own request).
|
|
1080
|
+
"""
|
|
1081
|
+
if not config.pipeline.auto_dispatch_stalled:
|
|
1082
|
+
return StalledDispatchAction(
|
|
1083
|
+
kind="disabled", detail="pipeline.auto_dispatch_stalled is False",
|
|
1084
|
+
)
|
|
1085
|
+
|
|
1086
|
+
if _stalled_row_has_live_session(board, work):
|
|
1087
|
+
return StalledDispatchAction(
|
|
1088
|
+
kind="skipped_live_session",
|
|
1089
|
+
detail=(
|
|
1090
|
+
f"a running/pending assignment already exists for "
|
|
1091
|
+
f"{work.repo_name}#{work.issue_number} — not acting "
|
|
1092
|
+
"underneath a live session (#602)"
|
|
1093
|
+
),
|
|
1094
|
+
)
|
|
1095
|
+
|
|
1096
|
+
if detection.reason == "review_request_changes_no_fix":
|
|
1097
|
+
from coord.auto_loop import process_review_completion # noqa: PLC0415
|
|
1098
|
+
|
|
1099
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
1100
|
+
review = next(
|
|
1101
|
+
(
|
|
1102
|
+
a for a in all_assignments
|
|
1103
|
+
if a.review_of_assignment_id == work.assignment_id and a.type == "review"
|
|
1104
|
+
),
|
|
1105
|
+
None,
|
|
1106
|
+
)
|
|
1107
|
+
if review is None:
|
|
1108
|
+
return StalledDispatchAction(
|
|
1109
|
+
kind="no_action", detail="review no longer found on board",
|
|
1110
|
+
)
|
|
1111
|
+
machine_host = next(
|
|
1112
|
+
(m.host for m in config.machines if m.name == review.machine_name), None,
|
|
1113
|
+
)
|
|
1114
|
+
actions = process_review_completion(
|
|
1115
|
+
review, board, config,
|
|
1116
|
+
machine_host=machine_host, terminal_cache=terminal_cache,
|
|
1117
|
+
)
|
|
1118
|
+
kind_set = {a.kind for a in actions}
|
|
1119
|
+
kinds = ", ".join(a.kind for a in actions) or "no_action"
|
|
1120
|
+
details = "; ".join(a.detail for a in actions if a.detail)
|
|
1121
|
+
detail_msg = f"process_review_completion → {kinds}" + (f" ({details})" if details else "")
|
|
1122
|
+
# #1478 review fix: `process_review_completion` mutates `board` in
|
|
1123
|
+
# place for several outcomes besides `fix_dispatched` — an
|
|
1124
|
+
# `approved`/`approved_with_nits`/`terminal_skip` resolution still
|
|
1125
|
+
# flips `review.review_verdict`/`work.review_state` and refreshes the
|
|
1126
|
+
# merge-queue entry (see that function's own "caller is responsible
|
|
1127
|
+
# for persisting the board" contract). Classifying those as
|
|
1128
|
+
# `no_action` silently dropped the mutation (the sweep's `board_dirty`
|
|
1129
|
+
# never got set) while the one-shot ledger still marked the row
|
|
1130
|
+
# notified — permanently losing the transition. Any kind in
|
|
1131
|
+
# `_MUTATING_REVIEW_COMPLETION_KINDS` must therefore map to a
|
|
1132
|
+
# `_STALLED_DISPATCH_KINDS` member so `_sweep_stalled_pipeline`
|
|
1133
|
+
# persists it.
|
|
1134
|
+
if "fix_dispatched" in kind_set:
|
|
1135
|
+
return StalledDispatchAction(kind="fix_dispatch_attempted", detail=detail_msg)
|
|
1136
|
+
if kind_set & _MUTATING_REVIEW_COMPLETION_KINDS:
|
|
1137
|
+
return StalledDispatchAction(kind="review_transition_applied", detail=detail_msg)
|
|
1138
|
+
return StalledDispatchAction(kind="no_action", detail=detail_msg)
|
|
1139
|
+
|
|
1140
|
+
if detection.reason == "review_done_no_verdict":
|
|
1141
|
+
# #1582: a review finalised `done` with no verdict ever captured
|
|
1142
|
+
# (#812). Reuse the SAME two steps `coord diagnose --stage review
|
|
1143
|
+
# [--reset]` runs for this exact shape, rather than re-deriving new
|
|
1144
|
+
# recovery/reset logic — `_recover_review`/`_reset_review_stage` are
|
|
1145
|
+
# the private functions behind that command for this branch. Called
|
|
1146
|
+
# directly (not through the full `diagnose_stage` orchestration),
|
|
1147
|
+
# which skips that command's tmux session-state probe and
|
|
1148
|
+
# issue-wide phantom-row cleanup — the review here is already
|
|
1149
|
+
# terminal, so neither applies, and both would add real
|
|
1150
|
+
# subprocess/ssh cost to every notify sweep tick.
|
|
1151
|
+
from coord.diagnose import ( # noqa: PLC0415
|
|
1152
|
+
DiagnoseResult,
|
|
1153
|
+
_recover_review,
|
|
1154
|
+
_reset_review_stage,
|
|
1155
|
+
)
|
|
1156
|
+
|
|
1157
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
1158
|
+
review = next(
|
|
1159
|
+
(
|
|
1160
|
+
a for a in all_assignments
|
|
1161
|
+
if a.review_of_assignment_id == work.assignment_id and a.type == "review"
|
|
1162
|
+
),
|
|
1163
|
+
None,
|
|
1164
|
+
)
|
|
1165
|
+
if review is None:
|
|
1166
|
+
return StalledDispatchAction(
|
|
1167
|
+
kind="no_action", detail="review no longer found on board",
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
diag = DiagnoseResult(
|
|
1171
|
+
repo_name=work.repo_name, issue_number=work.issue_number, stage="review",
|
|
1172
|
+
)
|
|
1173
|
+
# `state="unknown"` is safe: `_recover_review`'s live/dead-session
|
|
1174
|
+
# branches are only reached when `latest.status != "done"`, which
|
|
1175
|
+
# can't happen here (`detect_stalled_pipeline` only flags this
|
|
1176
|
+
# reason for a `status="done"` review).
|
|
1177
|
+
_recover_review(board, config, review, "unknown", diag, dry_run=False)
|
|
1178
|
+
|
|
1179
|
+
if diag.recovered:
|
|
1180
|
+
# A verdict was recovered from the session transcript and
|
|
1181
|
+
# durably persisted (#617's `_review_findings_from_transcript` →
|
|
1182
|
+
# `issue_store.post_result`). Run it through the SAME auto-loop
|
|
1183
|
+
# chokepoint a live review completion would have used — mirrors
|
|
1184
|
+
# `review_request_changes_no_fix` just above — so a recovered
|
|
1185
|
+
# `request-changes` still gets its fix worker and a recovered
|
|
1186
|
+
# `approve` still advances the pipeline.
|
|
1187
|
+
from coord.auto_loop import process_review_completion # noqa: PLC0415
|
|
1188
|
+
|
|
1189
|
+
machine_host = next(
|
|
1190
|
+
(m.host for m in config.machines if m.name == review.machine_name), None,
|
|
1191
|
+
)
|
|
1192
|
+
actions = process_review_completion(
|
|
1193
|
+
review, board, config,
|
|
1194
|
+
machine_host=machine_host, terminal_cache=terminal_cache,
|
|
1195
|
+
)
|
|
1196
|
+
kind_set = {a.kind for a in actions}
|
|
1197
|
+
kinds = ", ".join(a.kind for a in actions) or "no_action"
|
|
1198
|
+
details = "; ".join(a.detail for a in actions if a.detail)
|
|
1199
|
+
detail_msg = (
|
|
1200
|
+
"recovered verdict from the session transcript → "
|
|
1201
|
+
f"process_review_completion → {kinds}" + (f" ({details})" if details else "")
|
|
1202
|
+
)
|
|
1203
|
+
if "fix_dispatched" in kind_set:
|
|
1204
|
+
return StalledDispatchAction(kind="fix_dispatch_attempted", detail=detail_msg)
|
|
1205
|
+
if kind_set & _MUTATING_REVIEW_COMPLETION_KINDS:
|
|
1206
|
+
return StalledDispatchAction(kind="review_transition_applied", detail=detail_msg)
|
|
1207
|
+
return StalledDispatchAction(kind="review_verdict_recovered", detail=detail_msg)
|
|
1208
|
+
|
|
1209
|
+
if not diag.needs_reset:
|
|
1210
|
+
return StalledDispatchAction(
|
|
1211
|
+
kind="no_action", detail="; ".join(diag.findings) or "nothing to do",
|
|
1212
|
+
)
|
|
1213
|
+
|
|
1214
|
+
# Nothing recoverable — reset the review stage (delete the review
|
|
1215
|
+
# rows, clear review_state — #1180's `_reset_review_stage`, KEEPS
|
|
1216
|
+
# the branch/commits) and re-dispatch a fresh review.
|
|
1217
|
+
reset_res = DiagnoseResult(
|
|
1218
|
+
repo_name=work.repo_name, issue_number=work.issue_number, stage="review",
|
|
1219
|
+
)
|
|
1220
|
+
_reset_review_stage(
|
|
1221
|
+
config, work.repo_name, work.issue_number, reset_res,
|
|
1222
|
+
dry_run=False, assignment_id=work.assignment_id,
|
|
1223
|
+
)
|
|
1224
|
+
if not reset_res.reset_performed:
|
|
1225
|
+
return StalledDispatchAction(
|
|
1226
|
+
kind="no_action",
|
|
1227
|
+
detail="reset did not complete: " + "; ".join(reset_res.findings),
|
|
1228
|
+
)
|
|
1229
|
+
|
|
1230
|
+
# `_reset_review_stage` writes the canonical DB directly (the same
|
|
1231
|
+
# seam `coord diagnose --reset` uses — see commands/status.py's
|
|
1232
|
+
# "NOTE: deliberately NO save_board" comment for why) WITHOUT
|
|
1233
|
+
# touching `board`. Mirror the same two writes on `board` in place
|
|
1234
|
+
# so a later `write_board` upsert of the now-stale `review`/`work`
|
|
1235
|
+
# objects doesn't resurrect the just-deleted review row or clobber
|
|
1236
|
+
# the just-cleared review_state back to its wedged value.
|
|
1237
|
+
board.active[:] = [
|
|
1238
|
+
a for a in board.active
|
|
1239
|
+
if not (a.type == "review" and a.review_of_assignment_id == work.assignment_id)
|
|
1240
|
+
]
|
|
1241
|
+
board.completed[:] = [
|
|
1242
|
+
a for a in board.completed
|
|
1243
|
+
if not (a.type == "review" and a.review_of_assignment_id == work.assignment_id)
|
|
1244
|
+
]
|
|
1245
|
+
work.review_state = "pending"
|
|
1246
|
+
work.review_verdict = None
|
|
1247
|
+
work.review_posted_at = None
|
|
1248
|
+
|
|
1249
|
+
from coord.review import dispatch_review # noqa: PLC0415
|
|
1250
|
+
|
|
1251
|
+
new_review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
|
|
1252
|
+
if new_review is None:
|
|
1253
|
+
return StalledDispatchAction(
|
|
1254
|
+
kind="no_action",
|
|
1255
|
+
detail=(
|
|
1256
|
+
"review stage reset (no verdict recoverable) but "
|
|
1257
|
+
"re-dispatch declined (no machine / already in flight / gate)"
|
|
1258
|
+
),
|
|
1259
|
+
)
|
|
1260
|
+
return StalledDispatchAction(
|
|
1261
|
+
kind="review_reset_redispatched",
|
|
1262
|
+
detail=(
|
|
1263
|
+
"no verdict recoverable from transcript — reset the review "
|
|
1264
|
+
f"stage and re-dispatched as {new_review.assignment_id} to "
|
|
1265
|
+
f"{new_review.machine_name}"
|
|
1266
|
+
),
|
|
1267
|
+
)
|
|
1268
|
+
|
|
1269
|
+
if detection.reason == "done_no_review":
|
|
1270
|
+
from coord.review import dispatch_review # noqa: PLC0415
|
|
1271
|
+
|
|
1272
|
+
review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
|
|
1273
|
+
if review is None:
|
|
1274
|
+
return StalledDispatchAction(
|
|
1275
|
+
kind="no_action",
|
|
1276
|
+
detail="dispatch_review declined (no machine / already in flight / gate)",
|
|
1277
|
+
)
|
|
1278
|
+
return StalledDispatchAction(
|
|
1279
|
+
kind="review_dispatched",
|
|
1280
|
+
detail=f"review {review.assignment_id} dispatched to {review.machine_name}",
|
|
1281
|
+
)
|
|
1282
|
+
|
|
1283
|
+
if detection.reason == "review_failed_no_verdict":
|
|
1284
|
+
# #1584: the previous review died with no verdict — recovery is
|
|
1285
|
+
# identical to `done_no_review` above: `work` itself is still
|
|
1286
|
+
# `status="done"` (only the review it spawned failed), so a fresh
|
|
1287
|
+
# `dispatch_review` call is a normal, ungated re-dispatch. Reusing
|
|
1288
|
+
# the same call (rather than e.g. `coord retry` against the dead
|
|
1289
|
+
# review row) also picks up any board state that changed since —
|
|
1290
|
+
# same reasoning `done_no_review` already relies on.
|
|
1291
|
+
from coord.review import dispatch_review # noqa: PLC0415
|
|
1292
|
+
|
|
1293
|
+
# Belt-and-braces against the usage-limit kill (#1461/#1584):
|
|
1294
|
+
# `detect_stalled_pipeline` already skips those rows at
|
|
1295
|
+
# classification, but this function is public and is also reachable
|
|
1296
|
+
# with a caller-built detection, or after a race in which the
|
|
1297
|
+
# usage-limit `failure_reason` was stamped onto the review row
|
|
1298
|
+
# between detection and dispatch. Re-dispatching into an
|
|
1299
|
+
# account-wide exhausted budget only produces another corpse, so
|
|
1300
|
+
# decline — mirroring `_decide_review`'s WAIT in `coord/drive.py`.
|
|
1301
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
1302
|
+
dead_review = next(
|
|
1303
|
+
(
|
|
1304
|
+
a for a in all_assignments
|
|
1305
|
+
if a.review_of_assignment_id == work.assignment_id
|
|
1306
|
+
and a.type == "review"
|
|
1307
|
+
and a.status == "failed"
|
|
1308
|
+
),
|
|
1309
|
+
None,
|
|
1310
|
+
)
|
|
1311
|
+
if dead_review is not None and is_usage_limit_reason(dead_review.failure_reason):
|
|
1312
|
+
return StalledDispatchAction(
|
|
1313
|
+
kind="no_action",
|
|
1314
|
+
detail=(
|
|
1315
|
+
f"review {dead_review.assignment_id} was killed by the "
|
|
1316
|
+
f"usage limit ({dead_review.failure_reason}) — waiting "
|
|
1317
|
+
"for the reset instead of re-dispatching"
|
|
1318
|
+
),
|
|
1319
|
+
)
|
|
1320
|
+
|
|
1321
|
+
review = dispatch_review(work, board, config, terminal_cache=terminal_cache)
|
|
1322
|
+
if review is None:
|
|
1323
|
+
return StalledDispatchAction(
|
|
1324
|
+
kind="no_action",
|
|
1325
|
+
detail="dispatch_review declined (no machine / already in flight / gate)",
|
|
1326
|
+
)
|
|
1327
|
+
return StalledDispatchAction(
|
|
1328
|
+
kind="review_dispatched",
|
|
1329
|
+
detail=f"review {review.assignment_id} dispatched to {review.machine_name}",
|
|
1330
|
+
)
|
|
1331
|
+
|
|
1332
|
+
if detection.reason == "approved_not_queued":
|
|
1333
|
+
from coord.merge_queue import enqueue_approved_work, load_queue # noqa: PLC0415
|
|
1334
|
+
|
|
1335
|
+
changed = enqueue_approved_work(config, board)
|
|
1336
|
+
if work.assignment_id in changed:
|
|
1337
|
+
return StalledDispatchAction(
|
|
1338
|
+
kind="enqueued", detail=f"{work.assignment_id} enqueued for merge",
|
|
1339
|
+
)
|
|
1340
|
+
# #1478 review non-blocking finding: `enqueue_approved_work` bulk-
|
|
1341
|
+
# enqueues EVERY eligible row on `board.completed`, not just this one.
|
|
1342
|
+
# If an earlier row in the same sweep tick already triggered the
|
|
1343
|
+
# enqueue for this assignment, this call's `changed` list comes back
|
|
1344
|
+
# without it (nothing new to do) even though it genuinely is queued —
|
|
1345
|
+
# checking `changed` alone would misreport a real outcome as
|
|
1346
|
+
# `no_action`. Check queue membership directly instead of relying
|
|
1347
|
+
# solely on `changed`.
|
|
1348
|
+
if any(m.assignment_id == work.assignment_id for m in load_queue()):
|
|
1349
|
+
return StalledDispatchAction(
|
|
1350
|
+
kind="enqueued",
|
|
1351
|
+
detail=(
|
|
1352
|
+
f"{work.assignment_id} already enqueued for merge (queued "
|
|
1353
|
+
"earlier in this sweep tick)"
|
|
1354
|
+
),
|
|
1355
|
+
)
|
|
1356
|
+
return StalledDispatchAction(
|
|
1357
|
+
kind="no_action",
|
|
1358
|
+
detail="enqueue_approved_work made no change for this assignment",
|
|
1359
|
+
)
|
|
1360
|
+
|
|
1361
|
+
if detection.reason == "merge_conflict_unresolved":
|
|
1362
|
+
from coord.conflict_fix import ( # noqa: PLC0415
|
|
1363
|
+
dispatch_conflict_fix,
|
|
1364
|
+
has_prior_conflict_fix,
|
|
1365
|
+
)
|
|
1366
|
+
from coord.merge_queue import load_queue # noqa: PLC0415
|
|
1367
|
+
|
|
1368
|
+
entry = next(
|
|
1369
|
+
(m for m in load_queue() if m.assignment_id == work.assignment_id), None,
|
|
1370
|
+
)
|
|
1371
|
+
if entry is None:
|
|
1372
|
+
return StalledDispatchAction(
|
|
1373
|
+
kind="no_action", detail="merge queue entry no longer found",
|
|
1374
|
+
)
|
|
1375
|
+
if has_prior_conflict_fix(board, entry.assignment_id):
|
|
1376
|
+
return StalledDispatchAction(
|
|
1377
|
+
kind="skipped_human_required",
|
|
1378
|
+
detail="conflict-fix already active or its retry cap was already hit",
|
|
1379
|
+
)
|
|
1380
|
+
fix = dispatch_conflict_fix(entry, board, config, prefer_machine=work.machine_name)
|
|
1381
|
+
if fix is None:
|
|
1382
|
+
return StalledDispatchAction(
|
|
1383
|
+
kind="no_action",
|
|
1384
|
+
detail="dispatch_conflict_fix declined (no machine / no repo_path)",
|
|
1385
|
+
)
|
|
1386
|
+
return StalledDispatchAction(
|
|
1387
|
+
kind="conflict_fix_dispatched",
|
|
1388
|
+
detail=f"conflict-fix {fix.assignment_id} dispatched to {fix.machine_name}",
|
|
1389
|
+
)
|
|
1390
|
+
|
|
1391
|
+
return StalledDispatchAction(
|
|
1392
|
+
kind="no_action", detail=f"no dispatch arm for reason={detection.reason!r}",
|
|
1393
|
+
)
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def post_stalled_pipeline_dispatch(
|
|
1397
|
+
detection: StalledDetection, action: StalledDispatchAction, config: Config,
|
|
1398
|
+
) -> None:
|
|
1399
|
+
"""Post the #1478 auto-dispatch outcome comment and mark notified.
|
|
1400
|
+
|
|
1401
|
+
Posted INSTEAD OF :func:`post_stalled_pipeline` when
|
|
1402
|
+
:func:`dispatch_stalled_pipeline_action` actually dispatched something
|
|
1403
|
+
for this row (see that function's *kind* values) — the two write to the
|
|
1404
|
+
same GitHub thread, so posting both would leave a directly
|
|
1405
|
+
contradictory "nothing was dispatched automatically" comment sitting
|
|
1406
|
+
right above this one.
|
|
1407
|
+
"""
|
|
1408
|
+
repo = config.repo(detection.repo_name)
|
|
1409
|
+
repo_github = repo.github if repo is not None else None
|
|
1410
|
+
if not repo_github:
|
|
1411
|
+
return
|
|
1412
|
+
body = format_stalled_pipeline_dispatch(
|
|
1413
|
+
assignment_id=detection.assignment_id,
|
|
1414
|
+
repo_name=detection.repo_name,
|
|
1415
|
+
issue_number=detection.issue_number,
|
|
1416
|
+
reason=detection.reason,
|
|
1417
|
+
action_kind=action.kind,
|
|
1418
|
+
action_detail=action.detail,
|
|
1419
|
+
)
|
|
1420
|
+
github_ops.post_issue_comment(repo_github, detection.issue_number, body)
|
|
1421
|
+
mark_notified(_stalled_notified_key(detection.assignment_id), EVENT_STALLED)
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def _agent_status(host: str, port: int = AGENT_PORT, timeout: float = 5.0) -> dict | None:
|
|
1425
|
+
try:
|
|
1426
|
+
resp = httpx.get(f"http://{host}:{port}/status", timeout=timeout)
|
|
1427
|
+
resp.raise_for_status()
|
|
1428
|
+
return resp.json()
|
|
1429
|
+
except (httpx.HTTPError, httpx.TimeoutException):
|
|
1430
|
+
return None
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
def detect_transitions(config: Config) -> list[tuple[Transition, dict, dict]]:
|
|
1434
|
+
"""Return (transition, dispatch_record, agent_assignment) for each
|
|
1435
|
+
assignment whose terminal state has not yet been notified.
|
|
1436
|
+
|
|
1437
|
+
Splitting detection from posting makes the loop testable without
|
|
1438
|
+
mocking GitHub.
|
|
1439
|
+
"""
|
|
1440
|
+
dispatched = load_dispatched()
|
|
1441
|
+
if not dispatched:
|
|
1442
|
+
return []
|
|
1443
|
+
notified = load_notified()
|
|
1444
|
+
by_id = {r["assignment_id"]: r for r in dispatched}
|
|
1445
|
+
|
|
1446
|
+
# Collect machine hostnames we care about
|
|
1447
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
1448
|
+
needed = {r["machine_name"] for r in dispatched if r["assignment_id"] not in notified}
|
|
1449
|
+
|
|
1450
|
+
transitions: list[tuple[Transition, dict, dict]] = []
|
|
1451
|
+
for machine_name in needed:
|
|
1452
|
+
machine = machines_by_name.get(machine_name)
|
|
1453
|
+
if machine is None:
|
|
1454
|
+
continue
|
|
1455
|
+
status = _agent_status(machine.host)
|
|
1456
|
+
if status is None:
|
|
1457
|
+
continue
|
|
1458
|
+
for entry in status.get("completed", []):
|
|
1459
|
+
aid = entry.get("id")
|
|
1460
|
+
record = by_id.get(aid)
|
|
1461
|
+
if record is None or aid in notified:
|
|
1462
|
+
continue
|
|
1463
|
+
entry_status = entry.get("status")
|
|
1464
|
+
# Cancelled-on-agent for an assignment the DB already marks done
|
|
1465
|
+
# is cleanup noise (e.g. operator ran POST /cancel to unstick a
|
|
1466
|
+
# hung reap). Don't post a false failure for it.
|
|
1467
|
+
db_status = (record.get("status") or "").lower()
|
|
1468
|
+
if entry_status == "cancelled" and db_status == "done":
|
|
1469
|
+
continue
|
|
1470
|
+
if entry_status == "done":
|
|
1471
|
+
event = EVENT_COMPLETION
|
|
1472
|
+
elif entry_status in ("failed", "cancelled"):
|
|
1473
|
+
event = EVENT_FAILURE
|
|
1474
|
+
elif entry_status == "advisory":
|
|
1475
|
+
# #448: advisory (0-commit clean exit) — post a distinctive
|
|
1476
|
+
# GitHub comment so operators who rely on GitHub (not just
|
|
1477
|
+
# coord status) know the worker finished with no code change
|
|
1478
|
+
# and that human review is needed.
|
|
1479
|
+
event = EVENT_ADVISORY
|
|
1480
|
+
else:
|
|
1481
|
+
continue
|
|
1482
|
+
transitions.append(
|
|
1483
|
+
(
|
|
1484
|
+
Transition(
|
|
1485
|
+
assignment_id=aid,
|
|
1486
|
+
machine_name=record["machine_name"],
|
|
1487
|
+
repo_name=record["repo_name"],
|
|
1488
|
+
issue_number=record["issue_number"],
|
|
1489
|
+
event=event,
|
|
1490
|
+
exit_code=entry.get("exit_code"),
|
|
1491
|
+
),
|
|
1492
|
+
record,
|
|
1493
|
+
entry,
|
|
1494
|
+
)
|
|
1495
|
+
)
|
|
1496
|
+
return transitions
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def detect_stuck(config: Config) -> list[tuple[StuckDetection, dict]]:
|
|
1500
|
+
"""Scan active worker logs for STUCK signals.
|
|
1501
|
+
|
|
1502
|
+
Returns (StuckDetection, dispatch_record) for each stuck worker that
|
|
1503
|
+
hasn't already been notified as stuck.
|
|
1504
|
+
"""
|
|
1505
|
+
dispatched = load_dispatched()
|
|
1506
|
+
if not dispatched:
|
|
1507
|
+
return []
|
|
1508
|
+
notified = load_notified()
|
|
1509
|
+
by_id = {r["assignment_id"]: r for r in dispatched}
|
|
1510
|
+
|
|
1511
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
1512
|
+
|
|
1513
|
+
# Only look at assignments that haven't been notified at all (still active)
|
|
1514
|
+
# and haven't already been notified as stuck.
|
|
1515
|
+
active_records = [
|
|
1516
|
+
r for r in dispatched
|
|
1517
|
+
if r["assignment_id"] not in notified
|
|
1518
|
+
and _stuck_notified_key(r["assignment_id"]) not in notified
|
|
1519
|
+
]
|
|
1520
|
+
if not active_records:
|
|
1521
|
+
return []
|
|
1522
|
+
|
|
1523
|
+
# Group by machine
|
|
1524
|
+
by_machine: dict[str, list[dict]] = {}
|
|
1525
|
+
for r in active_records:
|
|
1526
|
+
by_machine.setdefault(r["machine_name"], []).append(r)
|
|
1527
|
+
|
|
1528
|
+
results: list[tuple[StuckDetection, dict]] = []
|
|
1529
|
+
for machine_name, records in by_machine.items():
|
|
1530
|
+
machine = machines_by_name.get(machine_name)
|
|
1531
|
+
if machine is None:
|
|
1532
|
+
continue
|
|
1533
|
+
status = _agent_status(machine.host)
|
|
1534
|
+
if status is None:
|
|
1535
|
+
continue
|
|
1536
|
+
|
|
1537
|
+
# Build lookup of active entries by id
|
|
1538
|
+
active_by_id: dict[str, dict] = {}
|
|
1539
|
+
for entry in status.get("active", []):
|
|
1540
|
+
eid = entry.get("id")
|
|
1541
|
+
if eid:
|
|
1542
|
+
active_by_id[eid] = entry
|
|
1543
|
+
|
|
1544
|
+
for record in records:
|
|
1545
|
+
aid = record["assignment_id"]
|
|
1546
|
+
entry = active_by_id.get(aid)
|
|
1547
|
+
if entry is None:
|
|
1548
|
+
continue
|
|
1549
|
+
|
|
1550
|
+
stuck_message: str | None = None
|
|
1551
|
+
log_path: str | None = None
|
|
1552
|
+
|
|
1553
|
+
# Check progress data from agent status
|
|
1554
|
+
progress = entry.get("progress")
|
|
1555
|
+
if progress and progress.get("stuck"):
|
|
1556
|
+
stuck_message = progress["stuck"]
|
|
1557
|
+
log_path = entry.get("log_path")
|
|
1558
|
+
|
|
1559
|
+
# Also try parsing the log file directly
|
|
1560
|
+
entry_log = entry.get("log_path")
|
|
1561
|
+
if entry_log and not stuck_message:
|
|
1562
|
+
try:
|
|
1563
|
+
# #1710: thread the dispatch record's resolved provider
|
|
1564
|
+
# name through so a non-claude worker's log parses via
|
|
1565
|
+
# its own provider rather than always assuming claude.
|
|
1566
|
+
parsed = parse_progress(
|
|
1567
|
+
entry_log, provider_name=record.get("provider_name"),
|
|
1568
|
+
)
|
|
1569
|
+
if parsed.stuck:
|
|
1570
|
+
stuck_message = parsed.stuck
|
|
1571
|
+
log_path = entry_log
|
|
1572
|
+
except Exception: # noqa: BLE001
|
|
1573
|
+
pass
|
|
1574
|
+
|
|
1575
|
+
if stuck_message:
|
|
1576
|
+
results.append(
|
|
1577
|
+
(
|
|
1578
|
+
StuckDetection(
|
|
1579
|
+
assignment_id=aid,
|
|
1580
|
+
machine_name=record["machine_name"],
|
|
1581
|
+
repo_name=record["repo_name"],
|
|
1582
|
+
issue_number=record["issue_number"],
|
|
1583
|
+
stuck_message=stuck_message,
|
|
1584
|
+
log_path=log_path,
|
|
1585
|
+
),
|
|
1586
|
+
record,
|
|
1587
|
+
)
|
|
1588
|
+
)
|
|
1589
|
+
|
|
1590
|
+
return results
|
|
1591
|
+
|
|
1592
|
+
|
|
1593
|
+
def post_stuck(detection: StuckDetection, record: dict) -> None:
|
|
1594
|
+
"""Post a stuck comment to GitHub and mark notified."""
|
|
1595
|
+
body = format_stuck(
|
|
1596
|
+
assignment_id=detection.assignment_id,
|
|
1597
|
+
machine_name=detection.machine_name,
|
|
1598
|
+
repo_name=detection.repo_name,
|
|
1599
|
+
issue_number=detection.issue_number,
|
|
1600
|
+
stuck_message=detection.stuck_message,
|
|
1601
|
+
)
|
|
1602
|
+
github_ops.post_issue_comment(
|
|
1603
|
+
record["repo_github"], detection.issue_number, body
|
|
1604
|
+
)
|
|
1605
|
+
mark_notified(_stuck_notified_key(detection.assignment_id), EVENT_STUCK)
|
|
1606
|
+
|
|
1607
|
+
|
|
1608
|
+
def _capture_completion_summary(transition: Transition, entry: dict) -> None:
|
|
1609
|
+
"""#874: parse the worker's ### Summary block and persist it on the row.
|
|
1610
|
+
|
|
1611
|
+
Tries the local log first, then falls back to the agent's /logs/<id>
|
|
1612
|
+
endpoint for remote-agent assignments. Silent on failure — a worker
|
|
1613
|
+
that emits no summary leaves the field NULL without error.
|
|
1614
|
+
"""
|
|
1615
|
+
from coord.progress import ( # noqa: PLC0415
|
|
1616
|
+
parse_completion_summary_from_agent,
|
|
1617
|
+
parse_completion_summary_from_log,
|
|
1618
|
+
)
|
|
1619
|
+
from coord.state import update_assignment_completion_summary # noqa: PLC0415
|
|
1620
|
+
|
|
1621
|
+
prose: str | None = None
|
|
1622
|
+
log_path = entry.get("log_path")
|
|
1623
|
+
if log_path:
|
|
1624
|
+
try:
|
|
1625
|
+
prose = parse_completion_summary_from_log(Path(log_path))
|
|
1626
|
+
except Exception as exc: # noqa: BLE001
|
|
1627
|
+
log.debug(
|
|
1628
|
+
"_capture_completion_summary: failed to parse local log for %s: %s",
|
|
1629
|
+
transition.assignment_id, exc,
|
|
1630
|
+
)
|
|
1631
|
+
|
|
1632
|
+
if prose is None:
|
|
1633
|
+
# Local log unavailable (remote-agent assignment) — fetch via the
|
|
1634
|
+
# agent's /logs/<id> endpoint. Same fallback used by smoke tests.
|
|
1635
|
+
host = _agent_host(transition.machine_name)
|
|
1636
|
+
if host:
|
|
1637
|
+
try:
|
|
1638
|
+
prose = parse_completion_summary_from_agent(host, transition.assignment_id)
|
|
1639
|
+
except Exception as exc: # noqa: BLE001
|
|
1640
|
+
log.debug(
|
|
1641
|
+
"_capture_completion_summary: failed to fetch from agent %s for %s: %s",
|
|
1642
|
+
host, transition.assignment_id, exc,
|
|
1643
|
+
)
|
|
1644
|
+
|
|
1645
|
+
if prose is None:
|
|
1646
|
+
# No ### Summary block anywhere — leave completion_summary NULL.
|
|
1647
|
+
return
|
|
1648
|
+
try:
|
|
1649
|
+
update_assignment_completion_summary(transition.assignment_id, prose)
|
|
1650
|
+
except Exception as exc: # noqa: BLE001
|
|
1651
|
+
log.warning(
|
|
1652
|
+
"_capture_completion_summary: failed to persist summary for %s: %s",
|
|
1653
|
+
transition.assignment_id, exc,
|
|
1654
|
+
)
|
|
1655
|
+
|
|
1656
|
+
|
|
1657
|
+
def _capture_smoke_tests(transition: Transition, entry: dict) -> None:
|
|
1658
|
+
"""#252: parse the worker's SMOKE_TESTS block and persist it on the row.
|
|
1659
|
+
|
|
1660
|
+
Tries the local log first, then falls back to the agent's /logs/<id>
|
|
1661
|
+
endpoint for remote-agent assignments (mirrors the plan and review
|
|
1662
|
+
capture paths). Silent on failure.
|
|
1663
|
+
"""
|
|
1664
|
+
from coord.progress import ( # noqa: PLC0415
|
|
1665
|
+
parse_smoke_tests_from_agent,
|
|
1666
|
+
parse_smoke_tests_from_log,
|
|
1667
|
+
)
|
|
1668
|
+
from coord.state import update_assignment_smoke_tests # noqa: PLC0415
|
|
1669
|
+
|
|
1670
|
+
parsed: list[str] | None = None
|
|
1671
|
+
log_path = entry.get("log_path")
|
|
1672
|
+
if log_path:
|
|
1673
|
+
try:
|
|
1674
|
+
parsed = parse_smoke_tests_from_log(Path(log_path))
|
|
1675
|
+
except Exception as exc: # noqa: BLE001
|
|
1676
|
+
log.debug(
|
|
1677
|
+
"_capture_smoke_tests: failed to parse local log for %s: %s",
|
|
1678
|
+
transition.assignment_id, exc,
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
if parsed is None:
|
|
1682
|
+
# Local log unavailable (remote-agent assignment) — fetch via the
|
|
1683
|
+
# agent's /logs/<id> endpoint. Same fallback the plan and review
|
|
1684
|
+
# paths use.
|
|
1685
|
+
host = _agent_host(transition.machine_name)
|
|
1686
|
+
if host:
|
|
1687
|
+
try:
|
|
1688
|
+
parsed = parse_smoke_tests_from_agent(host, transition.assignment_id)
|
|
1689
|
+
except Exception as exc: # noqa: BLE001
|
|
1690
|
+
log.debug(
|
|
1691
|
+
"_capture_smoke_tests: failed to fetch from agent %s for %s: %s",
|
|
1692
|
+
host, transition.assignment_id, exc,
|
|
1693
|
+
)
|
|
1694
|
+
|
|
1695
|
+
if parsed is None:
|
|
1696
|
+
# No SMOKE_TESTS block anywhere — leave smoke_tests NULL so the
|
|
1697
|
+
# TUI shows the graceful-degradation placeholder.
|
|
1698
|
+
return
|
|
1699
|
+
try:
|
|
1700
|
+
update_assignment_smoke_tests(transition.assignment_id, parsed)
|
|
1701
|
+
except Exception as exc: # noqa: BLE001
|
|
1702
|
+
log.warning(
|
|
1703
|
+
"_capture_smoke_tests: failed to persist list for %s: %s",
|
|
1704
|
+
transition.assignment_id, exc,
|
|
1705
|
+
)
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
def _capture_cost(transition: Transition, entry: dict, record: dict | None = None) -> None:
|
|
1709
|
+
"""#208/#546: parse the worker's final cost+tokens and persist them.
|
|
1710
|
+
|
|
1711
|
+
Preferred source is the local stream-json log (cheap, no network).
|
|
1712
|
+
Falls back to the agent's status entry, which carries ``cost_so_far``
|
|
1713
|
+
/ ``total_cost_usd`` reported live by the worker. Tokens are only
|
|
1714
|
+
available from the log (not from the agent status dict), so they are
|
|
1715
|
+
captured when the local log exists. Either path is best-effort —
|
|
1716
|
+
failure is silent so it can't block the comment post.
|
|
1717
|
+
|
|
1718
|
+
#1710: *record* (the dispatch record from ``load_dispatched()``) carries
|
|
1719
|
+
``provider_name`` — threaded into :func:`coord.usage.parse_usage_from_log`
|
|
1720
|
+
so cost/token parsing uses the assignment's actual provider instead of
|
|
1721
|
+
always assuming claude. ``None`` (no record, or predates #324) falls back
|
|
1722
|
+
to the claude default, unchanged from before #1710.
|
|
1723
|
+
"""
|
|
1724
|
+
from coord.state import update_assignment_cost, update_assignment_tokens # noqa: PLC0415
|
|
1725
|
+
from coord.usage import parse_usage_from_log # noqa: PLC0415
|
|
1726
|
+
|
|
1727
|
+
cost: float | None = None
|
|
1728
|
+
input_tokens: int = 0
|
|
1729
|
+
output_tokens: int = 0
|
|
1730
|
+
cache_creation_tokens: int = 0
|
|
1731
|
+
cache_read_tokens: int = 0
|
|
1732
|
+
provider_name = (record or {}).get("provider_name")
|
|
1733
|
+
|
|
1734
|
+
log_path = entry.get("log_path")
|
|
1735
|
+
if log_path:
|
|
1736
|
+
try:
|
|
1737
|
+
parsed = parse_usage_from_log(Path(log_path), provider_name=provider_name)
|
|
1738
|
+
if parsed is not None:
|
|
1739
|
+
if parsed.total_cost_usd > 0:
|
|
1740
|
+
cost = parsed.total_cost_usd
|
|
1741
|
+
# #546: also capture token counts from the same parse.
|
|
1742
|
+
input_tokens = parsed.input_tokens
|
|
1743
|
+
output_tokens = parsed.output_tokens
|
|
1744
|
+
cache_creation_tokens = parsed.cache_creation_tokens
|
|
1745
|
+
cache_read_tokens = parsed.cache_read_tokens
|
|
1746
|
+
except Exception as exc: # noqa: BLE001
|
|
1747
|
+
log.debug(
|
|
1748
|
+
"_capture_cost: failed to parse log for %s: %s",
|
|
1749
|
+
transition.assignment_id, exc,
|
|
1750
|
+
)
|
|
1751
|
+
|
|
1752
|
+
if cost is None:
|
|
1753
|
+
# Fall back to the live value the agent had at reap time.
|
|
1754
|
+
remote_cost = entry.get("total_cost_usd") or entry.get("cost_so_far")
|
|
1755
|
+
if remote_cost is not None:
|
|
1756
|
+
try:
|
|
1757
|
+
cost = float(remote_cost)
|
|
1758
|
+
except (TypeError, ValueError):
|
|
1759
|
+
cost = None
|
|
1760
|
+
|
|
1761
|
+
# #667: token fallback — when the local log was absent/unreadable the
|
|
1762
|
+
# token counts are still 0. The agent now includes them in the /status
|
|
1763
|
+
# completed entry, so read them from there.
|
|
1764
|
+
if input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens == 0:
|
|
1765
|
+
try:
|
|
1766
|
+
input_tokens = int(entry.get("input_tokens") or 0)
|
|
1767
|
+
output_tokens = int(entry.get("output_tokens") or 0)
|
|
1768
|
+
cache_creation_tokens = int(entry.get("cache_creation_tokens") or 0)
|
|
1769
|
+
cache_read_tokens = int(entry.get("cache_read_tokens") or 0)
|
|
1770
|
+
except (TypeError, ValueError):
|
|
1771
|
+
pass
|
|
1772
|
+
|
|
1773
|
+
if cost is not None and cost > 0:
|
|
1774
|
+
try:
|
|
1775
|
+
update_assignment_cost(transition.assignment_id, cost)
|
|
1776
|
+
except Exception as exc: # noqa: BLE001
|
|
1777
|
+
log.warning(
|
|
1778
|
+
"_capture_cost: failed to persist cost for %s: %s",
|
|
1779
|
+
transition.assignment_id, exc,
|
|
1780
|
+
)
|
|
1781
|
+
|
|
1782
|
+
# #546: persist token counts (best-effort; silent on missing columns).
|
|
1783
|
+
if input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens > 0:
|
|
1784
|
+
try:
|
|
1785
|
+
update_assignment_tokens(
|
|
1786
|
+
transition.assignment_id,
|
|
1787
|
+
input_tokens=input_tokens,
|
|
1788
|
+
output_tokens=output_tokens,
|
|
1789
|
+
cache_creation_tokens=cache_creation_tokens,
|
|
1790
|
+
cache_read_tokens=cache_read_tokens,
|
|
1791
|
+
)
|
|
1792
|
+
except Exception as exc: # noqa: BLE001
|
|
1793
|
+
log.warning(
|
|
1794
|
+
"_capture_cost: failed to persist tokens for %s: %s",
|
|
1795
|
+
transition.assignment_id, exc,
|
|
1796
|
+
)
|
|
1797
|
+
|
|
1798
|
+
|
|
1799
|
+
def _persist_review_verdict(assignment_id: str, verdict: str) -> None:
|
|
1800
|
+
"""Store the parsed reviewer verdict on the review assignment row.
|
|
1801
|
+
|
|
1802
|
+
#253: consumed by ``coord.merge_queue.has_approved_review`` so the merge
|
|
1803
|
+
gate can refuse to merge work whose review hasn't approved. Best-effort;
|
|
1804
|
+
a DB error is logged and swallowed (the merge gate falls back to "no
|
|
1805
|
+
approval found" which is the safe answer).
|
|
1806
|
+
"""
|
|
1807
|
+
if verdict not in ("approve", "request-changes"):
|
|
1808
|
+
return
|
|
1809
|
+
try:
|
|
1810
|
+
from coord.db import get_connection # noqa: PLC0415
|
|
1811
|
+
|
|
1812
|
+
conn = get_connection()
|
|
1813
|
+
with conn:
|
|
1814
|
+
conn.execute(
|
|
1815
|
+
"UPDATE assignments SET review_verdict = ? WHERE assignment_id = ?",
|
|
1816
|
+
(verdict, assignment_id),
|
|
1817
|
+
)
|
|
1818
|
+
except Exception as exc: # noqa: BLE001
|
|
1819
|
+
log.warning(
|
|
1820
|
+
"Failed to persist review_verdict for %s: %s", assignment_id, exc
|
|
1821
|
+
)
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
def _persist_review_findings(assignment_id: str, verdict: str, body: str) -> None:
|
|
1825
|
+
"""#bounce: persist both verdict + findings body in one shot.
|
|
1826
|
+
|
|
1827
|
+
Mirrors `_persist_review_verdict` (which we keep for callers that
|
|
1828
|
+
only have the verdict) but also caches the body so `coord bounce`
|
|
1829
|
+
can skip the slow HTTP log fetch. Best-effort; a DB error is
|
|
1830
|
+
logged and swallowed.
|
|
1831
|
+
"""
|
|
1832
|
+
if verdict not in ("approve", "request-changes"):
|
|
1833
|
+
return
|
|
1834
|
+
try:
|
|
1835
|
+
from coord.state import update_assignment_review_findings # noqa: PLC0415
|
|
1836
|
+
|
|
1837
|
+
update_assignment_review_findings(
|
|
1838
|
+
assignment_id, verdict=verdict, body=body,
|
|
1839
|
+
)
|
|
1840
|
+
except Exception as exc: # noqa: BLE001
|
|
1841
|
+
log.warning(
|
|
1842
|
+
"Failed to persist review_findings for %s: %s", assignment_id, exc
|
|
1843
|
+
)
|
|
1844
|
+
|
|
1845
|
+
|
|
1846
|
+
def _fetch_raw_log_text(transition: Transition, entry: dict) -> str | None:
|
|
1847
|
+
"""Best-effort raw log text for #1956/#1348 diagnostics.
|
|
1848
|
+
|
|
1849
|
+
Mirrors the local-file-then-agent-fetch fallback :func:`_try_parse_and_post_review`
|
|
1850
|
+
itself uses to PARSE the log, but returns the raw text instead — the
|
|
1851
|
+
diagnostic detectors (:func:`coord.review.detect_end_review_without_verdict`,
|
|
1852
|
+
:func:`coord.review.detect_unparsed_review_marker`) need the text the
|
|
1853
|
+
strict parser already rejected, not another parse attempt. Returns
|
|
1854
|
+
``None`` on any I/O failure — diagnostics are best-effort by design and
|
|
1855
|
+
must never be the reason ``coord notify`` raises.
|
|
1856
|
+
"""
|
|
1857
|
+
log_path = entry.get("log_path")
|
|
1858
|
+
if log_path:
|
|
1859
|
+
try:
|
|
1860
|
+
return Path(log_path).read_text(encoding="utf-8", errors="replace")
|
|
1861
|
+
except OSError:
|
|
1862
|
+
pass
|
|
1863
|
+
host = _agent_host(transition.machine_name)
|
|
1864
|
+
if host:
|
|
1865
|
+
try:
|
|
1866
|
+
resp = httpx.get(
|
|
1867
|
+
f"http://{host}:{AGENT_PORT}/logs/{transition.assignment_id}",
|
|
1868
|
+
timeout=15.0,
|
|
1869
|
+
)
|
|
1870
|
+
resp.raise_for_status()
|
|
1871
|
+
return resp.text
|
|
1872
|
+
except (httpx.HTTPError, httpx.TimeoutException):
|
|
1873
|
+
return None
|
|
1874
|
+
return None
|
|
1875
|
+
|
|
1876
|
+
|
|
1877
|
+
def _warn_missing_review_verdict(
|
|
1878
|
+
transition: Transition, entry: dict, diagnostic: list,
|
|
1879
|
+
) -> None:
|
|
1880
|
+
"""#1956: when a review's structured verdict could not be parsed, make it
|
|
1881
|
+
LOUD instead of silent — run the #1348/#1956 diagnostics against the raw
|
|
1882
|
+
log text and ``log.warning`` a recovery command.
|
|
1883
|
+
|
|
1884
|
+
Before this, a review that reached ``END_REVIEW`` with a full body but
|
|
1885
|
+
no ``REVIEW_VERDICT:`` header (quadraui#533's live incident — grepping
|
|
1886
|
+
the raw log found the string exactly once, inside the briefing's own
|
|
1887
|
+
instructions, never in an assistant message) landed ``status="done"``
|
|
1888
|
+
with ``review_verdict IS NULL`` and nothing anywhere said so; the merge
|
|
1889
|
+
gate just read ``review_required`` forever. Appends whichever
|
|
1890
|
+
diagnostic fired (if any) to *diagnostic* so :func:`post_transition` can
|
|
1891
|
+
tailor the GitHub-visible completion comment too — the operator should
|
|
1892
|
+
not have to go spelunking in ``coord notify``'s own log to learn this.
|
|
1893
|
+
Best-effort throughout: a failure to even fetch the raw text is
|
|
1894
|
+
swallowed, matching this module's "never crash notify" contract.
|
|
1895
|
+
"""
|
|
1896
|
+
from coord.review import ( # noqa: PLC0415
|
|
1897
|
+
detect_end_review_without_verdict,
|
|
1898
|
+
detect_unparsed_review_marker,
|
|
1899
|
+
)
|
|
1900
|
+
|
|
1901
|
+
text = _fetch_raw_log_text(transition, entry)
|
|
1902
|
+
if not text:
|
|
1903
|
+
return
|
|
1904
|
+
aid = transition.assignment_id
|
|
1905
|
+
log_path = entry.get("log_path")
|
|
1906
|
+
recover_hint = (
|
|
1907
|
+
f"coord report-result --assignment {aid} "
|
|
1908
|
+
"--verdict <approve|request-changes> --verdict-source recovered "
|
|
1909
|
+
'--verdict-reason "<why>" --body-file <extracted-review.md>'
|
|
1910
|
+
)
|
|
1911
|
+
|
|
1912
|
+
end_marker = detect_end_review_without_verdict(text, transcript_path=log_path)
|
|
1913
|
+
if end_marker is not None:
|
|
1914
|
+
log.warning(
|
|
1915
|
+
"review %s: reviewer wrote END_REVIEW but never emitted "
|
|
1916
|
+
"REVIEW_VERDICT: anywhere (#1956) — this is NOT a crashed/"
|
|
1917
|
+
"truncated session, the verdict is very likely recoverable "
|
|
1918
|
+
"from the transcript. Recover with:\n %s\nExcerpt before "
|
|
1919
|
+
"END_REVIEW:\n%s",
|
|
1920
|
+
aid, recover_hint, end_marker.excerpt,
|
|
1921
|
+
)
|
|
1922
|
+
diagnostic.append(end_marker)
|
|
1923
|
+
return
|
|
1924
|
+
|
|
1925
|
+
marker = detect_unparsed_review_marker(text, transcript_path=log_path)
|
|
1926
|
+
if marker is not None:
|
|
1927
|
+
log.warning(
|
|
1928
|
+
"review %s: a REVIEW_VERDICT: marker is present but malformed "
|
|
1929
|
+
"(#1348, detected word=%r) — the strict parser rejected it. "
|
|
1930
|
+
"Recover with:\n %s",
|
|
1931
|
+
aid, marker.verdict_word, recover_hint,
|
|
1932
|
+
)
|
|
1933
|
+
diagnostic.append(marker)
|
|
1934
|
+
return
|
|
1935
|
+
|
|
1936
|
+
log.debug(
|
|
1937
|
+
"review %s: no REVIEW_VERDICT:/END_REVIEW markers found at all — "
|
|
1938
|
+
"likely a crashed or truncated session, not a #1956/#1348 "
|
|
1939
|
+
"recoverable case",
|
|
1940
|
+
aid,
|
|
1941
|
+
)
|
|
1942
|
+
|
|
1943
|
+
|
|
1944
|
+
def _try_parse_and_post_review(
|
|
1945
|
+
transition: Transition,
|
|
1946
|
+
record: dict,
|
|
1947
|
+
entry: dict,
|
|
1948
|
+
duration: float | None,
|
|
1949
|
+
*,
|
|
1950
|
+
_diagnostic: list | None = None,
|
|
1951
|
+
) -> bool:
|
|
1952
|
+
"""Parse reviewer findings from the log and post as a PR review or issue comment.
|
|
1953
|
+
|
|
1954
|
+
Returns True if a review was successfully posted (either as a ``gh pr review``
|
|
1955
|
+
or as an issue comment when no PR number is available), False on any failure.
|
|
1956
|
+
Silently swallows all errors so callers can fall back gracefully.
|
|
1957
|
+
|
|
1958
|
+
*_diagnostic* (#1956): optional out-parameter, mirroring
|
|
1959
|
+
``coord.interactive``'s identically-shaped convention for #1348. When a
|
|
1960
|
+
list is supplied and the structured verdict cannot be parsed, whichever
|
|
1961
|
+
of :func:`coord.review.detect_end_review_without_verdict` /
|
|
1962
|
+
:func:`coord.review.detect_unparsed_review_marker` fires is appended to
|
|
1963
|
+
it, so the caller can tailor the fallback GitHub comment instead of a
|
|
1964
|
+
generic "could not be extracted" message every single time.
|
|
1965
|
+
"""
|
|
1966
|
+
from coord.review import parse_review_from_log, parse_review_from_agent # noqa: PLC0415
|
|
1967
|
+
|
|
1968
|
+
log_path = entry.get("log_path")
|
|
1969
|
+
findings = None
|
|
1970
|
+
if log_path:
|
|
1971
|
+
try:
|
|
1972
|
+
findings = parse_review_from_log(log_path)
|
|
1973
|
+
except Exception as exc: # noqa: BLE001
|
|
1974
|
+
log.warning("Failed to parse review log for %s: %s", transition.assignment_id, exc)
|
|
1975
|
+
|
|
1976
|
+
# Local file unavailable (worker ran on a remote agent whose log isn't on
|
|
1977
|
+
# this filesystem) — fetch via the agent's /logs endpoint and parse the
|
|
1978
|
+
# same way. Agents never use gh; the coordinator pulls + posts.
|
|
1979
|
+
if findings is None:
|
|
1980
|
+
host = _agent_host(transition.machine_name)
|
|
1981
|
+
if host:
|
|
1982
|
+
try:
|
|
1983
|
+
findings = parse_review_from_agent(host, transition.assignment_id)
|
|
1984
|
+
except Exception as exc: # noqa: BLE001
|
|
1985
|
+
log.warning(
|
|
1986
|
+
"Failed to fetch review log from agent %s for %s: %s",
|
|
1987
|
+
host, transition.assignment_id, exc,
|
|
1988
|
+
)
|
|
1989
|
+
|
|
1990
|
+
if findings is None:
|
|
1991
|
+
if _diagnostic is not None:
|
|
1992
|
+
try:
|
|
1993
|
+
_warn_missing_review_verdict(transition, entry, _diagnostic)
|
|
1994
|
+
except Exception as exc: # noqa: BLE001 — diagnostics must never crash notify
|
|
1995
|
+
log.debug(
|
|
1996
|
+
"review %s: #1956 diagnostic itself failed: %s",
|
|
1997
|
+
transition.assignment_id, exc,
|
|
1998
|
+
)
|
|
1999
|
+
return False
|
|
2000
|
+
|
|
2001
|
+
# #253: persist the parsed verdict on the review assignment so the merge
|
|
2002
|
+
# gate can refuse to merge work whose review hasn't approved. Independent
|
|
2003
|
+
# of auto_loop (which may be disabled in config).
|
|
2004
|
+
# #bounce: also persist the findings.body so `coord bounce` (and the
|
|
2005
|
+
# future per-stage display) can read it from the DB without re-fetching
|
|
2006
|
+
# the worker's full log.
|
|
2007
|
+
_persist_review_findings(
|
|
2008
|
+
transition.assignment_id, findings.verdict, findings.body
|
|
2009
|
+
)
|
|
2010
|
+
|
|
2011
|
+
review_target = record.get("review_target")
|
|
2012
|
+
repo_github = record["repo_github"]
|
|
2013
|
+
|
|
2014
|
+
# Determine whether review_target is a PR number (integer string) or a branch.
|
|
2015
|
+
pr_number: int | None = None
|
|
2016
|
+
if review_target:
|
|
2017
|
+
try:
|
|
2018
|
+
pr_number = int(review_target)
|
|
2019
|
+
except (ValueError, TypeError):
|
|
2020
|
+
pr_number = None
|
|
2021
|
+
|
|
2022
|
+
# #248: prepend a machine-readable header so the TUI / coordinator can
|
|
2023
|
+
# surface the verdict + counts without re-ingesting the prose body.
|
|
2024
|
+
body_with_header = _attach_review_header(
|
|
2025
|
+
findings.body,
|
|
2026
|
+
verdict=findings.verdict,
|
|
2027
|
+
reviewer_machine=transition.machine_name,
|
|
2028
|
+
assignment_id=transition.assignment_id,
|
|
2029
|
+
)
|
|
2030
|
+
|
|
2031
|
+
if pr_number is not None:
|
|
2032
|
+
try:
|
|
2033
|
+
github_ops.post_pr_review(repo_github, pr_number, findings.verdict, body_with_header)
|
|
2034
|
+
mark_review_posted(transition.assignment_id)
|
|
2035
|
+
return True
|
|
2036
|
+
except Exception as exc: # noqa: BLE001
|
|
2037
|
+
# GitHub rejects self-reviews (same user who opened the PR can't
|
|
2038
|
+
# review it via the API). Log the actual error and fall through to
|
|
2039
|
+
# post the findings as an issue comment instead of silently failing.
|
|
2040
|
+
log.warning(
|
|
2041
|
+
"Failed to post PR review for %s PR#%s via gh: %s — "
|
|
2042
|
+
"falling back to issue comment",
|
|
2043
|
+
transition.assignment_id, pr_number, exc,
|
|
2044
|
+
)
|
|
2045
|
+
# Fall through to the issue-comment path below.
|
|
2046
|
+
|
|
2047
|
+
# No PR number available, or gh pr review was rejected — post findings as
|
|
2048
|
+
# an issue comment so they are never silently lost.
|
|
2049
|
+
verdict_label = "✅ Approved" if findings.verdict == "approve" else "⚠️ Changes Requested"
|
|
2050
|
+
if pr_number is not None:
|
|
2051
|
+
preamble = (
|
|
2052
|
+
f"*Reviewer findings could not be posted directly to PR #{pr_number} "
|
|
2053
|
+
f"(gh pr review was rejected — likely a self-review restriction). "
|
|
2054
|
+
f"Findings are reproduced here.*"
|
|
2055
|
+
)
|
|
2056
|
+
else:
|
|
2057
|
+
preamble = (
|
|
2058
|
+
"*Reviewer could not post directly to a PR (no PR number available). "
|
|
2059
|
+
"Findings are reproduced here.*"
|
|
2060
|
+
)
|
|
2061
|
+
body = (
|
|
2062
|
+
f"## Review Complete — {verdict_label}\n\n"
|
|
2063
|
+
f"{preamble}\n\n"
|
|
2064
|
+
f"{body_with_header}"
|
|
2065
|
+
)
|
|
2066
|
+
try:
|
|
2067
|
+
github_ops.post_issue_comment(repo_github, transition.issue_number, body)
|
|
2068
|
+
mark_review_posted(transition.assignment_id)
|
|
2069
|
+
return True
|
|
2070
|
+
except Exception as exc: # noqa: BLE001
|
|
2071
|
+
log.warning(
|
|
2072
|
+
"Failed to post review comment for %s: %s", transition.assignment_id, exc
|
|
2073
|
+
)
|
|
2074
|
+
return False
|
|
2075
|
+
|
|
2076
|
+
|
|
2077
|
+
def _attach_review_header(
|
|
2078
|
+
body: str,
|
|
2079
|
+
*,
|
|
2080
|
+
verdict: str,
|
|
2081
|
+
reviewer_machine: str | None = None,
|
|
2082
|
+
assignment_id: str | None = None,
|
|
2083
|
+
) -> str:
|
|
2084
|
+
"""#248: prepend the machine-readable header line to a review *body*.
|
|
2085
|
+
|
|
2086
|
+
Counts are derived heuristically from the body's markdown sections.
|
|
2087
|
+
The header always carries the verdict; counts/identity fields are
|
|
2088
|
+
omitted when unavailable.
|
|
2089
|
+
"""
|
|
2090
|
+
from coord.review import ( # noqa: PLC0415 — local import keeps import graph clean
|
|
2091
|
+
estimate_review_counts, format_review_header,
|
|
2092
|
+
)
|
|
2093
|
+
blocking, nonblocking, nits = estimate_review_counts(body)
|
|
2094
|
+
header = format_review_header(
|
|
2095
|
+
verdict=verdict,
|
|
2096
|
+
reviewer_machine=reviewer_machine,
|
|
2097
|
+
assignment_id=assignment_id,
|
|
2098
|
+
blocking=blocking,
|
|
2099
|
+
nonblocking=nonblocking,
|
|
2100
|
+
nits=nits,
|
|
2101
|
+
)
|
|
2102
|
+
return f"{header}\n\n{body}"
|
|
2103
|
+
|
|
2104
|
+
|
|
2105
|
+
def _try_parse_and_post_plan(
|
|
2106
|
+
transition: Transition,
|
|
2107
|
+
record: dict,
|
|
2108
|
+
entry: dict,
|
|
2109
|
+
duration: float | None,
|
|
2110
|
+
) -> bool:
|
|
2111
|
+
"""Try to parse a WorkerPlan from the worker log and post it to GitHub.
|
|
2112
|
+
|
|
2113
|
+
Returns True if a plan comment was successfully posted, False otherwise.
|
|
2114
|
+
Silently swallows all errors so callers can fall back gracefully.
|
|
2115
|
+
"""
|
|
2116
|
+
from coord.plan_parser import parse_plan_from_log, parse_plan_from_agent # noqa: PLC0415
|
|
2117
|
+
|
|
2118
|
+
log_path = entry.get("log_path")
|
|
2119
|
+
worker_plan = None
|
|
2120
|
+
if log_path:
|
|
2121
|
+
try:
|
|
2122
|
+
worker_plan = parse_plan_from_log(log_path)
|
|
2123
|
+
except Exception as exc: # noqa: BLE001
|
|
2124
|
+
log.warning("Failed to parse plan log for %s: %s", transition.assignment_id, exc)
|
|
2125
|
+
|
|
2126
|
+
# Local log unavailable (worker ran on a remote agent — entry.log_path
|
|
2127
|
+
# is the agent's filesystem path, not the coordinator's). Mirror the
|
|
2128
|
+
# review path: fall back to the agent's /logs/<id> endpoint. Without
|
|
2129
|
+
# this, every remote-agent plan got posted as a generic "completion"
|
|
2130
|
+
# comment and the structured plan was lost (we hit this on quadraui#264).
|
|
2131
|
+
if worker_plan is None or worker_plan.is_empty():
|
|
2132
|
+
host = _agent_host(transition.machine_name)
|
|
2133
|
+
if host:
|
|
2134
|
+
try:
|
|
2135
|
+
worker_plan = parse_plan_from_agent(host, transition.assignment_id)
|
|
2136
|
+
except Exception as exc: # noqa: BLE001
|
|
2137
|
+
log.warning(
|
|
2138
|
+
"Failed to fetch plan log from agent %s for %s: %s",
|
|
2139
|
+
host, transition.assignment_id, exc,
|
|
2140
|
+
)
|
|
2141
|
+
|
|
2142
|
+
if worker_plan is None or worker_plan.is_empty():
|
|
2143
|
+
return False
|
|
2144
|
+
|
|
2145
|
+
try:
|
|
2146
|
+
body = format_plan(
|
|
2147
|
+
assignment_id=transition.assignment_id,
|
|
2148
|
+
machine_name=transition.machine_name,
|
|
2149
|
+
repo_name=transition.repo_name,
|
|
2150
|
+
issue_number=transition.issue_number,
|
|
2151
|
+
plan=worker_plan,
|
|
2152
|
+
duration_seconds=duration,
|
|
2153
|
+
)
|
|
2154
|
+
github_ops.post_issue_comment(
|
|
2155
|
+
record["repo_github"], transition.issue_number, body
|
|
2156
|
+
)
|
|
2157
|
+
# Cache the parsed plan in the state directory.
|
|
2158
|
+
save_plan(transition.assignment_id, worker_plan.to_dict())
|
|
2159
|
+
except Exception as exc: # noqa: BLE001
|
|
2160
|
+
log.warning("Failed to post plan comment for %s: %s", transition.assignment_id, exc)
|
|
2161
|
+
return False
|
|
2162
|
+
|
|
2163
|
+
return True
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
def _capture_claude_session_id(transition: Transition, entry: dict) -> None:
|
|
2167
|
+
"""#315: persist the worker's claude session ID to the coordinator DB.
|
|
2168
|
+
|
|
2169
|
+
The agent captures this from the ``system.init`` event in the worker log
|
|
2170
|
+
and includes it in the ``/status`` response. Once stored in the DB,
|
|
2171
|
+
``coord chat-continue`` can read it and pass ``--resume <id>`` to the
|
|
2172
|
+
next worker so it loads the prior conversation. Best-effort; a missing
|
|
2173
|
+
ID just means chat-continue will refuse with a clear error.
|
|
2174
|
+
"""
|
|
2175
|
+
session_id = entry.get("claude_session_id")
|
|
2176
|
+
if not isinstance(session_id, str) or not session_id:
|
|
2177
|
+
return
|
|
2178
|
+
try:
|
|
2179
|
+
from coord.state import update_assignment_claude_session_id # noqa: PLC0415
|
|
2180
|
+
update_assignment_claude_session_id(transition.assignment_id, session_id)
|
|
2181
|
+
except Exception as exc: # noqa: BLE001
|
|
2182
|
+
log.warning(
|
|
2183
|
+
"_capture_claude_session_id: failed for %s: %s",
|
|
2184
|
+
transition.assignment_id, exc,
|
|
2185
|
+
)
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
def post_transition(transition: Transition, record: dict, entry: dict) -> None:
|
|
2189
|
+
"""Post the GitHub comment for one transition and mark it notified."""
|
|
2190
|
+
started = entry.get("started_at")
|
|
2191
|
+
finished = entry.get("finished_at")
|
|
2192
|
+
duration = (finished - started) if (started and finished) else None
|
|
2193
|
+
# #208: capture worker cost as soon as the assignment completes — the
|
|
2194
|
+
# value is in the worker's final stream-json result event and would
|
|
2195
|
+
# otherwise be lost when the agent prunes the log. Best-effort:
|
|
2196
|
+
# local log → remote agent entry → skip.
|
|
2197
|
+
_capture_cost(transition, entry, record)
|
|
2198
|
+
# #252: capture the worker-emitted SMOKE_TESTS block at the same
|
|
2199
|
+
# moment so the TUI can render it under the Test stage. Same
|
|
2200
|
+
# best-effort discipline — failure is silent.
|
|
2201
|
+
_capture_smoke_tests(transition, entry)
|
|
2202
|
+
# #874: capture the worker's ### Summary prose block at the same moment
|
|
2203
|
+
# so the board has a durable, queryable summary field. Best-effort.
|
|
2204
|
+
_capture_completion_summary(transition, entry)
|
|
2205
|
+
# #315: persist the worker's claude session ID so chat-continue can
|
|
2206
|
+
# pass --resume to the next worker. Best-effort; silent on failure.
|
|
2207
|
+
_capture_claude_session_id(transition, entry)
|
|
2208
|
+
common = dict(
|
|
2209
|
+
assignment_id=transition.assignment_id,
|
|
2210
|
+
machine_name=transition.machine_name,
|
|
2211
|
+
repo_github=record["repo_github"],
|
|
2212
|
+
repo_name=transition.repo_name,
|
|
2213
|
+
issue_number=transition.issue_number,
|
|
2214
|
+
duration_seconds=duration,
|
|
2215
|
+
log_path=entry.get("log_path"),
|
|
2216
|
+
)
|
|
2217
|
+
assignment_type = record.get("type", "work")
|
|
2218
|
+
if transition.event == EVENT_COMPLETION and assignment_type in (
|
|
2219
|
+
"refinement",
|
|
2220
|
+
"milestone-chat",
|
|
2221
|
+
):
|
|
2222
|
+
# #315: refinement chat turns are developer-side conversation — do NOT
|
|
2223
|
+
# post completion comments to GitHub. Each turn would spam the issue
|
|
2224
|
+
# with identical "assignment completed" noise. We still capture cost,
|
|
2225
|
+
# smoke tests, and session ID above; just skip the GitHub post.
|
|
2226
|
+
# #770: milestone-chat is dispatched AGAINST the tracking issue
|
|
2227
|
+
# itself (unlike refinement's target issue, this one is the live
|
|
2228
|
+
# planning document a human reads) — a generic completion comment on
|
|
2229
|
+
# every conversational turn would be even noisier here. The
|
|
2230
|
+
# meaningful GitHub-visible effect is the tracking issue's body
|
|
2231
|
+
# update via `coord milestone write-order`, not a completion comment.
|
|
2232
|
+
mark_notified(
|
|
2233
|
+
transition.assignment_id,
|
|
2234
|
+
transition.event,
|
|
2235
|
+
branch=entry.get("branch"),
|
|
2236
|
+
)
|
|
2237
|
+
elif transition.event == EVENT_COMPLETION and assignment_type == "plan":
|
|
2238
|
+
# For plan assignments, post the structured plan comment. Fall back
|
|
2239
|
+
# to a standard completion comment if the log can't be parsed.
|
|
2240
|
+
posted = _try_parse_and_post_plan(transition, record, entry, duration)
|
|
2241
|
+
if not posted:
|
|
2242
|
+
post_completion(exit_code=transition.exit_code or 0, **common)
|
|
2243
|
+
mark_notified(
|
|
2244
|
+
transition.assignment_id,
|
|
2245
|
+
EVENT_PLAN if posted else EVENT_COMPLETION,
|
|
2246
|
+
branch=entry.get("branch"),
|
|
2247
|
+
)
|
|
2248
|
+
elif transition.event == EVENT_COMPLETION and assignment_type == "review":
|
|
2249
|
+
# For review assignments, parse the structured findings and post as a
|
|
2250
|
+
# PR review (or issue comment when no PR number is available). Fall
|
|
2251
|
+
# back to a plain completion comment noting the parse failure — #1956:
|
|
2252
|
+
# tailored per-diagnostic instead of one generic message, so an
|
|
2253
|
+
# operator reading GitHub (not `coord notify`'s own log) can ALSO see
|
|
2254
|
+
# that a verdict is recoverable, not just that parsing failed.
|
|
2255
|
+
_diag: list = []
|
|
2256
|
+
posted = _try_parse_and_post_review(
|
|
2257
|
+
transition, record, entry, duration, _diagnostic=_diag,
|
|
2258
|
+
)
|
|
2259
|
+
if not posted:
|
|
2260
|
+
from coord.review import EndReviewWithoutVerdict # noqa: PLC0415
|
|
2261
|
+
|
|
2262
|
+
if _diag and isinstance(_diag[0], EndReviewWithoutVerdict):
|
|
2263
|
+
fallback_summary = (
|
|
2264
|
+
"Review assignment completed and the reviewer wrote END_REVIEW, "
|
|
2265
|
+
"but never emitted the machine-readable REVIEW_VERDICT: header "
|
|
2266
|
+
"(#1956) — this is NOT a crashed/truncated session, the verdict "
|
|
2267
|
+
"is very likely recoverable from the transcript. Recover with: "
|
|
2268
|
+
f"`coord report-result --assignment {transition.assignment_id} "
|
|
2269
|
+
"--verdict <approve|request-changes> --verdict-source recovered "
|
|
2270
|
+
'--verdict-reason "<why>" --body-file <extracted-review.md>`.'
|
|
2271
|
+
)
|
|
2272
|
+
elif _diag:
|
|
2273
|
+
fallback_summary = (
|
|
2274
|
+
"Review assignment completed but a REVIEW_VERDICT: marker in "
|
|
2275
|
+
"the worker log was malformed and could not be parsed (#1348) "
|
|
2276
|
+
"— the verdict is likely still recoverable from the transcript. "
|
|
2277
|
+
"Recover with: "
|
|
2278
|
+
f"`coord report-result --assignment {transition.assignment_id} "
|
|
2279
|
+
"--verdict <approve|request-changes> --verdict-source recovered "
|
|
2280
|
+
'--verdict-reason "<why>" --body-file <extracted-review.md>`.'
|
|
2281
|
+
)
|
|
2282
|
+
else:
|
|
2283
|
+
fallback_summary = (
|
|
2284
|
+
"Review assignment completed but findings could not be extracted "
|
|
2285
|
+
"from the worker log. The reviewer may not have produced the "
|
|
2286
|
+
"expected structured output (REVIEW_VERDICT / REVIEW_BODY / END_REVIEW)."
|
|
2287
|
+
)
|
|
2288
|
+
post_completion(
|
|
2289
|
+
exit_code=transition.exit_code or 0,
|
|
2290
|
+
summary=fallback_summary,
|
|
2291
|
+
**common,
|
|
2292
|
+
)
|
|
2293
|
+
mark_notified(
|
|
2294
|
+
transition.assignment_id,
|
|
2295
|
+
transition.event,
|
|
2296
|
+
branch=entry.get("branch"),
|
|
2297
|
+
)
|
|
2298
|
+
elif transition.event == EVENT_COMPLETION and assignment_type == "conflict-fix":
|
|
2299
|
+
post_completion(exit_code=transition.exit_code or 0, **common)
|
|
2300
|
+
mark_notified(
|
|
2301
|
+
transition.assignment_id,
|
|
2302
|
+
transition.event,
|
|
2303
|
+
branch=entry.get("branch"),
|
|
2304
|
+
)
|
|
2305
|
+
# Re-enqueue the parent merge entry so the next `coord merge` retries.
|
|
2306
|
+
# This mirrors the reconcile() path — whichever runs first wins.
|
|
2307
|
+
parent_id = record.get("review_of_assignment_id")
|
|
2308
|
+
if parent_id:
|
|
2309
|
+
from coord.reconcile import on_conflict_fix_done # noqa: PLC0415
|
|
2310
|
+
on_conflict_fix_done(
|
|
2311
|
+
parent_assignment_id=parent_id,
|
|
2312
|
+
fix_assignment_id=transition.assignment_id,
|
|
2313
|
+
machine_name=transition.machine_name,
|
|
2314
|
+
succeeded=True,
|
|
2315
|
+
)
|
|
2316
|
+
elif transition.event == EVENT_COMPLETION and assignment_type == "smoke":
|
|
2317
|
+
# #1021: propagate the headless smoke exit code to the parent work
|
|
2318
|
+
# row's Test verdict so the merge gate is satisfied automatically.
|
|
2319
|
+
post_completion(exit_code=transition.exit_code or 0, **common)
|
|
2320
|
+
mark_notified(
|
|
2321
|
+
transition.assignment_id,
|
|
2322
|
+
transition.event,
|
|
2323
|
+
branch=entry.get("branch"),
|
|
2324
|
+
)
|
|
2325
|
+
parent_id = record.get("review_of_assignment_id")
|
|
2326
|
+
if parent_id:
|
|
2327
|
+
# Guard: only auto-certify when the issue's test-mode is "auto"
|
|
2328
|
+
# or unset (no label). A "smoke" label means the TUI offers an
|
|
2329
|
+
# interactive smoke agent — do NOT auto-certify here.
|
|
2330
|
+
from coord.state import get_issue_test_mode, record_test_verdict # noqa: PLC0415
|
|
2331
|
+
test_mode = get_issue_test_mode(
|
|
2332
|
+
transition.repo_name, transition.issue_number
|
|
2333
|
+
)
|
|
2334
|
+
if test_mode != "smoke":
|
|
2335
|
+
succeeded = (transition.exit_code or 0) == 0
|
|
2336
|
+
# #1384: no `smoke_test=` argument needed — the writer
|
|
2337
|
+
# (`state._record_test_verdict_local`) derives the legacy
|
|
2338
|
+
# mirror from `test_state`, so a headless smoke FAILURE lands
|
|
2339
|
+
# as test_state='failed' AND smoke_test='fail' and stays
|
|
2340
|
+
# reachable from `coord fix`.
|
|
2341
|
+
record_test_verdict(
|
|
2342
|
+
assignment_id=parent_id,
|
|
2343
|
+
test_state="passed" if succeeded else "failed",
|
|
2344
|
+
test_reason="headless smoke",
|
|
2345
|
+
)
|
|
2346
|
+
elif transition.event == EVENT_FAILURE and assignment_type == "smoke":
|
|
2347
|
+
# #1605: the Test-stage WORKER itself died (a dead agent, a killed
|
|
2348
|
+
# process group, a terminal API error — anything short of the
|
|
2349
|
+
# worker actually printing `SMOKE: pass`/`SMOKE: fail`) without ever
|
|
2350
|
+
# producing a verdict. Mirrors the EVENT_COMPLETION branch above
|
|
2351
|
+
# (#1021) but for the terminal-FAILED case that branch never
|
|
2352
|
+
# covered: before this, a failed smoke row left the parent's
|
|
2353
|
+
# `test_state` at whatever `dispatch_smoke` set it to (almost always
|
|
2354
|
+
# `"running"`, #1426) — forever, since no gate ever resolves
|
|
2355
|
+
# `"running"` on its own. That is the #1598 incident: a smoke worker
|
|
2356
|
+
# died on a terminal API error and the issue was permanently
|
|
2357
|
+
# stranded with the board reporting a plausible in-progress state.
|
|
2358
|
+
# #1797: `push_failure_reason` is the same column too — see the
|
|
2359
|
+
# identical `or` chain in `coord.reconcile.reconcile_completed_assignments`.
|
|
2360
|
+
_failure_reason = (
|
|
2361
|
+
entry.get("usage_limit_reason")
|
|
2362
|
+
or entry.get("api_error_reason")
|
|
2363
|
+
or entry.get("push_failure_reason")
|
|
2364
|
+
)
|
|
2365
|
+
post_failure(
|
|
2366
|
+
exit_code=transition.exit_code,
|
|
2367
|
+
error=entry.get("error") or _failure_reason or "",
|
|
2368
|
+
**common,
|
|
2369
|
+
)
|
|
2370
|
+
mark_notified(
|
|
2371
|
+
transition.assignment_id,
|
|
2372
|
+
transition.event,
|
|
2373
|
+
branch=entry.get("branch"),
|
|
2374
|
+
failure_reason=_failure_reason,
|
|
2375
|
+
exit_code=transition.exit_code,
|
|
2376
|
+
)
|
|
2377
|
+
parent_id = record.get("review_of_assignment_id")
|
|
2378
|
+
if parent_id:
|
|
2379
|
+
from coord.reconcile import ( # noqa: PLC0415
|
|
2380
|
+
propagate_smoke_terminal_failure,
|
|
2381
|
+
)
|
|
2382
|
+
propagate_smoke_terminal_failure(
|
|
2383
|
+
parent_assignment_id=parent_id,
|
|
2384
|
+
failure_reason=_failure_reason,
|
|
2385
|
+
)
|
|
2386
|
+
elif transition.event == EVENT_COMPLETION:
|
|
2387
|
+
post_completion(exit_code=transition.exit_code or 0, **common)
|
|
2388
|
+
mark_notified(
|
|
2389
|
+
transition.assignment_id,
|
|
2390
|
+
transition.event,
|
|
2391
|
+
branch=entry.get("branch"),
|
|
2392
|
+
)
|
|
2393
|
+
elif transition.event == EVENT_ADVISORY:
|
|
2394
|
+
# #448: 0-commit clean exit — post a distinctive advisory comment.
|
|
2395
|
+
# No ❌ emoji, no re-dispatch suggestion; just surfaces the advisory
|
|
2396
|
+
# state on GitHub so operators not watching coord status are informed.
|
|
2397
|
+
post_advisory(
|
|
2398
|
+
reason=entry.get("zero_commit_reason") or "",
|
|
2399
|
+
**common,
|
|
2400
|
+
)
|
|
2401
|
+
mark_notified(
|
|
2402
|
+
transition.assignment_id,
|
|
2403
|
+
transition.event,
|
|
2404
|
+
branch=entry.get("branch"),
|
|
2405
|
+
)
|
|
2406
|
+
else:
|
|
2407
|
+
# #1605/#1797: carry the agent's own diagnostic (a usage-limit kill,
|
|
2408
|
+
# a terminal API-error classification, or an auth-shaped push
|
|
2409
|
+
# failure — all stamped by `AgentServer._reap` onto this same
|
|
2410
|
+
# `/status` completed entry — see
|
|
2411
|
+
# `coord.reconcile.reconcile_completed_assignments`'s identical
|
|
2412
|
+
# `or`) through to `mark_notified` so a `status='failed'` row is
|
|
2413
|
+
# never left with both `failure_reason` and `exit_code` null. This
|
|
2414
|
+
# is the branch a `type="work"` push-auth failure actually hits
|
|
2415
|
+
# (none of the type-specific `elif`s above match "work"), so
|
|
2416
|
+
# `_failure_reason` also feeds `error=` below — otherwise the
|
|
2417
|
+
# posted GitHub failure comment's `error` field is blank for
|
|
2418
|
+
# exactly the failure #1797 exists to surface.
|
|
2419
|
+
_failure_reason = (
|
|
2420
|
+
entry.get("usage_limit_reason")
|
|
2421
|
+
or entry.get("api_error_reason")
|
|
2422
|
+
or entry.get("push_failure_reason")
|
|
2423
|
+
)
|
|
2424
|
+
post_failure(
|
|
2425
|
+
exit_code=transition.exit_code,
|
|
2426
|
+
error=entry.get("error") or _failure_reason or "",
|
|
2427
|
+
**common,
|
|
2428
|
+
)
|
|
2429
|
+
mark_notified(
|
|
2430
|
+
transition.assignment_id,
|
|
2431
|
+
transition.event,
|
|
2432
|
+
branch=entry.get("branch"),
|
|
2433
|
+
failure_reason=_failure_reason,
|
|
2434
|
+
exit_code=transition.exit_code,
|
|
2435
|
+
)
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
def post_orphaned_review_findings(
|
|
2439
|
+
config: Config,
|
|
2440
|
+
repo_name: str | None = None,
|
|
2441
|
+
) -> list[str]:
|
|
2442
|
+
"""Walk done-review assignments with unposted findings and attempt to post.
|
|
2443
|
+
|
|
2444
|
+
Handles two scenarios that cause findings to be lost:
|
|
2445
|
+
|
|
2446
|
+
1. The agent reported the assignment as 'done' but notify never ran (or
|
|
2447
|
+
ran at the wrong time) — no notification record in the DB at all.
|
|
2448
|
+
2. Notify ran and posted a fallback completion comment (because the log
|
|
2449
|
+
couldn't be parsed at that time), but findings were never extracted.
|
|
2450
|
+
|
|
2451
|
+
In both cases ``review_posted_at`` is NULL on the assignment row.
|
|
2452
|
+
|
|
2453
|
+
The function queries each relevant agent server to discover the log path,
|
|
2454
|
+
then re-parses and re-posts. If the agent is offline or its completed
|
|
2455
|
+
list no longer contains the assignment, the entry is silently skipped
|
|
2456
|
+
so ``coord notify`` stays non-fatal.
|
|
2457
|
+
|
|
2458
|
+
Returns a list of assignment_ids for which findings were successfully posted.
|
|
2459
|
+
Optionally filter to a single *repo_name*.
|
|
2460
|
+
"""
|
|
2461
|
+
from coord.review import parse_review_from_log # noqa: PLC0415
|
|
2462
|
+
|
|
2463
|
+
candidates = load_done_reviews_needing_post(repo_name=repo_name)
|
|
2464
|
+
if not candidates:
|
|
2465
|
+
return []
|
|
2466
|
+
|
|
2467
|
+
notified = load_notified()
|
|
2468
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
2469
|
+
|
|
2470
|
+
# Group by machine so we query each agent server once.
|
|
2471
|
+
by_machine: dict[str, list[dict]] = {}
|
|
2472
|
+
for row in candidates:
|
|
2473
|
+
by_machine.setdefault(row["machine_name"], []).append(row)
|
|
2474
|
+
|
|
2475
|
+
posted_ids: list[str] = []
|
|
2476
|
+
for machine_name, rows in by_machine.items():
|
|
2477
|
+
machine = machines_by_name.get(machine_name)
|
|
2478
|
+
if machine is None:
|
|
2479
|
+
log.debug("post_orphaned: unknown machine %r — skipping %d assignment(s)", machine_name, len(rows))
|
|
2480
|
+
continue
|
|
2481
|
+
|
|
2482
|
+
status = _agent_status(machine.host)
|
|
2483
|
+
log_by_id: dict[str, str] = {}
|
|
2484
|
+
if status:
|
|
2485
|
+
for entry in status.get("completed", []):
|
|
2486
|
+
eid = entry.get("id")
|
|
2487
|
+
lp = entry.get("log_path")
|
|
2488
|
+
if eid and lp:
|
|
2489
|
+
log_by_id[eid] = lp
|
|
2490
|
+
|
|
2491
|
+
for row in rows:
|
|
2492
|
+
aid = row["assignment_id"]
|
|
2493
|
+
log_path = log_by_id.get(aid)
|
|
2494
|
+
findings = None
|
|
2495
|
+
# Try local file first (cheap) — works when notify runs on the
|
|
2496
|
+
# same host as the agent. Falls back to fetching via HTTP so the
|
|
2497
|
+
# coordinator can post reviews from any machine.
|
|
2498
|
+
if log_path:
|
|
2499
|
+
try:
|
|
2500
|
+
findings = parse_review_from_log(log_path)
|
|
2501
|
+
except Exception as exc: # noqa: BLE001
|
|
2502
|
+
log.warning("post_orphaned: failed to parse local log for %s: %s", aid, exc)
|
|
2503
|
+
if findings is None and machine.host:
|
|
2504
|
+
from coord.review import parse_review_from_agent # noqa: PLC0415
|
|
2505
|
+
try:
|
|
2506
|
+
findings = parse_review_from_agent(machine.host, aid)
|
|
2507
|
+
except Exception as exc: # noqa: BLE001
|
|
2508
|
+
log.warning(
|
|
2509
|
+
"post_orphaned: failed to fetch log from agent %s for %s: %s",
|
|
2510
|
+
machine.host, aid, exc,
|
|
2511
|
+
)
|
|
2512
|
+
if findings is None:
|
|
2513
|
+
log.debug("post_orphaned: no findings (local + agent both missed) for %s", aid)
|
|
2514
|
+
continue
|
|
2515
|
+
|
|
2516
|
+
# #bounce: cache the parsed findings so coord bounce + the
|
|
2517
|
+
# per-stage display can skip the HTTP fetch on later runs.
|
|
2518
|
+
_persist_review_findings(aid, findings.verdict, findings.body)
|
|
2519
|
+
|
|
2520
|
+
review_target = row.get("review_target")
|
|
2521
|
+
repo_github = row.get("repo_github") or ""
|
|
2522
|
+
issue_number = row.get("issue_number", 0)
|
|
2523
|
+
|
|
2524
|
+
pr_number: int | None = None
|
|
2525
|
+
if review_target:
|
|
2526
|
+
try:
|
|
2527
|
+
pr_number = int(review_target)
|
|
2528
|
+
except (ValueError, TypeError):
|
|
2529
|
+
pr_number = None
|
|
2530
|
+
|
|
2531
|
+
# Build a preamble that distinguishes retroactive posts from fresh ones.
|
|
2532
|
+
already_notified = aid in notified
|
|
2533
|
+
if already_notified:
|
|
2534
|
+
retro_note = (
|
|
2535
|
+
"\n\n*Note: a completion comment was posted earlier but findings "
|
|
2536
|
+
"could not be extracted at that time. These are the retroactive findings.*"
|
|
2537
|
+
)
|
|
2538
|
+
else:
|
|
2539
|
+
retro_note = ""
|
|
2540
|
+
|
|
2541
|
+
# #248: same header injection as the live path.
|
|
2542
|
+
body_with_header = _attach_review_header(
|
|
2543
|
+
findings.body,
|
|
2544
|
+
verdict=findings.verdict,
|
|
2545
|
+
reviewer_machine=machine.name,
|
|
2546
|
+
assignment_id=aid,
|
|
2547
|
+
)
|
|
2548
|
+
|
|
2549
|
+
posted = False
|
|
2550
|
+
if pr_number is not None:
|
|
2551
|
+
try:
|
|
2552
|
+
github_ops.post_pr_review(repo_github, pr_number, findings.verdict, body_with_header + retro_note)
|
|
2553
|
+
posted = True
|
|
2554
|
+
except Exception as exc: # noqa: BLE001
|
|
2555
|
+
log.warning(
|
|
2556
|
+
"post_orphaned: failed gh pr review for %s PR#%s: %s — "
|
|
2557
|
+
"falling back to issue comment",
|
|
2558
|
+
aid, pr_number, exc,
|
|
2559
|
+
)
|
|
2560
|
+
|
|
2561
|
+
if not posted:
|
|
2562
|
+
verdict_label = "✅ Approved" if findings.verdict == "approve" else "⚠️ Changes Requested"
|
|
2563
|
+
if pr_number is not None:
|
|
2564
|
+
preamble = (
|
|
2565
|
+
f"*Reviewer findings could not be posted directly to PR #{pr_number} "
|
|
2566
|
+
f"(gh pr review was rejected — likely a self-review restriction). "
|
|
2567
|
+
f"Findings are reproduced here.*"
|
|
2568
|
+
)
|
|
2569
|
+
else:
|
|
2570
|
+
preamble = (
|
|
2571
|
+
"*Reviewer could not post directly to a PR (no PR number available). "
|
|
2572
|
+
"Findings are reproduced here.*"
|
|
2573
|
+
)
|
|
2574
|
+
body = (
|
|
2575
|
+
f"## Review Complete — {verdict_label}\n\n"
|
|
2576
|
+
f"{preamble}{retro_note}\n\n"
|
|
2577
|
+
f"{body_with_header}"
|
|
2578
|
+
)
|
|
2579
|
+
try:
|
|
2580
|
+
github_ops.post_issue_comment(repo_github, issue_number, body)
|
|
2581
|
+
posted = True
|
|
2582
|
+
except Exception as exc: # noqa: BLE001
|
|
2583
|
+
log.warning("post_orphaned: failed to post comment for %s: %s", aid, exc)
|
|
2584
|
+
|
|
2585
|
+
if posted:
|
|
2586
|
+
mark_review_posted(aid)
|
|
2587
|
+
if not already_notified:
|
|
2588
|
+
mark_notified(aid, EVENT_COMPLETION)
|
|
2589
|
+
posted_ids.append(aid)
|
|
2590
|
+
log.info("post_orphaned: posted findings for review %s", aid)
|
|
2591
|
+
|
|
2592
|
+
return posted_ids
|
|
2593
|
+
|
|
2594
|
+
|
|
2595
|
+
def _dispatch_board_pending_smoke(config: Config) -> None:
|
|
2596
|
+
"""Load the board, dispatch any pending Test-stage smoke, and save.
|
|
2597
|
+
|
|
2598
|
+
#1426: `dispatch_pending_smoke` (:mod:`coord.smoke`) was previously only
|
|
2599
|
+
ever called from `reconcile()`'s per-item loop, and the ONLY sanctioned
|
|
2600
|
+
caller of the full `reconcile()` is `coord resume`, a human-invoked
|
|
2601
|
+
command. A thin-client setup driven purely by `coord-notify.timer` (which
|
|
2602
|
+
calls `notify.run()`, not `reconcile()`) never dispatched the Test stage
|
|
2603
|
+
at all — the exact gap `scripts/drive-issue.sh` had to paper over with a
|
|
2604
|
+
local `scripts/coord-test-runner.sh` subprocess (#1395). Mirrors
|
|
2605
|
+
:func:`_dispatch_board_pending_reviews` exactly, and is safe to call even
|
|
2606
|
+
when the board file doesn't exist.
|
|
2607
|
+
"""
|
|
2608
|
+
from coord.board_service import read_board, write_board
|
|
2609
|
+
from coord.smoke import dispatch_pending_smoke
|
|
2610
|
+
|
|
2611
|
+
board = read_board()
|
|
2612
|
+
dispatched = dispatch_pending_smoke(board, config)
|
|
2613
|
+
if dispatched:
|
|
2614
|
+
write_board(board)
|
|
2615
|
+
|
|
2616
|
+
|
|
2617
|
+
def _dispatch_board_pending_reviews(config: Config) -> None:
|
|
2618
|
+
"""Load the board, dispatch any pending reviews, and save.
|
|
2619
|
+
|
|
2620
|
+
Mirrors the review-dispatch loop in reconcile() so that ``coord notify``
|
|
2621
|
+
also triggers review dispatch — not just ``coord status --reconcile``.
|
|
2622
|
+
Safe to call even when the board file doesn't exist.
|
|
2623
|
+
"""
|
|
2624
|
+
from coord.board_service import read_board, write_board
|
|
2625
|
+
from coord.review import dispatch_pending_reviews, dispatch_scoped_reviews_for_queue
|
|
2626
|
+
|
|
2627
|
+
# #749: read_board()/write_board() route through the daemon when
|
|
2628
|
+
# board_service is configured, so this no longer silently no-ops on a
|
|
2629
|
+
# thin client's empty local DB — read_board() falls back to an
|
|
2630
|
+
# effectively-empty board when nothing has been saved yet, which is
|
|
2631
|
+
# exactly as harmless as the old "return early" guard.
|
|
2632
|
+
board = read_board()
|
|
2633
|
+
|
|
2634
|
+
# #465: review fires immediately on work completion — no manual smoke
|
|
2635
|
+
# prerequisite. Mirrors reconcile(). dispatch_pending_reviews() enforces
|
|
2636
|
+
# the bulk-dispatch flood guard (per-pass cap + surge gate, incident
|
|
2637
|
+
# 2026-06-08) and the #459 active-fix dedupe, so notify can't flood either.
|
|
2638
|
+
dispatched = dispatch_pending_reviews(board, config)
|
|
2639
|
+
|
|
2640
|
+
# #1476: same scoped-re-review dispatch reconcile() runs, so a conflict-fix
|
|
2641
|
+
# that voids an approval by changing content gets a delta-scoped re-review
|
|
2642
|
+
# from `coord notify` too, not just `coord status --reconcile`.
|
|
2643
|
+
dispatched = dispatched + dispatch_scoped_reviews_for_queue(board, config)
|
|
2644
|
+
|
|
2645
|
+
if dispatched:
|
|
2646
|
+
write_board(board)
|
|
2647
|
+
|
|
2648
|
+
|
|
2649
|
+
def _sweep_stalled_pipeline(
|
|
2650
|
+
config: Config, *, terminal_cache: dict | None = None,
|
|
2651
|
+
) -> list[StalledDetection]:
|
|
2652
|
+
"""Detect #1441 stalled-pipeline rows, post one comment per row, and —
|
|
2653
|
+
when ``config.pipeline.auto_dispatch_stalled`` is on — dispatch the
|
|
2654
|
+
action the original transition would have taken (#1478).
|
|
2655
|
+
|
|
2656
|
+
Loads its own board (rather than accepting one) so mutations from a
|
|
2657
|
+
dispatched action (a freshly-enqueued merge entry, a newly dispatched
|
|
2658
|
+
review/fix/conflict-fix, ``board.review_state`` flips) can be persisted
|
|
2659
|
+
back via ``write_board`` — mirrors ``_dispatch_board_pending_reviews``/
|
|
2660
|
+
``_dispatch_board_pending_smoke`` above. A comment-posting failure for
|
|
2661
|
+
one row must not stop the sweep from reaching the rest (matches every
|
|
2662
|
+
other best-effort loop in this module) — the ``continue`` on failure
|
|
2663
|
+
means that row's ``notified`` key is never set, so it is picked back up
|
|
2664
|
+
on the next tick rather than silently dropped.
|
|
2665
|
+
|
|
2666
|
+
An unexpected exception *from* ``dispatch_stalled_pipeline_action``
|
|
2667
|
+
itself (e.g. a momentarily-unreachable agent during ``dispatch_review``/
|
|
2668
|
+
``dispatch_conflict_fix``) gets the same treatment, not the "declined"
|
|
2669
|
+
treatment: no comment is posted and the row is NOT marked notified, so
|
|
2670
|
+
it is retried on the next tick rather than permanently foreclosed. A
|
|
2671
|
+
considered decline (``no_action`` returned normally — no capable
|
|
2672
|
+
machine, gate not satisfied, entry vanished) still posts the diagnostic
|
|
2673
|
+
comment and marks notified per the one-shot "act once" guardrail; only a
|
|
2674
|
+
genuine raised exception gets the retry treatment.
|
|
2675
|
+
"""
|
|
2676
|
+
from coord.board_service import read_board, write_board
|
|
2677
|
+
|
|
2678
|
+
board = read_board()
|
|
2679
|
+
detections = detect_stalled_pipeline(config, board=board, terminal_cache=terminal_cache)
|
|
2680
|
+
|
|
2681
|
+
posted: list[StalledDetection] = []
|
|
2682
|
+
board_dirty = False
|
|
2683
|
+
for detection, work in detections:
|
|
2684
|
+
try:
|
|
2685
|
+
action = dispatch_stalled_pipeline_action(
|
|
2686
|
+
detection, work, board, config, terminal_cache=terminal_cache,
|
|
2687
|
+
)
|
|
2688
|
+
except Exception: # noqa: BLE001
|
|
2689
|
+
log.exception(
|
|
2690
|
+
"dispatch_stalled_pipeline_action: unexpected error for %s — "
|
|
2691
|
+
"not marking notified so this row is retried next tick",
|
|
2692
|
+
detection.assignment_id,
|
|
2693
|
+
)
|
|
2694
|
+
continue
|
|
2695
|
+
|
|
2696
|
+
dispatched = action.kind in _STALLED_DISPATCH_KINDS
|
|
2697
|
+
try:
|
|
2698
|
+
if dispatched:
|
|
2699
|
+
post_stalled_pipeline_dispatch(detection, action, config)
|
|
2700
|
+
else:
|
|
2701
|
+
post_stalled_pipeline(detection, config)
|
|
2702
|
+
except Exception: # noqa: BLE001
|
|
2703
|
+
continue
|
|
2704
|
+
posted.append(detection)
|
|
2705
|
+
if dispatched:
|
|
2706
|
+
board_dirty = True
|
|
2707
|
+
|
|
2708
|
+
# #1478 guardrail: "log every auto-dispatch to the audit trail with
|
|
2709
|
+
# the detection that triggered it" — business-tier (never dropped
|
|
2710
|
+
# by the operational/business audit-level gate) for an actual
|
|
2711
|
+
# dispatch; operational-tier for a no-op/skip, so the "nothing
|
|
2712
|
+
# happened" rows don't inflate the business audit stream but are
|
|
2713
|
+
# still reconstructable when `audit.level` includes operational.
|
|
2714
|
+
try:
|
|
2715
|
+
from coord.audit import record_audit # noqa: PLC0415
|
|
2716
|
+
|
|
2717
|
+
record_audit(
|
|
2718
|
+
tier="business" if dispatched else "operational",
|
|
2719
|
+
category="pipeline",
|
|
2720
|
+
event_type="stalled_pipeline_auto_dispatch",
|
|
2721
|
+
actor="coordinator",
|
|
2722
|
+
summary=(
|
|
2723
|
+
f"stalled-pipeline sweep ({detection.reason}) -> {action.kind} "
|
|
2724
|
+
f"for {detection.repo_name}#{detection.issue_number}"
|
|
2725
|
+
),
|
|
2726
|
+
repo=detection.repo_name,
|
|
2727
|
+
issue=detection.issue_number,
|
|
2728
|
+
assignment_id=detection.assignment_id,
|
|
2729
|
+
machine=detection.machine_name,
|
|
2730
|
+
details={
|
|
2731
|
+
"stalled_reason": detection.reason,
|
|
2732
|
+
"stalled_detail": detection.detail,
|
|
2733
|
+
"action_kind": action.kind,
|
|
2734
|
+
"action_detail": action.detail,
|
|
2735
|
+
},
|
|
2736
|
+
)
|
|
2737
|
+
except Exception: # noqa: BLE001
|
|
2738
|
+
log.exception(
|
|
2739
|
+
"record_audit failed for stalled dispatch %s", detection.assignment_id,
|
|
2740
|
+
)
|
|
2741
|
+
|
|
2742
|
+
if board_dirty:
|
|
2743
|
+
try:
|
|
2744
|
+
write_board(board)
|
|
2745
|
+
except Exception: # noqa: BLE001
|
|
2746
|
+
log.exception("write_board failed after stalled-pipeline dispatch")
|
|
2747
|
+
|
|
2748
|
+
return posted
|
|
2749
|
+
|
|
2750
|
+
|
|
2751
|
+
@dataclass(frozen=True)
|
|
2752
|
+
class DrainResult:
|
|
2753
|
+
"""What one :func:`run_drain` pass actually did.
|
|
2754
|
+
|
|
2755
|
+
``skipped_locked`` is the "someone else is draining" outcome, which is a
|
|
2756
|
+
success, not an error — the next tick picks the work up.
|
|
2757
|
+
|
|
2758
|
+
``propagated_verdicts`` (#1663) lists the review assignment IDs whose
|
|
2759
|
+
verdict this pass wrote through onto the parent **work** row. Never
|
|
2760
|
+
implies a fix worker was dispatched — the drain cannot dispatch one.
|
|
2761
|
+
"""
|
|
2762
|
+
|
|
2763
|
+
transitions: list[Transition] = field(default_factory=list)
|
|
2764
|
+
orphaned_findings: list[str] = field(default_factory=list)
|
|
2765
|
+
propagated_verdicts: list[str] = field(default_factory=list)
|
|
2766
|
+
skipped_locked: bool = False
|
|
2767
|
+
|
|
2768
|
+
def __bool__(self) -> bool:
|
|
2769
|
+
"""Truthy when this pass advanced something (for terse log guards)."""
|
|
2770
|
+
return bool(
|
|
2771
|
+
self.transitions or self.orphaned_findings or self.propagated_verdicts
|
|
2772
|
+
)
|
|
2773
|
+
|
|
2774
|
+
|
|
2775
|
+
def run_drain(
|
|
2776
|
+
config: Config,
|
|
2777
|
+
*,
|
|
2778
|
+
lock_path: "Path | None" = None,
|
|
2779
|
+
lock_timeout: float = 0.0,
|
|
2780
|
+
) -> DrainResult:
|
|
2781
|
+
"""The pipeline's **clock** (#1616) — advance terminal rows' side effects.
|
|
2782
|
+
|
|
2783
|
+
``reconcile_completed_assignments`` (the daemon's passive tick) writes
|
|
2784
|
+
``status='done'`` and stops there, by contract. Everything downstream —
|
|
2785
|
+
``finished_at``, the completion comment, the #1076/#1152 test-gate
|
|
2786
|
+
backfill, the Test-stage smoke dispatch, the review dispatch, the #1610
|
|
2787
|
+
``finalizing`` → verdict capture — is a side effect of ``coord notify``.
|
|
2788
|
+
On this fleet ``coord-notify.timer`` is deliberately disabled and the only
|
|
2789
|
+
caller of ``coord notify`` is a live ``coord drive``'s **stall nudge**, so
|
|
2790
|
+
a completed stage sat until the stall detector gave up (9 min on #1123,
|
|
2791
|
+
47 min on #1122) — and rows with no drive at all (vimcode#611/#613) sat
|
|
2792
|
+
until a human poked the daemon. This function is what the daemon tick
|
|
2793
|
+
calls so the pipeline advances on a clock instead of on an accident.
|
|
2794
|
+
|
|
2795
|
+
**Scope is the whole point — this is deliberately NOT ``run()``.**
|
|
2796
|
+
``coord notify`` triggers five side effects; four are bookkeeping with no
|
|
2797
|
+
race and no cost if repeated, and one spawns a metered worker. The line
|
|
2798
|
+
sits at exactly one place:
|
|
2799
|
+
|
|
2800
|
+
========================================== ======= ===================================
|
|
2801
|
+
side effect here? why
|
|
2802
|
+
========================================== ======= ===================================
|
|
2803
|
+
``finished_at`` stamped yes no race, no cost
|
|
2804
|
+
completion comment posted yes ``coord:`` markers make it idempotent
|
|
2805
|
+
test-gate backfill (#1076/#1152) yes no race, no cost
|
|
2806
|
+
Test-stage smoke dispatch (#1426) yes the gate review waits on; see below
|
|
2807
|
+
orphaned review findings posted yes comment + verdict capture only
|
|
2808
|
+
review dispatch yes guarded; see below
|
|
2809
|
+
verdict → parent work row (#1663) yes no race, no cost; see below
|
|
2810
|
+
merge enqueue n/a the daemon tick already runs
|
|
2811
|
+
``enqueue_approved_work`` right after
|
|
2812
|
+
**work dispatch** **no** stays with a drive or a human
|
|
2813
|
+
**fix-round dispatch** (``auto_loop``) **no** this is where #476/#477 lives
|
|
2814
|
+
**stalled-pipeline sweep/dispatch** **no** can dispatch work (#1478)
|
|
2815
|
+
========================================== ======= ===================================
|
|
2816
|
+
|
|
2817
|
+
#1663 is what the "verdict → parent work row" row costs to learn. That
|
|
2818
|
+
write — ``work.review_state='done'``, ``work.review_verdict=<verdict>``,
|
|
2819
|
+
``record_work_review_verdict``, the merge-queue refresh — is bookkeeping by
|
|
2820
|
+
every criterion in this table, but it lived *inside*
|
|
2821
|
+
``auto_loop.process_review_completion`` alongside the fix dispatch, and
|
|
2822
|
+
excluding the function excluded both. So every verdict the daemon consumed
|
|
2823
|
+
instead of a human's ``coord notify`` was captured on the review row and
|
|
2824
|
+
dropped on the way to the work row, for **both** verdicts — the approve
|
|
2825
|
+
case stayed invisible only because ``merge_queue.has_approved_review``
|
|
2826
|
+
reads the *review* row. ``coord drive``, the TUI's Review stage and the
|
|
2827
|
+
auto-loop all read the *work* row, so an approved issue simply stopped:
|
|
2828
|
+
2026-08-01's overnight batch reviewed five issues clean and merged none of
|
|
2829
|
+
them in 4h02m. The propagation half is now separately callable
|
|
2830
|
+
(``auto_loop.propagate_review_verdict_for_transition``) and step 5 calls
|
|
2831
|
+
only that; fix dispatch is as unreachable from here as it ever was.
|
|
2832
|
+
|
|
2833
|
+
Why review dispatch is in and fix dispatch is out — the asymmetry is the
|
|
2834
|
+
whole argument. #476/#477, the incident that got ``coord-notify.timer``
|
|
2835
|
+
disabled, was duplicate **fix-workers**: they create conflicting branches
|
|
2836
|
+
on the same issue and cost real recovery work. A duplicate *review* costs
|
|
2837
|
+
a few dollars and a redundant comment. Withholding reviews inherits a
|
|
2838
|
+
mitigation for a risk that does not apply to them. And bookkeeping-only
|
|
2839
|
+
is not sufficient: work→review is the most frequent boundary in the
|
|
2840
|
+
pipeline and the one that stalled #1122, so a drain that stamps state but
|
|
2841
|
+
will not dispatch reviews fixes the *watched* half and leaves the
|
|
2842
|
+
unwatched half exactly as broken as before.
|
|
2843
|
+
|
|
2844
|
+
Smoke dispatch rides along because ``dispatch_pending_reviews`` holds
|
|
2845
|
+
review dispatch until ``test_state`` is passed/skipped when
|
|
2846
|
+
``pipeline.test_precedes_review()`` (#1612). Draining reviews without
|
|
2847
|
+
ever dispatching the Test stage would just move the stall one box left —
|
|
2848
|
+
that is #1605. It is a Test-stage worker on the work's own branch, not a
|
|
2849
|
+
second author on a fresh branch, so it carries none of the #476/#477
|
|
2850
|
+
shape.
|
|
2851
|
+
|
|
2852
|
+
Stuck / needs-attention detection is deliberately absent: those are
|
|
2853
|
+
*notifications*, not pipeline advancement, and giving the daemon a
|
|
2854
|
+
periodic detector is #1632's job (which is blocked on this).
|
|
2855
|
+
|
|
2856
|
+
**Concurrency.** The whole pass runs under ``~/.coord/notify.lock`` —
|
|
2857
|
+
literally :class:`coord.filelock.FileLock`, the same class on the same
|
|
2858
|
+
path ``coord drive``'s ``run_notify()`` takes — so a drive's nudge and the
|
|
2859
|
+
daemon's clock can never both be inside ``dispatch_pending_reviews``,
|
|
2860
|
+
which reads ``review_state == 'pending'`` and writes ``'dispatched'``
|
|
2861
|
+
non-atomically (two concurrent passes would both see ``pending`` and
|
|
2862
|
+
dispatch two reviews). ``lock_timeout`` defaults to **0.0**
|
|
2863
|
+
(non-blocking): if another drain holds it, return ``skipped_locked`` and
|
|
2864
|
+
let the next tick retry rather than pinning a threadpool worker.
|
|
2865
|
+
|
|
2866
|
+
Every step is independently try/except'd — one failing side effect must
|
|
2867
|
+
never sink the rest of the pass, and a drain must never crash the daemon.
|
|
2868
|
+
"""
|
|
2869
|
+
from coord.filelock import FileLock, LockBusy, notify_lock_path # noqa: PLC0415
|
|
2870
|
+
|
|
2871
|
+
lock = FileLock(lock_path if lock_path is not None else notify_lock_path())
|
|
2872
|
+
try:
|
|
2873
|
+
lock.acquire(timeout=lock_timeout)
|
|
2874
|
+
except LockBusy:
|
|
2875
|
+
log.debug("notify drain: %s held elsewhere — skipping this pass", lock.path)
|
|
2876
|
+
return DrainResult(skipped_locked=True)
|
|
2877
|
+
try:
|
|
2878
|
+
return _run_drain_locked(config)
|
|
2879
|
+
finally:
|
|
2880
|
+
lock.release()
|
|
2881
|
+
|
|
2882
|
+
|
|
2883
|
+
def _run_drain_locked(config: Config) -> DrainResult:
|
|
2884
|
+
""":func:`run_drain`'s body, with the lock already held.
|
|
2885
|
+
|
|
2886
|
+
Split out so tests can exercise the side effects without the lock and the
|
|
2887
|
+
lock without the side effects.
|
|
2888
|
+
"""
|
|
2889
|
+
# Refresh the agent-host cache so _try_parse_and_post_review (and any other
|
|
2890
|
+
# helper using _agent_host) can resolve hostnames without threading config
|
|
2891
|
+
# through every call. Mirrors run().
|
|
2892
|
+
global _AGENT_HOSTS
|
|
2893
|
+
_AGENT_HOSTS = {m.name: m.host for m in config.machines}
|
|
2894
|
+
|
|
2895
|
+
# Step 1: post completion/failure/advisory/plan/review comments for rows
|
|
2896
|
+
# the agent reports terminal. This is what stamps `finished_at` (via
|
|
2897
|
+
# mark_notified) and captures cost / SMOKE_TESTS / summary / session id /
|
|
2898
|
+
# the review verdict + findings. Idempotent: detect_transitions skips any
|
|
2899
|
+
# assignment already in the `notifications` table, so a second drain over
|
|
2900
|
+
# the same board posts nothing.
|
|
2901
|
+
posted: list[Transition] = []
|
|
2902
|
+
# #1663: (transition, record, entry) for every review that completed in
|
|
2903
|
+
# THIS pass, so step 5 can propagate its verdict onto the parent work row.
|
|
2904
|
+
review_completions: list[tuple[Transition, dict, dict]] = []
|
|
2905
|
+
try:
|
|
2906
|
+
from coord.comments import EVENT_COMPLETION # noqa: PLC0415
|
|
2907
|
+
|
|
2908
|
+
for transition, record, entry in detect_transitions(config):
|
|
2909
|
+
try:
|
|
2910
|
+
post_transition(transition, record, entry)
|
|
2911
|
+
except Exception: # noqa: BLE001 — one bad row must not sink the pass
|
|
2912
|
+
log.exception(
|
|
2913
|
+
"notify drain: post_transition failed for %s",
|
|
2914
|
+
transition.assignment_id,
|
|
2915
|
+
)
|
|
2916
|
+
continue
|
|
2917
|
+
posted.append(transition)
|
|
2918
|
+
if (
|
|
2919
|
+
record.get("type") == "review"
|
|
2920
|
+
and transition.event == EVENT_COMPLETION
|
|
2921
|
+
):
|
|
2922
|
+
review_completions.append((transition, record, entry))
|
|
2923
|
+
except Exception: # noqa: BLE001
|
|
2924
|
+
log.exception("notify drain: detect_transitions failed")
|
|
2925
|
+
|
|
2926
|
+
# Step 2: dispatch pending Test-stage smoke (#1426). Runs BEFORE review
|
|
2927
|
+
# dispatch to mirror the pipeline's Work -> Test -> Review order.
|
|
2928
|
+
try:
|
|
2929
|
+
_dispatch_board_pending_smoke(config)
|
|
2930
|
+
except Exception: # noqa: BLE001
|
|
2931
|
+
log.exception("notify drain: smoke dispatch failed")
|
|
2932
|
+
|
|
2933
|
+
# Step 3: dispatch pending reviews. Carries the #1612 test-precedes-review
|
|
2934
|
+
# gate, the #1076/#1152 test-gate backfill, the #946 enqueue gate, the
|
|
2935
|
+
# 2026-06-08 flood guard (per-pass cap + surge gate) and the #459 active-fix
|
|
2936
|
+
# dedupe — this is calling existing machinery from a clock, not new
|
|
2937
|
+
# machinery.
|
|
2938
|
+
try:
|
|
2939
|
+
_dispatch_board_pending_reviews(config)
|
|
2940
|
+
except Exception: # noqa: BLE001
|
|
2941
|
+
log.exception("notify drain: review dispatch failed")
|
|
2942
|
+
|
|
2943
|
+
# Step 4: post findings for done-review assignments that were never
|
|
2944
|
+
# processed (agent reported 'cancelled', a human marked the row done, or
|
|
2945
|
+
# notify ran at the wrong time). Comment + verdict capture only.
|
|
2946
|
+
orphaned: list[str] = []
|
|
2947
|
+
try:
|
|
2948
|
+
orphaned = post_orphaned_review_findings(config) or []
|
|
2949
|
+
except Exception: # noqa: BLE001
|
|
2950
|
+
log.exception("notify drain: post_orphaned_review_findings failed")
|
|
2951
|
+
|
|
2952
|
+
# Step 5 (#1663): propagate each captured verdict onto its parent WORK row.
|
|
2953
|
+
#
|
|
2954
|
+
# Steps 1 and 4 both stamp the verdict on the *review* row and stop there.
|
|
2955
|
+
# Everything that reads the *work* row — `coord drive`, the TUI's Review
|
|
2956
|
+
# stage, `_stalled_pipeline`, any state-derived recovery — therefore saw
|
|
2957
|
+
# `review_state='dispatched'` / `review_verdict=NULL` for every verdict the
|
|
2958
|
+
# daemon consumed instead of a human's `coord notify`. The 2026-08-01
|
|
2959
|
+
# overnight batch is the receipt: five issues reviewed, four clean approves,
|
|
2960
|
+
# not one reached its work row, 4h02m of wall clock and zero merges.
|
|
2961
|
+
#
|
|
2962
|
+
# This is the bookkeeping half ONLY — `propagate_review_verdict_for_
|
|
2963
|
+
# transition` cannot reach `_dispatch_fix_for_review`, so the #476/#477
|
|
2964
|
+
# line (no metered fix worker from a clock) is exactly where it was. The
|
|
2965
|
+
# exclusion used to sit at function granularity and took the parent-row
|
|
2966
|
+
# write down with the dispatch; it now sits at side-effect granularity,
|
|
2967
|
+
# which is where the table above always said it belonged.
|
|
2968
|
+
_propagated: list[str] = []
|
|
2969
|
+
if review_completions or orphaned:
|
|
2970
|
+
try:
|
|
2971
|
+
from coord.auto_loop import ( # noqa: PLC0415
|
|
2972
|
+
propagate_review_verdict_for_transition,
|
|
2973
|
+
)
|
|
2974
|
+
|
|
2975
|
+
seen: set[str] = set()
|
|
2976
|
+
# Orphaned rows have no transition tuple (their comment was posted
|
|
2977
|
+
# on an earlier pass, or never). `_load_review_findings` reads the
|
|
2978
|
+
# DB findings cache first — which step 4 just populated — so an
|
|
2979
|
+
# empty record/entry still resolves the verdict without any I/O.
|
|
2980
|
+
pending: list[tuple[str, dict, dict]] = [
|
|
2981
|
+
(t.assignment_id, record, entry)
|
|
2982
|
+
for t, record, entry in review_completions
|
|
2983
|
+
] + [(aid, {"type": "review"}, {}) for aid in orphaned]
|
|
2984
|
+
|
|
2985
|
+
for aid, record, entry in pending:
|
|
2986
|
+
if not aid or aid in seen:
|
|
2987
|
+
continue
|
|
2988
|
+
seen.add(aid)
|
|
2989
|
+
try:
|
|
2990
|
+
actions = propagate_review_verdict_for_transition(
|
|
2991
|
+
aid, record, entry, config,
|
|
2992
|
+
)
|
|
2993
|
+
except Exception: # noqa: BLE001 — never sink the pass
|
|
2994
|
+
log.exception(
|
|
2995
|
+
"notify drain: verdict propagation failed for %s", aid,
|
|
2996
|
+
)
|
|
2997
|
+
continue
|
|
2998
|
+
for action in actions:
|
|
2999
|
+
log.info(
|
|
3000
|
+
"notify drain: verdict propagation %s: %s (assignment=%s)",
|
|
3001
|
+
action.kind, action.detail, action.assignment_id,
|
|
3002
|
+
)
|
|
3003
|
+
if action.kind in (
|
|
3004
|
+
"approved", "approved_with_nits", "verdict_propagated",
|
|
3005
|
+
"terminal_skip",
|
|
3006
|
+
):
|
|
3007
|
+
_propagated.append(aid)
|
|
3008
|
+
except Exception: # noqa: BLE001
|
|
3009
|
+
log.exception("notify drain: verdict propagation loop failed")
|
|
3010
|
+
|
|
3011
|
+
return DrainResult(
|
|
3012
|
+
transitions=posted,
|
|
3013
|
+
orphaned_findings=orphaned,
|
|
3014
|
+
propagated_verdicts=_propagated,
|
|
3015
|
+
)
|
|
3016
|
+
|
|
3017
|
+
|
|
3018
|
+
def run(
|
|
3019
|
+
config: Config,
|
|
3020
|
+
) -> tuple[
|
|
3021
|
+
list[Transition],
|
|
3022
|
+
list[StuckDetection],
|
|
3023
|
+
list[NeedsAttentionDetection],
|
|
3024
|
+
list[StalledDetection],
|
|
3025
|
+
list[LivenessStallDetection],
|
|
3026
|
+
]:
|
|
3027
|
+
"""Detect and post all pending transitions, stuck signals, #846
|
|
3028
|
+
needs-attention detections, #1441 stalled-pipeline detections, and
|
|
3029
|
+
#2048 liveness-auditor stalls.
|
|
3030
|
+
|
|
3031
|
+
Also dispatches any pending reviews found on the saved board so that
|
|
3032
|
+
``coord notify`` acts as a reliable review-dispatch trigger in addition
|
|
3033
|
+
to ``coord status --reconcile``.
|
|
3034
|
+
|
|
3035
|
+
Returns (posted_transitions, posted_stuck, posted_needs_attention,
|
|
3036
|
+
posted_stalled, posted_liveness). The liveness entry is new in #2048 —
|
|
3037
|
+
appended rather than inserted, following #1441's own precedent: any
|
|
3038
|
+
existing caller unpacking a 4-tuple positionally breaks loudly, which
|
|
3039
|
+
is a good thing (it means the CLI/board/TUI surfacing was actually
|
|
3040
|
+
wired up, not silently skipped). ``posted_liveness`` is always ``[]``
|
|
3041
|
+
when ``config.pipeline.liveness_auditor.enabled`` is ``False`` (the
|
|
3042
|
+
default).
|
|
3043
|
+
"""
|
|
3044
|
+
# Refresh the agent-host cache so _try_parse_and_post_review (and any
|
|
3045
|
+
# other helper using _agent_host) can resolve hostnames without
|
|
3046
|
+
# threading config through every call.
|
|
3047
|
+
global _AGENT_HOSTS
|
|
3048
|
+
_AGENT_HOSTS = {m.name: m.host for m in config.machines}
|
|
3049
|
+
|
|
3050
|
+
# #522: one terminal-state cache shared across every gh-hitting check in
|
|
3051
|
+
# this notify run (the auto-loop review/fix dispatches below, and the
|
|
3052
|
+
# #1441 stalled-pipeline sweep at the end), so a burst of activity for
|
|
3053
|
+
# the same merged/closed issue (the #349 ×4 case) costs a single `gh`
|
|
3054
|
+
# round-trip, not one per caller.
|
|
3055
|
+
terminal_cache: dict = {}
|
|
3056
|
+
|
|
3057
|
+
# Collect (transition, record, entry) tuples for review completions so we
|
|
3058
|
+
# can feed them to the auto-loop after all notifications are posted.
|
|
3059
|
+
review_completions: list[tuple[Transition, dict, dict]] = []
|
|
3060
|
+
# Collect (transition, record) tuples for completed fix workers so we can
|
|
3061
|
+
# dispatch a fresh review against each one after notifications are posted.
|
|
3062
|
+
fix_completions: list[tuple[Transition, dict]] = []
|
|
3063
|
+
|
|
3064
|
+
posted: list[Transition] = []
|
|
3065
|
+
for transition, record, entry in detect_transitions(config):
|
|
3066
|
+
try:
|
|
3067
|
+
post_transition(transition, record, entry)
|
|
3068
|
+
except Exception: # noqa: BLE001 — surface to caller; continue with rest
|
|
3069
|
+
continue
|
|
3070
|
+
posted.append(transition)
|
|
3071
|
+
# Track completed reviews for auto-loop processing below.
|
|
3072
|
+
from coord.comments import EVENT_COMPLETION # noqa: PLC0415
|
|
3073
|
+
from coord.auto_loop import FIX_DISPATCH_TYPES # noqa: PLC0415
|
|
3074
|
+
if (
|
|
3075
|
+
record.get("type") == "review"
|
|
3076
|
+
and transition.event == EVENT_COMPLETION
|
|
3077
|
+
):
|
|
3078
|
+
review_completions.append((transition, record, entry))
|
|
3079
|
+
# Track completed fix workers (type in FIX_DISPATCH_TYPES,
|
|
3080
|
+
# review_of_assignment_id set, title starts with "[fix-") for
|
|
3081
|
+
# auto-loop re-review dispatch. #1176 review: this used to hardcode
|
|
3082
|
+
# type == "work", which meant a completed type="test-author" fix
|
|
3083
|
+
# (added by #1176 itself) never reached run_for_fix_transition —
|
|
3084
|
+
# the same class of bug as #1141 ("test-author was never added to
|
|
3085
|
+
# WORK_LIKE_TYPES"). FIX_DISPATCH_TYPES is the single source of
|
|
3086
|
+
# truth for what _dispatch_fix can emit, so a future fix-dispatch
|
|
3087
|
+
# type can't reintroduce this gap silently.
|
|
3088
|
+
elif (
|
|
3089
|
+
record.get("type") in FIX_DISPATCH_TYPES
|
|
3090
|
+
and transition.event == EVENT_COMPLETION
|
|
3091
|
+
and record.get("review_of_assignment_id")
|
|
3092
|
+
and (record.get("issue_title") or "").startswith("[fix-")
|
|
3093
|
+
):
|
|
3094
|
+
fix_completions.append((transition, record))
|
|
3095
|
+
|
|
3096
|
+
# Also detect and post stuck signals
|
|
3097
|
+
stuck_posted: list[StuckDetection] = []
|
|
3098
|
+
for detection, record in detect_stuck(config):
|
|
3099
|
+
try:
|
|
3100
|
+
post_stuck(detection, record)
|
|
3101
|
+
except Exception: # noqa: BLE001
|
|
3102
|
+
continue
|
|
3103
|
+
stuck_posted.append(detection)
|
|
3104
|
+
|
|
3105
|
+
# #846: coordinator backstop for long-running / non-converging
|
|
3106
|
+
# assignments. Best-effort, non-fatal — one bad record must not sink the
|
|
3107
|
+
# rest of the notify run.
|
|
3108
|
+
needs_attention_posted: list[NeedsAttentionDetection] = []
|
|
3109
|
+
try:
|
|
3110
|
+
for detection, record in detect_needs_attention(config):
|
|
3111
|
+
try:
|
|
3112
|
+
post_needs_attention(detection, record)
|
|
3113
|
+
except Exception: # noqa: BLE001
|
|
3114
|
+
continue
|
|
3115
|
+
needs_attention_posted.append(detection)
|
|
3116
|
+
except Exception: # noqa: BLE001
|
|
3117
|
+
log.exception("detect_needs_attention: unexpected error")
|
|
3118
|
+
|
|
3119
|
+
# Dispatch pending Test-stage smoke from the saved board (#1426;
|
|
3120
|
+
# best-effort, non-fatal). Runs BEFORE review dispatch to mirror the
|
|
3121
|
+
# pipeline's Work -> Test -> Review order, though ordering isn't load-
|
|
3122
|
+
# bearing here: dispatch_pending_reviews already holds review dispatch
|
|
3123
|
+
# until test_state is passed/skipped regardless of which runs first in
|
|
3124
|
+
# a given pass.
|
|
3125
|
+
try:
|
|
3126
|
+
_dispatch_board_pending_smoke(config)
|
|
3127
|
+
except Exception: # noqa: BLE001
|
|
3128
|
+
pass
|
|
3129
|
+
|
|
3130
|
+
# Dispatch pending reviews from the saved board (best-effort, non-fatal).
|
|
3131
|
+
try:
|
|
3132
|
+
_dispatch_board_pending_reviews(config)
|
|
3133
|
+
except Exception: # noqa: BLE001
|
|
3134
|
+
pass
|
|
3135
|
+
|
|
3136
|
+
# Post findings for done-review assignments that were never processed
|
|
3137
|
+
# (e.g. agent reported 'cancelled', user manually marked done, or notify
|
|
3138
|
+
# ran at the wrong time). Best-effort, non-fatal.
|
|
3139
|
+
try:
|
|
3140
|
+
post_orphaned_review_findings(config)
|
|
3141
|
+
except Exception: # noqa: BLE001
|
|
3142
|
+
log.exception("post_orphaned_review_findings: unexpected error")
|
|
3143
|
+
|
|
3144
|
+
# Auto-loop: for each completed review, optionally dispatch a fix worker.
|
|
3145
|
+
# Runs after notify posts the completion comment so GitHub has the full
|
|
3146
|
+
# review body before any fix briefing references "previous findings".
|
|
3147
|
+
if review_completions:
|
|
3148
|
+
try:
|
|
3149
|
+
from coord.auto_loop import run_for_review_transition # noqa: PLC0415
|
|
3150
|
+
for transition, record, entry in review_completions:
|
|
3151
|
+
try:
|
|
3152
|
+
actions = run_for_review_transition(
|
|
3153
|
+
transition.assignment_id, record, entry, config,
|
|
3154
|
+
terminal_cache=terminal_cache,
|
|
3155
|
+
)
|
|
3156
|
+
for action in actions:
|
|
3157
|
+
log.info(
|
|
3158
|
+
"auto_loop %s: %s (assignment=%s)",
|
|
3159
|
+
action.kind, action.detail, action.assignment_id,
|
|
3160
|
+
)
|
|
3161
|
+
except Exception: # noqa: BLE001
|
|
3162
|
+
log.exception(
|
|
3163
|
+
"auto_loop: error processing review %s",
|
|
3164
|
+
transition.assignment_id,
|
|
3165
|
+
)
|
|
3166
|
+
except Exception: # noqa: BLE001
|
|
3167
|
+
log.exception("auto_loop: unexpected error in review completion loop")
|
|
3168
|
+
|
|
3169
|
+
# Auto-loop: for each completed fix worker, dispatch a fresh review so
|
|
3170
|
+
# the review → fix → re-review cycle closes without manual coord pr invocations.
|
|
3171
|
+
# Runs after review_completions so a simultaneous review + fix completion
|
|
3172
|
+
# in the same notify run is handled review-first.
|
|
3173
|
+
if fix_completions:
|
|
3174
|
+
try:
|
|
3175
|
+
from coord.auto_loop import run_for_fix_transition # noqa: PLC0415
|
|
3176
|
+
for transition, _record in fix_completions:
|
|
3177
|
+
try:
|
|
3178
|
+
actions = run_for_fix_transition(
|
|
3179
|
+
transition.assignment_id, config,
|
|
3180
|
+
terminal_cache=terminal_cache,
|
|
3181
|
+
)
|
|
3182
|
+
for action in actions:
|
|
3183
|
+
log.info(
|
|
3184
|
+
"auto_loop fix_transition %s: %s (assignment=%s)",
|
|
3185
|
+
action.kind, action.detail, action.assignment_id,
|
|
3186
|
+
)
|
|
3187
|
+
except Exception: # noqa: BLE001
|
|
3188
|
+
log.exception(
|
|
3189
|
+
"auto_loop: error processing fix completion %s",
|
|
3190
|
+
transition.assignment_id,
|
|
3191
|
+
)
|
|
3192
|
+
except Exception: # noqa: BLE001
|
|
3193
|
+
log.exception("auto_loop: unexpected error in fix completion loop")
|
|
3194
|
+
|
|
3195
|
+
# #1441/#1478: sweep for pipeline rows whose auto-loop transition
|
|
3196
|
+
# already fired once but which are now stuck on a precondition that
|
|
3197
|
+
# landed too late (the vimcode #602 reference case), post a diagnostic
|
|
3198
|
+
# (or, when `pipeline.auto_dispatch_stalled` is on, act). Runs last,
|
|
3199
|
+
# after the review/fix auto-loop above has had a chance to act on THIS
|
|
3200
|
+
# pass's transitions, so a row that just got a fresh fix/review
|
|
3201
|
+
# dispatched above is not also flagged as stalled in the same pass.
|
|
3202
|
+
# Best-effort, non-fatal — mirrors the #846 needs-attention block above;
|
|
3203
|
+
# the crucial difference from `reconcile()`-only sweepers (see
|
|
3204
|
+
# docs/OPERATING_GOTCHAS.md §7) is that this runs from `coord notify`.
|
|
3205
|
+
stalled_posted: list[StalledDetection] = []
|
|
3206
|
+
try:
|
|
3207
|
+
stalled_posted = _sweep_stalled_pipeline(config, terminal_cache=terminal_cache)
|
|
3208
|
+
except Exception: # noqa: BLE001
|
|
3209
|
+
log.exception("detect_stalled_pipeline: unexpected error")
|
|
3210
|
+
|
|
3211
|
+
# #2048: cheap per-turn liveness auditor. Best-effort, non-fatal —
|
|
3212
|
+
# mirrors the #846 needs-attention block above. Entirely a no-op
|
|
3213
|
+
# (returns [] immediately, no subprocess, no DB write) unless
|
|
3214
|
+
# config.pipeline.liveness_auditor.enabled is set.
|
|
3215
|
+
liveness_posted: list[LivenessStallDetection] = []
|
|
3216
|
+
try:
|
|
3217
|
+
for detection, record in detect_liveness_stall(config):
|
|
3218
|
+
try:
|
|
3219
|
+
post_liveness_stall(detection, record)
|
|
3220
|
+
except Exception: # noqa: BLE001
|
|
3221
|
+
continue
|
|
3222
|
+
liveness_posted.append(detection)
|
|
3223
|
+
except Exception: # noqa: BLE001
|
|
3224
|
+
log.exception("detect_liveness_stall: unexpected error")
|
|
3225
|
+
|
|
3226
|
+
return posted, stuck_posted, needs_attention_posted, stalled_posted, liveness_posted
|