code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/drive.py
ADDED
|
@@ -0,0 +1,3101 @@
|
|
|
1
|
+
"""``coord drive`` — drive ONE issue from dispatch to merge, unattended (#1392).
|
|
2
|
+
|
|
3
|
+
The Python port of ``scripts/drive-issue.sh`` (742 lines of bash, deleted in
|
|
4
|
+
the same change). The port buys **testability and shippability**, not fewer
|
|
5
|
+
processes: every ``coord`` invocation is still a subprocess, deliberately.
|
|
6
|
+
|
|
7
|
+
THE CLI IS THE CONTRACT; INTERNAL FUNCTIONS ARE NOT. The obvious "win" of
|
|
8
|
+
a Python port is to stop shelling out — call ``record_test_verdict()``
|
|
9
|
+
instead of ``coord test --passed``. Do NOT do this. It is exactly #1384:
|
|
10
|
+
the ``coord test`` CLI mirrors ``test_state`` → the legacy ``smoke_test``
|
|
11
|
+
field and ``record_test_verdict()`` alone does not, so calling the function
|
|
12
|
+
directly silently reintroduces the bug that makes ``coord fix`` refuse to
|
|
13
|
+
dispatch. Every board mutation this driver performs goes through the CLI.
|
|
14
|
+
|
|
15
|
+
WHAT IT IS. The pipeline is Work → Test → Review → Merge
|
|
16
|
+
(``pipeline.default_gates``). coord automates all of it (#1426): the ``coord
|
|
17
|
+
serve`` tick loop reconciles and enqueues, and the ``coord-notify.timer``
|
|
18
|
+
(5 min, on the daemon host) posts completions, auto-dispatches the Test-stage
|
|
19
|
+
smoke assignment (``dispatch_pending_smoke``), dispatches reviews, and runs the
|
|
20
|
+
review → fix → re-review auto-loop. One thing is still missing, and this
|
|
21
|
+
supplies it:
|
|
22
|
+
|
|
23
|
+
NOTHING SEQUENCES THE STAGES FOR A SINGLE ISSUE. ``coord wait`` is
|
|
24
|
+
per-assignment (and reads the LOCAL dispatched ledger, so it does not work
|
|
25
|
+
from a thin client at all). → This is a resumable state machine over the
|
|
26
|
+
daemon's board: it dispatches the WORK assignment, then OBSERVES
|
|
27
|
+
Test/Review/Merge — coord dispatches all three itself — nudging ``coord
|
|
28
|
+
notify`` (``--notify``) when nothing has changed for ``--stall`` minutes.
|
|
29
|
+
|
|
30
|
+
A FAILING TEST IS A LOOP ITERATION, NOT A DEAD END. On a genuine test failure
|
|
31
|
+
this runs ``coord fix``, which dispatches a headless follow-up worker on the
|
|
32
|
+
SAME branch with the model escalated (sonnet → opus, every round) and the
|
|
33
|
+
failure quoted in its briefing. The loop re-tests and repeats, bounded by
|
|
34
|
+
``--max-fix-rounds``. A fix round that legitimately changes nothing exits
|
|
35
|
+
``done``, not ``advisory`` — the zero-commit heuristic is per-branch and the
|
|
36
|
+
branch already carries the original work's commit — so a no-op fix does not
|
|
37
|
+
wedge the pipeline (observed on #1445).
|
|
38
|
+
|
|
39
|
+
Everywhere coord ALREADY has a path, this observes rather than acts — in
|
|
40
|
+
particular it never dispatches the Test-stage smoke assignment (coord's own
|
|
41
|
+
``dispatch_pending_smoke`` does) or a REVIEW fix (the notify timer's auto-loop
|
|
42
|
+
does) — two drivers racing to dispatch the same thing is exactly the
|
|
43
|
+
2026-06-07 duplicate-fix-worker incident (#476/#477).
|
|
44
|
+
|
|
45
|
+
Re-running it on the same issue is safe and resumes from wherever the board
|
|
46
|
+
actually is.
|
|
47
|
+
|
|
48
|
+
THE ORACLE LOOP (#1453, docs/ORACLE_LOOP.md). When this issue's milestone
|
|
49
|
+
already has a merged Gate-A contract and the repo has an acceptance driver
|
|
50
|
+
configured, dispatching ``coord assign`` straight away would just hit the
|
|
51
|
+
#1138 hard gate (``coord.dispatch.enforce_oracle_readiness``) and refuse —
|
|
52
|
+
the issue's JIT acceptance slice hasn't been authored yet. Rather than dead-
|
|
53
|
+
end there, :func:`resolve_oracle_decision` (resolved ONCE, at preflight —
|
|
54
|
+
mirrors ``tui/src/app/pipeline.rs``'s ``gate_a_contract_exists_for`` and
|
|
55
|
+
``coord.milestone_dispatch.gate_a_status``, all three keyed on
|
|
56
|
+
:func:`coord.acceptance.gate_a_contract_path`) puts this run into "oracle
|
|
57
|
+
drive" mode: :func:`_dispatch_work_stage` authors the slice first (``coord
|
|
58
|
+
acceptance author <repo> <tracking_issue> --issue <N>``, plus ``--for-path``
|
|
59
|
+
when the repo's driver is routed — resolved from the milestone's Gate-A
|
|
60
|
+
mock kind via the SHARED :func:`coord.acceptance.resolve_for_path`, so this
|
|
61
|
+
never drifts from whatever eventually resolves it for the TUI's own menu,
|
|
62
|
+
#1460) and :func:`_decide_acceptance_author` drives it through to a landed
|
|
63
|
+
merge (``status='merged'``, #609) before ever calling ``coord assign``.
|
|
64
|
+
The slice's Test and Review stages are dispatched by coord's own passive tick
|
|
65
|
+
exactly like a normal work row's, so this only observes those; its MERGE is
|
|
66
|
+
not (``serve_app._auto_drain_tick`` is gated on ``merge.auto_drain``, which is
|
|
67
|
+
off by default and off in the standing fleet config), so
|
|
68
|
+
:func:`_decide_acceptance_landing` performs it — the same bounded ``coord
|
|
69
|
+
merge --only <aid>`` this driver already runs for the work row (#2079: before
|
|
70
|
+
that, every oracle issue idled through ``2 × --deadline`` waiting for a drain
|
|
71
|
+
loop that was switched off, then landed in a terminal ``blocked`` state).
|
|
72
|
+
An ``advisory`` JIT-slice exit is handled exactly like the main work row's
|
|
73
|
+
(``--accept-advisory``, #1357) rather than waited on forever.
|
|
74
|
+
``--no-acceptance`` opts out back to the pre-#1453 behaviour.
|
|
75
|
+
|
|
76
|
+
STRUCTURE. All decision logic lives in :func:`decide` and :func:`preflight`,
|
|
77
|
+
which are pure functions over an :class:`~coord.drive_state.IssueState` plus
|
|
78
|
+
injected verifiers. :class:`Driver` is the thin I/O shell: poll, execute the
|
|
79
|
+
returned :class:`Action`, sleep. Every bug the bash version shipped was in the
|
|
80
|
+
decision half, which is why that half is where the tests are.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
from __future__ import annotations
|
|
84
|
+
|
|
85
|
+
import json
|
|
86
|
+
import os
|
|
87
|
+
import re
|
|
88
|
+
import shutil
|
|
89
|
+
import subprocess
|
|
90
|
+
import sys
|
|
91
|
+
import time
|
|
92
|
+
from dataclasses import dataclass, field, replace
|
|
93
|
+
from pathlib import Path
|
|
94
|
+
from typing import Any, Callable, Protocol, Sequence
|
|
95
|
+
|
|
96
|
+
from coord.filelock import FileLock, LockBusy, notify_lock_path
|
|
97
|
+
from coord.drive_state import (
|
|
98
|
+
BoardFetcher,
|
|
99
|
+
DriveStateError,
|
|
100
|
+
IssueState,
|
|
101
|
+
project,
|
|
102
|
+
scratch_dir,
|
|
103
|
+
)
|
|
104
|
+
from coord.interactive import (
|
|
105
|
+
DRIVE_SESSION_PREFIX,
|
|
106
|
+
TmuxHost,
|
|
107
|
+
tmux_available,
|
|
108
|
+
tmux_session_alive,
|
|
109
|
+
)
|
|
110
|
+
from coord.dead_end import DeadEnd, detect_dead_end
|
|
111
|
+
from coord.failure_class import classify_failure, plan_usage_limit_resume
|
|
112
|
+
from coord.usage_limits import PlanLimits, evaluate_usage_gate, get_plan_limits
|
|
113
|
+
# Lost in the #1584-onto-#1590 rebase: _decide_review() calls this, but the
|
|
114
|
+
# import lived in a hunk #1590 rewrote, so the merge came out textually clean
|
|
115
|
+
# and semantically broken (NameError at coord/drive.py:1162). Same symbol and
|
|
116
|
+
# same source as coord/notify.py:53 — deliberately NOT re-pointed at #1590's
|
|
117
|
+
# newer classify_failure(), which would change reviewed behaviour during a
|
|
118
|
+
# conflict resolution.
|
|
119
|
+
#
|
|
120
|
+
# #1710 inventory: kept as a direct import — same trivial-predicate reasoning
|
|
121
|
+
# as coord/notify.py's identical import: `is_usage_limit_reason` is a
|
|
122
|
+
# string-prefix check over `Assignment.failure_reason`/`review_failure_reason`
|
|
123
|
+
# (a coordinator-authored value stamped by `format_usage_limit_reason`), not a
|
|
124
|
+
# per-provider log-format parse. Any provider's failure_reason is checked the
|
|
125
|
+
# same way, so there is no `provider.parse_log()` equivalent to route through.
|
|
126
|
+
from coord.worker_events import is_usage_limit_reason
|
|
127
|
+
# #1769: the stale-vs-missing smoke-verdict predicate has exactly ONE
|
|
128
|
+
# implementation, in the module that emits both of the wordings it matches.
|
|
129
|
+
# See `_STALE_SMOKE_MARKERS` / `_is_stale_smoke_reason` below.
|
|
130
|
+
from coord.merge_queue import (
|
|
131
|
+
STALE_SMOKE_MARKERS as _mq_stale_smoke_markers,
|
|
132
|
+
is_ci_infra_reason,
|
|
133
|
+
is_ci_pending_reason,
|
|
134
|
+
is_stale_smoke_reason as _mq_is_stale_smoke_reason,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# ── exit codes (unchanged from drive-issue.sh) ───────────────────────────────
|
|
138
|
+
|
|
139
|
+
EXIT_OK = 0
|
|
140
|
+
EXIT_TERMINAL_FAILURE = 1
|
|
141
|
+
EXIT_USAGE = 2
|
|
142
|
+
EXIT_DEADLINE = 3
|
|
143
|
+
# #1505: distinct from EXIT_TERMINAL_FAILURE so `coord drive`'s exit code
|
|
144
|
+
# alone tells a wrapper/notify path "a human decision is waiting on the
|
|
145
|
+
# board" apart from "something actually broke" — see `_escalate_merge`.
|
|
146
|
+
EXIT_ESCALATED = 4
|
|
147
|
+
# #1844: distinct from EXIT_TERMINAL_FAILURE for the ONE failure shape that
|
|
148
|
+
# is deterministic rather than transient — `coord.dispatch.DispatchRefused`
|
|
149
|
+
# (raised by `enforce_oracle_readiness`/`enforce_epic_dispatch_guard`, a
|
|
150
|
+
# `ValueError` subclass) reaching `coord assign`/`coord approve-plan`/`coord
|
|
151
|
+
# fix`'s own dispatch call, refusing the exact dispatch this run just
|
|
152
|
+
# attempted.
|
|
153
|
+
# Nothing in a retry changes the condition that caused the refusal — no
|
|
154
|
+
# acceptance slice appears, no label gets added — so retrying costs a full
|
|
155
|
+
# tick cycle and changes nothing. `coord drive`'s own subprocess call
|
|
156
|
+
# (`Driver._spawn`) is what SEES this code on the `coord assign`/
|
|
157
|
+
# `approve-plan` child process; `_loop`'s RUN-action handling then re-raises
|
|
158
|
+
# with this SAME code (not EXIT_TERMINAL_FAILURE) so the distinction survives
|
|
159
|
+
# into `_drive_exit_summary`'s `drive_exited` audit row, which is the one
|
|
160
|
+
# thing `coord/drive_queue.py`'s tick can actually read after the process is
|
|
161
|
+
# gone. See the 2026-08-04/05 overnight run (#1817): two identical, fully
|
|
162
|
+
# actionable refusals were retried and exhausted as "drive session died",
|
|
163
|
+
# discarding the guard's own remedy in the process.
|
|
164
|
+
EXIT_DISPATCH_REFUSED = 5
|
|
165
|
+
# #2019: the row is TERMINAL AND UNACTIONABLE — every stage finished, nothing
|
|
166
|
+
# is active on the fleet, and no gate transition is available to any amount of
|
|
167
|
+
# polling. Same *class* as EXIT_DISPATCH_REFUSED above (a condition retrying
|
|
168
|
+
# cannot change) but a different *cause*: nothing refused a dispatch here;
|
|
169
|
+
# the board simply came to rest in a shape with no legal move. Kept distinct
|
|
170
|
+
# so `coord/drive_queue.py`'s tick can block the entry with the RIGHT reason
|
|
171
|
+
# rather than the pre-dispatch-guard wording, and so an operator reading an
|
|
172
|
+
# exit code alone can tell "a guard said no" from "the pipeline dead-ended".
|
|
173
|
+
# See `coord.dead_end.detect_dead_end` for what qualifies, and #1956 /
|
|
174
|
+
# vimcode#635 for the two live incidents (140 minutes and ~25 minutes of a
|
|
175
|
+
# held queue slot, respectively, producing nothing).
|
|
176
|
+
EXIT_DEAD_END = 6
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class DriveError(Exception):
|
|
180
|
+
"""A configuration/usage problem — reported, never polled through."""
|
|
181
|
+
|
|
182
|
+
def __init__(self, message: str, exit_code: int = EXIT_USAGE) -> None:
|
|
183
|
+
super().__init__(message)
|
|
184
|
+
self.exit_code = exit_code
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# ── options ──────────────────────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass
|
|
191
|
+
class DriveOptions:
|
|
192
|
+
"""Resolved flags. Field names mirror the bash variables one-for-one."""
|
|
193
|
+
|
|
194
|
+
machine: str = ""
|
|
195
|
+
model: str = ""
|
|
196
|
+
briefing_file: str = ""
|
|
197
|
+
do_plan: bool = False
|
|
198
|
+
max_fix_rounds: int = 3
|
|
199
|
+
skip_test: bool = False
|
|
200
|
+
repo_path: str = ""
|
|
201
|
+
poll: float = 60.0
|
|
202
|
+
max_work_retries: int = 1
|
|
203
|
+
deadline_mins: float = 240.0
|
|
204
|
+
stall_mins: float = 20.0
|
|
205
|
+
notify: bool = False
|
|
206
|
+
do_merge: bool = True
|
|
207
|
+
merge_method: str = "rebase"
|
|
208
|
+
accept_advisory: bool = False
|
|
209
|
+
force_review: bool = False
|
|
210
|
+
dry_run: bool = False
|
|
211
|
+
max_merge_attempts: int = 3
|
|
212
|
+
# #1453: skip the oracle-loop JIT slice authoring step below even when
|
|
213
|
+
# this issue's milestone has a merged Gate-A contract — an escape hatch
|
|
214
|
+
# for "the contract is stale/wrong for this issue" or "I want a plain
|
|
215
|
+
# run", matching the opt-out every other oracle-loop gate offers
|
|
216
|
+
# (`oracle:exempt` label, `exempt:` manifest list).
|
|
217
|
+
no_acceptance: bool = False
|
|
218
|
+
# Threaded onto every `coord` subprocess so a `coord drive --config X` run
|
|
219
|
+
# cannot dispatch against a *different* config than it is reading. The
|
|
220
|
+
# bash driver ran a bare `coord` and silently had this gap. Empty means
|
|
221
|
+
# "let each subprocess resolve the default" ($COORD_CONFIG →
|
|
222
|
+
# ~/.coord/coordinator.yml → ./coordinator.yml), i.e. today's behaviour.
|
|
223
|
+
config_path: str = ""
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def stall_secs(self) -> float:
|
|
227
|
+
return self.stall_mins * 60.0
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def deadline_secs(self) -> float:
|
|
231
|
+
return self.deadline_mins * 60.0
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@dataclass
|
|
235
|
+
class DriveCounters:
|
|
236
|
+
"""Bounds on every retry loop. Unbounded merge retries was a real bug."""
|
|
237
|
+
|
|
238
|
+
work_retries: int = 0
|
|
239
|
+
# ONE budget for BOTH fix arms (#1692): a failed test and a
|
|
240
|
+
# request-changes review are two shapes of the same "the work needs
|
|
241
|
+
# another round" loop, and a drive that spends three rounds bouncing
|
|
242
|
+
# between them has spent three rounds. Bounded by `opts.max_fix_rounds`.
|
|
243
|
+
fix_rounds: int = 0
|
|
244
|
+
merge_attempts: int = 0
|
|
245
|
+
review_dispatches: int = 0
|
|
246
|
+
# #1584: bounded retry for a review WORKER that died (transient API
|
|
247
|
+
# error, network drop, ...) before producing a verdict — the review-side
|
|
248
|
+
# analogue of `work_retries`, bounded the same way (`opts.max_work_retries`).
|
|
249
|
+
review_retries: int = 0
|
|
250
|
+
# #1692: NOT a second budget — `fix_rounds` above is the budget. This is a
|
|
251
|
+
# de-duplication latch: the assignment id of the review this driver has
|
|
252
|
+
# already spent a fix round on. `coord fix` returns as soon as the fix
|
|
253
|
+
# worker is dispatched, but the board this driver polls needs a beat to
|
|
254
|
+
# show the new row; until it does, the state is byte-for-byte identical to
|
|
255
|
+
# the one that triggered the dispatch. Without this latch the next poll
|
|
256
|
+
# re-fires `coord fix` against the same review and spawns a SECOND fix
|
|
257
|
+
# worker on the same branch — the #476/#477 shape, in a new dispatcher.
|
|
258
|
+
# Cleared implicitly rather than explicitly: the next review round is a
|
|
259
|
+
# different review row (`drive_state.project` keys the review on the
|
|
260
|
+
# current work id), so its id simply doesn't match this one.
|
|
261
|
+
review_fix_dispatched_for: str = ""
|
|
262
|
+
# #2078: the combined stdout+stderr of the MOST RECENT `coord merge
|
|
263
|
+
# --only <aid>` attempt (`Driver._loop` captures it via `run_coord`'s own
|
|
264
|
+
# `_last_run_output`, right after each merge Action runs). When the board
|
|
265
|
+
# carries no merge-queue entry at all (`merge_status == ""`), this is the
|
|
266
|
+
# ONLY place the real reason lives: `coord merge --only` prints
|
|
267
|
+
# `_explain_missing_only_entry`'s diagnosis (naming the blocking
|
|
268
|
+
# review/smoke gate, or "identifier didn't resolve", or "all gates
|
|
269
|
+
# pass — not enqueued yet") on every such attempt, but the driver used to
|
|
270
|
+
# discard it and echo the board's empty fields instead. Threaded through
|
|
271
|
+
# `DriveCounters` (not returned some other way) because `_decide_merge`
|
|
272
|
+
# is pure and only ever sees this value on the NEXT poll after the
|
|
273
|
+
# attempt that produced it — one poll's staleness is the price of
|
|
274
|
+
# keeping the I/O boundary in `Driver`, not in `decide()`.
|
|
275
|
+
last_merge_diagnostic: str = ""
|
|
276
|
+
# #2079: a SECOND, independent budget of exactly the same shape, spent
|
|
277
|
+
# only on landing the oracle-mode JIT acceptance slice
|
|
278
|
+
# (`_decide_acceptance_landing`). Separate rather than shared because the
|
|
279
|
+
# slice and the issue's own work row are two different PRs with two
|
|
280
|
+
# different merge queues and two different `coord merge --only` targets:
|
|
281
|
+
# three attempts spent landing the slice must not silently leave the work
|
|
282
|
+
# row's own merge with zero. Lazily created (`slice_budget`) so the
|
|
283
|
+
# overwhelming majority of drives — non-oracle ones — never allocate it,
|
|
284
|
+
# and `--dry-run`'s counter snapshot stays unchanged for them.
|
|
285
|
+
acceptance: "DriveCounters | None" = None
|
|
286
|
+
|
|
287
|
+
def slice_budget(self) -> "DriveCounters":
|
|
288
|
+
"""This run's slice-landing budget, created on first use (#2079)."""
|
|
289
|
+
if self.acceptance is None:
|
|
290
|
+
self.acceptance = DriveCounters()
|
|
291
|
+
return self.acceptance
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# ── actions ──────────────────────────────────────────────────────────────────
|
|
295
|
+
|
|
296
|
+
WAIT = "wait"
|
|
297
|
+
RUN = "run"
|
|
298
|
+
EXIT = "exit"
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
@dataclass(frozen=True)
|
|
302
|
+
class Action:
|
|
303
|
+
"""What the loop should do next. The only thing :func:`decide` returns.
|
|
304
|
+
|
|
305
|
+
``command`` is the ``coord`` subcommand argv **without** the ``coord``
|
|
306
|
+
binary itself — the driver prepends it. Keeping it here (rather than
|
|
307
|
+
building argv inside the executor) is what lets a unit test assert the
|
|
308
|
+
exact CLI contract, e.g. that a skipped Test gate really is
|
|
309
|
+
``coord test --skipped --reason ... <aid>`` and not a direct
|
|
310
|
+
``record_test_verdict()`` call (#1384).
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
kind: str
|
|
314
|
+
label: str = ""
|
|
315
|
+
message: str = ""
|
|
316
|
+
exit_code: int = 0
|
|
317
|
+
command: tuple[str, ...] = ()
|
|
318
|
+
sleep_after: float | None = None # None → the poll interval
|
|
319
|
+
on_error: str = "die" # "die" | "warn"
|
|
320
|
+
error_message: str = ""
|
|
321
|
+
serialize_merge: bool = False
|
|
322
|
+
warnings: tuple[str, ...] = ()
|
|
323
|
+
# #2079: which merge this Action's `coord merge --only` attempt belongs
|
|
324
|
+
# to — the issue's own work row ("work", the default and the only value
|
|
325
|
+
# before #2079) or the oracle-mode JIT acceptance slice ("acceptance").
|
|
326
|
+
# Read by `Driver._loop` to file the captured diagnostic against the
|
|
327
|
+
# matching `DriveCounters` (see `DriveCounters.acceptance`); a slice
|
|
328
|
+
# attempt's `_explain_missing_only_entry` output must not overwrite what
|
|
329
|
+
# the work row's own last attempt reported, or `_decide_merge` would
|
|
330
|
+
# diagnose one PR using the other PR's gates.
|
|
331
|
+
merge_scope: str = "work"
|
|
332
|
+
|
|
333
|
+
@property
|
|
334
|
+
def is_exit(self) -> bool:
|
|
335
|
+
return self.kind == EXIT
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _wait(sleep_after: float | None = None, label: str = "") -> Action:
|
|
339
|
+
return Action(kind=WAIT, label=label, sleep_after=sleep_after)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _succeed(message: str) -> Action:
|
|
343
|
+
return Action(kind=EXIT, message=message, exit_code=EXIT_OK)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _die(message: str, exit_code: int = EXIT_TERMINAL_FAILURE) -> Action:
|
|
347
|
+
return Action(kind=EXIT, message=message, exit_code=exit_code)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# ── oracle-loop JIT slice authoring (#1453) ─────────────────────────────────
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class AcceptanceGateChecker(Protocol):
|
|
354
|
+
"""The GitHub questions :func:`resolve_oracle_decision` and
|
|
355
|
+
:func:`_decide_acceptance_author` cannot answer from the board payload
|
|
356
|
+
alone: has Gate A's contract actually merged, and (for a routed repo)
|
|
357
|
+
which subtree does this milestone's slice belong to?"""
|
|
358
|
+
|
|
359
|
+
def contract_exists(self, repo_name: str, milestone_number: int) -> bool: ...
|
|
360
|
+
|
|
361
|
+
def resolve_for_path(self, repo_name: str, milestone_number: int) -> str | None: ...
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@dataclass
|
|
365
|
+
class GitHubAcceptanceGateChecker:
|
|
366
|
+
"""Real implementation: reuses ``coord.milestone_dispatch.gate_a_status``
|
|
367
|
+
— the SAME check ``coord milestone dispatch``'s Gate A gate and the
|
|
368
|
+
#1138 ``issue_oracle_ready`` hard gate already run — rather than
|
|
369
|
+
re-deriving the ``tests/acceptance/ms-NN/contract.md`` path here. That
|
|
370
|
+
function returns ``None`` for two different reasons ("no driver
|
|
371
|
+
configured" or "contract exists"); callers of this checker have already
|
|
372
|
+
confirmed ``config.acceptance.has_driver(repo_name)`` themselves
|
|
373
|
+
(:func:`resolve_oracle_decision` does), so ``None`` is unambiguous here.
|
|
374
|
+
"""
|
|
375
|
+
|
|
376
|
+
config: Any
|
|
377
|
+
|
|
378
|
+
def contract_exists(self, repo_name: str, milestone_number: int) -> bool:
|
|
379
|
+
from coord.milestone_dispatch import gate_a_status # noqa: PLC0415
|
|
380
|
+
|
|
381
|
+
repo_cfg = self.config.repo(repo_name)
|
|
382
|
+
if repo_cfg is None:
|
|
383
|
+
return False
|
|
384
|
+
return gate_a_status(repo_cfg, self.config, milestone_number) is None
|
|
385
|
+
|
|
386
|
+
def resolve_for_path(self, repo_name: str, milestone_number: int) -> str | None:
|
|
387
|
+
"""#1453 review finding 1: the ``--for-path`` a routed repo's JIT
|
|
388
|
+
acceptance-author dispatch needs. Delegates to
|
|
389
|
+
:func:`coord.acceptance.resolve_for_path` (the SHARED derivation —
|
|
390
|
+
see its docstring); raises :class:`coord.acceptance.
|
|
391
|
+
ForPathResolutionError` unchanged so callers report it verbatim.
|
|
392
|
+
"""
|
|
393
|
+
from coord.acceptance import resolve_for_path # noqa: PLC0415
|
|
394
|
+
|
|
395
|
+
repo_cfg = self.config.repo(repo_name)
|
|
396
|
+
if repo_cfg is None:
|
|
397
|
+
return None
|
|
398
|
+
return resolve_for_path(self.config, repo_cfg, milestone_number)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
@dataclass(frozen=True)
|
|
402
|
+
class OracleDecision:
|
|
403
|
+
"""Resolved ONCE per run (at preflight time, alongside machine
|
|
404
|
+
resolution) — never recomputed per poll, since *gate_checker* costs a
|
|
405
|
+
GitHub fetch and a milestone's Gate-A status does not change mid-run.
|
|
406
|
+
|
|
407
|
+
``active`` gates the JIT-authoring branch in :func:`_dispatch_work_stage`;
|
|
408
|
+
``reason`` is what the preflight banner prints so an operator never has
|
|
409
|
+
to guess which mode a run is in. ``tracking_issue`` is set iff ``active``
|
|
410
|
+
— the argument :func:`_decide_acceptance_author` needs to build ``coord
|
|
411
|
+
acceptance author <repo> <tracking_issue> --issue <N>``.
|
|
412
|
+
"""
|
|
413
|
+
|
|
414
|
+
active: bool
|
|
415
|
+
reason: str
|
|
416
|
+
tracking_issue: int | None = None
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def resolve_oracle_decision(
|
|
420
|
+
state: IssueState,
|
|
421
|
+
opts: DriveOptions,
|
|
422
|
+
config: Any,
|
|
423
|
+
gate_checker: AcceptanceGateChecker,
|
|
424
|
+
) -> OracleDecision:
|
|
425
|
+
"""The #1453 gate: does this issue's Work dispatch get preceded by an
|
|
426
|
+
independent JIT acceptance-slice authoring session?
|
|
427
|
+
|
|
428
|
+
Mirrors — and must never drift from — the same rule the TUI's
|
|
429
|
+
``gate_a_contract_exists_for`` (``tui/src/app/pipeline.rs``) and
|
|
430
|
+
``coord.milestone_dispatch.gate_a_status`` already enforce, both via
|
|
431
|
+
:func:`coord.acceptance.gate_a_contract_path`: a repo with a configured
|
|
432
|
+
acceptance driver, an issue that resolves to a milestone with a tracking
|
|
433
|
+
issue, and a Gate-A contract already merged for that milestone. This
|
|
434
|
+
complements (does not replace) the #1138 hard gate
|
|
435
|
+
(``coord.dispatch.enforce_oracle_readiness``), which would otherwise
|
|
436
|
+
just refuse the eventual ``coord assign``/``coord approve-plan`` with no
|
|
437
|
+
explanation once an oracle-opted-in milestone's issue reaches it — this
|
|
438
|
+
proactively drives the authoring + merge to completion FIRST so a plain
|
|
439
|
+
``coord drive`` doesn't dead-end on that refusal.
|
|
440
|
+
|
|
441
|
+
#2079: "drives the merge to completion" is now literally true. Until
|
|
442
|
+
#2079 this module said so here and said the opposite in
|
|
443
|
+
:func:`_decide_acceptance_author` ("this only observes"), and the
|
|
444
|
+
observing version was the one that shipped — so every oracle issue burned
|
|
445
|
+
``2 × --deadline`` waiting for ``serve_app._auto_drain_tick``, which is
|
|
446
|
+
off (``merge.auto_drain: false``) in the standing fleet config and merges
|
|
447
|
+
nothing, ever. :func:`_decide_acceptance_landing` is the reconciliation.
|
|
448
|
+
"""
|
|
449
|
+
if opts.no_acceptance:
|
|
450
|
+
return OracleDecision(False, "--no-acceptance set — normal drive")
|
|
451
|
+
if not config.acceptance.has_driver(state.repo):
|
|
452
|
+
return OracleDecision(
|
|
453
|
+
False, f"{state.repo!r} has no acceptance.drivers entry — normal drive"
|
|
454
|
+
)
|
|
455
|
+
if state.milestone_number is None:
|
|
456
|
+
return OracleDecision(
|
|
457
|
+
False, f"#{state.issue} has no GitHub milestone — normal drive"
|
|
458
|
+
)
|
|
459
|
+
if state.milestone_tracking_issue is None:
|
|
460
|
+
return OracleDecision(
|
|
461
|
+
False,
|
|
462
|
+
f"#{state.issue} isn't a member of a tracked milestone work order — "
|
|
463
|
+
"normal drive",
|
|
464
|
+
)
|
|
465
|
+
if not gate_checker.contract_exists(state.repo, state.milestone_number):
|
|
466
|
+
from coord.acceptance import gate_a_contract_path # noqa: PLC0415
|
|
467
|
+
|
|
468
|
+
path = gate_a_contract_path(state.milestone_number)
|
|
469
|
+
return OracleDecision(
|
|
470
|
+
False,
|
|
471
|
+
f"Gate A contract {path!r} not merged yet on "
|
|
472
|
+
f"{state.repo_default_branch!r} — normal drive (run `coord "
|
|
473
|
+
f"acceptance mock {state.repo} {state.milestone_tracking_issue}` "
|
|
474
|
+
"first for the oracle loop, docs/ORACLE_LOOP.md)",
|
|
475
|
+
)
|
|
476
|
+
return OracleDecision(
|
|
477
|
+
True,
|
|
478
|
+
f"ORACLE DRIVE — ms-{state.milestone_number}'s Gate-A contract is "
|
|
479
|
+
f"merged: authoring the sealed JIT slice for #{state.issue} "
|
|
480
|
+
f"(`coord acceptance author {state.repo} "
|
|
481
|
+
f"{state.milestone_tracking_issue} --issue {state.issue}`) before "
|
|
482
|
+
"dispatching work",
|
|
483
|
+
tracking_issue=state.milestone_tracking_issue,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _decide_acceptance_author(
|
|
488
|
+
state: IssueState,
|
|
489
|
+
oracle: OracleDecision,
|
|
490
|
+
opts: DriveOptions,
|
|
491
|
+
counters: DriveCounters,
|
|
492
|
+
machine: str,
|
|
493
|
+
gate_checker: AcceptanceGateChecker,
|
|
494
|
+
verifier: MergeVerifier,
|
|
495
|
+
) -> Action | None:
|
|
496
|
+
"""The #1453 JIT-slice gate itself. ``None`` means "landed — fall
|
|
497
|
+
through to dispatching work normally" (only ever called when
|
|
498
|
+
``oracle.active``).
|
|
499
|
+
|
|
500
|
+
Drives a `type="test-author"` assignment scoped to THIS issue
|
|
501
|
+
(``for_issue_number == state.issue`` — #1171/#1138 key the JIT slice's
|
|
502
|
+
row on the milestone's TRACKING issue via `issue_number`, so it never
|
|
503
|
+
shows up as this issue's own ``work_aid``; see ``IssueState``'s
|
|
504
|
+
docstring) all the way to ``status='merged'`` (#609) — the identical
|
|
505
|
+
terminal signal :func:`decide`'s own merged check uses for the real work
|
|
506
|
+
row.
|
|
507
|
+
|
|
508
|
+
**#2079 — how much of that landing is actually somebody else's job.**
|
|
509
|
+
The slice row is `WORK_LIKE` (``coord.models.WORK_LIKE_TYPES`` contains
|
|
510
|
+
``"test-author"``), so the daemon's passive tick really does run its
|
|
511
|
+
Test and Review stages and really does enqueue it
|
|
512
|
+
(``dispatch_pending_smoke`` / ``dispatch_pending_reviews`` /
|
|
513
|
+
``merge_queue.enqueue_approved_work``) with zero help from this driver.
|
|
514
|
+
Every one of those steps runs unconditionally. Exactly ONE does not: the
|
|
515
|
+
final drain, ``serve_app._auto_drain_tick``, gated on
|
|
516
|
+
``merge.auto_drain`` — ``false`` by default and ``false`` in the standing
|
|
517
|
+
fleet config. So the pre-#2079 comment here ("this only observes") was
|
|
518
|
+
describing a pipeline with its last stage switched off: the slice reached
|
|
519
|
+
READY with a green, ``MERGEABLE``/``CLEAN`` PR and then nothing merged
|
|
520
|
+
it, ever, while this driver idled to ``--deadline`` twice and left the
|
|
521
|
+
issue ``blocked`` (terminal — a manual ``remove`` + ``add`` to clear).
|
|
522
|
+
Landing that last step is what :func:`_decide_acceptance_landing` does,
|
|
523
|
+
with the same bounded ``coord merge --only <aid>`` call
|
|
524
|
+
:func:`_decide_merge` already makes for the work row.
|
|
525
|
+
"""
|
|
526
|
+
aid = state.acceptance_author_aid
|
|
527
|
+
status = state.acceptance_author_status
|
|
528
|
+
|
|
529
|
+
if not aid:
|
|
530
|
+
command = [
|
|
531
|
+
"acceptance", "author", state.repo, str(oracle.tracking_issue),
|
|
532
|
+
"--issue", str(state.issue),
|
|
533
|
+
]
|
|
534
|
+
# #1453 review finding 1: a ROUTED repo's `coord acceptance author`
|
|
535
|
+
# hard-refuses with no --for-path (coord.test_author.
|
|
536
|
+
# dispatch_test_author's "no route matched" RuntimeError) — resolve
|
|
537
|
+
# it from the milestone's Gate-A mock kind (the SHARED
|
|
538
|
+
# coord.acceptance.resolve_for_path helper) before ever dispatching,
|
|
539
|
+
# so a routed repo's very first JIT-authoring attempt doesn't die.
|
|
540
|
+
from coord.acceptance import ForPathResolutionError # noqa: PLC0415
|
|
541
|
+
|
|
542
|
+
try:
|
|
543
|
+
for_path = gate_checker.resolve_for_path(state.repo, state.milestone_number)
|
|
544
|
+
except ForPathResolutionError as exc:
|
|
545
|
+
return _die(
|
|
546
|
+
f"could not resolve --for-path for {state.repo}'s JIT "
|
|
547
|
+
f"acceptance slice on #{state.issue}: {exc}"
|
|
548
|
+
)
|
|
549
|
+
if for_path:
|
|
550
|
+
command += ["--for-path", for_path]
|
|
551
|
+
|
|
552
|
+
return Action(
|
|
553
|
+
kind=RUN,
|
|
554
|
+
label=(
|
|
555
|
+
"ACCEPTANCE: authoring sealed JIT slice → coord acceptance "
|
|
556
|
+
f"author {state.repo} {oracle.tracking_issue} --issue "
|
|
557
|
+
f"{state.issue}"
|
|
558
|
+
+ (f" --for-path {for_path}" if for_path else "")
|
|
559
|
+
),
|
|
560
|
+
command=tuple(command),
|
|
561
|
+
error_message=(
|
|
562
|
+
f"coord acceptance author failed to dispatch for #{state.issue}. "
|
|
563
|
+
"Check coordinator.yml's acceptance.drivers entry for "
|
|
564
|
+
f"{state.repo!r}, or re-run coord drive with --no-acceptance "
|
|
565
|
+
"to skip JIT authoring."
|
|
566
|
+
),
|
|
567
|
+
)
|
|
568
|
+
|
|
569
|
+
if status == "merged":
|
|
570
|
+
return None
|
|
571
|
+
|
|
572
|
+
if status == "failed":
|
|
573
|
+
return _die(
|
|
574
|
+
f"acceptance author {aid} failed — inspect: coord log {aid} "
|
|
575
|
+
f"--machine {state.acceptance_author_machine or machine}\n"
|
|
576
|
+
" Continue by hand, or re-run coord drive with "
|
|
577
|
+
"--no-acceptance to skip JIT authoring."
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
if status == "cancelled":
|
|
581
|
+
return _die(
|
|
582
|
+
f"acceptance author {aid} was cancelled — re-dispatch by hand: "
|
|
583
|
+
f"coord acceptance author {state.repo} {oracle.tracking_issue} "
|
|
584
|
+
f"--issue {state.issue}\n"
|
|
585
|
+
" or re-run coord drive with --no-acceptance."
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
if status == "advisory":
|
|
589
|
+
# #1453 review finding 2: this is the #1386 bug class reborn — an
|
|
590
|
+
# ``advisory`` row is TERMINAL (drive_state.TERMINAL_STATUSES) and
|
|
591
|
+
# is explicitly excluded from coord's Test/Review/Merge auto-loop
|
|
592
|
+
# (coord.reconcile's "review_state = 'advisory'" skip), so it will
|
|
593
|
+
# NEVER transition to 'merged' on its own — treating it as
|
|
594
|
+
# "still landing" below would spin forever. Mirror `_decide_advisory`
|
|
595
|
+
# exactly: a real 0-commit exit is terminal outright; a #1357-style
|
|
596
|
+
# false positive (commits present) needs the same
|
|
597
|
+
# `--accept-advisory` opt-in the main work row uses, not a silent
|
|
598
|
+
# pass-through.
|
|
599
|
+
branch = state.acceptance_author_branch
|
|
600
|
+
probe = replace(state, work_branch=branch) if branch else state
|
|
601
|
+
if not branch or not verifier.branch_has_commits(probe):
|
|
602
|
+
return _die(
|
|
603
|
+
f"acceptance author {aid} exited ADVISORY with no commits on "
|
|
604
|
+
"its branch — nothing was authored, so there is no slice to "
|
|
605
|
+
"land.\n"
|
|
606
|
+
f" inspect: coord log {aid} --machine "
|
|
607
|
+
f"{state.acceptance_author_machine or machine}\n"
|
|
608
|
+
" Continue by hand, or re-run coord drive with "
|
|
609
|
+
"--no-acceptance to skip JIT authoring."
|
|
610
|
+
)
|
|
611
|
+
if not opts.accept_advisory:
|
|
612
|
+
return _die(
|
|
613
|
+
f"acceptance author {aid} is ADVISORY, but its branch carries "
|
|
614
|
+
"real commits (the #1357 signature — see _decide_advisory).\n"
|
|
615
|
+
" Proceed anyway with --accept-advisory, or re-run coord "
|
|
616
|
+
"drive with --no-acceptance."
|
|
617
|
+
)
|
|
618
|
+
# #2079: "proceeding per --accept-advisory" now proceeds. This used
|
|
619
|
+
# to be a bare WAIT, which for an ADVISORY row is unreachable-by-
|
|
620
|
+
# construction: `coord.reconcile` explicitly skips advisory rows in
|
|
621
|
+
# the Test/Review/Merge auto-loop (the very fact the comment above
|
|
622
|
+
# cites), so the thing being waited for could not happen even with
|
|
623
|
+
# `merge.auto_drain` on.
|
|
624
|
+
return replace(
|
|
625
|
+
_decide_acceptance_landing(state, oracle, opts, counters, machine),
|
|
626
|
+
warnings=(
|
|
627
|
+
f"ACCEPTANCE: JIT slice {aid} is ADVISORY with commits present "
|
|
628
|
+
"— proceeding per --accept-advisory (#1357)",
|
|
629
|
+
),
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
if status == "done":
|
|
633
|
+
# #1535: `done` is TERMINAL (drive_state.TERMINAL_STATUSES) exactly
|
|
634
|
+
# like `advisory` — it will never transition to `merged` on its own
|
|
635
|
+
# if nothing was ever pushed (a #1534-style false "done", a reap, or
|
|
636
|
+
# a worker that forgot to push, the recurring shape). Re-polling
|
|
637
|
+
# can't help a terminal status, so waiting to `--deadline` with no
|
|
638
|
+
# diagnosis is the #1526 merge-gate defect reborn here. Mirror the
|
|
639
|
+
# `advisory` branch's probe exactly.
|
|
640
|
+
branch = state.acceptance_author_branch
|
|
641
|
+
probe = replace(state, work_branch=branch) if branch else state
|
|
642
|
+
if not branch or not verifier.branch_has_commits(probe):
|
|
643
|
+
branch_display = repr(branch) if branch else "(none)"
|
|
644
|
+
return _die(
|
|
645
|
+
f"acceptance author {aid} exited DONE, but its branch "
|
|
646
|
+
f"{branch_display} carries no commits — nothing was "
|
|
647
|
+
"authored, so there is no slice to land, and DONE is "
|
|
648
|
+
"terminal: it will never change on its own.\n"
|
|
649
|
+
f" inspect: coord log {aid} --machine "
|
|
650
|
+
f"{state.acceptance_author_machine or machine}\n"
|
|
651
|
+
" Re-author by hand: coord acceptance author "
|
|
652
|
+
f"{state.repo} {oracle.tracking_issue} --issue "
|
|
653
|
+
f"{state.issue}\n"
|
|
654
|
+
" or re-run coord drive with --no-acceptance to skip JIT "
|
|
655
|
+
"authoring."
|
|
656
|
+
)
|
|
657
|
+
# Authoring finished and the branch carries commits: hand over to the
|
|
658
|
+
# landing driver, which observes the daemon-driven Test/Review stages
|
|
659
|
+
# and performs the one step the daemon will not (#2079 — the merge).
|
|
660
|
+
return _decide_acceptance_landing(state, oracle, opts, counters, machine)
|
|
661
|
+
|
|
662
|
+
# "" / running: still authoring — nothing to drive yet.
|
|
663
|
+
return _wait(
|
|
664
|
+
label=(
|
|
665
|
+
f"ACCEPTANCE: JIT slice {aid} status={status or '(none)'} — authoring"
|
|
666
|
+
)
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _decide_acceptance_landing(
|
|
671
|
+
state: IssueState,
|
|
672
|
+
oracle: OracleDecision,
|
|
673
|
+
opts: DriveOptions,
|
|
674
|
+
counters: DriveCounters,
|
|
675
|
+
machine: str,
|
|
676
|
+
) -> Action:
|
|
677
|
+
"""Land the authored JIT acceptance slice (#2079).
|
|
678
|
+
|
|
679
|
+
Reached only once the slice's own ``test-author`` row is terminal WITH
|
|
680
|
+
commits on its branch — i.e. there is a real PR to land. From here the
|
|
681
|
+
slice walks the identical Test → Review → Merge path a work row does, and
|
|
682
|
+
this function takes the identical posture :func:`decide` takes for the
|
|
683
|
+
work row: **observe the stages the daemon dispatches, perform the merge
|
|
684
|
+
itself.**
|
|
685
|
+
|
|
686
|
+
The split is not a style choice, it is where the daemon's tick actually
|
|
687
|
+
stops. ``dispatch_pending_smoke``, ``dispatch_pending_reviews`` and
|
|
688
|
+
``merge_queue.enqueue_approved_work`` all run unconditionally on
|
|
689
|
+
``serve_app._passive_tick``; ``_auto_drain_tick`` — the step that turns a
|
|
690
|
+
READY queue entry into a merged PR — runs only when ``merge.auto_drain``
|
|
691
|
+
is on, and it is off. So waiting for the first three is waiting for
|
|
692
|
+
something that will happen, and waiting for the fourth is the #1526
|
|
693
|
+
defect: an unbounded wait for an event that cannot occur.
|
|
694
|
+
|
|
695
|
+
Three shapes get an immediate, actionable exit instead of a wait,
|
|
696
|
+
because for each of them the corrective action belongs to a loop that
|
|
697
|
+
will never run for this row:
|
|
698
|
+
|
|
699
|
+
* a FAILED slice test — the work row's equivalent dispatches
|
|
700
|
+
``coord fix``, but nothing dispatches one for a ``test-author`` row
|
|
701
|
+
whose Test stage failed;
|
|
702
|
+
* a ``request-changes`` slice review — ``auto_loop`` accepts a
|
|
703
|
+
``test-author`` fix (``FIX_DISPATCH_TYPES``), but the daemon drain
|
|
704
|
+
deliberately excludes fix dispatch (#476/#477, and #1692's own
|
|
705
|
+
analysis), so for the work row it is THIS driver that runs
|
|
706
|
+
``coord fix``; there was no such arm for the slice;
|
|
707
|
+
* ``--no-merge`` — the slice merge is a hard prerequisite for
|
|
708
|
+
dispatching any work at all (#1138), so with merging switched off the
|
|
709
|
+
run cannot progress, and saying so beats idling to the deadline.
|
|
710
|
+
|
|
711
|
+
The merge itself reuses :func:`_decide_merge` verbatim against a shadow
|
|
712
|
+
:class:`~coord.drive_state.IssueState` whose "work row" IS the slice —
|
|
713
|
+
that is what carries #1891 (CI not reported → wait, don't retry), #1892
|
|
714
|
+
(CI infra failure → wait), #1505 (a status no retry can fix →
|
|
715
|
+
escalate), #1526 (driver/gate divergence) and #2078 (quote the real
|
|
716
|
+
``coord merge --only`` diagnostic) into the slice lane without a second
|
|
717
|
+
implementation of any of them.
|
|
718
|
+
"""
|
|
719
|
+
aid = state.acceptance_author_aid
|
|
720
|
+
merge_status = (state.acceptance_merge_status or "").upper()
|
|
721
|
+
|
|
722
|
+
# Already landed on GitHub; the board row just hasn't been reconciled to
|
|
723
|
+
# `status='merged'` yet. Checked FIRST: MERGED is not in
|
|
724
|
+
# `_RETRYABLE_MERGE_STATUSES`, so handing it to `_decide_merge` would
|
|
725
|
+
# escalate a success.
|
|
726
|
+
if merge_status == "MERGED":
|
|
727
|
+
return _wait(
|
|
728
|
+
label=(
|
|
729
|
+
f"ACCEPTANCE: JIT slice {aid} PR is MERGED — waiting for the "
|
|
730
|
+
"board row to reconcile to status='merged'"
|
|
731
|
+
)
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
test_state = state.acceptance_author_test_state
|
|
735
|
+
if test_state == "failed":
|
|
736
|
+
return _die(
|
|
737
|
+
f"the JIT acceptance slice {aid} FAILED its Test stage — nothing "
|
|
738
|
+
"will fix it on its own (the review→fix loop that covers a work "
|
|
739
|
+
"row is not dispatched for a test-author row).\n"
|
|
740
|
+
f" inspect: coord log {aid} --machine "
|
|
741
|
+
f"{state.acceptance_author_machine or machine}\n"
|
|
742
|
+
f" Fix it: coord fix {aid}\n"
|
|
743
|
+
" or re-run coord drive with --no-acceptance to skip JIT "
|
|
744
|
+
"authoring."
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
if state.acceptance_review_verdict == "request-changes":
|
|
748
|
+
return _die(
|
|
749
|
+
f"the JIT acceptance slice {aid} was reviewed REQUEST-CHANGES "
|
|
750
|
+
f"(review {state.acceptance_review_aid or 'unknown'}) — it will "
|
|
751
|
+
"never reach the merge queue until that is addressed, and no "
|
|
752
|
+
"loop dispatches the fix for a test-author row on its own.\n"
|
|
753
|
+
f" Findings: coord log {state.acceptance_review_aid or aid}\n"
|
|
754
|
+
f" Fix it: coord fix {state.acceptance_review_aid or aid}\n"
|
|
755
|
+
" or re-run coord drive with --no-acceptance to skip JIT "
|
|
756
|
+
"authoring."
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
if not opts.do_merge:
|
|
760
|
+
pr = state.acceptance_merge_pr_url or "(no PR recorded yet)"
|
|
761
|
+
return _die(
|
|
762
|
+
f"the JIT acceptance slice {aid} is authored but NOT landed, and "
|
|
763
|
+
"--no-merge is set.\n"
|
|
764
|
+
f" Its PR: {pr}\n"
|
|
765
|
+
" #1138 refuses to dispatch work for this issue until the slice "
|
|
766
|
+
"is merged, so this run cannot progress.\n"
|
|
767
|
+
f" Land it by hand: coord merge --only {aid} --method "
|
|
768
|
+
f"{opts.merge_method}\n"
|
|
769
|
+
" or re-run coord drive without --no-merge (or with "
|
|
770
|
+
"--no-acceptance to skip JIT authoring)."
|
|
771
|
+
)
|
|
772
|
+
|
|
773
|
+
# The shadow state: the slice IS the work row. `issue` becomes the
|
|
774
|
+
# milestone's TRACKING issue because that is what the slice's own board
|
|
775
|
+
# row and merge-queue entry are keyed on — so every `coord diagnose
|
|
776
|
+
# <repo> <issue>` / `coord escalate record <repo> <issue>` command
|
|
777
|
+
# `_decide_merge` and `_escalate_merge` compose points at the row a human
|
|
778
|
+
# would actually have to fix.
|
|
779
|
+
shadow = replace(
|
|
780
|
+
state,
|
|
781
|
+
issue=oracle.tracking_issue or state.issue,
|
|
782
|
+
work_aid=aid,
|
|
783
|
+
work_branch=state.acceptance_author_branch,
|
|
784
|
+
work_machine=state.acceptance_author_machine,
|
|
785
|
+
work_test_state=test_state,
|
|
786
|
+
work_test_reason="",
|
|
787
|
+
review_aid=state.acceptance_review_aid,
|
|
788
|
+
review_verdict=state.acceptance_review_verdict,
|
|
789
|
+
merge_status=state.acceptance_merge_status,
|
|
790
|
+
merge_reason=state.acceptance_merge_reason,
|
|
791
|
+
merge_aid=state.acceptance_merge_aid,
|
|
792
|
+
merge_pr_url=state.acceptance_merge_pr_url,
|
|
793
|
+
)
|
|
794
|
+
action = _decide_merge(shadow, opts, counters.slice_budget())
|
|
795
|
+
return replace(
|
|
796
|
+
action,
|
|
797
|
+
label=_acceptance_label(action.label),
|
|
798
|
+
message=_acceptance_message(action.message, state),
|
|
799
|
+
merge_scope="acceptance",
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _acceptance_label(label: str) -> str:
|
|
804
|
+
"""Re-badge a :func:`_decide_merge` label as a slice-lane one (#2079).
|
|
805
|
+
|
|
806
|
+
Without this the pane prints ``MERGE: attempt 1/3`` for the SLICE's merge
|
|
807
|
+
while the issue's own work row does not exist yet — the single most
|
|
808
|
+
confusing line this driver could emit.
|
|
809
|
+
"""
|
|
810
|
+
if not label:
|
|
811
|
+
return ""
|
|
812
|
+
return "ACCEPTANCE/" + label
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _acceptance_message(message: str, state: IssueState) -> str:
|
|
816
|
+
"""The same re-badging for an EXIT action's message (#2079).
|
|
817
|
+
|
|
818
|
+
The exit message is what reaches the issue comment
|
|
819
|
+
(``Driver._post_escalation_comment``) and the drive-queue's stop reason —
|
|
820
|
+
i.e. the only two places a human reads it after the tmux pane is gone. A
|
|
821
|
+
bare "merge attempted 3 times without landing" there points them at the
|
|
822
|
+
issue's own PR, which does not exist yet.
|
|
823
|
+
"""
|
|
824
|
+
if not message:
|
|
825
|
+
return ""
|
|
826
|
+
return (
|
|
827
|
+
f"the JIT acceptance slice for #{state.issue} could not be landed "
|
|
828
|
+
f"(slice {state.acceptance_author_aid}, branch "
|
|
829
|
+
f"{state.acceptance_author_branch or '(none)'}) — no work can be "
|
|
830
|
+
"dispatched for this issue until it merges (#1138/#2079):\n"
|
|
831
|
+
f"{message}"
|
|
832
|
+
)
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
# ── merge verification ───────────────────────────────────────────────────────
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
class MergeVerifier(Protocol):
|
|
839
|
+
"""The two git/GitHub questions the state machine cannot answer itself."""
|
|
840
|
+
|
|
841
|
+
def branch_has_commits(self, state: IssueState) -> bool: ...
|
|
842
|
+
|
|
843
|
+
def verify_merged(self, state: IssueState) -> bool: ...
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
@dataclass
|
|
847
|
+
class GitMergeVerifier:
|
|
848
|
+
"""Real implementation: ``git`` for commits, ``gh`` for merge state.
|
|
849
|
+
|
|
850
|
+
``repo_path`` is the local checkout used for fetches; defaults to
|
|
851
|
+
``~/src/<repo>``.
|
|
852
|
+
"""
|
|
853
|
+
|
|
854
|
+
repo_path: str = ""
|
|
855
|
+
warn: Callable[[str], None] = lambda msg: None
|
|
856
|
+
|
|
857
|
+
def _base(self, state: IssueState) -> Path | None:
|
|
858
|
+
base = Path(self.repo_path).expanduser() if self.repo_path else (
|
|
859
|
+
Path.home() / "src" / state.repo
|
|
860
|
+
)
|
|
861
|
+
return base if (base / ".git").exists() else None
|
|
862
|
+
|
|
863
|
+
@staticmethod
|
|
864
|
+
def _git(base: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
865
|
+
return subprocess.run(
|
|
866
|
+
["git", "-C", str(base), *args],
|
|
867
|
+
capture_output=True,
|
|
868
|
+
text=True,
|
|
869
|
+
check=False,
|
|
870
|
+
)
|
|
871
|
+
|
|
872
|
+
def branch_has_commits(self, state: IssueState) -> bool:
|
|
873
|
+
"""True when *branch* exists on the remote and carries a commit the
|
|
874
|
+
default branch does not.
|
|
875
|
+
|
|
876
|
+
Used to tell a REAL zero-commit advisory apart from the #1357 false
|
|
877
|
+
positive, where the agent downgrades a good DONE over an artifact glob
|
|
878
|
+
that matched nothing.
|
|
879
|
+
"""
|
|
880
|
+
branch = state.work_branch
|
|
881
|
+
if not branch:
|
|
882
|
+
return False
|
|
883
|
+
base = self._base(state)
|
|
884
|
+
if base is None:
|
|
885
|
+
return False
|
|
886
|
+
target = state.repo_default_branch or "main"
|
|
887
|
+
if self._git(base, "fetch", "--quiet", "origin", target).returncode != 0:
|
|
888
|
+
return False
|
|
889
|
+
if self._git(base, "fetch", "--quiet", "origin", branch).returncode != 0:
|
|
890
|
+
return False
|
|
891
|
+
proc = self._git(base, "rev-list", "--count", f"origin/{target}..FETCH_HEAD")
|
|
892
|
+
if proc.returncode != 0:
|
|
893
|
+
return False
|
|
894
|
+
try:
|
|
895
|
+
return int((proc.stdout or "0").strip() or 0) > 0
|
|
896
|
+
except ValueError:
|
|
897
|
+
return False
|
|
898
|
+
|
|
899
|
+
def verify_merged(self, state: IssueState) -> bool:
|
|
900
|
+
"""Confirm the branch actually landed on the target.
|
|
901
|
+
|
|
902
|
+
NOTE: ``merge-base --is-ancestor`` is the WRONG test here. ``coord
|
|
903
|
+
merge`` defaults to ``--method rebase`` (and supports squash), both of
|
|
904
|
+
which rewrite the commits — so a fully-merged branch's tip SHA is never
|
|
905
|
+
an ancestor of the target. Verified against #1344: merged via PR
|
|
906
|
+
#1355, two commits on main, and ``--is-ancestor`` still says no.
|
|
907
|
+
"""
|
|
908
|
+
branch = state.work_branch
|
|
909
|
+
target = state.repo_default_branch or "main"
|
|
910
|
+
if not branch:
|
|
911
|
+
return False
|
|
912
|
+
|
|
913
|
+
# Primary: ask GitHub. Authoritative for every merge method, and still
|
|
914
|
+
# correct after the merged branch has been deleted from the remote.
|
|
915
|
+
# Routed through the github_ops seam (#1483) rather than shelling out
|
|
916
|
+
# to `gh` directly here — no `shutil.which` probe, so behaviour never
|
|
917
|
+
# silently varies with whether `gh` happens to be on this host's PATH.
|
|
918
|
+
if state.repo_github:
|
|
919
|
+
from coord import github_ops # noqa: PLC0415
|
|
920
|
+
|
|
921
|
+
pr_state = github_ops.get_pr_state_for_branch(state.repo_github, branch) or ""
|
|
922
|
+
if pr_state == "MERGED":
|
|
923
|
+
return True
|
|
924
|
+
if pr_state:
|
|
925
|
+
self.warn(f"PR for {branch} is {pr_state}, not MERGED")
|
|
926
|
+
return False
|
|
927
|
+
|
|
928
|
+
# Fallback: patch-equivalence. Every commit of a landed branch has an
|
|
929
|
+
# equivalent upstream, which is exactly what `git cherry` marks with
|
|
930
|
+
# '-'; a '+' means that commit is genuinely not on the target yet.
|
|
931
|
+
base = self._base(state)
|
|
932
|
+
if base is None:
|
|
933
|
+
return False
|
|
934
|
+
vref = f"refs/remotes/coord-verify/{branch}"
|
|
935
|
+
if self._git(base, "fetch", "--quiet", "origin", target).returncode != 0:
|
|
936
|
+
return False
|
|
937
|
+
fetched = self._git(
|
|
938
|
+
base, "fetch", "--quiet", "origin", f"refs/heads/{branch}:{vref}"
|
|
939
|
+
)
|
|
940
|
+
if fetched.returncode != 0:
|
|
941
|
+
return False
|
|
942
|
+
try:
|
|
943
|
+
proc = self._git(base, "cherry", f"origin/{target}", f"coord-verify/{branch}")
|
|
944
|
+
if proc.returncode != 0:
|
|
945
|
+
return False
|
|
946
|
+
unmerged = [
|
|
947
|
+
line for line in (proc.stdout or "").splitlines() if line.startswith("+")
|
|
948
|
+
]
|
|
949
|
+
return not unmerged
|
|
950
|
+
finally:
|
|
951
|
+
self._git(base, "update-ref", "-d", vref)
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
# ── preflight (pure) ─────────────────────────────────────────────────────────
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
@dataclass(frozen=True)
|
|
958
|
+
class Preflight:
|
|
959
|
+
"""The resolved machine plus anything worth warning about before looping."""
|
|
960
|
+
|
|
961
|
+
machine: str
|
|
962
|
+
warnings: tuple[str, ...] = ()
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def preflight(
|
|
966
|
+
state: IssueState,
|
|
967
|
+
opts: DriveOptions,
|
|
968
|
+
config: Any = None,
|
|
969
|
+
*,
|
|
970
|
+
usage_limits: PlanLimits | None = None,
|
|
971
|
+
) -> Preflight:
|
|
972
|
+
"""Resolve the machine and refuse the runs that can never win.
|
|
973
|
+
|
|
974
|
+
Raises :class:`DriveError` for a configuration problem or for interactive
|
|
975
|
+
work with no review (see below).
|
|
976
|
+
|
|
977
|
+
*usage_limits* (#1466) is the ALREADY-PROBED Max-plan 5h/weekly usage
|
|
978
|
+
snapshot — this function stays pure and never shells out itself, mirroring
|
|
979
|
+
the *verifier*/*gate_checker* injection pattern used elsewhere in this
|
|
980
|
+
module. ``None`` (every pre-#1466 caller, and any caller that skips the
|
|
981
|
+
probe) is treated exactly like an unavailable probe: the gate silently
|
|
982
|
+
lets the run proceed. *config* is likewise optional — ``None`` skips the
|
|
983
|
+
usage gate entirely (no ``usage_gate`` section to consult), which is what
|
|
984
|
+
every pre-#1466 test in this file's suite still passes.
|
|
985
|
+
|
|
986
|
+
#1906: ``state.picked_machine`` is now itself provider-capability-aware
|
|
987
|
+
(:func:`coord.drive_state.pick_machine_choice`, resolved once in
|
|
988
|
+
:func:`project`) whenever an auto-pick happens — this function no
|
|
989
|
+
longer needs to (and does not) resolve the provider itself; it only
|
|
990
|
+
reports the *result*, including the distinct "no capable machine" vs.
|
|
991
|
+
"no host at all" failure state on ``state``.
|
|
992
|
+
"""
|
|
993
|
+
machine = opts.machine or state.picked_machine
|
|
994
|
+
if not machine:
|
|
995
|
+
# #1906: an explicit `--machine` always wins (never reaches here —
|
|
996
|
+
# `opts.machine` short-circuits `or` above) and hits #1711's own
|
|
997
|
+
# refusal downstream in `coord.dispatch.dispatch()` instead, exactly
|
|
998
|
+
# like today. This branch is only the AUTO-pick failing, and it now
|
|
999
|
+
# has two distinct causes that must not collapse into one message:
|
|
1000
|
+
# no unpaused machine hosts the repo at all, vs. at least one does
|
|
1001
|
+
# but none advertise the resolved provider (`state.picked_machine`'s
|
|
1002
|
+
# own `coord.drive_state.pick_machine_choice` already did the
|
|
1003
|
+
# capability filtering — see `IssueState.picked_machine_no_capable`).
|
|
1004
|
+
if state.picked_machine_no_capable:
|
|
1005
|
+
raise DriveError(
|
|
1006
|
+
f"no unpaused machine advertises provider "
|
|
1007
|
+
f"{state.picked_machine_provider!r} for {state.repo} — pass "
|
|
1008
|
+
"--machine, or add the capability to a machine's "
|
|
1009
|
+
"coordinator.yml machines[].capabilities",
|
|
1010
|
+
EXIT_USAGE,
|
|
1011
|
+
)
|
|
1012
|
+
raise DriveError(
|
|
1013
|
+
f"no unpaused machine hosts {state.repo} — pass --machine",
|
|
1014
|
+
EXIT_USAGE,
|
|
1015
|
+
)
|
|
1016
|
+
|
|
1017
|
+
warnings: list[str] = []
|
|
1018
|
+
|
|
1019
|
+
if config is not None:
|
|
1020
|
+
gate_cfg = config.usage_gate
|
|
1021
|
+
limits = usage_limits if usage_limits is not None else PlanLimits(status="unknown")
|
|
1022
|
+
gate_result = evaluate_usage_gate(limits, gate_cfg)
|
|
1023
|
+
if gate_result.action == "block":
|
|
1024
|
+
raise DriveError(
|
|
1025
|
+
f"{gate_result.message} (usage_gate.mode: block) — refusing to "
|
|
1026
|
+
"dispatch. Wait for the window to reset, or lower urgency by "
|
|
1027
|
+
"raising the threshold / setting usage_gate.mode: warn in "
|
|
1028
|
+
"coordinator.yml.",
|
|
1029
|
+
EXIT_USAGE,
|
|
1030
|
+
)
|
|
1031
|
+
if gate_result.action == "warn":
|
|
1032
|
+
warnings.append(f"{gate_result.message} (usage_gate.mode: warn — proceeding anyway)")
|
|
1033
|
+
|
|
1034
|
+
if not state.auto_loop:
|
|
1035
|
+
warnings.append(
|
|
1036
|
+
"pipeline.auto_loop is OFF — the review→fix path is switched off."
|
|
1037
|
+
)
|
|
1038
|
+
warnings.append(
|
|
1039
|
+
"A request-changes verdict will be REPORTED and this run will stop "
|
|
1040
|
+
"(#1692): `coord fix` refuses while auto_loop is off, so there is "
|
|
1041
|
+
"no fix to dispatch."
|
|
1042
|
+
)
|
|
1043
|
+
|
|
1044
|
+
# INTERACTIVE WORK NEVER GETS AN AUTOMATIC REVIEW.
|
|
1045
|
+
#
|
|
1046
|
+
# `dispatch_pending_reviews` carries `and c.provider_name != "claude-pty"`
|
|
1047
|
+
# (#555): a metered headless review must never silently follow a
|
|
1048
|
+
# human-attended session. So for work done interactively, the review is
|
|
1049
|
+
# not "late" — it is never coming, and waiting for it is an infinite stall.
|
|
1050
|
+
#
|
|
1051
|
+
# Checked HERE, at preflight, rather than at the review gate: otherwise a
|
|
1052
|
+
# run burns the full test suite (~6 min) before parking on a wait it can
|
|
1053
|
+
# never win. That is exactly what happened driving #1357 — test gate
|
|
1054
|
+
# passed at 4642/4642, then 90 minutes of nothing.
|
|
1055
|
+
if state.work_aid and state.work_provider == "claude-pty" and not state.review_aid:
|
|
1056
|
+
if opts.force_review:
|
|
1057
|
+
warnings.append(
|
|
1058
|
+
f"work {state.work_aid} is INTERACTIVE (claude-pty) — no "
|
|
1059
|
+
"automatic review (#555)."
|
|
1060
|
+
)
|
|
1061
|
+
warnings.append(
|
|
1062
|
+
"--force-review set: this run will request the review explicitly."
|
|
1063
|
+
)
|
|
1064
|
+
else:
|
|
1065
|
+
raise DriveError(
|
|
1066
|
+
f"work {state.work_aid} was completed INTERACTIVELY "
|
|
1067
|
+
"(provider=claude-pty).\n"
|
|
1068
|
+
" coord's #555 guard permanently excludes interactive work from "
|
|
1069
|
+
"automatic\n"
|
|
1070
|
+
" review dispatch, so waiting for one would stall forever.\n\n"
|
|
1071
|
+
" Either drive it unattended: re-run with --force-review\n"
|
|
1072
|
+
" or review it human-attended: coord assign --interactive "
|
|
1073
|
+
f"--review-of {state.work_aid}",
|
|
1074
|
+
EXIT_USAGE,
|
|
1075
|
+
)
|
|
1076
|
+
|
|
1077
|
+
return Preflight(machine=machine, warnings=tuple(warnings))
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
# ── the state machine (pure) ─────────────────────────────────────────────────
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _escalate_dead_end(state: IssueState, dead_end: DeadEnd) -> Action:
|
|
1084
|
+
"""Build the EXIT action for a terminal-and-unactionable row (#2019).
|
|
1085
|
+
|
|
1086
|
+
Structurally identical to :func:`_escalate_merge` — same ``coord escalate
|
|
1087
|
+
record`` argv, same "this function stays pure, the write happens in
|
|
1088
|
+
:meth:`Driver._loop`'s exit handling" split — because the operator-facing
|
|
1089
|
+
outcome is the same: a board-visible record naming the blocker and the
|
|
1090
|
+
command that clears it, instead of a counter ticking against an event that
|
|
1091
|
+
can never happen.
|
|
1092
|
+
|
|
1093
|
+
The exit code is what differs. :data:`EXIT_ESCALATED` means "a human
|
|
1094
|
+
decision is waiting"; :data:`EXIT_DEAD_END` additionally means "and no
|
|
1095
|
+
relaunch of this drive can change it", which is the fact
|
|
1096
|
+
``coord/drive_queue.py``'s tick needs to block the entry WITHOUT spending
|
|
1097
|
+
an attempt (the #1844 posture, applied to a second cause).
|
|
1098
|
+
"""
|
|
1099
|
+
gates_summary = " | ".join(f"{k}={v}" for k, v in dead_end.gates)
|
|
1100
|
+
|
|
1101
|
+
command: list[str] = [
|
|
1102
|
+
"escalate", "record", state.repo, str(state.issue),
|
|
1103
|
+
"--stage", dead_end.stage,
|
|
1104
|
+
"--reason", f"{dead_end.kind}: {dead_end.reason}",
|
|
1105
|
+
]
|
|
1106
|
+
for key, value in dead_end.gates:
|
|
1107
|
+
command += ["--gate", f"{key}={value}"]
|
|
1108
|
+
command += ["--command", dead_end.recovery]
|
|
1109
|
+
if dead_end.assignment_id:
|
|
1110
|
+
command += ["--assignment", dead_end.assignment_id]
|
|
1111
|
+
|
|
1112
|
+
return Action(
|
|
1113
|
+
kind=EXIT,
|
|
1114
|
+
exit_code=EXIT_DEAD_END,
|
|
1115
|
+
message=(
|
|
1116
|
+
f"DEAD END [{dead_end.kind}] — this row is terminal and "
|
|
1117
|
+
"unactionable; exiting instead of polling (#2019).\n"
|
|
1118
|
+
f" {dead_end.reason}\n"
|
|
1119
|
+
f" gates: {gates_summary}\n"
|
|
1120
|
+
f" Recover: {dead_end.recovery}\n"
|
|
1121
|
+
f" Recorded on the board — see: coord escalate list --repo "
|
|
1122
|
+
f"{state.repo}"
|
|
1123
|
+
),
|
|
1124
|
+
command=tuple(command),
|
|
1125
|
+
error_message=(
|
|
1126
|
+
"failed to record the dead-end escalation on the board (exiting "
|
|
1127
|
+
f"anyway — resolve by hand: {dead_end.recovery})"
|
|
1128
|
+
),
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def decide(
|
|
1133
|
+
state: IssueState,
|
|
1134
|
+
opts: DriveOptions,
|
|
1135
|
+
counters: DriveCounters,
|
|
1136
|
+
verifier: MergeVerifier,
|
|
1137
|
+
*,
|
|
1138
|
+
machine: str = "",
|
|
1139
|
+
oracle: OracleDecision | None = None,
|
|
1140
|
+
gate_checker: AcceptanceGateChecker | None = None,
|
|
1141
|
+
) -> Action:
|
|
1142
|
+
"""One step of the state machine: given the board, what next?
|
|
1143
|
+
|
|
1144
|
+
Pure apart from the injected *verifier* (git/GitHub) and the bounded
|
|
1145
|
+
counters it increments. Every branch here was a bash ``case`` arm; the
|
|
1146
|
+
ordering is identical, and — critically — **no terminal status falls
|
|
1147
|
+
through to a bare wait**. An ``advisory`` work row doing exactly that was
|
|
1148
|
+
a silent 240-minute spin (fixed in PR #1386, and now unit-tested).
|
|
1149
|
+
|
|
1150
|
+
*oracle* (#1453) is resolved ONCE per run by :func:`resolve_oracle_decision`
|
|
1151
|
+
and threaded through unchanged on every call — ``None`` (the default,
|
|
1152
|
+
every pre-#1453 caller) behaves exactly as before: no JIT slice, straight
|
|
1153
|
+
to ``coord assign``. *gate_checker* is only consulted when *oracle* is
|
|
1154
|
+
active (to resolve a routed repo's ``--for-path``, #1453 review finding
|
|
1155
|
+
1) — unused, like *oracle*, on every pre-#1453 call site.
|
|
1156
|
+
"""
|
|
1157
|
+
machine = machine or opts.machine or state.picked_machine
|
|
1158
|
+
|
|
1159
|
+
# ---- terminal: merged ---------------------------------------------------
|
|
1160
|
+
if state.work_status == "merged" or state.merge_status == "MERGED":
|
|
1161
|
+
target = state.repo_default_branch or "main"
|
|
1162
|
+
if state.work_branch and verifier.verify_merged(state):
|
|
1163
|
+
return _succeed(f"✓ MERGED — {state.work_branch} has landed on {target}")
|
|
1164
|
+
base = opts.repo_path or f"~/src/{state.repo}"
|
|
1165
|
+
return _die(
|
|
1166
|
+
f"board says merged but {state.work_branch} has NOT landed on {target}\n"
|
|
1167
|
+
f" verify by hand: git -C {base} log --oneline origin/{target}"
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
# ---- something is running: just wait -----------------------------------
|
|
1171
|
+
if state.active_count > 0:
|
|
1172
|
+
return _wait()
|
|
1173
|
+
|
|
1174
|
+
# ---- no work yet: plan and/or dispatch ---------------------------------
|
|
1175
|
+
if not state.work_aid:
|
|
1176
|
+
return _dispatch_work_stage(
|
|
1177
|
+
state, opts, counters, machine, oracle, gate_checker, verifier
|
|
1178
|
+
)
|
|
1179
|
+
|
|
1180
|
+
# ---- work died from hitting the account's usage limit: wait -----------
|
|
1181
|
+
#
|
|
1182
|
+
# #1461: a usage-limit kill is NOT a defect and NOT the #1357 zero-commit
|
|
1183
|
+
# advisory signature — it is the ONE terminal state known safe to
|
|
1184
|
+
# re-dispatch unchanged, once the reset time passes. Falling into the
|
|
1185
|
+
# bounded-retry branch below (or `coord fix`'s model escalation, on the
|
|
1186
|
+
# advisory side) would just re-dispatch straight into the same exhausted
|
|
1187
|
+
# budget and fail again for no diagnostic reason — exactly the confusion
|
|
1188
|
+
# the issue is about. Detected via the `usage limit — resets ...` prefix
|
|
1189
|
+
# that `coord.worker_events.format_usage_limit_reason` stamps onto
|
|
1190
|
+
# `failure_reason` regardless of whether the agent's own reap landed on
|
|
1191
|
+
# FAILED or ADVISORY (#1461's own worked example hit both in one
|
|
1192
|
+
# session). Deliberately does NOT auto-retry here — retrying before the
|
|
1193
|
+
# reset only produces more of the same; a human (or a future reset-aware
|
|
1194
|
+
# auto-retry) re-runs `coord retry` once the window reopens.
|
|
1195
|
+
#
|
|
1196
|
+
# #1590: routed through `coord.failure_class` so this branch and the
|
|
1197
|
+
# sequencer's budget agree on what "environmental" means, and the surfaced
|
|
1198
|
+
# warning now names *when* the node could resume — the `reset_at_raw` the
|
|
1199
|
+
# detector has always parsed and nobody ever used.
|
|
1200
|
+
if state.work_status in ("failed", "advisory"):
|
|
1201
|
+
classification = classify_failure(
|
|
1202
|
+
failure_reason=state.work_failure_reason or None
|
|
1203
|
+
)
|
|
1204
|
+
if classification.is_usage_limit:
|
|
1205
|
+
resume = plan_usage_limit_resume(
|
|
1206
|
+
reset_at_raw=classification.reset_at_raw
|
|
1207
|
+
)
|
|
1208
|
+
when = (
|
|
1209
|
+
resume.resume_at.isoformat(timespec="minutes")
|
|
1210
|
+
if resume.from_reset_time
|
|
1211
|
+
else "unknown (reset time not parseable)"
|
|
1212
|
+
)
|
|
1213
|
+
return Action(
|
|
1214
|
+
kind=WAIT,
|
|
1215
|
+
label=(
|
|
1216
|
+
f"WORK: {state.work_aid} killed by the usage limit — waiting "
|
|
1217
|
+
"for the reset, not retrying"
|
|
1218
|
+
),
|
|
1219
|
+
warnings=(
|
|
1220
|
+
f"usage-limit kill detected on {state.work_aid}: "
|
|
1221
|
+
f"{state.work_failure_reason} — waiting for the reset instead "
|
|
1222
|
+
"of retrying (#1461)",
|
|
1223
|
+
f"{classification.reason}; earliest resume {when} (#1590)",
|
|
1224
|
+
),
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
# ---- work failed: bounded retry ----------------------------------------
|
|
1228
|
+
if state.work_status == "failed":
|
|
1229
|
+
if counters.work_retries >= opts.max_work_retries:
|
|
1230
|
+
# #1590 part 6: name the actual cause. "failed 3 retries in: <prose>"
|
|
1231
|
+
# sent the morning triage looking at the work even when the provider
|
|
1232
|
+
# was the problem; the class is now stated up front.
|
|
1233
|
+
classification = classify_failure(
|
|
1234
|
+
failure_reason=state.work_failure_reason or None
|
|
1235
|
+
)
|
|
1236
|
+
return _die(
|
|
1237
|
+
f"work {state.work_aid} failed {counters.work_retries} retr(ies) in: "
|
|
1238
|
+
f"{state.work_failure_reason or 'no reason recorded'}\n"
|
|
1239
|
+
f" cause: {classification.reason}\n"
|
|
1240
|
+
f" inspect: coord log {state.work_aid} --machine "
|
|
1241
|
+
f"{state.work_machine or machine}"
|
|
1242
|
+
)
|
|
1243
|
+
counters.work_retries += 1
|
|
1244
|
+
return Action(
|
|
1245
|
+
kind=RUN,
|
|
1246
|
+
label=(
|
|
1247
|
+
f"WORK: failed → coord retry {state.work_aid} "
|
|
1248
|
+
f"(attempt {counters.work_retries}/{opts.max_work_retries})"
|
|
1249
|
+
),
|
|
1250
|
+
command=("retry", state.work_aid),
|
|
1251
|
+
error_message=f"coord retry failed for {state.work_aid}",
|
|
1252
|
+
)
|
|
1253
|
+
|
|
1254
|
+
# ---- work reached a terminal state that is not 'done' ------------------
|
|
1255
|
+
#
|
|
1256
|
+
# Every status here is TERMINAL (a non-terminal row would have been caught
|
|
1257
|
+
# by the active_count wait above), so none of them may fall through to a
|
|
1258
|
+
# bare wait — that spins silently until the deadline instead of reporting
|
|
1259
|
+
# anything.
|
|
1260
|
+
warnings: tuple[str, ...] = ()
|
|
1261
|
+
if state.work_status == "done":
|
|
1262
|
+
pass
|
|
1263
|
+
elif state.work_status == "advisory":
|
|
1264
|
+
advisory = _decide_advisory(state, opts, machine, verifier)
|
|
1265
|
+
if advisory.is_exit:
|
|
1266
|
+
return advisory
|
|
1267
|
+
warnings = advisory.warnings
|
|
1268
|
+
elif state.work_status == "cancelled":
|
|
1269
|
+
return _die(
|
|
1270
|
+
f"work {state.work_aid} was cancelled — re-dispatch with: "
|
|
1271
|
+
f"coord assign {machine} {state.repo} {state.issue} --force"
|
|
1272
|
+
)
|
|
1273
|
+
else:
|
|
1274
|
+
return _die(
|
|
1275
|
+
f"unexpected terminal work status '{state.work_status}' for "
|
|
1276
|
+
f"{state.work_aid} —\n"
|
|
1277
|
+
f" refusing to guess. Inspect: coord log {state.work_aid} --machine "
|
|
1278
|
+
f"{state.work_machine or machine}"
|
|
1279
|
+
)
|
|
1280
|
+
|
|
1281
|
+
# A 'done' row with no branch never pushed anything either.
|
|
1282
|
+
if not state.work_branch:
|
|
1283
|
+
return _die(
|
|
1284
|
+
f"work {state.work_aid} finished with no branch — nothing was pushed "
|
|
1285
|
+
"(0-commit advisory).\n"
|
|
1286
|
+
f" inspect: coord log {state.work_aid} --machine "
|
|
1287
|
+
f"{state.work_machine or machine}"
|
|
1288
|
+
)
|
|
1289
|
+
|
|
1290
|
+
# ---- the dead-end predicate (#2019) ------------------------------------
|
|
1291
|
+
#
|
|
1292
|
+
# Positioned HERE — after the merged/active/work-status arms above have
|
|
1293
|
+
# all had their say, before the Test and Review gates — on purpose:
|
|
1294
|
+
#
|
|
1295
|
+
# * everything above it is either terminal-and-already-reported (merged,
|
|
1296
|
+
# cancelled, an unexpected status) or genuinely actionable (a bounded
|
|
1297
|
+
# work retry, a usage-limit wait, an advisory), so the predicate can
|
|
1298
|
+
# never steal a live move from them;
|
|
1299
|
+
# * everything below it is a gate that, on the shapes the predicate
|
|
1300
|
+
# recognises, would return a bare `_wait()` and spin — which is the
|
|
1301
|
+
# entire bug (#1956: 140 minutes of `no state change`, with `active=0`
|
|
1302
|
+
# printed on every line).
|
|
1303
|
+
#
|
|
1304
|
+
# `detect_dead_end` itself refuses to fire while `active_count > 0`, so a
|
|
1305
|
+
# healthy long-running stage is structurally incapable of reaching this.
|
|
1306
|
+
# #2024: `--skip-test` is a live Test-stage move (`_decide_test` records
|
|
1307
|
+
# `skipped`), so the human-attended-Test shape must not escalate past it.
|
|
1308
|
+
dead_end = detect_dead_end(state, can_waive_test_gate=opts.skip_test)
|
|
1309
|
+
if dead_end is not None:
|
|
1310
|
+
return replace(
|
|
1311
|
+
_escalate_dead_end(state, dead_end), warnings=warnings
|
|
1312
|
+
)
|
|
1313
|
+
|
|
1314
|
+
test = _decide_test(state, opts, counters, machine)
|
|
1315
|
+
if test is not None:
|
|
1316
|
+
return replace(test, warnings=warnings + test.warnings)
|
|
1317
|
+
|
|
1318
|
+
review = _decide_review(state, opts, counters, machine)
|
|
1319
|
+
if review is not None:
|
|
1320
|
+
return replace(review, warnings=warnings + review.warnings)
|
|
1321
|
+
|
|
1322
|
+
if not opts.do_merge:
|
|
1323
|
+
return replace(
|
|
1324
|
+
_succeed(
|
|
1325
|
+
"✓ review approved — stopping here (--no-merge)\n"
|
|
1326
|
+
f" merge with: coord merge --only {state.work_aid}"
|
|
1327
|
+
),
|
|
1328
|
+
warnings=warnings,
|
|
1329
|
+
)
|
|
1330
|
+
|
|
1331
|
+
merge = _decide_merge(state, opts, counters)
|
|
1332
|
+
return replace(merge, warnings=warnings + merge.warnings)
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def _dispatch_work_stage(
|
|
1336
|
+
state: IssueState,
|
|
1337
|
+
opts: DriveOptions,
|
|
1338
|
+
counters: DriveCounters,
|
|
1339
|
+
machine: str,
|
|
1340
|
+
oracle: OracleDecision | None = None,
|
|
1341
|
+
gate_checker: AcceptanceGateChecker | None = None,
|
|
1342
|
+
verifier: MergeVerifier | None = None,
|
|
1343
|
+
) -> Action:
|
|
1344
|
+
"""No work row yet: run the optional plan stage, then dispatch the work.
|
|
1345
|
+
|
|
1346
|
+
#1453: when *oracle* is active, the sealed JIT acceptance slice for this
|
|
1347
|
+
issue is authored — and driven through to a landed merge (#2079) — BEFORE
|
|
1348
|
+
either the plan or the direct-assign path below. Otherwise the #1138
|
|
1349
|
+
hard gate (``coord.dispatch.enforce_oracle_readiness``) would simply
|
|
1350
|
+
refuse the eventual ``coord assign``/``coord approve-plan`` once an
|
|
1351
|
+
oracle-opted-in milestone's issue reaches it, with this driver never
|
|
1352
|
+
having explained why.
|
|
1353
|
+
|
|
1354
|
+
*counters* is threaded in for the slice's own merge budget (#2079 —
|
|
1355
|
+
``DriveCounters.acceptance``); every other decision here is stateless.
|
|
1356
|
+
"""
|
|
1357
|
+
if oracle is not None and oracle.active:
|
|
1358
|
+
assert gate_checker is not None and verifier is not None, (
|
|
1359
|
+
"oracle.active implies resolve_oracle_decision ran with a real "
|
|
1360
|
+
"gate_checker; decide()/Driver always thread one through"
|
|
1361
|
+
)
|
|
1362
|
+
gate = _decide_acceptance_author(
|
|
1363
|
+
state, oracle, opts, counters, machine, gate_checker, verifier
|
|
1364
|
+
)
|
|
1365
|
+
if gate is not None:
|
|
1366
|
+
return gate
|
|
1367
|
+
|
|
1368
|
+
# #1499: durable provenance stamped on every assignment this driver
|
|
1369
|
+
# dispatches via `coord assign` — the piece that survives the driver
|
|
1370
|
+
# process exiting (see coord.models.Assignment.driven_by / Proposal.driven_by).
|
|
1371
|
+
driven_by = f"drive:{state.repo}#{state.issue}"
|
|
1372
|
+
|
|
1373
|
+
if opts.do_plan:
|
|
1374
|
+
if not state.plan_aid:
|
|
1375
|
+
args = [
|
|
1376
|
+
"assign", "--plan-only", machine, state.repo, str(state.issue),
|
|
1377
|
+
"--driven-by", driven_by,
|
|
1378
|
+
]
|
|
1379
|
+
if opts.model:
|
|
1380
|
+
args += ["--model", opts.model]
|
|
1381
|
+
return Action(
|
|
1382
|
+
kind=RUN,
|
|
1383
|
+
label=(
|
|
1384
|
+
f"PLAN: coord assign --plan-only {machine} {state.repo} "
|
|
1385
|
+
f"{state.issue}"
|
|
1386
|
+
),
|
|
1387
|
+
command=tuple(args),
|
|
1388
|
+
)
|
|
1389
|
+
if state.plan_status == "done":
|
|
1390
|
+
return Action(
|
|
1391
|
+
kind=RUN,
|
|
1392
|
+
label=f"PLAN: approved → coord approve-plan {state.plan_aid}",
|
|
1393
|
+
command=("approve-plan", state.plan_aid),
|
|
1394
|
+
)
|
|
1395
|
+
if state.plan_status == "failed":
|
|
1396
|
+
return _die(
|
|
1397
|
+
f"plan assignment {state.plan_aid} failed — inspect: "
|
|
1398
|
+
f"coord log {state.plan_aid} --machine {machine}"
|
|
1399
|
+
)
|
|
1400
|
+
return _wait()
|
|
1401
|
+
|
|
1402
|
+
args = ["assign", machine, state.repo, str(state.issue), "--driven-by", driven_by]
|
|
1403
|
+
if opts.model:
|
|
1404
|
+
args += ["--model", opts.model]
|
|
1405
|
+
if opts.briefing_file:
|
|
1406
|
+
args += ["--briefing-file", opts.briefing_file]
|
|
1407
|
+
return Action(
|
|
1408
|
+
kind=RUN,
|
|
1409
|
+
label=f"WORK: coord assign {machine} {state.repo} {state.issue}",
|
|
1410
|
+
command=tuple(args),
|
|
1411
|
+
)
|
|
1412
|
+
|
|
1413
|
+
|
|
1414
|
+
def _decide_advisory(
|
|
1415
|
+
state: IssueState,
|
|
1416
|
+
opts: DriveOptions,
|
|
1417
|
+
machine: str,
|
|
1418
|
+
verifier: MergeVerifier,
|
|
1419
|
+
) -> Action:
|
|
1420
|
+
"""The #448 downgrade: the agent flagged a zero-commit / stash-miss exit.
|
|
1421
|
+
|
|
1422
|
+
#1357 makes this a FALSE POSITIVE for every Python-only headless assignment
|
|
1423
|
+
in claude-coordinator — its only artifact glob is
|
|
1424
|
+
``tui/target/debug/coord-tui``, which a Python diff never produces, so
|
|
1425
|
+
#1323's stash-miss check downgrades a perfectly good DONE. Ask git which
|
|
1426
|
+
case this actually is rather than trusting the status.
|
|
1427
|
+
"""
|
|
1428
|
+
if not state.work_branch or not verifier.branch_has_commits(state):
|
|
1429
|
+
return _die(
|
|
1430
|
+
f"work {state.work_aid} exited ADVISORY with no commits on its branch —\n"
|
|
1431
|
+
" nothing was pushed, so there is nothing to test, review, or merge.\n"
|
|
1432
|
+
f" inspect: coord log {state.work_aid} --machine "
|
|
1433
|
+
f"{state.work_machine or machine}"
|
|
1434
|
+
)
|
|
1435
|
+
if not opts.accept_advisory:
|
|
1436
|
+
return _die(
|
|
1437
|
+
f"work {state.work_aid} is ADVISORY, but its branch carries real "
|
|
1438
|
+
"commits.\n"
|
|
1439
|
+
" This is the #1357 signature: since v0.4.75 every Python-only "
|
|
1440
|
+
"headless\n"
|
|
1441
|
+
" assignment in this repo is downgraded DONE→ADVISORY by an "
|
|
1442
|
+
"artifact glob\n"
|
|
1443
|
+
" that a Python diff can never match.\n"
|
|
1444
|
+
" Proceed anyway with --accept-advisory (and fix #1357 to stop "
|
|
1445
|
+
"needing it)."
|
|
1446
|
+
)
|
|
1447
|
+
return Action(
|
|
1448
|
+
kind=WAIT,
|
|
1449
|
+
warnings=(
|
|
1450
|
+
"ADVISORY with commits present — proceeding per --accept-advisory (#1357)",
|
|
1451
|
+
),
|
|
1452
|
+
)
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def _decide_test(
|
|
1456
|
+
state: IssueState,
|
|
1457
|
+
opts: DriveOptions,
|
|
1458
|
+
counters: DriveCounters,
|
|
1459
|
+
machine: str,
|
|
1460
|
+
) -> Action | None:
|
|
1461
|
+
"""The TEST gate. ``None`` means "passed/skipped, fall through".
|
|
1462
|
+
|
|
1463
|
+
#1426: coord dispatches this stage itself (``dispatch_smoke`` via the
|
|
1464
|
+
``coord serve`` tick loop or ``coord notify``) onto a capability-matched
|
|
1465
|
+
machine; this only OBSERVES ``test_state``, exactly like the review gate
|
|
1466
|
+
below. ``--skip-test`` is the one Test-stage action taken here, and it is
|
|
1467
|
+
a ``coord test --skipped`` CLI call — never a direct
|
|
1468
|
+
``record_test_verdict()`` (#1384).
|
|
1469
|
+
"""
|
|
1470
|
+
test_state = state.work_test_state
|
|
1471
|
+
if test_state in ("passed", "skipped"):
|
|
1472
|
+
return None
|
|
1473
|
+
|
|
1474
|
+
# #1605: the Test-stage CHILD assignment (`type="smoke"`) itself reached
|
|
1475
|
+
# a terminal FAILED/cancelled status — a dead agent, a killed process
|
|
1476
|
+
# group, a terminal API error, anything short of the worker actually
|
|
1477
|
+
# printing `SMOKE: pass`/`SMOKE: fail` — without ever producing a
|
|
1478
|
+
# verdict. Before this, `test_state` could be left at `"running"`
|
|
1479
|
+
# (`dispatch_smoke`'s own marker, #1426) forever: every gate treats
|
|
1480
|
+
# `"running"` as "no verdict yet" (#1395), so nothing downstream ever
|
|
1481
|
+
# resolves it and this function prints "TEST: in progress" every poll,
|
|
1482
|
+
# unbounded — the exact #1598 incident this closes (2.5 hours against
|
|
1483
|
+
# three idle machines). `reconcile_completed_assignments` /
|
|
1484
|
+
# `coord diagnose --stage test` normally resolve this from the daemon's
|
|
1485
|
+
# own tick, but a live `coord drive` loop must not depend on that
|
|
1486
|
+
# timing — detect the contradiction directly from the child's own board
|
|
1487
|
+
# fields and stop with an actionable message rather than poll forever.
|
|
1488
|
+
# Scoped to smoke FAILED/cancelled (not "done"): a fresh `done` smoke
|
|
1489
|
+
# completion has an expected, bounded propagation lag before `coord
|
|
1490
|
+
# notify` records its verdict — that is NOT this bug.
|
|
1491
|
+
if state.smoke_status in ("failed", "cancelled") and test_state == "running":
|
|
1492
|
+
return _die(
|
|
1493
|
+
"test stage is stuck: work.test_state='running' but its "
|
|
1494
|
+
f"Test-stage worker {state.smoke_aid} already finished "
|
|
1495
|
+
f"(status={state.smoke_status!r}, reason="
|
|
1496
|
+
f"{state.smoke_failure_reason or 'none recorded'!r}) — the "
|
|
1497
|
+
"parent verdict was never resolved (#1605).\n"
|
|
1498
|
+
f" Recover: coord diagnose {state.repo} {state.issue} --stage "
|
|
1499
|
+
"test\n"
|
|
1500
|
+
" (add --reset if the diagnosis alone doesn't clear it)"
|
|
1501
|
+
)
|
|
1502
|
+
|
|
1503
|
+
if test_state == "":
|
|
1504
|
+
if opts.skip_test:
|
|
1505
|
+
return Action(
|
|
1506
|
+
kind=RUN,
|
|
1507
|
+
label="TEST: --skip-test → recording 'skipped'",
|
|
1508
|
+
command=(
|
|
1509
|
+
"test",
|
|
1510
|
+
"--skipped",
|
|
1511
|
+
"--reason",
|
|
1512
|
+
"coord drive --skip-test",
|
|
1513
|
+
state.work_aid,
|
|
1514
|
+
),
|
|
1515
|
+
sleep_after=5.0,
|
|
1516
|
+
)
|
|
1517
|
+
# Waiting for coord to dispatch the Test stage itself. The stall
|
|
1518
|
+
# detector nudges `coord notify` (--notify) after --stall minutes of no
|
|
1519
|
+
# state change — no need to force it here on every poll.
|
|
1520
|
+
return _wait()
|
|
1521
|
+
|
|
1522
|
+
if test_state == "running":
|
|
1523
|
+
return _wait(label="TEST: in progress on a capability-matched machine")
|
|
1524
|
+
|
|
1525
|
+
if test_state == "failed":
|
|
1526
|
+
if counters.fix_rounds >= opts.max_fix_rounds:
|
|
1527
|
+
return _die(
|
|
1528
|
+
f"test still failing after {counters.fix_rounds} fix round(s) — "
|
|
1529
|
+
"stopping.\n"
|
|
1530
|
+
f" Reason: {state.work_test_reason or 'none recorded'}\n"
|
|
1531
|
+
f" Inspect: coord log {state.work_aid} --machine "
|
|
1532
|
+
f"{state.work_machine or machine}\n"
|
|
1533
|
+
f" Continue by hand: coord assign --interactive --fix-of "
|
|
1534
|
+
f"{state.work_aid}"
|
|
1535
|
+
)
|
|
1536
|
+
counters.fix_rounds += 1
|
|
1537
|
+
# `coord fix` gates on the assignment's legacy `smoke_test == "fail"`
|
|
1538
|
+
# field — which `coord test --fail` mirrors from `test_state` — and
|
|
1539
|
+
# dispatches a follow-up worker with `inherit_branch=True`, so the fix
|
|
1540
|
+
# continues the SAME branch rather than orphaning it on a fresh one. It
|
|
1541
|
+
# also escalates the model (sonnet → opus) and quotes the stored test
|
|
1542
|
+
# output in the briefing. This is why a test failure is a loop
|
|
1543
|
+
# iteration and not a dead end. (The interactive `--fix-of` and `coord
|
|
1544
|
+
# bounce` paths are NOT usable here: `--fix-of` requires --interactive,
|
|
1545
|
+
# and `bounce` needs a request-changes REVIEW id, not a failed test.)
|
|
1546
|
+
return Action(
|
|
1547
|
+
kind=RUN,
|
|
1548
|
+
label=(
|
|
1549
|
+
f"TEST: failed → fix round {counters.fix_rounds}/"
|
|
1550
|
+
f"{opts.max_fix_rounds} (coord fix {state.work_aid})"
|
|
1551
|
+
),
|
|
1552
|
+
command=("fix", state.work_aid),
|
|
1553
|
+
error_message=(
|
|
1554
|
+
f"coord fix {state.work_aid} failed to dispatch.\n"
|
|
1555
|
+
" Most likely the assignment's legacy smoke_test field is not "
|
|
1556
|
+
"'fail' — that is\n"
|
|
1557
|
+
" what `coord fix` gates on, and only `coord test --fail` sets "
|
|
1558
|
+
"it.\n"
|
|
1559
|
+
f" Check: coord log {state.work_aid} / continue by hand: "
|
|
1560
|
+
f"coord assign --interactive --fix-of {state.work_aid}"
|
|
1561
|
+
),
|
|
1562
|
+
)
|
|
1563
|
+
|
|
1564
|
+
return Action(kind=WAIT, warnings=(f"unexpected test_state '{test_state}'",))
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
def _decide_review(
|
|
1568
|
+
state: IssueState,
|
|
1569
|
+
opts: DriveOptions,
|
|
1570
|
+
counters: DriveCounters,
|
|
1571
|
+
machine: str,
|
|
1572
|
+
) -> Action | None:
|
|
1573
|
+
"""The REVIEW gate. ``None`` means "approved, fall through to merge".
|
|
1574
|
+
|
|
1575
|
+
coord dispatches the review itself once the test verdict lands (the notify
|
|
1576
|
+
timer's ``dispatch_pending_reviews``), so this mostly observes — the
|
|
1577
|
+
exceptions are the #555 interactive case (one explicit request), the #1584
|
|
1578
|
+
dead-reviewer retry, and the #1692 request-changes fix round.
|
|
1579
|
+
|
|
1580
|
+
**#1692 — a request-changes verdict dispatches ``coord fix`` here.** It
|
|
1581
|
+
used to ``_wait()`` on a comment that read "the auto-loop dispatches the
|
|
1582
|
+
fix", which stopped being true when #1616 replaced the ``coord notify``
|
|
1583
|
+
timer with the daemon drain: the drain's responsibility table deliberately
|
|
1584
|
+
excludes fix dispatch (#476/#477), ``run_for_review_transition`` never sees
|
|
1585
|
+
the transition because the drain already consumed it, and the #1478 stalled
|
|
1586
|
+
sweeper is off by default. Three mechanisms, three defensible declines, one
|
|
1587
|
+
hole — a 50-minute park to the deadline with nothing dispatched. This arm
|
|
1588
|
+
now mirrors the test arm one-for-one: same ``coord fix`` command, same
|
|
1589
|
+
``counters.fix_rounds`` budget, one extra de-duplication latch because a
|
|
1590
|
+
review row (unlike a failed test) is not re-created by the fix it triggers.
|
|
1591
|
+
"""
|
|
1592
|
+
verdict = state.review_verdict
|
|
1593
|
+
if verdict == "approve":
|
|
1594
|
+
return None
|
|
1595
|
+
|
|
1596
|
+
# #1584: the review WORKER itself died (transient API error, network
|
|
1597
|
+
# drop, ...) before ever producing a verdict. Before #1584 that worker
|
|
1598
|
+
# was mislabelled `done` with `review_verdict == ""`, which fell through
|
|
1599
|
+
# to the `verdict == ""` branch below and either waited for a dispatch
|
|
1600
|
+
# that would never come or (if `state.work_review_state == "done"`)
|
|
1601
|
+
# died with a "REVIEW_VERDICT block failed to parse" message that no
|
|
1602
|
+
# longer applies now that the review is correctly `failed`. Checked
|
|
1603
|
+
# BEFORE `verdict == ""` so it can never fall through to that stale
|
|
1604
|
+
# message or to a silent `_wait()` — the exact regression this issue's
|
|
1605
|
+
# own evidence (#1563) was filed over.
|
|
1606
|
+
#
|
|
1607
|
+
# Re-dispatch via ``coord review <work_aid>`` — NOT ``coord retry
|
|
1608
|
+
# <review_aid>``. ``coord retry``'s underlying `_reassign` (coord/
|
|
1609
|
+
# reconcile.py) hardcodes `type="work"` on every re-dispatch regardless
|
|
1610
|
+
# of the failed assignment's own type (it exists solely to retry WORK
|
|
1611
|
+
# rows); pointing it at a review assignment id would silently create a
|
|
1612
|
+
# bogus fresh `type="work"` assignment on this issue instead of a
|
|
1613
|
+
# review. ``coord review`` is the existing #555 escape-hatch command
|
|
1614
|
+
# (already used a few lines below for the interactive case) — a thin,
|
|
1615
|
+
# type-correct wrapper over `coord.review.dispatch_review` keyed on the
|
|
1616
|
+
# WORK row, which is still `status="done"` (only the review it spawned
|
|
1617
|
+
# failed). Bounded the same way as the WORK failed-retry loop in
|
|
1618
|
+
# `decide()` (usage-limit-aware wait, then a bounded re-dispatch).
|
|
1619
|
+
if state.review_status == "failed":
|
|
1620
|
+
if is_usage_limit_reason(state.review_failure_reason):
|
|
1621
|
+
return Action(
|
|
1622
|
+
kind=WAIT,
|
|
1623
|
+
label=(
|
|
1624
|
+
f"REVIEW: {state.review_aid} killed by the usage limit — "
|
|
1625
|
+
"waiting for the reset, not retrying"
|
|
1626
|
+
),
|
|
1627
|
+
warnings=(
|
|
1628
|
+
f"usage-limit kill detected on {state.review_aid}: "
|
|
1629
|
+
f"{state.review_failure_reason} — waiting for the reset "
|
|
1630
|
+
"instead of retrying (#1461/#1584)",
|
|
1631
|
+
),
|
|
1632
|
+
)
|
|
1633
|
+
if counters.review_retries >= opts.max_work_retries:
|
|
1634
|
+
return _die(
|
|
1635
|
+
f"review {state.review_aid} failed "
|
|
1636
|
+
f"{counters.review_retries} retr(ies) in: "
|
|
1637
|
+
f"{state.review_failure_reason or 'no reason recorded'}\n"
|
|
1638
|
+
f" inspect: coord log {state.review_aid}"
|
|
1639
|
+
)
|
|
1640
|
+
counters.review_retries += 1
|
|
1641
|
+
return Action(
|
|
1642
|
+
kind=RUN,
|
|
1643
|
+
label=(
|
|
1644
|
+
f"REVIEW: failed → coord review {state.work_aid} "
|
|
1645
|
+
f"(attempt {counters.review_retries}/{opts.max_work_retries})"
|
|
1646
|
+
),
|
|
1647
|
+
command=("review", state.work_aid),
|
|
1648
|
+
error_message=f"coord review failed for {state.work_aid}",
|
|
1649
|
+
)
|
|
1650
|
+
|
|
1651
|
+
if verdict == "request-changes":
|
|
1652
|
+
# The OUTER cap, and it stays FIRST: an exhausted review loop is
|
|
1653
|
+
# terminal no matter how much of this drive's own fix budget is left,
|
|
1654
|
+
# and `_dispatch_fix_for_review` would refuse the dispatch anyway
|
|
1655
|
+
# (`next_iteration > max_review_iterations` → `max_iterations`, which
|
|
1656
|
+
# `coord fix` turns into a non-zero exit). Dying here reports the cap
|
|
1657
|
+
# instead of reporting a subprocess failure.
|
|
1658
|
+
if state.work_review_iter >= state.max_review_iterations:
|
|
1659
|
+
return _die(
|
|
1660
|
+
"review requested changes and the fix loop is exhausted\n"
|
|
1661
|
+
f" ({state.work_review_iter} rounds, cap "
|
|
1662
|
+
f"{state.max_review_iterations}).\n"
|
|
1663
|
+
f" Findings: coord log {state.review_aid}\n"
|
|
1664
|
+
f" Continue by hand: coord assign --interactive --fix-of "
|
|
1665
|
+
f"{state.review_aid}"
|
|
1666
|
+
)
|
|
1667
|
+
# #1692: `coord fix` routes through `auto_loop.process_review_completion`,
|
|
1668
|
+
# whose very first line refuses when `pipeline.auto_loop` is off. Say so
|
|
1669
|
+
# here rather than dispatching a subprocess that can only fail — and
|
|
1670
|
+
# rather than the pre-#1692 infinite wait, which is what the preflight
|
|
1671
|
+
# warning ("this run will report the verdict and stop") already promised
|
|
1672
|
+
# not to do.
|
|
1673
|
+
if not state.auto_loop:
|
|
1674
|
+
return _die(
|
|
1675
|
+
"review requested changes but pipeline.auto_loop is OFF — the "
|
|
1676
|
+
"review→fix\n"
|
|
1677
|
+
" path is switched off in coordinator.yml, so no fix can be "
|
|
1678
|
+
"dispatched.\n"
|
|
1679
|
+
f" Findings: coord log {state.review_aid}\n"
|
|
1680
|
+
f" Continue by hand: coord assign --interactive --fix-of "
|
|
1681
|
+
f"{state.review_aid}"
|
|
1682
|
+
)
|
|
1683
|
+
# Belt-and-braces: `review_verdict` and `review_aid` are read off the
|
|
1684
|
+
# SAME board row (`drive_state.project`), so a verdict without an id is
|
|
1685
|
+
# impossible today. Assert it rather than assume it — everything below
|
|
1686
|
+
# spends money keyed on that id, and `coord fix ""` is not a refusal
|
|
1687
|
+
# this arm should ever have to interpret.
|
|
1688
|
+
if not state.review_aid:
|
|
1689
|
+
return _die(
|
|
1690
|
+
"review verdict is 'request-changes' but no review assignment "
|
|
1691
|
+
"id is on the board —\n"
|
|
1692
|
+
" refusing to guess which review to fix. Inspect: coord "
|
|
1693
|
+
f"gates {state.repo} {state.issue}"
|
|
1694
|
+
)
|
|
1695
|
+
# Already spent a round on THIS review row and the board hasn't caught
|
|
1696
|
+
# up yet. Waiting is the only safe answer: dispatching again would put
|
|
1697
|
+
# a second fix worker on the same branch (#476/#477). Once the fix row
|
|
1698
|
+
# lands, `decide()`'s `active_count > 0` guard takes over, and when it
|
|
1699
|
+
# completes the review row changes id and this latch stops matching.
|
|
1700
|
+
if counters.review_fix_dispatched_for == state.review_aid:
|
|
1701
|
+
return _wait(
|
|
1702
|
+
label=(
|
|
1703
|
+
f"REVIEW: fix already dispatched for {state.review_aid} — "
|
|
1704
|
+
"waiting for the fix row to appear on the board"
|
|
1705
|
+
)
|
|
1706
|
+
)
|
|
1707
|
+
# The driver-side twin of the test arm's bound, sharing ONE budget with
|
|
1708
|
+
# it (see `DriveCounters.fix_rounds`): `max_review_iterations` bounds
|
|
1709
|
+
# the *issue's* review loop across every drive that ever touches it,
|
|
1710
|
+
# `max_fix_rounds` bounds what THIS drive is willing to spend.
|
|
1711
|
+
if counters.fix_rounds >= opts.max_fix_rounds:
|
|
1712
|
+
return _die(
|
|
1713
|
+
f"review requested changes after {counters.fix_rounds} fix "
|
|
1714
|
+
"round(s) this drive — stopping.\n"
|
|
1715
|
+
f" (review iteration {state.work_review_iter}/"
|
|
1716
|
+
f"{state.max_review_iterations} is NOT exhausted; this drive's "
|
|
1717
|
+
"own --max-fix-rounds is.)\n"
|
|
1718
|
+
f" Findings: coord log {state.review_aid}\n"
|
|
1719
|
+
f" Continue by hand: coord assign --interactive --fix-of "
|
|
1720
|
+
f"{state.review_aid}"
|
|
1721
|
+
)
|
|
1722
|
+
counters.fix_rounds += 1
|
|
1723
|
+
counters.review_fix_dispatched_for = state.review_aid
|
|
1724
|
+
# #1622 widened `coord fix` to take a REVIEW id whose verdict was
|
|
1725
|
+
# request-changes; #1692 is the review arm finally walking through that
|
|
1726
|
+
# door. The REVIEW id, not `state.work_aid`: the work-id form gates on
|
|
1727
|
+
# the legacy `smoke_test == "fail"` field and would be refused here.
|
|
1728
|
+
# It is not a second implementation of fix dispatch — it hands the row
|
|
1729
|
+
# to `auto_loop.process_review_completion` → the single `_dispatch_fix`
|
|
1730
|
+
# chokepoint, so `pipeline.auto_loop`, the #476/#1456
|
|
1731
|
+
# approve-with-nits gate, the #522 terminal-work guard and
|
|
1732
|
+
# `max_review_iterations` all still apply. No `sleep_after`: the test
|
|
1733
|
+
# arm above dispatches the same command on the same plain poll
|
|
1734
|
+
# interval, and the latch — not a timing guess — is what makes the
|
|
1735
|
+
# next poll safe.
|
|
1736
|
+
return Action(
|
|
1737
|
+
kind=RUN,
|
|
1738
|
+
label=(
|
|
1739
|
+
f"REVIEW: request-changes → fix round {counters.fix_rounds}/"
|
|
1740
|
+
f"{opts.max_fix_rounds} (coord fix {state.review_aid}, review "
|
|
1741
|
+
f"iteration {state.work_review_iter + 1}/"
|
|
1742
|
+
f"{state.max_review_iterations})"
|
|
1743
|
+
),
|
|
1744
|
+
command=("fix", state.review_aid),
|
|
1745
|
+
error_message=(
|
|
1746
|
+
f"coord fix {state.review_aid} failed to dispatch the review "
|
|
1747
|
+
"fix.\n"
|
|
1748
|
+
" Its refusals are all guards doing their job: auto_loop "
|
|
1749
|
+
"disabled, no structured\n"
|
|
1750
|
+
" findings, approve-with-nits (#476), max_review_iterations, "
|
|
1751
|
+
"or the #522\n"
|
|
1752
|
+
" terminal-work guard — the message above names which.\n"
|
|
1753
|
+
f" Check: coord log {state.review_aid} / continue by hand: "
|
|
1754
|
+
f"coord assign --interactive --fix-of {state.review_aid}"
|
|
1755
|
+
),
|
|
1756
|
+
)
|
|
1757
|
+
|
|
1758
|
+
if verdict == "":
|
|
1759
|
+
if state.work_review_state == "done":
|
|
1760
|
+
return _die(
|
|
1761
|
+
f"review {state.review_aid} finished but recorded NO verdict — the\n"
|
|
1762
|
+
" REVIEW_VERDICT block failed to parse (#1346/#1348 class).\n"
|
|
1763
|
+
" Recover: coord post-pending-reviews, or read the transcript "
|
|
1764
|
+
"directly."
|
|
1765
|
+
)
|
|
1766
|
+
# No review row at all. For interactive work that is terminal, not
|
|
1767
|
+
# transient (#555) — request one explicitly, once, rather than waiting
|
|
1768
|
+
# on a dispatch that will never happen. Preflight already refused this
|
|
1769
|
+
# case unless --force-review was given.
|
|
1770
|
+
if not state.review_aid and state.work_provider == "claude-pty":
|
|
1771
|
+
if not opts.force_review:
|
|
1772
|
+
return _die(
|
|
1773
|
+
f"no review for interactive work {state.work_aid} and "
|
|
1774
|
+
"--force-review not set (#555)."
|
|
1775
|
+
)
|
|
1776
|
+
if counters.review_dispatches >= 1:
|
|
1777
|
+
return _die(
|
|
1778
|
+
f"requested a review for {state.work_aid} but none appeared "
|
|
1779
|
+
"on the board.\n"
|
|
1780
|
+
" Check for an eligible reviewer machine: coord status"
|
|
1781
|
+
)
|
|
1782
|
+
counters.review_dispatches += 1
|
|
1783
|
+
return Action(
|
|
1784
|
+
kind=RUN,
|
|
1785
|
+
label="REVIEW: requesting explicitly (interactive work, #555)",
|
|
1786
|
+
command=("review", state.work_aid),
|
|
1787
|
+
error_message=(
|
|
1788
|
+
f"explicit review dispatch failed for {state.work_aid}"
|
|
1789
|
+
),
|
|
1790
|
+
)
|
|
1791
|
+
return _wait()
|
|
1792
|
+
|
|
1793
|
+
return Action(kind=WAIT, warnings=(f"unexpected review verdict '{verdict}'",))
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
# #1505: merge statuses a bounded `coord merge --only` retry can actually
|
|
1797
|
+
# change. "" / PENDING / READY / MERGING are normal in-flight states — the
|
|
1798
|
+
# next `coord merge` tick is expected to move them forward. CONFLICT is
|
|
1799
|
+
# retried too, deliberately: that's what runs `classify_conflict` +
|
|
1800
|
+
# `dispatch_conflict_fix` (#1474, see this function's docstring). Everything
|
|
1801
|
+
# else — most commonly NEEDS_ATTENTION, or any status this driver has never
|
|
1802
|
+
# seen before — cannot be resolved by retrying, so it escalates instead of
|
|
1803
|
+
# spinning the attempt cap down to zero on a no-op.
|
|
1804
|
+
#
|
|
1805
|
+
# #1505 review fix: `merge_queue.plan()`'s `_state_to_plan_status` collapses
|
|
1806
|
+
# CONFLICT, HUMAN_REQUIRED, and SKIPPED into a single "NEEDS_ATTENTION"
|
|
1807
|
+
# status for operator display, and `merge_plan` (not the raw `merge_queue`
|
|
1808
|
+
# table) is what a normal daemon-backed `/board` build actually populates —
|
|
1809
|
+
# so a literal `status == "CONFLICT"` almost never reaches this function
|
|
1810
|
+
# without help. `drive_state._merge_entry` is where that help lives: it
|
|
1811
|
+
# cross-checks the raw `merge_queue` row and reports its un-collapsed state
|
|
1812
|
+
# whenever the plan says NEEDS_ATTENTION, so a fresh, still-auto-fixable
|
|
1813
|
+
# conflict lands here as "CONFLICT" (retried) rather than "NEEDS_ATTENTION"
|
|
1814
|
+
# (escalated on sight). See `_merge_entry`'s docstring for the full story.
|
|
1815
|
+
_RETRYABLE_MERGE_STATUSES = frozenset({"", "PENDING", "READY", "MERGING", "CONFLICT"})
|
|
1816
|
+
|
|
1817
|
+
# #2078: matches the "enqueue blocked by <gate> gate — <reason> (waive with
|
|
1818
|
+
# <flag>)" line `coord.commands.merge._explain_missing_only_entry` prints for
|
|
1819
|
+
# each board row a failed `coord merge --only <aid>` matched but could not
|
|
1820
|
+
# enqueue. `re.MULTILINE` + `$` (not the whole string) so it still finds the
|
|
1821
|
+
# line even when `_explain_missing_only_entry` reports several matching rows
|
|
1822
|
+
# (rare — normally there is exactly one board row per (repo, issue)).
|
|
1823
|
+
_ENQUEUE_BLOCKED_RE = re.compile(r"enqueue blocked by (.+)$", re.MULTILINE)
|
|
1824
|
+
|
|
1825
|
+
_PR_NUMBER_RE = re.compile(r"/pull/(\d+)")
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
def _extract_gate_block_reason(diagnostic: str) -> str | None:
|
|
1829
|
+
"""Pull the named gate failure out of a captured `coord merge --only`
|
|
1830
|
+
diagnostic (#2078).
|
|
1831
|
+
|
|
1832
|
+
`coord merge --only <aid>`, when it finds no queue entry for *aid*,
|
|
1833
|
+
prints `_explain_missing_only_entry`'s diagnosis (commands/merge.py) —
|
|
1834
|
+
one line per matching board row, each either naming the blocking
|
|
1835
|
+
review/smoke gate (what this extracts) or reporting that every gate
|
|
1836
|
+
already passes / that no board row matched at all (neither of which this
|
|
1837
|
+
matches). `_decide_merge`'s empty-status arm uses a non-``None`` result
|
|
1838
|
+
to tell "a real, persistent gate failure" — worth waiting on, like a
|
|
1839
|
+
BLOCKED board entry — apart from "not enqueued yet" or "identifier
|
|
1840
|
+
didn't resolve", which are still worth retrying. Returns ``None`` for an
|
|
1841
|
+
empty *diagnostic* (no attempt has run yet this drive) or one with no
|
|
1842
|
+
matching line.
|
|
1843
|
+
"""
|
|
1844
|
+
match = _ENQUEUE_BLOCKED_RE.search(diagnostic)
|
|
1845
|
+
return match.group(1).strip() if match else None
|
|
1846
|
+
|
|
1847
|
+
|
|
1848
|
+
def _extract_pr_number(pr_url: str) -> int | None:
|
|
1849
|
+
"""Best-effort PR number out of a GitHub PR URL, or ``None``."""
|
|
1850
|
+
if not pr_url:
|
|
1851
|
+
return None
|
|
1852
|
+
m = _PR_NUMBER_RE.search(pr_url)
|
|
1853
|
+
return int(m.group(1)) if m else None
|
|
1854
|
+
|
|
1855
|
+
|
|
1856
|
+
# ── #1526: driver/gate divergence ───────────────────────────────────────────
|
|
1857
|
+
#
|
|
1858
|
+
# `coord drive` decides `test=`/`review=` are satisfied from
|
|
1859
|
+
# `work_test_state`/`review_verdict` — the same fields `_decide_test`/
|
|
1860
|
+
# `_decide_review` above already let through (that is WHY `decide()` ever
|
|
1861
|
+
# reaches `_decide_merge` at all). `coord merge` enforces a DIFFERENT, fresher
|
|
1862
|
+
# check (`merge_queue.has_smoke_verdict`/`has_approved_review` — SHA/patch-id
|
|
1863
|
+
# -anchored freshness for smoke, patch-id voiding for review) and can refuse
|
|
1864
|
+
# for a reason this driver's own view never sees coming. When it does, the
|
|
1865
|
+
# refusal text is left on the board as `merge_reason` — persisted on the raw
|
|
1866
|
+
# queue row's `.error` by `merge_queue.process()` — even while `merge_status`
|
|
1867
|
+
# itself often still reads a RETRYABLE value like READY, because the
|
|
1868
|
+
# board-render gate check in `merge_queue.plan()`'s `_entry_gate_status`
|
|
1869
|
+
# doesn't have the live SHA data a real merge attempt fetches (see that
|
|
1870
|
+
# function's docstring). Retrying `coord merge` unchanged cannot resolve two
|
|
1871
|
+
# READS of the board disagreeing with each other; only a human (or a fresh
|
|
1872
|
+
# verdict) can — see `_merge_gate_divergence`.
|
|
1873
|
+
# #1640 added two more wordings for the SAME smoke gate: "smoke test verdict
|
|
1874
|
+
# is stale: …" (merge_queue.process) and "test verdict stale (…)"
|
|
1875
|
+
# (merge_queue.plan / staging). Both name a recorded-but-stale verdict, which
|
|
1876
|
+
# is still the smoke gate — and still the case _merge_gate_divergence exists
|
|
1877
|
+
# to catch, since `work_test_state` reads "passed" while the merge refuses.
|
|
1878
|
+
_SMOKE_GATE_MARKERS = (
|
|
1879
|
+
"smoke test required",
|
|
1880
|
+
"test verdict missing",
|
|
1881
|
+
"smoke test verdict is stale",
|
|
1882
|
+
"test verdict stale",
|
|
1883
|
+
)
|
|
1884
|
+
_REVIEW_GATE_MARKERS = ("review required", "review not approved")
|
|
1885
|
+
|
|
1886
|
+
|
|
1887
|
+
def _merge_gate_kind(reason: str) -> str | None:
|
|
1888
|
+
"""Classify a merge-queue block *reason* as the gate it names, or
|
|
1889
|
+
``None`` when it isn't one of the two this module knows a corrective
|
|
1890
|
+
action for.
|
|
1891
|
+
|
|
1892
|
+
Matches both `merge_queue.process()`'s live-attempt wording ("smoke test
|
|
1893
|
+
required but no verdict recorded" / "review required but not approved")
|
|
1894
|
+
and `merge_queue.plan()`'s board-render wording ("test verdict missing" /
|
|
1895
|
+
"review not approved") — the two functions describe the identical gates
|
|
1896
|
+
in different words.
|
|
1897
|
+
"""
|
|
1898
|
+
r = (reason or "").lower()
|
|
1899
|
+
if any(marker in r for marker in _SMOKE_GATE_MARKERS):
|
|
1900
|
+
return "smoke"
|
|
1901
|
+
if any(marker in r for marker in _REVIEW_GATE_MARKERS):
|
|
1902
|
+
return "review"
|
|
1903
|
+
return None
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
# #1738: the two wordings that name a verdict recorded-but-STALE specifically
|
|
1907
|
+
# (`merge_queue.process`'s live-attempt text and `merge_queue.plan`'s
|
|
1908
|
+
# board-render text — see the module comment above `_SMOKE_GATE_MARKERS`) —
|
|
1909
|
+
# a strict subset of `_SMOKE_GATE_MARKERS`, which also matches "no verdict at
|
|
1910
|
+
# all" ("smoke test required"/"test verdict missing"). Only the stale case
|
|
1911
|
+
# has a safe, bounded, fully-automatable fix: re-run the Test stage against
|
|
1912
|
+
# the CURRENT base and let a fresh verdict land. A missing-verdict divergence
|
|
1913
|
+
# is the #1640 lost-write shape instead — driver and gate disagree about
|
|
1914
|
+
# whether a verdict exists at all, which a re-test can't safely paper over —
|
|
1915
|
+
# so that one still escalates to a human on first encounter, unchanged.
|
|
1916
|
+
#
|
|
1917
|
+
# #1769: this used to be defined HERE, and #1769 added a second consumer in
|
|
1918
|
+
# the merge lane (`coord merge --revalidate`). Rather than let a second copy
|
|
1919
|
+
# of the same string matching drift silently apart from the code that emits
|
|
1920
|
+
# the strings, the definition was lifted to `coord.merge_queue` — which is
|
|
1921
|
+
# where both wordings are actually produced (`SmokeVerdictStatus.message` /
|
|
1922
|
+
# `.short_reason`) and which both lanes already depend on. These are aliases,
|
|
1923
|
+
# not copies: `tests/test_merge_queue.py` asserts identity.
|
|
1924
|
+
_STALE_SMOKE_MARKERS = _mq_stale_smoke_markers
|
|
1925
|
+
_is_stale_smoke_reason = _mq_is_stale_smoke_reason
|
|
1926
|
+
|
|
1927
|
+
|
|
1928
|
+
def _merge_gate_divergence(state: IssueState) -> str | None:
|
|
1929
|
+
"""``"smoke"``/``"review"`` when *state* shows the #1526 divergence,
|
|
1930
|
+
else ``None``.
|
|
1931
|
+
|
|
1932
|
+
The divergence: `state.merge_reason` names a smoke/review block while
|
|
1933
|
+
this SAME state's `work_test_state`/`review_verdict` says the opposite.
|
|
1934
|
+
That contradiction can only come from `coord merge` checking something
|
|
1935
|
+
this driver's view does not (freshness against the CURRENT branch/base,
|
|
1936
|
+
not just the terminal verdict) — never from a retry, since neither
|
|
1937
|
+
input changes by running `coord merge` again unchanged.
|
|
1938
|
+
"""
|
|
1939
|
+
kind = _merge_gate_kind(state.merge_reason)
|
|
1940
|
+
if kind == "smoke" and state.work_test_state in ("passed", "skipped"):
|
|
1941
|
+
return "smoke"
|
|
1942
|
+
if kind == "review" and state.review_verdict == "approve":
|
|
1943
|
+
return "review"
|
|
1944
|
+
return None
|
|
1945
|
+
|
|
1946
|
+
|
|
1947
|
+
def _escalate_merge(
|
|
1948
|
+
state: IssueState, status: str, *, gate_kind: str | None = None
|
|
1949
|
+
) -> Action:
|
|
1950
|
+
"""Build the EXIT action for a merge status retrying cannot fix (#1505).
|
|
1951
|
+
|
|
1952
|
+
Escalates on the FIRST encounter rather than after exhausting
|
|
1953
|
+
``max_merge_attempts`` — a merge attempt is expensive (a whole
|
|
1954
|
+
``coord merge`` run) and, for these statuses, guaranteed to be a no-op;
|
|
1955
|
+
an escalation record is cheap and actionable instead.
|
|
1956
|
+
|
|
1957
|
+
This function stays pure like every other decision in this module (see
|
|
1958
|
+
the "STRUCTURE" section of the module docstring) — it only *describes*
|
|
1959
|
+
the escalation via the returned :class:`Action`'s ``command``. The
|
|
1960
|
+
actual write happens in :meth:`Driver.run`'s exit handling, which runs
|
|
1961
|
+
that command through the CLI exactly like any other board mutation this
|
|
1962
|
+
driver performs (``coord escalate record ...``, never a direct
|
|
1963
|
+
``coord.state`` call).
|
|
1964
|
+
|
|
1965
|
+
*gate_kind* (#1526) is set by :func:`_decide_merge` when
|
|
1966
|
+
:func:`_merge_gate_divergence` fired — a smoke/review gate refusal this
|
|
1967
|
+
driver's own ``work_test_state``/``review_verdict`` view contradicts. The
|
|
1968
|
+
proposed command in that case names the specific, safe corrective action
|
|
1969
|
+
(re-confirm the test verdict, or a scoped/full re-review) rather than the
|
|
1970
|
+
generic "inspect the plan" fallback below.
|
|
1971
|
+
|
|
1972
|
+
Otherwise, the proposed command mirrors the #1477 resolution this issue
|
|
1973
|
+
was opened over: when a PR is known, ``gh pr merge --rebase`` + ``coord
|
|
1974
|
+
reconcile-merges`` is the sanctioned escape hatch (also documented in
|
|
1975
|
+
docs/OPERATING_GOTCHAS.md). Without a known PR number there is nothing
|
|
1976
|
+
concrete to propose beyond pointing at the plan for a human to read.
|
|
1977
|
+
"""
|
|
1978
|
+
pr_number = _extract_pr_number(state.merge_pr_url)
|
|
1979
|
+
if gate_kind == "smoke":
|
|
1980
|
+
# #1738: lead with re-dispatching the Test stage, not with the
|
|
1981
|
+
# hand-recorded `coord test --passed` — that command records a
|
|
1982
|
+
# verdict for a run that never happened if pasted without actually
|
|
1983
|
+
# re-running the suite, and it was the path of least resistance at
|
|
1984
|
+
# 2am on an issue everyone already believed was green. This
|
|
1985
|
+
# escalation only fires once the automated re-test arm in
|
|
1986
|
+
# `_decide_merge` has already spent its `fix_rounds` budget (or hit
|
|
1987
|
+
# the "missing verdict" divergence that arm deliberately doesn't
|
|
1988
|
+
# touch), so the safe, verified remedy is offered FIRST; the
|
|
1989
|
+
# hand-recorded form is still here as the explicit fallback for a
|
|
1990
|
+
# human who has actually re-run the suite themselves.
|
|
1991
|
+
proposed = (
|
|
1992
|
+
f"coord diagnose {state.repo} {state.issue} --stage test "
|
|
1993
|
+
"--reset # re-run the Test stage against the CURRENT base "
|
|
1994
|
+
"(preferred) — or, ONLY if you have personally just re-run the "
|
|
1995
|
+
f"suite against the current base yourself: coord test "
|
|
1996
|
+
f"{state.work_aid} --passed"
|
|
1997
|
+
)
|
|
1998
|
+
elif gate_kind == "review":
|
|
1999
|
+
proposed = (
|
|
2000
|
+
f"coord review-reaffirm {state.work_aid} --reason '<why this "
|
|
2001
|
+
f"delta is safe>' # or a full re-review: coord review "
|
|
2002
|
+
f"{state.work_aid}"
|
|
2003
|
+
)
|
|
2004
|
+
elif pr_number is not None:
|
|
2005
|
+
proposed = f"gh pr merge {pr_number} --rebase && coord reconcile-merges"
|
|
2006
|
+
else:
|
|
2007
|
+
proposed = (
|
|
2008
|
+
f"coord merge --plan --repo {state.repo} "
|
|
2009
|
+
"# inspect the gates, then decide"
|
|
2010
|
+
)
|
|
2011
|
+
|
|
2012
|
+
if gate_kind is not None:
|
|
2013
|
+
driver_view = (
|
|
2014
|
+
f"test_state={state.work_test_state!r}"
|
|
2015
|
+
if gate_kind == "smoke"
|
|
2016
|
+
else f"review_verdict={state.review_verdict!r}"
|
|
2017
|
+
)
|
|
2018
|
+
reason = (
|
|
2019
|
+
f"{gate_kind}_required — coord merge's own gate reports "
|
|
2020
|
+
f"{state.merge_reason!r}, but this driver's OWN view already "
|
|
2021
|
+
f"shows {driver_view} — the two cannot converge by retrying the "
|
|
2022
|
+
"identical `coord merge` command (#1526); a human must "
|
|
2023
|
+
"reconcile them"
|
|
2024
|
+
)
|
|
2025
|
+
else:
|
|
2026
|
+
reason = (
|
|
2027
|
+
f"merge_status={status or '(empty)'} — no number of retries changes "
|
|
2028
|
+
"this; escalating on first encounter instead of burning the "
|
|
2029
|
+
"merge-attempt budget (#1505)"
|
|
2030
|
+
)
|
|
2031
|
+
gate_pairs = (
|
|
2032
|
+
("merge_status", status or "(empty)"),
|
|
2033
|
+
("merge_reason", state.merge_reason or "(none)"),
|
|
2034
|
+
("review_verdict", state.review_verdict or "(none)"),
|
|
2035
|
+
("test_state", state.work_test_state or "(none)"),
|
|
2036
|
+
("pr_url", state.merge_pr_url or "(none)"),
|
|
2037
|
+
)
|
|
2038
|
+
gates_summary = " | ".join(f"{k}={v}" for k, v in gate_pairs)
|
|
2039
|
+
aid = state.merge_aid or state.work_aid
|
|
2040
|
+
|
|
2041
|
+
command: list[str] = [
|
|
2042
|
+
"escalate", "record", state.repo, str(state.issue),
|
|
2043
|
+
"--stage", "merge",
|
|
2044
|
+
"--reason", reason,
|
|
2045
|
+
]
|
|
2046
|
+
for k, v in gate_pairs:
|
|
2047
|
+
command += ["--gate", f"{k}={v}"]
|
|
2048
|
+
command += ["--command", proposed]
|
|
2049
|
+
if aid:
|
|
2050
|
+
command += ["--assignment", aid]
|
|
2051
|
+
|
|
2052
|
+
return Action(
|
|
2053
|
+
kind=EXIT,
|
|
2054
|
+
exit_code=EXIT_ESCALATED,
|
|
2055
|
+
message=(
|
|
2056
|
+
f"merge escalated: {reason}\n"
|
|
2057
|
+
f" gates: {gates_summary}\n"
|
|
2058
|
+
f" proposed: {proposed}\n"
|
|
2059
|
+
f" Recorded on the board — see: coord escalate list --repo {state.repo}"
|
|
2060
|
+
),
|
|
2061
|
+
command=tuple(command),
|
|
2062
|
+
error_message=(
|
|
2063
|
+
"failed to record the escalation on the board (exiting anyway — "
|
|
2064
|
+
f"resolve by hand: coord escalate record {state.repo} {state.issue} "
|
|
2065
|
+
"--reason ... --command ...)"
|
|
2066
|
+
),
|
|
2067
|
+
)
|
|
2068
|
+
|
|
2069
|
+
|
|
2070
|
+
def _decide_merge(
|
|
2071
|
+
state: IssueState, opts: DriveOptions, counters: DriveCounters
|
|
2072
|
+
) -> Action:
|
|
2073
|
+
"""The MERGE stage.
|
|
2074
|
+
|
|
2075
|
+
#1474: a CONFLICT status must NOT be a bare wait. ``dispatch_conflict_fix``
|
|
2076
|
+
(coord.conflict_fix) has exactly two sanctioned callers — inside an actual
|
|
2077
|
+
``coord merge`` run, and the semantic-escalation variant behind
|
|
2078
|
+
``pipeline.escalate_semantic_conflicts`` that only ``coord resume``
|
|
2079
|
+
(human-invoked) reaches — so nothing ever dispatches the fix worker while
|
|
2080
|
+
this function just parks on ``_wait()``. That was the exact deadlock that
|
|
2081
|
+
stalled #1453/#1461 for ~14 hours despite ``classify_conflict()`` correctly
|
|
2082
|
+
saying ``rebaseable`` and a capable machine being idle: the coordinator's
|
|
2083
|
+
own #241 auto-rebase machinery was never invoked.
|
|
2084
|
+
|
|
2085
|
+
The fix is to fall through to the same bounded ``coord merge --only <aid>``
|
|
2086
|
+
retry below every other non-terminal status already uses — that call is
|
|
2087
|
+
what runs ``classify_conflict`` + ``dispatch_conflict_fix`` (or discovers
|
|
2088
|
+
one is already in flight / already failed and escalates to
|
|
2089
|
+
``HUMAN_REQUIRED``, which stays terminal via the check above). Once a
|
|
2090
|
+
conflict-fix is actually dispatched, it shows up as a `type="conflict-fix"`
|
|
2091
|
+
row for this same issue, so the very first check in :func:`decide`
|
|
2092
|
+
(``state.active_count > 0`` → wait) parks the run there on the next poll —
|
|
2093
|
+
:func:`decide` never even reaches this function while it is running. That
|
|
2094
|
+
is what keeps this from double-dispatching or fighting an in-flight fix;
|
|
2095
|
+
:func:`coord.conflict_fix.has_prior_conflict_fix` /
|
|
2096
|
+
:func:`~coord.conflict_fix._has_active_conflict_fix` are the belt-and-
|
|
2097
|
+
braces guard inside ``dispatch_conflict_fix`` itself.
|
|
2098
|
+
|
|
2099
|
+
#1526: the driver/gate divergence (:func:`_merge_gate_divergence`) is
|
|
2100
|
+
checked FIRST, before the status switch below — it can hide behind
|
|
2101
|
+
EITHER a nominally-blocking status (BLOCKED, if ``merge_queue.plan()``'s
|
|
2102
|
+
own render-time gate check caught the same disagreement) or a
|
|
2103
|
+
nominally-retryable one (READY/PENDING/"", if only a live ``coord
|
|
2104
|
+
merge`` attempt caught it and left its reason on the board — see that
|
|
2105
|
+
function's docstring for why the two checks can disagree). Escalating
|
|
2106
|
+
here, before either branch runs, is what stops the driver from spending
|
|
2107
|
+
its whole ``--max-merge-attempts`` budget retrying a merge that cannot
|
|
2108
|
+
succeed until a human — or a fresh verdict — reconciles the two
|
|
2109
|
+
readings; retrying the identical ``coord merge`` command changes neither
|
|
2110
|
+
side of the disagreement.
|
|
2111
|
+
"""
|
|
2112
|
+
divergence = _merge_gate_divergence(state)
|
|
2113
|
+
if divergence == "smoke" and _is_stale_smoke_reason(state.merge_reason):
|
|
2114
|
+
# #1738: a STALE (not missing) smoke verdict has a safe, bounded
|
|
2115
|
+
# self-service fix this driver can take without a human — re-run the
|
|
2116
|
+
# Test stage against the current base via the same non-destructive
|
|
2117
|
+
# reset `coord diagnose --stage test --reset` already performs
|
|
2118
|
+
# (clears `test_state` so `dispatch_pending_smoke` picks the work
|
|
2119
|
+
# back up on its own next tick; the branch/commits are untouched).
|
|
2120
|
+
# Bounded by the SAME `fix_rounds` budget the test-failed and
|
|
2121
|
+
# review-request-changes arms already share, so a verdict that keeps
|
|
2122
|
+
# going stale (e.g. a base that keeps moving under it) still
|
|
2123
|
+
# converges to an escalation instead of spinning forever.
|
|
2124
|
+
if counters.fix_rounds >= opts.max_fix_rounds:
|
|
2125
|
+
return _escalate_merge(state, state.merge_status, gate_kind=divergence)
|
|
2126
|
+
counters.fix_rounds += 1
|
|
2127
|
+
return Action(
|
|
2128
|
+
kind=RUN,
|
|
2129
|
+
label=(
|
|
2130
|
+
"MERGE: smoke verdict stale → re-test round "
|
|
2131
|
+
f"{counters.fix_rounds}/{opts.max_fix_rounds} "
|
|
2132
|
+
f"(coord diagnose {state.repo} {state.issue} --stage test --reset)"
|
|
2133
|
+
),
|
|
2134
|
+
command=(
|
|
2135
|
+
"diagnose", state.repo, str(state.issue),
|
|
2136
|
+
"--stage", "test", "--reset",
|
|
2137
|
+
),
|
|
2138
|
+
error_message=(
|
|
2139
|
+
f"coord diagnose {state.repo} {state.issue} --stage test "
|
|
2140
|
+
"--reset failed to clear the stale verdict.\n"
|
|
2141
|
+
f" Continue by hand: coord test {state.work_aid} --passed "
|
|
2142
|
+
"# ONLY if the suite genuinely still passes against the "
|
|
2143
|
+
"CURRENT base — otherwise dispatch a fresh smoke test"
|
|
2144
|
+
),
|
|
2145
|
+
)
|
|
2146
|
+
if divergence is not None:
|
|
2147
|
+
return _escalate_merge(state, state.merge_status, gate_kind=divergence)
|
|
2148
|
+
|
|
2149
|
+
# #1891: a CI verdict that has not arrived is not a CI verdict of "no" —
|
|
2150
|
+
# checked BEFORE the `status` switch below (and regardless of what
|
|
2151
|
+
# `status` itself reads) because `merge_reason` is the more robust of the
|
|
2152
|
+
# two: `drive_state._merge_entry` falls back to the raw queue row's own
|
|
2153
|
+
# *persisted* `error` whenever the live plan's re-evaluation comes back
|
|
2154
|
+
# empty (e.g. `_gate_refresher`'s periodic snapshot lagging or gapping a
|
|
2155
|
+
# real `coord merge` attempt's fresher read — see
|
|
2156
|
+
# `coord.merge_queue.CI_PENDING_PREFIX`'s docstring), while `merge_status`
|
|
2157
|
+
# has no such fallback and can still read `""`/`"PENDING"`/`"READY"` in
|
|
2158
|
+
# exactly that gap. #1891's incident was a drive burning its whole
|
|
2159
|
+
# `--max-merge-attempts` budget (and then a drive-queue launch attempt)
|
|
2160
|
+
# retrying a merge that only more real time — never another retry —
|
|
2161
|
+
# could resolve. No number of retries makes a check that hasn't reported
|
|
2162
|
+
# yet report sooner, so this is a bare wait, exactly like BLOCKED below,
|
|
2163
|
+
# and it never spends `counters.merge_attempts`.
|
|
2164
|
+
if is_ci_pending_reason(state.merge_reason):
|
|
2165
|
+
return _wait(
|
|
2166
|
+
label=(
|
|
2167
|
+
"MERGE: CI checks have not reported yet — waiting, not "
|
|
2168
|
+
f"retrying (#1891): {state.merge_reason}"
|
|
2169
|
+
)
|
|
2170
|
+
)
|
|
2171
|
+
|
|
2172
|
+
# #1892: the sibling case — a CI verdict DID arrive, but every failing
|
|
2173
|
+
# check said nothing about the code (never assigned a runner, or died
|
|
2174
|
+
# before checkout). `coord merge`'s own live attempt is already
|
|
2175
|
+
# auto-rerunning CI for this (see `coord.merge_queue.MAX_CI_INFRA_RERUNS`)
|
|
2176
|
+
# — retrying `coord merge` here would just re-observe the same in-flight
|
|
2177
|
+
# rerun and spend an attempt for nothing. Same bare wait as the #1891
|
|
2178
|
+
# case above, and for the identical reason: only more real time (here,
|
|
2179
|
+
# the rerun landing) resolves it, never another `coord merge` retry.
|
|
2180
|
+
if is_ci_infra_reason(state.merge_reason):
|
|
2181
|
+
return _wait(
|
|
2182
|
+
label=(
|
|
2183
|
+
"MERGE: CI failed with no verdict about the code — "
|
|
2184
|
+
f"auto-rerunning, not retrying (#1892): {state.merge_reason}"
|
|
2185
|
+
)
|
|
2186
|
+
)
|
|
2187
|
+
|
|
2188
|
+
status = state.merge_status
|
|
2189
|
+
if status.upper() == "HUMAN_REQUIRED":
|
|
2190
|
+
return _die(
|
|
2191
|
+
f"merge entry is HUMAN_REQUIRED: {state.merge_reason or 'no reason recorded'}\n"
|
|
2192
|
+
" An automated conflict-fix already gave up. Resolve by hand, or "
|
|
2193
|
+
"override:\n"
|
|
2194
|
+
f" coord merge --only {state.merge_aid or state.work_aid} "
|
|
2195
|
+
"--override-human-required '<reason>'"
|
|
2196
|
+
)
|
|
2197
|
+
if status.upper() == "BLOCKED":
|
|
2198
|
+
return _wait(
|
|
2199
|
+
label=(
|
|
2200
|
+
"MERGE: blocked — "
|
|
2201
|
+
f"{state.merge_reason or 'gate not satisfied'}; re-checking"
|
|
2202
|
+
)
|
|
2203
|
+
)
|
|
2204
|
+
|
|
2205
|
+
# #2078: an EMPTY status means "the board has no merge-queue entry for
|
|
2206
|
+
# this issue at all" — a fundamentally different fact from a real
|
|
2207
|
+
# PENDING/READY/MERGING/CONFLICT status, even though `""` sits in
|
|
2208
|
+
# `_RETRYABLE_MERGE_STATUSES` alongside them and used to retry
|
|
2209
|
+
# identically. `coord merge --only <aid>`, when it finds no entry, prints
|
|
2210
|
+
# exactly why (`_explain_missing_only_entry`, #1695) — every attempt
|
|
2211
|
+
# below captures that text into `counters.last_merge_diagnostic`
|
|
2212
|
+
# (`Driver._loop`), so by the SECOND empty-status poll this driver
|
|
2213
|
+
# already knows, from its own prior attempt, whether the row is simply
|
|
2214
|
+
# not enqueued yet (self-heals — fall through to the bounded retry) or
|
|
2215
|
+
# genuinely blocked on a review/smoke gate that a fourth identical
|
|
2216
|
+
# `--only` cannot change (wait, like the BLOCKED arm above, instead of
|
|
2217
|
+
# spending another attempt on a no-op). A CI-only block stays invisible
|
|
2218
|
+
# here — CI is only evaluated once a row is enqueued — and falls through
|
|
2219
|
+
# to the retry below, where the die message now quotes whatever the
|
|
2220
|
+
# driver actually observed instead of echoing the board's empty fields.
|
|
2221
|
+
if status == "":
|
|
2222
|
+
gate_reason = _extract_gate_block_reason(counters.last_merge_diagnostic)
|
|
2223
|
+
if gate_reason:
|
|
2224
|
+
return _wait(
|
|
2225
|
+
label=(
|
|
2226
|
+
"MERGE: blocked (not yet enqueued) — "
|
|
2227
|
+
f"{gate_reason}; re-checking"
|
|
2228
|
+
)
|
|
2229
|
+
)
|
|
2230
|
+
|
|
2231
|
+
# #1505: a status no retry can fix (most commonly NEEDS_ATTENTION)
|
|
2232
|
+
# escalates immediately rather than falling into the bounded retry below
|
|
2233
|
+
# — see `_RETRYABLE_MERGE_STATUSES` and `_escalate_merge`.
|
|
2234
|
+
if status.upper() not in _RETRYABLE_MERGE_STATUSES:
|
|
2235
|
+
return _escalate_merge(state, status)
|
|
2236
|
+
|
|
2237
|
+
# Cap the attempts: without this, a merge that fails for a reason the board
|
|
2238
|
+
# never reflects (so merge_status stays empty) would re-run `coord merge`
|
|
2239
|
+
# on every poll until the deadline. The same cap bounds the CONFLICT case
|
|
2240
|
+
# (#1474) — a `coord merge --only` that keeps landing back on CONFLICT
|
|
2241
|
+
# (e.g. a fresh conflict on every rebase attempt) must still terminate
|
|
2242
|
+
# rather than spin forever.
|
|
2243
|
+
if counters.merge_attempts >= opts.max_merge_attempts:
|
|
2244
|
+
# #2078: quote the last captured `coord merge --only` diagnostic
|
|
2245
|
+
# instead of just the board's (possibly still-empty) status/reason
|
|
2246
|
+
# fields — `coord merge --plan` was never actually run here, but the
|
|
2247
|
+
# SAME underlying gate check (`_explain_missing_only_entry`) already
|
|
2248
|
+
# ran, on every attempt above, and its output is exactly the
|
|
2249
|
+
# diagnosis a human would otherwise have to go fetch by hand.
|
|
2250
|
+
diagnostic = counters.last_merge_diagnostic.strip()
|
|
2251
|
+
diagnostic_block = (
|
|
2252
|
+
"\n".join(f" {line}" for line in diagnostic.splitlines())
|
|
2253
|
+
if diagnostic
|
|
2254
|
+
else " (no output captured from the merge attempts)"
|
|
2255
|
+
)
|
|
2256
|
+
return _die(
|
|
2257
|
+
f"merge attempted {counters.merge_attempts} times without landing.\n"
|
|
2258
|
+
f" Last board state: status='{status or 'none'}' "
|
|
2259
|
+
f"reason='{state.merge_reason or 'none'}'\n"
|
|
2260
|
+
f" Last `coord merge --only` diagnostic:\n"
|
|
2261
|
+
f"{diagnostic_block}\n"
|
|
2262
|
+
f" Inspect the gates: coord merge --plan --repo {state.repo}"
|
|
2263
|
+
)
|
|
2264
|
+
counters.merge_attempts += 1
|
|
2265
|
+
aid = state.merge_aid or state.work_aid
|
|
2266
|
+
# Tolerant on purpose: the first attempt often lands before the daemon's
|
|
2267
|
+
# tick has run `enqueue_approved_work`, so `--only <aid>` finds no queue
|
|
2268
|
+
# entry. That is a "try again next poll", not a reason to abort the run —
|
|
2269
|
+
# the attempt cap above is what bounds it. A CONFLICT entry that is
|
|
2270
|
+
# already CONFLICT (not PENDING) similarly errors out of `--only`
|
|
2271
|
+
# (`coord merge --only` refuses a non-PENDING entry) rather than
|
|
2272
|
+
# reclassifying it — that message still counts against the same cap, so
|
|
2273
|
+
# a genuinely stuck entry dies with a clear pointer instead of spinning.
|
|
2274
|
+
conflict_note = (
|
|
2275
|
+
" — retrying via coord merge's own conflict-fix dispatch (#241)"
|
|
2276
|
+
if status.upper() == "CONFLICT"
|
|
2277
|
+
else ""
|
|
2278
|
+
)
|
|
2279
|
+
return Action(
|
|
2280
|
+
kind=RUN,
|
|
2281
|
+
label=(
|
|
2282
|
+
f"MERGE: attempt {counters.merge_attempts}/{opts.max_merge_attempts} "
|
|
2283
|
+
f"(coord merge --only {aid} --method {opts.merge_method})"
|
|
2284
|
+
f"{conflict_note}"
|
|
2285
|
+
),
|
|
2286
|
+
command=("merge", "--only", aid, "--method", opts.merge_method),
|
|
2287
|
+
on_error="warn",
|
|
2288
|
+
error_message=(
|
|
2289
|
+
"coord merge returned non-zero (or the merge lock timed out) — "
|
|
2290
|
+
"re-checking next poll"
|
|
2291
|
+
),
|
|
2292
|
+
serialize_merge=True,
|
|
2293
|
+
)
|
|
2294
|
+
|
|
2295
|
+
|
|
2296
|
+
# ── locking ──────────────────────────────────────────────────────────────────
|
|
2297
|
+
|
|
2298
|
+
# #1616: ``FileLock``/``LockBusy`` moved to :mod:`coord.filelock` so the daemon's
|
|
2299
|
+
# pipeline-clock drain (``coord.notify.run_drain``) takes literally the same lock
|
|
2300
|
+
# class this module's ``run_notify()`` takes on ``~/.coord/notify.lock`` — a
|
|
2301
|
+
# second implementation agreeing on a filename is not mutual exclusion, it is a
|
|
2302
|
+
# coincidence. Re-exported here so every existing ``from coord.drive import
|
|
2303
|
+
# FileLock`` (tests included) keeps working unchanged.
|
|
2304
|
+
_ = (FileLock, LockBusy, notify_lock_path) # re-export; see coord.filelock
|
|
2305
|
+
|
|
2306
|
+
|
|
2307
|
+
# ── the driver (I/O) ─────────────────────────────────────────────────────────
|
|
2308
|
+
|
|
2309
|
+
|
|
2310
|
+
def coord_argv() -> list[str]:
|
|
2311
|
+
"""The ``coord`` invocation prefix.
|
|
2312
|
+
|
|
2313
|
+
Prefers the installed console script (the same thing a human types). Falls
|
|
2314
|
+
back to ``python -m coord.cli`` when it is not on PATH — which happens under
|
|
2315
|
+
a venv whose ``bin`` is not exported, e.g. a worker with the agent venv
|
|
2316
|
+
stripped (#402). Overridable with ``$COORD_DRIVE_COORD_BIN`` for tests.
|
|
2317
|
+
"""
|
|
2318
|
+
override = os.environ.get("COORD_DRIVE_COORD_BIN")
|
|
2319
|
+
if override:
|
|
2320
|
+
return override.split()
|
|
2321
|
+
found = shutil.which("coord")
|
|
2322
|
+
if found:
|
|
2323
|
+
return [found]
|
|
2324
|
+
return [sys.executable, "-m", "coord.cli"]
|
|
2325
|
+
|
|
2326
|
+
|
|
2327
|
+
# ── tmux launch (`coord drive --tmux`, #1398) ─────────────────────────────────
|
|
2328
|
+
#
|
|
2329
|
+
# A drive runs 60-90 minutes. `--tmux` launches it DETACHED in a
|
|
2330
|
+
# `coord-drive-<repo>-<issue>` tmux session instead of running inline, so the
|
|
2331
|
+
# run survives the launching terminal closing, a TUI restart, or an ssh drop
|
|
2332
|
+
# — the same rationale, and the same `TmuxHost`/`tmux_available`/
|
|
2333
|
+
# `tmux_session_alive` seam, as the `coord-<assignment_id>` interactive
|
|
2334
|
+
# sessions in `coord/interactive.py` and the free-floating `coord-term-*`
|
|
2335
|
+
# terminals in `coord/commands/terminal.py`. Unlike both of those, a drive
|
|
2336
|
+
# session is LOCAL ONLY — the driver runs on the operator's machine, reading
|
|
2337
|
+
# the daemon's board over the network, so there is no remote/ssh variant
|
|
2338
|
+
# here (see the class docstring's "Out of scope" note in #1398).
|
|
2339
|
+
#
|
|
2340
|
+
# Killing the tmux session IS Stop: the per-issue `flock` in `Driver.run()`
|
|
2341
|
+
# is released when the process's file descriptor closes (the OS does this on
|
|
2342
|
+
# any process exit, including SIGHUP from a killed tmux pane) — no separate
|
|
2343
|
+
# cleanup code is needed for cancellation to be correct.
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
def drive_session_name(repo: str, issue: int) -> str:
|
|
2347
|
+
"""Return the canonical tmux session name for a ``coord drive --tmux`` run."""
|
|
2348
|
+
return f"{DRIVE_SESSION_PREFIX}{repo}-{issue}"
|
|
2349
|
+
|
|
2350
|
+
|
|
2351
|
+
def parse_drive_session_name(session_name: str) -> tuple[str, int] | None:
|
|
2352
|
+
"""Parse a ``coord-drive-<repo>-<issue>`` session name back to ``(repo, issue)``.
|
|
2353
|
+
|
|
2354
|
+
Returns ``None`` when *session_name* doesn't carry the drive prefix, or
|
|
2355
|
+
the segment after the LAST hyphen isn't a bare issue number (repo names
|
|
2356
|
+
may themselves contain hyphens, so the issue number — always numeric —
|
|
2357
|
+
is what anchors the split).
|
|
2358
|
+
"""
|
|
2359
|
+
if not session_name.startswith(DRIVE_SESSION_PREFIX):
|
|
2360
|
+
return None
|
|
2361
|
+
rest = session_name[len(DRIVE_SESSION_PREFIX):]
|
|
2362
|
+
repo, sep, issue_str = rest.rpartition("-")
|
|
2363
|
+
if not sep or not repo or not issue_str.isdigit():
|
|
2364
|
+
return None
|
|
2365
|
+
return repo, int(issue_str)
|
|
2366
|
+
|
|
2367
|
+
|
|
2368
|
+
def list_drive_sessions(*, host: TmuxHost = TmuxHost(None)) -> list[dict[str, Any]]:
|
|
2369
|
+
"""Return live ``coord-drive-*`` tmux sessions on *host* as parsed dicts.
|
|
2370
|
+
|
|
2371
|
+
Each entry: ``{"repo": str, "issue": int, "session_name": str, "attached": bool}``.
|
|
2372
|
+
Mirrors :func:`coord.commands.terminal.list_tmux_terminal_sessions` — a
|
|
2373
|
+
single ``tmux list-sessions`` call; returns ``[]`` when tmux is
|
|
2374
|
+
unavailable, has no server running, or has no matching sessions.
|
|
2375
|
+
"""
|
|
2376
|
+
try:
|
|
2377
|
+
result = subprocess.run(
|
|
2378
|
+
host.cmd([
|
|
2379
|
+
"list-sessions", "-F",
|
|
2380
|
+
"#{session_name}\t#{session_attached}",
|
|
2381
|
+
]),
|
|
2382
|
+
capture_output=True,
|
|
2383
|
+
text=True,
|
|
2384
|
+
timeout=5.0,
|
|
2385
|
+
)
|
|
2386
|
+
except (subprocess.SubprocessError, OSError):
|
|
2387
|
+
return []
|
|
2388
|
+
if result.returncode != 0:
|
|
2389
|
+
return []
|
|
2390
|
+
|
|
2391
|
+
sessions: list[dict[str, Any]] = []
|
|
2392
|
+
for raw_line in result.stdout.splitlines():
|
|
2393
|
+
parts = raw_line.split("\t")
|
|
2394
|
+
if len(parts) < 2:
|
|
2395
|
+
continue
|
|
2396
|
+
name, attached_raw = parts[0].strip(), parts[1].strip()
|
|
2397
|
+
parsed = parse_drive_session_name(name)
|
|
2398
|
+
if parsed is None:
|
|
2399
|
+
continue
|
|
2400
|
+
repo, issue = parsed
|
|
2401
|
+
sessions.append({
|
|
2402
|
+
"repo": repo,
|
|
2403
|
+
"issue": issue,
|
|
2404
|
+
"session_name": name,
|
|
2405
|
+
"attached": attached_raw not in ("", "0"),
|
|
2406
|
+
})
|
|
2407
|
+
return sessions
|
|
2408
|
+
|
|
2409
|
+
|
|
2410
|
+
def launch_drive_in_tmux(
|
|
2411
|
+
cmd: Sequence[str],
|
|
2412
|
+
*,
|
|
2413
|
+
repo: str,
|
|
2414
|
+
issue: int,
|
|
2415
|
+
host: TmuxHost = TmuxHost(None),
|
|
2416
|
+
verify_checks: int = 16,
|
|
2417
|
+
verify_interval: float = 0.5,
|
|
2418
|
+
sleeper: Callable[[float], None] = time.sleep,
|
|
2419
|
+
) -> str:
|
|
2420
|
+
"""Create a detached tmux session named for *(repo, issue)* running *cmd*.
|
|
2421
|
+
|
|
2422
|
+
*cmd* is a full argv (e.g. ``coord_argv() + ["drive", repo, str(issue),
|
|
2423
|
+
...]``) — each element is passed to tmux as a SEPARATE argument, which
|
|
2424
|
+
tmux hands to ``execve`` unmodified (no shell re-splitting), so a path
|
|
2425
|
+
containing spaces (``--briefing-file``, ``--config``) survives intact.
|
|
2426
|
+
|
|
2427
|
+
Returns the session name. Raises :class:`DriveError` when tmux is
|
|
2428
|
+
unavailable, a session for this *(repo, issue)* is already alive (the
|
|
2429
|
+
CLI checks aliveness first for a friendlier message, but this guards
|
|
2430
|
+
direct/test callers too), or — #1606 — ``tmux new-session`` succeeded
|
|
2431
|
+
but the launched process never actually got a drive loop running.
|
|
2432
|
+
|
|
2433
|
+
#1606: ``tmux new-session`` returning 0 only proves tmux itself started
|
|
2434
|
+
a process; it says nothing about whether *that* process stayed up. The
|
|
2435
|
+
observed failure (drive dispatched with ``--accept-advisory`` onto a
|
|
2436
|
+
zero-commit advisory, decided there was nothing to do, and exited
|
|
2437
|
+
immediately) left the session dead and ``Driver.run()``'s own log
|
|
2438
|
+
untouched — while this function still returned success and the CLI
|
|
2439
|
+
printed the "driving ... in tmux session" banner. ``~/.coord/drive-
|
|
2440
|
+
epic.py`` treats *any* zero-exit ``coord drive --tmux`` as a live
|
|
2441
|
+
attempt and increments its ledger, so an unreported instant-death here
|
|
2442
|
+
silently burns a retry budget without ever running the issue once.
|
|
2443
|
+
After the session is created, poll (up to ``verify_checks *
|
|
2444
|
+
verify_interval`` seconds, default 8s) for the session to still be
|
|
2445
|
+
alive AND ``Driver.run()``'s own run log (``scratch_dir()/<repo>-
|
|
2446
|
+
<issue>.log`` — the same path ``Driver.run()`` computes) to have grown
|
|
2447
|
+
past whatever it held before this launch. Either check failing raises
|
|
2448
|
+
:class:`DriveError` instead of returning a session name — the caller
|
|
2449
|
+
must then report failure, not the success banner.
|
|
2450
|
+
|
|
2451
|
+
The growth check relies on ``Driver.run()`` writing a start marker to
|
|
2452
|
+
that log the instant its per-issue lock is acquired (see the
|
|
2453
|
+
``drive loop started`` line in ``Driver.run()``) — independent of
|
|
2454
|
+
whether a ``RUN`` action (the *only other* writer of this file, via
|
|
2455
|
+
``Driver._spawn``) ever actually fires. Without that marker, "log
|
|
2456
|
+
grew" would really mean "a subprocess happened to run first", which is
|
|
2457
|
+
false for the ordinary, majority-case launch of attaching to an issue
|
|
2458
|
+
that already has another assignment active: ``decide()``'s very first
|
|
2459
|
+
branch after "merged" is a pure ``WAIT`` with no command whenever
|
|
2460
|
+
``state.active_count > 0``, so that loop could legitimately sit
|
|
2461
|
+
alive-but-log-silent for a full ``--poll`` interval (default 60s) —
|
|
2462
|
+
far longer than this function's ~8s verification window — and get
|
|
2463
|
+
misdiagnosed as stuck.
|
|
2464
|
+
"""
|
|
2465
|
+
if not tmux_available():
|
|
2466
|
+
raise DriveError("tmux is not available on this machine.", EXIT_USAGE)
|
|
2467
|
+
session = drive_session_name(repo, issue)
|
|
2468
|
+
if tmux_session_alive(session, host=host):
|
|
2469
|
+
raise DriveError(
|
|
2470
|
+
f"already driving {repo} #{issue} (tmux session {session!r} is live).\n"
|
|
2471
|
+
f" attach with: coord drive-attach {repo} {issue}",
|
|
2472
|
+
EXIT_USAGE,
|
|
2473
|
+
)
|
|
2474
|
+
log_path = scratch_dir() / f"{repo}-{issue}.log"
|
|
2475
|
+
try:
|
|
2476
|
+
before_mtime = log_path.stat().st_mtime
|
|
2477
|
+
except OSError:
|
|
2478
|
+
before_mtime = None
|
|
2479
|
+
try:
|
|
2480
|
+
result = subprocess.run(
|
|
2481
|
+
host.cmd(["new-session", "-d", "-s", session, *cmd]),
|
|
2482
|
+
capture_output=True,
|
|
2483
|
+
text=True,
|
|
2484
|
+
timeout=15.0,
|
|
2485
|
+
)
|
|
2486
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
2487
|
+
raise DriveError(f"failed to create tmux session: {exc}", EXIT_USAGE) from exc
|
|
2488
|
+
if result.returncode != 0:
|
|
2489
|
+
raise DriveError(
|
|
2490
|
+
f"tmux new-session failed: {(result.stderr or '').strip()}", EXIT_USAGE
|
|
2491
|
+
)
|
|
2492
|
+
|
|
2493
|
+
alive = True
|
|
2494
|
+
grew = False
|
|
2495
|
+
for _ in range(max(verify_checks, 1)):
|
|
2496
|
+
sleeper(verify_interval)
|
|
2497
|
+
alive = tmux_session_alive(session, host=host)
|
|
2498
|
+
try:
|
|
2499
|
+
grew = log_path.stat().st_mtime > (before_mtime or 0)
|
|
2500
|
+
except OSError:
|
|
2501
|
+
grew = False
|
|
2502
|
+
if grew or not alive:
|
|
2503
|
+
break
|
|
2504
|
+
if not alive:
|
|
2505
|
+
detail = (
|
|
2506
|
+
"it did write to its log before exiting" if grew
|
|
2507
|
+
else "it never wrote anything to its log"
|
|
2508
|
+
)
|
|
2509
|
+
raise DriveError(
|
|
2510
|
+
f"tmux session {session!r} for {repo} #{issue} already exited "
|
|
2511
|
+
f"({detail}) — the drive loop did not stay running, so this is "
|
|
2512
|
+
f"not a live background run. Check the log: {log_path}\n"
|
|
2513
|
+
" Re-run without --tmux to see the failure inline instead.",
|
|
2514
|
+
EXIT_USAGE,
|
|
2515
|
+
)
|
|
2516
|
+
if not grew:
|
|
2517
|
+
raise DriveError(
|
|
2518
|
+
f"tmux session {session!r} for {repo} #{issue} is running but its "
|
|
2519
|
+
f"log ({log_path}) was never written to within "
|
|
2520
|
+
f"{verify_checks * verify_interval:.0f}s — the drive loop may be "
|
|
2521
|
+
f"stuck before its first log line. Attach to inspect: coord "
|
|
2522
|
+
f"drive-attach {repo} {issue}",
|
|
2523
|
+
EXIT_USAGE,
|
|
2524
|
+
)
|
|
2525
|
+
return session
|
|
2526
|
+
|
|
2527
|
+
|
|
2528
|
+
@dataclass
|
|
2529
|
+
class Driver:
|
|
2530
|
+
"""The resumable state machine's I/O shell: poll → decide → execute → sleep."""
|
|
2531
|
+
|
|
2532
|
+
repo: str
|
|
2533
|
+
issue: int
|
|
2534
|
+
opts: DriveOptions
|
|
2535
|
+
config: Any
|
|
2536
|
+
fetcher: BoardFetcher = field(default_factory=BoardFetcher)
|
|
2537
|
+
verifier: MergeVerifier | None = None
|
|
2538
|
+
oracle_gate: AcceptanceGateChecker | None = None
|
|
2539
|
+
out: Any = None
|
|
2540
|
+
err: Any = None
|
|
2541
|
+
sleeper: Callable[[float], None] = time.sleep
|
|
2542
|
+
clock: Callable[[], float] = time.monotonic
|
|
2543
|
+
# #1466: injected so tests can stub the Max-plan usage probe without a
|
|
2544
|
+
# real `claude -p "/usage"` subprocess — mirrors *verifier*/*oracle_gate*
|
|
2545
|
+
# above. Defaults to the real (cached, ~60s) probe.
|
|
2546
|
+
usage_prober: Callable[[], PlanLimits] = get_plan_limits
|
|
2547
|
+
|
|
2548
|
+
_run_log: Path | None = field(default=None, init=False, repr=False)
|
|
2549
|
+
# #1499: the terminating Action's own message, captured by `_loop()` right
|
|
2550
|
+
# before it returns — `run()` folds this into the `drive_exited` audit
|
|
2551
|
+
# summary/details so a non-exceptional terminal exit (e.g. `decide()`
|
|
2552
|
+
# returning a `_die(...)` Action for a genuinely failed work assignment,
|
|
2553
|
+
# as opposed to a raised DriveError) still narrates WHY, not just the
|
|
2554
|
+
# bare exit code.
|
|
2555
|
+
_last_exit_message: str = field(default="", init=False, repr=False)
|
|
2556
|
+
# #1844: the most recent `_spawn`ed subprocess's combined stdout+stderr —
|
|
2557
|
+
# the ONLY place the real text of a `coord assign`/`approve-plan`
|
|
2558
|
+
# refusal exists once the subprocess has exited (`_append_run_log`
|
|
2559
|
+
# writes the same bytes to disk, but nothing downstream re-reads that
|
|
2560
|
+
# file). `_loop`'s RUN-action handling reaches for this when the child
|
|
2561
|
+
# exits `EXIT_DISPATCH_REFUSED`, so the guard's own message — remedy
|
|
2562
|
+
# included — becomes the raised `DriveError`'s message instead of a
|
|
2563
|
+
# generic "coord assign ... exited 5". Overwritten on every `_spawn`
|
|
2564
|
+
# call, so it is only ever trustworthy read immediately after one, which
|
|
2565
|
+
# is exactly how `_loop` uses it.
|
|
2566
|
+
_last_run_output: str = field(default="", init=False, repr=False)
|
|
2567
|
+
|
|
2568
|
+
def __post_init__(self) -> None:
|
|
2569
|
+
self.out = self.out or sys.stdout
|
|
2570
|
+
self.err = self.err or sys.stderr
|
|
2571
|
+
if self.verifier is None:
|
|
2572
|
+
self.verifier = GitMergeVerifier(
|
|
2573
|
+
repo_path=self.opts.repo_path, warn=self.warn
|
|
2574
|
+
)
|
|
2575
|
+
if self.oracle_gate is None:
|
|
2576
|
+
self.oracle_gate = GitHubAcceptanceGateChecker(config=self.config)
|
|
2577
|
+
|
|
2578
|
+
# ── logging ─────────────────────────────────────────────────────────
|
|
2579
|
+
@staticmethod
|
|
2580
|
+
def _stamp() -> str:
|
|
2581
|
+
return time.strftime("%H:%M:%S")
|
|
2582
|
+
|
|
2583
|
+
def log(self, message: str) -> None:
|
|
2584
|
+
print(f"{self._stamp()} {message}", file=self.out, flush=True)
|
|
2585
|
+
|
|
2586
|
+
def warn(self, message: str) -> None:
|
|
2587
|
+
print(f"{self._stamp()} !! {message}", file=self.err, flush=True)
|
|
2588
|
+
|
|
2589
|
+
def _append_run_log(self, text: str) -> None:
|
|
2590
|
+
if self._run_log is None or not text:
|
|
2591
|
+
return
|
|
2592
|
+
try:
|
|
2593
|
+
with self._run_log.open("a") as fh:
|
|
2594
|
+
fh.write(text)
|
|
2595
|
+
except OSError:
|
|
2596
|
+
pass
|
|
2597
|
+
|
|
2598
|
+
# ── state ───────────────────────────────────────────────────────────
|
|
2599
|
+
def read_state(self) -> IssueState | None:
|
|
2600
|
+
"""Project the current board, or ``None`` on a transport blip.
|
|
2601
|
+
|
|
2602
|
+
A blip must never be a traceback: the loop just retries next poll.
|
|
2603
|
+
"""
|
|
2604
|
+
try:
|
|
2605
|
+
payload = self.fetcher.fetch()
|
|
2606
|
+
except Exception as exc: # noqa: BLE001 — transport, not logic
|
|
2607
|
+
self.warn(f"state read failed: {exc}")
|
|
2608
|
+
return None
|
|
2609
|
+
try:
|
|
2610
|
+
return project(payload, self.repo, self.issue, self.config)
|
|
2611
|
+
except DriveStateError as exc:
|
|
2612
|
+
raise DriveError(str(exc), EXIT_USAGE) from exc
|
|
2613
|
+
|
|
2614
|
+
# ── execution ───────────────────────────────────────────────────────
|
|
2615
|
+
def run_coord(self, args: tuple[str, ...], *, serialize_merge: bool = False) -> int:
|
|
2616
|
+
"""Run a ``coord`` subcommand, echoing its output and appending to the log.
|
|
2617
|
+
|
|
2618
|
+
Output is captured and echoed after the process exits rather than
|
|
2619
|
+
streamed through a pipe. The bash version used ``tee``, whose exit
|
|
2620
|
+
code masks the command's (one of the sharp edges #1392 set out to
|
|
2621
|
+
remove) — here the return code is unambiguous, and the run log still
|
|
2622
|
+
gets every byte.
|
|
2623
|
+
"""
|
|
2624
|
+
argv = [*coord_argv(), *args]
|
|
2625
|
+
if self.opts.config_path:
|
|
2626
|
+
# Click parses options interspersed with arguments, so appending is
|
|
2627
|
+
# safe for every subcommand this driver invokes (all of which carry
|
|
2628
|
+
# the shared --config option).
|
|
2629
|
+
argv += ["--config", self.opts.config_path]
|
|
2630
|
+
if serialize_merge:
|
|
2631
|
+
# Merges are serialized on THIS HOST even when the runs themselves
|
|
2632
|
+
# are parallel. #1400 fixed the daemon-side cross-talk (the
|
|
2633
|
+
# process-global `redirect_stdout` in POST /merge), so this is now
|
|
2634
|
+
# belt-and-braces for same-host callers rather than the only thing
|
|
2635
|
+
# preventing fleet-wide cross-talk; it still earns its keep by
|
|
2636
|
+
# keeping this host's own queue submissions ordered (two branches
|
|
2637
|
+
# rebasing onto a moving main at once is how pile-ups start) and by
|
|
2638
|
+
# failing fast locally instead of piling up blocked daemon requests.
|
|
2639
|
+
lock = FileLock(scratch_dir() / "merge.lock")
|
|
2640
|
+
try:
|
|
2641
|
+
lock.acquire(timeout=1800.0)
|
|
2642
|
+
except LockBusy:
|
|
2643
|
+
self.warn("merge lock timed out after 30m — re-checking next poll")
|
|
2644
|
+
return 1
|
|
2645
|
+
try:
|
|
2646
|
+
return self._spawn(argv)
|
|
2647
|
+
finally:
|
|
2648
|
+
lock.release()
|
|
2649
|
+
return self._spawn(argv)
|
|
2650
|
+
|
|
2651
|
+
def _spawn(self, argv: list[str]) -> int:
|
|
2652
|
+
proc = subprocess.run(argv, capture_output=True, text=True, check=False)
|
|
2653
|
+
combined = (proc.stdout or "") + (proc.stderr or "")
|
|
2654
|
+
if combined:
|
|
2655
|
+
print(combined.rstrip("\n"), file=self.out, flush=True)
|
|
2656
|
+
self._append_run_log(combined)
|
|
2657
|
+
self._last_run_output = combined.strip()
|
|
2658
|
+
return proc.returncode
|
|
2659
|
+
|
|
2660
|
+
def run_notify(self) -> None:
|
|
2661
|
+
"""Nudge ``coord notify`` under the shared lock, to cut timer latency."""
|
|
2662
|
+
if not self.opts.notify:
|
|
2663
|
+
return
|
|
2664
|
+
self.log("nudging: coord notify (flock ~/.coord/notify.lock)")
|
|
2665
|
+
lock = FileLock(notify_lock_path())
|
|
2666
|
+
try:
|
|
2667
|
+
lock.acquire(timeout=300.0)
|
|
2668
|
+
except LockBusy:
|
|
2669
|
+
self.warn("could not take ~/.coord/notify.lock within 5m — skipping nudge")
|
|
2670
|
+
return
|
|
2671
|
+
try:
|
|
2672
|
+
if self.run_coord(("notify",)) != 0:
|
|
2673
|
+
self.warn("coord notify returned non-zero")
|
|
2674
|
+
finally:
|
|
2675
|
+
lock.release()
|
|
2676
|
+
|
|
2677
|
+
def _post_escalation_comment(self, state: IssueState, message: str) -> None:
|
|
2678
|
+
"""#1526: durably surface an escalation's reason onto the issue
|
|
2679
|
+
itself, not just this run's tmux pane and the ``coord escalate``
|
|
2680
|
+
board row.
|
|
2681
|
+
|
|
2682
|
+
Best-effort and never raises: a failed post must not mask the
|
|
2683
|
+
escalation that already happened via the exit code (``EXIT_
|
|
2684
|
+
ESCALATED``) and the board record `action.command` just wrote — the
|
|
2685
|
+
two durable channels this already has. This is a THIRD channel, not
|
|
2686
|
+
a replacement for either.
|
|
2687
|
+
"""
|
|
2688
|
+
if not state.repo_github:
|
|
2689
|
+
return
|
|
2690
|
+
try:
|
|
2691
|
+
from coord import github_ops # noqa: PLC0415
|
|
2692
|
+
|
|
2693
|
+
github_ops.post_issue_comment(
|
|
2694
|
+
state.repo_github,
|
|
2695
|
+
state.issue,
|
|
2696
|
+
"🚧 **`coord drive` escalated — a human decision is needed.**\n\n"
|
|
2697
|
+
f"{message}\n\n"
|
|
2698
|
+
f"Run it: `coord escalate run {state.repo} {state.issue}`\n"
|
|
2699
|
+
f"Dismiss it: `coord escalate dismiss {state.repo} {state.issue}`",
|
|
2700
|
+
)
|
|
2701
|
+
except Exception as exc: # noqa: BLE001 — best-effort, never mask the exit
|
|
2702
|
+
self.warn(f"could not post the escalation comment to GitHub: {exc}")
|
|
2703
|
+
|
|
2704
|
+
# ── audit boundaries (#1499) ────────────────────────────────────────
|
|
2705
|
+
def _record_drive_audit(
|
|
2706
|
+
self,
|
|
2707
|
+
event_type: str,
|
|
2708
|
+
summary: str,
|
|
2709
|
+
*,
|
|
2710
|
+
details: dict[str, Any] | None = None,
|
|
2711
|
+
) -> None:
|
|
2712
|
+
"""Emit a ``category="drive"``, ``actor="drive"`` audit row.
|
|
2713
|
+
|
|
2714
|
+
``coord.audit.record_audit`` is itself best-effort (never raises into
|
|
2715
|
+
the caller — disk-full/locked-DB/schema-drift are swallowed there),
|
|
2716
|
+
so this needs no try/except of its own: a broken audit_log must never
|
|
2717
|
+
take down the drive loop.
|
|
2718
|
+
"""
|
|
2719
|
+
from coord.audit import record_audit # noqa: PLC0415
|
|
2720
|
+
|
|
2721
|
+
record_audit(
|
|
2722
|
+
tier="business",
|
|
2723
|
+
category="drive",
|
|
2724
|
+
event_type=event_type,
|
|
2725
|
+
actor="drive",
|
|
2726
|
+
summary=summary,
|
|
2727
|
+
repo=self.repo,
|
|
2728
|
+
issue=self.issue,
|
|
2729
|
+
details=details,
|
|
2730
|
+
)
|
|
2731
|
+
|
|
2732
|
+
def _drive_exit_summary(
|
|
2733
|
+
self, exit_code: int | None, exc: BaseException | None
|
|
2734
|
+
) -> tuple[str, dict[str, Any]]:
|
|
2735
|
+
"""Human summary + machine details for the terminating ``drive_exited``
|
|
2736
|
+
row — the piece that answers "what did the driver do, and why did it
|
|
2737
|
+
stop?" retroactively, after the tmux session (if any) is long gone."""
|
|
2738
|
+
ident = f"{self.repo}#{self.issue}"
|
|
2739
|
+
if exc is not None:
|
|
2740
|
+
if isinstance(exc, DriveError):
|
|
2741
|
+
summary = f"drive exited for {ident}: {exc} (exit_code={exc.exit_code})"
|
|
2742
|
+
return summary, {"exit_code": exc.exit_code, "error": str(exc)}
|
|
2743
|
+
summary = f"drive exited for {ident}: unexpected error ({exc!r})"
|
|
2744
|
+
return summary, {"exit_code": None, "error": repr(exc)}
|
|
2745
|
+
# Non-exceptional terminal exit — decide() returned a `_die(...)` (or
|
|
2746
|
+
# `_succeed(...)`) Action directly (`_loop`'s `action.is_exit` branch
|
|
2747
|
+
# returns the code without raising). `_last_exit_message` carries that
|
|
2748
|
+
# Action's own `message` — the same text this run's log already
|
|
2749
|
+
# printed via self.log/self.warn — so the audit row narrates WHY,
|
|
2750
|
+
# not just the bare exit code.
|
|
2751
|
+
reason = self._last_exit_message.strip()
|
|
2752
|
+
if not reason:
|
|
2753
|
+
reason = {EXIT_OK: "ok", EXIT_DEADLINE: "deadline exceeded"}.get(
|
|
2754
|
+
exit_code, f"exit_code={exit_code}"
|
|
2755
|
+
)
|
|
2756
|
+
summary = f"drive exited for {ident} (exit_code={exit_code}): {reason}"
|
|
2757
|
+
return summary, {"exit_code": exit_code, "reason": reason}
|
|
2758
|
+
|
|
2759
|
+
# ── the loop ────────────────────────────────────────────────────────
|
|
2760
|
+
def run(self) -> int:
|
|
2761
|
+
scratch = scratch_dir()
|
|
2762
|
+
self._run_log = scratch / f"{self.repo}-{self.issue}.log"
|
|
2763
|
+
|
|
2764
|
+
# PER-ISSUE lock. Two drivers on DIFFERENT issues are fine; two on the
|
|
2765
|
+
# SAME issue are not — they would double-dispatch work and
|
|
2766
|
+
# double-record verdicts.
|
|
2767
|
+
lock = FileLock(scratch / f"lock-{self.repo}-{self.issue}")
|
|
2768
|
+
holder = scratch / f"holder-{self.repo}-{self.issue}"
|
|
2769
|
+
try:
|
|
2770
|
+
lock.acquire(timeout=0.0)
|
|
2771
|
+
except LockBusy:
|
|
2772
|
+
try:
|
|
2773
|
+
who = holder.read_text().strip()
|
|
2774
|
+
except OSError:
|
|
2775
|
+
who = "another run"
|
|
2776
|
+
# No `drive_started`/`drive_exited` pair here — this run never
|
|
2777
|
+
# actually started (another driver already holds the per-issue
|
|
2778
|
+
# lock), so there is nothing new to narrate in the audit log.
|
|
2779
|
+
raise DriveError(
|
|
2780
|
+
f"already driving {self.repo} #{self.issue} ({who}).\n"
|
|
2781
|
+
" A second driver on the SAME issue would double-dispatch work.\n"
|
|
2782
|
+
" Other issues can be driven concurrently.\n"
|
|
2783
|
+
f" Lock file: {lock.path}",
|
|
2784
|
+
EXIT_USAGE,
|
|
2785
|
+
) from None
|
|
2786
|
+
try:
|
|
2787
|
+
holder.write_text(f"{self.repo} #{self.issue} (pid {os.getpid()})\n")
|
|
2788
|
+
except OSError:
|
|
2789
|
+
pass
|
|
2790
|
+
self._record_drive_audit(
|
|
2791
|
+
"drive_started", f"drive started for {self.repo}#{self.issue}"
|
|
2792
|
+
)
|
|
2793
|
+
# #1606: a start marker, written the instant the loop legitimately
|
|
2794
|
+
# begins — independent of whether a RUN action (`_spawn`, the only
|
|
2795
|
+
# other writer of this file) ever fires. `decide()`'s very first
|
|
2796
|
+
# branch after "merged" is a pure WAIT with no command whenever
|
|
2797
|
+
# another assignment is already active (coord/drive.py `decide()`),
|
|
2798
|
+
# so a drive attached to healthy in-flight work could sit
|
|
2799
|
+
# alive-but-log-silent for its entire first `--poll` interval
|
|
2800
|
+
# (default 60s). `launch_drive_in_tmux`'s post-launch verification
|
|
2801
|
+
# only waits ~8s for the log to grow, so without this marker it
|
|
2802
|
+
# would misdiagnose that ordinary, majority-case attach as a stuck
|
|
2803
|
+
# loop and kill a perfectly healthy session. This line makes "the
|
|
2804
|
+
# log grew" mean "the loop started", not "a subprocess happened to
|
|
2805
|
+
# run first".
|
|
2806
|
+
self._append_run_log(
|
|
2807
|
+
f"{self._stamp()} drive loop started for {self.repo}#{self.issue}\n"
|
|
2808
|
+
)
|
|
2809
|
+
try:
|
|
2810
|
+
exit_code = self._loop()
|
|
2811
|
+
except BaseException as exc: # noqa: BLE001 — narrate every exit, then re-raise unchanged
|
|
2812
|
+
summary, details = self._drive_exit_summary(None, exc)
|
|
2813
|
+
self._record_drive_audit("drive_exited", summary, details=details)
|
|
2814
|
+
raise
|
|
2815
|
+
else:
|
|
2816
|
+
summary, details = self._drive_exit_summary(exit_code, None)
|
|
2817
|
+
self._record_drive_audit("drive_exited", summary, details=details)
|
|
2818
|
+
return exit_code
|
|
2819
|
+
finally:
|
|
2820
|
+
try:
|
|
2821
|
+
holder.unlink()
|
|
2822
|
+
except OSError:
|
|
2823
|
+
pass
|
|
2824
|
+
lock.release()
|
|
2825
|
+
|
|
2826
|
+
def _loop(self) -> int:
|
|
2827
|
+
state = self.read_state()
|
|
2828
|
+
if state is None:
|
|
2829
|
+
raise DriveError("could not read board state", EXIT_USAGE)
|
|
2830
|
+
|
|
2831
|
+
# #1466: probe ONCE here (not per-poll) — the underlying `claude -p
|
|
2832
|
+
# "/usage"` call is itself cached ~60s (coord.usage_limits), but
|
|
2833
|
+
# there's no reason to re-shell-out every loop iteration for a
|
|
2834
|
+
# decision only made at the top of the run. Skipped entirely when
|
|
2835
|
+
# the gate is off, so a `disabled` config never pays the subprocess
|
|
2836
|
+
# cost.
|
|
2837
|
+
usage_limits = (
|
|
2838
|
+
self.usage_prober() if self.config.usage_gate.mode != "disabled" else None
|
|
2839
|
+
)
|
|
2840
|
+
pre = preflight(state, self.opts, self.config, usage_limits=usage_limits)
|
|
2841
|
+
machine = pre.machine
|
|
2842
|
+
|
|
2843
|
+
# #1453: resolved ONCE here (not per-poll) — the gate_checker inside
|
|
2844
|
+
# costs a GitHub fetch and a milestone's Gate-A status can't change
|
|
2845
|
+
# mid-run. Threaded unchanged into every decide() call below.
|
|
2846
|
+
oracle = resolve_oracle_decision(state, self.opts, self.config, self.oracle_gate)
|
|
2847
|
+
|
|
2848
|
+
self.log(f"driving {self.repo} #{self.issue}")
|
|
2849
|
+
self.log(f" machine : {machine}")
|
|
2850
|
+
if not self.opts.machine and state.picked_machine_provider_reason:
|
|
2851
|
+
# #1906: only meaningful for an AUTO-picked machine — an
|
|
2852
|
+
# explicit `--machine` never ran this selection's provider
|
|
2853
|
+
# resolution (it wins outright; #1711's dispatch-time guard is
|
|
2854
|
+
# the enforcement for it). Mirrors `coord assign --dry-run`'s
|
|
2855
|
+
# own "provider: ..." line (`describe_provider_choice`).
|
|
2856
|
+
self.log(f" provider : {state.picked_machine_provider_reason}")
|
|
2857
|
+
self.log(f" acceptance : {oracle.reason}")
|
|
2858
|
+
self.log(
|
|
2859
|
+
f" test command : {state.repo_test_command or '<none configured>'} "
|
|
2860
|
+
"(coord dispatches this itself — #1426; this observes)"
|
|
2861
|
+
)
|
|
2862
|
+
self.log(
|
|
2863
|
+
" merge : "
|
|
2864
|
+
+ (f"yes ({self.opts.merge_method})" if self.opts.do_merge else "no")
|
|
2865
|
+
)
|
|
2866
|
+
self.log(
|
|
2867
|
+
" auto-loop : "
|
|
2868
|
+
+ (
|
|
2869
|
+
"on (request-changes → this driver runs coord fix, #1692)"
|
|
2870
|
+
if state.auto_loop
|
|
2871
|
+
else "off (a request-changes verdict stops this run)"
|
|
2872
|
+
)
|
|
2873
|
+
)
|
|
2874
|
+
self.log(
|
|
2875
|
+
f" fix rounds : {self.opts.max_fix_rounds} this run, shared by "
|
|
2876
|
+
"the test and review arms (via coord fix)"
|
|
2877
|
+
)
|
|
2878
|
+
self.log(
|
|
2879
|
+
f" review fix cap : {state.max_review_iterations} "
|
|
2880
|
+
"(pipeline.max_review_iterations — per issue, across every drive)"
|
|
2881
|
+
)
|
|
2882
|
+
self.log(
|
|
2883
|
+
" notify nudge : "
|
|
2884
|
+
+ (
|
|
2885
|
+
"on"
|
|
2886
|
+
if self.opts.notify
|
|
2887
|
+
else "off (relying on the 5-min coord-notify.timer)"
|
|
2888
|
+
)
|
|
2889
|
+
)
|
|
2890
|
+
self.log(f" log : {self._run_log}")
|
|
2891
|
+
for warning in pre.warnings:
|
|
2892
|
+
self.warn(warning)
|
|
2893
|
+
|
|
2894
|
+
if self.opts.dry_run:
|
|
2895
|
+
self.log("current state:")
|
|
2896
|
+
print(
|
|
2897
|
+
json.dumps(state.as_flat_dict(), indent=2, default=str),
|
|
2898
|
+
file=self.out,
|
|
2899
|
+
flush=True,
|
|
2900
|
+
)
|
|
2901
|
+
return EXIT_OK
|
|
2902
|
+
|
|
2903
|
+
counters = DriveCounters()
|
|
2904
|
+
start = self.clock()
|
|
2905
|
+
deadline = start + self.opts.deadline_secs
|
|
2906
|
+
last_fingerprint = ""
|
|
2907
|
+
last_change = start
|
|
2908
|
+
# #1593: the nudge cadence is tracked SEPARATELY from `last_change`.
|
|
2909
|
+
# The one-shot latch (`nudged = False`/`True`, cleared only on a
|
|
2910
|
+
# fingerprint change) let a stage that stalls for 30-40 real minutes
|
|
2911
|
+
# get exactly one nudge near the start, then go completely silent —
|
|
2912
|
+
# `coord notify` correctly finds nothing to settle while the worker
|
|
2913
|
+
# is still running, and nothing ever re-checks after that. Re-nudging
|
|
2914
|
+
# every `stall_secs` while the fingerprint stays put, without
|
|
2915
|
+
# resetting `last_change`, keeps the staleness clock honest (so
|
|
2916
|
+
# `--stall` measures real elapsed idle time, not time-since-last-
|
|
2917
|
+
# nudge) while guaranteeing a stalled stage is never more than one
|
|
2918
|
+
# `stall_secs` window away from a fresh check.
|
|
2919
|
+
last_nudge: float | None = None
|
|
2920
|
+
|
|
2921
|
+
while True:
|
|
2922
|
+
now = self.clock()
|
|
2923
|
+
if now > deadline:
|
|
2924
|
+
self._last_exit_message = (
|
|
2925
|
+
f"deadline of {self.opts.deadline_mins:g}m exceeded"
|
|
2926
|
+
)
|
|
2927
|
+
self.warn(self._last_exit_message)
|
|
2928
|
+
if state is not None:
|
|
2929
|
+
print(
|
|
2930
|
+
json.dumps(state.as_flat_dict(), indent=2, default=str),
|
|
2931
|
+
file=self.err,
|
|
2932
|
+
flush=True,
|
|
2933
|
+
)
|
|
2934
|
+
return EXIT_DEADLINE
|
|
2935
|
+
|
|
2936
|
+
state = self.read_state()
|
|
2937
|
+
if state is None:
|
|
2938
|
+
self.sleeper(self.opts.poll)
|
|
2939
|
+
continue
|
|
2940
|
+
|
|
2941
|
+
fingerprint = state.fingerprint
|
|
2942
|
+
if fingerprint != last_fingerprint:
|
|
2943
|
+
last_fingerprint = fingerprint
|
|
2944
|
+
last_change = now
|
|
2945
|
+
last_nudge = None
|
|
2946
|
+
self.log(
|
|
2947
|
+
f"state: work={state.work_status or '-'} "
|
|
2948
|
+
f"test={state.work_test_state or '-'} "
|
|
2949
|
+
f"review={state.review_status or '-'}/"
|
|
2950
|
+
f"{state.review_verdict or '-'} "
|
|
2951
|
+
f"iter={state.work_review_iter} "
|
|
2952
|
+
f"merge={state.merge_status or '-'}"
|
|
2953
|
+
# #1526: print the merge gate's OWN reason right next to
|
|
2954
|
+
# its status — this is the line the 2026-07-27/28 stalls
|
|
2955
|
+
# never carried, so a "test=passed" operator watching the
|
|
2956
|
+
# pane had no way to see `coord merge` disagreeing until
|
|
2957
|
+
# it had already burned the whole retry budget.
|
|
2958
|
+
+ (f" ({state.merge_reason})" if state.merge_reason else "")
|
|
2959
|
+
+ f" active={state.active_count}"
|
|
2960
|
+
# #2079: while the oracle slice is landing, every field
|
|
2961
|
+
# above is empty by construction (the work row does not
|
|
2962
|
+
# exist yet) — so without this the one line that is
|
|
2963
|
+
# supposed to narrate progress narrated nothing at all
|
|
2964
|
+
# for hours. Only printed when a slice row exists, so a
|
|
2965
|
+
# normal drive's line is byte-for-byte unchanged.
|
|
2966
|
+
+ (
|
|
2967
|
+
f" | slice={state.acceptance_author_status or '-'}"
|
|
2968
|
+
f" test={state.acceptance_author_test_state or '-'}"
|
|
2969
|
+
f" review={state.acceptance_review_verdict or '-'}"
|
|
2970
|
+
f" merge={state.acceptance_merge_status or '-'}"
|
|
2971
|
+
+ (
|
|
2972
|
+
f" ({state.acceptance_merge_reason})"
|
|
2973
|
+
if state.acceptance_merge_reason
|
|
2974
|
+
else ""
|
|
2975
|
+
)
|
|
2976
|
+
if state.acceptance_author_aid
|
|
2977
|
+
else ""
|
|
2978
|
+
)
|
|
2979
|
+
)
|
|
2980
|
+
elif now - last_change > self.opts.stall_secs and (
|
|
2981
|
+
last_nudge is None or now - last_nudge > self.opts.stall_secs
|
|
2982
|
+
):
|
|
2983
|
+
# #1593: re-nudge on a `stall_secs` cadence for as long as the
|
|
2984
|
+
# fingerprint stays put, instead of firing once and going
|
|
2985
|
+
# silent. `last_change` is deliberately left untouched here —
|
|
2986
|
+
# only `last_nudge` advances — so the elapsed time below (and
|
|
2987
|
+
# `--stall`'s own monotonicity: smaller stall never means
|
|
2988
|
+
# FEWER nudges) keeps reflecting genuine staleness rather than
|
|
2989
|
+
# resetting every time this branch fires.
|
|
2990
|
+
self.warn(
|
|
2991
|
+
f"no state change in {(now - last_change) / 60.0:g}m "
|
|
2992
|
+
f"({','.join(state.active_types) or 'nothing'} active)"
|
|
2993
|
+
)
|
|
2994
|
+
self.run_notify()
|
|
2995
|
+
last_nudge = now
|
|
2996
|
+
|
|
2997
|
+
action = decide(
|
|
2998
|
+
state, self.opts, counters, self.verifier,
|
|
2999
|
+
machine=machine, oracle=oracle, gate_checker=self.oracle_gate,
|
|
3000
|
+
)
|
|
3001
|
+
for warning in action.warnings:
|
|
3002
|
+
self.warn(warning)
|
|
3003
|
+
|
|
3004
|
+
if action.is_exit:
|
|
3005
|
+
# #1499: capture the exit reason for the audit boundary before
|
|
3006
|
+
# anything below can fail — the escalation write is explicitly
|
|
3007
|
+
# best-effort, so it must not be able to cost us the reason.
|
|
3008
|
+
self._last_exit_message = action.message
|
|
3009
|
+
# #1505: an escalation exit still carries a `command` — the
|
|
3010
|
+
# `coord escalate record ...` write that makes the stop
|
|
3011
|
+
# reason board-visible after this process is gone. Run it
|
|
3012
|
+
# HERE (the I/O shell), not inside `decide()`, which stays a
|
|
3013
|
+
# pure function like every other decision in this module.
|
|
3014
|
+
# Best-effort: a failed write must never block the exit
|
|
3015
|
+
# itself (there is nothing left to retry), so this only
|
|
3016
|
+
# warns, never raises.
|
|
3017
|
+
if action.command:
|
|
3018
|
+
rc = self.run_coord(action.command)
|
|
3019
|
+
if rc != 0:
|
|
3020
|
+
self.warn(
|
|
3021
|
+
action.error_message
|
|
3022
|
+
or f"coord {' '.join(action.command)} exited {rc}"
|
|
3023
|
+
)
|
|
3024
|
+
# #1526: an escalation's reason must reach the issue itself,
|
|
3025
|
+
# not just this tmux pane (gone the moment the session ends)
|
|
3026
|
+
# and the `coord escalate` board row (invisible unless an
|
|
3027
|
+
# operator thinks to run `coord escalate list`). This is what
|
|
3028
|
+
# turned "drive died without closing the issue" into three
|
|
3029
|
+
# unexplained deaths during the 2026-07-27/28 overnight run.
|
|
3030
|
+
# #2019 rides the same rail: a dead end is exactly the
|
|
3031
|
+
# "nobody is coming" case this comment exists for, and the
|
|
3032
|
+
# tmux pane it would otherwise be trapped in dies with the
|
|
3033
|
+
# session.
|
|
3034
|
+
if action.exit_code in (EXIT_ESCALATED, EXIT_DEAD_END):
|
|
3035
|
+
self._post_escalation_comment(state, action.message)
|
|
3036
|
+
if action.exit_code == EXIT_OK:
|
|
3037
|
+
for line in action.message.splitlines():
|
|
3038
|
+
self.log(line)
|
|
3039
|
+
else:
|
|
3040
|
+
for line in action.message.splitlines():
|
|
3041
|
+
self.warn(line)
|
|
3042
|
+
return action.exit_code
|
|
3043
|
+
|
|
3044
|
+
if action.label:
|
|
3045
|
+
self.log(action.label)
|
|
3046
|
+
|
|
3047
|
+
if action.kind == RUN:
|
|
3048
|
+
rc = self.run_coord(
|
|
3049
|
+
action.command, serialize_merge=action.serialize_merge
|
|
3050
|
+
)
|
|
3051
|
+
if action.serialize_merge:
|
|
3052
|
+
# #2078: `serialize_merge` is set on exactly one Action —
|
|
3053
|
+
# `_decide_merge`'s bounded `coord merge --only <aid>`
|
|
3054
|
+
# retry — so this is the merge attempt's own diagnostic,
|
|
3055
|
+
# captured for `_decide_merge` to read back (via
|
|
3056
|
+
# `counters.last_merge_diagnostic`) on the NEXT poll: to
|
|
3057
|
+
# avoid a blind retry once a real gate block is already
|
|
3058
|
+
# known, and to name it in the give-up message instead of
|
|
3059
|
+
# the board's empty fields.
|
|
3060
|
+
#
|
|
3061
|
+
# #2079: `_decide_merge` now has two callers — the work
|
|
3062
|
+
# row and the oracle JIT slice — with one budget each, so
|
|
3063
|
+
# the diagnostic is filed against the budget that actually
|
|
3064
|
+
# spent the attempt. Cross-filing it would diagnose one PR
|
|
3065
|
+
# using the other PR's gates.
|
|
3066
|
+
budget = (
|
|
3067
|
+
counters.slice_budget()
|
|
3068
|
+
if action.merge_scope == "acceptance"
|
|
3069
|
+
else counters
|
|
3070
|
+
)
|
|
3071
|
+
budget.last_merge_diagnostic = self._last_run_output
|
|
3072
|
+
if rc != 0:
|
|
3073
|
+
# #1844: `coord assign`/`coord approve-plan` exits this
|
|
3074
|
+
# SAME code (see EXIT_DISPATCH_REFUSED's docstring) only
|
|
3075
|
+
# when a pre-dispatch guard refused deterministically —
|
|
3076
|
+
# never for a transient failure. That refusal's own
|
|
3077
|
+
# message (the guard's remedy, verbatim) is what the
|
|
3078
|
+
# child just printed to stdout/stderr, captured above by
|
|
3079
|
+
# `_spawn` into `_last_run_output`; `action.error_message`
|
|
3080
|
+
# is a STATIC string chosen when the Action was built and
|
|
3081
|
+
# cannot carry it. Re-raising with the SAME exit code (not
|
|
3082
|
+
# EXIT_TERMINAL_FAILURE) is what lets `_drive_exit_summary`
|
|
3083
|
+
# and, downstream, `coord/drive_queue.py`'s tick tell this
|
|
3084
|
+
# refusal apart from a genuine crash.
|
|
3085
|
+
if rc == EXIT_DISPATCH_REFUSED:
|
|
3086
|
+
msg = self._last_run_output or action.error_message or (
|
|
3087
|
+
f"coord {' '.join(action.command)} refused "
|
|
3088
|
+
f"(exit {rc})"
|
|
3089
|
+
)
|
|
3090
|
+
raise DriveError(msg, EXIT_DISPATCH_REFUSED)
|
|
3091
|
+
msg = action.error_message or (
|
|
3092
|
+
f"coord {' '.join(action.command)} exited {rc}"
|
|
3093
|
+
)
|
|
3094
|
+
if action.on_error == "warn":
|
|
3095
|
+
self.warn(msg)
|
|
3096
|
+
else:
|
|
3097
|
+
raise DriveError(msg, EXIT_TERMINAL_FAILURE)
|
|
3098
|
+
|
|
3099
|
+
self.sleeper(
|
|
3100
|
+
self.opts.poll if action.sleep_after is None else action.sleep_after
|
|
3101
|
+
)
|