code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/commands/status.py
ADDED
|
@@ -0,0 +1,2089 @@
|
|
|
1
|
+
"""`coord status`/`usage`/`show-plan`/`diagnose` — read-only board and
|
|
2
|
+
machine reporting. Extracted from coord/cli.py (#747)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
import click
|
|
12
|
+
|
|
13
|
+
from coord import __version__, github_ops
|
|
14
|
+
|
|
15
|
+
from coord.commands._common import _CONFIG_OPTION, _load_config
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from coord.config import Config
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _live_advisory_entries(
|
|
22
|
+
entries: list[dict],
|
|
23
|
+
cfg: "Config",
|
|
24
|
+
*,
|
|
25
|
+
cache: dict | None = None,
|
|
26
|
+
) -> list[dict]:
|
|
27
|
+
"""Drop advisory entries (#448) whose work is already terminal on GitHub.
|
|
28
|
+
|
|
29
|
+
#1472: an advisory entry is agent-local state served verbatim from the
|
|
30
|
+
worker's own completed-assignment map on every ``/status`` poll — nothing
|
|
31
|
+
tells the agent the issue closed or the branch merged, so a genuinely
|
|
32
|
+
resolved advisory (e.g. rescued, reviewed, and merged by hand) keeps
|
|
33
|
+
showing "The work is UNVERIFIED — review it before testing or merging."
|
|
34
|
+
forever. That trains the operator to skim past the whole advisory block,
|
|
35
|
+
which is exactly where a real 0-commit rescue needs to be noticed.
|
|
36
|
+
|
|
37
|
+
Reuses the shared #522 chokepoint guard
|
|
38
|
+
(:func:`coord.github_ops.work_is_terminal` — "issue closed OR PR merged")
|
|
39
|
+
rather than a second ad hoc check. **Fail-open**: an entry whose repo
|
|
40
|
+
isn't in *cfg* (or has no ``github`` slug configured) is kept rather than
|
|
41
|
+
silently dropped — ``work_is_terminal`` itself already fails open on any
|
|
42
|
+
GitHub/CLI error. *cache* defaults to a dict scoped to this call so a
|
|
43
|
+
repeated ``(repo, issue, branch)`` triple across entries costs one ``gh``
|
|
44
|
+
round-trip, not one per entry — this list renders on every ``coord
|
|
45
|
+
status``.
|
|
46
|
+
"""
|
|
47
|
+
if cache is None:
|
|
48
|
+
cache = {}
|
|
49
|
+
live = []
|
|
50
|
+
for e in entries:
|
|
51
|
+
spec = e.get("spec") or {}
|
|
52
|
+
repo_name = spec.get("repo_name")
|
|
53
|
+
repo_cfg = cfg.repo(repo_name) if repo_name else None
|
|
54
|
+
if repo_cfg is None or not repo_cfg.github:
|
|
55
|
+
live.append(e)
|
|
56
|
+
continue
|
|
57
|
+
if github_ops.work_is_terminal(
|
|
58
|
+
repo_cfg.github,
|
|
59
|
+
spec.get("issue_number"),
|
|
60
|
+
e.get("branch"),
|
|
61
|
+
cache=cache,
|
|
62
|
+
):
|
|
63
|
+
continue
|
|
64
|
+
live.append(e)
|
|
65
|
+
return live
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@click.command(help="Show all machines, assignments, and connectivity.")
|
|
69
|
+
@_CONFIG_OPTION
|
|
70
|
+
@click.option("--machine", "machine_filter", default=None, help="Only show this machine.")
|
|
71
|
+
@click.option("--timeout", default=3.0, show_default=True, type=float, help="Per-machine health-check timeout (seconds).")
|
|
72
|
+
@click.option("--no-reconcile", is_flag=True, help="Skip auto-reconciliation of the board with live agent state.")
|
|
73
|
+
@click.option(
|
|
74
|
+
"--freshness",
|
|
75
|
+
is_flag=True,
|
|
76
|
+
help="Also report per-machine repo freshness vs GitHub HEADs.",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def status(config_path: Path, machine_filter: str | None, no_reconcile: bool, timeout: float, freshness: bool) -> None:
|
|
81
|
+
from coord import freshness as fresh
|
|
82
|
+
from coord.deps import blocked_repos as compute_blocked, build_dep_graph
|
|
83
|
+
from coord.board_service import read_board, write_board
|
|
84
|
+
from coord.client import resolve_board_service
|
|
85
|
+
from coord.network import check_all, fetch_repos, fetch_status
|
|
86
|
+
from coord.state import load_dispatched, load_notified
|
|
87
|
+
|
|
88
|
+
# #584/#1080: when a board service is configured, read the board + config
|
|
89
|
+
# from the daemon instead of local SQLite. _load_config() itself now always
|
|
90
|
+
# fetches the daemon's config on a thin client (never trusts a local file
|
|
91
|
+
# that happens to exist — the config-fetch pre-step that used to live here
|
|
92
|
+
# was a redundant duplicate of that same buggy "local file exists" check,
|
|
93
|
+
# removed in #1080). `svc` below is still needed to gate the local-only
|
|
94
|
+
# reads (queue/notified/session) further down. Unset ⇒ unchanged local
|
|
95
|
+
# behaviour.
|
|
96
|
+
svc = resolve_board_service()
|
|
97
|
+
cfg = _load_config(config_path)
|
|
98
|
+
|
|
99
|
+
# Dependency graph (only when --machine isn't narrowing the view).
|
|
100
|
+
if not machine_filter:
|
|
101
|
+
graph = build_dep_graph(cfg.repos)
|
|
102
|
+
if any(deps for deps in graph.values()):
|
|
103
|
+
click.echo("Dependency graph:")
|
|
104
|
+
for repo in cfg.repos:
|
|
105
|
+
deps = graph.get(repo.name, [])
|
|
106
|
+
if deps:
|
|
107
|
+
click.echo(f" {repo.name} → {', '.join(deps)}")
|
|
108
|
+
else:
|
|
109
|
+
click.echo(f" {repo.name} (no dependencies)")
|
|
110
|
+
click.echo()
|
|
111
|
+
|
|
112
|
+
machines = cfg.machines
|
|
113
|
+
if machine_filter:
|
|
114
|
+
machines = [m for m in machines if m.name == machine_filter]
|
|
115
|
+
if not machines:
|
|
116
|
+
click.echo(
|
|
117
|
+
f"error: machine {machine_filter!r} not in coordinator.yml "
|
|
118
|
+
f"(have: {[m.name for m in cfg.machines]})",
|
|
119
|
+
err=True,
|
|
120
|
+
)
|
|
121
|
+
sys.exit(2)
|
|
122
|
+
|
|
123
|
+
# #1563: paused_set() is daemon-aware — on a thin client it fetches the
|
|
124
|
+
# daemon's own `/pause` copy, so this renders the state that actually
|
|
125
|
+
# governs dispatch instead of a host-local file the daemon never reads.
|
|
126
|
+
# #1862: passing `cfg.machines` folds quiet-hours windows into that same
|
|
127
|
+
# set (a no-op on any machine with no `quiet_hours:` block).
|
|
128
|
+
# #2101: a release cordon is IN `paused` (that is how every dispatcher
|
|
129
|
+
# honours it), so without the cordon map alongside it this line would
|
|
130
|
+
# render "PAUSED" for a machine nobody paused and no `coord unpause` will
|
|
131
|
+
# free — work stopping with no stated reason, which is the exact failure
|
|
132
|
+
# the cordon mechanism is supposed to stop repeating.
|
|
133
|
+
from coord.machine_pause import ( # noqa: PLC0415
|
|
134
|
+
cordons as fetch_cordons,
|
|
135
|
+
describe_pause_state,
|
|
136
|
+
paused_set,
|
|
137
|
+
)
|
|
138
|
+
paused = paused_set(cfg.machines)
|
|
139
|
+
cordons = fetch_cordons()
|
|
140
|
+
|
|
141
|
+
statuses = check_all(machines, timeout=timeout)
|
|
142
|
+
agent_completed: dict[str, dict] = {}
|
|
143
|
+
click.echo("Machines:")
|
|
144
|
+
for s in statuses:
|
|
145
|
+
m = s.machine
|
|
146
|
+
latency = f" ({s.latency_ms:.0f}ms)" if s.latency_ms is not None else ""
|
|
147
|
+
if s.is_online:
|
|
148
|
+
status_result = fetch_status(m, timeout=timeout)
|
|
149
|
+
if status_result.ok:
|
|
150
|
+
active = (status_result.data or {}).get("active", [])
|
|
151
|
+
if active:
|
|
152
|
+
a = active[0]
|
|
153
|
+
spec = a.get("spec", {})
|
|
154
|
+
spec_type = spec.get("type", "work")
|
|
155
|
+
badge_map = {"review": "[review] ", "smoke": "[smoke] ", "plan": "[plan] "}
|
|
156
|
+
badge = badge_map.get(spec_type, "")
|
|
157
|
+
target = spec.get("review_target")
|
|
158
|
+
if spec_type == "review" and target:
|
|
159
|
+
target_str = f" reviewing PR #{target}"
|
|
160
|
+
elif spec_type == "smoke" and target:
|
|
161
|
+
target_str = f" smoking branch `{target}`"
|
|
162
|
+
else:
|
|
163
|
+
target_str = ""
|
|
164
|
+
# #1707: the wire payload only carries `provider` when
|
|
165
|
+
# the resolved name differs from the implicit "claude"
|
|
166
|
+
# default (coord/dispatch.py's dispatch()), so this is
|
|
167
|
+
# absent for the common case and present exactly when a
|
|
168
|
+
# mixed fleet needs it surfaced.
|
|
169
|
+
provider_val = spec.get("provider")
|
|
170
|
+
provider_str = f" (provider={provider_val})" if provider_val else ""
|
|
171
|
+
detail = (
|
|
172
|
+
f"busy — {badge}#{spec.get('issue_number', '?')}: "
|
|
173
|
+
f"{spec.get('issue_title', '?')}{target_str}{provider_str}"
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
detail = "idle"
|
|
177
|
+
else:
|
|
178
|
+
active = []
|
|
179
|
+
detail = f"status unavailable ({status_result.error})"
|
|
180
|
+
if status_result.ok and status_result.data:
|
|
181
|
+
for entry in status_result.data.get("completed", []):
|
|
182
|
+
eid = entry.get("id") or entry.get("assignment_id")
|
|
183
|
+
if eid:
|
|
184
|
+
agent_completed[eid] = entry
|
|
185
|
+
label = f"{s.state} • {detail}{latency}"
|
|
186
|
+
else:
|
|
187
|
+
status_result = None
|
|
188
|
+
label = f"{s.state} — {s.reason}{latency}"
|
|
189
|
+
|
|
190
|
+
# #1563: surface pause state the same way regardless of whether
|
|
191
|
+
# `paused_set()` resolved it locally or from the daemon — a thin
|
|
192
|
+
# client that just ran `coord pause` needs to SEE that it took,
|
|
193
|
+
# otherwise a pause that silently didn't reach the daemon looks
|
|
194
|
+
# identical to one that did (the whole bug this closes).
|
|
195
|
+
# #1862: a quiet-hours pause must not look identical to a hand
|
|
196
|
+
# pause — an operator debugging a stalled queue at 1AM needs to
|
|
197
|
+
# know whether the machine will wake itself up or is waiting on
|
|
198
|
+
# `coord unpause`.
|
|
199
|
+
pause_state = describe_pause_state(m, paused, cordons=cordons)
|
|
200
|
+
if pause_state is not None and pause_state.kind == "cordon":
|
|
201
|
+
# #2101 trap E: name the version it is draining for, so a stopped
|
|
202
|
+
# machine reads as "the fleet is upgrading itself" rather than as
|
|
203
|
+
# a mystery.
|
|
204
|
+
label = f"{pause_state.detail.upper()} — {label}"
|
|
205
|
+
elif pause_state is not None and pause_state.kind == "hand":
|
|
206
|
+
label = f"PAUSED — {label}"
|
|
207
|
+
elif pause_state is not None and pause_state.kind == "quiet":
|
|
208
|
+
label = f"QUIET ({pause_state.detail}) — {label}"
|
|
209
|
+
elif pause_state is not None and pause_state.kind == "quiet_overridden":
|
|
210
|
+
label = f"{label} [quiet hours overridden]"
|
|
211
|
+
|
|
212
|
+
# Extract agent version from /status response (added in #104).
|
|
213
|
+
agent_version: str | None = None
|
|
214
|
+
if status_result and status_result.ok and status_result.data:
|
|
215
|
+
agent_version = status_result.data.get("version")
|
|
216
|
+
|
|
217
|
+
repos = ", ".join(m.repos) if m.repos else "(none)"
|
|
218
|
+
click.echo(f" {m.name:15s} [{label}]")
|
|
219
|
+
version_line = ""
|
|
220
|
+
if agent_version:
|
|
221
|
+
if agent_version != __version__:
|
|
222
|
+
version_line = f" agent-version: {agent_version} ⚠ (coord is {__version__})"
|
|
223
|
+
else:
|
|
224
|
+
version_line = f" agent-version: {agent_version}"
|
|
225
|
+
click.echo(f" host: {m.host} repos: {repos}{version_line}")
|
|
226
|
+
|
|
227
|
+
# #1886 Path B: `/health` exposes `installed_version` (a disk read
|
|
228
|
+
# that advances the instant `pip` writes to site-packages)
|
|
229
|
+
# separately from `version` (the running process's loaded module,
|
|
230
|
+
# fixed at import time — never advances without a restart). A
|
|
231
|
+
# process that never restarted after `coord agent update` — the
|
|
232
|
+
# execv-under-systemd stall, #404 — is otherwise invisible: `pip
|
|
233
|
+
# show` and this agent's own `version` field both "agree" with
|
|
234
|
+
# whatever was installed last, even though the code actually
|
|
235
|
+
# executing hasn't changed. Surfacing the drift here means it's
|
|
236
|
+
# visible on every `coord status`, not only when an update happens
|
|
237
|
+
# to be running.
|
|
238
|
+
if s.is_online and s.health:
|
|
239
|
+
installed_version = s.health.get("installed_version")
|
|
240
|
+
running_version = s.health.get("version")
|
|
241
|
+
if (
|
|
242
|
+
installed_version
|
|
243
|
+
and running_version
|
|
244
|
+
and installed_version != running_version
|
|
245
|
+
):
|
|
246
|
+
click.echo(
|
|
247
|
+
f" ⚠ running v{running_version} but installed "
|
|
248
|
+
f"v{installed_version} — process hasn't restarted since "
|
|
249
|
+
"its last update (`systemctl --user restart coord-agent` "
|
|
250
|
+
"on that machine, or `coord agent update` to retry)"
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# #1527: `/health`'s `degraded` dict names any configured repo whose
|
|
254
|
+
# `repo_path` is missing/unconfigured on this machine — the machine
|
|
255
|
+
# can still be green/idle above while every dispatch for that one
|
|
256
|
+
# repo silently 400s. Surface it here instead of leaving it only
|
|
257
|
+
# discoverable by sshing in and reading the filesystem.
|
|
258
|
+
degraded = (s.health or {}).get("degraded") if s.is_online else None
|
|
259
|
+
if degraded:
|
|
260
|
+
for repo_name, reason in degraded.items():
|
|
261
|
+
click.echo(f" ⚠ degraded: {repo_name} — {reason}")
|
|
262
|
+
|
|
263
|
+
if status_result and status_result.ok and status_result.data:
|
|
264
|
+
for entry in status_result.data.get("active", []):
|
|
265
|
+
progress = entry.get("progress")
|
|
266
|
+
if not progress:
|
|
267
|
+
continue
|
|
268
|
+
if progress.get("stuck"):
|
|
269
|
+
click.echo(f" !! STUCK: {progress['stuck']}")
|
|
270
|
+
for w in progress.get("warnings", []):
|
|
271
|
+
click.echo(f" !! {w}")
|
|
272
|
+
updates = progress.get("updates", [])
|
|
273
|
+
if updates:
|
|
274
|
+
click.echo(f" latest: {updates[-1]}")
|
|
275
|
+
|
|
276
|
+
# Reconcile board with live agent data
|
|
277
|
+
#
|
|
278
|
+
# #1631 (H-4): the fleet-health footer needs the SAME `fleet_health`
|
|
279
|
+
# block a thin client's `/board` GET already carries — fetching it a
|
|
280
|
+
# second time would double the board round-trip (this repo has hit
|
|
281
|
+
# multi-MB /board payloads before; see fleet_snapshot.py's own budget
|
|
282
|
+
# comment), so on a thin client this replaces the plain `read_board()`
|
|
283
|
+
# call with the raw-payload equivalent and pulls `fleet_health` off the
|
|
284
|
+
# SAME response instead of issuing a second GET. Host mode (no
|
|
285
|
+
# board_service) has no such payload to share, so it falls back to
|
|
286
|
+
# reassembling an equivalent block from the local DB.
|
|
287
|
+
if svc is not None:
|
|
288
|
+
from coord.client import board_from_payload, fetch_board_payload
|
|
289
|
+
|
|
290
|
+
board_payload = fetch_board_payload(svc)
|
|
291
|
+
board = board_from_payload(board_payload)
|
|
292
|
+
fleet_health_block = board_payload.get("fleet_health")
|
|
293
|
+
else:
|
|
294
|
+
from coord.health.aggregate import local_fleet_health_block
|
|
295
|
+
|
|
296
|
+
board = read_board()
|
|
297
|
+
fleet_health_block = local_fleet_health_block([m.name for m in cfg.machines])
|
|
298
|
+
if not no_reconcile and agent_completed:
|
|
299
|
+
# #749: write_board() routes to the daemon's /board upsert when a
|
|
300
|
+
# board service is configured, so a thin client's reconciliation now
|
|
301
|
+
# actually lands on the shared DB instead of being skipped entirely.
|
|
302
|
+
reconciled = 0
|
|
303
|
+
for a in board.active[:]:
|
|
304
|
+
if a.assignment_id is None:
|
|
305
|
+
continue
|
|
306
|
+
entry = agent_completed.get(a.assignment_id)
|
|
307
|
+
if entry is None:
|
|
308
|
+
continue
|
|
309
|
+
branch = entry.get("branch")
|
|
310
|
+
agent_status = entry.get("status")
|
|
311
|
+
if agent_status == "done":
|
|
312
|
+
done = board.mark_done_by_id(
|
|
313
|
+
a.assignment_id,
|
|
314
|
+
finished_at=entry.get("finished_at"),
|
|
315
|
+
branch=branch,
|
|
316
|
+
)
|
|
317
|
+
# #1566: mirror reconcile.py — a review agent reporting
|
|
318
|
+
# "done" has only finished the LLM session; the verdict is
|
|
319
|
+
# parsed + persisted by `coord notify`, a separate, slower
|
|
320
|
+
# step. Leaving status="done" here would show a finished
|
|
321
|
+
# review with no verdict, indistinguishable from a dropped
|
|
322
|
+
# one. "finalizing" isn't in drive_state.TERMINAL_STATUSES,
|
|
323
|
+
# so `coord drive` still correctly waits on it.
|
|
324
|
+
if done is not None and done.type == "review":
|
|
325
|
+
done.status = "finalizing"
|
|
326
|
+
elif agent_status == "advisory":
|
|
327
|
+
# #448: 0-commit clean exit — treat as done on the board so
|
|
328
|
+
# the assignment doesn't block; the advisory section below
|
|
329
|
+
# flags it for human attention. Mirror reconcile.py: set
|
|
330
|
+
# status="advisory" (mark_done_by_id leaves it as "done")
|
|
331
|
+
# and review_state="advisory" on work assignments so that
|
|
332
|
+
# the review-dispatch loop in coord notify skips them.
|
|
333
|
+
done = board.mark_done_by_id(
|
|
334
|
+
a.assignment_id,
|
|
335
|
+
finished_at=entry.get("finished_at"),
|
|
336
|
+
branch=branch,
|
|
337
|
+
)
|
|
338
|
+
if done is not None:
|
|
339
|
+
done.status = "advisory"
|
|
340
|
+
if done.type == "work":
|
|
341
|
+
done.review_state = "advisory"
|
|
342
|
+
else:
|
|
343
|
+
board.mark_failed_by_id(
|
|
344
|
+
a.assignment_id,
|
|
345
|
+
finished_at=entry.get("finished_at"),
|
|
346
|
+
)
|
|
347
|
+
# #1461: stamp a usage-limit-kill diagnostic (if the agent
|
|
348
|
+
# flagged one — AgentServer._reap, agent.py) onto the persisted
|
|
349
|
+
# `failure_reason` column. Routed through the already-daemon-aware
|
|
350
|
+
# `set_assignment_failure_reason` (#618) rather than a raw
|
|
351
|
+
# get_connection() write — `coord status` is thin-client
|
|
352
|
+
# reachable, and a raw local write would silently land on the
|
|
353
|
+
# thin client's empty local DB instead of the daemon's (the same
|
|
354
|
+
# #906 audit gap `get_issue_test_mode` was fixed for). This also
|
|
355
|
+
# normalises status to 'failed' even when the branch above set
|
|
356
|
+
# 'advisory' — a usage-limit kill is the one terminal state known
|
|
357
|
+
# safe to re-dispatch unchanged (drive.py's FAILED bucket),
|
|
358
|
+
# mirrors coord.reconcile._record_usage_limit_reason exactly.
|
|
359
|
+
usage_limit_reason = entry.get("usage_limit_reason")
|
|
360
|
+
if usage_limit_reason and a.assignment_id:
|
|
361
|
+
try:
|
|
362
|
+
from coord.state import set_assignment_failure_reason
|
|
363
|
+
|
|
364
|
+
set_assignment_failure_reason(a.assignment_id, usage_limit_reason)
|
|
365
|
+
except Exception: # noqa: BLE001 — diagnostic only
|
|
366
|
+
pass
|
|
367
|
+
reconciled += 1
|
|
368
|
+
if reconciled:
|
|
369
|
+
write_board(board)
|
|
370
|
+
click.echo(f"\n (reconciled {reconciled} assignment(s) from live agent data)")
|
|
371
|
+
|
|
372
|
+
# #1461: surface usage-limit kills as a distinct fleet-level condition —
|
|
373
|
+
# a known-safe-to-retry-once-reset wait, not a defect. Shown ahead of (and
|
|
374
|
+
# excluded from) the Advisory/plain-failure buckets below so a confusing
|
|
375
|
+
# evening reads as "3 killed by the usage limit, resets 8:30pm" instead of
|
|
376
|
+
# N unrelated-looking advisory/failed rows.
|
|
377
|
+
usage_limit_entries = [
|
|
378
|
+
e for e in agent_completed.values()
|
|
379
|
+
if e.get("usage_limit_reason")
|
|
380
|
+
]
|
|
381
|
+
if usage_limit_entries:
|
|
382
|
+
click.echo("")
|
|
383
|
+
click.echo(
|
|
384
|
+
"⏳ Usage limit (worker killed by the account's usage limit — "
|
|
385
|
+
"safe to retry unchanged once reset):"
|
|
386
|
+
)
|
|
387
|
+
for e in usage_limit_entries:
|
|
388
|
+
spec = e.get("spec", {})
|
|
389
|
+
click.echo(
|
|
390
|
+
f" #{spec.get('issue_number', '?')}: "
|
|
391
|
+
f"{spec.get('issue_title', '?')} "
|
|
392
|
+
f"[{spec.get('repo_name', '?')}] — {e['usage_limit_reason']}"
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
# #448: surface advisory assignments (0 commits, clean exit) so the
|
|
396
|
+
# operator knows they need attention without having to dig into logs.
|
|
397
|
+
# Usage-limit kills are excluded — surfaced separately above, since they
|
|
398
|
+
# are a wait condition rather than something needing human attention.
|
|
399
|
+
#
|
|
400
|
+
# #1472: an advisory entry is agent-local state — it lives in the
|
|
401
|
+
# worker's own completed-assignment map (`_COMPLETED_HISTORY_CAP` prunes
|
|
402
|
+
# it by *count*, not by GitHub outcome) — so it keeps being re-served
|
|
403
|
+
# here forever, even after the issue closes or the branch merges out
|
|
404
|
+
# from under it. Re-check terminal state on every render rather than
|
|
405
|
+
# trusting the agent to have cleared it.
|
|
406
|
+
#
|
|
407
|
+
# Both filters apply: #1461 drops usage-limit kills (a wait condition,
|
|
408
|
+
# shown above) and #1472 drops work that has since gone terminal.
|
|
409
|
+
advisory_entries = _live_advisory_entries(
|
|
410
|
+
[
|
|
411
|
+
e for e in agent_completed.values()
|
|
412
|
+
if e.get("status") == "advisory" and not e.get("usage_limit_reason")
|
|
413
|
+
],
|
|
414
|
+
cfg,
|
|
415
|
+
)
|
|
416
|
+
if advisory_entries:
|
|
417
|
+
click.echo("")
|
|
418
|
+
click.echo("⚠ Advisory (needs attention — worker exited cleanly with 0 commits):")
|
|
419
|
+
for e in advisory_entries:
|
|
420
|
+
spec = e.get("spec", {})
|
|
421
|
+
reason = e.get("zero_commit_reason") or "0 commits pushed"
|
|
422
|
+
click.echo(
|
|
423
|
+
f" #{spec.get('issue_number', '?')}: "
|
|
424
|
+
f"{spec.get('issue_title', '?')} "
|
|
425
|
+
f"[{spec.get('repo_name', '?')}] — {reason}"
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
blocked = compute_blocked(cfg.repos, board.active)
|
|
429
|
+
if blocked:
|
|
430
|
+
click.echo("")
|
|
431
|
+
click.echo("Blocked repos:")
|
|
432
|
+
for repo_name, reasons in blocked.items():
|
|
433
|
+
click.echo(f" {repo_name}:")
|
|
434
|
+
for reason in reasons:
|
|
435
|
+
click.echo(f" - {reason}")
|
|
436
|
+
|
|
437
|
+
if freshness:
|
|
438
|
+
click.echo("")
|
|
439
|
+
click.echo("Repo freshness:")
|
|
440
|
+
github_heads: dict[str, str | None] = {}
|
|
441
|
+
for repo_cfg in cfg.repos:
|
|
442
|
+
try:
|
|
443
|
+
github_heads[repo_cfg.name] = github_ops.get_default_branch_head(
|
|
444
|
+
repo_cfg.github, repo_cfg.default_branch
|
|
445
|
+
)
|
|
446
|
+
except RuntimeError as e:
|
|
447
|
+
github_heads[repo_cfg.name] = None
|
|
448
|
+
click.echo(f" (github HEAD lookup failed for {repo_cfg.name}: {e})", err=True)
|
|
449
|
+
for s in statuses:
|
|
450
|
+
if not s.is_online:
|
|
451
|
+
click.echo(f" {s.machine.name}: (offline, skipping)")
|
|
452
|
+
continue
|
|
453
|
+
agent_repos = fetch_repos(s.machine, timeout=timeout) or {}
|
|
454
|
+
click.echo(f" {s.machine.name}:")
|
|
455
|
+
for repo_name in s.machine.repos:
|
|
456
|
+
rf = fresh.compare(repo_name, agent_repos.get(repo_name), github_heads.get(repo_name))
|
|
457
|
+
local = (rf.local_sha or "?")[:7]
|
|
458
|
+
remote = (rf.remote_sha or "?")[:7]
|
|
459
|
+
tag = f"[{rf.state}]"
|
|
460
|
+
detail = f"local {local} remote {remote}"
|
|
461
|
+
if rf.dirty:
|
|
462
|
+
detail += " (dirty)"
|
|
463
|
+
if rf.error:
|
|
464
|
+
detail += f" — {rf.error}"
|
|
465
|
+
click.echo(f" {repo_name:20s} {tag:10s} {detail}")
|
|
466
|
+
|
|
467
|
+
# Merge queue
|
|
468
|
+
from coord import merge_queue as mq
|
|
469
|
+
|
|
470
|
+
# #584: merge_queue lives in the (host-local) DB; skip it for a thin client.
|
|
471
|
+
queue = [] if svc else mq.load_queue()
|
|
472
|
+
by_repo = mq.pending_summary(queue) if queue else {}
|
|
473
|
+
if by_repo:
|
|
474
|
+
click.echo("")
|
|
475
|
+
click.echo("Merge queue:")
|
|
476
|
+
for repo_name, entries in sorted(by_repo.items()):
|
|
477
|
+
click.echo(f" {repo_name}:")
|
|
478
|
+
for e in entries:
|
|
479
|
+
size = f"+{e.size}" if e.size is not None else "?"
|
|
480
|
+
pr = f"PR #{e.pr_number}" if e.pr_number else "no PR yet"
|
|
481
|
+
tag = f"[{e.state}]"
|
|
482
|
+
line = f" {tag:11s} #{e.issue_number} ({e.branch} → {e.target_branch}) {pr} size={size}"
|
|
483
|
+
click.echo(line)
|
|
484
|
+
# #420: recompute the review/smoke gate error live rather than
|
|
485
|
+
# echoing the stored string verbatim — it's only refreshed on
|
|
486
|
+
# a real merge attempt, so an approval/verdict that landed
|
|
487
|
+
# since then would otherwise show as stale as "blocked".
|
|
488
|
+
live_error = mq.display_error(e, board, cfg)
|
|
489
|
+
if live_error:
|
|
490
|
+
click.echo(f" error: {live_error}")
|
|
491
|
+
|
|
492
|
+
# #920: sibling-overlap warnings — approved (PENDING) queue entries that
|
|
493
|
+
# touch overlapping files and have been aging. Mirrors the merge-queue
|
|
494
|
+
# skip above: the queue is host-local, so this is a no-op on a thin
|
|
495
|
+
# client (use `coord merge --plan`, which fetches the daemon-computed
|
|
496
|
+
# equivalent via /board, from a thin client instead).
|
|
497
|
+
if not svc:
|
|
498
|
+
from coord.commands.merge import _print_sibling_overlap_warnings
|
|
499
|
+
|
|
500
|
+
overlaps = mq.find_sibling_overlaps(board, cfg)
|
|
501
|
+
_print_sibling_overlap_warnings(overlaps)
|
|
502
|
+
|
|
503
|
+
# Auto-loop iteration-cap blockers: assignments where the review→fix loop
|
|
504
|
+
# exhausted all allowed iterations without receiving an approval. These
|
|
505
|
+
# require manual intervention (bump pipeline.max_review_iterations or
|
|
506
|
+
# dispatch a fix with `coord assign`) and are shown prominently so the
|
|
507
|
+
# operator notices them on the first `coord status` after the cap fires.
|
|
508
|
+
cap_hit_blocked = [
|
|
509
|
+
a for a in board.completed
|
|
510
|
+
if a.type == "work" and a.review_state == "cap_hit"
|
|
511
|
+
]
|
|
512
|
+
if cap_hit_blocked:
|
|
513
|
+
click.echo("")
|
|
514
|
+
click.echo("⚠ Auto-loop blockers (manual action required):")
|
|
515
|
+
for a in cap_hit_blocked:
|
|
516
|
+
click.echo(
|
|
517
|
+
f" #{a.issue_number}: {a.issue_title} ({a.repo_name})"
|
|
518
|
+
f" [iteration cap hit]"
|
|
519
|
+
)
|
|
520
|
+
click.echo(
|
|
521
|
+
f" Options: bump pipeline.max_review_iterations in coordinator.yml"
|
|
522
|
+
f" or 'coord assign' to dispatch a fix manually,"
|
|
523
|
+
f" or 'coord merge --force-merge' to merge as-is."
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
# #586: branch-not-on-remote blockers — work that completed but the branch
|
|
527
|
+
# was never pushed. Downstream review/fix dispatch is blocked until the
|
|
528
|
+
# operator pushes from the original worker machine.
|
|
529
|
+
branch_not_pushed = [
|
|
530
|
+
a for a in board.completed
|
|
531
|
+
if a.type == "work" and a.review_state == "branch_not_on_remote"
|
|
532
|
+
]
|
|
533
|
+
if branch_not_pushed:
|
|
534
|
+
click.echo("")
|
|
535
|
+
click.echo("⚠ Push required (review blocked — branch not on remote):")
|
|
536
|
+
for a in branch_not_pushed:
|
|
537
|
+
click.echo(
|
|
538
|
+
f" #{a.issue_number}: {a.issue_title} ({a.repo_name})"
|
|
539
|
+
f" [branch not on remote]"
|
|
540
|
+
)
|
|
541
|
+
click.echo(
|
|
542
|
+
f" Branch '{a.branch}' exists only on {a.machine_name}."
|
|
543
|
+
f" Push it with: ssh {a.machine_name} 'cd <repo-path> && git push origin {a.branch}'"
|
|
544
|
+
f" then re-run 'coord notify' to retry review dispatch."
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
# #904: no-eligible-reviewer blockers — every configured candidate machine
|
|
548
|
+
# definitively rejected the review dispatch (drifted coordinator.yml vs.
|
|
549
|
+
# an agent's actual `/health` repos list, most commonly). Mirrors the
|
|
550
|
+
# branch-not-on-remote block above so this stall is operator-visible
|
|
551
|
+
# instead of only a log.error() line.
|
|
552
|
+
no_eligible_reviewer = [
|
|
553
|
+
a for a in board.completed
|
|
554
|
+
if a.type == "work" and a.review_state == "no_eligible_reviewer"
|
|
555
|
+
]
|
|
556
|
+
if no_eligible_reviewer:
|
|
557
|
+
click.echo("")
|
|
558
|
+
click.echo("⚠ No reviewer available (review blocked — all candidates rejected):")
|
|
559
|
+
for a in no_eligible_reviewer:
|
|
560
|
+
click.echo(
|
|
561
|
+
f" #{a.issue_number}: {a.issue_title} ({a.repo_name})"
|
|
562
|
+
f" [no eligible reviewer]"
|
|
563
|
+
)
|
|
564
|
+
click.echo(
|
|
565
|
+
" Every configured machine for this repo rejected the dispatch."
|
|
566
|
+
" Check that each agent's /health 'repos' list matches coordinator.yml,"
|
|
567
|
+
" then re-run 'coord notify' to retry review dispatch."
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
# Show completed work assignments with review lifecycle state.
|
|
571
|
+
_REVIEW_STATE_TAGS = {
|
|
572
|
+
"pending": "[awaiting review]",
|
|
573
|
+
"dispatched": "[review dispatched]",
|
|
574
|
+
"done": "[review done]",
|
|
575
|
+
"cap_hit": "[⚠ iteration cap hit — manual action required]",
|
|
576
|
+
"branch_not_on_remote": "[⚠ branch not on remote — push required]",
|
|
577
|
+
"no_eligible_reviewer": "[⚠ no reviewer available — check agent /health vs coordinator.yml]",
|
|
578
|
+
# #1534: the branch carries no commits over its base, so there is
|
|
579
|
+
# nothing to review. Almost always means the work assignment produced
|
|
580
|
+
# nothing (e.g. a usage-limit kill) and should be re-dispatched.
|
|
581
|
+
"zero_commits": "[⚠ branch has 0 commits — nothing to review, re-dispatch the work]",
|
|
582
|
+
}
|
|
583
|
+
work_completed = [a for a in board.completed if a.type == "work"]
|
|
584
|
+
if work_completed:
|
|
585
|
+
by_time = sorted(work_completed, key=lambda a: a.finished_at or 0, reverse=True)[:10]
|
|
586
|
+
click.echo("")
|
|
587
|
+
click.echo("Completed work assignments:")
|
|
588
|
+
for a in by_time:
|
|
589
|
+
# test_state="failed" takes priority: the review gate is correctly
|
|
590
|
+
# held but "[awaiting review]" would mislead the operator into
|
|
591
|
+
# thinking the item is queued to move forward (real incident: #1116).
|
|
592
|
+
if getattr(a, "test_state", None) == "failed":
|
|
593
|
+
rs_tag = "[✗ test FAILED — needs fix]"
|
|
594
|
+
else:
|
|
595
|
+
rs_tag = _REVIEW_STATE_TAGS.get(a.review_state or "", "")
|
|
596
|
+
rs_suffix = f" {rs_tag}" if rs_tag else ""
|
|
597
|
+
# #1707: surface the resolved provider (already persisted at
|
|
598
|
+
# dispatch time, coord/models.py Assignment.provider_name) so a
|
|
599
|
+
# mixed claude/opencode fleet is legible here — not just in
|
|
600
|
+
# `coord gates`. Omit the plain "claude" case (the overwhelming
|
|
601
|
+
# majority of rows) so the common path stays uncluttered; only a
|
|
602
|
+
# non-default backend earns the tag.
|
|
603
|
+
provider_tag = (
|
|
604
|
+
f" [provider={a.provider_name}]"
|
|
605
|
+
if a.provider_name and a.provider_name != "claude"
|
|
606
|
+
else ""
|
|
607
|
+
)
|
|
608
|
+
click.echo(
|
|
609
|
+
f" #{a.issue_number}: {a.issue_title} ({a.repo_name})"
|
|
610
|
+
f"{provider_tag}{rs_suffix}"
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
notified = {} if svc else load_notified()
|
|
614
|
+
if notified:
|
|
615
|
+
dispatched_by_id = {r["assignment_id"]: r for r in load_dispatched()}
|
|
616
|
+
items = sorted(notified.items(), key=lambda kv: kv[1].get("posted_at", 0), reverse=True)[:5]
|
|
617
|
+
click.echo("")
|
|
618
|
+
click.echo("Recent issue comment activity:")
|
|
619
|
+
for aid, info in items:
|
|
620
|
+
record = dispatched_by_id.get(aid, {})
|
|
621
|
+
repo = record.get("repo_github", "?")
|
|
622
|
+
issue = record.get("issue_number", "?")
|
|
623
|
+
click.echo(f" [{info['event']}] {repo}#{issue} (assignment {aid})")
|
|
624
|
+
|
|
625
|
+
# Burn-rate warning: show a one-liner when spend rate is high.
|
|
626
|
+
try:
|
|
627
|
+
from coord.state import load_session
|
|
628
|
+
from coord.usage import build_session_usage, format_burn_rate_line
|
|
629
|
+
import datetime
|
|
630
|
+
|
|
631
|
+
sess = None if svc else load_session()
|
|
632
|
+
started_at: float | None = None
|
|
633
|
+
if sess and sess.get("started_at"):
|
|
634
|
+
try:
|
|
635
|
+
dt = datetime.datetime.fromisoformat(
|
|
636
|
+
sess["started_at"].rstrip("Z").replace("Z", "+00:00")
|
|
637
|
+
)
|
|
638
|
+
started_at = dt.replace(tzinfo=datetime.timezone.utc).timestamp()
|
|
639
|
+
except (ValueError, AttributeError):
|
|
640
|
+
pass
|
|
641
|
+
|
|
642
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
643
|
+
session_usage = build_session_usage(all_assignments, started_at=started_at)
|
|
644
|
+
burn_line = format_burn_rate_line(session_usage)
|
|
645
|
+
if burn_line:
|
|
646
|
+
click.echo("")
|
|
647
|
+
click.echo(burn_line)
|
|
648
|
+
except (ImportError, OSError, ValueError, KeyError):
|
|
649
|
+
pass # Never let usage tracking break the status command.
|
|
650
|
+
|
|
651
|
+
# #1631 (H-4): the always-visible fleet-health footer. Printed
|
|
652
|
+
# unconditionally, every run — including the all-OK case ("OK states its
|
|
653
|
+
# OK-ness rather than printing nothing": a check nobody ever sees run is
|
|
654
|
+
# indistinguishable from a check that's silently broken, the exact
|
|
655
|
+
# failure mode #1631 exists to close). Aggregation itself lives in
|
|
656
|
+
# coord.health.aggregate — this command only renders it.
|
|
657
|
+
try:
|
|
658
|
+
from coord.health.aggregate import render_fleet_footer, summarize_fleet_health
|
|
659
|
+
|
|
660
|
+
click.echo("")
|
|
661
|
+
click.echo(render_fleet_footer(summarize_fleet_health(fleet_health_block)))
|
|
662
|
+
except Exception: # noqa: BLE001 — the footer must never break `coord status`
|
|
663
|
+
click.echo("")
|
|
664
|
+
click.echo("FLEET: ? (health footer unavailable — coord health for detail)")
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def _health_vs_config_lines(machine, health: dict) -> list[tuple[bool, str]]:
|
|
668
|
+
"""Cross-check a machine's ``/health`` against what ``coordinator.yml``
|
|
669
|
+
declares for it. Returns ``(is_problem, line)`` pairs (#1712).
|
|
670
|
+
|
|
671
|
+
A machine that publishes ``capabilities: []`` while the config declares
|
|
672
|
+
capabilities for it is a **misconfiguration**, not an absence — and the two
|
|
673
|
+
are indistinguishable from ``/health`` alone, which is exactly how #1673
|
|
674
|
+
stayed "unexplained" for so long: precision was silently ineligible for
|
|
675
|
+
every ``rust``/``python``/``gtk`` dispatch and nothing anywhere said so.
|
|
676
|
+
Same shape for ``repos: []`` (#1485's review-router misread).
|
|
677
|
+
|
|
678
|
+
#1801: that inference only holds for a **standing** agent — one with its
|
|
679
|
+
own ``coordinator.yml`` that is expected to publish from it. A
|
|
680
|
+
**config-free** agent (``health["config_free"]`` set) is DESIGNED to
|
|
681
|
+
publish empty capabilities/repos: the coordinator supplies both at
|
|
682
|
+
dispatch time instead. Before this fix, a config-free agent whose
|
|
683
|
+
machine entry in the coordinator's OWN ``coordinator.yml`` happened to
|
|
684
|
+
declare capabilities/repos (azure-epic1709: ``['rust', 'python']`` /
|
|
685
|
+
``['claude-coordinator']``) still hit the CRIT branch below — and the
|
|
686
|
+
CRIT's own detail line ("the agent is running config-free") flatly
|
|
687
|
+
contradicted the CRIT's headline ("every dispatch ... will be refused"),
|
|
688
|
+
while dispatch in fact worked. A config-free mismatch is now reported at
|
|
689
|
+
WARN (not a problem), while a *configured* agent publishing nothing
|
|
690
|
+
still CRITs — that's the #1485/#1712 case this check exists for.
|
|
691
|
+
|
|
692
|
+
Pure function — no I/O — so it's testable without a live fleet.
|
|
693
|
+
"""
|
|
694
|
+
out: list[tuple[bool, str]] = []
|
|
695
|
+
# None on the normal path; a reason string when the agent came up with no
|
|
696
|
+
# config at all (#1712). Absent entirely on agents predating #1712.
|
|
697
|
+
config_free = health.get("config_free")
|
|
698
|
+
|
|
699
|
+
declared_caps = list(getattr(machine, "capabilities", None) or [])
|
|
700
|
+
published_caps = list(health.get("capabilities") or [])
|
|
701
|
+
declared_repos = list(getattr(machine, "repos", None) or [])
|
|
702
|
+
published_repos = list(health.get("repos") or [])
|
|
703
|
+
degraded = health.get("degraded") or {}
|
|
704
|
+
|
|
705
|
+
if config_free and not declared_caps and not declared_repos:
|
|
706
|
+
# Legitimately config-free (ephemeral worker) AND the config agrees
|
|
707
|
+
# there's nothing to declare — worth surfacing, but not a failure.
|
|
708
|
+
out.append((False, f" ⚠ running config-free — {config_free}"))
|
|
709
|
+
|
|
710
|
+
if declared_caps and not published_caps:
|
|
711
|
+
if config_free:
|
|
712
|
+
# #1801: expected shape for a config-free agent — the coordinator
|
|
713
|
+
# supplies capabilities at dispatch time, not this machine's own
|
|
714
|
+
# config, so an empty /health is not a dispatch blocker.
|
|
715
|
+
out.append((
|
|
716
|
+
False,
|
|
717
|
+
f" ⚠ capabilities: coordinator.yml declares {declared_caps} "
|
|
718
|
+
"but /health publishes none — the agent is running "
|
|
719
|
+
f"config-free ({config_free}); capabilities come from the "
|
|
720
|
+
"coordinator at dispatch time, not this machine's own "
|
|
721
|
+
"config, so this is expected, not a dispatch blocker (#1801)",
|
|
722
|
+
))
|
|
723
|
+
else:
|
|
724
|
+
out.append((
|
|
725
|
+
True,
|
|
726
|
+
f" ✗ CRIT capabilities: coordinator.yml declares "
|
|
727
|
+
f"{declared_caps} but /health publishes none — this machine "
|
|
728
|
+
"is silently ineligible for every capability-matched "
|
|
729
|
+
"dispatch (#1712)",
|
|
730
|
+
))
|
|
731
|
+
out.append((
|
|
732
|
+
True,
|
|
733
|
+
" the agent published no capabilities despite a "
|
|
734
|
+
"loadable config: check that its unit's --machine names this "
|
|
735
|
+
"machine, then `coord agent update` + restart coord-agent",
|
|
736
|
+
))
|
|
737
|
+
|
|
738
|
+
if declared_repos and not published_repos:
|
|
739
|
+
if config_free:
|
|
740
|
+
out.append((
|
|
741
|
+
False,
|
|
742
|
+
f" ⚠ repos: coordinator.yml declares {declared_repos} but "
|
|
743
|
+
"/health advertises none — the agent is running config-free "
|
|
744
|
+
f"({config_free}); repos come from the coordinator at "
|
|
745
|
+
"dispatch time, not this machine's own config, so this is "
|
|
746
|
+
"expected, not a dispatch blocker (#1801)",
|
|
747
|
+
))
|
|
748
|
+
else:
|
|
749
|
+
out.append((
|
|
750
|
+
True,
|
|
751
|
+
f" ✗ CRIT repos: coordinator.yml declares {declared_repos} "
|
|
752
|
+
"but /health advertises none — every dispatch to this "
|
|
753
|
+
"machine will be refused, and any reader trusting /health "
|
|
754
|
+
"sees a repo-less machine (#1485/#1712)",
|
|
755
|
+
))
|
|
756
|
+
for repo, reason in sorted(degraded.items()):
|
|
757
|
+
out.append((True, f" {repo}: {reason}"))
|
|
758
|
+
|
|
759
|
+
return out
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def _unit_drift_lines(health: dict) -> list[tuple[bool, str]]:
|
|
763
|
+
"""Render a machine's ``unit_drift`` H-1 results (``coord/health/checks/
|
|
764
|
+
unit_drift.py``, #1831) as ``coord doctor`` lines.
|
|
765
|
+
|
|
766
|
+
``/health``'s ``health`` block already carries every machine/checkout
|
|
767
|
+
check this agent ran (``_cached_local_health`` in ``coord/agent.py``) —
|
|
768
|
+
this just projects the one check id ``coord doctor`` cares about into its
|
|
769
|
+
own report, the same way the tool_versions section below projects
|
|
770
|
+
``health["tool_versions"]``. An agent predating #1831 simply has no
|
|
771
|
+
``unit_drift`` entries here, so this renders nothing for it — never a
|
|
772
|
+
false "clean".
|
|
773
|
+
|
|
774
|
+
Pure function — no I/O — so it's testable without a live fleet.
|
|
775
|
+
"""
|
|
776
|
+
out: list[tuple[bool, str]] = []
|
|
777
|
+
results = ((health.get("health") or {}).get("results") or [])
|
|
778
|
+
for r in results:
|
|
779
|
+
if r.get("check_id") != "unit_drift":
|
|
780
|
+
continue
|
|
781
|
+
severity = r.get("severity")
|
|
782
|
+
subject = r.get("subject")
|
|
783
|
+
headroom = r.get("headroom", "")
|
|
784
|
+
label = f" {subject}" if subject else ""
|
|
785
|
+
if severity == "crit":
|
|
786
|
+
out.append((True, f" ✗ CRIT unit drift{label}: {headroom}"))
|
|
787
|
+
detail = r.get("detail")
|
|
788
|
+
if detail:
|
|
789
|
+
out.append((True, f" {detail}"))
|
|
790
|
+
elif severity == "warn":
|
|
791
|
+
out.append((True, f" ⚠ unit drift{label}: {headroom}"))
|
|
792
|
+
detail = r.get("detail")
|
|
793
|
+
if detail:
|
|
794
|
+
out.append((True, f" fix: {detail}"))
|
|
795
|
+
elif severity == "unknown":
|
|
796
|
+
out.append((False, f" ? unit drift{label}: {headroom}"))
|
|
797
|
+
return out
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _unit_enablement_lines(health: dict) -> list[tuple[bool, str]]:
|
|
801
|
+
"""Render a machine's ``unit_enablement`` H-1 results (``coord/health/
|
|
802
|
+
checks/unit_enablement.py``, #2098) as ``coord doctor`` lines.
|
|
803
|
+
|
|
804
|
+
Mirrors ``_unit_drift_lines`` immediately above: ``unit_drift`` answers
|
|
805
|
+
"does an installed unit's content match ``deploy/``", this answers "is
|
|
806
|
+
an installed, manifest-listed unit actually ``systemctl --user
|
|
807
|
+
enable``d" — the state that hid ``coord-release-propagate.timer`` for a
|
|
808
|
+
day, because a disabled timer and a deferring one produce identical
|
|
809
|
+
evidence otherwise. Before this renderer existed, that WARN only
|
|
810
|
+
surfaced in the per-machine aggregate ``severity`` (the "FLEET: WARN"
|
|
811
|
+
footer / coord-tui indicator) and in ``coord health``'s own per-unit
|
|
812
|
+
detail — an operator reading ``coord doctor``'s printed report, which
|
|
813
|
+
``docs/AGENT_OPERATIONS.md`` explicitly points to for this, saw nothing
|
|
814
|
+
naming which unit was disabled or how to fix it.
|
|
815
|
+
|
|
816
|
+
An agent predating #2098 simply has no ``unit_enablement`` entries here,
|
|
817
|
+
so this renders nothing for it — never a false "clean".
|
|
818
|
+
|
|
819
|
+
Pure function — no I/O — so it's testable without a live fleet.
|
|
820
|
+
"""
|
|
821
|
+
out: list[tuple[bool, str]] = []
|
|
822
|
+
results = ((health.get("health") or {}).get("results") or [])
|
|
823
|
+
for r in results:
|
|
824
|
+
if r.get("check_id") != "unit_enablement":
|
|
825
|
+
continue
|
|
826
|
+
severity = r.get("severity")
|
|
827
|
+
subject = r.get("subject")
|
|
828
|
+
headroom = r.get("headroom", "")
|
|
829
|
+
label = f" {subject}" if subject else ""
|
|
830
|
+
if severity == "warn":
|
|
831
|
+
out.append((True, f" ⚠ unit enablement{label}: {headroom}"))
|
|
832
|
+
detail = r.get("detail")
|
|
833
|
+
if detail:
|
|
834
|
+
out.append((True, f" fix: {detail}"))
|
|
835
|
+
elif severity == "unknown":
|
|
836
|
+
out.append((False, f" ? unit enablement{label}: {headroom}"))
|
|
837
|
+
return out
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _dispatch_blocker_lines_for_config_free(machine, cfg) -> list[tuple[bool, str]]:
|
|
841
|
+
"""Real dispatch blockers on a **config-free** agent's machine (#1801).
|
|
842
|
+
|
|
843
|
+
The #1712 capabilities/repos cross-check above CRIT'd azure-epic1709 for
|
|
844
|
+
the wrong reason (an expected config-free shape) while staying silent
|
|
845
|
+
about the two things that actually *would* refuse dispatch there: a
|
|
846
|
+
declared repo with no ``repo_paths`` entry (``coord.dispatch.dispatch``
|
|
847
|
+
raises ``ValueError`` on exactly this), and a declared repo whose
|
|
848
|
+
resolved provider (``Repo.provider`` > ``providers.default``) the
|
|
849
|
+
machine hasn't declared support for via ``provider:<type>`` (the #1711
|
|
850
|
+
structural gate, ``coord.providers.guard_provider_machine_capability``).
|
|
851
|
+
|
|
852
|
+
Both are derivable from ``coordinator.yml`` alone — no ``/health``
|
|
853
|
+
round-trip needed — so this runs regardless of whether the machine is
|
|
854
|
+
currently reachable. Scoped to config-free machines because that's the
|
|
855
|
+
gap #1801 found; a standing machine's own config predates this check and
|
|
856
|
+
is out of scope here.
|
|
857
|
+
|
|
858
|
+
Pure function — no I/O — so it's testable without a live fleet.
|
|
859
|
+
"""
|
|
860
|
+
from coord.config import provider_capability
|
|
861
|
+
from coord.providers import (
|
|
862
|
+
machine_supports_provider,
|
|
863
|
+
provider_type_for,
|
|
864
|
+
resolve_provider_name,
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
out: list[tuple[bool, str]] = []
|
|
868
|
+
|
|
869
|
+
missing_paths = sorted(r for r in machine.repos if not machine.repo_path(r))
|
|
870
|
+
if missing_paths:
|
|
871
|
+
out.append((
|
|
872
|
+
True,
|
|
873
|
+
f" ✗ CRIT repo_paths: {missing_paths} declared under repos but "
|
|
874
|
+
"have no repo_paths entry in coordinator.yml — dispatch to this "
|
|
875
|
+
"machine for these repos will be refused (#1801)",
|
|
876
|
+
))
|
|
877
|
+
|
|
878
|
+
missing_caps: list[str] = []
|
|
879
|
+
for repo_name in machine.repos:
|
|
880
|
+
repo = cfg.repo(repo_name)
|
|
881
|
+
repo_provider = repo.provider if repo is not None else None
|
|
882
|
+
provider_name = resolve_provider_name(None, repo_provider, cfg.providers)
|
|
883
|
+
if not machine_supports_provider(machine, provider_name, cfg.providers):
|
|
884
|
+
ptype = provider_type_for(provider_name, cfg.providers)
|
|
885
|
+
cap = provider_capability(ptype)
|
|
886
|
+
if cap not in missing_caps:
|
|
887
|
+
missing_caps.append(cap)
|
|
888
|
+
if missing_caps:
|
|
889
|
+
out.append((
|
|
890
|
+
True,
|
|
891
|
+
" ✗ CRIT provider capability: this machine's declared repos "
|
|
892
|
+
f"expect {sorted(missing_caps)} but coordinator.yml capabilities "
|
|
893
|
+
"don't include it — dispatch with that provider will be refused "
|
|
894
|
+
"(#1711/#1801)",
|
|
895
|
+
))
|
|
896
|
+
|
|
897
|
+
return out
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _release_lag_lines(report) -> list[tuple[bool, str]]:
|
|
901
|
+
"""Render a :class:`coord.release_verify.VerifyReport`'s findings as
|
|
902
|
+
``coord doctor`` lines (#2082).
|
|
903
|
+
|
|
904
|
+
#2082's whole complaint: ``coord release verify`` already computes
|
|
905
|
+
whether the fleet's running version matches the released one, and
|
|
906
|
+
already returns CRIT when it doesn't — nothing *routine* ever called
|
|
907
|
+
it, so a fleet could sit eleven releases behind PyPI with every other
|
|
908
|
+
readout silent. This projects that SAME computation (not a second one —
|
|
909
|
+
see the epic's "two surfaces, one function" rule) into doctor's output,
|
|
910
|
+
the same way :func:`_unit_drift_lines` above projects ``unit_drift``.
|
|
911
|
+
|
|
912
|
+
Filters out ``unit `` findings: those are ``unit_drift`` results folded
|
|
913
|
+
into ``coord release verify`` (#1834), and doctor already renders that
|
|
914
|
+
same data via :func:`_unit_drift_lines` — showing it twice would be
|
|
915
|
+
noise, not a second opinion. UNKNOWN findings (an unreachable host, a
|
|
916
|
+
lane with no data yet, "no expected version to grade against") are
|
|
917
|
+
likewise not surfaced here: they never made a problem elsewhere in this
|
|
918
|
+
command either, and their causes are already visible above (the
|
|
919
|
+
"unreachable" line, tool_versions gaps).
|
|
920
|
+
"""
|
|
921
|
+
out: list[tuple[bool, str]] = []
|
|
922
|
+
for f in report.findings:
|
|
923
|
+
if f.severity not in ("crit", "warn"):
|
|
924
|
+
continue
|
|
925
|
+
if f.lane.startswith("unit "):
|
|
926
|
+
continue
|
|
927
|
+
mark = "✗ CRIT" if f.severity == "crit" else "⚠ WARN"
|
|
928
|
+
out.append((True, f" {mark} release version: {f.host}/{f.lane}: {f.summary}"))
|
|
929
|
+
if f.detail:
|
|
930
|
+
out.append((True, f" {f.detail}"))
|
|
931
|
+
return out
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
@click.command(
|
|
935
|
+
help=(
|
|
936
|
+
"Fleet-wide prereq report: is this machine fit to be routed work?\n\n"
|
|
937
|
+
"#1570 E -- \"One command, whole fleet, prereq status per machine. "
|
|
938
|
+
"Would have answered [the #1564 gh-version incident] in seconds.\" "
|
|
939
|
+
"Reads each machine's already-probed tool_versions straight out of "
|
|
940
|
+
"/health (#1570 B), so it costs exactly what `coord status` costs "
|
|
941
|
+
"-- no SSHing around the fleet by hand.\n\n"
|
|
942
|
+
"Exits 1 if any machine is unreachable, hasn't upgraded to publish "
|
|
943
|
+
"tool_versions yet, fails a baseline prereq, or claims a capability "
|
|
944
|
+
"its own probe can't back up."
|
|
945
|
+
)
|
|
946
|
+
)
|
|
947
|
+
@_CONFIG_OPTION
|
|
948
|
+
@click.option("--machine", "machine_filter", default=None, help="Only check this machine.")
|
|
949
|
+
@click.option(
|
|
950
|
+
"--timeout", default=3.0, show_default=True, type=float,
|
|
951
|
+
help="Per-machine /health timeout (seconds).",
|
|
952
|
+
)
|
|
953
|
+
@click.option(
|
|
954
|
+
"--expected", default=None,
|
|
955
|
+
help=(
|
|
956
|
+
"The version every lane should be on (leading 'v' optional) — same "
|
|
957
|
+
"semantics as `coord release verify --expected` (#2082)."
|
|
958
|
+
),
|
|
959
|
+
)
|
|
960
|
+
@click.option(
|
|
961
|
+
"--pypi/--no-pypi", "use_pypi", default=True, show_default=True,
|
|
962
|
+
help=(
|
|
963
|
+
"Resolve --expected from the PyPI simple index when not given "
|
|
964
|
+
"explicitly, so a fleet that is uniformly behind the released "
|
|
965
|
+
"version reads as CRIT here rather than clean (#2052's lesson, "
|
|
966
|
+
"applied to doctor by #2082) — same default `coord release verify "
|
|
967
|
+
"--pypi` uses, and the same resolution (`_resolve_expected`)."
|
|
968
|
+
),
|
|
969
|
+
)
|
|
970
|
+
def doctor(
|
|
971
|
+
config_path: Path,
|
|
972
|
+
machine_filter: str | None,
|
|
973
|
+
timeout: float,
|
|
974
|
+
expected: str | None,
|
|
975
|
+
use_pypi: bool,
|
|
976
|
+
) -> None:
|
|
977
|
+
from coord.network import check_all
|
|
978
|
+
from coord.prereqs import ToolProbe, unmet_capabilities
|
|
979
|
+
|
|
980
|
+
cfg = _load_config(config_path)
|
|
981
|
+
machines = cfg.machines
|
|
982
|
+
if machine_filter:
|
|
983
|
+
machines = [m for m in machines if m.name == machine_filter]
|
|
984
|
+
if not machines:
|
|
985
|
+
click.echo(
|
|
986
|
+
f"error: machine {machine_filter!r} not in coordinator.yml "
|
|
987
|
+
f"(have: {[m.name for m in cfg.machines]})",
|
|
988
|
+
err=True,
|
|
989
|
+
)
|
|
990
|
+
sys.exit(2)
|
|
991
|
+
|
|
992
|
+
statuses = check_all(machines, timeout=timeout)
|
|
993
|
+
any_problem = False
|
|
994
|
+
for s in statuses:
|
|
995
|
+
m = s.machine
|
|
996
|
+
click.echo(f"{m.name} ({m.host}):")
|
|
997
|
+
if not s.is_online:
|
|
998
|
+
click.echo(f" ✗ unreachable — {s.reason}")
|
|
999
|
+
any_problem = True
|
|
1000
|
+
continue
|
|
1001
|
+
|
|
1002
|
+
health = s.health or {}
|
|
1003
|
+
# #1712: run the config-vs-/health cross-check BEFORE the
|
|
1004
|
+
# tool_versions early-continue below — "declares capabilities,
|
|
1005
|
+
# publishes none" is the loudest thing this command can say about a
|
|
1006
|
+
# machine, and it must not be skipped just because the agent is also
|
|
1007
|
+
# too old to report tool_versions.
|
|
1008
|
+
for is_problem, line in _health_vs_config_lines(m, health):
|
|
1009
|
+
click.echo(line)
|
|
1010
|
+
if is_problem:
|
|
1011
|
+
any_problem = True
|
|
1012
|
+
|
|
1013
|
+
# #1801: for a config-free agent, the #1712 check above is silenced
|
|
1014
|
+
# (empty /health capabilities/repos is the designed shape) — so run
|
|
1015
|
+
# the checks that actually catch what blocks dispatch there instead:
|
|
1016
|
+
# missing repo_paths, missing provider:* capability. Both come
|
|
1017
|
+
# straight out of coordinator.yml.
|
|
1018
|
+
if health.get("config_free"):
|
|
1019
|
+
for is_problem, line in _dispatch_blocker_lines_for_config_free(m, cfg):
|
|
1020
|
+
click.echo(line)
|
|
1021
|
+
if is_problem:
|
|
1022
|
+
any_problem = True
|
|
1023
|
+
|
|
1024
|
+
# #1831: installed systemd units silently drift from deploy/ — this
|
|
1025
|
+
# is the same class of blind spot #1712 closed for capabilities/repos,
|
|
1026
|
+
# projected from the machine's own unit_drift H-1 check rather than
|
|
1027
|
+
# SSHing in to diff unit files by hand.
|
|
1028
|
+
for is_problem, line in _unit_drift_lines(health):
|
|
1029
|
+
click.echo(line)
|
|
1030
|
+
if is_problem:
|
|
1031
|
+
any_problem = True
|
|
1032
|
+
|
|
1033
|
+
# #2098: same class of blind spot as unit_drift immediately above,
|
|
1034
|
+
# but for enablement rather than content — an installed unit that
|
|
1035
|
+
# the manifest says this host should run and that isn't actually
|
|
1036
|
+
# `systemctl --user enable`d.
|
|
1037
|
+
for is_problem, line in _unit_enablement_lines(health):
|
|
1038
|
+
click.echo(line)
|
|
1039
|
+
if is_problem:
|
|
1040
|
+
any_problem = True
|
|
1041
|
+
|
|
1042
|
+
raw_probes = health.get("tool_versions")
|
|
1043
|
+
if not raw_probes:
|
|
1044
|
+
click.echo(
|
|
1045
|
+
" ⚠ no tool_versions in /health — agent predates #1570 B "
|
|
1046
|
+
"(`coord agent update` to see prereq status)"
|
|
1047
|
+
)
|
|
1048
|
+
any_problem = True
|
|
1049
|
+
continue
|
|
1050
|
+
|
|
1051
|
+
probes = {
|
|
1052
|
+
tool: ToolProbe(
|
|
1053
|
+
tool=tool,
|
|
1054
|
+
capability=info.get("capability"),
|
|
1055
|
+
found=bool(info.get("found", False)),
|
|
1056
|
+
version=info.get("version"),
|
|
1057
|
+
min_version=info.get("min_version"),
|
|
1058
|
+
meets_floor=info.get("meets_floor"),
|
|
1059
|
+
what_breaks="",
|
|
1060
|
+
)
|
|
1061
|
+
for tool, info in raw_probes.items()
|
|
1062
|
+
if isinstance(info, dict)
|
|
1063
|
+
}
|
|
1064
|
+
for tool, p in sorted(probes.items()):
|
|
1065
|
+
marker = "✓" if p.ok else "✗"
|
|
1066
|
+
if not p.ok:
|
|
1067
|
+
any_problem = True
|
|
1068
|
+
if not p.found:
|
|
1069
|
+
detail = "not found"
|
|
1070
|
+
else:
|
|
1071
|
+
detail = p.version or "found (version unknown)"
|
|
1072
|
+
floor = f" (>= {p.min_version} required)" if p.min_version else ""
|
|
1073
|
+
click.echo(f" {marker} {tool}: {detail}{floor}")
|
|
1074
|
+
|
|
1075
|
+
unmet = unmet_capabilities(m.capabilities, probes)
|
|
1076
|
+
for cap, reasons in unmet.items():
|
|
1077
|
+
any_problem = True
|
|
1078
|
+
for reason in reasons:
|
|
1079
|
+
click.echo(f" ✗ capability {cap!r} claimed but unmet — {reason}")
|
|
1080
|
+
|
|
1081
|
+
# #2082: is the fleet actually running the released version? On
|
|
1082
|
+
# 2026-08-10 it was eleven releases behind (v0.5.15 vs PyPI's v0.5.26)
|
|
1083
|
+
# and nothing routine said so — `coord release verify` already computed
|
|
1084
|
+
# this exact comparison and already returned CRIT, it was just never
|
|
1085
|
+
# called anywhere an operator would see without being told to look.
|
|
1086
|
+
# Reuses that SAME function (not a second comparison — #2096's "two
|
|
1087
|
+
# surfaces, one function" rule) over the `/health` bodies this command
|
|
1088
|
+
# already fetched above, so this costs no extra per-machine round trip —
|
|
1089
|
+
# only the one-time PyPI resolution below.
|
|
1090
|
+
from coord import release_verify as rv # noqa: PLC0415
|
|
1091
|
+
from coord.commands.release import _resolve_expected # noqa: PLC0415
|
|
1092
|
+
|
|
1093
|
+
index_url = getattr(getattr(cfg, "health", None), "pypi_index_url",
|
|
1094
|
+
"https://pypi.org/simple")
|
|
1095
|
+
resolved_expected, resolve_warning = _resolve_expected(
|
|
1096
|
+
expected, use_pypi=use_pypi, index_url=index_url, timeout=timeout
|
|
1097
|
+
)
|
|
1098
|
+
if resolve_warning:
|
|
1099
|
+
click.echo(f"⚠ {resolve_warning}")
|
|
1100
|
+
|
|
1101
|
+
machine_health = {s.machine.name: (s.health if s.is_online else None) for s in statuses}
|
|
1102
|
+
unreachable = {
|
|
1103
|
+
s.machine.name: (s.reason or "offline") for s in statuses if not s.is_online
|
|
1104
|
+
}
|
|
1105
|
+
release_report = rv.verify(
|
|
1106
|
+
machine_health=machine_health, unreachable=unreachable, expected=resolved_expected,
|
|
1107
|
+
)
|
|
1108
|
+
lag_lines = _release_lag_lines(release_report)
|
|
1109
|
+
if lag_lines:
|
|
1110
|
+
click.echo("")
|
|
1111
|
+
click.echo("release version (coord release verify):")
|
|
1112
|
+
for is_problem, line in lag_lines:
|
|
1113
|
+
click.echo(line)
|
|
1114
|
+
if is_problem:
|
|
1115
|
+
any_problem = True
|
|
1116
|
+
|
|
1117
|
+
# #1862: a quiet-hours window that removes the only machine with a
|
|
1118
|
+
# capability makes matching work silently unroutable (`dispatch_smoke`
|
|
1119
|
+
# already has this failure shape — #1678: it refuses to route and the
|
|
1120
|
+
# Test stage retries forever with no error). Cheap heuristic, not full
|
|
1121
|
+
# time-overlap math: if EVERY machine advertising a capability has some
|
|
1122
|
+
# `quiet_hours` window, coverage isn't guaranteed around the clock; if
|
|
1123
|
+
# at least one machine offering it has none, it's always coverable.
|
|
1124
|
+
# Runs over the FULL fleet regardless of `--machine`.
|
|
1125
|
+
caps_with_quiet_hours: dict[str, list[str]] = {}
|
|
1126
|
+
caps_without_quiet_hours: set[str] = set()
|
|
1127
|
+
for m in cfg.machines:
|
|
1128
|
+
for cap in m.capabilities:
|
|
1129
|
+
if m.quiet_hours is not None:
|
|
1130
|
+
caps_with_quiet_hours.setdefault(cap, []).append(m.name)
|
|
1131
|
+
else:
|
|
1132
|
+
caps_without_quiet_hours.add(cap)
|
|
1133
|
+
for cap, quiet_machine_names in sorted(caps_with_quiet_hours.items()):
|
|
1134
|
+
if cap in caps_without_quiet_hours:
|
|
1135
|
+
continue
|
|
1136
|
+
any_problem = True
|
|
1137
|
+
names = ", ".join(sorted(quiet_machine_names))
|
|
1138
|
+
click.echo(
|
|
1139
|
+
f"⚠ capability {cap!r} is only ever offered by machine(s) with "
|
|
1140
|
+
f"quiet_hours configured ({names}) — an overlapping window "
|
|
1141
|
+
"could leave it with no awake machine to route to"
|
|
1142
|
+
)
|
|
1143
|
+
|
|
1144
|
+
if any_problem:
|
|
1145
|
+
sys.exit(1)
|
|
1146
|
+
|
|
1147
|
+
|
|
1148
|
+
@click.command("show-plan", help="Pretty-print the structured plan for a plan-only assignment.")
|
|
1149
|
+
@click.argument("assignment_id")
|
|
1150
|
+
def show_plan(assignment_id: str) -> None:
|
|
1151
|
+
from coord.board_service import read_board
|
|
1152
|
+
from coord.plan_parser import WorkerPlan, parse_plan_from_log
|
|
1153
|
+
from coord.state import COORD_DIR, load_plans
|
|
1154
|
+
|
|
1155
|
+
board = read_board()
|
|
1156
|
+
assignment = board.find_by_id(assignment_id)
|
|
1157
|
+
if assignment is None:
|
|
1158
|
+
click.echo(f"error: assignment {assignment_id!r} not found in board", err=True)
|
|
1159
|
+
sys.exit(1)
|
|
1160
|
+
|
|
1161
|
+
if assignment.type != "plan":
|
|
1162
|
+
atype = assignment.type
|
|
1163
|
+
click.echo(
|
|
1164
|
+
f"error: assignment {assignment_id} is type {atype!r}, not 'plan'",
|
|
1165
|
+
err=True,
|
|
1166
|
+
)
|
|
1167
|
+
sys.exit(1)
|
|
1168
|
+
|
|
1169
|
+
# 1. Try the plan cached on the board/assignment record.
|
|
1170
|
+
plan_dict = assignment.plan
|
|
1171
|
+
if plan_dict is None:
|
|
1172
|
+
plans = load_plans()
|
|
1173
|
+
plan_dict = plans.get(assignment_id)
|
|
1174
|
+
|
|
1175
|
+
# 2. Fall back to parsing the log directly (works when agent is local).
|
|
1176
|
+
if plan_dict is None:
|
|
1177
|
+
local_log = COORD_DIR / "logs" / f"{assignment_id}.log"
|
|
1178
|
+
try:
|
|
1179
|
+
worker_plan = parse_plan_from_log(local_log)
|
|
1180
|
+
except Exception: # noqa: BLE001
|
|
1181
|
+
worker_plan = None
|
|
1182
|
+
if worker_plan is not None:
|
|
1183
|
+
plan_dict = worker_plan.to_dict()
|
|
1184
|
+
|
|
1185
|
+
if plan_dict is None:
|
|
1186
|
+
click.echo(
|
|
1187
|
+
f"No structured plan found for assignment {assignment_id}.\n"
|
|
1188
|
+
"Possible reasons: the worker has not completed yet, the log is on "
|
|
1189
|
+
"a remote machine, or the worker did not output plan sections.\n"
|
|
1190
|
+
"Run 'coord notify' after the worker finishes to parse and cache the plan."
|
|
1191
|
+
)
|
|
1192
|
+
return
|
|
1193
|
+
|
|
1194
|
+
_display_plan(WorkerPlan.from_dict(plan_dict), assignment)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
def _display_plan(plan: object, assignment: object) -> None:
|
|
1198
|
+
"""Pretty-print a WorkerPlan to stdout."""
|
|
1199
|
+
from coord.plan_parser import WorkerPlan # noqa: PLC0415
|
|
1200
|
+
|
|
1201
|
+
assert isinstance(plan, WorkerPlan)
|
|
1202
|
+
|
|
1203
|
+
repo_name = getattr(assignment, "repo_name", "?")
|
|
1204
|
+
issue_number = getattr(assignment, "issue_number", "?")
|
|
1205
|
+
issue_title = getattr(assignment, "issue_title", "")
|
|
1206
|
+
machine_name = getattr(assignment, "machine_name", "?")
|
|
1207
|
+
assignment_id = getattr(assignment, "assignment_id", "?")
|
|
1208
|
+
|
|
1209
|
+
click.echo(
|
|
1210
|
+
f"## Plan — {repo_name} #{issue_number}: {issue_title}"
|
|
1211
|
+
)
|
|
1212
|
+
click.echo(f"Assignment: {assignment_id} Machine: {machine_name}")
|
|
1213
|
+
|
|
1214
|
+
if plan.plan:
|
|
1215
|
+
click.echo("")
|
|
1216
|
+
click.echo("### Summary")
|
|
1217
|
+
click.echo(plan.plan)
|
|
1218
|
+
|
|
1219
|
+
if plan.files_read:
|
|
1220
|
+
click.echo("")
|
|
1221
|
+
click.echo("### Files Read")
|
|
1222
|
+
for f in plan.files_read:
|
|
1223
|
+
click.echo(f" {f}")
|
|
1224
|
+
|
|
1225
|
+
if plan.files_modify:
|
|
1226
|
+
click.echo("")
|
|
1227
|
+
click.echo("### Files to Modify")
|
|
1228
|
+
for f in plan.files_modify:
|
|
1229
|
+
click.echo(f" {f}")
|
|
1230
|
+
|
|
1231
|
+
if plan.approach:
|
|
1232
|
+
click.echo("")
|
|
1233
|
+
click.echo("### Approach")
|
|
1234
|
+
click.echo(plan.approach)
|
|
1235
|
+
|
|
1236
|
+
if plan.risks:
|
|
1237
|
+
click.echo("")
|
|
1238
|
+
click.echo("### Risks")
|
|
1239
|
+
click.echo(plan.risks)
|
|
1240
|
+
|
|
1241
|
+
if plan.estimate:
|
|
1242
|
+
click.echo("")
|
|
1243
|
+
click.echo("### Estimate")
|
|
1244
|
+
click.echo(plan.estimate)
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def _diagnose_via_daemon(svc, params: dict) -> None:
|
|
1248
|
+
"""#diagnose: run ``coord diagnose`` on the daemon host (canonical board +
|
|
1249
|
+
gh + ssh access to the fleet) and relay its output, so the per-stage doctor
|
|
1250
|
+
does real work from a thin client instead of no-opping against an empty
|
|
1251
|
+
local board. Mirrors ``_reconcile_via_daemon``."""
|
|
1252
|
+
from coord.client import post_record # noqa: PLC0415
|
|
1253
|
+
|
|
1254
|
+
try:
|
|
1255
|
+
resp = post_record(svc, "/diagnose", params, timeout=180.0)
|
|
1256
|
+
except Exception as exc: # noqa: BLE001
|
|
1257
|
+
click.echo(f"error: diagnose via daemon failed: {exc}", err=True)
|
|
1258
|
+
sys.exit(1)
|
|
1259
|
+
output = resp.get("output") or ""
|
|
1260
|
+
if output:
|
|
1261
|
+
click.echo(output, nl=False)
|
|
1262
|
+
if resp.get("error"):
|
|
1263
|
+
click.echo(f"error: {resp['error']}", err=True)
|
|
1264
|
+
code = resp.get("exit_code") or 0
|
|
1265
|
+
if code:
|
|
1266
|
+
sys.exit(int(code))
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
@click.command(
|
|
1270
|
+
help=(
|
|
1271
|
+
"Diagnose and fix a specific pipeline stage of an issue.\n\n"
|
|
1272
|
+
"Inspects the stage (phantom 'running' rows, dropped review findings, "
|
|
1273
|
+
"stale-but-live sessions, merged-but-grey boxes, orphaned worktrees), "
|
|
1274
|
+
"makes a BEST-EFFORT non-destructive recovery of THAT stage (finalize, "
|
|
1275
|
+
"recover review findings from the session transcript, reconcile "
|
|
1276
|
+
"merges), and ALWAYS scans this issue's OTHER board rows for phantom "
|
|
1277
|
+
"'running' rows. When recovery isn't possible it reports "
|
|
1278
|
+
"needs_reset=true; re-run with --reset to clear the stage's "
|
|
1279
|
+
"rows/claim/worktree and stop a live session — KEEPING the branch + "
|
|
1280
|
+
"commits, so the stage is re-dispatchable. A phantom row found on the "
|
|
1281
|
+
"issue-wide scan (#1658) is only ever a RECOMMENDATION without "
|
|
1282
|
+
"--reset — it is reported, never finalized, until --reset is passed.\n\n"
|
|
1283
|
+
"Pass --orphan-worktrees instead of REPO/ISSUE to run a local fleet sweep "
|
|
1284
|
+
"that removes coordinator worktrees with no live tmux session and no "
|
|
1285
|
+
"uncommitted work. Dirty worktrees are reported but never auto-deleted."
|
|
1286
|
+
)
|
|
1287
|
+
)
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
@click.argument("repo", required=False, default=None)
|
|
1291
|
+
@click.argument("issue", type=int, required=False, default=None)
|
|
1292
|
+
@click.option(
|
|
1293
|
+
"--stage",
|
|
1294
|
+
# #2087: "smoke" added — it was previously unreachable via an explicit
|
|
1295
|
+
# --stage (rejected here before ever reaching diagnose_stage()) even
|
|
1296
|
+
# though an implicit (no --stage) pick could already land on a
|
|
1297
|
+
# type="smoke" row and then dead-end inside diagnose_stage() with "no
|
|
1298
|
+
# diagnosis available". See STAGE_ASSIGNMENT_TYPES["smoke"] (diagnose.py).
|
|
1299
|
+
type=click.Choice(["plan", "work", "review", "test", "merge", "smoke"]),
|
|
1300
|
+
default=None,
|
|
1301
|
+
help="Which stage to diagnose (default: the issue's most-recent stage).",
|
|
1302
|
+
)
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
@click.option(
|
|
1306
|
+
"--reset",
|
|
1307
|
+
is_flag=True,
|
|
1308
|
+
help="Non-destructive reset: clear the stage's rows/claim/worktree and stop "
|
|
1309
|
+
"a live session, KEEPING the branch + commits (stage re-dispatchable).",
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
@click.option("--dry-run", is_flag=True, help="Report findings without writing.")
|
|
1314
|
+
@click.option(
|
|
1315
|
+
"--json",
|
|
1316
|
+
"output_json",
|
|
1317
|
+
is_flag=True,
|
|
1318
|
+
help=(
|
|
1319
|
+
"#935: emit the DiagnoseResult as a JSON object on stdout (in addition to "
|
|
1320
|
+
"the human-readable lines and the DIAGNOSE_RESULT trailer). The JSON block "
|
|
1321
|
+
"is printed BEFORE the trailer so callers can parse it without grepping."
|
|
1322
|
+
),
|
|
1323
|
+
)
|
|
1324
|
+
@click.option(
|
|
1325
|
+
"--orphan-worktrees",
|
|
1326
|
+
is_flag=True,
|
|
1327
|
+
help=(
|
|
1328
|
+
"#618: local fleet sweep — find and remove coordinator worktrees "
|
|
1329
|
+
"(~/.coord/worktrees/*) whose assignment has no live tmux session and "
|
|
1330
|
+
"no uncommitted work. Dirty worktrees are reported but never deleted."
|
|
1331
|
+
),
|
|
1332
|
+
)
|
|
1333
|
+
@click.option(
|
|
1334
|
+
"--graph",
|
|
1335
|
+
"graph_health",
|
|
1336
|
+
is_flag=True,
|
|
1337
|
+
help=(
|
|
1338
|
+
"Report graphify knowledge-graph freshness for this machine's local "
|
|
1339
|
+
"checkouts: whether the graph matches HEAD, and whether "
|
|
1340
|
+
"core.hooksPath is set so worktrees get a linked graph. Read-only."
|
|
1341
|
+
),
|
|
1342
|
+
)
|
|
1343
|
+
@click.option(
|
|
1344
|
+
"--config-provenance",
|
|
1345
|
+
"config_provenance_check",
|
|
1346
|
+
is_flag=True,
|
|
1347
|
+
help=(
|
|
1348
|
+
"#1779: report whether THIS machine's live ~/.coord/coordinator.yml "
|
|
1349
|
+
"is still a symlink into the coord-settings checkout (vs. having "
|
|
1350
|
+
"been silently replaced by `coord init`, scp, or an editor), "
|
|
1351
|
+
"whether that checkout has uncommitted changes, and whether it's "
|
|
1352
|
+
"behind/ahead of origin. Neutral skip on any machine with no "
|
|
1353
|
+
"coord-settings checkout — that is normal everywhere except the "
|
|
1354
|
+
"daemon host / operator box. Read-only, no network required."
|
|
1355
|
+
),
|
|
1356
|
+
)
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
@_CONFIG_OPTION
|
|
1360
|
+
def diagnose(
|
|
1361
|
+
repo: str | None,
|
|
1362
|
+
issue: int | None,
|
|
1363
|
+
stage: str | None,
|
|
1364
|
+
reset: bool,
|
|
1365
|
+
dry_run: bool,
|
|
1366
|
+
config_path: Path,
|
|
1367
|
+
output_json: bool = False,
|
|
1368
|
+
orphan_worktrees: bool = False,
|
|
1369
|
+
graph_health: bool = False,
|
|
1370
|
+
config_provenance_check: bool = False,
|
|
1371
|
+
) -> None:
|
|
1372
|
+
"""Per-stage doctor — diagnose, best-effort recover, optional reset."""
|
|
1373
|
+
# ── graphify graph freshness sweep (read-only) ───────────────────────────
|
|
1374
|
+
if graph_health:
|
|
1375
|
+
_diagnose_graph_health(config_path)
|
|
1376
|
+
return
|
|
1377
|
+
|
|
1378
|
+
# ── #1779: fleet coordinator.yml provenance (read-only) ─────────────────
|
|
1379
|
+
if config_provenance_check:
|
|
1380
|
+
_diagnose_config_provenance()
|
|
1381
|
+
return
|
|
1382
|
+
|
|
1383
|
+
# ── #618: --orphan-worktrees fleet sweep ─────────────────────────────────
|
|
1384
|
+
if orphan_worktrees:
|
|
1385
|
+
_diagnose_orphan_worktrees(config_path, dry_run=dry_run)
|
|
1386
|
+
return
|
|
1387
|
+
|
|
1388
|
+
if repo is None or issue is None:
|
|
1389
|
+
click.echo(
|
|
1390
|
+
"error: REPO and ISSUE are required (or pass --orphan-worktrees for a fleet sweep).",
|
|
1391
|
+
err=True,
|
|
1392
|
+
)
|
|
1393
|
+
sys.exit(2)
|
|
1394
|
+
|
|
1395
|
+
# #584: the canonical board + gh + fleet ssh live on the daemon host, so on
|
|
1396
|
+
# a thin client this must run there (an empty local board would no-op).
|
|
1397
|
+
# COORD_DIAGNOSE_ON_DAEMON guards the daemon against re-routing to itself.
|
|
1398
|
+
from coord.board_service import daemon_reroute_target # noqa: PLC0415
|
|
1399
|
+
|
|
1400
|
+
_svc = daemon_reroute_target("COORD_DIAGNOSE_ON_DAEMON")
|
|
1401
|
+
if _svc is not None:
|
|
1402
|
+
_diagnose_via_daemon(
|
|
1403
|
+
_svc,
|
|
1404
|
+
{
|
|
1405
|
+
"repo": repo,
|
|
1406
|
+
"issue": issue,
|
|
1407
|
+
"stage": stage,
|
|
1408
|
+
"reset": reset,
|
|
1409
|
+
"dry_run": dry_run,
|
|
1410
|
+
"output_json": output_json,
|
|
1411
|
+
},
|
|
1412
|
+
)
|
|
1413
|
+
return
|
|
1414
|
+
|
|
1415
|
+
from coord.diagnose import current_stage, diagnose_stage # noqa: PLC0415
|
|
1416
|
+
from coord.state import build_board # noqa: PLC0415
|
|
1417
|
+
|
|
1418
|
+
cfg = _load_config(config_path)
|
|
1419
|
+
board = build_board()
|
|
1420
|
+
resolved_stage = stage or current_stage(board, repo, issue)
|
|
1421
|
+
res = diagnose_stage(
|
|
1422
|
+
board, cfg, repo, issue, resolved_stage, reset=reset, dry_run=dry_run
|
|
1423
|
+
)
|
|
1424
|
+
# NOTE: deliberately NO save_board here. Every diagnose write goes through
|
|
1425
|
+
# the authoritative seam (finalize→post_completion, recover→post_result,
|
|
1426
|
+
# reconcile→state.update_*), which writes the canonical DB directly. A
|
|
1427
|
+
# save_board would persist the STALE in-memory snapshot (built before those
|
|
1428
|
+
# seam writes) and clobber them — e.g. flip a just-finalized phantom back to
|
|
1429
|
+
# 'running' (caught live on quadraui #366).
|
|
1430
|
+
|
|
1431
|
+
click.echo(f"diagnose {repo} #{issue} — stage={resolved_stage}"
|
|
1432
|
+
+ (" [reset]" if reset else "") + (" [dry-run]" if dry_run else ""))
|
|
1433
|
+
for f in res.findings:
|
|
1434
|
+
click.echo(f" · {f}")
|
|
1435
|
+
for a in res.actions_taken:
|
|
1436
|
+
click.echo(f" ✓ {a}")
|
|
1437
|
+
if res.needs_reset and not reset:
|
|
1438
|
+
click.echo(" ⚠ still wedged — re-run with --reset to clear the stage "
|
|
1439
|
+
"(keeps the branch + commits).")
|
|
1440
|
+
# #935 Part C: emit JSON dict before the trailer when --json is requested.
|
|
1441
|
+
# The daemon handler also passes output_json through so remote calls relay it.
|
|
1442
|
+
if output_json:
|
|
1443
|
+
import json # noqa: PLC0415
|
|
1444
|
+
click.echo("DIAGNOSE_JSON:" + json.dumps(res.to_json_dict()))
|
|
1445
|
+
click.echo(res.summary_line())
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _diagnose_graph_health(config_path: Path) -> None:
|
|
1449
|
+
"""Report graphify graph freshness for this machine's local checkouts.
|
|
1450
|
+
|
|
1451
|
+
Read-only by design. The graph is a *navigation* aid, so a stale one is a
|
|
1452
|
+
warning, not a failure — the point is to make drift visible in a routine
|
|
1453
|
+
health check instead of leaving an agent to discover it mid-task. It also
|
|
1454
|
+
checks ``core.hooksPath``, the one-time per-machine setting that decides
|
|
1455
|
+
whether worktrees on this box get a linked graph at all.
|
|
1456
|
+
|
|
1457
|
+
Local-machine only, same scope as ``--orphan-worktrees``: it inspects the
|
|
1458
|
+
checkouts named in ``coordinator.yml`` that actually exist here.
|
|
1459
|
+
"""
|
|
1460
|
+
from coord.graph_health import ( # noqa: PLC0415
|
|
1461
|
+
format_status_lines,
|
|
1462
|
+
graph_status,
|
|
1463
|
+
hooks_path_status,
|
|
1464
|
+
)
|
|
1465
|
+
|
|
1466
|
+
cfg = _load_config(config_path)
|
|
1467
|
+
|
|
1468
|
+
seen: set[Path] = set()
|
|
1469
|
+
checkouts: list[tuple[str, Path]] = []
|
|
1470
|
+
for machine in cfg.machines:
|
|
1471
|
+
for repo_cfg in cfg.repos:
|
|
1472
|
+
rp = machine.repo_path(repo_cfg.name)
|
|
1473
|
+
if not rp:
|
|
1474
|
+
continue
|
|
1475
|
+
path = Path(rp).expanduser()
|
|
1476
|
+
if path in seen or not (path / ".git").exists():
|
|
1477
|
+
continue
|
|
1478
|
+
seen.add(path)
|
|
1479
|
+
checkouts.append((repo_cfg.name, path))
|
|
1480
|
+
|
|
1481
|
+
if not checkouts:
|
|
1482
|
+
click.echo("no local checkouts from coordinator.yml exist on this machine.")
|
|
1483
|
+
return
|
|
1484
|
+
|
|
1485
|
+
stale_count = 0
|
|
1486
|
+
for repo_name, path in checkouts:
|
|
1487
|
+
click.echo(f"── {repo_name}")
|
|
1488
|
+
st = graph_status(path)
|
|
1489
|
+
for line in format_status_lines(st):
|
|
1490
|
+
click.echo(f" {line}")
|
|
1491
|
+
if st.stale:
|
|
1492
|
+
stale_count += 1
|
|
1493
|
+
click.echo(
|
|
1494
|
+
" fix: run `graphify update .` in the checkout "
|
|
1495
|
+
"(the hooks skip rebase/merge/cherry-pick and reset --hard)"
|
|
1496
|
+
)
|
|
1497
|
+
ok, detail = hooks_path_status(path)
|
|
1498
|
+
click.echo(f" {'✓' if ok else '⚠'} {detail}")
|
|
1499
|
+
|
|
1500
|
+
click.echo(
|
|
1501
|
+
f"GRAPH_HEALTH: checkouts={len(checkouts)} stale={stale_count}"
|
|
1502
|
+
)
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
def _diagnose_config_provenance() -> None:
|
|
1506
|
+
"""Report whether THIS machine's live ``coordinator.yml`` is still the
|
|
1507
|
+
reviewed one (#1779).
|
|
1508
|
+
|
|
1509
|
+
Read-only, local-machine only — same family as ``--graph`` and
|
|
1510
|
+
``--orphan-worktrees``. Neutral (not a warning) when the coord-settings
|
|
1511
|
+
checkout is absent, which is the normal, correct state on every machine
|
|
1512
|
+
except the daemon host and the operator box: the checkout is
|
|
1513
|
+
deliberately excluded from the fleet's own repo list so a dispatched
|
|
1514
|
+
worker can never edit the file governing its own concurrency limits,
|
|
1515
|
+
capability routing, and review gates (see ``coord/fleet_config_health.py``
|
|
1516
|
+
for the three failure modes this distinguishes).
|
|
1517
|
+
|
|
1518
|
+
Deliberately takes no ``config_path`` — unlike ``--graph``/
|
|
1519
|
+
``--orphan-worktrees`` this does not read ``coordinator.yml`` for a list
|
|
1520
|
+
of checkouts to inspect; it inspects one fixed pair of paths
|
|
1521
|
+
(``$COORD_CONFIG``/``~/.coord/coordinator.yml`` and
|
|
1522
|
+
``$COORD_SETTINGS_DIR``/``~/src/coord-settings``).
|
|
1523
|
+
"""
|
|
1524
|
+
from coord.fleet_config_health import ( # noqa: PLC0415
|
|
1525
|
+
config_provenance,
|
|
1526
|
+
format_provenance_lines,
|
|
1527
|
+
summary_line,
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
prov = config_provenance()
|
|
1531
|
+
for line in format_provenance_lines(prov):
|
|
1532
|
+
click.echo(f" {line}")
|
|
1533
|
+
click.echo(summary_line(prov))
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
def _diagnose_orphan_worktrees(config_path: Path, *, dry_run: bool) -> None:
|
|
1537
|
+
"""#618: local fleet sweep — find and prune orphaned coordinator worktrees.
|
|
1538
|
+
|
|
1539
|
+
An orphaned worktree is one under ``~/.coord/worktrees/`` whose
|
|
1540
|
+
assignment_id has no live tmux session and no running/pending DB row.
|
|
1541
|
+
Dirty worktrees (uncommitted changes) are reported but never deleted.
|
|
1542
|
+
|
|
1543
|
+
#1445: also runs :func:`coord.agent.check_worktree_writable` against
|
|
1544
|
+
``~/.coord/worktrees/`` first and reports a DEGRADED line naming the
|
|
1545
|
+
path and (when it's a permission rule at fault) the exact rule/file —
|
|
1546
|
+
this machine invariant is otherwise invisible until a dispatched worker
|
|
1547
|
+
burns a full session discovering it can't save anything. This check is
|
|
1548
|
+
LOCAL-machine only, same as the rest of this sweep; it does not reach
|
|
1549
|
+
out to other machines in the fleet.
|
|
1550
|
+
|
|
1551
|
+
#1445 review: ``--dry-run`` means "report findings without writing," so
|
|
1552
|
+
the OS-level half of the writability probe — which ``mkdir(parents=True,
|
|
1553
|
+
exist_ok=True)``s ``worktrees_dir`` (a real, persistent creation on a
|
|
1554
|
+
machine that never had one) and does a write+unlink of a probe file — is
|
|
1555
|
+
skipped under ``--dry-run``. The deny-rule scan
|
|
1556
|
+
(:func:`coord.agent.find_blocking_deny_rule`) is read-only and still runs.
|
|
1557
|
+
On a machine that has never had a worktree, ``--dry-run`` therefore
|
|
1558
|
+
reports "does not exist yet" rather than creating it just to say it's
|
|
1559
|
+
empty.
|
|
1560
|
+
"""
|
|
1561
|
+
from coord.diagnose import ( # noqa: PLC0415
|
|
1562
|
+
_find_orphaned_worktrees,
|
|
1563
|
+
_prune_orphaned_worktrees,
|
|
1564
|
+
)
|
|
1565
|
+
from coord.interactive import ( # noqa: PLC0415
|
|
1566
|
+
tmux_available,
|
|
1567
|
+
tmux_session_name,
|
|
1568
|
+
tmux_session_alive,
|
|
1569
|
+
)
|
|
1570
|
+
from coord.agent import check_worktree_writable, find_blocking_deny_rule # noqa: PLC0415
|
|
1571
|
+
from coord.board_service import read_board # noqa: PLC0415
|
|
1572
|
+
from coord.state import COORD_DIR # noqa: PLC0415
|
|
1573
|
+
|
|
1574
|
+
cfg = _load_config(config_path)
|
|
1575
|
+
board = read_board()
|
|
1576
|
+
worktrees_dir = COORD_DIR / "worktrees"
|
|
1577
|
+
|
|
1578
|
+
# #1445: surface a machine that can't write its own worktrees as
|
|
1579
|
+
# DEGRADED rather than silently idle-and-ready — this is the same
|
|
1580
|
+
# fleet invariant `AgentServer.assign()` now preflights before every
|
|
1581
|
+
# dispatch, checked here so an operator (or `scripts/drive-issue.sh`)
|
|
1582
|
+
# can catch it ahead of time with a plain `coord diagnose --orphan-worktrees`.
|
|
1583
|
+
if dry_run:
|
|
1584
|
+
if not worktrees_dir.exists():
|
|
1585
|
+
click.echo(
|
|
1586
|
+
f"~/.coord/worktrees/ does not exist yet — nothing to sweep "
|
|
1587
|
+
f"(dry-run: skipping writability probe to avoid creating it)."
|
|
1588
|
+
)
|
|
1589
|
+
return
|
|
1590
|
+
blocked_by = find_blocking_deny_rule(worktrees_dir)
|
|
1591
|
+
if blocked_by is not None:
|
|
1592
|
+
click.echo(
|
|
1593
|
+
f"⚠ DEGRADED: workers on this machine cannot write to "
|
|
1594
|
+
f"{worktrees_dir}: a Claude Code permission rule denies "
|
|
1595
|
+
f"Edit/Write under {worktrees_dir}: {blocked_by}"
|
|
1596
|
+
)
|
|
1597
|
+
else:
|
|
1598
|
+
click.echo(
|
|
1599
|
+
f"✓ {worktrees_dir} is writable by workers "
|
|
1600
|
+
f"(dry-run: OS-level write probe skipped)."
|
|
1601
|
+
)
|
|
1602
|
+
else:
|
|
1603
|
+
write_issue = check_worktree_writable(worktrees_dir)
|
|
1604
|
+
if write_issue is not None:
|
|
1605
|
+
click.echo(f"⚠ DEGRADED: workers on this machine cannot write to {worktrees_dir}: {write_issue}")
|
|
1606
|
+
else:
|
|
1607
|
+
click.echo(f"✓ {worktrees_dir} is writable by workers.")
|
|
1608
|
+
|
|
1609
|
+
# Outside --dry-run, check_worktree_writable() just mkdir'd worktrees_dir
|
|
1610
|
+
# (parents=True, exist_ok=True) as part of its probe, so it always exists
|
|
1611
|
+
# by this point — an empty directory just means no worktrees are
|
|
1612
|
+
# currently checked out. Under --dry-run we already returned above when
|
|
1613
|
+
# it didn't exist, so it's safe to iterate here too.
|
|
1614
|
+
if not any(worktrees_dir.iterdir()):
|
|
1615
|
+
click.echo("~/.coord/worktrees/ has no worktrees — nothing to sweep.")
|
|
1616
|
+
return
|
|
1617
|
+
|
|
1618
|
+
# Collect all assignment_ids with live tmux sessions.
|
|
1619
|
+
tmux_ok = tmux_available()
|
|
1620
|
+
live_tmux: set[str] = set()
|
|
1621
|
+
if tmux_ok:
|
|
1622
|
+
for entry in worktrees_dir.iterdir():
|
|
1623
|
+
if not entry.is_dir():
|
|
1624
|
+
continue
|
|
1625
|
+
aid = entry.name
|
|
1626
|
+
if tmux_session_alive(tmux_session_name(aid)):
|
|
1627
|
+
live_tmux.add(aid)
|
|
1628
|
+
|
|
1629
|
+
# All running/pending assignment_ids from the board (includes live tmux ones
|
|
1630
|
+
# from the board's active set; combine with live_tmux for sessions whose DB
|
|
1631
|
+
# rows may already be gone).
|
|
1632
|
+
running_ids: set[str] = {
|
|
1633
|
+
a.assignment_id
|
|
1634
|
+
for a in board.active
|
|
1635
|
+
if a.assignment_id
|
|
1636
|
+
}
|
|
1637
|
+
active_ids = running_ids | live_tmux
|
|
1638
|
+
|
|
1639
|
+
total_removed: list[Path] = []
|
|
1640
|
+
total_skipped: list[Path] = []
|
|
1641
|
+
|
|
1642
|
+
for repo in cfg.repos:
|
|
1643
|
+
# Find any local checkout for this repo.
|
|
1644
|
+
repo_path: Path | None = None
|
|
1645
|
+
for machine in cfg.machines:
|
|
1646
|
+
rp = machine.repo_path(repo.name)
|
|
1647
|
+
if rp:
|
|
1648
|
+
candidate = Path(rp).expanduser()
|
|
1649
|
+
if candidate.exists():
|
|
1650
|
+
repo_path = candidate
|
|
1651
|
+
break
|
|
1652
|
+
if repo_path is None:
|
|
1653
|
+
continue
|
|
1654
|
+
|
|
1655
|
+
# Delegate porcelain parsing to the shared helper (branch=None → any branch).
|
|
1656
|
+
orphans = _find_orphaned_worktrees(
|
|
1657
|
+
repo_path, None, active_assignment_ids=active_ids, worktrees_dir=worktrees_dir
|
|
1658
|
+
)
|
|
1659
|
+
if not orphans:
|
|
1660
|
+
continue
|
|
1661
|
+
|
|
1662
|
+
click.echo(f"{repo.name}: found {len(orphans)} orphaned worktree(s)")
|
|
1663
|
+
for wt in orphans:
|
|
1664
|
+
click.echo(f" {wt}")
|
|
1665
|
+
if dry_run:
|
|
1666
|
+
click.echo(f" (dry-run) would prune {len(orphans)} worktree(s)")
|
|
1667
|
+
total_skipped.extend(orphans)
|
|
1668
|
+
continue
|
|
1669
|
+
|
|
1670
|
+
removed, skipped = _prune_orphaned_worktrees(repo_path, orphans)
|
|
1671
|
+
for wt in removed:
|
|
1672
|
+
click.echo(f" ✓ removed {wt}")
|
|
1673
|
+
for wt in skipped:
|
|
1674
|
+
click.echo(f" ⚠ skipped (uncommitted work) {wt}")
|
|
1675
|
+
total_removed.extend(removed)
|
|
1676
|
+
total_skipped.extend(skipped)
|
|
1677
|
+
|
|
1678
|
+
click.echo(
|
|
1679
|
+
f"orphan-worktrees sweep: {len(total_removed)} removed"
|
|
1680
|
+
+ (f", {len(total_skipped)} skipped (dirty — inspect manually)" if total_skipped else "")
|
|
1681
|
+
+ (" [dry-run]" if dry_run else "")
|
|
1682
|
+
)
|
|
1683
|
+
|
|
1684
|
+
|
|
1685
|
+
@click.command(help="Show per-assignment and per-model cost breakdown with burn rate.")
|
|
1686
|
+
@_CONFIG_OPTION
|
|
1687
|
+
@click.option(
|
|
1688
|
+
"--remote",
|
|
1689
|
+
is_flag=True,
|
|
1690
|
+
help="Fetch cost data from agent servers for assignments without local logs.",
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
@click.option(
|
|
1695
|
+
"--timeout",
|
|
1696
|
+
default=3.0,
|
|
1697
|
+
show_default=True,
|
|
1698
|
+
type=float,
|
|
1699
|
+
help="Per-machine HTTP timeout for --remote lookups (seconds).",
|
|
1700
|
+
)
|
|
1701
|
+
@click.option(
|
|
1702
|
+
"--today",
|
|
1703
|
+
is_flag=True,
|
|
1704
|
+
help="Limit the view to the local calendar day (#1115).",
|
|
1705
|
+
)
|
|
1706
|
+
@click.option(
|
|
1707
|
+
"--week",
|
|
1708
|
+
is_flag=True,
|
|
1709
|
+
help="Limit the view to the current ISO week, Monday 00:00 -> next Monday (#1119).",
|
|
1710
|
+
)
|
|
1711
|
+
@click.option(
|
|
1712
|
+
"--month",
|
|
1713
|
+
is_flag=True,
|
|
1714
|
+
help="Limit the view to the current calendar month (#1119).",
|
|
1715
|
+
)
|
|
1716
|
+
@click.option(
|
|
1717
|
+
"--since",
|
|
1718
|
+
"since_spec",
|
|
1719
|
+
default=None,
|
|
1720
|
+
help="Limit the view to legs since <ISO date | Nd | Nh> (#1115).",
|
|
1721
|
+
)
|
|
1722
|
+
@click.option(
|
|
1723
|
+
"--by-issue",
|
|
1724
|
+
"by_issue",
|
|
1725
|
+
is_flag=True,
|
|
1726
|
+
help="Group daemon-board usage by GitHub issue for the time window, sorted desc (#1115).",
|
|
1727
|
+
)
|
|
1728
|
+
@click.option(
|
|
1729
|
+
"--issue",
|
|
1730
|
+
"issue_number",
|
|
1731
|
+
type=int,
|
|
1732
|
+
default=None,
|
|
1733
|
+
help="Per-stage drill-down for one issue number — all legs, oldest-first (#1115).",
|
|
1734
|
+
)
|
|
1735
|
+
@click.option(
|
|
1736
|
+
"--by",
|
|
1737
|
+
"by_dim",
|
|
1738
|
+
type=click.Choice(["repo", "week", "month", "issue"]),
|
|
1739
|
+
default=None,
|
|
1740
|
+
help="Cross-cut daemon-board usage by dimension: repo (cross-repo rollup), "
|
|
1741
|
+
"week/month (time-bucketed spend series), or issue (same as --by-issue) (#1119).",
|
|
1742
|
+
)
|
|
1743
|
+
@click.option(
|
|
1744
|
+
"--by-time",
|
|
1745
|
+
"by_time",
|
|
1746
|
+
is_flag=True,
|
|
1747
|
+
help="Time-spent view: rank wall-clock by stage-type, or by issue when combined "
|
|
1748
|
+
"with --by issue (#1119).",
|
|
1749
|
+
)
|
|
1750
|
+
@click.option(
|
|
1751
|
+
"--sort",
|
|
1752
|
+
"sort_by",
|
|
1753
|
+
type=click.Choice(["cost", "tokens", "time"]),
|
|
1754
|
+
default=None,
|
|
1755
|
+
help="Sort order (always descending). Default: cost for rollup views, time for --by-time.",
|
|
1756
|
+
)
|
|
1757
|
+
@click.option(
|
|
1758
|
+
"--limits",
|
|
1759
|
+
"show_limits",
|
|
1760
|
+
is_flag=True,
|
|
1761
|
+
help="Show the account's Max-plan 5h/weekly usage-window probe (#1466) "
|
|
1762
|
+
"instead of the cost breakdown — server-side/account-wide, ~60s cached.",
|
|
1763
|
+
)
|
|
1764
|
+
@click.option(
|
|
1765
|
+
"--json",
|
|
1766
|
+
"as_json",
|
|
1767
|
+
is_flag=True,
|
|
1768
|
+
help="With --limits, emit the probe as structured JSON instead of text.",
|
|
1769
|
+
)
|
|
1770
|
+
def usage(
|
|
1771
|
+
config_path: Path,
|
|
1772
|
+
remote: bool,
|
|
1773
|
+
timeout: float,
|
|
1774
|
+
today: bool,
|
|
1775
|
+
week: bool,
|
|
1776
|
+
month: bool,
|
|
1777
|
+
since_spec: str | None,
|
|
1778
|
+
by_issue: bool,
|
|
1779
|
+
issue_number: int | None,
|
|
1780
|
+
by_dim: str | None,
|
|
1781
|
+
by_time: bool,
|
|
1782
|
+
sort_by: str | None,
|
|
1783
|
+
show_limits: bool,
|
|
1784
|
+
as_json: bool,
|
|
1785
|
+
) -> None:
|
|
1786
|
+
if as_json and not show_limits:
|
|
1787
|
+
raise click.BadParameter(
|
|
1788
|
+
"--json only applies to --limits", param_hint="'--json'"
|
|
1789
|
+
)
|
|
1790
|
+
if show_limits:
|
|
1791
|
+
import json as _json
|
|
1792
|
+
|
|
1793
|
+
from coord.usage_limits import format_plan_limits, get_plan_limits
|
|
1794
|
+
|
|
1795
|
+
limits = get_plan_limits()
|
|
1796
|
+
if as_json:
|
|
1797
|
+
click.echo(_json.dumps(limits.to_dict(), indent=2))
|
|
1798
|
+
else:
|
|
1799
|
+
click.echo(format_plan_limits(limits))
|
|
1800
|
+
return
|
|
1801
|
+
if issue_number is not None:
|
|
1802
|
+
_usage_issue_drill(config_path, issue_number, today=today, week=week, month=month, since_spec=since_spec)
|
|
1803
|
+
return
|
|
1804
|
+
if by_time:
|
|
1805
|
+
if by_dim in ("repo", "week", "month"):
|
|
1806
|
+
raise click.BadParameter(
|
|
1807
|
+
f"--by-time only supports --by issue (or no --by); got --by {by_dim}",
|
|
1808
|
+
param_hint="'--by'",
|
|
1809
|
+
)
|
|
1810
|
+
_usage_by_time(
|
|
1811
|
+
config_path,
|
|
1812
|
+
today=today, week=week, month=month, since_spec=since_spec,
|
|
1813
|
+
by_dim=by_dim, sort_by=sort_by,
|
|
1814
|
+
)
|
|
1815
|
+
return
|
|
1816
|
+
if by_issue or by_dim == "issue":
|
|
1817
|
+
_usage_by_issue(
|
|
1818
|
+
config_path,
|
|
1819
|
+
today=today, week=week, month=month, since_spec=since_spec,
|
|
1820
|
+
sort_by=_usage_resolve_sort(sort_by, default="cost"),
|
|
1821
|
+
)
|
|
1822
|
+
return
|
|
1823
|
+
if by_dim in ("repo", "week", "month"):
|
|
1824
|
+
_usage_by_dim(
|
|
1825
|
+
config_path, by_dim,
|
|
1826
|
+
today=today, week=week, month=month, since_spec=since_spec,
|
|
1827
|
+
sort_by=_usage_resolve_sort(sort_by, default="cost"),
|
|
1828
|
+
)
|
|
1829
|
+
return
|
|
1830
|
+
|
|
1831
|
+
from coord.board_service import read_board
|
|
1832
|
+
from coord.state import load_session
|
|
1833
|
+
from coord.usage import build_session_usage, filter_assignments_in_window, format_usage_report
|
|
1834
|
+
|
|
1835
|
+
board = read_board()
|
|
1836
|
+
all_assignments = list(board.active) + list(board.completed)
|
|
1837
|
+
|
|
1838
|
+
# Resolve + apply the window flags to the legacy (no --by/--by-time/
|
|
1839
|
+
# --by-issue/--issue) view too (#1119 review finding #1) — previously
|
|
1840
|
+
# --today/--week/--month/--since were silently ignored here, with no
|
|
1841
|
+
# validation (even --week --month together fell through to this branch
|
|
1842
|
+
# unchecked). Calling _usage_resolve_window unconditionally means the
|
|
1843
|
+
# existing n_set > 1 guard now fires for the bare case as well. Only set
|
|
1844
|
+
# window_label when a flag was actually given, so the default
|
|
1845
|
+
# (unwindowed) report stays byte-for-byte unchanged.
|
|
1846
|
+
window_label: str | None = None
|
|
1847
|
+
if today or week or month or since_spec:
|
|
1848
|
+
window = _usage_resolve_window(today, week, month, since_spec)
|
|
1849
|
+
window_label = window.label
|
|
1850
|
+
all_assignments = filter_assignments_in_window(all_assignments, window)
|
|
1851
|
+
|
|
1852
|
+
# Resolve session start time from session.json
|
|
1853
|
+
started_at: float | None = None
|
|
1854
|
+
sess = load_session()
|
|
1855
|
+
if sess and sess.get("started_at"):
|
|
1856
|
+
import datetime
|
|
1857
|
+
try:
|
|
1858
|
+
dt = datetime.datetime.fromisoformat(
|
|
1859
|
+
sess["started_at"].rstrip("Z").replace("Z", "+00:00")
|
|
1860
|
+
)
|
|
1861
|
+
started_at = dt.replace(tzinfo=datetime.timezone.utc).timestamp()
|
|
1862
|
+
except (ValueError, AttributeError):
|
|
1863
|
+
pass
|
|
1864
|
+
|
|
1865
|
+
# Optionally fetch remote cost data for assignments without local logs.
|
|
1866
|
+
remote_by_id: dict[str, dict] = {}
|
|
1867
|
+
if remote and all_assignments:
|
|
1868
|
+
cfg = _load_config(config_path)
|
|
1869
|
+
from coord.network import fetch_status
|
|
1870
|
+
|
|
1871
|
+
# Build a map from machine_name → assignments on that machine.
|
|
1872
|
+
by_machine: dict[str, list] = {}
|
|
1873
|
+
for a in all_assignments:
|
|
1874
|
+
if a.assignment_id:
|
|
1875
|
+
by_machine.setdefault(a.machine_name, []).append(a)
|
|
1876
|
+
|
|
1877
|
+
for machine in cfg.machines:
|
|
1878
|
+
if machine.name not in by_machine:
|
|
1879
|
+
continue
|
|
1880
|
+
try:
|
|
1881
|
+
data = fetch_status(machine, timeout=timeout)
|
|
1882
|
+
except Exception:
|
|
1883
|
+
continue
|
|
1884
|
+
if not data:
|
|
1885
|
+
continue
|
|
1886
|
+
for entry in (data.get("active") or []) + (data.get("completed") or []):
|
|
1887
|
+
aid = entry.get("id") or entry.get("assignment_id")
|
|
1888
|
+
if aid:
|
|
1889
|
+
remote_by_id[aid] = entry
|
|
1890
|
+
|
|
1891
|
+
session = build_session_usage(
|
|
1892
|
+
all_assignments,
|
|
1893
|
+
remote_by_id=remote_by_id if remote_by_id else None,
|
|
1894
|
+
started_at=started_at,
|
|
1895
|
+
)
|
|
1896
|
+
click.echo(format_usage_report(session, window_label=window_label))
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
_SINCE_WEEKS_RE = re.compile(r"^(\d+)\s*w$", re.IGNORECASE)
|
|
1900
|
+
|
|
1901
|
+
|
|
1902
|
+
def _normalize_since_spec(spec: str) -> str:
|
|
1903
|
+
"""Expand a ``Nw`` (weeks) shorthand to ``(N*7)d`` before handing it to
|
|
1904
|
+
Core's ``Window.since`` (#1119 requirement 1 / acceptance example
|
|
1905
|
+
``--since 8w``). Core's ``since`` regex only knows ``Nd``/``Nh`` (see
|
|
1906
|
+
``coord.usage_rollup._SINCE_RELATIVE_RE``) — this is a thin CLI-side
|
|
1907
|
+
syntactic convenience, not new window-resolution logic, so it stays here
|
|
1908
|
+
rather than in Core. Anything else (ISO date, ``Nd``, ``Nh``) passes
|
|
1909
|
+
through unchanged for Core to validate.
|
|
1910
|
+
"""
|
|
1911
|
+
match = _SINCE_WEEKS_RE.match(spec.strip())
|
|
1912
|
+
if match:
|
|
1913
|
+
return f"{int(match.group(1)) * 7}d"
|
|
1914
|
+
return spec
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
def _usage_resolve_window(today: bool, week: bool, month: bool, since_spec: str | None):
|
|
1918
|
+
"""Resolve the ``--today``/``--week``/``--month``/``--since`` flags to a
|
|
1919
|
+
:class:`coord.usage_rollup.TimeWindow` for the daemon-sourced rollup views
|
|
1920
|
+
(#1115/#1119). At most one of the four may be given. None given falls back
|
|
1921
|
+
to the current session's start time (open-ended); no session at all falls
|
|
1922
|
+
back to an unbounded window.
|
|
1923
|
+
|
|
1924
|
+
``--week``/``--month`` are resolved via :mod:`coord.usage_rollup`'s own
|
|
1925
|
+
``window_week``/``window_month`` presets — called, not reimplemented,
|
|
1926
|
+
per #1119's "consumes Core, no new aggregation logic" scope.
|
|
1927
|
+
"""
|
|
1928
|
+
n_set = sum([bool(today), bool(week), bool(month), bool(since_spec)])
|
|
1929
|
+
if n_set > 1:
|
|
1930
|
+
raise click.BadParameter(
|
|
1931
|
+
"pass at most one of --today, --week, --month, --since",
|
|
1932
|
+
param_hint="'--today'/'--week'/'--month'/'--since'",
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
from coord.usage_rollup import Window, window_month, window_week
|
|
1936
|
+
|
|
1937
|
+
if today:
|
|
1938
|
+
return Window.today()
|
|
1939
|
+
if week:
|
|
1940
|
+
return window_week()
|
|
1941
|
+
if month:
|
|
1942
|
+
return window_month()
|
|
1943
|
+
if since_spec:
|
|
1944
|
+
try:
|
|
1945
|
+
window = Window.since(_normalize_since_spec(since_spec))
|
|
1946
|
+
except ValueError as e:
|
|
1947
|
+
raise click.BadParameter(str(e), param_hint="'--since'") from e
|
|
1948
|
+
# Preserve the human-readable spec the user actually typed in the
|
|
1949
|
+
# printed label (e.g. "since 8w"), even though it was expanded to
|
|
1950
|
+
# "56d" for Core's date math above.
|
|
1951
|
+
from dataclasses import replace
|
|
1952
|
+
|
|
1953
|
+
return replace(window, label=f"since {since_spec}")
|
|
1954
|
+
|
|
1955
|
+
from coord.state import load_session
|
|
1956
|
+
|
|
1957
|
+
sess = load_session()
|
|
1958
|
+
if sess and sess.get("started_at"):
|
|
1959
|
+
import datetime
|
|
1960
|
+
try:
|
|
1961
|
+
dt = datetime.datetime.fromisoformat(
|
|
1962
|
+
sess["started_at"].rstrip("Z").replace("Z", "+00:00")
|
|
1963
|
+
)
|
|
1964
|
+
started_at = dt.replace(tzinfo=datetime.timezone.utc).timestamp()
|
|
1965
|
+
return Window(start=started_at, end=None, label="session")
|
|
1966
|
+
except (ValueError, AttributeError):
|
|
1967
|
+
pass
|
|
1968
|
+
return Window(start=None, end=None, label="all")
|
|
1969
|
+
|
|
1970
|
+
|
|
1971
|
+
def _usage_resolve_sort(sort_by: str | None, *, default: str) -> str:
|
|
1972
|
+
"""Resolve the ``--sort`` flag: explicit value wins, else *default*
|
|
1973
|
+
(#1119 — different views default to a different natural ranking key)."""
|
|
1974
|
+
return sort_by or default
|
|
1975
|
+
|
|
1976
|
+
|
|
1977
|
+
def _usage_sort_key(sort_by: str):
|
|
1978
|
+
"""Key function for sorting an :func:`~coord.usage_rollup.aggregate`
|
|
1979
|
+
``groups`` list by *sort_by* (``cost``/``tokens``/``time``), descending."""
|
|
1980
|
+
if sort_by == "tokens":
|
|
1981
|
+
def _total_tokens(group: dict) -> int:
|
|
1982
|
+
t = group["tokens"]
|
|
1983
|
+
return t["input"] + t["output"] + t["cache_read"] + t["cache_creation"]
|
|
1984
|
+
return _total_tokens
|
|
1985
|
+
if sort_by == "time":
|
|
1986
|
+
return lambda group: group["duration_secs"]
|
|
1987
|
+
return lambda group: group["cost_total"]
|
|
1988
|
+
|
|
1989
|
+
|
|
1990
|
+
def _usage_by_issue(
|
|
1991
|
+
config_path: Path, *, today: bool, week: bool, month: bool, since_spec: str | None, sort_by: str
|
|
1992
|
+
) -> None:
|
|
1993
|
+
"""``coord usage --by-issue`` (contract Mock 1, #1115) — daemon-board-
|
|
1994
|
+
sourced per-issue cost/token rollup for the resolved time window."""
|
|
1995
|
+
from coord.usage import fetch_usage_rows, format_usage_by_issue, pricing_dict_from_config
|
|
1996
|
+
from coord.usage_rollup import aggregate
|
|
1997
|
+
|
|
1998
|
+
cfg = _load_config(config_path)
|
|
1999
|
+
window = _usage_resolve_window(today, week, month, since_spec)
|
|
2000
|
+
rows = fetch_usage_rows()
|
|
2001
|
+
pricing = pricing_dict_from_config(cfg.pricing)
|
|
2002
|
+
result = aggregate(rows, by="issue", window=window, pricing=pricing)
|
|
2003
|
+
result["groups"].sort(key=_usage_sort_key(sort_by), reverse=True)
|
|
2004
|
+
|
|
2005
|
+
click.echo(format_usage_by_issue(result, window.label))
|
|
2006
|
+
|
|
2007
|
+
|
|
2008
|
+
def _usage_issue_drill(
|
|
2009
|
+
config_path: Path, issue_number: int, *, today: bool, week: bool, month: bool, since_spec: str | None
|
|
2010
|
+
) -> None:
|
|
2011
|
+
"""``coord usage --issue N`` (contract Mock 2, #1115) — per-stage drill
|
|
2012
|
+
for one issue's legs. Unbounded (all history) unless --today/--week/
|
|
2013
|
+
--month/--since is also given."""
|
|
2014
|
+
from coord.usage import fetch_usage_rows, format_usage_issue_drill
|
|
2015
|
+
from coord.usage_rollup import leg_in_window, row_issue_number
|
|
2016
|
+
|
|
2017
|
+
cfg = _load_config(config_path)
|
|
2018
|
+
has_window_flag = today or week or month or since_spec
|
|
2019
|
+
window = _usage_resolve_window(today, week, month, since_spec) if has_window_flag else None
|
|
2020
|
+
# #1553: select by the *attributed* issue, matching the `--by issue`
|
|
2021
|
+
# summary above. Selecting on the raw `issue_number` while the summary
|
|
2022
|
+
# groups on `for_issue_number` would make the two views disagree — the
|
|
2023
|
+
# epic's drill would list slice legs the summary had already moved to
|
|
2024
|
+
# the child, and the child's drill would be empty.
|
|
2025
|
+
rows = [
|
|
2026
|
+
row
|
|
2027
|
+
for row in fetch_usage_rows()
|
|
2028
|
+
if row_issue_number(row) == issue_number
|
|
2029
|
+
and (window is None or leg_in_window(row, window))
|
|
2030
|
+
]
|
|
2031
|
+
click.echo(format_usage_issue_drill(rows, issue_number, cfg.pricing))
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
def _usage_by_dim(
|
|
2035
|
+
config_path: Path,
|
|
2036
|
+
by_dim: str,
|
|
2037
|
+
*,
|
|
2038
|
+
today: bool,
|
|
2039
|
+
week: bool,
|
|
2040
|
+
month: bool,
|
|
2041
|
+
since_spec: str | None,
|
|
2042
|
+
sort_by: str,
|
|
2043
|
+
) -> None:
|
|
2044
|
+
"""``coord usage --by repo|week|month`` (#1119) — cross-cut daemon-board
|
|
2045
|
+
usage by *by_dim* for the resolved time window. ``repo`` is the
|
|
2046
|
+
cross-repo rollup (contract Mock 3); ``week``/``month`` are a
|
|
2047
|
+
time-bucketed spend series over a wider window (e.g. ``--since 8w --by
|
|
2048
|
+
week``)."""
|
|
2049
|
+
from coord.usage import fetch_usage_rows, format_usage_by_group, pricing_dict_from_config
|
|
2050
|
+
from coord.usage_rollup import aggregate
|
|
2051
|
+
|
|
2052
|
+
cfg = _load_config(config_path)
|
|
2053
|
+
window = _usage_resolve_window(today, week, month, since_spec)
|
|
2054
|
+
rows = fetch_usage_rows()
|
|
2055
|
+
pricing = pricing_dict_from_config(cfg.pricing)
|
|
2056
|
+
result = aggregate(rows, by=by_dim, window=window, pricing=pricing)
|
|
2057
|
+
result["groups"].sort(key=_usage_sort_key(sort_by), reverse=True)
|
|
2058
|
+
|
|
2059
|
+
click.echo(format_usage_by_group(result, window.label, by_dim))
|
|
2060
|
+
|
|
2061
|
+
|
|
2062
|
+
def _usage_by_time(
|
|
2063
|
+
config_path: Path,
|
|
2064
|
+
*,
|
|
2065
|
+
today: bool,
|
|
2066
|
+
week: bool,
|
|
2067
|
+
month: bool,
|
|
2068
|
+
since_spec: str | None,
|
|
2069
|
+
by_dim: str | None,
|
|
2070
|
+
sort_by: str | None,
|
|
2071
|
+
) -> None:
|
|
2072
|
+
"""``coord usage --by-time`` (contract Mock 4, #1119) — ranks where
|
|
2073
|
+
wall-clock is going: by stage-type (default) or, combined with
|
|
2074
|
+
``--by issue``, by issue. Defaults to ranking by ``time`` (that's the
|
|
2075
|
+
point of the view) unless ``--sort`` explicitly overrides."""
|
|
2076
|
+
from coord.usage import fetch_usage_rows, format_usage_by_time, pricing_dict_from_config
|
|
2077
|
+
from coord.usage_rollup import aggregate
|
|
2078
|
+
|
|
2079
|
+
dim = "issue" if by_dim == "issue" else "stage"
|
|
2080
|
+
resolved_sort = _usage_resolve_sort(sort_by, default="time")
|
|
2081
|
+
|
|
2082
|
+
cfg = _load_config(config_path)
|
|
2083
|
+
window = _usage_resolve_window(today, week, month, since_spec)
|
|
2084
|
+
rows = fetch_usage_rows()
|
|
2085
|
+
pricing = pricing_dict_from_config(cfg.pricing)
|
|
2086
|
+
result = aggregate(rows, by=dim, window=window, pricing=pricing)
|
|
2087
|
+
result["groups"].sort(key=_usage_sort_key(resolved_sort), reverse=True)
|
|
2088
|
+
|
|
2089
|
+
click.echo(format_usage_by_time(result, window.label, dim))
|