code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/interactive.py
ADDED
|
@@ -0,0 +1,4286 @@
|
|
|
1
|
+
"""Human-attended interactive `claude` launcher (#437).
|
|
2
|
+
|
|
3
|
+
This module owns the **only** path that may launch a provider whose
|
|
4
|
+
:attr:`~coord.providers.base.Capabilities.human_attended_only` flag is
|
|
5
|
+
``True`` — interactive Claude Code on a Max/Pro subscription. It is
|
|
6
|
+
invoked from ``coord assign --interactive`` and attaches the child to
|
|
7
|
+
the operator's local TTY via one of two strategies:
|
|
8
|
+
|
|
9
|
+
**PTY relay (fallback, no tmux)**
|
|
10
|
+
|
|
11
|
+
* A new pty pair is opened; the child execs ``claude`` with the slave fd
|
|
12
|
+
as its stdin/stdout/stderr.
|
|
13
|
+
* The parent relays bytes between the operator's TTY and the master fd
|
|
14
|
+
(raw-mode stdin → master, master → stdout), so the human drives the
|
|
15
|
+
session as if they had typed ``claude`` themselves.
|
|
16
|
+
* Once the TUI emits the bracketed-paste-enable DECSET (``ESC[?2004h``),
|
|
17
|
+
the rendered input box (:data:`~coord.providers.claude_pty.INPUT_BOX_MARKER`)
|
|
18
|
+
has appeared, AND the render has gone quiet, the briefing is PRE-FILLED
|
|
19
|
+
into the input box as a single bracketed paste — NO trailing carriage
|
|
20
|
+
return. The operator presses Enter to submit. The paste is then
|
|
21
|
+
**verified** (re-observing the master output for a fingerprint of the
|
|
22
|
+
briefing) and **retried** on a miss (#865: a mistimed paste — e.g. one
|
|
23
|
+
that races an async startup banner like "Fable 5 is back" — used to be
|
|
24
|
+
silently dropped with no way to tell). See :class:`_PrefillState`.
|
|
25
|
+
* Window-size changes (SIGWINCH) are forwarded to the child via
|
|
26
|
+
``TIOCSWINSZ`` so the TUI re-flows correctly when the operator
|
|
27
|
+
resizes their terminal.
|
|
28
|
+
|
|
29
|
+
**tmux session (preferred, when tmux is available and assignment_id is
|
|
30
|
+
provided — #487)**
|
|
31
|
+
|
|
32
|
+
* A named tmux session ``coord-<assignment_id>`` is created (or reused).
|
|
33
|
+
* ``claude`` runs directly inside the tmux session; tmux provides the pty.
|
|
34
|
+
* The briefing is injected via :func:`_inject_briefing_into_tmux_session`:
|
|
35
|
+
a marker+quiescence-anchored poll on the pane output, then
|
|
36
|
+
``tmux paste-buffer -p`` (bracketed-paste mode), then a re-``capture-pane``
|
|
37
|
+
verification with retry on a miss — same #865 rationale as the PTY path.
|
|
38
|
+
* The operator's terminal ATTACHES to the tmux session
|
|
39
|
+
(``tmux attach-session -t coord-<aid>``); if the TUI crashes the
|
|
40
|
+
attachment disconnects but **the tmux session and claude keep running**.
|
|
41
|
+
* On TUI restart the user can reattach with ``coord reattach <aid>`` or
|
|
42
|
+
via the TUI's Pipeline→Terminal tab (which checks for a live
|
|
43
|
+
``coord-*`` session for the open issue and offers reattach).
|
|
44
|
+
|
|
45
|
+
The session is HUMAN-CLOSED — the relay loop / tmux session exits when
|
|
46
|
+
the child exits and the launcher reports the child's exit code to the
|
|
47
|
+
caller. This module deliberately contains **no** content-based
|
|
48
|
+
completion detection and **no** TTY scraper. The structural
|
|
49
|
+
ToS-compliance posture the abandoned #426 was missing.
|
|
50
|
+
|
|
51
|
+
After exit, the CLI caller invokes :func:`finalize_interactive_exit`
|
|
52
|
+
(the #466 git-floor backstop) which computes the worktree's
|
|
53
|
+
commits-ahead using the #448 ``_commits_ahead`` primitive, pushes any
|
|
54
|
+
local commits, and writes the terminal completion through the
|
|
55
|
+
:mod:`coord.issue_store` seam. That guarantees the board always gets a
|
|
56
|
+
recorded completion regardless of whether the interactive agent
|
|
57
|
+
remembered to call ``coord report-result`` first.
|
|
58
|
+
|
|
59
|
+
When the tmux path is used and the TUI crashes (the attach disconnects
|
|
60
|
+
while the session is still live), the CLI caller skips
|
|
61
|
+
``finalize_interactive_exit`` and instead shows a reattach hint.
|
|
62
|
+
Finalize runs when the user reattaches via ``coord reattach`` and the
|
|
63
|
+
session eventually ends.
|
|
64
|
+
|
|
65
|
+
Imports are deferred / Unix-only — the stdlib ``pty`` / ``termios`` /
|
|
66
|
+
``fcntl`` modules are not present on Windows, but agent machines are
|
|
67
|
+
Linux/macOS only.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
from __future__ import annotations
|
|
71
|
+
|
|
72
|
+
import json
|
|
73
|
+
import logging
|
|
74
|
+
import os
|
|
75
|
+
import re
|
|
76
|
+
import select
|
|
77
|
+
import shlex
|
|
78
|
+
import shutil
|
|
79
|
+
import signal
|
|
80
|
+
import socket
|
|
81
|
+
import struct
|
|
82
|
+
import subprocess
|
|
83
|
+
import sys
|
|
84
|
+
import tempfile
|
|
85
|
+
import time
|
|
86
|
+
from dataclasses import dataclass, field
|
|
87
|
+
from pathlib import Path
|
|
88
|
+
from typing import TYPE_CHECKING, Sequence
|
|
89
|
+
|
|
90
|
+
if TYPE_CHECKING:
|
|
91
|
+
from coord.agent import MergeVerify
|
|
92
|
+
from coord.config import Config
|
|
93
|
+
from coord.models import Board
|
|
94
|
+
|
|
95
|
+
from coord.providers.claude_pty import (
|
|
96
|
+
BRACKETED_PASTE_ENABLE,
|
|
97
|
+
BRACKETED_PASTE_END,
|
|
98
|
+
BRACKETED_PASTE_START,
|
|
99
|
+
INPUT_BOX_MARKER,
|
|
100
|
+
INPUT_BOX_MARKER_BYTES,
|
|
101
|
+
briefing_fingerprint,
|
|
102
|
+
paste_landed,
|
|
103
|
+
paste_landed_bytes,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
__all__ = [
|
|
107
|
+
"launch_human_attended_interactive",
|
|
108
|
+
"finalize_interactive_exit",
|
|
109
|
+
"reap_stale_interactive_sessions",
|
|
110
|
+
"reap_stale_remote_interactive_sessions",
|
|
111
|
+
"InteractiveFinalizeResult",
|
|
112
|
+
"TMUX_SESSION_PREFIX",
|
|
113
|
+
"TERM_SESSION_PREFIX",
|
|
114
|
+
"DRIVE_SESSION_PREFIX",
|
|
115
|
+
"TmuxHost",
|
|
116
|
+
"tmux_session_name",
|
|
117
|
+
"tmux_available",
|
|
118
|
+
"tmux_session_alive",
|
|
119
|
+
"tmux_pane_dead",
|
|
120
|
+
"list_coord_tmux_sessions",
|
|
121
|
+
"TMUX_ATTACH_WARNING",
|
|
122
|
+
"PTY_RELAY_NO_DETACH_WARNING",
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ── tmux session management (#487) ───────────────────────────────────────────
|
|
127
|
+
|
|
128
|
+
#: Prefix for all coordinator-managed tmux sessions. The session name is
|
|
129
|
+
#: ``coord-<assignment_id>`` so that a ``tmux ls`` output can be filtered
|
|
130
|
+
#: cheaply and the assignment_id is directly recoverable.
|
|
131
|
+
TMUX_SESSION_PREFIX = "coord-"
|
|
132
|
+
|
|
133
|
+
#: Prefix for free-floating "plain terminal" tmux sessions (#952) — shells
|
|
134
|
+
#: not tied to any issue/assignment, created via ``coord terminal new``.
|
|
135
|
+
#: Deliberately a DISTINCT, longer prefix than :data:`TMUX_SESSION_PREFIX`
|
|
136
|
+
#: (which it also starts with) so the two kinds never collide on a name but
|
|
137
|
+
#: each side can still filter its own: assignment-session discovery
|
|
138
|
+
#: (:func:`list_coord_tmux_sessions`) explicitly excludes this prefix, and
|
|
139
|
+
#: ``coord terminal list`` (``coord/commands/terminal.py``) matches only it.
|
|
140
|
+
TERM_SESSION_PREFIX = "coord-term-"
|
|
141
|
+
|
|
142
|
+
#: Prefix for detached ``coord drive --tmux`` sessions (#1398) — the
|
|
143
|
+
#: unattended single-issue driver running in the background so it survives
|
|
144
|
+
#: the launching terminal (or the TUI) closing. Session name is
|
|
145
|
+
#: ``coord-drive-<repo>-<issue>``. Distinct from :data:`TMUX_SESSION_PREFIX`
|
|
146
|
+
#: for the same reason :data:`TERM_SESSION_PREFIX` is: it also starts with
|
|
147
|
+
#: ``"coord-"``, so assignment-session discovery
|
|
148
|
+
#: (:func:`list_coord_tmux_sessions`) explicitly excludes it too — a drive
|
|
149
|
+
#: session carries a repo/issue, not an assignment_id, and must not show up
|
|
150
|
+
#: as a phantom "(unknown issue)" row in ``coord sessions``/reattach
|
|
151
|
+
#: discovery. ``coord drive-sessions`` (``coord/drive.py``) is the
|
|
152
|
+
#: dedicated surface for it.
|
|
153
|
+
DRIVE_SESSION_PREFIX = "coord-drive-"
|
|
154
|
+
|
|
155
|
+
#: SSH connection-multiplexing options for remote (#486/#494) tmux calls.
|
|
156
|
+
#: One interactive launch fires ~5+ separate ssh invocations (has-session →
|
|
157
|
+
#: new-session → a 50 ms ``capture-pane`` readiness poll → load/paste-buffer →
|
|
158
|
+
#: attach → alive-check). Without multiplexing each is a fresh SSH auth, so a
|
|
159
|
+
#: passphrase-protected key prompts once *per call* — a wall of prompts the
|
|
160
|
+
#: operator has to clear. ``ControlMaster=auto`` opens the connection once and
|
|
161
|
+
#: every subsequent call reuses it over the same socket (one auth per launch,
|
|
162
|
+
#: and zero within ``ControlPersist`` of the previous launch). ``%C`` hashes
|
|
163
|
+
#: host/port/user so the socket path stays short and per-destination. Applied
|
|
164
|
+
#: only on the remote path; the local (``ssh_target=None``) argv is unchanged.
|
|
165
|
+
_SSH_MUX_OPTS = [
|
|
166
|
+
"-o", "ControlMaster=auto",
|
|
167
|
+
"-o", "ControlPath=~/.ssh/coord-mux-%C",
|
|
168
|
+
"-o", "ControlPersist=120s",
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
# ── #1102 attach/kill guard ─────────────────────────────────────────────────
|
|
172
|
+
#
|
|
173
|
+
# A tmux-hosted coord session has exactly one pane. Detach (Ctrl-b d) drops
|
|
174
|
+
# the operator's client but leaves the session (and the ``claude`` process
|
|
175
|
+
# inside it) running server-side -- the intended "stop watching here" action.
|
|
176
|
+
# Kill-pane (Ctrl-b x, ``exit``, closing the window) instead kills the pane's
|
|
177
|
+
# process, and since it's the session's only pane, tmux immediately destroys
|
|
178
|
+
# the whole session with it -- silently, with no confirmation and no way
|
|
179
|
+
# back. Printed at every attach point (fresh launch, reuse-after-crash, and
|
|
180
|
+
# ``coord reattach``) so the distinction is in front of the operator at the
|
|
181
|
+
# moment it matters, not just documented in a docstring nobody re-reads
|
|
182
|
+
# mid-session.
|
|
183
|
+
TMUX_ATTACH_WARNING = (
|
|
184
|
+
"\n"
|
|
185
|
+
"=== live session -- read before you touch the keyboard =====================\n"
|
|
186
|
+
" Detach Ctrl-b d -> claude keeps running; reattach anytime with\n"
|
|
187
|
+
" `coord reattach <assignment-id>`.\n"
|
|
188
|
+
" DO NOT Ctrl-b x / exit / -> KILLS the running claude process, and since\n"
|
|
189
|
+
" close this window this session has exactly one pane, tmux\n"
|
|
190
|
+
" destroys the WHOLE session with it. No undo.\n"
|
|
191
|
+
"==============================================================================\n"
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
#: Companion warning for the no-tmux PTY-relay fallback (:func:`_launch_via_pty`).
|
|
195
|
+
#: There is no tmux layer here at all, so there is no detach: the operator's
|
|
196
|
+
#: terminal *is* the session, and closing it (or Ctrl-C) ends ``claude``
|
|
197
|
+
#: immediately with no way to pick it back up.
|
|
198
|
+
PTY_RELAY_NO_DETACH_WARNING = (
|
|
199
|
+
"\n"
|
|
200
|
+
"=== no detach available (tmux not in use for this session) =================\n"
|
|
201
|
+
" This session is attached directly to your terminal -- there is no\n"
|
|
202
|
+
" detach. Closing this window or Ctrl-C ends the claude process\n"
|
|
203
|
+
" immediately and it cannot be resumed.\n"
|
|
204
|
+
"==============================================================================\n"
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
#: How long to hold the terminal after printing
|
|
208
|
+
#: :data:`PTY_RELAY_NO_DETACH_WARNING` and before ``pty.fork()`` hands the
|
|
209
|
+
#: screen to the child (#1102 follow-up). Long enough for the banner to
|
|
210
|
+
#: register with the operator, short enough not to feel like a hang.
|
|
211
|
+
_PTY_NO_DETACH_WARNING_PAUSE_S = 1.5
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _get_local_short_hostname() -> str:
|
|
215
|
+
"""Return the short hostname of the local machine (split on '.' and lowercased).
|
|
216
|
+
|
|
217
|
+
Isolated into a helper so tests can patch ``coord.interactive._get_local_short_hostname``
|
|
218
|
+
without monkey-patching the global ``socket`` module.
|
|
219
|
+
"""
|
|
220
|
+
return socket.gethostname().split(".")[0].lower()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True)
|
|
224
|
+
class TmuxHost:
|
|
225
|
+
"""Seam that resolves tmux subprocess commands for local or remote hosts.
|
|
226
|
+
|
|
227
|
+
``ssh_target=None`` (the default) means the local machine; subprocess
|
|
228
|
+
calls are plain ``["tmux", ...]``. When ``ssh_target`` is set, calls
|
|
229
|
+
become ``["ssh", ssh_target, "tmux", ...]`` — optionally with ``-t``
|
|
230
|
+
for commands that need a TTY (e.g. ``attach-session``).
|
|
231
|
+
|
|
232
|
+
This seam is introduced in #493 to unblock #486b (remote tmux). No
|
|
233
|
+
remote callers exist yet; all production call-sites use the default
|
|
234
|
+
``TmuxHost(None)`` which preserves the exact same subprocess argv as
|
|
235
|
+
before.
|
|
236
|
+
|
|
237
|
+
``batch=True`` adds ``BatchMode=yes`` (+ a short ``ConnectTimeout``) so the
|
|
238
|
+
ssh call NEVER prompts for a passphrase/password — it fails fast instead.
|
|
239
|
+
Use it for **non-interactive, background** probes (e.g. the `coord sessions
|
|
240
|
+
--remote` fleet sweep the TUI runs at startup): without it, ssh grabs the
|
|
241
|
+
controlling terminal to ask for the key passphrase, corrupting the TUI
|
|
242
|
+
display (#486 Leg 4 regression). Leave it ``False`` for human-attended
|
|
243
|
+
paths (launch / reattach) that legitimately prompt once.
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
ssh_target: str | None # None => local
|
|
247
|
+
batch: bool = False # True => BatchMode=yes (never prompt; fail fast)
|
|
248
|
+
|
|
249
|
+
def cmd(self, tmux_args: list[str], *, tty: bool = False) -> list[str]:
|
|
250
|
+
"""Build the full subprocess argv for a tmux invocation.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
tmux_args: The tmux sub-command and its arguments, *without*
|
|
254
|
+
the leading ``"tmux"`` token.
|
|
255
|
+
tty: When ``True`` and the host is remote, ``-t`` is inserted
|
|
256
|
+
after ``"ssh"`` so the remote side allocates a pseudo-TTY.
|
|
257
|
+
Use ``True`` only for interactive commands like
|
|
258
|
+
``attach-session``; leave ``False`` (the default) for all
|
|
259
|
+
control commands (has-session, ls, capture-pane, etc.).
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
A complete ``subprocess.run``-ready command list.
|
|
263
|
+
"""
|
|
264
|
+
if self.ssh_target is None:
|
|
265
|
+
return ["tmux", *tmux_args]
|
|
266
|
+
# BatchMode must precede the destination; it disables every interactive
|
|
267
|
+
# auth prompt (passphrase/password) so a background probe degrades to a
|
|
268
|
+
# fast failure instead of hijacking the terminal. An existing
|
|
269
|
+
# ControlMaster socket / ssh-agent key still authenticates silently.
|
|
270
|
+
batch_opts = (
|
|
271
|
+
["-o", "BatchMode=yes", "-o", "ConnectTimeout=4"] if self.batch else []
|
|
272
|
+
)
|
|
273
|
+
# ssh space-joins the remote command args (NO escaping) and runs the
|
|
274
|
+
# result through the remote LOGIN SHELL. Passing a tmux format like
|
|
275
|
+
# ``#{session_name}`` as a bare token means that shell sees ``#`` — a
|
|
276
|
+
# comment marker — and silently truncates the command
|
|
277
|
+
# (``tmux ls -F #{session_name}`` → ``tmux ls -F``, which errors and
|
|
278
|
+
# returns nothing). This broke remote session discovery for `coord
|
|
279
|
+
# sessions --remote` and the TUI reattach sweep. Shell-quote each tmux
|
|
280
|
+
# arg so the remote shell receives it verbatim. Plain tokens (``ls``,
|
|
281
|
+
# ``-t``, ``coord-<id>``) quote to themselves, so the launch/attach
|
|
282
|
+
# argv is byte-for-byte unchanged; only shell-special chars (``#{}``,
|
|
283
|
+
# spaces) get protected.
|
|
284
|
+
return [
|
|
285
|
+
"ssh",
|
|
286
|
+
*(["-t"] if tty else []),
|
|
287
|
+
*batch_opts,
|
|
288
|
+
*_SSH_MUX_OPTS,
|
|
289
|
+
self.ssh_target,
|
|
290
|
+
"tmux",
|
|
291
|
+
*(shlex.quote(a) for a in tmux_args),
|
|
292
|
+
]
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def tmux_session_name(assignment_id: str) -> str:
|
|
296
|
+
"""Return the canonical tmux session name for *assignment_id*."""
|
|
297
|
+
return f"{TMUX_SESSION_PREFIX}{assignment_id}"
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def tmux_available() -> bool:
|
|
301
|
+
"""Return ``True`` when ``tmux`` is on the current ``PATH``."""
|
|
302
|
+
return shutil.which("tmux") is not None
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def tmux_session_alive(
|
|
306
|
+
session_name: str,
|
|
307
|
+
*,
|
|
308
|
+
host: TmuxHost = TmuxHost(None),
|
|
309
|
+
) -> bool:
|
|
310
|
+
"""Return ``True`` when the named tmux session exists and is running.
|
|
311
|
+
|
|
312
|
+
Uses ``tmux has-session`` which exits 0 when the session is alive and 1
|
|
313
|
+
when it does not exist. Subprocess / OS errors are treated as
|
|
314
|
+
"not alive" so callers don't need try/except.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
session_name: The tmux session name to probe.
|
|
318
|
+
host: Target host. Defaults to ``TmuxHost(None)`` (local).
|
|
319
|
+
"""
|
|
320
|
+
try:
|
|
321
|
+
result = subprocess.run(
|
|
322
|
+
host.cmd(["has-session", "-t", session_name]),
|
|
323
|
+
capture_output=True,
|
|
324
|
+
timeout=5.0,
|
|
325
|
+
)
|
|
326
|
+
return result.returncode == 0
|
|
327
|
+
except (subprocess.SubprocessError, OSError):
|
|
328
|
+
return False
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def tmux_pane_dead(
|
|
332
|
+
session_name: str,
|
|
333
|
+
*,
|
|
334
|
+
host: TmuxHost = TmuxHost(None),
|
|
335
|
+
) -> bool:
|
|
336
|
+
"""Return ``True`` when the named session exists but its pane process has exited.
|
|
337
|
+
|
|
338
|
+
A "dead pane" means the tmux session is still up (``tmux has-session``
|
|
339
|
+
returns 0) but the child process that was running inside the pane —
|
|
340
|
+
typically ``claude`` — has exited. This happens when the operator detaches
|
|
341
|
+
while the session is running and claude finishes in the background.
|
|
342
|
+
|
|
343
|
+
Uses ``tmux list-panes -F "#{pane_dead}" -t <session>``, which emits
|
|
344
|
+
``1`` when a pane's child process has exited and ``0`` while it is still
|
|
345
|
+
running. Returns ``True`` only when **all** panes in the session report
|
|
346
|
+
dead (i.e. at least one alive pane keeps the session active).
|
|
347
|
+
|
|
348
|
+
Subprocess / OS errors return ``False`` (treat as alive to avoid spurious
|
|
349
|
+
reaping of sessions we can't probe).
|
|
350
|
+
|
|
351
|
+
Args:
|
|
352
|
+
session_name: The tmux session name to probe.
|
|
353
|
+
host: Target host. Defaults to ``TmuxHost(None)`` (local).
|
|
354
|
+
"""
|
|
355
|
+
try:
|
|
356
|
+
result = subprocess.run(
|
|
357
|
+
host.cmd(["list-panes", "-F", "#{pane_dead}", "-t", session_name]),
|
|
358
|
+
capture_output=True,
|
|
359
|
+
text=True,
|
|
360
|
+
timeout=5.0,
|
|
361
|
+
)
|
|
362
|
+
if result.returncode != 0:
|
|
363
|
+
return False
|
|
364
|
+
lines = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
|
|
365
|
+
if not lines:
|
|
366
|
+
return False
|
|
367
|
+
# All panes must be dead for the session to be considered dead-pane.
|
|
368
|
+
return all(ln == "1" for ln in lines)
|
|
369
|
+
except (subprocess.SubprocessError, OSError):
|
|
370
|
+
return False
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def list_coord_tmux_sessions(
|
|
374
|
+
*,
|
|
375
|
+
host: TmuxHost = TmuxHost(None),
|
|
376
|
+
) -> list[dict[str, str]]:
|
|
377
|
+
"""Return a list of live ``coord-*`` tmux sessions.
|
|
378
|
+
|
|
379
|
+
Each entry is a ``dict`` with keys:
|
|
380
|
+
|
|
381
|
+
* ``"session_name"`` — the tmux session name (``coord-<assignment_id>``).
|
|
382
|
+
* ``"pane_dead"`` — ``"1"`` when the session's pane process has exited
|
|
383
|
+
(``claude`` finished but the tmux session is still up — the
|
|
384
|
+
detach-and-abandon case), ``"0"`` while the pane is still running.
|
|
385
|
+
* ``"attached"`` — ``bool``, ``True`` when a client is currently attached
|
|
386
|
+
to the tmux session (#1031). Mirrors
|
|
387
|
+
:func:`coord.commands.terminal.list_tmux_terminal_sessions`'s
|
|
388
|
+
``#{session_attached}`` handling so the TUI Sessions panel can flag
|
|
389
|
+
orphaned/detached sessions at a glance.
|
|
390
|
+
|
|
391
|
+
Returns an empty list when tmux is not available, not running, or has
|
|
392
|
+
no matching sessions.
|
|
393
|
+
|
|
394
|
+
Uses ``tmux list-panes -a -F "#{session_name}\t#{pane_dead}\t#{session_attached}"``
|
|
395
|
+
to fetch the session name, pane-dead status, and attached status in a
|
|
396
|
+
single subprocess call. When a session has multiple panes the *most
|
|
397
|
+
conservative* (alive=0) value wins for ``pane_dead`` — i.e. the session
|
|
398
|
+
is only marked dead when every pane has exited; ``session_attached`` is
|
|
399
|
+
a session-level (not pane-level) tmux value, so every pane row for a
|
|
400
|
+
given session reports the same value, but any row showing attached wins
|
|
401
|
+
(defensive OR) to avoid a stray malformed line flipping it to detached.
|
|
402
|
+
|
|
403
|
+
Free-floating ``coord-term-*`` sessions (#952, ``coord terminal new``)
|
|
404
|
+
and ``coord-drive-*`` unattended-driver sessions (#1398, ``coord drive
|
|
405
|
+
--tmux``) are explicitly excluded even though they also start with
|
|
406
|
+
:data:`TMUX_SESSION_PREFIX` — neither carries an assignment_id, so
|
|
407
|
+
both would otherwise show up as a phantom "(unknown issue)" row in
|
|
408
|
+
``coord sessions``/reattach discovery. ``coord terminal list`` and
|
|
409
|
+
``coord drive-sessions`` are the dedicated surfaces for them.
|
|
410
|
+
|
|
411
|
+
Args:
|
|
412
|
+
host: Target host. Defaults to ``TmuxHost(None)`` (local). This
|
|
413
|
+
same function drives the ``--remote`` sweep in ``coord sessions``
|
|
414
|
+
(a raw ``ssh <host> tmux list-panes ...`` call — never remote
|
|
415
|
+
``coord sessions --json``), so ``attached`` is carried through
|
|
416
|
+
for remote hosts with no agent release / ``coord agent update``
|
|
417
|
+
required (#1031).
|
|
418
|
+
|
|
419
|
+
Example::
|
|
420
|
+
|
|
421
|
+
[
|
|
422
|
+
{"session_name": "coord-abc123", "pane_dead": "0", "attached": True},
|
|
423
|
+
{"session_name": "coord-def456", "pane_dead": "1", "attached": False},
|
|
424
|
+
]
|
|
425
|
+
"""
|
|
426
|
+
try:
|
|
427
|
+
result = subprocess.run(
|
|
428
|
+
host.cmd([
|
|
429
|
+
"list-panes", "-a", "-F",
|
|
430
|
+
"#{session_name}\t#{pane_dead}\t#{session_attached}",
|
|
431
|
+
]),
|
|
432
|
+
capture_output=True,
|
|
433
|
+
text=True,
|
|
434
|
+
timeout=5.0,
|
|
435
|
+
)
|
|
436
|
+
if result.returncode != 0:
|
|
437
|
+
return []
|
|
438
|
+
# Collect per-session: "0" (alive) beats "1" (dead) — any alive pane
|
|
439
|
+
# keeps the session active. Attached: any row reporting attached wins.
|
|
440
|
+
pane_dead_per_session: dict[str, str] = {}
|
|
441
|
+
attached_per_session: dict[str, bool] = {}
|
|
442
|
+
for raw_line in result.stdout.splitlines():
|
|
443
|
+
parts = raw_line.split("\t")
|
|
444
|
+
if len(parts) < 2:
|
|
445
|
+
continue
|
|
446
|
+
name, pane_dead = parts[0].strip(), parts[1].strip()
|
|
447
|
+
if not name.startswith(TMUX_SESSION_PREFIX):
|
|
448
|
+
continue
|
|
449
|
+
if name.startswith(TERM_SESSION_PREFIX):
|
|
450
|
+
continue # #952: free-floating terminal, not an assignment session
|
|
451
|
+
if name.startswith(DRIVE_SESSION_PREFIX):
|
|
452
|
+
continue # #1398: unattended driver session, not an assignment session
|
|
453
|
+
existing = pane_dead_per_session.get(name)
|
|
454
|
+
# "0" (alive) wins over "1" (dead).
|
|
455
|
+
if existing is None or pane_dead == "0":
|
|
456
|
+
pane_dead_per_session[name] = pane_dead
|
|
457
|
+
attached_raw = parts[2].strip() if len(parts) > 2 else ""
|
|
458
|
+
is_attached = attached_raw not in ("", "0")
|
|
459
|
+
attached_per_session[name] = attached_per_session.get(name, False) or is_attached
|
|
460
|
+
return [
|
|
461
|
+
{
|
|
462
|
+
"session_name": name,
|
|
463
|
+
"pane_dead": pd,
|
|
464
|
+
"attached": attached_per_session.get(name, False),
|
|
465
|
+
}
|
|
466
|
+
for name, pd in pane_dead_per_session.items()
|
|
467
|
+
]
|
|
468
|
+
except (subprocess.SubprocessError, OSError):
|
|
469
|
+
return []
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _inject_briefing_into_tmux_session(
|
|
473
|
+
session_name: str,
|
|
474
|
+
briefing: str,
|
|
475
|
+
*,
|
|
476
|
+
timeout: float = 12.0,
|
|
477
|
+
host: TmuxHost = TmuxHost(None),
|
|
478
|
+
) -> bool:
|
|
479
|
+
"""Wait for the tmux pane to stabilise, inject *briefing*, then VERIFY it landed.
|
|
480
|
+
|
|
481
|
+
#865 fix: quiescence alone can't tell "static startup banner" from "input
|
|
482
|
+
box settled" — Claude Code's TUI paints async startup content (promo
|
|
483
|
+
banners, MCP/auth notices) over several seconds, and a paste that lands
|
|
484
|
+
mid-repaint used to be silently dropped. The injection is now two
|
|
485
|
+
phases:
|
|
486
|
+
|
|
487
|
+
1. **Readiness wait** — poll ``tmux capture-pane -p`` every 50 ms until
|
|
488
|
+
the pane both contains :data:`~coord.providers.claude_pty.INPUT_BOX_MARKER`
|
|
489
|
+
(the rendered input box, not just any static content) AND has been
|
|
490
|
+
unchanged for :data:`_READY_QUIESCE_S` seconds, or *timeout* lapses
|
|
491
|
+
(degraded fallback: proceed anyway rather than hang forever on an
|
|
492
|
+
unrecognised render).
|
|
493
|
+
2. **Paste + verify + retry** — load the briefing into a tmux named
|
|
494
|
+
buffer (``coord-brief``) via stdin and ``paste-buffer -p`` (bracketed
|
|
495
|
+
paste), then re-capture the pane and check whether a fingerprint of
|
|
496
|
+
the briefing (see :func:`~coord.providers.claude_pty.briefing_fingerprint`)
|
|
497
|
+
actually rendered. Retries up to :data:`_INJECT_MAX_ATTEMPTS` times
|
|
498
|
+
with a short backoff between attempts. If the pane can't be
|
|
499
|
+
captured at all (tmux/session gone), verification is impossible and
|
|
500
|
+
the single paste already issued is treated as best-effort success —
|
|
501
|
+
there's nothing more to learn by retrying blind. If every
|
|
502
|
+
observable attempt misses, a hard failure is logged (never silent)
|
|
503
|
+
and ``False`` is returned.
|
|
504
|
+
|
|
505
|
+
The ``-p`` flag makes tmux send the content wrapped in bracketed-paste
|
|
506
|
+
markers (``ESC[200~`` … ``ESC[201~``) **if** the target application has
|
|
507
|
+
requested bracketed-paste mode. Since ``claude``'s TUI always enables
|
|
508
|
+
bracketed paste, this is equivalent to the PTY relay's manual
|
|
509
|
+
bracketed-paste block.
|
|
510
|
+
|
|
511
|
+
The stdin-based ``load-buffer -`` approach avoids creating a local
|
|
512
|
+
temporary file, which is important for the remote-host path (#486b):
|
|
513
|
+
no ``scp`` or ``ssh`` file-transfer is needed.
|
|
514
|
+
|
|
515
|
+
Args:
|
|
516
|
+
session_name: The tmux session to inject into.
|
|
517
|
+
briefing: Text to pre-fill in the TUI input box.
|
|
518
|
+
timeout: Readiness-wait deadline in seconds. The paste+verify+retry
|
|
519
|
+
phase spends additional (bounded) time beyond this.
|
|
520
|
+
host: Target host. Defaults to ``TmuxHost(None)`` (local).
|
|
521
|
+
|
|
522
|
+
Returns:
|
|
523
|
+
``True`` when the briefing was injected (verified, or verification
|
|
524
|
+
was impossible and the paste was issued). ``False`` when every
|
|
525
|
+
verifiable attempt confirmed the briefing did NOT land.
|
|
526
|
+
"""
|
|
527
|
+
if not briefing.strip():
|
|
528
|
+
return True # nothing to inject — trivially OK
|
|
529
|
+
|
|
530
|
+
fingerprint = briefing_fingerprint(briefing)
|
|
531
|
+
|
|
532
|
+
def _capture() -> str | None:
|
|
533
|
+
try:
|
|
534
|
+
cap = subprocess.run(
|
|
535
|
+
host.cmd(["capture-pane", "-p", "-t", session_name]),
|
|
536
|
+
capture_output=True,
|
|
537
|
+
text=True,
|
|
538
|
+
timeout=2.0,
|
|
539
|
+
)
|
|
540
|
+
except (subprocess.SubprocessError, OSError):
|
|
541
|
+
return None
|
|
542
|
+
if cap.returncode != 0:
|
|
543
|
+
return None
|
|
544
|
+
return cap.stdout
|
|
545
|
+
|
|
546
|
+
def _paste_once() -> None:
|
|
547
|
+
# Load briefing into a named tmux buffer via stdin. Using "-" as
|
|
548
|
+
# the source tells tmux to read from stdin, which works for both
|
|
549
|
+
# local and remote hosts without temporary files.
|
|
550
|
+
subprocess.run(
|
|
551
|
+
host.cmd(["load-buffer", "-b", "coord-brief", "-"]),
|
|
552
|
+
input=briefing.rstrip("\n"),
|
|
553
|
+
text=True,
|
|
554
|
+
capture_output=True,
|
|
555
|
+
timeout=5.0,
|
|
556
|
+
)
|
|
557
|
+
# Paste with bracketed-paste enabled (``-p``).
|
|
558
|
+
subprocess.run(
|
|
559
|
+
host.cmd(["paste-buffer", "-p", "-t", session_name, "-b", "coord-brief"]),
|
|
560
|
+
capture_output=True,
|
|
561
|
+
timeout=5.0,
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
try:
|
|
565
|
+
# ── Phase 1: wait for the input box to render AND settle ───────────
|
|
566
|
+
deadline = time.monotonic() + timeout
|
|
567
|
+
prev_content: str | None = None
|
|
568
|
+
quiescent_since: float | None = None
|
|
569
|
+
|
|
570
|
+
while time.monotonic() < deadline:
|
|
571
|
+
time.sleep(0.05)
|
|
572
|
+
content = _capture()
|
|
573
|
+
if content is None:
|
|
574
|
+
break # session/tmux gone — degrade to a blind paste below
|
|
575
|
+
|
|
576
|
+
now = time.monotonic()
|
|
577
|
+
stable = bool(content.strip())
|
|
578
|
+
if content != prev_content:
|
|
579
|
+
prev_content = content
|
|
580
|
+
quiescent_since = now if stable else None
|
|
581
|
+
elif stable and quiescent_since is not None:
|
|
582
|
+
# A recognised input box needs less settle time to trust —
|
|
583
|
+
# we've seen the actual prompt render, not just SOME static
|
|
584
|
+
# content. When the marker never shows (older CLI, unusual
|
|
585
|
+
# render, or a session that died before drawing anything),
|
|
586
|
+
# fall back to a longer quiescence window rather than
|
|
587
|
+
# spinning for the full ``timeout`` (#865).
|
|
588
|
+
quiet_for = now - quiescent_since
|
|
589
|
+
threshold = (
|
|
590
|
+
_READY_QUIESCE_S
|
|
591
|
+
if INPUT_BOX_MARKER in content
|
|
592
|
+
else _READY_QUIESCE_NO_MARKER_S
|
|
593
|
+
)
|
|
594
|
+
if quiet_for >= threshold:
|
|
595
|
+
break # settled — inject
|
|
596
|
+
|
|
597
|
+
# ── Phase 2: paste, verify, retry on a miss (#865 / #896) ─────────
|
|
598
|
+
# Capture a baseline snapshot of the input-box region BEFORE the
|
|
599
|
+
# first paste so ``paste_landed`` can detect "box changed from
|
|
600
|
+
# empty placeholder" even when the paste-chip hides the literal
|
|
601
|
+
# fingerprint (#896).
|
|
602
|
+
baseline = _capture()
|
|
603
|
+
|
|
604
|
+
for attempt in range(1, _INJECT_MAX_ATTEMPTS + 1):
|
|
605
|
+
if attempt > 1:
|
|
606
|
+
# Idempotent retry (#896): clear any stacked paste chips
|
|
607
|
+
# before re-pasting so a false-negative on a successfully-
|
|
608
|
+
# landed paste doesn't stack duplicate chips.
|
|
609
|
+
subprocess.run(
|
|
610
|
+
host.cmd(["send-keys", "-t", session_name, "Escape"]),
|
|
611
|
+
capture_output=True,
|
|
612
|
+
timeout=2.0,
|
|
613
|
+
check=False,
|
|
614
|
+
)
|
|
615
|
+
time.sleep(0.05)
|
|
616
|
+
subprocess.run(
|
|
617
|
+
host.cmd(["send-keys", "-t", session_name, "C-u"]),
|
|
618
|
+
capture_output=True,
|
|
619
|
+
timeout=2.0,
|
|
620
|
+
check=False,
|
|
621
|
+
)
|
|
622
|
+
_paste_once()
|
|
623
|
+
time.sleep(_INJECT_VERIFY_SETTLE_S)
|
|
624
|
+
content = _capture()
|
|
625
|
+
if content is None:
|
|
626
|
+
# Can't observe the pane — nothing more to learn from a
|
|
627
|
+
# retry. The paste itself was issued; treat as best-effort
|
|
628
|
+
# success (matches pre-#865 behaviour when tmux is broken).
|
|
629
|
+
return True
|
|
630
|
+
if paste_landed(content, fingerprint=fingerprint, baseline=baseline):
|
|
631
|
+
return True
|
|
632
|
+
if attempt < _INJECT_MAX_ATTEMPTS:
|
|
633
|
+
time.sleep(_INJECT_RETRY_BACKOFF_S)
|
|
634
|
+
|
|
635
|
+
logging.error(
|
|
636
|
+
"briefing injection unverified after %d attempt(s) for tmux "
|
|
637
|
+
"session %r — the briefing may not have landed in the input "
|
|
638
|
+
"box; the operator should check and paste manually",
|
|
639
|
+
_INJECT_MAX_ATTEMPTS,
|
|
640
|
+
session_name,
|
|
641
|
+
)
|
|
642
|
+
return False
|
|
643
|
+
except (subprocess.SubprocessError, OSError):
|
|
644
|
+
return False
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _with_coord_on_path(shell_cmd: str) -> str:
|
|
648
|
+
"""Prefix *shell_cmd* so ``coord`` resolves inside the interactive session
|
|
649
|
+
(#606 PATH-fix).
|
|
650
|
+
|
|
651
|
+
Interactive sessions run under the machine's ``$SHELL``, which often lacks
|
|
652
|
+
the coord venv bin on ``PATH`` (same class as the claude-not-on-ssh-PATH
|
|
653
|
+
issue). A human-attended REVIEW agent then can't run ``coord
|
|
654
|
+
report-result`` and falls back to printing ``REVIEW_VERDICT:`` (recovered by
|
|
655
|
+
the #606 transcript-floor). Prepend the agent's coord venv bin — the
|
|
656
|
+
``~/.coord-venv`` install INVARIANT, kept as a literal ``$HOME`` so the
|
|
657
|
+
session's own shell expands it on the machine the session runs on — so the
|
|
658
|
+
agent's PREFERRED self-report path works. Additive: a non-existent PATH
|
|
659
|
+
entry is simply ignored, so this is harmless when coord lives elsewhere.
|
|
660
|
+
"""
|
|
661
|
+
return f'export PATH="$HOME/.coord-venv/bin:$PATH"; {shell_cmd}'
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _launch_via_tmux(
|
|
665
|
+
argv: Sequence[str],
|
|
666
|
+
briefing: str,
|
|
667
|
+
session_name: str,
|
|
668
|
+
*,
|
|
669
|
+
cwd: str | None = None,
|
|
670
|
+
host: TmuxHost = TmuxHost(None),
|
|
671
|
+
raw_shell_cmd: str | None = None,
|
|
672
|
+
) -> int | None:
|
|
673
|
+
"""Create (or reuse) a named tmux session running *argv* and attach.
|
|
674
|
+
|
|
675
|
+
This is the preferred path when ``tmux`` is available and
|
|
676
|
+
*session_name* is provided (#487). The strategy:
|
|
677
|
+
|
|
678
|
+
1. If the session does not already exist, create it with
|
|
679
|
+
``tmux new-session -d`` running ``argv`` as the session command.
|
|
680
|
+
The briefing is then injected via
|
|
681
|
+
:func:`_inject_briefing_into_tmux_session`.
|
|
682
|
+
2. If the session already exists (reattach after TUI crash), skip
|
|
683
|
+
creation and injection — the session is already running.
|
|
684
|
+
3. On fresh creation, a ``pane-died`` hook is set on the session
|
|
685
|
+
(#1102) so an unintended pane death (e.g. an accidental kill-pane)
|
|
686
|
+
has a chance to surface via ``display-message`` rather than the
|
|
687
|
+
session just silently vanishing — best-effort, since a single-pane
|
|
688
|
+
session's pane-death and session-death are simultaneous.
|
|
689
|
+
4. Immediately before attaching (both fresh and reuse), print
|
|
690
|
+
:data:`TMUX_ATTACH_WARNING` to the operator's terminal so the
|
|
691
|
+
kill-vs-detach distinction is the last thing they read before tmux
|
|
692
|
+
takes over (#1102).
|
|
693
|
+
5. Attach the current terminal with
|
|
694
|
+
``tmux attach-session -t <session_name>``. If the TUI process is
|
|
695
|
+
killed (e.g. SIGHUP on TUI crash), only the *attach* subprocess
|
|
696
|
+
dies; the tmux session and ``claude`` inside it keep running.
|
|
697
|
+
|
|
698
|
+
Returns ``tmux attach-session``'s exit code (typically ``0`` for both
|
|
699
|
+
clean exits and user-initiated detach with ``Ctrl-b d``). Callers
|
|
700
|
+
should check :func:`tmux_session_alive` after this returns to
|
|
701
|
+
distinguish "session ended" from "user detached — session still live".
|
|
702
|
+
|
|
703
|
+
Returns ``None`` when session creation fails so that the caller
|
|
704
|
+
(:func:`launch_human_attended_interactive`) can fall back to the
|
|
705
|
+
PTY relay without double-echoing the briefing.
|
|
706
|
+
|
|
707
|
+
Args:
|
|
708
|
+
argv: Worker command. Used as-is when *raw_shell_cmd* is ``None``.
|
|
709
|
+
briefing: Text to pre-fill in the TUI input box.
|
|
710
|
+
session_name: tmux session name (``coord-<assignment_id>``).
|
|
711
|
+
cwd: Starting directory for the session (local) or the tmux ``-c``
|
|
712
|
+
start-directory (remote). For remote worktree launches
|
|
713
|
+
(#486b) this is typically ``None`` because the *raw_shell_cmd*
|
|
714
|
+
itself ``cd``s into the worktree.
|
|
715
|
+
host: Target host. ``TmuxHost(None)`` means local; a non-``None``
|
|
716
|
+
``ssh_target`` triggers the remote-safe single-string SSH path.
|
|
717
|
+
raw_shell_cmd: When provided, use this string verbatim as the
|
|
718
|
+
tmux session command instead of ``shlex.join(argv)``. Intended
|
|
719
|
+
for the remote launch path (#486b) where the command includes
|
|
720
|
+
``$HOME`` paths and shell operators (``&&``, ``||``) that must
|
|
721
|
+
survive round-tripping through SSH and the remote shell without
|
|
722
|
+
being re-split or having ``~`` masked by single-quoting.
|
|
723
|
+
"""
|
|
724
|
+
# Determine terminal dimensions for the new session.
|
|
725
|
+
try:
|
|
726
|
+
cols, rows = os.get_terminal_size(sys.stdout.fileno())
|
|
727
|
+
except (OSError, AttributeError, ValueError):
|
|
728
|
+
cols, rows = 220, 50
|
|
729
|
+
|
|
730
|
+
already_alive = tmux_session_alive(session_name, host=host)
|
|
731
|
+
|
|
732
|
+
if not already_alive:
|
|
733
|
+
# ``raw_shell_cmd`` is used by the remote launch path (#486b) so that
|
|
734
|
+
# paths containing ``$HOME`` / ``~`` and shell operators are passed
|
|
735
|
+
# verbatim to the remote shell rather than being re-quoted by
|
|
736
|
+
# ``shlex.join``. When not provided, fall back to the default
|
|
737
|
+
# behaviour: quote the argv list.
|
|
738
|
+
shell_cmd = raw_shell_cmd if raw_shell_cmd is not None else shlex.join(list(argv))
|
|
739
|
+
# #606: make `coord` resolvable in the session (agent self-report path).
|
|
740
|
+
shell_cmd = _with_coord_on_path(shell_cmd)
|
|
741
|
+
|
|
742
|
+
if host.ssh_target is not None:
|
|
743
|
+
# Remote: build the entire ``tmux new-session`` invocation as a
|
|
744
|
+
# single shell string and pass it to SSH as ONE argument. If we
|
|
745
|
+
# used ``host.cmd([..., shell_cmd])`` (multiple list elements),
|
|
746
|
+
# SSH would concatenate everything after the hostname with spaces
|
|
747
|
+
# and the remote shell would split ``shell_cmd`` at operator
|
|
748
|
+
# tokens (``&&``, ``||``, …), breaking the intended multi-step
|
|
749
|
+
# command. Sending the whole tmux invocation as one string
|
|
750
|
+
# preserves quoting and lets the remote shell hand the
|
|
751
|
+
# properly-quoted ``shell_cmd`` to tmux as a single argument.
|
|
752
|
+
# Tmux then runs ``$SHELL -c <shell_cmd>`` where ``$HOME`` and
|
|
753
|
+
# other shell expansions work correctly.
|
|
754
|
+
_parts: list[str] = [
|
|
755
|
+
"tmux", "new-session", "-d",
|
|
756
|
+
"-s", shlex.quote(session_name),
|
|
757
|
+
"-x", str(max(cols, 40)),
|
|
758
|
+
"-y", str(max(rows, 10)),
|
|
759
|
+
]
|
|
760
|
+
if cwd:
|
|
761
|
+
_parts += ["-c", shlex.quote(cwd)]
|
|
762
|
+
# shlex.quote wraps shell_cmd so the remote shell treats it as
|
|
763
|
+
# exactly one argument to tmux new-session.
|
|
764
|
+
_parts.append(shlex.quote(shell_cmd))
|
|
765
|
+
create_cmd = ["ssh", *_SSH_MUX_OPTS, host.ssh_target, " ".join(_parts)]
|
|
766
|
+
else:
|
|
767
|
+
# Local: pass args directly to tmux via subprocess list form.
|
|
768
|
+
# No shell is involved, so shell_cmd is passed as ONE argument
|
|
769
|
+
# without any re-quoting or special-character issues.
|
|
770
|
+
create_cmd = host.cmd([
|
|
771
|
+
"new-session", "-d",
|
|
772
|
+
"-s", session_name,
|
|
773
|
+
"-x", str(max(cols, 40)),
|
|
774
|
+
"-y", str(max(rows, 10)),
|
|
775
|
+
])
|
|
776
|
+
if cwd:
|
|
777
|
+
create_cmd += ["-c", cwd]
|
|
778
|
+
create_cmd.append(shell_cmd)
|
|
779
|
+
|
|
780
|
+
try:
|
|
781
|
+
result = subprocess.run(
|
|
782
|
+
create_cmd,
|
|
783
|
+
capture_output=True,
|
|
784
|
+
text=True,
|
|
785
|
+
timeout=15.0,
|
|
786
|
+
)
|
|
787
|
+
except (subprocess.SubprocessError, OSError):
|
|
788
|
+
return None # signal caller to fall back to PTY relay
|
|
789
|
+
|
|
790
|
+
if result.returncode != 0:
|
|
791
|
+
# Session creation failed (name collision, tmux daemon error, …).
|
|
792
|
+
return None # signal caller to fall back to PTY relay
|
|
793
|
+
|
|
794
|
+
# #1102: best-effort operator-facing signal if the pane dies while
|
|
795
|
+
# attached (e.g. an accidental kill-pane) rather than the session
|
|
796
|
+
# just silently vanishing. Since a coord session has exactly one
|
|
797
|
+
# pane, pane-death and session-death are simultaneous, so this
|
|
798
|
+
# ``display-message`` may not always be visible before tmux tears
|
|
799
|
+
# the session down — it's cheap, harmless, and worth setting
|
|
800
|
+
# regardless (attached-but-not-focused clients, slower teardown on
|
|
801
|
+
# some tmux versions, etc. all still benefit). Failure to set the
|
|
802
|
+
# hook is non-fatal to the session itself.
|
|
803
|
+
try:
|
|
804
|
+
subprocess.run(
|
|
805
|
+
host.cmd([
|
|
806
|
+
"set-hook", "-t", session_name, "pane-died",
|
|
807
|
+
"display-message \"coord: pane died -- if this was not an "
|
|
808
|
+
"intentional exit, the claude session may be gone. Ctrl-b d "
|
|
809
|
+
"detaches without killing; kill-pane destroys the session.\"",
|
|
810
|
+
]),
|
|
811
|
+
capture_output=True,
|
|
812
|
+
timeout=5.0,
|
|
813
|
+
)
|
|
814
|
+
except (subprocess.SubprocessError, OSError):
|
|
815
|
+
pass
|
|
816
|
+
|
|
817
|
+
# Inject briefing (best-effort; a failure here is non-fatal to the
|
|
818
|
+
# session itself, but MUST be visible to the operator — #865 review
|
|
819
|
+
# follow-up: the previous code discarded this return value, so an
|
|
820
|
+
# exhausted verify+retry only produced a ``logging.error`` call.
|
|
821
|
+
# With no ``logging.basicConfig`` anywhere in the repo, that falls
|
|
822
|
+
# through to Python's "handler of last resort" — one stderr line
|
|
823
|
+
# that the very next statement (tmux attach-session, below) blows
|
|
824
|
+
# away by switching the terminal to the tmux alt-screen before the
|
|
825
|
+
# operator can read it. Print a message to the operator's OWN
|
|
826
|
+
# terminal (not logging) and pause for an explicit acknowledgment
|
|
827
|
+
# BEFORE attaching, so the failure can't be missed.
|
|
828
|
+
if briefing.strip():
|
|
829
|
+
injected = _inject_briefing_into_tmux_session(session_name, briefing, host=host)
|
|
830
|
+
if not injected:
|
|
831
|
+
print(
|
|
832
|
+
"\n"
|
|
833
|
+
"!!! coord: briefing injection could not be verified after "
|
|
834
|
+
"multiple attempts.\n"
|
|
835
|
+
"!!! The input box in the session below may be EMPTY — "
|
|
836
|
+
"paste the briefing yourself if so.\n",
|
|
837
|
+
file=sys.stderr,
|
|
838
|
+
)
|
|
839
|
+
try:
|
|
840
|
+
input("Press Enter to attach to the session... ")
|
|
841
|
+
except (EOFError, KeyboardInterrupt):
|
|
842
|
+
pass
|
|
843
|
+
|
|
844
|
+
# #1102: print the kill-vs-detach warning immediately before attaching —
|
|
845
|
+
# both for a fresh session and for reuse-after-crash — so it's the last
|
|
846
|
+
# thing the operator reads before tmux takes over the terminal.
|
|
847
|
+
try:
|
|
848
|
+
os.write(sys.stdout.fileno(), TMUX_ATTACH_WARNING.encode("utf-8"))
|
|
849
|
+
except OSError:
|
|
850
|
+
pass
|
|
851
|
+
|
|
852
|
+
# #1102 fix-iteration-1 (live human-attended smoke FAILED): printing the
|
|
853
|
+
# banner is not enough on its own -- the very next statement below
|
|
854
|
+
# (tmux attach-session / switch-client) switches the terminal to tmux's
|
|
855
|
+
# alt-screen buffer, which covers the banner before the operator's eyes
|
|
856
|
+
# can land on it ("it was too fast, I saw something flash by but could
|
|
857
|
+
# not read it" -- confirmed live; the operator then hit Ctrl-D instead
|
|
858
|
+
# of Ctrl-b d and the one-pane session was destroyed). Require an
|
|
859
|
+
# explicit acknowledgment here, mirroring the existing
|
|
860
|
+
# unverified-injection gate above (lines ~808-822) and the pre-existing
|
|
861
|
+
# #865 pattern it was modeled on. ``input()`` on a piped/non-interactive
|
|
862
|
+
# stdin (tests, CI, a scripted caller) raises immediately -- swallow
|
|
863
|
+
# that so this stays best-effort and never hangs a headless run.
|
|
864
|
+
try:
|
|
865
|
+
input("Press Enter to attach (read the warning above first)... ")
|
|
866
|
+
except (EOFError, KeyboardInterrupt, OSError):
|
|
867
|
+
pass
|
|
868
|
+
|
|
869
|
+
# Attach. ``subprocess.run`` (not ``os.execvp``) is intentional: we
|
|
870
|
+
# need this process to continue after the operator detaches so that
|
|
871
|
+
# the CLI caller (``coord assign``) can check whether the session is
|
|
872
|
+
# still alive and decide whether to run ``finalize_interactive_exit``.
|
|
873
|
+
# The attach-session call uses tty=True because it needs a pseudo-TTY
|
|
874
|
+
# on the remote side when the host is remote.
|
|
875
|
+
try:
|
|
876
|
+
if host.ssh_target is None and os.environ.get("TMUX"):
|
|
877
|
+
# The operator is already inside a tmux client (nested).
|
|
878
|
+
# ``attach-session`` refuses to nest ("sessions should be nested
|
|
879
|
+
# with care, unset $TMUX to force") and exits 1, orphaning the
|
|
880
|
+
# session so the terminal never opens. ``switch-client`` switches
|
|
881
|
+
# the current client to the new session instead — it works from
|
|
882
|
+
# inside tmux and lands the operator in the claude session. It
|
|
883
|
+
# returns immediately (unlike ``attach-session`` it does not block
|
|
884
|
+
# until detach), so the caller's ``tmux_session_alive`` check then
|
|
885
|
+
# treats the live session as "operator is now in it".
|
|
886
|
+
cmd: list[str] = ["tmux", "switch-client", "-t", session_name]
|
|
887
|
+
else:
|
|
888
|
+
cmd = list(host.cmd(["attach-session", "-t", session_name], tty=True))
|
|
889
|
+
attach_result = subprocess.run(cmd)
|
|
890
|
+
return attach_result.returncode
|
|
891
|
+
except (subprocess.SubprocessError, OSError):
|
|
892
|
+
return 1
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
# ── Quiescence constants for the PTY relay path ───────────────────────────────
|
|
896
|
+
|
|
897
|
+
# Match the agent-side readiness constants so the operator-launched path
|
|
898
|
+
# behaves the same way as the agent-spawned one once #437 lands the
|
|
899
|
+
# guardrail on the unattended side. Bracketed-paste-enable fires while
|
|
900
|
+
# the TUI is still drawing; we additionally wait for render quiescence
|
|
901
|
+
# before pasting because a paste at the enable instant is silently
|
|
902
|
+
# dropped (verified live against interactive ``claude``).
|
|
903
|
+
_READY_QUIESCE_S = 0.8
|
|
904
|
+
_READY_QUIESCE_CAP_S = 8.0
|
|
905
|
+
|
|
906
|
+
# #865: requiring INPUT_BOX_MARKER is a STRONGER readiness signal than bare
|
|
907
|
+
# quiescence (it means we've actually seen the rendered prompt, not just
|
|
908
|
+
# some static content — a promo banner mid-paint, say), but making it a hard
|
|
909
|
+
# requirement for the fast quiescence exit would regress the case where the
|
|
910
|
+
# render never emits a recognisable marker at all (older CLI, unusual
|
|
911
|
+
# terminal, or a session that exits before drawing anything — caught by the
|
|
912
|
+
# test suite). So: exit fast (after _READY_QUIESCE_S) once the marker has
|
|
913
|
+
# been seen; fall back to this longer window when it hasn't, rather than
|
|
914
|
+
# spinning all the way to the outer cap/timeout.
|
|
915
|
+
_READY_QUIESCE_NO_MARKER_S = 1.6
|
|
916
|
+
|
|
917
|
+
# #865: paste-verify-retry constants shared by the tmux path
|
|
918
|
+
# (:func:`_inject_briefing_into_tmux_session`) and the PTY relay path
|
|
919
|
+
# (:func:`_launch_via_pty`'s prefill state machine). A miss re-pastes after
|
|
920
|
+
# a short backoff rather than silently giving up — this is the core #865
|
|
921
|
+
# fix, since fire-and-forget (no verification at all) was the root defect.
|
|
922
|
+
_INJECT_MAX_ATTEMPTS = 3
|
|
923
|
+
_INJECT_VERIFY_SETTLE_S = 0.5
|
|
924
|
+
_INJECT_RETRY_BACKOFF_S = 0.4
|
|
925
|
+
|
|
926
|
+
#: Cap on the in-memory copy of recent master-fd output the PTY relay keeps
|
|
927
|
+
#: for readiness/verification (#865). ~32KB comfortably holds a full TUI
|
|
928
|
+
#: frame plus ANSI escapes; older bytes are dropped so a long-lived session
|
|
929
|
+
#: doesn't grow this unbounded.
|
|
930
|
+
_PREFILL_SCREEN_BUF_CAP = 32768
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
@dataclass
|
|
934
|
+
class _PrefillState:
|
|
935
|
+
"""Pure state for the PTY relay's pre-fill-then-verify state machine (#865).
|
|
936
|
+
|
|
937
|
+
Deliberately free of any I/O — :func:`_prefill_on_master_data` folds
|
|
938
|
+
observed master-fd bytes in, :func:`_prefill_step` decides (given the
|
|
939
|
+
current time) whether :func:`_launch_via_pty` should write
|
|
940
|
+
``paste_block`` to the master fd right now. Splitting the decision out
|
|
941
|
+
as pure functions means the readiness+verify+retry logic can be unit
|
|
942
|
+
tested without a real pty, and keeps the relay's ``select()`` loop
|
|
943
|
+
(which must stay non-blocking — it's also servicing the operator's
|
|
944
|
+
keystrokes and the child's live output) free of any ``time.sleep()``.
|
|
945
|
+
|
|
946
|
+
Attributes:
|
|
947
|
+
fingerprint: The whitespace-normalized briefing snippet (see
|
|
948
|
+
:func:`~coord.providers.claude_pty.briefing_fingerprint`) used to
|
|
949
|
+
confirm a paste landed.
|
|
950
|
+
started: ``time.monotonic()`` when the relay loop began — anchors
|
|
951
|
+
the overall degraded-fallback cap.
|
|
952
|
+
last_master_activity: ``time.monotonic()`` of the most recent
|
|
953
|
+
master-fd read — anchors the pre-paste quiescence check.
|
|
954
|
+
done: ``True`` once the state machine has either verified success or
|
|
955
|
+
exhausted its retry budget — the caller stops driving it.
|
|
956
|
+
seen_enable: Whether the bracketed-paste-enable DECSET has been
|
|
957
|
+
observed in the master output yet.
|
|
958
|
+
screen_buf: Rolling tail of recent master-fd bytes (capped at
|
|
959
|
+
:data:`_PREFILL_SCREEN_BUF_CAP`), used both to look for
|
|
960
|
+
:data:`~coord.providers.claude_pty.INPUT_BOX_MARKER_BYTES`
|
|
961
|
+
(readiness) and the fingerprint (post-paste verification).
|
|
962
|
+
paste_attempts: How many times the paste has been written so far.
|
|
963
|
+
next_paste_at: ``time.monotonic()`` deadline for the next paste
|
|
964
|
+
attempt when a retry is pending; ``None`` otherwise.
|
|
965
|
+
verify_deadline: ``time.monotonic()`` deadline to check for the
|
|
966
|
+
fingerprint after the most recent paste; ``None`` when not
|
|
967
|
+
currently waiting on a verification window.
|
|
968
|
+
"""
|
|
969
|
+
|
|
970
|
+
fingerprint: str
|
|
971
|
+
started: float
|
|
972
|
+
last_master_activity: float
|
|
973
|
+
done: bool = False
|
|
974
|
+
seen_enable: bool = False
|
|
975
|
+
screen_buf: bytearray = field(default_factory=bytearray)
|
|
976
|
+
paste_attempts: int = 0
|
|
977
|
+
next_paste_at: float | None = None
|
|
978
|
+
verify_deadline: float | None = None
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _prefill_on_master_data(state: _PrefillState, data: bytes, now: float) -> None:
|
|
982
|
+
"""Fold newly-observed master-fd *data* into *state*.
|
|
983
|
+
|
|
984
|
+
Call this for every chunk read from the master fd, BEFORE the next
|
|
985
|
+
:func:`_prefill_step` call, while ``state.done`` is still ``False``.
|
|
986
|
+
"""
|
|
987
|
+
if state.done:
|
|
988
|
+
return
|
|
989
|
+
state.screen_buf.extend(data)
|
|
990
|
+
overflow = len(state.screen_buf) - _PREFILL_SCREEN_BUF_CAP
|
|
991
|
+
if overflow > 0:
|
|
992
|
+
del state.screen_buf[:overflow]
|
|
993
|
+
if not state.seen_enable and BRACKETED_PASTE_ENABLE in data:
|
|
994
|
+
state.seen_enable = True
|
|
995
|
+
state.last_master_activity = now
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _prefill_step(state: _PrefillState, now: float) -> bool:
|
|
999
|
+
"""Advance the pre-fill state machine one tick.
|
|
1000
|
+
|
|
1001
|
+
Returns ``True`` exactly when the caller should write the bracketed-paste
|
|
1002
|
+
block to the master fd right now (once for the initial attempt, again
|
|
1003
|
+
for each retry). Mutates *state* in place; check ``state.done`` after
|
|
1004
|
+
calling to know whether the machine has finished (verified success, or
|
|
1005
|
+
exhausted its retry budget — see :data:`_INJECT_MAX_ATTEMPTS`).
|
|
1006
|
+
"""
|
|
1007
|
+
if state.done:
|
|
1008
|
+
return False
|
|
1009
|
+
|
|
1010
|
+
if state.verify_deadline is not None:
|
|
1011
|
+
if now < state.verify_deadline:
|
|
1012
|
+
return False
|
|
1013
|
+
# Verification window elapsed — did the paste land? Broadened
|
|
1014
|
+
# (#896 review follow-up) from the bare fingerprint check to
|
|
1015
|
+
# ``paste_landed_bytes`` so a collapsed paste-chip or a scrolled-tail
|
|
1016
|
+
# input box also counts as evidence — this is the third of the three
|
|
1017
|
+
# call-sites documented above; it was missed in the first #896 pass.
|
|
1018
|
+
if paste_landed_bytes(bytes(state.screen_buf), state.fingerprint):
|
|
1019
|
+
state.done = True
|
|
1020
|
+
elif state.paste_attempts >= _INJECT_MAX_ATTEMPTS:
|
|
1021
|
+
state.done = True
|
|
1022
|
+
logging.error(
|
|
1023
|
+
"briefing pre-fill unverified after %d attempt(s) in the "
|
|
1024
|
+
"PTY relay session — the briefing may not have landed in "
|
|
1025
|
+
"the input box; the operator should check and paste "
|
|
1026
|
+
"manually",
|
|
1027
|
+
state.paste_attempts,
|
|
1028
|
+
)
|
|
1029
|
+
else:
|
|
1030
|
+
state.next_paste_at = now + _INJECT_RETRY_BACKOFF_S
|
|
1031
|
+
state.verify_deadline = None
|
|
1032
|
+
return False
|
|
1033
|
+
|
|
1034
|
+
if state.next_paste_at is not None:
|
|
1035
|
+
if now < state.next_paste_at:
|
|
1036
|
+
return False
|
|
1037
|
+
state.next_paste_at = None
|
|
1038
|
+
state.paste_attempts += 1
|
|
1039
|
+
state.verify_deadline = now + _INJECT_VERIFY_SETTLE_S
|
|
1040
|
+
return True
|
|
1041
|
+
|
|
1042
|
+
# No attempt in flight yet — wait for bracketed-paste-enable AND render
|
|
1043
|
+
# quiescence. A recognised input box (INPUT_BOX_MARKER) needs only the
|
|
1044
|
+
# short _READY_QUIESCE_S settle window; without it (older CLI, unusual
|
|
1045
|
+
# render, or a session that never draws anything — see the two-tier
|
|
1046
|
+
# rationale on _READY_QUIESCE_NO_MARKER_S above) fall back to the longer
|
|
1047
|
+
# window instead of spinning all the way to the overall cap (degraded
|
|
1048
|
+
# fallback — paste anyway rather than hang forever).
|
|
1049
|
+
quiet_for = now - state.last_master_activity
|
|
1050
|
+
marker_seen = INPUT_BOX_MARKER_BYTES in bytes(state.screen_buf)
|
|
1051
|
+
threshold = _READY_QUIESCE_S if marker_seen else _READY_QUIESCE_NO_MARKER_S
|
|
1052
|
+
ready_quiet = state.seen_enable and quiet_for >= threshold
|
|
1053
|
+
ready_cap = now - state.started >= _READY_QUIESCE_CAP_S
|
|
1054
|
+
if ready_quiet or ready_cap:
|
|
1055
|
+
state.paste_attempts += 1
|
|
1056
|
+
state.verify_deadline = now + _INJECT_VERIFY_SETTLE_S
|
|
1057
|
+
return True
|
|
1058
|
+
return False
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def launch_human_attended_interactive(
|
|
1062
|
+
argv: Sequence[str],
|
|
1063
|
+
briefing: str,
|
|
1064
|
+
*,
|
|
1065
|
+
assignment_id: str | None = None,
|
|
1066
|
+
cwd: str | None = None,
|
|
1067
|
+
) -> int:
|
|
1068
|
+
"""Run *argv* attached to the current TTY with *briefing* pre-filled.
|
|
1069
|
+
|
|
1070
|
+
Used exclusively by ``coord assign --interactive``. The function
|
|
1071
|
+
returns the child's exit code; the caller decides what (if anything)
|
|
1072
|
+
to record on the board. No GitHub comments are posted from inside
|
|
1073
|
+
this function — the human owns the session lifecycle.
|
|
1074
|
+
|
|
1075
|
+
When *assignment_id* is provided and ``tmux`` is available, the session
|
|
1076
|
+
is hosted in a named tmux session ``coord-<assignment_id>`` (#487).
|
|
1077
|
+
This means the session survives a TUI crash — the operator can reattach
|
|
1078
|
+
later with ``coord reattach <assignment_id>``. If tmux is not available
|
|
1079
|
+
or *assignment_id* is omitted, falls back to the PTY relay path.
|
|
1080
|
+
|
|
1081
|
+
Args:
|
|
1082
|
+
argv: The worker command (typically built via
|
|
1083
|
+
:meth:`ClaudePtyProvider.build_command`).
|
|
1084
|
+
briefing: The text to PRE-FILL in the TUI's input box. Wrapped
|
|
1085
|
+
in a bracketed-paste block; the operator presses Enter to
|
|
1086
|
+
submit. An empty string disables the pre-fill entirely.
|
|
1087
|
+
assignment_id: Optional coordinator assignment ID. When provided
|
|
1088
|
+
and tmux is available, the session runs inside a persistent
|
|
1089
|
+
tmux session named ``coord-<assignment_id>``.
|
|
1090
|
+
cwd: Working directory for the child. ``None`` keeps the
|
|
1091
|
+
parent's cwd.
|
|
1092
|
+
|
|
1093
|
+
Returns:
|
|
1094
|
+
The child's exit status (``0`` on clean exit; ``128 + signum``
|
|
1095
|
+
on termination by signal). For the tmux path, returns
|
|
1096
|
+
``tmux attach-session``'s exit code — callers should check
|
|
1097
|
+
:func:`tmux_session_alive` afterwards to distinguish
|
|
1098
|
+
"session ended" from "user detached with Ctrl-b d".
|
|
1099
|
+
"""
|
|
1100
|
+
# #487 — prefer the tmux path when assignment_id is available.
|
|
1101
|
+
if assignment_id and tmux_available():
|
|
1102
|
+
sname = tmux_session_name(assignment_id)
|
|
1103
|
+
# Echo briefing BEFORE the tmux session starts (same as PTY path).
|
|
1104
|
+
# Important: only echo here on the tmux path. _launch_via_pty()
|
|
1105
|
+
# handles its own echo so that it works correctly when called
|
|
1106
|
+
# stand-alone (no assignment_id / no tmux).
|
|
1107
|
+
if briefing.strip():
|
|
1108
|
+
_hdr = (
|
|
1109
|
+
"--- seeded briefing -- review below; "
|
|
1110
|
+
"submit the pre-filled input in Claude to send ---"
|
|
1111
|
+
)
|
|
1112
|
+
_ftr = "-" * len(_hdr)
|
|
1113
|
+
_preview = f"\n{_hdr}\n{briefing.rstrip()}\n{_ftr}\n\n"
|
|
1114
|
+
try:
|
|
1115
|
+
os.write(sys.stdout.fileno(), _preview.encode("utf-8"))
|
|
1116
|
+
except OSError:
|
|
1117
|
+
pass
|
|
1118
|
+
rc = _launch_via_tmux(argv, briefing, sname, cwd=cwd)
|
|
1119
|
+
if rc is not None:
|
|
1120
|
+
return rc
|
|
1121
|
+
# tmux session creation failed — fall back to the PTY relay.
|
|
1122
|
+
# Note: _launch_via_pty will echo the briefing again; that is
|
|
1123
|
+
# acceptable for this rare fallback case.
|
|
1124
|
+
|
|
1125
|
+
return _launch_via_pty(argv, briefing, cwd=cwd)
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def _launch_via_pty(
|
|
1129
|
+
argv: Sequence[str],
|
|
1130
|
+
briefing: str,
|
|
1131
|
+
*,
|
|
1132
|
+
cwd: str | None = None,
|
|
1133
|
+
) -> int:
|
|
1134
|
+
"""PTY relay implementation — the original pty.fork() path.
|
|
1135
|
+
|
|
1136
|
+
Kept as a separate function so :func:`launch_human_attended_interactive`
|
|
1137
|
+
can delegate to it both directly (no tmux) and as a fallback from the
|
|
1138
|
+
tmux path.
|
|
1139
|
+
"""
|
|
1140
|
+
import fcntl # stdlib, Unix-only — deferred for platform safety # noqa: PLC0415
|
|
1141
|
+
import pty # stdlib, Unix-only — deferred for platform safety # noqa: PLC0415
|
|
1142
|
+
import termios # stdlib, Unix-only — deferred for platform safety # noqa: PLC0415
|
|
1143
|
+
import tty # stdlib, Unix-only — deferred for platform safety # noqa: PLC0415
|
|
1144
|
+
|
|
1145
|
+
fd_in = sys.stdin.fileno()
|
|
1146
|
+
fd_out = sys.stdout.fileno()
|
|
1147
|
+
|
|
1148
|
+
# Capture original TTY attrs so we can restore on exit. ``setraw``
|
|
1149
|
+
# may fail if stdin isn't a TTY (e.g. piped input under tests); in
|
|
1150
|
+
# that case we proceed without raw mode — the pre-fill still works,
|
|
1151
|
+
# only line discipline is lost.
|
|
1152
|
+
try:
|
|
1153
|
+
orig_attr = termios.tcgetattr(fd_in)
|
|
1154
|
+
except termios.error:
|
|
1155
|
+
orig_attr = None
|
|
1156
|
+
|
|
1157
|
+
# ── #485 human-readable briefing preview ────────────────────────────────
|
|
1158
|
+
# Echo the briefing to the operator's terminal BEFORE pty.fork() while
|
|
1159
|
+
# the terminal is still in cooked mode (before tty.setraw). This lands
|
|
1160
|
+
# in scrollback just above Claude's TUI so the operator can read the
|
|
1161
|
+
# briefing before pressing Enter to submit the pre-filled chip.
|
|
1162
|
+
#
|
|
1163
|
+
# IMPORTANT: do NOT move this into the relay ``while True`` loop. At
|
|
1164
|
+
# that point Claude is actively drawing its screen; injecting bytes onto
|
|
1165
|
+
# fd_out there corrupts the display.
|
|
1166
|
+
if briefing.strip():
|
|
1167
|
+
_hdr = (
|
|
1168
|
+
"--- seeded briefing -- review below; "
|
|
1169
|
+
"submit the pre-filled input in Claude to send ---"
|
|
1170
|
+
)
|
|
1171
|
+
_ftr = "-" * len(_hdr)
|
|
1172
|
+
_preview = f"\n{_hdr}\n{briefing.rstrip()}\n{_ftr}\n\n"
|
|
1173
|
+
try:
|
|
1174
|
+
os.write(fd_out, _preview.encode("utf-8"))
|
|
1175
|
+
except OSError:
|
|
1176
|
+
pass
|
|
1177
|
+
|
|
1178
|
+
# #1102: this path has no tmux underneath it at all, so — unlike the
|
|
1179
|
+
# tmux-hosted path — there is no detach to protect: the operator's
|
|
1180
|
+
# terminal IS the session. Say so explicitly rather than leaving the
|
|
1181
|
+
# operator to assume the same Ctrl-b d semantics apply here.
|
|
1182
|
+
try:
|
|
1183
|
+
os.write(fd_out, PTY_RELAY_NO_DETACH_WARNING.encode("utf-8"))
|
|
1184
|
+
except OSError:
|
|
1185
|
+
pass
|
|
1186
|
+
|
|
1187
|
+
# #1102 follow-up: same visibility risk as the tmux path above -- the
|
|
1188
|
+
# ``pty.fork()`` a few lines down hands the terminal straight to the
|
|
1189
|
+
# child, and a real full-screen TUI (``claude``) typically clears the
|
|
1190
|
+
# screen and starts drawing well inside a second, so this banner can be
|
|
1191
|
+
# gone before the operator reads it. Unlike the tmux path there is no
|
|
1192
|
+
# ``input()`` gate here: this function's ``fd_in`` is sometimes a
|
|
1193
|
+
# script-driven pipe rather than a real keyboard (see the real-pty
|
|
1194
|
+
# relay-loop tests), and a blocking read on a pipe nobody writes to
|
|
1195
|
+
# would hang forever rather than raise. Give the banner a fixed beat of
|
|
1196
|
+
# on-screen time instead, and only when actually attached to a real
|
|
1197
|
+
# terminal -- headless/test callers (stdout is not a tty) proceed
|
|
1198
|
+
# immediately, unaffected.
|
|
1199
|
+
try:
|
|
1200
|
+
if os.isatty(fd_out):
|
|
1201
|
+
time.sleep(_PTY_NO_DETACH_WARNING_PAUSE_S)
|
|
1202
|
+
except OSError:
|
|
1203
|
+
pass
|
|
1204
|
+
|
|
1205
|
+
pid, master_fd = pty.fork()
|
|
1206
|
+
if pid == 0:
|
|
1207
|
+
# child
|
|
1208
|
+
try:
|
|
1209
|
+
if cwd:
|
|
1210
|
+
os.chdir(cwd)
|
|
1211
|
+
# #606: ensure `coord` resolves for the agent's self-report (mirrors
|
|
1212
|
+
# the tmux path's _with_coord_on_path; the PTY fallback execs argv
|
|
1213
|
+
# directly, so set PATH in the child env here).
|
|
1214
|
+
os.environ["PATH"] = (
|
|
1215
|
+
os.path.expanduser("~/.coord-venv/bin") + ":" + os.environ.get("PATH", "")
|
|
1216
|
+
)
|
|
1217
|
+
os.execvp(argv[0], list(argv))
|
|
1218
|
+
except OSError as e:
|
|
1219
|
+
sys.stderr.write(f"exec failed: {e}\n")
|
|
1220
|
+
os._exit(127)
|
|
1221
|
+
|
|
1222
|
+
# parent: forward initial terminal size to the child so the TUI
|
|
1223
|
+
# renders at the operator's actual viewport size.
|
|
1224
|
+
def _send_winsize() -> None:
|
|
1225
|
+
try:
|
|
1226
|
+
cols, rows = os.get_terminal_size(fd_out)
|
|
1227
|
+
fcntl.ioctl(
|
|
1228
|
+
master_fd, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)
|
|
1229
|
+
)
|
|
1230
|
+
except (OSError, AttributeError, ValueError):
|
|
1231
|
+
pass
|
|
1232
|
+
|
|
1233
|
+
_send_winsize()
|
|
1234
|
+
|
|
1235
|
+
# Forward SIGWINCH so terminal resizes propagate to the child while
|
|
1236
|
+
# the session is live. Restore default handler on exit.
|
|
1237
|
+
def _winch_handler(_signum, _frame) -> None:
|
|
1238
|
+
_send_winsize()
|
|
1239
|
+
|
|
1240
|
+
prev_winch = signal.signal(signal.SIGWINCH, _winch_handler)
|
|
1241
|
+
|
|
1242
|
+
if orig_attr is not None:
|
|
1243
|
+
try:
|
|
1244
|
+
tty.setraw(fd_in)
|
|
1245
|
+
except termios.error:
|
|
1246
|
+
pass
|
|
1247
|
+
|
|
1248
|
+
paste_block = b""
|
|
1249
|
+
if briefing.strip():
|
|
1250
|
+
paste_block = (
|
|
1251
|
+
BRACKETED_PASTE_START
|
|
1252
|
+
+ briefing.rstrip("\n").encode("utf-8")
|
|
1253
|
+
+ BRACKETED_PASTE_END
|
|
1254
|
+
)
|
|
1255
|
+
|
|
1256
|
+
prefilled = not bool(paste_block)
|
|
1257
|
+
started = time.monotonic()
|
|
1258
|
+
# #865: pure paste/verify/retry state machine — see _PrefillState. Only
|
|
1259
|
+
# constructed when there's actually a briefing to pre-fill; when
|
|
1260
|
+
# ``prefilled`` starts ``True`` (empty briefing) it's never touched.
|
|
1261
|
+
prefill_state = _PrefillState(
|
|
1262
|
+
fingerprint=briefing_fingerprint(briefing) if paste_block else "",
|
|
1263
|
+
started=started,
|
|
1264
|
+
last_master_activity=started,
|
|
1265
|
+
)
|
|
1266
|
+
# Capture the raw wait-status from the WNOHANG poll so that if the
|
|
1267
|
+
# child is already reaped when we reach the post-loop waitpid we can
|
|
1268
|
+
# still extract the correct exit code (see ChildProcessError handler
|
|
1269
|
+
# below).
|
|
1270
|
+
_reaped_status: int | None = None
|
|
1271
|
+
|
|
1272
|
+
try:
|
|
1273
|
+
while True:
|
|
1274
|
+
try:
|
|
1275
|
+
r, _, _ = select.select([fd_in, master_fd], [], [], 0.05)
|
|
1276
|
+
except (OSError, ValueError):
|
|
1277
|
+
break
|
|
1278
|
+
|
|
1279
|
+
if fd_in in r:
|
|
1280
|
+
try:
|
|
1281
|
+
data = os.read(fd_in, 4096)
|
|
1282
|
+
except OSError:
|
|
1283
|
+
data = b""
|
|
1284
|
+
if data:
|
|
1285
|
+
try:
|
|
1286
|
+
os.write(master_fd, data)
|
|
1287
|
+
except OSError:
|
|
1288
|
+
break
|
|
1289
|
+
|
|
1290
|
+
if master_fd in r:
|
|
1291
|
+
try:
|
|
1292
|
+
data = os.read(master_fd, 4096)
|
|
1293
|
+
except OSError:
|
|
1294
|
+
break
|
|
1295
|
+
if not data:
|
|
1296
|
+
break
|
|
1297
|
+
try:
|
|
1298
|
+
os.write(fd_out, data)
|
|
1299
|
+
except OSError:
|
|
1300
|
+
break
|
|
1301
|
+
if not prefilled:
|
|
1302
|
+
_prefill_on_master_data(prefill_state, data, time.monotonic())
|
|
1303
|
+
|
|
1304
|
+
if not prefilled:
|
|
1305
|
+
# #865: readiness-anchored paste, verified and retried by
|
|
1306
|
+
# _prefill_step — a mistimed/lost paste is no longer
|
|
1307
|
+
# fire-and-forget. ``_prefill_step`` is deliberately I/O-free
|
|
1308
|
+
# (see _PrefillState's docstring), so it can't send the
|
|
1309
|
+
# clear-before-retry keys itself; instead it increments
|
|
1310
|
+
# ``paste_attempts`` before returning True, and a value > 1
|
|
1311
|
+
# here means this write is a retry (the initial attempt is
|
|
1312
|
+
# always 1) — the same signal ``_paste_once``'s caller and
|
|
1313
|
+
# agent.py's PTY relay use their own attempt counters for.
|
|
1314
|
+
if _prefill_step(prefill_state, time.monotonic()):
|
|
1315
|
+
try:
|
|
1316
|
+
if prefill_state.paste_attempts > 1:
|
|
1317
|
+
# Idempotent retry (#896): clear any stacked
|
|
1318
|
+
# paste chip before re-pasting so a false-
|
|
1319
|
+
# negative on a successfully-landed paste
|
|
1320
|
+
# doesn't stack duplicate chips.
|
|
1321
|
+
os.write(master_fd, b"\x1b\x15") # Escape + Ctrl-U
|
|
1322
|
+
os.write(master_fd, paste_block)
|
|
1323
|
+
except OSError:
|
|
1324
|
+
pass
|
|
1325
|
+
prefilled = prefill_state.done
|
|
1326
|
+
|
|
1327
|
+
# Poll child status without blocking.
|
|
1328
|
+
try:
|
|
1329
|
+
done_pid, _reaped_status = os.waitpid(pid, os.WNOHANG)
|
|
1330
|
+
except ChildProcessError:
|
|
1331
|
+
break
|
|
1332
|
+
if done_pid != 0:
|
|
1333
|
+
# Drain any final master output before exiting the loop.
|
|
1334
|
+
while True:
|
|
1335
|
+
try:
|
|
1336
|
+
data = os.read(master_fd, 4096)
|
|
1337
|
+
except OSError:
|
|
1338
|
+
break
|
|
1339
|
+
if not data:
|
|
1340
|
+
break
|
|
1341
|
+
try:
|
|
1342
|
+
os.write(fd_out, data)
|
|
1343
|
+
except OSError:
|
|
1344
|
+
break
|
|
1345
|
+
break
|
|
1346
|
+
finally:
|
|
1347
|
+
if orig_attr is not None:
|
|
1348
|
+
try:
|
|
1349
|
+
termios.tcsetattr(fd_in, termios.TCSAFLUSH, orig_attr)
|
|
1350
|
+
except termios.error:
|
|
1351
|
+
pass
|
|
1352
|
+
try:
|
|
1353
|
+
os.close(master_fd)
|
|
1354
|
+
except OSError:
|
|
1355
|
+
pass
|
|
1356
|
+
try:
|
|
1357
|
+
signal.signal(signal.SIGWINCH, prev_winch)
|
|
1358
|
+
except (OSError, ValueError, TypeError):
|
|
1359
|
+
pass
|
|
1360
|
+
|
|
1361
|
+
try:
|
|
1362
|
+
_, status = os.waitpid(pid, 0)
|
|
1363
|
+
except ChildProcessError:
|
|
1364
|
+
# The zombie was already reaped by the WNOHANG poll inside the
|
|
1365
|
+
# relay loop. Use the status captured there so non-zero exit
|
|
1366
|
+
# codes are correctly propagated (without this fix the function
|
|
1367
|
+
# would always return 0 for the normal-exit path).
|
|
1368
|
+
if _reaped_status is not None:
|
|
1369
|
+
if os.WIFEXITED(_reaped_status):
|
|
1370
|
+
return os.WEXITSTATUS(_reaped_status)
|
|
1371
|
+
if os.WIFSIGNALED(_reaped_status):
|
|
1372
|
+
return 128 + os.WTERMSIG(_reaped_status)
|
|
1373
|
+
return 0
|
|
1374
|
+
if os.WIFEXITED(status):
|
|
1375
|
+
return os.WEXITSTATUS(status)
|
|
1376
|
+
if os.WIFSIGNALED(status):
|
|
1377
|
+
return 128 + os.WTERMSIG(status)
|
|
1378
|
+
return 0
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
# ── #466 git-floor backstop ─────────────────────────────────────────────────
|
|
1382
|
+
|
|
1383
|
+
|
|
1384
|
+
def _worktree_sandbox_root(wt_path: Path) -> Path:
|
|
1385
|
+
"""Directory an *unregistered* worktree must sit inside to be removable.
|
|
1386
|
+
|
|
1387
|
+
#1693: coord always places worktrees at ``<state_dir>/worktrees/<id>``, so
|
|
1388
|
+
the sandbox is that parent when it is named ``worktrees`` and the coord
|
|
1389
|
+
state dir's otherwise. This is only the second line of defence — the
|
|
1390
|
+
base-checkout identity check inside
|
|
1391
|
+
:func:`coord.agent._safe_remove_worktree` is the primary one.
|
|
1392
|
+
"""
|
|
1393
|
+
from coord.state import COORD_DIR # noqa: PLC0415
|
|
1394
|
+
|
|
1395
|
+
parent = wt_path.parent
|
|
1396
|
+
if parent.name == "worktrees":
|
|
1397
|
+
return parent
|
|
1398
|
+
return COORD_DIR / "worktrees"
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def _remove_worktree(repo_path: Path, wt_path: Path) -> bool:
|
|
1402
|
+
"""Best-effort removal of a git worktree for an interactive session.
|
|
1403
|
+
|
|
1404
|
+
Mirrors :meth:`coord.agent.AgentServer._cleanup_worktree` without pulling
|
|
1405
|
+
in the full AgentServer graph. Returns ``True`` on success, ``False`` if
|
|
1406
|
+
every removal strategy fails.
|
|
1407
|
+
|
|
1408
|
+
#1693: this used to carry its own ``git worktree remove`` →
|
|
1409
|
+
``shutil.rmtree`` fallback, one of four independent copies of that idiom,
|
|
1410
|
+
none of which checked whether the path was the base checkout. It now
|
|
1411
|
+
delegates to :func:`coord.agent._safe_remove_worktree`, the single place
|
|
1412
|
+
allowed to recursively delete a worktree.
|
|
1413
|
+
"""
|
|
1414
|
+
from coord.agent import _safe_remove_worktree # noqa: PLC0415
|
|
1415
|
+
|
|
1416
|
+
removed = _safe_remove_worktree(
|
|
1417
|
+
repo_path,
|
|
1418
|
+
wt_path,
|
|
1419
|
+
sandbox_root=_worktree_sandbox_root(wt_path),
|
|
1420
|
+
prune=False,
|
|
1421
|
+
)
|
|
1422
|
+
|
|
1423
|
+
# Prune the stale git admin entry regardless of whether the directory
|
|
1424
|
+
# was physically removed — a stale entry blocks the next dispatch on
|
|
1425
|
+
# the same branch.
|
|
1426
|
+
try:
|
|
1427
|
+
subprocess.run(
|
|
1428
|
+
["git", "worktree", "prune"],
|
|
1429
|
+
cwd=str(repo_path),
|
|
1430
|
+
capture_output=True,
|
|
1431
|
+
timeout=10.0,
|
|
1432
|
+
)
|
|
1433
|
+
except (subprocess.SubprocessError, OSError):
|
|
1434
|
+
pass
|
|
1435
|
+
|
|
1436
|
+
return removed
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
@dataclass
|
|
1440
|
+
class InteractiveFinalizeResult:
|
|
1441
|
+
"""What :func:`finalize_interactive_exit` ended up writing.
|
|
1442
|
+
|
|
1443
|
+
Returned for callers (and tests) so the chosen terminal state and
|
|
1444
|
+
commit count can be asserted without re-reading the DB.
|
|
1445
|
+
|
|
1446
|
+
Attributes:
|
|
1447
|
+
terminal_status: ``done`` | ``advisory`` | ``failed`` — what the
|
|
1448
|
+
issue-store seam ultimately recorded.
|
|
1449
|
+
commits_ahead: Commits the worktree was ahead of *base* at exit
|
|
1450
|
+
time. ``None`` means the commit-count primitive couldn't
|
|
1451
|
+
determine it (e.g. detached HEAD); the seam treats this as
|
|
1452
|
+
non-zero per #448 policy.
|
|
1453
|
+
push_ok: ``True`` if ``git push`` succeeded (or was skipped
|
|
1454
|
+
because there were zero commits to push). ``False`` if a
|
|
1455
|
+
push was attempted and the remote rejected.
|
|
1456
|
+
push_error: Stderr from the failed push, when ``push_ok`` is
|
|
1457
|
+
``False``.
|
|
1458
|
+
already_recorded: ``True`` when ``coord report-result`` already
|
|
1459
|
+
wrote a terminal status for this assignment before the
|
|
1460
|
+
backstop ran. The backstop respects the agent's report and
|
|
1461
|
+
does NOT clobber it; this field tells the caller that.
|
|
1462
|
+
seam_outcome: The :class:`coord.issue_store.StoreOutcome` for
|
|
1463
|
+
the seam call (``None`` when the backstop deferred to an
|
|
1464
|
+
existing report).
|
|
1465
|
+
worktree_removed: ``True`` when the interactive worktree was
|
|
1466
|
+
successfully removed by the backstop. ``False`` when no
|
|
1467
|
+
*repo_path* was supplied (the caller owns cleanup) or when
|
|
1468
|
+
the removal failed.
|
|
1469
|
+
smoke_restored_paths: Human-readable descriptions of what the
|
|
1470
|
+
live-checkout restore-on-exit safety net reverted (#1256) —
|
|
1471
|
+
e.g. a path checked back out to ``HEAD``, or a branch moved
|
|
1472
|
+
back. Empty for every call that didn't pass
|
|
1473
|
+
``smoke_repo_path``, or that did but found no snapshot to
|
|
1474
|
+
restore (the overwhelmingly common case).
|
|
1475
|
+
smoke_restore_error: Set when the restore step itself failed
|
|
1476
|
+
outright (as opposed to "nothing to restore"). ``None``
|
|
1477
|
+
otherwise.
|
|
1478
|
+
artifacts_stashed: Number of files copied by the artifact stash
|
|
1479
|
+
step (#1295), or ``None`` when no stash was attempted at all
|
|
1480
|
+
(no ``artifact_paths`` configured, no worktree present, or —
|
|
1481
|
+
on the remote path — the ssh call couldn't be confirmed).
|
|
1482
|
+
``0`` is a real result: the stash ran but nothing matched the
|
|
1483
|
+
configured globs, which is exactly the silent-failure mode the
|
|
1484
|
+
issue's fix items #2/#3 asked to stop hiding — callers should
|
|
1485
|
+
treat ``0`` as worth a warning, not as "nothing happened".
|
|
1486
|
+
test_verdict_recovered: ``"passed"`` / ``"failed"`` when the #1351
|
|
1487
|
+
Test transcript-floor recovered a ``TEST_VERDICT:`` block and
|
|
1488
|
+
recorded it on the WORK row (only possible when the caller
|
|
1489
|
+
passed ``smoke_of``). ``None`` when no ``smoke_of`` was given,
|
|
1490
|
+
the work row already had a verdict, or nothing was recovered —
|
|
1491
|
+
the operator-prompt backstop (`_prompt_and_relay_test_verdict`)
|
|
1492
|
+
is still the ultimate fallback in every one of those cases.
|
|
1493
|
+
"""
|
|
1494
|
+
|
|
1495
|
+
terminal_status: str
|
|
1496
|
+
commits_ahead: int | None
|
|
1497
|
+
push_ok: bool
|
|
1498
|
+
push_error: str | None
|
|
1499
|
+
already_recorded: bool
|
|
1500
|
+
seam_outcome: object | None = None # StoreOutcome | None
|
|
1501
|
+
worktree_removed: bool = field(default=False)
|
|
1502
|
+
# #604: populated only on the --merge-of path (verify_merge=True). Carries
|
|
1503
|
+
# the merge verification (default_ahead / added / foreign) so the caller can
|
|
1504
|
+
# surface the offending commits even after the worktree is removed. None
|
|
1505
|
+
# for every other interactive flavour.
|
|
1506
|
+
merge_verify: MergeVerify | None = field(default=None)
|
|
1507
|
+
# #1256: populated only when the caller passed `smoke_repo_path` to
|
|
1508
|
+
# finalize_interactive_exit. See attribute docs above.
|
|
1509
|
+
smoke_restored_paths: list[str] = field(default_factory=list)
|
|
1510
|
+
smoke_restore_error: str | None = field(default=None)
|
|
1511
|
+
artifacts_stashed: int | None = field(default=None)
|
|
1512
|
+
test_verdict_recovered: str | None = field(default=None)
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def _git_push(wt_path: Path, *, timeout: float = 60.0) -> tuple[bool, str | None]:
|
|
1516
|
+
"""``git push -u origin HEAD`` with a clean (ok, error) return shape.
|
|
1517
|
+
|
|
1518
|
+
Caller decides what to do with a failed push. ``(True, None)`` is
|
|
1519
|
+
the happy path; ``(False, stderr)`` surfaces the remote's error.
|
|
1520
|
+
"""
|
|
1521
|
+
try:
|
|
1522
|
+
result = subprocess.run(
|
|
1523
|
+
["git", "push", "-u", "origin", "HEAD"],
|
|
1524
|
+
cwd=str(wt_path),
|
|
1525
|
+
capture_output=True,
|
|
1526
|
+
text=True,
|
|
1527
|
+
timeout=timeout,
|
|
1528
|
+
)
|
|
1529
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
1530
|
+
return False, str(exc)
|
|
1531
|
+
if result.returncode != 0:
|
|
1532
|
+
return False, (result.stderr or "git push failed").strip()
|
|
1533
|
+
return True, None
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
def _current_branch(wt_path: Path) -> str | None:
|
|
1537
|
+
"""Best-effort current-branch name for the worktree. ``None`` on
|
|
1538
|
+
detached HEAD or git failure — the caller passes the value through
|
|
1539
|
+
to the seam where ``None`` is acceptable (the assignment row already
|
|
1540
|
+
has the dispatch-time branch slug)."""
|
|
1541
|
+
try:
|
|
1542
|
+
result = subprocess.run(
|
|
1543
|
+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
|
1544
|
+
cwd=str(wt_path),
|
|
1545
|
+
capture_output=True,
|
|
1546
|
+
text=True,
|
|
1547
|
+
timeout=10.0,
|
|
1548
|
+
check=True,
|
|
1549
|
+
)
|
|
1550
|
+
except (subprocess.SubprocessError, OSError):
|
|
1551
|
+
return None
|
|
1552
|
+
branch = result.stdout.strip()
|
|
1553
|
+
if not branch or branch == "HEAD":
|
|
1554
|
+
return None
|
|
1555
|
+
return branch
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
# ── Live-checkout restore-on-exit for interactive smoke sessions (#1256) ────
|
|
1559
|
+
#
|
|
1560
|
+
# `--smoke-of` runs the human-attended smoke agent directly in the LIVE
|
|
1561
|
+
# checkout (no worktree — #1010), because exercising an *agent-side* file
|
|
1562
|
+
# (e.g. `coord/agent.py`, only live via a PyPI release + `coord agent
|
|
1563
|
+
# update`) requires that branch's version of the file to actually sit in the
|
|
1564
|
+
# editable-install path a scratch worktree elsewhere can't provide. The
|
|
1565
|
+
# operator/agent's only way to do that is a path-scoped
|
|
1566
|
+
# `git checkout <branch> -- <path>` in the live tree, which used to be left
|
|
1567
|
+
# dirty forever once the session ended.
|
|
1568
|
+
#
|
|
1569
|
+
# The pair below is the safety net: `snapshot_live_checkout_for_smoke` is
|
|
1570
|
+
# called ONCE, right before an interactive smoke session launches, and
|
|
1571
|
+
# records the checkout's pre-session branch + dirty-path baseline into two
|
|
1572
|
+
# marker files living inside the checkout's own `.git/` dir — keyed by
|
|
1573
|
+
# assignment_id. `restore_live_checkout_from_smoke_snapshot` is called from
|
|
1574
|
+
# `finalize_interactive_exit` every time a session finalizes (normal exit,
|
|
1575
|
+
# self-report via `coord report-result`, transcript-floor, or a later `coord
|
|
1576
|
+
# reattach` from a different operator machine) and, if a matching snapshot
|
|
1577
|
+
# exists, moves HEAD back and reverts any path that's dirty now but wasn't
|
|
1578
|
+
# in the baseline — leaving any *pre-existing* dirt untouched. Storing the
|
|
1579
|
+
# marker inside `.git/` (rather than under `~/.coord/`) means the restore
|
|
1580
|
+
# works identically wherever it happens to run from, with no cross-machine
|
|
1581
|
+
# bookkeeping.
|
|
1582
|
+
#
|
|
1583
|
+
# Both functions are best-effort and never raise: a failure here must never
|
|
1584
|
+
# block dispatch or clobber the session's actual test verdict.
|
|
1585
|
+
|
|
1586
|
+
def _smoke_snapshot_marker_paths(assignment_id: str) -> tuple[str, str]:
|
|
1587
|
+
"""Marker filenames (relative to a checkout's top level) for #1256."""
|
|
1588
|
+
return (
|
|
1589
|
+
f".git/coord-smoke-branch-{assignment_id}",
|
|
1590
|
+
f".git/coord-smoke-status-{assignment_id}",
|
|
1591
|
+
)
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
# Placeholders `__BRANCH_FILE__` / `__STATUS_FILE__` are substituted with the
|
|
1595
|
+
# marker paths via plain `.replace()` (not `.format()`/f-string) so the
|
|
1596
|
+
# shell's own `${...}` / `$(...)` syntax below needs no brace-escaping.
|
|
1597
|
+
_SMOKE_SNAPSHOT_SCRIPT = """\
|
|
1598
|
+
git rev-parse --is-inside-work-tree >/dev/null 2>&1 || exit 0
|
|
1599
|
+
BR=$(git rev-parse --abbrev-ref HEAD 2>/dev/null)
|
|
1600
|
+
if [ -z "$BR" ] || [ "$BR" = "HEAD" ]; then
|
|
1601
|
+
BR="DETACHED:$(git rev-parse HEAD 2>/dev/null)"
|
|
1602
|
+
fi
|
|
1603
|
+
printf '%s\\n' "$BR" > __BRANCH_FILE__
|
|
1604
|
+
git status --porcelain > __STATUS_FILE__
|
|
1605
|
+
"""
|
|
1606
|
+
|
|
1607
|
+
_SMOKE_RESTORE_SCRIPT = """\
|
|
1608
|
+
git rev-parse --is-inside-work-tree >/dev/null 2>&1 || exit 0
|
|
1609
|
+
if [ ! -f __BRANCH_FILE__ ]; then
|
|
1610
|
+
exit 0
|
|
1611
|
+
fi
|
|
1612
|
+
ORIG=$(cat __BRANCH_FILE__)
|
|
1613
|
+
CUR=$(git rev-parse --abbrev-ref HEAD 2>/dev/null)
|
|
1614
|
+
case "$ORIG" in
|
|
1615
|
+
DETACHED:*) : ;;
|
|
1616
|
+
*)
|
|
1617
|
+
if [ -n "$ORIG" ] && [ "$CUR" != "$ORIG" ]; then
|
|
1618
|
+
if git checkout "$ORIG" >/dev/null 2>&1; then
|
|
1619
|
+
echo "__BRANCH_RESTORED__:$ORIG"
|
|
1620
|
+
else
|
|
1621
|
+
echo "__BRANCH_RESTORE_FAILED__:$ORIG"
|
|
1622
|
+
fi
|
|
1623
|
+
fi
|
|
1624
|
+
;;
|
|
1625
|
+
esac
|
|
1626
|
+
git status --porcelain | while IFS= read -r line; do
|
|
1627
|
+
path=${line#?? }
|
|
1628
|
+
[ -z "$path" ] && continue
|
|
1629
|
+
if [ -f __STATUS_FILE__ ] && grep -qxF -- "$line" __STATUS_FILE__; then
|
|
1630
|
+
continue
|
|
1631
|
+
fi
|
|
1632
|
+
if git checkout HEAD -- "$path" >/dev/null 2>&1; then
|
|
1633
|
+
echo "__RESTORED_PATH__:$path"
|
|
1634
|
+
else
|
|
1635
|
+
git rm -f --cached -- "$path" >/dev/null 2>&1
|
|
1636
|
+
rm -f -- "$path"
|
|
1637
|
+
echo "__RESTORED_PATH__:$path"
|
|
1638
|
+
fi
|
|
1639
|
+
done
|
|
1640
|
+
rm -f __BRANCH_FILE__ __STATUS_FILE__
|
|
1641
|
+
echo "__SMOKE_RESTORE_DONE__"
|
|
1642
|
+
"""
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
def snapshot_live_checkout_for_smoke(
|
|
1646
|
+
repo_path: str,
|
|
1647
|
+
assignment_id: str,
|
|
1648
|
+
*,
|
|
1649
|
+
ssh_target: str | None = None,
|
|
1650
|
+
timeout: float = 15.0,
|
|
1651
|
+
) -> None:
|
|
1652
|
+
"""Record *repo_path*'s current branch + dirty-path baseline (#1256).
|
|
1653
|
+
|
|
1654
|
+
Call exactly once, right before an interactive ``--smoke-of`` session
|
|
1655
|
+
launches in the live checkout. Paired with
|
|
1656
|
+
:func:`restore_live_checkout_from_smoke_snapshot`.
|
|
1657
|
+
|
|
1658
|
+
When *ssh_target* is given, *repo_path* must be a path the REMOTE shell
|
|
1659
|
+
can expand itself (e.g. ``$HOME/src/foo``, unquoted) — mirrors
|
|
1660
|
+
``remote_repo_sh`` elsewhere in this module. Best-effort: any failure
|
|
1661
|
+
only means the restore-on-exit safety net won't activate for this
|
|
1662
|
+
session; it must never block dispatch.
|
|
1663
|
+
"""
|
|
1664
|
+
branch_file, status_file = _smoke_snapshot_marker_paths(assignment_id)
|
|
1665
|
+
script = _SMOKE_SNAPSHOT_SCRIPT.replace(
|
|
1666
|
+
"__BRANCH_FILE__", branch_file
|
|
1667
|
+
).replace("__STATUS_FILE__", status_file)
|
|
1668
|
+
try:
|
|
1669
|
+
if ssh_target is None:
|
|
1670
|
+
subprocess.run(
|
|
1671
|
+
["bash", "-c", script],
|
|
1672
|
+
cwd=repo_path,
|
|
1673
|
+
capture_output=True,
|
|
1674
|
+
text=True,
|
|
1675
|
+
timeout=timeout,
|
|
1676
|
+
)
|
|
1677
|
+
else:
|
|
1678
|
+
remote_cmd = f"cd {repo_path} 2>/dev/null && ( {script} )"
|
|
1679
|
+
subprocess.run(
|
|
1680
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
1681
|
+
capture_output=True,
|
|
1682
|
+
text=True,
|
|
1683
|
+
timeout=timeout,
|
|
1684
|
+
)
|
|
1685
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
1686
|
+
logging.warning(
|
|
1687
|
+
"smoke snapshot failed for assignment %s (repo=%s ssh_target=%s): "
|
|
1688
|
+
"%s — the live-checkout restore-on-exit safety net will not run "
|
|
1689
|
+
"for this session (#1256)",
|
|
1690
|
+
assignment_id, repo_path, ssh_target, exc,
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def restore_live_checkout_from_smoke_snapshot(
|
|
1695
|
+
repo_path: str,
|
|
1696
|
+
assignment_id: str,
|
|
1697
|
+
*,
|
|
1698
|
+
ssh_target: str | None = None,
|
|
1699
|
+
timeout: float = 20.0,
|
|
1700
|
+
) -> tuple[list[str], str | None]:
|
|
1701
|
+
"""Best-effort counterpart to :func:`snapshot_live_checkout_for_smoke`.
|
|
1702
|
+
|
|
1703
|
+
A no-op — returns ``([], None)`` — unless a snapshot marker for
|
|
1704
|
+
*assignment_id* exists in *repo_path*'s ``.git/`` dir, so calling this
|
|
1705
|
+
unconditionally from every interactive finalize path is safe: every
|
|
1706
|
+
flavour other than smoke (and a smoke session that already restored
|
|
1707
|
+
once) simply finds no marker and does nothing.
|
|
1708
|
+
|
|
1709
|
+
When a marker IS found: moves HEAD back to the captured branch if it
|
|
1710
|
+
drifted, then reverts any path dirty now but not in the captured
|
|
1711
|
+
baseline — the mutation the smoke session introduced — leaving any
|
|
1712
|
+
dirt that pre-dates the session untouched. Deletes the marker files
|
|
1713
|
+
when done.
|
|
1714
|
+
|
|
1715
|
+
Returns ``(restored_paths, error)`` — human-readable strings describing
|
|
1716
|
+
what was reverted (or ``[]``), and an error string on outright failure.
|
|
1717
|
+
Never raises.
|
|
1718
|
+
"""
|
|
1719
|
+
branch_file, status_file = _smoke_snapshot_marker_paths(assignment_id)
|
|
1720
|
+
script = _SMOKE_RESTORE_SCRIPT.replace(
|
|
1721
|
+
"__BRANCH_FILE__", branch_file
|
|
1722
|
+
).replace("__STATUS_FILE__", status_file)
|
|
1723
|
+
try:
|
|
1724
|
+
if ssh_target is None:
|
|
1725
|
+
result = subprocess.run(
|
|
1726
|
+
["bash", "-c", script],
|
|
1727
|
+
cwd=repo_path,
|
|
1728
|
+
capture_output=True,
|
|
1729
|
+
text=True,
|
|
1730
|
+
timeout=timeout,
|
|
1731
|
+
)
|
|
1732
|
+
else:
|
|
1733
|
+
remote_cmd = f"cd {repo_path} 2>/dev/null && ( {script} )"
|
|
1734
|
+
result = subprocess.run(
|
|
1735
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
1736
|
+
capture_output=True,
|
|
1737
|
+
text=True,
|
|
1738
|
+
timeout=timeout,
|
|
1739
|
+
)
|
|
1740
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
1741
|
+
return [], str(exc)
|
|
1742
|
+
|
|
1743
|
+
restored: list[str] = []
|
|
1744
|
+
for line in (result.stdout or "").splitlines():
|
|
1745
|
+
if line.startswith("__RESTORED_PATH__:"):
|
|
1746
|
+
restored.append(line.split(":", 1)[1])
|
|
1747
|
+
elif line.startswith("__BRANCH_RESTORED__:"):
|
|
1748
|
+
restored.append(f"(branch restored to {line.split(':', 1)[1]})")
|
|
1749
|
+
elif line.startswith("__BRANCH_RESTORE_FAILED__:"):
|
|
1750
|
+
restored.append(
|
|
1751
|
+
f"(WARNING: failed to restore branch to {line.split(':', 1)[1]})"
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
if result.returncode != 0:
|
|
1755
|
+
err = (result.stderr or "").strip() or (
|
|
1756
|
+
f"restore script exited {result.returncode}"
|
|
1757
|
+
)
|
|
1758
|
+
return restored, err
|
|
1759
|
+
return restored, None
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
def _assignment_status(assignment_id: str) -> str | None:
|
|
1763
|
+
"""Read an assignment's ``status`` — daemon-aware (#883).
|
|
1764
|
+
|
|
1765
|
+
On a thin client the canonical status lives on the daemon (a reviewer's
|
|
1766
|
+
``coord report-result`` routes there via ``issue_store``), so a local-only
|
|
1767
|
+
read sees a stale ``running`` and misses the terminal write — the sibling of
|
|
1768
|
+
the #877 ``load_assignment_review_findings`` bug. Prefer the daemon board
|
|
1769
|
+
(``GET /board`` filtered by ``assignment_id``) when ``board_service`` is set;
|
|
1770
|
+
fall back to the local DB when unset (daemon host, where the local DB IS
|
|
1771
|
+
canonical) or the fetch fails. Returns ``None`` when the row is absent.
|
|
1772
|
+
"""
|
|
1773
|
+
try:
|
|
1774
|
+
from coord.board_service import resolve as _resolve_svc # noqa: PLC0415
|
|
1775
|
+
|
|
1776
|
+
svc = _resolve_svc()
|
|
1777
|
+
except Exception: # noqa: BLE001
|
|
1778
|
+
svc = None
|
|
1779
|
+
if svc is not None:
|
|
1780
|
+
try:
|
|
1781
|
+
from coord.client import fetch_assignment, fetch_board_payload # noqa: PLC0415
|
|
1782
|
+
|
|
1783
|
+
# #1336 invariant 3: one row via the point endpoint, not the whole
|
|
1784
|
+
# /board collection. 404 → pre-#1336 daemon (or genuinely absent):
|
|
1785
|
+
# one compatibility pass through the collection payload.
|
|
1786
|
+
row = fetch_assignment(svc, assignment_id)
|
|
1787
|
+
if row is not None:
|
|
1788
|
+
return row.get("status")
|
|
1789
|
+
payload = fetch_board_payload(svc)
|
|
1790
|
+
for a in payload.get("assignments", []):
|
|
1791
|
+
if a.get("assignment_id") == assignment_id:
|
|
1792
|
+
return a.get("status")
|
|
1793
|
+
# Daemon is canonical: assignment absent ⇒ no terminal write yet.
|
|
1794
|
+
return None
|
|
1795
|
+
except Exception: # noqa: BLE001 — daemon unreachable → local fallback
|
|
1796
|
+
pass
|
|
1797
|
+
try:
|
|
1798
|
+
from coord.state import get_connection # noqa: PLC0415
|
|
1799
|
+
|
|
1800
|
+
conn = get_connection()
|
|
1801
|
+
row = conn.execute(
|
|
1802
|
+
"SELECT status FROM assignments WHERE assignment_id=?",
|
|
1803
|
+
(assignment_id,),
|
|
1804
|
+
).fetchone()
|
|
1805
|
+
except Exception: # noqa: BLE001
|
|
1806
|
+
return None
|
|
1807
|
+
if row is None:
|
|
1808
|
+
return None
|
|
1809
|
+
return row["status"] if hasattr(row, "keys") else row[0]
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
def _assignment_already_recorded(assignment_id: str) -> bool:
|
|
1813
|
+
"""Did ``coord report-result`` already write a terminal state for
|
|
1814
|
+
this assignment? We check the assignments row's ``status`` column.
|
|
1815
|
+
The backstop runs after the human session exits, so if the agent
|
|
1816
|
+
invoked ``coord report-result`` before exiting we must NOT clobber
|
|
1817
|
+
that decision with the git-floor's heuristic verdict.
|
|
1818
|
+
|
|
1819
|
+
#883: the status read is daemon-aware (see :func:`_assignment_status`) — on
|
|
1820
|
+
a thin client the reviewer's terminal write lands on the daemon, not the
|
|
1821
|
+
local DB, so a local-only read made this fire the backstop on every
|
|
1822
|
+
interactive review even after a clean ``report-result``.
|
|
1823
|
+
"""
|
|
1824
|
+
if not assignment_id:
|
|
1825
|
+
return False
|
|
1826
|
+
status = _assignment_status(assignment_id)
|
|
1827
|
+
# Anything that's not 'running'/'pending' means a terminal write
|
|
1828
|
+
# already landed (done/advisory/failed/cancelled). Respect it.
|
|
1829
|
+
return status not in (None, "", "running", "pending")
|
|
1830
|
+
|
|
1831
|
+
|
|
1832
|
+
def _finalize_merge_blocked(
|
|
1833
|
+
*,
|
|
1834
|
+
merge_verify, # agent.MergeVerify (not ok)
|
|
1835
|
+
assignment_id: str,
|
|
1836
|
+
machine_name: str,
|
|
1837
|
+
repo_name: str,
|
|
1838
|
+
repo_github: str,
|
|
1839
|
+
issue_number: int,
|
|
1840
|
+
base_branch: str,
|
|
1841
|
+
worktree_path: str | None,
|
|
1842
|
+
repo_path: str | None,
|
|
1843
|
+
started_at: float | None,
|
|
1844
|
+
log_path: str | None,
|
|
1845
|
+
branch: str | None = None,
|
|
1846
|
+
smoke_restored_paths: list[str] | None = None,
|
|
1847
|
+
smoke_restore_error: str | None = None,
|
|
1848
|
+
test_verdict_recovered: str | None = None,
|
|
1849
|
+
) -> InteractiveFinalizeResult:
|
|
1850
|
+
"""Record a botched ``--merge-of`` rebase as blocked → ``failed`` (#604).
|
|
1851
|
+
|
|
1852
|
+
Routes through the same ``post_result(status=blocked)`` seam ``coord
|
|
1853
|
+
report-result`` uses (so a thin client still lands the write on the daemon),
|
|
1854
|
+
OVERRIDING any ``done`` the agent self-reported. ``blocked`` maps to a
|
|
1855
|
+
``failed`` board state — there is no literal ``blocked`` status the rest of
|
|
1856
|
+
the pipeline understands; the override's whole purpose is to keep the entry
|
|
1857
|
+
out of the merge-ready set. The offending commits ride in the summary,
|
|
1858
|
+
posted on the issue, so the operator sees exactly why. Cleans up the
|
|
1859
|
+
worktree afterwards, matching the normal-path discipline.
|
|
1860
|
+
|
|
1861
|
+
*smoke_restored_paths*/*smoke_restore_error*/*test_verdict_recovered*
|
|
1862
|
+
(#1256, #1351): this gate is unreachable for a smoke session in practice
|
|
1863
|
+
(every ``smoke_repo_path``/``smoke_of`` caller also passes
|
|
1864
|
+
``worktree_path=None``, and the gate above requires ``verify_merge and
|
|
1865
|
+
worktree_path``) — but the restore mutation and the Test transcript-floor
|
|
1866
|
+
in :func:`finalize_interactive_exit` both run unconditionally before this
|
|
1867
|
+
gate is even checked, so if that invariant ever changes, their results
|
|
1868
|
+
must still be threaded through rather than silently dropped.
|
|
1869
|
+
"""
|
|
1870
|
+
from coord.issue_store import ( # noqa: PLC0415
|
|
1871
|
+
STATUS_BLOCKED,
|
|
1872
|
+
ResultRecord,
|
|
1873
|
+
post_result,
|
|
1874
|
+
)
|
|
1875
|
+
|
|
1876
|
+
wt_p = Path(worktree_path) if worktree_path else None
|
|
1877
|
+
branch_now = (
|
|
1878
|
+
_current_branch(wt_p) if wt_p is not None and wt_p.exists() else None
|
|
1879
|
+
)
|
|
1880
|
+
# Capture whether the agent had already self-reported BEFORE we override it,
|
|
1881
|
+
# so the caller can tell "we overrode a self-reported done" from "we were
|
|
1882
|
+
# the only writer".
|
|
1883
|
+
prior_recorded = _assignment_already_recorded(assignment_id)
|
|
1884
|
+
|
|
1885
|
+
duration: float | None = None
|
|
1886
|
+
if started_at is not None:
|
|
1887
|
+
duration = max(0.0, time.time() - started_at)
|
|
1888
|
+
|
|
1889
|
+
outcome = post_result(
|
|
1890
|
+
ResultRecord(
|
|
1891
|
+
assignment_id=assignment_id,
|
|
1892
|
+
machine_name=machine_name,
|
|
1893
|
+
repo_name=repo_name,
|
|
1894
|
+
repo_github=repo_github,
|
|
1895
|
+
issue_number=issue_number,
|
|
1896
|
+
status=STATUS_BLOCKED,
|
|
1897
|
+
verdict=None,
|
|
1898
|
+
summary=merge_verify.block_summary(base_branch),
|
|
1899
|
+
duration_seconds=duration,
|
|
1900
|
+
log_path=log_path,
|
|
1901
|
+
branch=branch_now or branch,
|
|
1902
|
+
)
|
|
1903
|
+
)
|
|
1904
|
+
|
|
1905
|
+
worktree_removed = False
|
|
1906
|
+
if repo_path is not None and wt_p is not None:
|
|
1907
|
+
worktree_removed = _remove_worktree(Path(repo_path), wt_p)
|
|
1908
|
+
|
|
1909
|
+
# #546: mark as interactive + capture tokens from the session transcript.
|
|
1910
|
+
_persist_interactive_tokens(assignment_id, started_at, worktree_path)
|
|
1911
|
+
|
|
1912
|
+
return InteractiveFinalizeResult(
|
|
1913
|
+
terminal_status=outcome.status, # "failed" (blocked → failed)
|
|
1914
|
+
commits_ahead=merge_verify.default_ahead,
|
|
1915
|
+
push_ok=True,
|
|
1916
|
+
push_error=None,
|
|
1917
|
+
already_recorded=prior_recorded,
|
|
1918
|
+
seam_outcome=outcome,
|
|
1919
|
+
worktree_removed=worktree_removed,
|
|
1920
|
+
merge_verify=merge_verify,
|
|
1921
|
+
smoke_restored_paths=list(smoke_restored_paths or []),
|
|
1922
|
+
smoke_restore_error=smoke_restore_error,
|
|
1923
|
+
test_verdict_recovered=test_verdict_recovered,
|
|
1924
|
+
)
|
|
1925
|
+
|
|
1926
|
+
|
|
1927
|
+
def _transcript_names_issue(body: str, issue_number: int) -> bool:
|
|
1928
|
+
"""True when a review body explicitly names *issue_number* (#617).
|
|
1929
|
+
|
|
1930
|
+
Accepts BOTH the ``#<N>`` form (how reviewers refer to an issue in prose —
|
|
1931
|
+
the real #607 review body used ``#607``) and the ``issue-<N>`` form (the
|
|
1932
|
+
branch / briefing tag). The original floor gated on a literal
|
|
1933
|
+
``issue-<N>`` only, so a review whose body said ``#607`` was silently
|
|
1934
|
+
rejected and its findings lost — a hidden contributor to the recurring
|
|
1935
|
+
drop, caught by live-smoking the real transcript. The trailing-digit
|
|
1936
|
+
negative lookahead stops ``#607`` from matching ``#6070``; gating on an
|
|
1937
|
+
explicit reference still prevents mis-attributing a concurrent, unrelated
|
|
1938
|
+
review's verdict.
|
|
1939
|
+
"""
|
|
1940
|
+
return re.search(rf"(?:#|issue-){issue_number}(?!\d)", body) is not None
|
|
1941
|
+
|
|
1942
|
+
|
|
1943
|
+
def _transcript_names_assignment(body: str, assignment_id: str) -> bool:
|
|
1944
|
+
"""True when a transcript carries this exact *assignment_id* (#989).
|
|
1945
|
+
|
|
1946
|
+
The interactive review briefing embeds the assignment id verbatim in its
|
|
1947
|
+
very first prompt — ``[Coordinator review assignment <id>]`` and
|
|
1948
|
+
``coord report-result --assignment <id>`` (``_dispatch_review_of`` in
|
|
1949
|
+
``coord/commands/dispatch_workers.py``) — so only the transcript of the
|
|
1950
|
+
session THIS assignment id was actually dispatched to can ever contain
|
|
1951
|
+
it. Unlike an issue number (a small integer that sibling issues under the
|
|
1952
|
+
same epic routinely cross-reference in prose/docs — the #989 incident:
|
|
1953
|
+
#930's transcript legitimately contained the substring ``issue-931``
|
|
1954
|
+
even though the session ran entirely for #930), a freshly generated
|
|
1955
|
+
``uuid4().hex[:12]`` cannot plausibly appear in an unrelated session's
|
|
1956
|
+
transcript by coincidence. This is the primary discriminator between two
|
|
1957
|
+
concurrent review sessions that both happen to name the same issue;
|
|
1958
|
+
``_transcript_names_issue`` is kept as an additional (much weaker) check
|
|
1959
|
+
on top of it.
|
|
1960
|
+
"""
|
|
1961
|
+
return bool(assignment_id) and assignment_id in body
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def _fetch_remote_review_findings(
|
|
1965
|
+
issue_number: int,
|
|
1966
|
+
cutoff: float,
|
|
1967
|
+
ssh_target: str,
|
|
1968
|
+
*,
|
|
1969
|
+
assignment_id: str,
|
|
1970
|
+
timeout: float = 30.0,
|
|
1971
|
+
_diagnostic: list | None = None,
|
|
1972
|
+
):
|
|
1973
|
+
"""Remote twin of the transcript-floor: parse the review block from the
|
|
1974
|
+
Claude transcript on the SESSION'S OWN host (#617).
|
|
1975
|
+
|
|
1976
|
+
When an interactive review ran on a remote machine and the operator
|
|
1977
|
+
reattached + exited from a DIFFERENT machine, the local
|
|
1978
|
+
``~/.claude/projects`` scan is blind — the transcript lives on the remote
|
|
1979
|
+
host, so the findings vanished and only a verdict-less operator prompt was
|
|
1980
|
+
left (the #607 incident). This lists the remote host's recent transcripts
|
|
1981
|
+
over ssh (newest-first, active at/after *cutoff*), streams each candidate
|
|
1982
|
+
back, and parses it with the same :func:`coord.review.parse_review_from_log`
|
|
1983
|
+
used locally. Returns the first ``ReviewFindings`` that parses as a
|
|
1984
|
+
review AND names this issue (``issue-<N>``) AND carries this exact
|
|
1985
|
+
*assignment_id* (#989 — the issue-number check alone is not sufficient:
|
|
1986
|
+
sibling issues under the same epic legitimately cross-reference each
|
|
1987
|
+
other's numbers, so a DIFFERENT assignment's transcript can satisfy it),
|
|
1988
|
+
or ``None`` (ssh failure / no match) so the caller falls through to the
|
|
1989
|
+
operator-prompt backstop.
|
|
1990
|
+
|
|
1991
|
+
The ``cutoff`` filter already bounds the candidate set to the session
|
|
1992
|
+
window — no additional per-scan cap is applied (#619: the original
|
|
1993
|
+
``max_candidates=6`` dropped the target transcript under concurrent-session
|
|
1994
|
+
load, exactly the condition when reviews run).
|
|
1995
|
+
|
|
1996
|
+
*_diagnostic* (#1348): optional out-parameter. When a non-``None`` list
|
|
1997
|
+
is provided and a candidate transcript passes the attribution gates but the
|
|
1998
|
+
strict parse fails, :func:`coord.review.detect_unparsed_review_marker` is
|
|
1999
|
+
called and its result (if non-``None``) appended — at most once, for the
|
|
2000
|
+
newest matching transcript. The happy-path return type is unchanged;
|
|
2001
|
+
callers that don't need diagnostics pass the default ``None``.
|
|
2002
|
+
"""
|
|
2003
|
+
from coord.review import parse_review_from_log # noqa: PLC0415
|
|
2004
|
+
|
|
2005
|
+
# GNU find on the fleet (Linux): list every recent transcript with its
|
|
2006
|
+
# mtime, newest-first. `$HOME` is expanded by the REMOTE shell.
|
|
2007
|
+
# head -200 is a safety rail against enormous project dirs; the cutoff
|
|
2008
|
+
# filter below is the real bound.
|
|
2009
|
+
list_cmd = (
|
|
2010
|
+
'find "$HOME/.claude/projects" -maxdepth 2 -name "*.jsonl" '
|
|
2011
|
+
r"-printf '%T@\t%p\n' 2>/dev/null | sort -rn | head -200"
|
|
2012
|
+
)
|
|
2013
|
+
try:
|
|
2014
|
+
listing = subprocess.run(
|
|
2015
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, list_cmd],
|
|
2016
|
+
capture_output=True,
|
|
2017
|
+
text=True,
|
|
2018
|
+
timeout=timeout,
|
|
2019
|
+
)
|
|
2020
|
+
except (subprocess.SubprocessError, OSError):
|
|
2021
|
+
return None
|
|
2022
|
+
if listing.returncode != 0:
|
|
2023
|
+
return None
|
|
2024
|
+
|
|
2025
|
+
candidates: list[str] = []
|
|
2026
|
+
for raw in (listing.stdout or "").splitlines():
|
|
2027
|
+
if "\t" not in raw:
|
|
2028
|
+
continue
|
|
2029
|
+
mtime_s, path = raw.split("\t", 1)
|
|
2030
|
+
try:
|
|
2031
|
+
mtime = float(mtime_s)
|
|
2032
|
+
except ValueError:
|
|
2033
|
+
continue
|
|
2034
|
+
path = path.strip()
|
|
2035
|
+
# Collect ALL candidates within the cutoff window — no early break.
|
|
2036
|
+
# The cutoff is the authoritative bound; stopping at an arbitrary N
|
|
2037
|
+
# is what caused the #619 miss under concurrent-session load.
|
|
2038
|
+
if path and mtime >= cutoff:
|
|
2039
|
+
candidates.append(path)
|
|
2040
|
+
|
|
2041
|
+
for remote_path in candidates:
|
|
2042
|
+
try:
|
|
2043
|
+
cat = subprocess.run(
|
|
2044
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, f"cat {shlex.quote(remote_path)}"],
|
|
2045
|
+
capture_output=True,
|
|
2046
|
+
text=True,
|
|
2047
|
+
timeout=timeout,
|
|
2048
|
+
)
|
|
2049
|
+
except (subprocess.SubprocessError, OSError):
|
|
2050
|
+
continue
|
|
2051
|
+
if cat.returncode != 0 or not cat.stdout:
|
|
2052
|
+
continue
|
|
2053
|
+
tmp_path: Path | None = None
|
|
2054
|
+
try:
|
|
2055
|
+
with tempfile.NamedTemporaryFile(
|
|
2056
|
+
"w", suffix=".jsonl", delete=False, encoding="utf-8"
|
|
2057
|
+
) as tf:
|
|
2058
|
+
tf.write(cat.stdout)
|
|
2059
|
+
tmp_path = Path(tf.name)
|
|
2060
|
+
findings = parse_review_from_log(tmp_path)
|
|
2061
|
+
finally:
|
|
2062
|
+
if tmp_path is not None:
|
|
2063
|
+
try:
|
|
2064
|
+
tmp_path.unlink()
|
|
2065
|
+
except OSError:
|
|
2066
|
+
pass
|
|
2067
|
+
# Gate on the WHOLE transcript, not just the parsed review prose: a
|
|
2068
|
+
# reviewer describes the code, not the issue number, so the body often
|
|
2069
|
+
# never says "#362" (real #362 miss, 2026-06-18) — but the review
|
|
2070
|
+
# BRIEFING seeds the issue/branch (`issue-<N>`), so the transcript does.
|
|
2071
|
+
# #989: the issue-number check ALONE is not sufficient — also require
|
|
2072
|
+
# this exact assignment_id, since a sibling issue's own review session
|
|
2073
|
+
# can legitimately name this issue number in prose.
|
|
2074
|
+
if (
|
|
2075
|
+
findings is not None
|
|
2076
|
+
and _transcript_names_issue(cat.stdout, issue_number)
|
|
2077
|
+
and _transcript_names_assignment(cat.stdout, assignment_id)
|
|
2078
|
+
):
|
|
2079
|
+
return findings
|
|
2080
|
+
# #1348: strict parse failed — when the caller wants diagnostics and
|
|
2081
|
+
# the attribution gates pass, check whether the transcript contains a
|
|
2082
|
+
# REVIEW_VERDICT: marker that the strict parser rejected. Only collect
|
|
2083
|
+
# the first (newest) hit so `_diagnostic[0]` is always the most recent
|
|
2084
|
+
# candidate; `not _diagnostic` prevents overwriting with an older one.
|
|
2085
|
+
if (
|
|
2086
|
+
findings is None
|
|
2087
|
+
and _diagnostic is not None
|
|
2088
|
+
and not _diagnostic
|
|
2089
|
+
and _transcript_names_issue(cat.stdout, issue_number)
|
|
2090
|
+
and _transcript_names_assignment(cat.stdout, assignment_id)
|
|
2091
|
+
):
|
|
2092
|
+
from coord.review import detect_unparsed_review_marker # noqa: PLC0415
|
|
2093
|
+
_marker = detect_unparsed_review_marker(
|
|
2094
|
+
cat.stdout,
|
|
2095
|
+
transcript_path=remote_path, # path on the remote host
|
|
2096
|
+
host=ssh_target,
|
|
2097
|
+
)
|
|
2098
|
+
if _marker is not None:
|
|
2099
|
+
_diagnostic.append(_marker)
|
|
2100
|
+
return None
|
|
2101
|
+
|
|
2102
|
+
|
|
2103
|
+
def _tokens_from_transcript(
|
|
2104
|
+
started_at: float | None,
|
|
2105
|
+
*,
|
|
2106
|
+
worktree_path: str | None = None,
|
|
2107
|
+
projects_dir: Path | None = None,
|
|
2108
|
+
) -> tuple[int, int, int, int]:
|
|
2109
|
+
"""#546: sum token usage from Claude Code session transcripts for this session.
|
|
2110
|
+
|
|
2111
|
+
Claude Code persists every session as a JSONL transcript under
|
|
2112
|
+
``~/.claude/projects/<cwd>/``. Each ``assistant`` message carries a
|
|
2113
|
+
``usage`` dict with ``input_tokens``, ``output_tokens``,
|
|
2114
|
+
``cache_creation_input_tokens``, and ``cache_read_input_tokens``.
|
|
2115
|
+
|
|
2116
|
+
When *worktree_path* is given, we restrict the scan to the project
|
|
2117
|
+
directory that corresponds to that CWD (the same mapping Claude Code uses:
|
|
2118
|
+
replace ``/`` with ``-``). This prevents attributing a concurrent
|
|
2119
|
+
unrelated session's tokens to this assignment. Without *worktree_path*
|
|
2120
|
+
we fall back to scanning ALL project directories that were active since
|
|
2121
|
+
*started_at* — broader but still bounded by the time window.
|
|
2122
|
+
|
|
2123
|
+
Returns ``(input_tokens, output_tokens, cache_creation_tokens,
|
|
2124
|
+
cache_read_tokens)``. Returns ``(0, 0, 0, 0)`` when no tokens can be
|
|
2125
|
+
recovered — callers treat that as "no data" and skip the write rather
|
|
2126
|
+
than zeroing out existing columns.
|
|
2127
|
+
"""
|
|
2128
|
+
if started_at is None:
|
|
2129
|
+
return 0, 0, 0, 0
|
|
2130
|
+
|
|
2131
|
+
base = projects_dir if projects_dir is not None else (Path.home() / ".claude" / "projects")
|
|
2132
|
+
if not base.is_dir():
|
|
2133
|
+
return 0, 0, 0, 0
|
|
2134
|
+
|
|
2135
|
+
# Determine which project directories to search. When we have the CWD
|
|
2136
|
+
# (worktree_path) we can be targeted; otherwise we scan all dirs that were
|
|
2137
|
+
# touched during the session window.
|
|
2138
|
+
if worktree_path is not None:
|
|
2139
|
+
# Claude Code names project dirs by replacing every '/' in the CWD
|
|
2140
|
+
# with '-'. Example: /home/john/.coord/worktrees/abc → -home-john--coord-worktrees-abc
|
|
2141
|
+
proj_name = str(worktree_path).replace("/", "-")
|
|
2142
|
+
proj_dir = base / proj_name
|
|
2143
|
+
search_dirs = [proj_dir] if proj_dir.is_dir() else []
|
|
2144
|
+
else:
|
|
2145
|
+
search_dirs = []
|
|
2146
|
+
|
|
2147
|
+
# Fallback: scan ALL project dirs and filter by file mtime.
|
|
2148
|
+
if not search_dirs:
|
|
2149
|
+
try:
|
|
2150
|
+
search_dirs = [d for d in base.iterdir() if d.is_dir()]
|
|
2151
|
+
except OSError:
|
|
2152
|
+
return 0, 0, 0, 0
|
|
2153
|
+
|
|
2154
|
+
cutoff = started_at - 5.0 # small clock-skew buffer
|
|
2155
|
+
input_tokens = output_tokens = cache_creation = cache_read = 0
|
|
2156
|
+
|
|
2157
|
+
for proj_dir in search_dirs:
|
|
2158
|
+
try:
|
|
2159
|
+
jsonl_files = list(proj_dir.glob("*.jsonl"))
|
|
2160
|
+
except OSError:
|
|
2161
|
+
continue
|
|
2162
|
+
for p in jsonl_files:
|
|
2163
|
+
try:
|
|
2164
|
+
if p.stat().st_mtime < cutoff:
|
|
2165
|
+
continue
|
|
2166
|
+
text = p.read_text(encoding="utf-8", errors="replace")
|
|
2167
|
+
except OSError:
|
|
2168
|
+
continue
|
|
2169
|
+
for line in text.splitlines():
|
|
2170
|
+
if not line or not line.startswith("{"):
|
|
2171
|
+
continue
|
|
2172
|
+
try:
|
|
2173
|
+
obj = json.loads(line)
|
|
2174
|
+
except json.JSONDecodeError:
|
|
2175
|
+
continue
|
|
2176
|
+
if obj.get("type") != "assistant":
|
|
2177
|
+
continue
|
|
2178
|
+
msg = obj.get("message") or {}
|
|
2179
|
+
usage = msg.get("usage") or {}
|
|
2180
|
+
input_tokens += int(usage.get("input_tokens") or 0)
|
|
2181
|
+
output_tokens += int(usage.get("output_tokens") or 0)
|
|
2182
|
+
cache_creation += int(usage.get("cache_creation_input_tokens") or 0)
|
|
2183
|
+
cache_read += int(usage.get("cache_read_input_tokens") or 0)
|
|
2184
|
+
|
|
2185
|
+
return input_tokens, output_tokens, cache_creation, cache_read
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
def _review_findings_from_transcript(
|
|
2189
|
+
issue_number: int,
|
|
2190
|
+
started_at: float | None,
|
|
2191
|
+
*,
|
|
2192
|
+
assignment_id: str,
|
|
2193
|
+
projects_dir: Path | None = None,
|
|
2194
|
+
ssh_target: str | None = None,
|
|
2195
|
+
_diagnostic: list | None = None,
|
|
2196
|
+
):
|
|
2197
|
+
"""Recover a human-attended review's verdict + findings from the Claude
|
|
2198
|
+
session transcript — the **transcript-floor** backstop (#606).
|
|
2199
|
+
|
|
2200
|
+
A review session emits its verdict as ``REVIEW_VERDICT:`` / ``REVIEW_BODY:``
|
|
2201
|
+
in its output, but often cannot run ``coord report-result`` because ``coord``
|
|
2202
|
+
is not on the interactive session's PATH (same class as the ``claude``-not-
|
|
2203
|
+
on-ssh-PATH issue). Reviews also have no git-floor (read-only, 0 commits),
|
|
2204
|
+
so without this the findings are lost the moment the operator exits Claude.
|
|
2205
|
+
|
|
2206
|
+
Claude Code persists every session as a JSONL transcript under
|
|
2207
|
+
``~/.claude/projects/<cwd>/``; we parse the most-recent one(s) active during
|
|
2208
|
+
this session with the existing :func:`coord.review.parse_review_from_log`
|
|
2209
|
+
(which already handles the transcript's stream-json shape). Returns the
|
|
2210
|
+
parsed ``ReviewFindings`` or ``None`` — independent of the agent running any
|
|
2211
|
+
command.
|
|
2212
|
+
|
|
2213
|
+
Robustness: only a transcript that parses as a review AND names this issue
|
|
2214
|
+
(``issue-<N>``, the format the review briefing produces) AND carries this
|
|
2215
|
+
exact *assignment_id* is trusted — newest-first, so the just-exited session
|
|
2216
|
+
wins. The issue-number check alone is NOT sufficient (#989): #930 and #931
|
|
2217
|
+
were sibling issues under the same epic that cross-referenced each other's
|
|
2218
|
+
numbers throughout their bodies/docs, so #930's own (legitimately dead-
|
|
2219
|
+
unrelated) review transcript satisfied the ``issue-931`` substring check
|
|
2220
|
+
and got recorded as #931's verdict, on BOTH of #931's review assignment
|
|
2221
|
+
slots. ``assignment_id`` is a fresh ``uuid4().hex[:12]`` embedded verbatim
|
|
2222
|
+
in the review briefing's first prompt (``_dispatch_review_of``) — it
|
|
2223
|
+
cannot plausibly appear in a different assignment's transcript, so
|
|
2224
|
+
requiring it closes that cross-issue (and cross-assignment) attribution
|
|
2225
|
+
hole. There is deliberately NO "guess the only review in the window"
|
|
2226
|
+
fallback: that could mis-attribute an unrelated/concurrent review (and
|
|
2227
|
+
record its verdict against the wrong assignment id). Self-gating — a work
|
|
2228
|
+
session's transcript carries no ``REVIEW_VERDICT`` block, so this returns
|
|
2229
|
+
``None`` and the caller falls through to the git-floor.
|
|
2230
|
+
|
|
2231
|
+
*_diagnostic* (#1348): optional out-parameter. When a non-``None`` list is
|
|
2232
|
+
provided and a candidate transcript passes the attribution gates but the
|
|
2233
|
+
strict parse fails, :func:`coord.review.detect_unparsed_review_marker` is
|
|
2234
|
+
called and the result (if non-``None``) appended — at most once, for the
|
|
2235
|
+
newest qualifying transcript. The happy-path return type is unchanged;
|
|
2236
|
+
callers that don't need diagnostics pass the default ``None``.
|
|
2237
|
+
"""
|
|
2238
|
+
from coord.review import parse_review_from_log # noqa: PLC0415
|
|
2239
|
+
|
|
2240
|
+
# No session start → no bounded window → don't scan (avoid matching a stale,
|
|
2241
|
+
# unrelated transcript). Production always passes a real started_at; only a
|
|
2242
|
+
# caller that can't bound the session (or a test) passes None.
|
|
2243
|
+
if started_at is None:
|
|
2244
|
+
return None
|
|
2245
|
+
# Only sessions active at/after this review started (small clock-skew buffer).
|
|
2246
|
+
cutoff = started_at - 5.0
|
|
2247
|
+
# #617: when the session ran on a REMOTE host (operator reattached + exited
|
|
2248
|
+
# from a different machine), its Claude transcript lives THERE — the local
|
|
2249
|
+
# projects dir is blind to it (the #607 silent-drop). Scan the session's
|
|
2250
|
+
# own host over ssh instead.
|
|
2251
|
+
if ssh_target is not None:
|
|
2252
|
+
result = _fetch_remote_review_findings(
|
|
2253
|
+
issue_number, cutoff, ssh_target, assignment_id=assignment_id,
|
|
2254
|
+
_diagnostic=_diagnostic, # #1348: thread through so remote floor can surface markers
|
|
2255
|
+
)
|
|
2256
|
+
if result is None:
|
|
2257
|
+
# One settle-and-retry: covers a transcript-flush blip where the
|
|
2258
|
+
# JSONL hasn't been fully flushed to disk at the instant we read it
|
|
2259
|
+
# (#619). 2 s is long enough for a local flush but short enough
|
|
2260
|
+
# to avoid annoying the operator on the fast path.
|
|
2261
|
+
time.sleep(2.0)
|
|
2262
|
+
result = _fetch_remote_review_findings(
|
|
2263
|
+
issue_number, cutoff, ssh_target, assignment_id=assignment_id,
|
|
2264
|
+
_diagnostic=_diagnostic, # #1348: second pass also collects markers
|
|
2265
|
+
)
|
|
2266
|
+
return result
|
|
2267
|
+
base = projects_dir if projects_dir is not None else (Path.home() / ".claude" / "projects")
|
|
2268
|
+
if not base.is_dir():
|
|
2269
|
+
return None
|
|
2270
|
+
candidates: list[tuple[float, Path]] = []
|
|
2271
|
+
for p in base.glob("*/*.jsonl"):
|
|
2272
|
+
try:
|
|
2273
|
+
mtime = p.stat().st_mtime
|
|
2274
|
+
except OSError:
|
|
2275
|
+
continue
|
|
2276
|
+
if mtime >= cutoff:
|
|
2277
|
+
candidates.append((mtime, p))
|
|
2278
|
+
candidates.sort(reverse=True) # newest first — the just-exited session
|
|
2279
|
+
|
|
2280
|
+
for _mtime, p in candidates:
|
|
2281
|
+
findings = parse_review_from_log(p)
|
|
2282
|
+
if findings is None:
|
|
2283
|
+
# #1348: strict parse failed — detect a REVIEW_VERDICT: marker that
|
|
2284
|
+
# the parser rejected (e.g. bolded markers like **REVIEW_VERDICT:**
|
|
2285
|
+
# or a missing END_REVIEW terminator) when the caller wants
|
|
2286
|
+
# diagnostics and attribution gates pass. Only the first (newest)
|
|
2287
|
+
# hit is collected; `not _diagnostic` prevents overwriting it with a
|
|
2288
|
+
# marker from an older transcript in a subsequent iteration.
|
|
2289
|
+
if _diagnostic is not None and not _diagnostic:
|
|
2290
|
+
try:
|
|
2291
|
+
_raw_for_detect = p.read_text(encoding="utf-8", errors="replace")
|
|
2292
|
+
except OSError:
|
|
2293
|
+
_raw_for_detect = ""
|
|
2294
|
+
if (
|
|
2295
|
+
_raw_for_detect
|
|
2296
|
+
and _transcript_names_issue(_raw_for_detect, issue_number)
|
|
2297
|
+
and _transcript_names_assignment(_raw_for_detect, assignment_id)
|
|
2298
|
+
):
|
|
2299
|
+
from coord.review import detect_unparsed_review_marker # noqa: PLC0415
|
|
2300
|
+
_marker = detect_unparsed_review_marker(
|
|
2301
|
+
_raw_for_detect, transcript_path=str(p)
|
|
2302
|
+
)
|
|
2303
|
+
if _marker is not None:
|
|
2304
|
+
_diagnostic.append(_marker)
|
|
2305
|
+
continue
|
|
2306
|
+
# Gate on the WHOLE transcript, not just the parsed review prose (the
|
|
2307
|
+
# body often never names the issue — the reviewer describes the code;
|
|
2308
|
+
# the BRIEFING seeds `issue-<N>`/the branch, so the transcript does).
|
|
2309
|
+
try:
|
|
2310
|
+
raw = p.read_text(encoding="utf-8", errors="replace")
|
|
2311
|
+
except OSError:
|
|
2312
|
+
raw = findings.body
|
|
2313
|
+
# #989: require BOTH the issue-number reference AND this exact
|
|
2314
|
+
# assignment_id — a sibling issue's transcript can satisfy the former
|
|
2315
|
+
# alone (cross-referenced issue numbers in shared epic prose/docs).
|
|
2316
|
+
if _transcript_names_issue(raw, issue_number) and _transcript_names_assignment(
|
|
2317
|
+
raw, assignment_id
|
|
2318
|
+
):
|
|
2319
|
+
return findings
|
|
2320
|
+
return None
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
def _fetch_remote_test_verdict(
|
|
2324
|
+
issue_number: int,
|
|
2325
|
+
cutoff: float,
|
|
2326
|
+
ssh_target: str,
|
|
2327
|
+
*,
|
|
2328
|
+
assignment_id: str,
|
|
2329
|
+
timeout: float = 30.0,
|
|
2330
|
+
):
|
|
2331
|
+
"""Remote twin of the Test transcript-floor (#1351): parse the
|
|
2332
|
+
``TEST_VERDICT:`` block from the Claude transcript on the SESSION'S OWN
|
|
2333
|
+
host.
|
|
2334
|
+
|
|
2335
|
+
Mirrors :func:`_fetch_remote_review_findings` exactly — same ssh listing
|
|
2336
|
+
(newest-first, active at/after *cutoff*, no early break per #619), same
|
|
2337
|
+
cat-back-to-a-tempfile-then-parse shape. Uses
|
|
2338
|
+
:func:`coord.review.parse_test_verdict_from_log` instead of
|
|
2339
|
+
``parse_review_from_log``, and gates on THIS smoke session's own
|
|
2340
|
+
*assignment_id* — embedded verbatim in the smoke briefing's
|
|
2341
|
+
``[Coordinator smoke assignment <id>]`` header (see
|
|
2342
|
+
``_smoke_report_reminder`` in ``coord/commands/dispatch_workers.py``) —
|
|
2343
|
+
plus the issue number, the same #989 double-gate the review floor uses.
|
|
2344
|
+
Returns the first ``TestVerdictFindings`` that parses AND passes both
|
|
2345
|
+
gates, or ``None``.
|
|
2346
|
+
"""
|
|
2347
|
+
from coord.review import parse_test_verdict_from_log # noqa: PLC0415
|
|
2348
|
+
|
|
2349
|
+
list_cmd = (
|
|
2350
|
+
'find "$HOME/.claude/projects" -maxdepth 2 -name "*.jsonl" '
|
|
2351
|
+
r"-printf '%T@\t%p\n' 2>/dev/null | sort -rn | head -200"
|
|
2352
|
+
)
|
|
2353
|
+
try:
|
|
2354
|
+
listing = subprocess.run(
|
|
2355
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, list_cmd],
|
|
2356
|
+
capture_output=True,
|
|
2357
|
+
text=True,
|
|
2358
|
+
timeout=timeout,
|
|
2359
|
+
)
|
|
2360
|
+
except (subprocess.SubprocessError, OSError):
|
|
2361
|
+
return None
|
|
2362
|
+
if listing.returncode != 0:
|
|
2363
|
+
return None
|
|
2364
|
+
|
|
2365
|
+
candidates: list[str] = []
|
|
2366
|
+
for raw in (listing.stdout or "").splitlines():
|
|
2367
|
+
if "\t" not in raw:
|
|
2368
|
+
continue
|
|
2369
|
+
mtime_s, path = raw.split("\t", 1)
|
|
2370
|
+
try:
|
|
2371
|
+
mtime = float(mtime_s)
|
|
2372
|
+
except ValueError:
|
|
2373
|
+
continue
|
|
2374
|
+
path = path.strip()
|
|
2375
|
+
if path and mtime >= cutoff:
|
|
2376
|
+
candidates.append(path)
|
|
2377
|
+
|
|
2378
|
+
for remote_path in candidates:
|
|
2379
|
+
try:
|
|
2380
|
+
cat = subprocess.run(
|
|
2381
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, f"cat {shlex.quote(remote_path)}"],
|
|
2382
|
+
capture_output=True,
|
|
2383
|
+
text=True,
|
|
2384
|
+
timeout=timeout,
|
|
2385
|
+
)
|
|
2386
|
+
except (subprocess.SubprocessError, OSError):
|
|
2387
|
+
continue
|
|
2388
|
+
if cat.returncode != 0 or not cat.stdout:
|
|
2389
|
+
continue
|
|
2390
|
+
tmp_path: Path | None = None
|
|
2391
|
+
try:
|
|
2392
|
+
with tempfile.NamedTemporaryFile(
|
|
2393
|
+
"w", suffix=".jsonl", delete=False, encoding="utf-8"
|
|
2394
|
+
) as tf:
|
|
2395
|
+
tf.write(cat.stdout)
|
|
2396
|
+
tmp_path = Path(tf.name)
|
|
2397
|
+
findings = parse_test_verdict_from_log(tmp_path)
|
|
2398
|
+
finally:
|
|
2399
|
+
if tmp_path is not None:
|
|
2400
|
+
try:
|
|
2401
|
+
tmp_path.unlink()
|
|
2402
|
+
except OSError:
|
|
2403
|
+
pass
|
|
2404
|
+
if (
|
|
2405
|
+
findings is not None
|
|
2406
|
+
and _transcript_names_issue(cat.stdout, issue_number)
|
|
2407
|
+
and _transcript_names_assignment(cat.stdout, assignment_id)
|
|
2408
|
+
):
|
|
2409
|
+
return findings
|
|
2410
|
+
return None
|
|
2411
|
+
|
|
2412
|
+
|
|
2413
|
+
def _test_verdict_from_transcript(
|
|
2414
|
+
issue_number: int,
|
|
2415
|
+
started_at: float | None,
|
|
2416
|
+
*,
|
|
2417
|
+
assignment_id: str,
|
|
2418
|
+
projects_dir: Path | None = None,
|
|
2419
|
+
ssh_target: str | None = None,
|
|
2420
|
+
):
|
|
2421
|
+
"""Recover a human-attended Test session's verdict from the Claude
|
|
2422
|
+
session transcript — the Test-gate twin of the #606 review
|
|
2423
|
+
transcript-floor (#1351).
|
|
2424
|
+
|
|
2425
|
+
A Test (smoke) session's only channel back to the board was, before
|
|
2426
|
+
this, ``coord test --passed|--fail`` running successfully inside the
|
|
2427
|
+
session — no structured block, no transcript floor, no fallback (unlike
|
|
2428
|
+
a review, which has had the #606/#651 ``REVIEW_VERDICT`` treatment all
|
|
2429
|
+
along). This recovers the same class of durable, PATH-independent
|
|
2430
|
+
verdict for the Test gate: it parses the most-recent transcript(s)
|
|
2431
|
+
active during this session for a ``TEST_VERDICT:``/``TEST_REASON:``/
|
|
2432
|
+
``END_TEST`` block via :func:`coord.review.parse_test_verdict_from_log`,
|
|
2433
|
+
gated on this exact smoke session's *assignment_id* AND the issue number
|
|
2434
|
+
— the same #989 double-gate :func:`_review_findings_from_transcript`
|
|
2435
|
+
uses, for the same reason (a sibling issue's transcript can legitimately
|
|
2436
|
+
name this issue number in prose; only the assignment id, a fresh
|
|
2437
|
+
``uuid4().hex[:12]`` embedded in the smoke briefing's own header, can't).
|
|
2438
|
+
|
|
2439
|
+
Returns the parsed :class:`coord.review.TestVerdictFindings` or ``None``.
|
|
2440
|
+
**The caller is responsible for recording the result against the WORK
|
|
2441
|
+
assignment id** (``smoke_of`` / ``review_of_assignment_id``), never this
|
|
2442
|
+
session's own row — this function only locates and parses the
|
|
2443
|
+
transcript, gated on the SESSION's own id, not the work id. See
|
|
2444
|
+
:func:`finalize_interactive_exit`'s ``smoke_of`` parameter, which follows
|
|
2445
|
+
the same rule :func:`coord.commands.review._prompt_and_relay_test_verdict`
|
|
2446
|
+
already does for its own (later, operator-prompt) backstop.
|
|
2447
|
+
"""
|
|
2448
|
+
from coord.review import parse_test_verdict_from_log # noqa: PLC0415
|
|
2449
|
+
|
|
2450
|
+
if started_at is None:
|
|
2451
|
+
return None
|
|
2452
|
+
cutoff = started_at - 5.0
|
|
2453
|
+
if ssh_target is not None:
|
|
2454
|
+
result = _fetch_remote_test_verdict(
|
|
2455
|
+
issue_number, cutoff, ssh_target, assignment_id=assignment_id,
|
|
2456
|
+
)
|
|
2457
|
+
if result is None:
|
|
2458
|
+
# One settle-and-retry, mirroring the review floor's #619 flush
|
|
2459
|
+
# race guard.
|
|
2460
|
+
time.sleep(2.0)
|
|
2461
|
+
result = _fetch_remote_test_verdict(
|
|
2462
|
+
issue_number, cutoff, ssh_target, assignment_id=assignment_id,
|
|
2463
|
+
)
|
|
2464
|
+
return result
|
|
2465
|
+
base = projects_dir if projects_dir is not None else (Path.home() / ".claude" / "projects")
|
|
2466
|
+
if not base.is_dir():
|
|
2467
|
+
return None
|
|
2468
|
+
candidates: list[tuple[float, Path]] = []
|
|
2469
|
+
for p in base.glob("*/*.jsonl"):
|
|
2470
|
+
try:
|
|
2471
|
+
mtime = p.stat().st_mtime
|
|
2472
|
+
except OSError:
|
|
2473
|
+
continue
|
|
2474
|
+
if mtime >= cutoff:
|
|
2475
|
+
candidates.append((mtime, p))
|
|
2476
|
+
candidates.sort(reverse=True) # newest first — the just-exited session
|
|
2477
|
+
|
|
2478
|
+
for _mtime, p in candidates:
|
|
2479
|
+
findings = parse_test_verdict_from_log(p)
|
|
2480
|
+
if findings is None:
|
|
2481
|
+
continue
|
|
2482
|
+
try:
|
|
2483
|
+
raw = p.read_text(encoding="utf-8", errors="replace")
|
|
2484
|
+
except OSError:
|
|
2485
|
+
raw = findings.reason
|
|
2486
|
+
if _transcript_names_issue(raw, issue_number) and _transcript_names_assignment(
|
|
2487
|
+
raw, assignment_id
|
|
2488
|
+
):
|
|
2489
|
+
return findings
|
|
2490
|
+
return None
|
|
2491
|
+
|
|
2492
|
+
|
|
2493
|
+
def _read_test_state(work_assignment_id: str) -> str | None:
|
|
2494
|
+
"""Best-effort read of ``test_state`` for *work_assignment_id* (#1351).
|
|
2495
|
+
|
|
2496
|
+
Point-lookup-first, mirroring the idempotency gate in
|
|
2497
|
+
:func:`coord.commands.review._prompt_and_relay_test_verdict` (#1349): a
|
|
2498
|
+
thin client hits the single-assignment endpoint rather than paying for
|
|
2499
|
+
the full board collection just to read one field off one row. Returns
|
|
2500
|
+
``None`` on any read failure (transport, unknown id, pre-#1336 daemon) or
|
|
2501
|
+
when no verdict is set — callers must treat that as "unknown, proceed"
|
|
2502
|
+
since the write it guards is itself idempotent, not as confirmation that
|
|
2503
|
+
no verdict exists.
|
|
2504
|
+
"""
|
|
2505
|
+
try:
|
|
2506
|
+
from coord.board_service import resolve as _resolve_bs # noqa: PLC0415
|
|
2507
|
+
|
|
2508
|
+
svc = _resolve_bs()
|
|
2509
|
+
if svc is not None:
|
|
2510
|
+
from coord.client import fetch_assignment, fetch_board_payload # noqa: PLC0415
|
|
2511
|
+
|
|
2512
|
+
row = fetch_assignment(svc, work_assignment_id)
|
|
2513
|
+
if row is None:
|
|
2514
|
+
payload = fetch_board_payload(svc)
|
|
2515
|
+
row = next(
|
|
2516
|
+
(
|
|
2517
|
+
a for a in payload.get("assignments", [])
|
|
2518
|
+
if a.get("assignment_id") == work_assignment_id
|
|
2519
|
+
),
|
|
2520
|
+
None,
|
|
2521
|
+
)
|
|
2522
|
+
if row is not None:
|
|
2523
|
+
return (row.get("test_state") or "").strip() or None
|
|
2524
|
+
else:
|
|
2525
|
+
from coord.board_service import read_board as _read_board_ts # noqa: PLC0415
|
|
2526
|
+
|
|
2527
|
+
work = _read_board_ts().find_by_id(work_assignment_id)
|
|
2528
|
+
if work is not None:
|
|
2529
|
+
return (work.test_state or "").strip() or None
|
|
2530
|
+
except Exception: # noqa: BLE001 — best-effort
|
|
2531
|
+
pass
|
|
2532
|
+
return None
|
|
2533
|
+
|
|
2534
|
+
|
|
2535
|
+
def _persist_interactive_tokens(
|
|
2536
|
+
assignment_id: str,
|
|
2537
|
+
started_at: float | None,
|
|
2538
|
+
worktree_path: str | None,
|
|
2539
|
+
) -> None:
|
|
2540
|
+
"""#546: flag as interactive + capture token counts from the session transcript.
|
|
2541
|
+
|
|
2542
|
+
Called before every return in :func:`finalize_interactive_exit`. Best-effort
|
|
2543
|
+
— any exception is swallowed so the caller's normal finalization path is never
|
|
2544
|
+
blocked.
|
|
2545
|
+
"""
|
|
2546
|
+
from coord.state import mark_assignment_interactive, update_assignment_tokens # noqa: PLC0415
|
|
2547
|
+
try:
|
|
2548
|
+
mark_assignment_interactive(assignment_id)
|
|
2549
|
+
except Exception: # noqa: BLE001
|
|
2550
|
+
pass
|
|
2551
|
+
if started_at is None:
|
|
2552
|
+
return
|
|
2553
|
+
try:
|
|
2554
|
+
inp, out, cc, cr = _tokens_from_transcript(
|
|
2555
|
+
started_at, worktree_path=worktree_path
|
|
2556
|
+
)
|
|
2557
|
+
if inp + out + cc + cr > 0:
|
|
2558
|
+
update_assignment_tokens(
|
|
2559
|
+
assignment_id,
|
|
2560
|
+
input_tokens=inp,
|
|
2561
|
+
output_tokens=out,
|
|
2562
|
+
cache_creation_tokens=cc,
|
|
2563
|
+
cache_read_tokens=cr,
|
|
2564
|
+
)
|
|
2565
|
+
except Exception: # noqa: BLE001
|
|
2566
|
+
pass
|
|
2567
|
+
|
|
2568
|
+
|
|
2569
|
+
def finalize_interactive_exit(
|
|
2570
|
+
*,
|
|
2571
|
+
assignment_id: str,
|
|
2572
|
+
repo_name: str,
|
|
2573
|
+
repo_github: str,
|
|
2574
|
+
issue_number: int,
|
|
2575
|
+
machine_name: str,
|
|
2576
|
+
worktree_path: str | None = None,
|
|
2577
|
+
base_branch: str,
|
|
2578
|
+
exit_code: int,
|
|
2579
|
+
started_at: float | None = None,
|
|
2580
|
+
log_path: str | None = None,
|
|
2581
|
+
repo_path: str | None = None,
|
|
2582
|
+
artifact_paths: list[str] | None = None,
|
|
2583
|
+
verify_merge: bool = False,
|
|
2584
|
+
ssh_target: str | None = None,
|
|
2585
|
+
branch: str | None = None,
|
|
2586
|
+
smoke_repo_path: str | None = None,
|
|
2587
|
+
smoke_of: str | None = None,
|
|
2588
|
+
) -> InteractiveFinalizeResult:
|
|
2589
|
+
"""Git-floor backstop for the interactive launcher exit path (#466).
|
|
2590
|
+
|
|
2591
|
+
Called AFTER :func:`launch_human_attended_interactive` returns.
|
|
2592
|
+
Performs three steps, then optionally removes the interactive worktree:
|
|
2593
|
+
|
|
2594
|
+
1. Push any local commits with ``git push -u origin HEAD`` (the same
|
|
2595
|
+
discipline the agent-side reap uses). Push errors are surfaced
|
|
2596
|
+
but never abort the backstop — the local DB write is still the
|
|
2597
|
+
authoritative record.
|
|
2598
|
+
2. Count commits ahead of *base_branch* using the #448 primitive
|
|
2599
|
+
(mirrored above to avoid importing the full AgentServer graph).
|
|
2600
|
+
3. Route the resulting ``(exit_code, commits_ahead)`` tuple through
|
|
2601
|
+
:func:`coord.issue_store.post_completion` — the seam decides
|
|
2602
|
+
whether this is ``done`` / ``advisory`` / ``failed``.
|
|
2603
|
+
|
|
2604
|
+
If ``coord report-result`` already wrote a terminal state for this
|
|
2605
|
+
assignment, this function returns without re-writing anything. The
|
|
2606
|
+
agent-reported result wins because the backstop's heuristic can't
|
|
2607
|
+
distinguish "0 commits because review session" from "0 commits
|
|
2608
|
+
because agent did nothing".
|
|
2609
|
+
|
|
2610
|
+
When *repo_path* is supplied (the interactive launcher always provides
|
|
2611
|
+
it after the worktree-per-session fix), the function removes the
|
|
2612
|
+
worktree after recording the terminal state — matching the cleanup
|
|
2613
|
+
discipline of :meth:`coord.agent.AgentServer._cleanup_worktree`.
|
|
2614
|
+
|
|
2615
|
+
When *smoke_repo_path* is supplied (the ``--smoke-of`` dispatch/reattach
|
|
2616
|
+
paths only), runs the live-checkout restore-on-exit safety net (#1256)
|
|
2617
|
+
FIRST, unconditionally, before any of the branches below — so it lands
|
|
2618
|
+
regardless of which one ultimately returns. See
|
|
2619
|
+
:func:`restore_live_checkout_from_smoke_snapshot`.
|
|
2620
|
+
|
|
2621
|
+
*smoke_of* (#1351): the WORK assignment id a Test (smoke) session was
|
|
2622
|
+
dispatched to validate (``review_of_assignment_id`` on the smoke row).
|
|
2623
|
+
When given, the #1351 Test transcript-floor runs FIRST, unconditionally,
|
|
2624
|
+
right after the restore-on-exit step above and BEFORE the
|
|
2625
|
+
``_assignment_already_recorded(assignment_id)`` gate below: it looks for
|
|
2626
|
+
a ``TEST_VERDICT:``/``TEST_REASON:``/``END_TEST`` block in THIS session's
|
|
2627
|
+
own transcript and, if found and the work row has no verdict yet,
|
|
2628
|
+
records it on *smoke_of* — never on this session's own ``assignment_id``.
|
|
2629
|
+
It must run before that gate because the gate reads the status of THIS
|
|
2630
|
+
session's own row (the smoke row), not the work row — and the smoke
|
|
2631
|
+
briefing unconditionally requires the agent to close the smoke row with
|
|
2632
|
+
``coord report-result`` as a terminal step regardless of whether it also
|
|
2633
|
+
ran ``coord test``. If the floor ran after that gate, the everyday case
|
|
2634
|
+
of "agent forgot to run `coord test` but still closed the smoke row"
|
|
2635
|
+
would return through that early return before the floor ever executed,
|
|
2636
|
+
silently disabling the very mechanism this issue added. This is the
|
|
2637
|
+
Test-gate twin of the #606 review transcript-floor below: a
|
|
2638
|
+
human-attended Test session's only channel back to the board was,
|
|
2639
|
+
before this, a successful ``coord test --passed|--fail`` run INSIDE the
|
|
2640
|
+
session, with no structured block, no transcript floor, and no fallback
|
|
2641
|
+
if that command never ran (#1351). ``None`` (every non-smoke caller)
|
|
2642
|
+
makes this an unconditional no-op.
|
|
2643
|
+
"""
|
|
2644
|
+
_effective_patterns = list(artifact_paths or [])
|
|
2645
|
+
|
|
2646
|
+
_smoke_restored: list[str] = []
|
|
2647
|
+
_smoke_restore_error: str | None = None
|
|
2648
|
+
if smoke_repo_path:
|
|
2649
|
+
_smoke_restored, _smoke_restore_error = (
|
|
2650
|
+
restore_live_checkout_from_smoke_snapshot(
|
|
2651
|
+
smoke_repo_path, assignment_id, ssh_target=ssh_target
|
|
2652
|
+
)
|
|
2653
|
+
)
|
|
2654
|
+
|
|
2655
|
+
# ── Test transcript-floor (#1351): durable Test-gate verdict capture ────
|
|
2656
|
+
# Mirrors the review transcript-floor further below, but for the Test
|
|
2657
|
+
# gate: a human-attended smoke session's ONLY channel back to the board
|
|
2658
|
+
# was, before this, `coord test --passed|--fail` running successfully
|
|
2659
|
+
# INSIDE the session — no structured block, no transcript floor, no
|
|
2660
|
+
# fallback. Recover a TEST_VERDICT:/TEST_REASON:/END_TEST block from
|
|
2661
|
+
# THIS session's own transcript and record it on the WORK row
|
|
2662
|
+
# (`smoke_of`), never on this session's own `assignment_id` — the same
|
|
2663
|
+
# rule `_prompt_and_relay_test_verdict` follows for its own (later)
|
|
2664
|
+
# backstop.
|
|
2665
|
+
#
|
|
2666
|
+
# Runs HERE — unconditionally, before the `_assignment_already_recorded`
|
|
2667
|
+
# gate below — and NOT after it, deliberately. That gate reads the
|
|
2668
|
+
# status of THIS session's own row (the smoke row), not the work row it
|
|
2669
|
+
# targets. The smoke briefing unconditionally requires the agent to close
|
|
2670
|
+
# the smoke row with `coord report-result` as a terminal step regardless
|
|
2671
|
+
# of whether `coord test` also ran, so the everyday failure mode is: the
|
|
2672
|
+
# agent forgets `coord test` but still closes the smoke row cleanly. If
|
|
2673
|
+
# this floor ran after that gate, that exact case would hit the early
|
|
2674
|
+
# return before the floor ever executed — silently disabling the
|
|
2675
|
+
# mechanism this issue exists to add. Self-gating on `smoke_of`: every
|
|
2676
|
+
# non-smoke caller passes `smoke_of=None` and this is a no-op, same as
|
|
2677
|
+
# the review floor is a no-op for a work session whose transcript
|
|
2678
|
+
# carries no REVIEW_VERDICT block.
|
|
2679
|
+
_test_verdict_recorded: str | None = None
|
|
2680
|
+
if smoke_of:
|
|
2681
|
+
try:
|
|
2682
|
+
_work_test_state = _read_test_state(smoke_of)
|
|
2683
|
+
if not _work_test_state:
|
|
2684
|
+
_tv = _test_verdict_from_transcript(
|
|
2685
|
+
issue_number, started_at, assignment_id=assignment_id,
|
|
2686
|
+
ssh_target=ssh_target,
|
|
2687
|
+
)
|
|
2688
|
+
if _tv is not None:
|
|
2689
|
+
from coord.state import record_test_verdict as _record_tv_floor # noqa: PLC0415
|
|
2690
|
+
|
|
2691
|
+
_record_tv_floor(
|
|
2692
|
+
assignment_id=smoke_of,
|
|
2693
|
+
test_state=_tv.verdict,
|
|
2694
|
+
test_reason=_tv.reason or None,
|
|
2695
|
+
)
|
|
2696
|
+
_test_verdict_recorded = _tv.verdict
|
|
2697
|
+
except Exception: # noqa: BLE001 — best-effort; the operator prompt
|
|
2698
|
+
# backstop is still the ultimate fallback if this fails.
|
|
2699
|
+
pass
|
|
2700
|
+
|
|
2701
|
+
# ── Merge-prep verification gate (#604) ──────────────────────────────────
|
|
2702
|
+
# For the interactive MERGE agent (--merge-of), GIT TRUTH OVERRIDES the
|
|
2703
|
+
# agent's self-report. A rebase that left the branch behind the target
|
|
2704
|
+
# branch, or force-pushed a polluted history dragging in unrelated
|
|
2705
|
+
# already-merged commits, must NEVER be recorded as `done` — even if the
|
|
2706
|
+
# agent ran `coord report-result --status done` (vimcode #494, 2026-06-15).
|
|
2707
|
+
#
|
|
2708
|
+
# This deliberately INVERTS the precedence used everywhere else in this
|
|
2709
|
+
# function, where `_assignment_already_recorded` wins and a 0-commit review
|
|
2710
|
+
# session is legitimately fine. For merge-prep the opposite is true: a
|
|
2711
|
+
# clean-exit self-report is exactly what a botched rebase looks like, so we
|
|
2712
|
+
# re-derive the truth from git and let it override. Do NOT "simplify" this
|
|
2713
|
+
# back to deferring to the report.
|
|
2714
|
+
merge_verify = None
|
|
2715
|
+
if verify_merge and worktree_path:
|
|
2716
|
+
wt_v = Path(worktree_path)
|
|
2717
|
+
if wt_v.exists():
|
|
2718
|
+
from coord.agent import ( # noqa: PLC0415
|
|
2719
|
+
resolve_closed_issue_numbers,
|
|
2720
|
+
verify_merge_branch,
|
|
2721
|
+
)
|
|
2722
|
+
|
|
2723
|
+
merge_verify = verify_merge_branch(
|
|
2724
|
+
wt_v, base=base_branch, issue_number=issue_number
|
|
2725
|
+
)
|
|
2726
|
+
# #1279: corroborate blocking foreign commits against GitHub's
|
|
2727
|
+
# closed-issue state before deciding — a typo'd reference to an
|
|
2728
|
+
# already-closed issue is downgraded to advisory, not blocking.
|
|
2729
|
+
closed = resolve_closed_issue_numbers(
|
|
2730
|
+
repo_github, merge_verify.foreign, issue_number
|
|
2731
|
+
)
|
|
2732
|
+
if closed:
|
|
2733
|
+
merge_verify = verify_merge_branch(
|
|
2734
|
+
wt_v,
|
|
2735
|
+
base=base_branch,
|
|
2736
|
+
issue_number=issue_number,
|
|
2737
|
+
closed_issue_numbers=closed,
|
|
2738
|
+
)
|
|
2739
|
+
if not merge_verify.ok:
|
|
2740
|
+
return _finalize_merge_blocked(
|
|
2741
|
+
merge_verify=merge_verify,
|
|
2742
|
+
assignment_id=assignment_id,
|
|
2743
|
+
machine_name=machine_name,
|
|
2744
|
+
repo_name=repo_name,
|
|
2745
|
+
repo_github=repo_github,
|
|
2746
|
+
issue_number=issue_number,
|
|
2747
|
+
base_branch=base_branch,
|
|
2748
|
+
worktree_path=worktree_path,
|
|
2749
|
+
repo_path=repo_path,
|
|
2750
|
+
started_at=started_at,
|
|
2751
|
+
log_path=log_path,
|
|
2752
|
+
branch=branch,
|
|
2753
|
+
smoke_restored_paths=_smoke_restored,
|
|
2754
|
+
smoke_restore_error=_smoke_restore_error,
|
|
2755
|
+
test_verdict_recovered=_test_verdict_recorded,
|
|
2756
|
+
)
|
|
2757
|
+
|
|
2758
|
+
# Respect an explicit `coord report-result` from the agent. Without
|
|
2759
|
+
# this check, every review session (which legitimately has 0 commits)
|
|
2760
|
+
# would have its agent-reported verdict overwritten with an advisory
|
|
2761
|
+
# the instant the human closed the TTY.
|
|
2762
|
+
if _assignment_already_recorded(assignment_id):
|
|
2763
|
+
worktree_removed = False
|
|
2764
|
+
_artifacts_stashed: int | None = None
|
|
2765
|
+
wt_p = Path(worktree_path) if worktree_path else None
|
|
2766
|
+
if wt_p is not None and wt_p.exists():
|
|
2767
|
+
# #562: stash before removing — `coord report-result` never stashes.
|
|
2768
|
+
if _effective_patterns:
|
|
2769
|
+
from coord.agent import ( # noqa: PLC0415
|
|
2770
|
+
stash_artifacts_for_branch as _stash_fn,
|
|
2771
|
+
)
|
|
2772
|
+
from coord.state import COORD_DIR as _CD # noqa: PLC0415
|
|
2773
|
+
_artifacts_stashed = _stash_fn(
|
|
2774
|
+
worktree_path=wt_p,
|
|
2775
|
+
branch=_current_branch(wt_p) or "",
|
|
2776
|
+
repo_name=repo_name,
|
|
2777
|
+
patterns=_effective_patterns,
|
|
2778
|
+
state_dir=_CD,
|
|
2779
|
+
assignment_id=assignment_id,
|
|
2780
|
+
log_path=log_path,
|
|
2781
|
+
)
|
|
2782
|
+
if repo_path is not None and wt_p is not None:
|
|
2783
|
+
worktree_removed = _remove_worktree(Path(repo_path), wt_p)
|
|
2784
|
+
# #546: mark as interactive + capture tokens from the session transcript.
|
|
2785
|
+
_persist_interactive_tokens(assignment_id, started_at, worktree_path)
|
|
2786
|
+
return InteractiveFinalizeResult(
|
|
2787
|
+
terminal_status="report-result", # informational only
|
|
2788
|
+
commits_ahead=None,
|
|
2789
|
+
push_ok=True,
|
|
2790
|
+
push_error=None,
|
|
2791
|
+
already_recorded=True,
|
|
2792
|
+
seam_outcome=None,
|
|
2793
|
+
worktree_removed=worktree_removed,
|
|
2794
|
+
merge_verify=merge_verify,
|
|
2795
|
+
smoke_restored_paths=_smoke_restored,
|
|
2796
|
+
smoke_restore_error=_smoke_restore_error,
|
|
2797
|
+
artifacts_stashed=_artifacts_stashed,
|
|
2798
|
+
test_verdict_recovered=_test_verdict_recorded,
|
|
2799
|
+
)
|
|
2800
|
+
|
|
2801
|
+
# ── Transcript-floor (#606): durable review capture ──────────────────────
|
|
2802
|
+
# A human-attended REVIEW emits its verdict (REVIEW_VERDICT:/REVIEW_BODY:)
|
|
2803
|
+
# but frequently can't run `coord report-result` (coord not on the session
|
|
2804
|
+
# PATH), and a review has no git-floor (0 commits) — so without this the
|
|
2805
|
+
# findings vanish when the operator exits Claude. Recover them from the
|
|
2806
|
+
# Claude session transcript and record them through the same issue_store
|
|
2807
|
+
# seam. Runs BEFORE the git-floor (which would otherwise stamp a verdict-
|
|
2808
|
+
# less advisory). Self-gating: a work session's transcript has no review
|
|
2809
|
+
# block, so this is a no-op there and the git-floor below still handles it.
|
|
2810
|
+
_tf = _review_findings_from_transcript(
|
|
2811
|
+
issue_number, started_at, assignment_id=assignment_id, ssh_target=ssh_target
|
|
2812
|
+
)
|
|
2813
|
+
if _tf is not None:
|
|
2814
|
+
try:
|
|
2815
|
+
from coord import issue_store # noqa: PLC0415
|
|
2816
|
+
|
|
2817
|
+
issue_store.post_result(
|
|
2818
|
+
issue_store.ResultRecord(
|
|
2819
|
+
assignment_id=assignment_id,
|
|
2820
|
+
machine_name=machine_name,
|
|
2821
|
+
repo_name=repo_name,
|
|
2822
|
+
repo_github=repo_github,
|
|
2823
|
+
issue_number=issue_number,
|
|
2824
|
+
status="done",
|
|
2825
|
+
verdict=_tf.verdict, # type: ignore[arg-type] # approve|request-changes
|
|
2826
|
+
summary="Review verdict recovered from the session transcript "
|
|
2827
|
+
"(agent could not run `coord report-result`).",
|
|
2828
|
+
findings_body=_tf.body,
|
|
2829
|
+
branch=branch,
|
|
2830
|
+
)
|
|
2831
|
+
)
|
|
2832
|
+
except Exception: # noqa: BLE001 — best-effort
|
|
2833
|
+
# Recording failed (e.g. board unreachable) — fall through to the
|
|
2834
|
+
# git-floor / human-prompt backstop rather than swallowing the review.
|
|
2835
|
+
_tf = None
|
|
2836
|
+
else:
|
|
2837
|
+
_wt = Path(worktree_path) if worktree_path else None
|
|
2838
|
+
_removed = (
|
|
2839
|
+
_remove_worktree(Path(repo_path), _wt)
|
|
2840
|
+
if repo_path is not None and _wt is not None and _wt.exists()
|
|
2841
|
+
else False
|
|
2842
|
+
)
|
|
2843
|
+
# #546: mark as interactive + capture tokens from the session transcript.
|
|
2844
|
+
_persist_interactive_tokens(assignment_id, started_at, worktree_path)
|
|
2845
|
+
return InteractiveFinalizeResult(
|
|
2846
|
+
terminal_status="transcript-floor",
|
|
2847
|
+
commits_ahead=None,
|
|
2848
|
+
push_ok=True,
|
|
2849
|
+
push_error=None,
|
|
2850
|
+
already_recorded=True,
|
|
2851
|
+
seam_outcome=None,
|
|
2852
|
+
worktree_removed=_removed,
|
|
2853
|
+
merge_verify=merge_verify,
|
|
2854
|
+
smoke_restored_paths=_smoke_restored,
|
|
2855
|
+
smoke_restore_error=_smoke_restore_error,
|
|
2856
|
+
test_verdict_recovered=_test_verdict_recorded,
|
|
2857
|
+
)
|
|
2858
|
+
|
|
2859
|
+
# worktree_path is None for a human-attended REVIEW (migration A1): the
|
|
2860
|
+
# review runs read-only in the LIVE checkout, so there is no session
|
|
2861
|
+
# worktree to push from, count commits in, or remove. Guard every git
|
|
2862
|
+
# step on a real worktree path so the review backstop still records a
|
|
2863
|
+
# terminal state (commits_ahead=None) without ever touching the checkout.
|
|
2864
|
+
# #1155: .expanduser() defensively — every known caller already builds
|
|
2865
|
+
# worktree_path from an expanded COORD_DIR, but a `~`-relative string
|
|
2866
|
+
# reaching here would make .exists() silently False even though the
|
|
2867
|
+
# worktree is physically present, skipping push/commit-count/branch
|
|
2868
|
+
# capture below and leaking the worktree.
|
|
2869
|
+
wt_path = Path(worktree_path).expanduser() if worktree_path else None
|
|
2870
|
+
|
|
2871
|
+
# Step 1 — push. Failure is non-fatal but recorded. Skip the push
|
|
2872
|
+
# entirely when there is no worktree (review) or its directory doesn't
|
|
2873
|
+
# exist (extremely rare, but possible if the operator nuked it).
|
|
2874
|
+
push_ok = True
|
|
2875
|
+
push_error: str | None = None
|
|
2876
|
+
if wt_path is not None and wt_path.exists():
|
|
2877
|
+
push_ok, push_error = _git_push(wt_path)
|
|
2878
|
+
|
|
2879
|
+
# Step 2 — count commits ahead of the base. None = git failed; the
|
|
2880
|
+
# seam treats that as "unknown, assume non-zero" so a git outage
|
|
2881
|
+
# never falsely flags an advisory.
|
|
2882
|
+
# Deferred import of the shared primitive from coord.agent (#466) so
|
|
2883
|
+
# this module stays importable in isolation without pulling in the full
|
|
2884
|
+
# AgentServer graph at module load time.
|
|
2885
|
+
from coord.agent import _commits_ahead # noqa: PLC0415
|
|
2886
|
+
commits = None
|
|
2887
|
+
if wt_path is not None and wt_path.exists():
|
|
2888
|
+
commits = _commits_ahead(wt_path, base_branch)
|
|
2889
|
+
|
|
2890
|
+
branch_now = (
|
|
2891
|
+
_current_branch(wt_path)
|
|
2892
|
+
if wt_path is not None and wt_path.exists()
|
|
2893
|
+
else None
|
|
2894
|
+
)
|
|
2895
|
+
|
|
2896
|
+
duration: float | None = None
|
|
2897
|
+
if started_at is not None:
|
|
2898
|
+
duration = max(0.0, time.time() - started_at)
|
|
2899
|
+
|
|
2900
|
+
# Step 3 — write the terminal state through the seam. Deferred import
|
|
2901
|
+
# so the launcher module stays importable in isolation (tests, agent
|
|
2902
|
+
# processes that just want the constants).
|
|
2903
|
+
from coord.issue_store import CompletionRecord, post_completion # noqa: PLC0415
|
|
2904
|
+
|
|
2905
|
+
# #717: For interactive WORK sessions the session exit_code is unreliable
|
|
2906
|
+
# (operator Ctrl-C after the push landed, tmux kill, a deny-listed `gh`
|
|
2907
|
+
# command at the tail end of the session, etc.). When commits_ahead >= 1
|
|
2908
|
+
# the branch IS pushed and the work IS reviewable — normalise to exit_code=0
|
|
2909
|
+
# so _post_completion_local takes the DONE path and the Test → Review →
|
|
2910
|
+
# Merge chain stays alive.
|
|
2911
|
+
# commits_ahead == 0 (no work produced) and None (git unreachable / no
|
|
2912
|
+
# worktree — covers review sessions) keep the original exit_code so:
|
|
2913
|
+
# • exit_code != 0, commits == 0 → still recorded as FAILED
|
|
2914
|
+
# • exit_code == 0, commits == 0 → still recorded as ADVISORY (#448)
|
|
2915
|
+
# • exit_code == 0, commits is None → still recorded as DONE, UNLESS this
|
|
2916
|
+
# is an interactive WORK session, in which case issue_store.py's
|
|
2917
|
+
# _interactive_work_has_pushed_branch does an authoritative `gh`
|
|
2918
|
+
# branch check and demotes to ADVISORY when GitHub confirms no branch
|
|
2919
|
+
# was ever pushed (#1155). Non-interactive / non-work sessions keep
|
|
2920
|
+
# the original unknown-commit heuristic (DONE).
|
|
2921
|
+
effective_exit_code = 0 if (commits is not None and commits >= 1) else exit_code
|
|
2922
|
+
|
|
2923
|
+
record = CompletionRecord(
|
|
2924
|
+
assignment_id=assignment_id,
|
|
2925
|
+
machine_name=machine_name,
|
|
2926
|
+
repo_name=repo_name,
|
|
2927
|
+
repo_github=repo_github,
|
|
2928
|
+
issue_number=issue_number,
|
|
2929
|
+
exit_code=effective_exit_code,
|
|
2930
|
+
commits_ahead=commits,
|
|
2931
|
+
branch=branch_now or branch,
|
|
2932
|
+
duration_seconds=duration,
|
|
2933
|
+
log_path=log_path,
|
|
2934
|
+
summary="",
|
|
2935
|
+
is_interactive=True,
|
|
2936
|
+
)
|
|
2937
|
+
outcome = post_completion(record)
|
|
2938
|
+
|
|
2939
|
+
# Step 3.5 — stash artifacts BEFORE removing the worktree (#562).
|
|
2940
|
+
# This is the missing link that caused "Artifact unavailable" in the TUI
|
|
2941
|
+
# for every interactive work session. Same discipline as the agent-side
|
|
2942
|
+
# stash: best-effort, runs regardless of outcome.status so partially
|
|
2943
|
+
# built artifacts on a failed session are still captured.
|
|
2944
|
+
artifacts_stashed: int | None = None
|
|
2945
|
+
if _effective_patterns and wt_path is not None and wt_path.exists():
|
|
2946
|
+
from coord.agent import stash_artifacts_for_branch as _stash_fn # noqa: PLC0415
|
|
2947
|
+
from coord.state import COORD_DIR as _COORD_DIR # noqa: PLC0415
|
|
2948
|
+
artifacts_stashed = _stash_fn(
|
|
2949
|
+
worktree_path=wt_path,
|
|
2950
|
+
branch=branch_now or "",
|
|
2951
|
+
repo_name=repo_name,
|
|
2952
|
+
patterns=_effective_patterns,
|
|
2953
|
+
state_dir=_COORD_DIR,
|
|
2954
|
+
assignment_id=assignment_id,
|
|
2955
|
+
log_path=log_path,
|
|
2956
|
+
)
|
|
2957
|
+
|
|
2958
|
+
# Step 4 — remove the interactive worktree when repo_path is provided.
|
|
2959
|
+
# Matches _cleanup_worktree discipline: always runs, best-effort.
|
|
2960
|
+
worktree_removed = False
|
|
2961
|
+
if repo_path is not None and wt_path is not None:
|
|
2962
|
+
worktree_removed = _remove_worktree(Path(repo_path), wt_path)
|
|
2963
|
+
# #1155: if wt_path didn't resolve (Steps 1-3 above skipped, so
|
|
2964
|
+
# commits/branch stayed None/empty), _remove_worktree treated the
|
|
2965
|
+
# directory as "already gone" without checking anywhere else. Every
|
|
2966
|
+
# interactive worktree is created at the canonical
|
|
2967
|
+
# <COORD_DIR>/worktrees/<assignment_id> location (see
|
|
2968
|
+
# agent.setup_interactive_worktree), independent of whatever string
|
|
2969
|
+
# the caller passed as worktree_path — so fall back to that fixed
|
|
2970
|
+
# location and remove it too if it's still on disk and wasn't the
|
|
2971
|
+
# path we just handled. Closes the orphaned-worktree leak even when
|
|
2972
|
+
# the root cause of the unresolved wt_path is never diagnosed.
|
|
2973
|
+
from coord.state import COORD_DIR as _COORD_DIR_WT # noqa: PLC0415
|
|
2974
|
+
_canonical_wt = _COORD_DIR_WT / "worktrees" / assignment_id
|
|
2975
|
+
if (
|
|
2976
|
+
assignment_id
|
|
2977
|
+
and _canonical_wt != wt_path
|
|
2978
|
+
and _canonical_wt.exists()
|
|
2979
|
+
):
|
|
2980
|
+
if _remove_worktree(Path(repo_path), _canonical_wt):
|
|
2981
|
+
worktree_removed = True
|
|
2982
|
+
|
|
2983
|
+
# #546: mark as interactive + capture tokens from the session transcript.
|
|
2984
|
+
_persist_interactive_tokens(assignment_id, started_at, worktree_path)
|
|
2985
|
+
|
|
2986
|
+
return InteractiveFinalizeResult(
|
|
2987
|
+
terminal_status=outcome.status,
|
|
2988
|
+
commits_ahead=commits,
|
|
2989
|
+
push_ok=push_ok,
|
|
2990
|
+
push_error=push_error,
|
|
2991
|
+
already_recorded=False,
|
|
2992
|
+
seam_outcome=outcome,
|
|
2993
|
+
worktree_removed=worktree_removed,
|
|
2994
|
+
merge_verify=merge_verify,
|
|
2995
|
+
smoke_restored_paths=_smoke_restored,
|
|
2996
|
+
smoke_restore_error=_smoke_restore_error,
|
|
2997
|
+
artifacts_stashed=artifacts_stashed,
|
|
2998
|
+
test_verdict_recovered=_test_verdict_recorded,
|
|
2999
|
+
)
|
|
3000
|
+
|
|
3001
|
+
|
|
3002
|
+
# ── Remote (#486d) finalize for a remote interactive FIX ──────────────────────
|
|
3003
|
+
|
|
3004
|
+
|
|
3005
|
+
def _remote_push_and_count(
|
|
3006
|
+
ssh_target: str,
|
|
3007
|
+
remote_worktree_sh: str,
|
|
3008
|
+
branch: str,
|
|
3009
|
+
base_branch: str,
|
|
3010
|
+
*,
|
|
3011
|
+
timeout: float = 90.0,
|
|
3012
|
+
) -> tuple[bool, str | None, int | None, str | None]:
|
|
3013
|
+
"""Over ssh: push the remote fix worktree's commits to ``origin/<branch>``
|
|
3014
|
+
(a fast-forward — the worktree started at ``origin/<branch>`` plus the
|
|
3015
|
+
session's commits), then read commits-ahead-of-base and the branch.
|
|
3016
|
+
|
|
3017
|
+
Returns ``(push_ok, push_error, commits_ahead, branch)``. Any push error
|
|
3018
|
+
(incl. a non-fast-forward) sets ``push_ok=False`` with the remote's
|
|
3019
|
+
stderr; the caller then PRESERVES the worktree instead of removing it, so
|
|
3020
|
+
the operator can recover the commits. ``remote_worktree_sh`` is a
|
|
3021
|
+
``$HOME``-form path the remote shell expands — it is NOT quoted (coord
|
|
3022
|
+
generates it: ``$HOME/.coord/worktrees/<hex>``). *branch*/*base_branch*
|
|
3023
|
+
are shell-quoted since they come from issue data.
|
|
3024
|
+
"""
|
|
3025
|
+
refspec = shlex.quote(f"HEAD:{branch}")
|
|
3026
|
+
base_ref = shlex.quote(f"origin/{base_branch}")
|
|
3027
|
+
remote_cmd = (
|
|
3028
|
+
f"cd {remote_worktree_sh} || exit 91; "
|
|
3029
|
+
f"git push origin {refspec}; echo \"__PUSH_RC=$?\"; "
|
|
3030
|
+
f"echo \"__COMMITS=$(git rev-list --count {base_ref}..HEAD 2>/dev/null)\"; "
|
|
3031
|
+
f"echo \"__BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null)\""
|
|
3032
|
+
)
|
|
3033
|
+
try:
|
|
3034
|
+
result = subprocess.run(
|
|
3035
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3036
|
+
capture_output=True,
|
|
3037
|
+
text=True,
|
|
3038
|
+
timeout=timeout,
|
|
3039
|
+
)
|
|
3040
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
3041
|
+
return False, str(exc), None, None
|
|
3042
|
+
|
|
3043
|
+
push_rc: int | None = None
|
|
3044
|
+
commits: int | None = None
|
|
3045
|
+
branch_now: str | None = None
|
|
3046
|
+
for raw in (result.stdout or "").splitlines():
|
|
3047
|
+
line = raw.strip()
|
|
3048
|
+
if line.startswith("__PUSH_RC="):
|
|
3049
|
+
try:
|
|
3050
|
+
push_rc = int(line.split("=", 1)[1])
|
|
3051
|
+
except ValueError:
|
|
3052
|
+
pass
|
|
3053
|
+
elif line.startswith("__COMMITS="):
|
|
3054
|
+
val = line.split("=", 1)[1].strip()
|
|
3055
|
+
if val.isdigit():
|
|
3056
|
+
commits = int(val)
|
|
3057
|
+
elif line.startswith("__BRANCH="):
|
|
3058
|
+
branch_now = line.split("=", 1)[1].strip() or None
|
|
3059
|
+
if branch_now in (None, "", "HEAD"):
|
|
3060
|
+
branch_now = None
|
|
3061
|
+
push_ok = push_rc == 0
|
|
3062
|
+
push_error = (
|
|
3063
|
+
None if push_ok
|
|
3064
|
+
else (result.stderr or result.stdout or "git push failed").strip()
|
|
3065
|
+
)
|
|
3066
|
+
return push_ok, push_error, commits, branch_now
|
|
3067
|
+
|
|
3068
|
+
|
|
3069
|
+
def _remote_worktree_remove(
|
|
3070
|
+
ssh_target: str,
|
|
3071
|
+
remote_repo_sh: str,
|
|
3072
|
+
remote_worktree_sh: str,
|
|
3073
|
+
*,
|
|
3074
|
+
timeout: float = 30.0,
|
|
3075
|
+
) -> bool:
|
|
3076
|
+
"""Best-effort ``git worktree remove --force`` + ``prune`` over ssh.
|
|
3077
|
+
|
|
3078
|
+
The worktree is a worktree OF the remote ``~/src/<repo>`` checkout, so
|
|
3079
|
+
removing it never touches the live checkout (the worker-worktree base).
|
|
3080
|
+
Returns ``True`` when the remote command ran to completion.
|
|
3081
|
+
"""
|
|
3082
|
+
remote_cmd = (
|
|
3083
|
+
f"git -C {remote_repo_sh} worktree remove {remote_worktree_sh} --force"
|
|
3084
|
+
f" 2>/dev/null; git -C {remote_repo_sh} worktree prune 2>/dev/null;"
|
|
3085
|
+
f" echo __WT_DONE"
|
|
3086
|
+
)
|
|
3087
|
+
try:
|
|
3088
|
+
result = subprocess.run(
|
|
3089
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3090
|
+
capture_output=True,
|
|
3091
|
+
text=True,
|
|
3092
|
+
timeout=timeout,
|
|
3093
|
+
)
|
|
3094
|
+
except (subprocess.SubprocessError, OSError):
|
|
3095
|
+
return False
|
|
3096
|
+
return "__WT_DONE" in (result.stdout or "")
|
|
3097
|
+
|
|
3098
|
+
|
|
3099
|
+
def _remote_orphan_is_safe_to_prune(
|
|
3100
|
+
ssh_target: str,
|
|
3101
|
+
remote_repo_sh: str,
|
|
3102
|
+
holder_abs_path: str,
|
|
3103
|
+
branch: str,
|
|
3104
|
+
*,
|
|
3105
|
+
timeout: float = 15.0,
|
|
3106
|
+
) -> bool:
|
|
3107
|
+
"""Return ``True`` only when a dead holder worktree is safe to auto-prune.
|
|
3108
|
+
|
|
3109
|
+
Safety gate for the auto-prune-and-retry path (#759). Both conditions
|
|
3110
|
+
must hold before the coordinator removes a worktree without operator
|
|
3111
|
+
intervention:
|
|
3112
|
+
|
|
3113
|
+
1. **Clean** — ``git status --porcelain`` inside *holder_abs_path* is
|
|
3114
|
+
empty (no staged or unstaged changes).
|
|
3115
|
+
2. **Fully pushed** — the holder's HEAD SHA matches the tip of
|
|
3116
|
+
``origin/refs/heads/<branch>`` (no local commits that haven't been
|
|
3117
|
+
pushed to the remote).
|
|
3118
|
+
|
|
3119
|
+
*holder_abs_path* must be the **absolute** path as returned by
|
|
3120
|
+
:func:`find_remote_branch_holder` (e.g.
|
|
3121
|
+
``/home/john/.coord/worktrees/<id>``).
|
|
3122
|
+
|
|
3123
|
+
*remote_repo_sh* is the ``$HOME``-form path to the repo checkout on
|
|
3124
|
+
the remote machine (e.g. ``$HOME/src/myrepo``); the remote shell
|
|
3125
|
+
expands ``$HOME`` correctly.
|
|
3126
|
+
|
|
3127
|
+
Returns ``False`` on any SSH / git failure — conservative default,
|
|
3128
|
+
never auto-removes when the check cannot be completed.
|
|
3129
|
+
"""
|
|
3130
|
+
holder_q = shlex.quote(holder_abs_path)
|
|
3131
|
+
# branch is always issue-<N>-<slug> (alphanumeric + hyphens) — safe to
|
|
3132
|
+
# interpolate directly, but quote defensively.
|
|
3133
|
+
branch_q = shlex.quote(branch)
|
|
3134
|
+
# The script runs on the remote shell. $HOME in remote_repo_sh is
|
|
3135
|
+
# expanded by the remote shell (passed unquoted); holder_q is a
|
|
3136
|
+
# single-quoted absolute path (no $HOME expansion needed).
|
|
3137
|
+
remote_cmd = (
|
|
3138
|
+
f"cd {holder_q} 2>/dev/null || {{ echo __NOT_SAFE; exit 0; }}; "
|
|
3139
|
+
f"STATUS=$(git status --porcelain 2>/dev/null); "
|
|
3140
|
+
f"HEAD_SHA=$(git rev-parse HEAD 2>/dev/null); "
|
|
3141
|
+
f"ORIGIN_SHA=$(git -C {remote_repo_sh} ls-remote origin"
|
|
3142
|
+
f" refs/heads/{branch_q} 2>/dev/null | awk '{{print $1}}'); "
|
|
3143
|
+
f"if [ -z \"$STATUS\" ] && [ -n \"$HEAD_SHA\" ] && [ -n \"$ORIGIN_SHA\" ]"
|
|
3144
|
+
f" && [ \"$HEAD_SHA\" = \"$ORIGIN_SHA\" ]; then"
|
|
3145
|
+
f" echo __SAFE; else echo __NOT_SAFE; fi"
|
|
3146
|
+
)
|
|
3147
|
+
try:
|
|
3148
|
+
result = subprocess.run(
|
|
3149
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3150
|
+
capture_output=True,
|
|
3151
|
+
text=True,
|
|
3152
|
+
timeout=timeout,
|
|
3153
|
+
)
|
|
3154
|
+
except (subprocess.SubprocessError, OSError):
|
|
3155
|
+
return False # SSH/timeout failure → conservative: do not auto-prune
|
|
3156
|
+
return "__SAFE" in (result.stdout or "")
|
|
3157
|
+
|
|
3158
|
+
|
|
3159
|
+
def remote_worktree_exists(
|
|
3160
|
+
ssh_target: str,
|
|
3161
|
+
remote_worktree_sh: str,
|
|
3162
|
+
*,
|
|
3163
|
+
timeout: float = 10.0,
|
|
3164
|
+
) -> bool:
|
|
3165
|
+
"""Return ``True`` if *remote_worktree_sh* exists as a directory on *ssh_target*.
|
|
3166
|
+
|
|
3167
|
+
Used by the remote interactive launch paths (#560) to distinguish a setup
|
|
3168
|
+
failure (git worktree add was refused — directory never created) from a
|
|
3169
|
+
worker failure (the directory was created but ``claude`` exited non-zero).
|
|
3170
|
+
|
|
3171
|
+
``remote_worktree_sh`` is a ``$HOME``-form path (e.g.
|
|
3172
|
+
``$HOME/.coord/worktrees/<id>``); the remote shell expands ``$HOME``
|
|
3173
|
+
correctly because the argument is passed as a single unquoted token inside
|
|
3174
|
+
a double-quoted string.
|
|
3175
|
+
|
|
3176
|
+
Returns ``True`` on SSH/timeout errors — safe fallback that prevents false
|
|
3177
|
+
"setup failed" diagnoses when connectivity is the real problem.
|
|
3178
|
+
"""
|
|
3179
|
+
# We intentionally pass remote_worktree_sh un-quoted so $HOME expands.
|
|
3180
|
+
# coord always generates this path from a hex assignment_id, so there is
|
|
3181
|
+
# no injection risk.
|
|
3182
|
+
remote_cmd = f"test -d {remote_worktree_sh} && echo __WT_EXISTS || echo __WT_MISSING"
|
|
3183
|
+
try:
|
|
3184
|
+
result = subprocess.run(
|
|
3185
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3186
|
+
capture_output=True,
|
|
3187
|
+
text=True,
|
|
3188
|
+
timeout=timeout,
|
|
3189
|
+
)
|
|
3190
|
+
except (subprocess.SubprocessError, OSError):
|
|
3191
|
+
return True # assume exists; prevents false "setup failed" on SSH error
|
|
3192
|
+
return "__WT_EXISTS" in (result.stdout or "")
|
|
3193
|
+
|
|
3194
|
+
|
|
3195
|
+
def find_remote_branch_holder(
|
|
3196
|
+
ssh_target: str,
|
|
3197
|
+
remote_repo_sh: str,
|
|
3198
|
+
branch: str,
|
|
3199
|
+
*,
|
|
3200
|
+
timeout: float = 15.0,
|
|
3201
|
+
) -> str | None:
|
|
3202
|
+
"""Return the worktree path that has *branch* checked out on *ssh_target*.
|
|
3203
|
+
|
|
3204
|
+
Runs ``git worktree prune`` then ``git worktree list --porcelain`` in
|
|
3205
|
+
*remote_repo_sh*; parses the output to find an entry whose ``branch``
|
|
3206
|
+
field matches ``refs/heads/<branch>``. Returns the worktree path string
|
|
3207
|
+
(as reported by git — a fully expanded absolute path, not a
|
|
3208
|
+
``$HOME``-style shell path) or ``None`` if the branch is not locked.
|
|
3209
|
+
|
|
3210
|
+
Used by the interactive launch paths (#560) to produce an actionable
|
|
3211
|
+
"branch already checked out at <path>" error instead of the raw git
|
|
3212
|
+
error surfaced via the tmux session exit.
|
|
3213
|
+
|
|
3214
|
+
``remote_repo_sh`` is a ``$HOME``-form path passed un-quoted so the
|
|
3215
|
+
remote shell expands ``$HOME``.
|
|
3216
|
+
"""
|
|
3217
|
+
remote_cmd = (
|
|
3218
|
+
f"cd {remote_repo_sh} 2>/dev/null || exit 0; "
|
|
3219
|
+
f"git worktree prune 2>/dev/null; "
|
|
3220
|
+
f"git worktree list --porcelain 2>/dev/null"
|
|
3221
|
+
)
|
|
3222
|
+
try:
|
|
3223
|
+
result = subprocess.run(
|
|
3224
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3225
|
+
capture_output=True,
|
|
3226
|
+
text=True,
|
|
3227
|
+
timeout=timeout,
|
|
3228
|
+
)
|
|
3229
|
+
except (subprocess.SubprocessError, OSError):
|
|
3230
|
+
return None
|
|
3231
|
+
if result.returncode != 0:
|
|
3232
|
+
return None
|
|
3233
|
+
|
|
3234
|
+
# Parse porcelain output. Each worktree block is separated by a blank
|
|
3235
|
+
# line; within a block the fields appear in order: worktree, HEAD,
|
|
3236
|
+
# branch (or "detached" / "bare").
|
|
3237
|
+
current_path: str | None = None
|
|
3238
|
+
for raw_line in result.stdout.splitlines():
|
|
3239
|
+
line = raw_line.strip()
|
|
3240
|
+
if line.startswith("worktree "):
|
|
3241
|
+
current_path = line[len("worktree "):].strip()
|
|
3242
|
+
elif line.startswith("branch ") and current_path is not None:
|
|
3243
|
+
branch_ref = line[len("branch "):].strip()
|
|
3244
|
+
if branch_ref == f"refs/heads/{branch}":
|
|
3245
|
+
return current_path
|
|
3246
|
+
elif not line:
|
|
3247
|
+
current_path = None # blank line: end of a worktree block
|
|
3248
|
+
return None
|
|
3249
|
+
|
|
3250
|
+
|
|
3251
|
+
def _holder_is_base_checkout(holder_abs_path: str) -> bool:
|
|
3252
|
+
"""Return ``True`` when *holder_abs_path* is the repo base checkout.
|
|
3253
|
+
|
|
3254
|
+
Coord always places worker worktrees under ``~/.coord/worktrees/<id>``.
|
|
3255
|
+
Any holder path that does NOT contain ``.coord/worktrees`` is therefore the
|
|
3256
|
+
base repo checkout (e.g. ``~/src/<repo>``) — the invariant path that must
|
|
3257
|
+
NEVER be pruned or removed (#814 / #561).
|
|
3258
|
+
|
|
3259
|
+
Used by the remote interactive launch paths to distinguish a stale orphan
|
|
3260
|
+
(safe to force-remove) from the base checkout (must be freed with
|
|
3261
|
+
``git checkout <default_branch>`` instead).
|
|
3262
|
+
"""
|
|
3263
|
+
return ".coord/worktrees" not in holder_abs_path
|
|
3264
|
+
|
|
3265
|
+
|
|
3266
|
+
def _remote_base_checkout_free_branch(
|
|
3267
|
+
ssh_target: str,
|
|
3268
|
+
remote_repo_sh: str,
|
|
3269
|
+
default_branch: str,
|
|
3270
|
+
*,
|
|
3271
|
+
timeout: float = 15.0,
|
|
3272
|
+
) -> bool:
|
|
3273
|
+
"""Check out *default_branch* in the remote base checkout to free a branch.
|
|
3274
|
+
|
|
3275
|
+
When ``~/src/<repo>`` is checked out on an issue branch, ``git worktree
|
|
3276
|
+
add`` refuses to create a new worktree for the same branch. The correct
|
|
3277
|
+
remedy is to switch the base back to *default_branch* so the branch ref is
|
|
3278
|
+
no longer occupied by the main checkout (#814 / #561).
|
|
3279
|
+
|
|
3280
|
+
*remote_repo_sh* is a ``$HOME``-form path (e.g. ``$HOME/src/myrepo``); the
|
|
3281
|
+
remote shell expands ``$HOME`` correctly because the path is passed unquoted
|
|
3282
|
+
inside the compound command.
|
|
3283
|
+
|
|
3284
|
+
Returns ``True`` when the remote ``git checkout`` succeeded.
|
|
3285
|
+
Returns ``False`` on SSH / timeout / git failure — the caller then falls
|
|
3286
|
+
back to printing a manual-fix command for the operator.
|
|
3287
|
+
"""
|
|
3288
|
+
default_branch_q = shlex.quote(default_branch)
|
|
3289
|
+
remote_cmd = (
|
|
3290
|
+
f"git -C {remote_repo_sh} checkout {default_branch_q} 2>/dev/null"
|
|
3291
|
+
f" && echo __FREE_DONE || echo __FREE_FAIL"
|
|
3292
|
+
)
|
|
3293
|
+
try:
|
|
3294
|
+
result = subprocess.run(
|
|
3295
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3296
|
+
capture_output=True,
|
|
3297
|
+
text=True,
|
|
3298
|
+
timeout=timeout,
|
|
3299
|
+
)
|
|
3300
|
+
except (subprocess.SubprocessError, OSError):
|
|
3301
|
+
return False
|
|
3302
|
+
return "__FREE_DONE" in (result.stdout or "")
|
|
3303
|
+
|
|
3304
|
+
|
|
3305
|
+
def _remote_stash_artifacts(
|
|
3306
|
+
ssh_target: str,
|
|
3307
|
+
remote_worktree_sh: str,
|
|
3308
|
+
repo_name: str,
|
|
3309
|
+
branch: str,
|
|
3310
|
+
patterns: list[str],
|
|
3311
|
+
assignment_id: str,
|
|
3312
|
+
*,
|
|
3313
|
+
timeout: float = 60.0,
|
|
3314
|
+
) -> int | None:
|
|
3315
|
+
"""Run :func:`coord.agent.stash_artifacts_for_branch` on the remote machine.
|
|
3316
|
+
|
|
3317
|
+
SSHes into *ssh_target* and invokes the standalone stash function via the
|
|
3318
|
+
remote's coord venv Python so the artifact-filter logic is identical to the
|
|
3319
|
+
local worker path. Must be called **before** :func:`_remote_worktree_remove`
|
|
3320
|
+
so the files are still present.
|
|
3321
|
+
|
|
3322
|
+
All dynamic values (worktree path, branch, patterns JSON, assignment_id)
|
|
3323
|
+
are passed as ``sys.argv`` positional arguments so no shell-within-shell
|
|
3324
|
+
quoting is needed — the outer ssh call handles quoting of the argv list.
|
|
3325
|
+
|
|
3326
|
+
Returns the number of files copied (the same int
|
|
3327
|
+
:func:`~coord.agent.stash_artifacts_for_branch` returns), or ``None``
|
|
3328
|
+
when the remote call could not be confirmed at all (ssh transport
|
|
3329
|
+
failure, or the sentinel line never came back). ``0`` is a real,
|
|
3330
|
+
meaningful result — the stash ran but nothing matched
|
|
3331
|
+
``artifact_paths`` — and callers must not conflate it with ``None``.
|
|
3332
|
+
Previously this returned a bare ``bool`` fed only by an unconditional
|
|
3333
|
+
``echo __STASH_DONE``, so a 0-copy stash and a fully successful one
|
|
3334
|
+
were indistinguishable to every caller (#1295).
|
|
3335
|
+
|
|
3336
|
+
Best-effort: SSH/import failures are silently ignored (``None``) by
|
|
3337
|
+
the caller.
|
|
3338
|
+
"""
|
|
3339
|
+
patterns_json = json.dumps(patterns)
|
|
3340
|
+
# The Python snippet reads all inputs from argv — no interpolation of
|
|
3341
|
+
# branch/repo/patterns inside the script string itself. This avoids any
|
|
3342
|
+
# quoting-within-quoting hazards.
|
|
3343
|
+
#
|
|
3344
|
+
# ``remote_worktree_sh`` is a ``$HOME``-form path (e.g.
|
|
3345
|
+
# ``$HOME/.coord/worktrees/<id>``). When passed via shlex.quote() it
|
|
3346
|
+
# becomes a single-quoted shell token, so the remote shell NEVER expands
|
|
3347
|
+
# ``$HOME``. ``Path.expanduser()`` only handles ``~``, not ``$HOME``.
|
|
3348
|
+
# Use ``os.path.expandvars()`` in the snippet to resolve ``$HOME`` on the
|
|
3349
|
+
# remote machine before constructing the Path.
|
|
3350
|
+
#
|
|
3351
|
+
# #1295: the snippet now prints its own result (``__STASH_COPIED:<n>``)
|
|
3352
|
+
# instead of relying purely on the outer shell's unconditional
|
|
3353
|
+
# ``&& echo __STASH_DONE`` — that old sentinel fired even when 0 files
|
|
3354
|
+
# were copied, which is exactly the silent-failure the issue reported.
|
|
3355
|
+
py_snippet = (
|
|
3356
|
+
"import sys,json,os; from pathlib import Path; "
|
|
3357
|
+
"from coord.agent import stash_artifacts_for_branch; "
|
|
3358
|
+
"n = stash_artifacts_for_branch("
|
|
3359
|
+
"worktree_path=Path(os.path.expandvars(sys.argv[1])),"
|
|
3360
|
+
"branch=sys.argv[2],"
|
|
3361
|
+
"repo_name=sys.argv[3],"
|
|
3362
|
+
"patterns=json.loads(sys.argv[4]),"
|
|
3363
|
+
"state_dir=Path.home()/'.coord',"
|
|
3364
|
+
"assignment_id=sys.argv[5]"
|
|
3365
|
+
"); print(f'__STASH_COPIED:{n}')"
|
|
3366
|
+
)
|
|
3367
|
+
argv_tail = [
|
|
3368
|
+
remote_worktree_sh,
|
|
3369
|
+
branch,
|
|
3370
|
+
repo_name,
|
|
3371
|
+
patterns_json,
|
|
3372
|
+
assignment_id,
|
|
3373
|
+
]
|
|
3374
|
+
snippet_q = shlex.quote(py_snippet)
|
|
3375
|
+
args_q = " ".join(shlex.quote(a) for a in argv_tail)
|
|
3376
|
+
# #1295: the previous version ran
|
|
3377
|
+
# ( ~/.coord-venv/bin/python3 -c SNIPPET ARGS 2>/dev/null
|
|
3378
|
+
# || python3 -c SNIPPET ARGS 2>/dev/null ) && echo __STASH_DONE
|
|
3379
|
+
# — both branches of the `||` unconditionally redirected stderr to
|
|
3380
|
+
# /dev/null, which is where Python's default logging handler sends
|
|
3381
|
+
# `stash_artifacts_for_branch`'s new `_log.warning("stash: 0 files
|
|
3382
|
+
# matched ...")` (no handler is configured in this bare `python3 -c`
|
|
3383
|
+
# process, so it falls back to `logging.lastResort`, a stderr
|
|
3384
|
+
# StreamHandler). The warning was therefore invisible on exactly this
|
|
3385
|
+
# path. Probing for the venv python with `command -v` first — instead
|
|
3386
|
+
# of trying it and swallowing whatever it prints on failure — means
|
|
3387
|
+
# only ONE interpreter ever actually runs the snippet, so its stderr
|
|
3388
|
+
# can be captured intact and surfaced below rather than discarded.
|
|
3389
|
+
remote_cmd = (
|
|
3390
|
+
"PYBIN=$(command -v ~/.coord-venv/bin/python3 2>/dev/null || command -v python3); "
|
|
3391
|
+
f"\"$PYBIN\" -c {snippet_q} {args_q}"
|
|
3392
|
+
)
|
|
3393
|
+
try:
|
|
3394
|
+
result = subprocess.run(
|
|
3395
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3396
|
+
capture_output=True,
|
|
3397
|
+
text=True,
|
|
3398
|
+
timeout=timeout,
|
|
3399
|
+
)
|
|
3400
|
+
except (subprocess.SubprocessError, OSError):
|
|
3401
|
+
logging.warning(
|
|
3402
|
+
"remote stash: ssh to %s failed for repo=%s branch=%s aid=%s "
|
|
3403
|
+
"— could not confirm artifact stash",
|
|
3404
|
+
ssh_target, repo_name, branch, assignment_id,
|
|
3405
|
+
)
|
|
3406
|
+
return None
|
|
3407
|
+
|
|
3408
|
+
copied: int | None = None
|
|
3409
|
+
for line in (result.stdout or "").splitlines():
|
|
3410
|
+
if line.startswith("__STASH_COPIED:"):
|
|
3411
|
+
try:
|
|
3412
|
+
copied = int(line.split(":", 1)[1])
|
|
3413
|
+
except ValueError:
|
|
3414
|
+
copied = None
|
|
3415
|
+
break
|
|
3416
|
+
|
|
3417
|
+
# #1295: a 0-copy or unconfirmed remote stash used to vanish silently
|
|
3418
|
+
# (stderr → /dev/null, and the caller only ever saw a bare `True`/`False`
|
|
3419
|
+
# it didn't act on). Surface it through the coordinator's own log —
|
|
3420
|
+
# this runs on the coordinator/dispatcher host, so it's a different
|
|
3421
|
+
# log than the agent-side `_log.warning` inside
|
|
3422
|
+
# `stash_artifacts_for_branch`, but it's the log a human debugging a
|
|
3423
|
+
# *remote* interactive finalize will actually be looking at.
|
|
3424
|
+
if copied is None or copied == 0:
|
|
3425
|
+
logging.warning(
|
|
3426
|
+
"remote stash: %s (repo=%s branch=%s aid=%s target=%s) "
|
|
3427
|
+
"remote stderr=%r",
|
|
3428
|
+
"could not confirm result" if copied is None else "0 files matched",
|
|
3429
|
+
repo_name, branch, assignment_id, ssh_target,
|
|
3430
|
+
(result.stderr or "").strip()[:2000],
|
|
3431
|
+
)
|
|
3432
|
+
return copied
|
|
3433
|
+
|
|
3434
|
+
|
|
3435
|
+
def _remote_verify_merge_branch(
|
|
3436
|
+
ssh_target: str,
|
|
3437
|
+
remote_worktree_sh: str,
|
|
3438
|
+
*,
|
|
3439
|
+
base: str,
|
|
3440
|
+
issue_number: int,
|
|
3441
|
+
timeout: float = 30.0,
|
|
3442
|
+
closed_issue_numbers: frozenset[int] = frozenset(),
|
|
3443
|
+
) -> MergeVerify:
|
|
3444
|
+
"""Remote (over ssh) analogue of :func:`coord.agent.verify_merge_branch`
|
|
3445
|
+
for a ``--merge-of`` session whose worktree lives on *ssh_target* rather
|
|
3446
|
+
than the local filesystem (#1007).
|
|
3447
|
+
|
|
3448
|
+
Runs the identical ``git rev-list`` / ``git log`` plumbing the local
|
|
3449
|
+
function uses — resolve ``origin/<base>`` then fall back to ``<base>``
|
|
3450
|
+
(mirrors :func:`coord.agent._resolve_base_ref`'s order), count commits
|
|
3451
|
+
HEAD is missing from that ref, and list the commits HEAD adds over it —
|
|
3452
|
+
but as a single ssh call into *remote_worktree_sh* (a ``$HOME``-form path,
|
|
3453
|
+
passed unquoted so the remote shell expands ``$HOME``). *base* is
|
|
3454
|
+
shell-quoted since it comes from repo config data.
|
|
3455
|
+
|
|
3456
|
+
Returns a :class:`~coord.agent.MergeVerify` with ``default_ahead=None``
|
|
3457
|
+
(not ``ok``) when the base ref can't be resolved on the remote side, the
|
|
3458
|
+
ssh call fails/times out, or the remote worktree directory is gone —
|
|
3459
|
+
same conservative "unverifiable ⇒ not ok" fallback as the local function.
|
|
3460
|
+
|
|
3461
|
+
Args:
|
|
3462
|
+
closed_issue_numbers: Mirrors the same parameter in
|
|
3463
|
+
:func:`coord.agent.verify_merge_branch` — issue numbers known to
|
|
3464
|
+
be closed/merged. Foreign-subject commits referencing only
|
|
3465
|
+
closed issues are downgraded to :attr:`~MergeVerify.advisory_foreign`
|
|
3466
|
+
rather than blocking ``foreign`` (#1279).
|
|
3467
|
+
"""
|
|
3468
|
+
from coord.agent import ( # noqa: PLC0415
|
|
3469
|
+
MergeVerify,
|
|
3470
|
+
_foreign_issue_refs,
|
|
3471
|
+
)
|
|
3472
|
+
|
|
3473
|
+
base_q = shlex.quote(base)
|
|
3474
|
+
remote_cmd = (
|
|
3475
|
+
f"cd {remote_worktree_sh} || exit 91; "
|
|
3476
|
+
f"if git rev-parse --verify --quiet origin/{base_q} >/dev/null 2>&1; then "
|
|
3477
|
+
f"REF=origin/{base_q}; "
|
|
3478
|
+
f"elif git rev-parse --verify --quiet {base_q} >/dev/null 2>&1; then "
|
|
3479
|
+
f"REF={base_q}; "
|
|
3480
|
+
f"else echo __REF_MISSING; exit 0; fi; "
|
|
3481
|
+
f'echo "__DEFAULT_AHEAD=$(git rev-list --count HEAD..$REF 2>/dev/null)"; '
|
|
3482
|
+
f'git log --format="__ADDED=%H%x09%s" "$REF"..HEAD 2>/dev/null'
|
|
3483
|
+
)
|
|
3484
|
+
try:
|
|
3485
|
+
result = subprocess.run(
|
|
3486
|
+
["ssh", *_SSH_MUX_OPTS, ssh_target, remote_cmd],
|
|
3487
|
+
capture_output=True,
|
|
3488
|
+
text=True,
|
|
3489
|
+
timeout=timeout,
|
|
3490
|
+
)
|
|
3491
|
+
except (subprocess.SubprocessError, OSError):
|
|
3492
|
+
return MergeVerify(default_ahead=None, added=[], foreign=[])
|
|
3493
|
+
|
|
3494
|
+
default_ahead: int | None = None
|
|
3495
|
+
added: list[tuple[str, str]] = []
|
|
3496
|
+
ref_missing = False
|
|
3497
|
+
for raw in (result.stdout or "").splitlines():
|
|
3498
|
+
line = raw.rstrip("\n")
|
|
3499
|
+
if line.strip() == "__REF_MISSING":
|
|
3500
|
+
ref_missing = True
|
|
3501
|
+
elif line.startswith("__DEFAULT_AHEAD="):
|
|
3502
|
+
val = line.split("=", 1)[1].strip()
|
|
3503
|
+
if val.isdigit():
|
|
3504
|
+
default_ahead = int(val)
|
|
3505
|
+
elif line.startswith("__ADDED="):
|
|
3506
|
+
payload = line[len("__ADDED="):]
|
|
3507
|
+
sha, _, subject = payload.partition("\t")
|
|
3508
|
+
if sha:
|
|
3509
|
+
added.append((sha, subject))
|
|
3510
|
+
|
|
3511
|
+
if ref_missing or default_ahead is None:
|
|
3512
|
+
return MergeVerify(default_ahead=None, added=[], foreign=[])
|
|
3513
|
+
|
|
3514
|
+
foreign: list[tuple[str, str]] = []
|
|
3515
|
+
advisory_foreign: list[tuple[str, str]] = []
|
|
3516
|
+
for sha, subj in added:
|
|
3517
|
+
foreign_refs = _foreign_issue_refs(subj, issue_number)
|
|
3518
|
+
if not foreign_refs:
|
|
3519
|
+
continue
|
|
3520
|
+
if closed_issue_numbers and foreign_refs.issubset(closed_issue_numbers):
|
|
3521
|
+
advisory_foreign.append((sha, subj))
|
|
3522
|
+
else:
|
|
3523
|
+
foreign.append((sha, subj))
|
|
3524
|
+
return MergeVerify(
|
|
3525
|
+
default_ahead=default_ahead,
|
|
3526
|
+
added=added,
|
|
3527
|
+
foreign=foreign,
|
|
3528
|
+
advisory_foreign=advisory_foreign,
|
|
3529
|
+
)
|
|
3530
|
+
|
|
3531
|
+
|
|
3532
|
+
def _finalize_remote_merge_blocked(
|
|
3533
|
+
*,
|
|
3534
|
+
merge_verify, # agent.MergeVerify (not ok)
|
|
3535
|
+
assignment_id: str,
|
|
3536
|
+
machine_name: str,
|
|
3537
|
+
repo_name: str,
|
|
3538
|
+
repo_github: str,
|
|
3539
|
+
issue_number: int,
|
|
3540
|
+
ssh_target: str,
|
|
3541
|
+
remote_repo_sh: str,
|
|
3542
|
+
remote_worktree_sh: str,
|
|
3543
|
+
branch: str | None,
|
|
3544
|
+
base_branch: str,
|
|
3545
|
+
started_at: float | None,
|
|
3546
|
+
) -> InteractiveFinalizeResult:
|
|
3547
|
+
"""Remote analogue of :func:`_finalize_merge_blocked` (#1007): record a
|
|
3548
|
+
botched remote ``--merge-of`` rebase as ``blocked`` → ``failed``, via the
|
|
3549
|
+
same ``post_result`` seam ``coord report-result`` uses, OVERRIDING any
|
|
3550
|
+
``done`` the agent self-reported — mirrors the local function's
|
|
3551
|
+
git-truth-wins ordering (vimcode #494). The remote worktree is removed
|
|
3552
|
+
afterwards, same as the local path (the offending commits already rode in
|
|
3553
|
+
*merge_verify* for forensics before the worktree + reflog disappear).
|
|
3554
|
+
"""
|
|
3555
|
+
from coord.issue_store import ( # noqa: PLC0415
|
|
3556
|
+
STATUS_BLOCKED,
|
|
3557
|
+
ResultRecord,
|
|
3558
|
+
post_result,
|
|
3559
|
+
)
|
|
3560
|
+
|
|
3561
|
+
prior_recorded = _assignment_already_recorded(assignment_id)
|
|
3562
|
+
|
|
3563
|
+
duration = (
|
|
3564
|
+
max(0.0, time.time() - started_at) if started_at is not None else None
|
|
3565
|
+
)
|
|
3566
|
+
|
|
3567
|
+
outcome = post_result(
|
|
3568
|
+
ResultRecord(
|
|
3569
|
+
assignment_id=assignment_id,
|
|
3570
|
+
machine_name=machine_name,
|
|
3571
|
+
repo_name=repo_name,
|
|
3572
|
+
repo_github=repo_github,
|
|
3573
|
+
issue_number=issue_number,
|
|
3574
|
+
status=STATUS_BLOCKED,
|
|
3575
|
+
verdict=None,
|
|
3576
|
+
summary=merge_verify.block_summary(base_branch),
|
|
3577
|
+
duration_seconds=duration,
|
|
3578
|
+
log_path=None,
|
|
3579
|
+
branch=branch,
|
|
3580
|
+
)
|
|
3581
|
+
)
|
|
3582
|
+
|
|
3583
|
+
worktree_removed = _remote_worktree_remove(
|
|
3584
|
+
ssh_target, remote_repo_sh, remote_worktree_sh,
|
|
3585
|
+
)
|
|
3586
|
+
|
|
3587
|
+
return InteractiveFinalizeResult(
|
|
3588
|
+
terminal_status=outcome.status, # "failed" (blocked → failed)
|
|
3589
|
+
commits_ahead=merge_verify.default_ahead,
|
|
3590
|
+
push_ok=True,
|
|
3591
|
+
push_error=None,
|
|
3592
|
+
already_recorded=prior_recorded,
|
|
3593
|
+
seam_outcome=outcome,
|
|
3594
|
+
worktree_removed=worktree_removed,
|
|
3595
|
+
merge_verify=merge_verify,
|
|
3596
|
+
)
|
|
3597
|
+
|
|
3598
|
+
|
|
3599
|
+
def finalize_remote_interactive_exit(
|
|
3600
|
+
*,
|
|
3601
|
+
assignment_id: str,
|
|
3602
|
+
repo_name: str,
|
|
3603
|
+
repo_github: str,
|
|
3604
|
+
issue_number: int,
|
|
3605
|
+
machine_name: str,
|
|
3606
|
+
ssh_target: str,
|
|
3607
|
+
remote_worktree_sh: str,
|
|
3608
|
+
remote_repo_sh: str,
|
|
3609
|
+
branch: str,
|
|
3610
|
+
base_branch: str,
|
|
3611
|
+
exit_code: int,
|
|
3612
|
+
started_at: float | None = None,
|
|
3613
|
+
artifact_paths: list[str] | None = None,
|
|
3614
|
+
verify_merge: bool = False,
|
|
3615
|
+
) -> InteractiveFinalizeResult:
|
|
3616
|
+
"""Remote (#486d) analog of :func:`finalize_interactive_exit` for a remote
|
|
3617
|
+
interactive FIX (and, since #1007, a remote ``--merge-of``).
|
|
3618
|
+
|
|
3619
|
+
The fix session ran in a worktree ON the remote machine, so the local
|
|
3620
|
+
git-floor backstop can't see its commits. This sshs in to:
|
|
3621
|
+
|
|
3622
|
+
1. push the worktree's commits to ``origin/<branch>`` (fast-forward) and
|
|
3623
|
+
read commits-ahead-of-base + the branch;
|
|
3624
|
+
2. record the completion through the issue_store seam LOCALLY (coordinator
|
|
3625
|
+
DB + GitHub comment) so the pipeline sees a normal completion and the
|
|
3626
|
+
re-review fires;
|
|
3627
|
+
2.5 (#562) stash configured artifacts ON THE REMOTE via ssh so the built
|
|
3628
|
+
binaries survive worktree removal and ``coord pull-artifact`` can fetch
|
|
3629
|
+
them. Stash runs before worktree removal regardless of push outcome.
|
|
3630
|
+
3. remove the remote worktree — but ONLY when the push succeeded, so
|
|
3631
|
+
unpushed commits are never silently destroyed.
|
|
3632
|
+
|
|
3633
|
+
Respects an existing ``coord report-result`` (same as the local backstop):
|
|
3634
|
+
if the row already holds a terminal status, the verdict/result wins and
|
|
3635
|
+
this only attempts worktree cleanup (after a best-effort remote stash).
|
|
3636
|
+
|
|
3637
|
+
``verify_merge=True`` (the remote ``--merge-of`` path, #1007) mirrors
|
|
3638
|
+
:func:`finalize_interactive_exit`'s #604 gate: GIT TRUTH on the remote
|
|
3639
|
+
worktree overrides everything else, including an already-recorded
|
|
3640
|
+
self-report, so a botched remote rebase is never silently accepted as
|
|
3641
|
+
``done``. This check runs FIRST, before the ``_assignment_already_recorded``
|
|
3642
|
+
fast path below — do NOT reorder it after that check.
|
|
3643
|
+
"""
|
|
3644
|
+
_effective_patterns = list(artifact_paths or [])
|
|
3645
|
+
|
|
3646
|
+
# ── Merge-prep verification gate (#604 / #1007) ──────────────────────────
|
|
3647
|
+
# Same precedence inversion as the local function: for merge-prep, a
|
|
3648
|
+
# clean-exit self-report is exactly what a botched rebase looks like, so
|
|
3649
|
+
# re-derive the truth from git on the remote worktree and let it override.
|
|
3650
|
+
merge_verify = None
|
|
3651
|
+
if verify_merge and remote_worktree_exists(ssh_target, remote_worktree_sh):
|
|
3652
|
+
from coord.agent import resolve_closed_issue_numbers # noqa: PLC0415
|
|
3653
|
+
|
|
3654
|
+
merge_verify = _remote_verify_merge_branch(
|
|
3655
|
+
ssh_target, remote_worktree_sh,
|
|
3656
|
+
base=base_branch, issue_number=issue_number,
|
|
3657
|
+
)
|
|
3658
|
+
# #1279: same corroboration as the local finalize path — re-verify
|
|
3659
|
+
# with the closed-issue downgrade signal only when the first (cheap,
|
|
3660
|
+
# git-only) pass actually found blocking foreign commits.
|
|
3661
|
+
closed = resolve_closed_issue_numbers(
|
|
3662
|
+
repo_github, merge_verify.foreign, issue_number
|
|
3663
|
+
)
|
|
3664
|
+
if closed:
|
|
3665
|
+
merge_verify = _remote_verify_merge_branch(
|
|
3666
|
+
ssh_target, remote_worktree_sh,
|
|
3667
|
+
base=base_branch, issue_number=issue_number,
|
|
3668
|
+
closed_issue_numbers=closed,
|
|
3669
|
+
)
|
|
3670
|
+
if not merge_verify.ok:
|
|
3671
|
+
return _finalize_remote_merge_blocked(
|
|
3672
|
+
merge_verify=merge_verify,
|
|
3673
|
+
assignment_id=assignment_id,
|
|
3674
|
+
machine_name=machine_name,
|
|
3675
|
+
repo_name=repo_name,
|
|
3676
|
+
repo_github=repo_github,
|
|
3677
|
+
issue_number=issue_number,
|
|
3678
|
+
ssh_target=ssh_target,
|
|
3679
|
+
remote_repo_sh=remote_repo_sh,
|
|
3680
|
+
remote_worktree_sh=remote_worktree_sh,
|
|
3681
|
+
branch=branch,
|
|
3682
|
+
base_branch=base_branch,
|
|
3683
|
+
started_at=started_at,
|
|
3684
|
+
)
|
|
3685
|
+
|
|
3686
|
+
if _assignment_already_recorded(assignment_id):
|
|
3687
|
+
# #562: stash artifacts before removing the worktree, even when the
|
|
3688
|
+
# agent already recorded via `coord report-result` — that path never
|
|
3689
|
+
# stashes on its own.
|
|
3690
|
+
_artifacts_stashed: int | None = None
|
|
3691
|
+
if _effective_patterns:
|
|
3692
|
+
_artifacts_stashed = _remote_stash_artifacts(
|
|
3693
|
+
ssh_target, remote_worktree_sh, repo_name, branch,
|
|
3694
|
+
_effective_patterns, assignment_id,
|
|
3695
|
+
)
|
|
3696
|
+
removed = _remote_worktree_remove(
|
|
3697
|
+
ssh_target, remote_repo_sh, remote_worktree_sh,
|
|
3698
|
+
)
|
|
3699
|
+
return InteractiveFinalizeResult(
|
|
3700
|
+
terminal_status="report-result",
|
|
3701
|
+
commits_ahead=None,
|
|
3702
|
+
push_ok=True,
|
|
3703
|
+
push_error=None,
|
|
3704
|
+
already_recorded=True,
|
|
3705
|
+
seam_outcome=None,
|
|
3706
|
+
worktree_removed=removed,
|
|
3707
|
+
merge_verify=merge_verify,
|
|
3708
|
+
artifacts_stashed=_artifacts_stashed,
|
|
3709
|
+
)
|
|
3710
|
+
|
|
3711
|
+
push_ok, push_error, commits, branch_now = _remote_push_and_count(
|
|
3712
|
+
ssh_target, remote_worktree_sh, branch, base_branch,
|
|
3713
|
+
)
|
|
3714
|
+
|
|
3715
|
+
duration = (
|
|
3716
|
+
max(0.0, time.time() - started_at) if started_at is not None else None
|
|
3717
|
+
)
|
|
3718
|
+
|
|
3719
|
+
from coord.issue_store import CompletionRecord, post_completion # noqa: PLC0415
|
|
3720
|
+
|
|
3721
|
+
# #717: same normalisation as the local path — a remote interactive session
|
|
3722
|
+
# that pushed ≥1 commit must not land as FAILED because the ssh session
|
|
3723
|
+
# exited non-zero (e.g. tmux kill after the push finished).
|
|
3724
|
+
effective_exit_code = 0 if (commits is not None and commits >= 1) else exit_code
|
|
3725
|
+
|
|
3726
|
+
record = CompletionRecord(
|
|
3727
|
+
assignment_id=assignment_id,
|
|
3728
|
+
machine_name=machine_name,
|
|
3729
|
+
repo_name=repo_name,
|
|
3730
|
+
repo_github=repo_github,
|
|
3731
|
+
issue_number=issue_number,
|
|
3732
|
+
exit_code=effective_exit_code,
|
|
3733
|
+
commits_ahead=commits,
|
|
3734
|
+
branch=branch_now or branch,
|
|
3735
|
+
duration_seconds=duration,
|
|
3736
|
+
log_path=None,
|
|
3737
|
+
summary="",
|
|
3738
|
+
is_interactive=True,
|
|
3739
|
+
)
|
|
3740
|
+
outcome = post_completion(record)
|
|
3741
|
+
|
|
3742
|
+
# #562: stash artifacts BEFORE removing the worktree so compiled outputs
|
|
3743
|
+
# survive cleanup — same discipline as the agent-side stash on workers.
|
|
3744
|
+
# Stash regardless of push_ok (the worktree files are still present even
|
|
3745
|
+
# when the push failed) so a failed-push session doesn't lose its build.
|
|
3746
|
+
artifacts_stashed: int | None = None
|
|
3747
|
+
if _effective_patterns:
|
|
3748
|
+
artifacts_stashed = _remote_stash_artifacts(
|
|
3749
|
+
ssh_target, remote_worktree_sh, repo_name, branch,
|
|
3750
|
+
_effective_patterns, assignment_id,
|
|
3751
|
+
)
|
|
3752
|
+
|
|
3753
|
+
# Clean up the remote worktree only on a successful push — a failed push
|
|
3754
|
+
# means the commits live nowhere but the worktree, so preserve it.
|
|
3755
|
+
worktree_removed = False
|
|
3756
|
+
if push_ok:
|
|
3757
|
+
worktree_removed = _remote_worktree_remove(
|
|
3758
|
+
ssh_target, remote_repo_sh, remote_worktree_sh,
|
|
3759
|
+
)
|
|
3760
|
+
|
|
3761
|
+
return InteractiveFinalizeResult(
|
|
3762
|
+
terminal_status=outcome.status,
|
|
3763
|
+
commits_ahead=commits,
|
|
3764
|
+
push_ok=push_ok,
|
|
3765
|
+
push_error=push_error,
|
|
3766
|
+
already_recorded=False,
|
|
3767
|
+
seam_outcome=outcome,
|
|
3768
|
+
worktree_removed=worktree_removed,
|
|
3769
|
+
merge_verify=merge_verify,
|
|
3770
|
+
artifacts_stashed=artifacts_stashed,
|
|
3771
|
+
)
|
|
3772
|
+
|
|
3773
|
+
|
|
3774
|
+
# ── Stale interactive-session reaper ─────────────────────────────────────────
|
|
3775
|
+
|
|
3776
|
+
|
|
3777
|
+
def reap_stale_interactive_sessions(
|
|
3778
|
+
board: "Board",
|
|
3779
|
+
config: "Config",
|
|
3780
|
+
*,
|
|
3781
|
+
worktrees_dir: Path | None = None,
|
|
3782
|
+
) -> list[str]:
|
|
3783
|
+
"""Sweep *board.active* for dead interactive sessions and release their claims.
|
|
3784
|
+
|
|
3785
|
+
Called from :func:`coord.reconcile.reconcile` on every pass so that
|
|
3786
|
+
``coord resume`` / ``coord notify`` automatically clean up orphaned
|
|
3787
|
+
worktrees and stale ``running`` DB rows for interactive assignments whose
|
|
3788
|
+
tmux session died without going through the normal ``/exit`` → finalize
|
|
3789
|
+
path.
|
|
3790
|
+
|
|
3791
|
+
**Detection strategy**: an assignment is stale when ALL of:
|
|
3792
|
+
|
|
3793
|
+
* ``provider_name == "claude-pty"`` (i.e. dispatched via ``--interactive``)
|
|
3794
|
+
* ``status`` is ``"running"`` or ``"pending"`` in the in-memory board
|
|
3795
|
+
* tmux is available on the local machine AND EITHER:
|
|
3796
|
+
|
|
3797
|
+
- ``coord-<assignment_id>`` is NOT alive (session gone), OR
|
|
3798
|
+
- the session IS alive but its pane has exited (dead-pane case — claude
|
|
3799
|
+
finished while the operator was detached).
|
|
3800
|
+
|
|
3801
|
+
When tmux is NOT available the function returns immediately — the PTY
|
|
3802
|
+
relay is handled synchronously inside ``coord assign``, so no orphan
|
|
3803
|
+
accumulates there.
|
|
3804
|
+
|
|
3805
|
+
**Remote sessions are skipped.** :func:`tmux_session_alive` probes only the
|
|
3806
|
+
local tmux server; a remote interactive session launched via ``coord assign
|
|
3807
|
+
--interactive <remote>`` stores ``machine_name=<remote>`` in the DB and
|
|
3808
|
+
runs its tmux session on that host. If the reaper were allowed to proceed,
|
|
3809
|
+
it would see ``tmux_session_alive() == False`` (no local session) and
|
|
3810
|
+
falsely stamp the remote session ``failed`` while it is still live. The
|
|
3811
|
+
reaper therefore skips any assignment whose machine does not resolve to the
|
|
3812
|
+
local host.
|
|
3813
|
+
|
|
3814
|
+
For each stale **local** session the function:
|
|
3815
|
+
|
|
3816
|
+
1. Counts commits the worktree is ahead of the base branch (before
|
|
3817
|
+
removing the worktree, so git can still run).
|
|
3818
|
+
1a. (Dead-pane only) Pushes any commits to the remote with
|
|
3819
|
+
``git push -u origin HEAD`` (best-effort) so the work is not lost when
|
|
3820
|
+
the worktree is removed. Failures are silently ignored.
|
|
3821
|
+
2. Removes the interactive worktree at
|
|
3822
|
+
``~/.coord/worktrees/<assignment_id>`` (best-effort via
|
|
3823
|
+
:func:`coord.agent._safe_remove_worktree`, which refuses any path that
|
|
3824
|
+
is not a linked worktree or a sandboxed orphan — #1693).
|
|
3825
|
+
3. Marks the assignment in the SQLite DB (only when the row is still
|
|
3826
|
+
``running`` / ``pending`` — a ``coord report-result`` that raced the
|
|
3827
|
+
reaper is left untouched):
|
|
3828
|
+
|
|
3829
|
+
- ``advisory`` when the worktree had 0 commits ahead of the base branch
|
|
3830
|
+
(the operator killed the session before producing any work).
|
|
3831
|
+
- ``failed`` otherwise (commits ≥ 1 or the count is unknown because the
|
|
3832
|
+
worktree was absent when the reaper ran).
|
|
3833
|
+
|
|
3834
|
+
4. Moves the assignment from ``board.active`` to ``board.completed``
|
|
3835
|
+
and sets the in-memory status to match the DB value.
|
|
3836
|
+
4a. (Dead-pane only) Kills the now-empty tmux session with
|
|
3837
|
+
``tmux kill-session`` (best-effort) so ``coord sessions`` stops
|
|
3838
|
+
listing it.
|
|
3839
|
+
4b. (``type == "smoke"`` only, #1256) Runs the live-checkout
|
|
3840
|
+
restore-on-exit safety net via
|
|
3841
|
+
:func:`restore_live_checkout_from_smoke_snapshot`. A ``--smoke-of``
|
|
3842
|
+
session has no worktree (steps 1/1a/2 above are all no-ops for it,
|
|
3843
|
+
since ``wt_path`` never exists) — it runs directly in the live
|
|
3844
|
+
checkout, so a dead tmux session here is exactly the "agent died
|
|
3845
|
+
mid-run" case the snapshot/restore pair exists for. Best-effort and
|
|
3846
|
+
a no-op when no snapshot marker is present.
|
|
3847
|
+
|
|
3848
|
+
Returns the assignment IDs that were reaped. The caller should include
|
|
3849
|
+
these in its ``changed`` list so ``save_board`` is triggered.
|
|
3850
|
+
"""
|
|
3851
|
+
if not tmux_available():
|
|
3852
|
+
return []
|
|
3853
|
+
|
|
3854
|
+
from coord.state import COORD_DIR, get_connection # noqa: PLC0415
|
|
3855
|
+
|
|
3856
|
+
if worktrees_dir is None:
|
|
3857
|
+
worktrees_dir = COORD_DIR / "worktrees"
|
|
3858
|
+
|
|
3859
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
3860
|
+
repos_by_name = {r.name: r for r in config.repos}
|
|
3861
|
+
_local_hn = _get_local_short_hostname()
|
|
3862
|
+
reaped: list[str] = []
|
|
3863
|
+
now = time.time()
|
|
3864
|
+
|
|
3865
|
+
for a in board.active[:]: # iterate a copy — we mutate board mid-loop
|
|
3866
|
+
if a.provider_name != "claude-pty":
|
|
3867
|
+
continue
|
|
3868
|
+
if a.status not in ("running", "pending"):
|
|
3869
|
+
continue
|
|
3870
|
+
if not a.assignment_id:
|
|
3871
|
+
continue
|
|
3872
|
+
|
|
3873
|
+
sname = tmux_session_name(a.assignment_id)
|
|
3874
|
+
_dead_pane_kill_needed = False
|
|
3875
|
+
if tmux_session_alive(sname):
|
|
3876
|
+
# Session is alive — but check whether the claude pane has already
|
|
3877
|
+
# exited (dead pane). This is the detach-and-abandon case: the
|
|
3878
|
+
# operator detached while claude was running, and claude finished in
|
|
3879
|
+
# the background. The tmux session is still up but the work is
|
|
3880
|
+
# stranded without finalize.
|
|
3881
|
+
if not tmux_pane_dead(sname):
|
|
3882
|
+
continue # session alive AND pane running — genuinely in progress
|
|
3883
|
+
# Pane exited; the session is now an empty shell. Fall through to
|
|
3884
|
+
# the reap logic and kill the session at the end of this iteration.
|
|
3885
|
+
_dead_pane_kill_needed = True
|
|
3886
|
+
|
|
3887
|
+
# ── Session is dead (or dead-pane) locally: check locality ──────────
|
|
3888
|
+
|
|
3889
|
+
# Reconstruct machine info for locality check and repo path lookup.
|
|
3890
|
+
machine = machines_by_name.get(a.machine_name or "")
|
|
3891
|
+
|
|
3892
|
+
# Remote-machine guard (#515 follow-up): tmux_session_alive() only
|
|
3893
|
+
# checks the coordinator's local tmux server. A remote session (coord
|
|
3894
|
+
# assign --interactive <remote_host>) runs on a different machine and
|
|
3895
|
+
# will always appear "not alive" locally even while it's running.
|
|
3896
|
+
# Skip the reap entirely so we don't falsely stamp a live remote
|
|
3897
|
+
# session as failed and release its dispatch claim.
|
|
3898
|
+
if machine is not None:
|
|
3899
|
+
_is_local = (
|
|
3900
|
+
machine.name.lower() == _local_hn
|
|
3901
|
+
or machine.host.split(".")[0].lower() == _local_hn
|
|
3902
|
+
)
|
|
3903
|
+
if not _is_local:
|
|
3904
|
+
continue # remote session — leave it alone
|
|
3905
|
+
elif a.machine_name:
|
|
3906
|
+
# machine_name is set but not found in config — unknown host;
|
|
3907
|
+
# skip rather than risk a false-positive reap.
|
|
3908
|
+
continue
|
|
3909
|
+
# else: machine_name is None/empty → coordinator-local session, proceed
|
|
3910
|
+
|
|
3911
|
+
# Reconstruct the repo root so ``git worktree remove`` can run
|
|
3912
|
+
# relative to it. When it is unavailable the removal below still
|
|
3913
|
+
# runs, but only for a path strictly inside ``worktrees_dir`` (#1693).
|
|
3914
|
+
repo_path_val: str | None = None
|
|
3915
|
+
if machine is not None and a.repo_name:
|
|
3916
|
+
rp = machine.repo_path(a.repo_name)
|
|
3917
|
+
if rp:
|
|
3918
|
+
repo_path_val = str(Path(rp).expanduser())
|
|
3919
|
+
|
|
3920
|
+
# ── Determine terminal status from commit count ────────────────────
|
|
3921
|
+
# Count commits ahead of the base branch BEFORE removing the worktree
|
|
3922
|
+
# so that git can still run against it.
|
|
3923
|
+
# advisory: 0 commits — session died before producing any work.
|
|
3924
|
+
# failed: ≥1 commits or unknown — abandoned work-in-progress.
|
|
3925
|
+
wt_path = worktrees_dir / a.assignment_id
|
|
3926
|
+
commits: int | None = None
|
|
3927
|
+
if wt_path.exists():
|
|
3928
|
+
repo = repos_by_name.get(a.repo_name or "")
|
|
3929
|
+
base_branch = repo.default_branch if repo is not None else "main"
|
|
3930
|
+
from coord.agent import _commits_ahead # noqa: PLC0415
|
|
3931
|
+
commits = _commits_ahead(wt_path, base_branch)
|
|
3932
|
+
terminal_status = "advisory" if commits == 0 else "failed"
|
|
3933
|
+
|
|
3934
|
+
# 1a. Push commits before removing the worktree so that any work the
|
|
3935
|
+
# operator produced survives. Best-effort: a push failure does NOT
|
|
3936
|
+
# abort the reap — the worktree is still removed and the DB still
|
|
3937
|
+
# updated. #949: this now runs for the session-gone case too, not
|
|
3938
|
+
# just dead-pane. The old "already gone → operator must have
|
|
3939
|
+
# finalized (and pushed)" assumption is false — a crashed/killed
|
|
3940
|
+
# session that vanished can hold unpushed commits, and since the
|
|
3941
|
+
# reaper *deletes* the worktree next, skipping the push loses that
|
|
3942
|
+
# work permanently. Non-force `git push -u origin HEAD`
|
|
3943
|
+
# fast-forwards real work and safely no-ops on a diverged branch.
|
|
3944
|
+
if wt_path.exists() and commits:
|
|
3945
|
+
try:
|
|
3946
|
+
subprocess.run(
|
|
3947
|
+
["git", "push", "-u", "origin", "HEAD"],
|
|
3948
|
+
cwd=str(wt_path),
|
|
3949
|
+
capture_output=True,
|
|
3950
|
+
timeout=30.0,
|
|
3951
|
+
)
|
|
3952
|
+
except Exception: # noqa: BLE001
|
|
3953
|
+
pass # non-fatal — work may already be on the remote
|
|
3954
|
+
|
|
3955
|
+
# 1. Remove worktree (best-effort). #1693: both branches go through
|
|
3956
|
+
# `_safe_remove_worktree`; the no-repo_path branch used to rmtree
|
|
3957
|
+
# with no git attempt and no guard at all.
|
|
3958
|
+
if wt_path.exists():
|
|
3959
|
+
from coord.agent import _safe_remove_worktree # noqa: PLC0415
|
|
3960
|
+
|
|
3961
|
+
try:
|
|
3962
|
+
_safe_remove_worktree(
|
|
3963
|
+
Path(repo_path_val) if repo_path_val is not None else None,
|
|
3964
|
+
wt_path,
|
|
3965
|
+
sandbox_root=worktrees_dir,
|
|
3966
|
+
)
|
|
3967
|
+
except Exception: # noqa: BLE001 - never abort the reap sweep
|
|
3968
|
+
pass
|
|
3969
|
+
|
|
3970
|
+
# 2. Mark ``terminal_status`` in the DB if the row is still live.
|
|
3971
|
+
try:
|
|
3972
|
+
conn = get_connection()
|
|
3973
|
+
conn.execute(
|
|
3974
|
+
"UPDATE assignments SET status=?, finished_at=? "
|
|
3975
|
+
"WHERE assignment_id=? AND status IN ('running', 'pending')",
|
|
3976
|
+
(terminal_status, now, a.assignment_id),
|
|
3977
|
+
)
|
|
3978
|
+
conn.commit()
|
|
3979
|
+
except Exception: # noqa: BLE001
|
|
3980
|
+
pass # non-fatal — the board update below still releases the claim
|
|
3981
|
+
|
|
3982
|
+
# 3. Update in-memory board so the claim is released immediately
|
|
3983
|
+
# (without waiting for the next build_board() call).
|
|
3984
|
+
moved = board.mark_failed_by_id(a.assignment_id, finished_at=now)
|
|
3985
|
+
# mark_failed_by_id always sets status="failed"; upgrade to advisory
|
|
3986
|
+
# when the commit count shows no work was produced.
|
|
3987
|
+
if moved is not None and terminal_status == "advisory":
|
|
3988
|
+
moved.status = "advisory"
|
|
3989
|
+
|
|
3990
|
+
# 4a. (Dead-pane only) Kill the now-empty tmux session so it disappears
|
|
3991
|
+
# from ``coord sessions`` immediately. Best-effort — if kill fails
|
|
3992
|
+
# the session will be gone on its own eventually (or the operator
|
|
3993
|
+
# can kill it manually).
|
|
3994
|
+
if _dead_pane_kill_needed:
|
|
3995
|
+
try:
|
|
3996
|
+
subprocess.run(
|
|
3997
|
+
["tmux", "kill-session", "-t", sname],
|
|
3998
|
+
capture_output=True,
|
|
3999
|
+
timeout=5.0,
|
|
4000
|
+
)
|
|
4001
|
+
except Exception: # noqa: BLE001
|
|
4002
|
+
pass # non-fatal
|
|
4003
|
+
|
|
4004
|
+
# 4b. (``type == "smoke"`` only, #1256) A --smoke-of session runs
|
|
4005
|
+
# directly in the live checkout with no worktree, so the
|
|
4006
|
+
# worktree-removal branches above (1/1a/2) never touch it — this
|
|
4007
|
+
# is the ONLY cleanup step that applies to a dead smoke session.
|
|
4008
|
+
# restore_live_checkout_from_smoke_snapshot() is a documented
|
|
4009
|
+
# no-op when no snapshot marker exists, so it's safe to call
|
|
4010
|
+
# even if `a.type` classification were ever wrong.
|
|
4011
|
+
if a.type == "smoke" and repo_path_val is not None:
|
|
4012
|
+
_restored, _restore_err = restore_live_checkout_from_smoke_snapshot(
|
|
4013
|
+
repo_path_val, a.assignment_id
|
|
4014
|
+
)
|
|
4015
|
+
if _restore_err:
|
|
4016
|
+
logging.warning(
|
|
4017
|
+
"reap: live-checkout restore failed for dead smoke "
|
|
4018
|
+
"session %s (repo=%s): %s",
|
|
4019
|
+
a.assignment_id, repo_path_val, _restore_err,
|
|
4020
|
+
)
|
|
4021
|
+
elif _restored:
|
|
4022
|
+
logging.info(
|
|
4023
|
+
"reap: restored live checkout for dead smoke session "
|
|
4024
|
+
"%s: %s",
|
|
4025
|
+
a.assignment_id, ", ".join(_restored),
|
|
4026
|
+
)
|
|
4027
|
+
|
|
4028
|
+
reaped.append(a.assignment_id)
|
|
4029
|
+
|
|
4030
|
+
return reaped
|
|
4031
|
+
|
|
4032
|
+
|
|
4033
|
+
# ── Remote stale-session reaper (#588) ───────────────────────────────────────
|
|
4034
|
+
|
|
4035
|
+
#: Per-session count of consecutive reconcile passes where SSH to the remote
|
|
4036
|
+
#: host timed out or refused. Reset to 0 on first successful SSH probe.
|
|
4037
|
+
#: Module-level (ephemeral — not persisted to DB) so the count survives
|
|
4038
|
+
#: multiple reconcile calls within a coordinator session.
|
|
4039
|
+
_REMOTE_SSH_UNREACHABLE_COUNTS: dict[str, int] = {}
|
|
4040
|
+
|
|
4041
|
+
|
|
4042
|
+
def _mark_stale_reap_in_db(assignment_id: str, status: str, finished_at: float) -> None:
|
|
4043
|
+
"""Best-effort DB update when a stale remote session is reaped without a
|
|
4044
|
+
full ``finalize_remote_interactive_exit`` call.
|
|
4045
|
+
|
|
4046
|
+
Called when either:
|
|
4047
|
+
|
|
4048
|
+
* ``finalize_remote_interactive_exit`` raised an exception mid-run, OR
|
|
4049
|
+
* There is no branch / repo path to pass to finalize (push cannot run).
|
|
4050
|
+
|
|
4051
|
+
The update is silently swallowed on DB error so the in-memory board
|
|
4052
|
+
update that follows always frees the machine slot regardless.
|
|
4053
|
+
"""
|
|
4054
|
+
try:
|
|
4055
|
+
from coord.state import get_connection # noqa: PLC0415
|
|
4056
|
+
conn = get_connection()
|
|
4057
|
+
conn.execute(
|
|
4058
|
+
"UPDATE assignments SET status=?, finished_at=? "
|
|
4059
|
+
"WHERE assignment_id=? AND status IN ('running', 'pending')",
|
|
4060
|
+
(status, finished_at, assignment_id),
|
|
4061
|
+
)
|
|
4062
|
+
conn.commit()
|
|
4063
|
+
except Exception: # noqa: BLE001
|
|
4064
|
+
pass
|
|
4065
|
+
|
|
4066
|
+
|
|
4067
|
+
def _probe_remote_tmux_alive(
|
|
4068
|
+
session_name: str,
|
|
4069
|
+
host: TmuxHost,
|
|
4070
|
+
) -> tuple[bool, bool]:
|
|
4071
|
+
"""Probe a remote tmux session and report liveness + SSH reachability.
|
|
4072
|
+
|
|
4073
|
+
Returns a ``(session_alive, ssh_ok)`` tuple:
|
|
4074
|
+
|
|
4075
|
+
* ``session_alive`` — ``True`` when ``tmux has-session`` exited 0.
|
|
4076
|
+
* ``ssh_ok`` — ``True`` when SSH connected successfully (even if the
|
|
4077
|
+
session was absent). ``False`` indicates an SSH transport failure
|
|
4078
|
+
(connection refused / timeout / auth error) rather than a tmux
|
|
4079
|
+
answer of "no such session".
|
|
4080
|
+
|
|
4081
|
+
SSH exit code 255 is the canonical "SSH could not connect" indicator.
|
|
4082
|
+
The ``BatchMode=yes`` + ``ConnectTimeout=4`` flags on the
|
|
4083
|
+
:class:`TmuxHost` (``batch=True``) ensure the call fails fast and
|
|
4084
|
+
never prompts for credentials.
|
|
4085
|
+
"""
|
|
4086
|
+
try:
|
|
4087
|
+
result = subprocess.run(
|
|
4088
|
+
host.cmd(["has-session", "-t", session_name]),
|
|
4089
|
+
capture_output=True,
|
|
4090
|
+
timeout=8.0,
|
|
4091
|
+
)
|
|
4092
|
+
# SSH itself returns 255 when it cannot connect (refused, timeout,
|
|
4093
|
+
# auth failure). tmux returns 0 (alive) or 1 (no session) — both
|
|
4094
|
+
# mean SSH succeeded.
|
|
4095
|
+
if result.returncode == 255:
|
|
4096
|
+
return False, False
|
|
4097
|
+
return result.returncode == 0, True
|
|
4098
|
+
except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError):
|
|
4099
|
+
return False, False
|
|
4100
|
+
|
|
4101
|
+
|
|
4102
|
+
def reap_stale_remote_interactive_sessions(
|
|
4103
|
+
board: "Board",
|
|
4104
|
+
config: "Config",
|
|
4105
|
+
) -> list[str]:
|
|
4106
|
+
"""Sweep *board.active* for remote interactive sessions whose tmux session
|
|
4107
|
+
is dead, and release their machine slots via
|
|
4108
|
+
:func:`finalize_remote_interactive_exit`.
|
|
4109
|
+
|
|
4110
|
+
This is the **remote** complement to :func:`reap_stale_interactive_sessions`,
|
|
4111
|
+
which deliberately skips remote assignments to avoid false-positive reaping
|
|
4112
|
+
of live sessions. This function only touches assignments that are:
|
|
4113
|
+
|
|
4114
|
+
* ``provider_name == "claude-pty"``
|
|
4115
|
+
* ``status`` is ``"running"`` or ``"pending"``
|
|
4116
|
+
* The assigned machine is **not** the local host
|
|
4117
|
+
* The assignment has been running for at least
|
|
4118
|
+
``concurrency.interactive_session_timeout_hours`` (default 12 h)
|
|
4119
|
+
|
|
4120
|
+
For each candidate it SSHes to the remote host (using ``BatchMode=yes``
|
|
4121
|
+
+ ``ConnectTimeout`` for a fast, non-prompting probe) and runs
|
|
4122
|
+
``tmux has-session -t coord-<id>``. Three outcomes:
|
|
4123
|
+
|
|
4124
|
+
1. **Session alive** — skip; the human is still working.
|
|
4125
|
+
2. **SSH ok but session dead** — the tmux session exited without going
|
|
4126
|
+
through the normal finalize path. Call
|
|
4127
|
+
:func:`finalize_remote_interactive_exit` to push any outstanding
|
|
4128
|
+
commits, record the terminal status, and remove the remote worktree.
|
|
4129
|
+
The assignment is moved to ``board.completed`` and the machine slot
|
|
4130
|
+
is freed.
|
|
4131
|
+
3. **SSH unreachable** — the host is down or the connection timed out.
|
|
4132
|
+
Increment a per-session counter in :data:`_REMOTE_SSH_UNREACHABLE_COUNTS`
|
|
4133
|
+
and emit a ``logging.warning()`` so the operator sees it in CLI
|
|
4134
|
+
output. The assignment is left ``running`` — a host that was up 22h
|
|
4135
|
+
ago might just be temporarily unreachable.
|
|
4136
|
+
|
|
4137
|
+
Returns the assignment IDs that were reaped (outcome 2 only). The
|
|
4138
|
+
caller should extend its ``changed`` list with these IDs so
|
|
4139
|
+
:func:`coord.state.save_board` is triggered.
|
|
4140
|
+
"""
|
|
4141
|
+
timeout_hours = config.concurrency.interactive_session_timeout_hours
|
|
4142
|
+
if timeout_hours <= 0:
|
|
4143
|
+
return [] # sweep disabled by config
|
|
4144
|
+
|
|
4145
|
+
timeout_secs = timeout_hours * 3600
|
|
4146
|
+
machines_by_name = {m.name: m for m in config.machines}
|
|
4147
|
+
repos_by_name = {r.name: r for r in config.repos}
|
|
4148
|
+
_local_hn = _get_local_short_hostname()
|
|
4149
|
+
reaped: list[str] = []
|
|
4150
|
+
now = time.time()
|
|
4151
|
+
|
|
4152
|
+
for a in board.active[:]: # iterate a copy — we mutate board mid-loop
|
|
4153
|
+
if a.provider_name != "claude-pty":
|
|
4154
|
+
continue
|
|
4155
|
+
if a.status not in ("running", "pending"):
|
|
4156
|
+
continue
|
|
4157
|
+
if not a.assignment_id:
|
|
4158
|
+
continue
|
|
4159
|
+
|
|
4160
|
+
# Only probe remote machines — local ones are handled by
|
|
4161
|
+
# reap_stale_interactive_sessions().
|
|
4162
|
+
machine = machines_by_name.get(a.machine_name or "")
|
|
4163
|
+
if machine is None:
|
|
4164
|
+
continue
|
|
4165
|
+
_is_local = (
|
|
4166
|
+
machine.name.lower() == _local_hn
|
|
4167
|
+
or machine.host.split(".")[0].lower() == _local_hn
|
|
4168
|
+
)
|
|
4169
|
+
if _is_local:
|
|
4170
|
+
continue
|
|
4171
|
+
|
|
4172
|
+
# Skip sessions that are too young to have plausibly gone stale.
|
|
4173
|
+
dispatched_at = a.dispatched_at or 0.0
|
|
4174
|
+
age_secs = now - dispatched_at
|
|
4175
|
+
if age_secs < timeout_secs:
|
|
4176
|
+
continue
|
|
4177
|
+
|
|
4178
|
+
# ── SSH probe ──────────────────────────────────────────────────────
|
|
4179
|
+
sname = tmux_session_name(a.assignment_id)
|
|
4180
|
+
tmux_host = TmuxHost(ssh_target=machine.host, batch=True)
|
|
4181
|
+
alive, ssh_ok = _probe_remote_tmux_alive(sname, tmux_host)
|
|
4182
|
+
|
|
4183
|
+
if alive:
|
|
4184
|
+
# Session is genuinely running — clear any unreachable counter
|
|
4185
|
+
# accumulated from prior SSH hiccups.
|
|
4186
|
+
_REMOTE_SSH_UNREACHABLE_COUNTS.pop(a.assignment_id, None)
|
|
4187
|
+
continue
|
|
4188
|
+
|
|
4189
|
+
if not ssh_ok:
|
|
4190
|
+
# SSH transport failure — host may be rebooting, not crashed.
|
|
4191
|
+
# Emit a warning and leave the slot occupied rather than
|
|
4192
|
+
# destroying potentially good work.
|
|
4193
|
+
count = _REMOTE_SSH_UNREACHABLE_COUNTS.get(a.assignment_id, 0) + 1
|
|
4194
|
+
_REMOTE_SSH_UNREACHABLE_COUNTS[a.assignment_id] = count
|
|
4195
|
+
age_h = age_secs / 3600
|
|
4196
|
+
logging.warning(
|
|
4197
|
+
"⚠ %s unreachable — interactive session %s may be stale; "
|
|
4198
|
+
"running for %.1fh (consecutive SSH failures: %d)",
|
|
4199
|
+
machine.name, a.assignment_id, age_h, count,
|
|
4200
|
+
)
|
|
4201
|
+
continue
|
|
4202
|
+
|
|
4203
|
+
# ── SSH ok, tmux session dead — reap it ───────────────────────────
|
|
4204
|
+
_REMOTE_SSH_UNREACHABLE_COUNTS.pop(a.assignment_id, None)
|
|
4205
|
+
|
|
4206
|
+
repo_cfg = repos_by_name.get(a.repo_name or "")
|
|
4207
|
+
repo_github = repo_cfg.github if repo_cfg is not None else ""
|
|
4208
|
+
base_branch = (repo_cfg.default_branch or "main") if repo_cfg is not None else "main"
|
|
4209
|
+
artifact_paths_val = list(repo_cfg.artifact_paths or []) if repo_cfg is not None else []
|
|
4210
|
+
|
|
4211
|
+
rp = machine.repo_path(a.repo_name or "")
|
|
4212
|
+
remote_repo_sh: str | None = None
|
|
4213
|
+
if rp:
|
|
4214
|
+
# Convert the machine's ``~/…`` repo path to a ``$HOME/…`` form
|
|
4215
|
+
# that the *remote* shell (not the coordinator's local shell) will
|
|
4216
|
+
# expand correctly.
|
|
4217
|
+
remote_repo_sh = (
|
|
4218
|
+
"$HOME/" + rp[2:]
|
|
4219
|
+
if rp.startswith("~/")
|
|
4220
|
+
else ("$HOME" if rp == "~" else rp)
|
|
4221
|
+
)
|
|
4222
|
+
|
|
4223
|
+
remote_worktree_sh = "$HOME/.coord/worktrees/" + a.assignment_id
|
|
4224
|
+
|
|
4225
|
+
# Recover the branch from the assignment record. When the branch
|
|
4226
|
+
# wasn't persisted at dispatch time (pre-#557 boards) derive it from
|
|
4227
|
+
# the remote worktree HEAD so we don't strand commits.
|
|
4228
|
+
branch_val = a.branch
|
|
4229
|
+
if not branch_val and machine.host:
|
|
4230
|
+
try:
|
|
4231
|
+
probe = subprocess.run(
|
|
4232
|
+
[
|
|
4233
|
+
"ssh", *_SSH_MUX_OPTS, machine.host,
|
|
4234
|
+
f"git -C {remote_worktree_sh}"
|
|
4235
|
+
" rev-parse --abbrev-ref HEAD 2>/dev/null",
|
|
4236
|
+
],
|
|
4237
|
+
capture_output=True,
|
|
4238
|
+
text=True,
|
|
4239
|
+
timeout=15,
|
|
4240
|
+
)
|
|
4241
|
+
if probe.returncode == 0:
|
|
4242
|
+
derived = probe.stdout.strip()
|
|
4243
|
+
if derived and derived != "HEAD":
|
|
4244
|
+
branch_val = derived
|
|
4245
|
+
except Exception: # noqa: BLE001
|
|
4246
|
+
pass
|
|
4247
|
+
|
|
4248
|
+
if branch_val and remote_repo_sh:
|
|
4249
|
+
# Full finalize: push commits, record completion, clean worktree.
|
|
4250
|
+
try:
|
|
4251
|
+
fr = finalize_remote_interactive_exit(
|
|
4252
|
+
assignment_id=a.assignment_id,
|
|
4253
|
+
repo_name=a.repo_name or "",
|
|
4254
|
+
repo_github=repo_github,
|
|
4255
|
+
issue_number=a.issue_number,
|
|
4256
|
+
machine_name=a.machine_name or "",
|
|
4257
|
+
ssh_target=machine.host,
|
|
4258
|
+
remote_worktree_sh=remote_worktree_sh,
|
|
4259
|
+
remote_repo_sh=remote_repo_sh,
|
|
4260
|
+
branch=branch_val,
|
|
4261
|
+
base_branch=base_branch,
|
|
4262
|
+
exit_code=1, # stale / timed-out exit
|
|
4263
|
+
started_at=a.dispatched_at,
|
|
4264
|
+
artifact_paths=artifact_paths_val,
|
|
4265
|
+
)
|
|
4266
|
+
terminal_status = fr.terminal_status
|
|
4267
|
+
if terminal_status not in ("done", "advisory", "failed", "report-result"):
|
|
4268
|
+
terminal_status = "failed"
|
|
4269
|
+
except Exception: # noqa: BLE001
|
|
4270
|
+
# Finalize failed (e.g. network error mid-push). Fall back to
|
|
4271
|
+
# a plain DB mark so the slot is still freed.
|
|
4272
|
+
terminal_status = "failed"
|
|
4273
|
+
_mark_stale_reap_in_db(a.assignment_id, terminal_status, now)
|
|
4274
|
+
else:
|
|
4275
|
+
# branch or remote_repo_sh unavailable — can't push via finalize;
|
|
4276
|
+
# fall back to a bare DB mark so the machine slot is freed.
|
|
4277
|
+
terminal_status = "failed"
|
|
4278
|
+
_mark_stale_reap_in_db(a.assignment_id, terminal_status, now)
|
|
4279
|
+
|
|
4280
|
+
# Update the in-memory board so the claim is released immediately.
|
|
4281
|
+
moved = board.mark_failed_by_id(a.assignment_id, finished_at=now)
|
|
4282
|
+
if moved is not None and terminal_status == "advisory":
|
|
4283
|
+
moved.status = "advisory"
|
|
4284
|
+
reaped.append(a.assignment_id)
|
|
4285
|
+
|
|
4286
|
+
return reaped
|