syncmill 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syncmill/__init__.py +11 -0
- syncmill/agents.py +2064 -0
- syncmill/artifact_io.py +201 -0
- syncmill/artifacts.py +1946 -0
- syncmill/board.py +2060 -0
- syncmill/breaker.py +327 -0
- syncmill/cli.py +4986 -0
- syncmill/command_errors.py +32 -0
- syncmill/config.py +1118 -0
- syncmill/dashboard.py +935 -0
- syncmill/gates.py +179 -0
- syncmill/gateway.py +234 -0
- syncmill/git_env.py +99 -0
- syncmill/isolated_session.py +431 -0
- syncmill/matcher.py +127 -0
- syncmill/memory.py +1067 -0
- syncmill/memory_backend.py +1907 -0
- syncmill/memory_migration.py +962 -0
- syncmill/model_routing.py +482 -0
- syncmill/models.py +244 -0
- syncmill/monitor.py +775 -0
- syncmill/phoenix_exporter.py +339 -0
- syncmill/planning.py +512 -0
- syncmill/platform_support.py +14 -0
- syncmill/preflight.py +1129 -0
- syncmill/project.py +1067 -0
- syncmill/report.py +502 -0
- syncmill/review_candidates.py +107 -0
- syncmill/room_compare.py +2380 -0
- syncmill/room_raw.py +484 -0
- syncmill/room_viewer.py +604 -0
- syncmill/rooms.py +4044 -0
- syncmill/session.py +444 -0
- syncmill/strategies/__init__.py +82 -0
- syncmill/strategies/base.py +910 -0
- syncmill/strategies/compete.py +233 -0
- syncmill/strategies/council.py +429 -0
- syncmill/strategies/decompose.py +570 -0
- syncmill/strategies/pipeline.py +221 -0
- syncmill/strategies/route.py +130 -0
- syncmill/supervisor.py +1423 -0
- syncmill/teamfile.py +571 -0
- syncmill/telemetry_hook.py +231 -0
- syncmill/terminal.py +245 -0
- syncmill/tracer.py +548 -0
- syncmill/tui.py +3537 -0
- syncmill/tui_collaboration.py +499 -0
- syncmill/tui_jobs.py +1407 -0
- syncmill/tui_render.py +156 -0
- syncmill/workflow.py +3746 -0
- syncmill/worktree.py +1045 -0
- syncmill-0.2.0.dist-info/METADATA +853 -0
- syncmill-0.2.0.dist-info/RECORD +56 -0
- syncmill-0.2.0.dist-info/WHEEL +4 -0
- syncmill-0.2.0.dist-info/entry_points.txt +2 -0
- syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
syncmill/monitor.py
ADDED
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
"""tmux live monitoring of concurrent agent runs (#16, v0.3).
|
|
2
|
+
|
|
3
|
+
A live view of a concurrent run: each agent's output streams into its own tmux
|
|
4
|
+
pane while it runs. This is NOT a strategy and NOT an agent — it's an
|
|
5
|
+
observability layer the Supervisor builds per run and threads through the
|
|
6
|
+
strategy (like the gate/breaker), reusing the ambient ``OutputSink`` seam in
|
|
7
|
+
``agents.py`` so agents stay pure ``(prompt, workdir, timeout)`` functions.
|
|
8
|
+
|
|
9
|
+
**Outer / inner split** (so auto-attach can coexist with the deliberately sync,
|
|
10
|
+
main-thread orchestration — a single sync process can't both run the fan-out and
|
|
11
|
+
hold the foreground for ``tmux attach``):
|
|
12
|
+
|
|
13
|
+
* The **OUTER** ``syncmill run`` (a normal invocation with ``--monitor``) builds a
|
|
14
|
+
detached tmux session whose control pane runs the REAL orchestration by
|
|
15
|
+
re-invoking ``syncmill run`` — same argv — with ``SYNCMILL_MONITOR_DIR`` set
|
|
16
|
+
(the INNER process). It then ``tmux attach``es (foreground, blocks) and, on
|
|
17
|
+
detach, reads the inner's ``result.json`` and prints/exits with its code.
|
|
18
|
+
* The **INNER** process detects itself via :func:`inner_dir` (env var + sentinel),
|
|
19
|
+
builds a :class:`TmuxFileMonitor`, runs the orchestration normally, tees each
|
|
20
|
+
agent's output to a per-agent log, lazily adds a live ``tail -F`` pane per agent
|
|
21
|
+
it actually runs (no outer/inner drift), and writes ``result.json`` at the end.
|
|
22
|
+
|
|
23
|
+
Every process stays sync + single-threaded. All tmux interaction funnels through
|
|
24
|
+
:func:`_tmux` / :func:`_tmux_attach` and tmux discovery through
|
|
25
|
+
:func:`tmux_available`, so the whole module is hermetically testable without ever
|
|
26
|
+
launching tmux. The module is intentionally FLAT (one file, like ``gateway.py``);
|
|
27
|
+
the OUTER launcher and INNER monitor halves are banner-separated and the OUTER
|
|
28
|
+
half imports nothing from the INNER half, so a future split is a clean cut.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import contextlib
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import shutil
|
|
37
|
+
import subprocess
|
|
38
|
+
import sys
|
|
39
|
+
import tempfile
|
|
40
|
+
import time
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Protocol, runtime_checkable
|
|
43
|
+
from uuid import uuid4
|
|
44
|
+
|
|
45
|
+
from syncmill.agents import stream_to
|
|
46
|
+
from syncmill.command_errors import command_error
|
|
47
|
+
from syncmill.artifacts import printable_name, printable_text
|
|
48
|
+
from syncmill.config import MonitorConfig
|
|
49
|
+
from syncmill.models import TaskResult, TaskStatus
|
|
50
|
+
from syncmill.terminal import NO_COMMAND, echo, printable_command, secho, shell_arg
|
|
51
|
+
|
|
52
|
+
# ── Shared: the outer↔inner handshake, tmux plumbing, result hand-off ──────
|
|
53
|
+
|
|
54
|
+
#: Env var the OUTER launcher sets on the INNER process, naming the shared run
|
|
55
|
+
#: dir. INTERNAL IPC, not user config — deliberately outside ``SYNCMILL_MONITOR__*``.
|
|
56
|
+
INNER_ENV = "SYNCMILL_MONITOR_DIR"
|
|
57
|
+
#: Env var carrying the OUTER-minted run_id to the INNER's ``resolve_run_id``
|
|
58
|
+
#: (report.py). INTERNAL IPC like ``INNER_ENV`` — it rides the same 0600 env
|
|
59
|
+
#: snapshot (never ``_inner_argv``, which stays deterministic/id-free), and the
|
|
60
|
+
#: resolver only honors it when ``inner_dir()`` says this process IS the inner,
|
|
61
|
+
#: so a stray exported ``SYNCMILL_RUN_ID`` can never leak into a normal run.
|
|
62
|
+
RUN_ID_ENV = "SYNCMILL_RUN_ID"
|
|
63
|
+
#: Sentinel file the OUTER writes into the run dir. ``inner_dir`` requires BOTH the
|
|
64
|
+
#: env var and this file, so a stray exported ``SYNCMILL_MONITOR_DIR`` can never
|
|
65
|
+
#: masquerade a normal run as the inner orchestration.
|
|
66
|
+
SENTINEL_NAME = ".syncmill-inner"
|
|
67
|
+
_RESULT_FILE = "result.json"
|
|
68
|
+
|
|
69
|
+
#: Grace window the OUTER polls for ``result.json`` after ``tmux attach`` returns,
|
|
70
|
+
#: covering the race where the user detaches just as the inner is writing it.
|
|
71
|
+
#: Module-level so tests can shrink it to 0.
|
|
72
|
+
_RESULT_GRACE_SECONDS = 2.0
|
|
73
|
+
_RESULT_POLL_INTERVAL = 0.1
|
|
74
|
+
|
|
75
|
+
#: Env vars NOT forwarded into the inner PANE via the snapshot: ``TMUX``/``TMUX_PANE``
|
|
76
|
+
#: identify the pane (forwarding the outer's would point the inner at the wrong one),
|
|
77
|
+
#: and ``TERM`` is set by tmux for the pane (forwarding the outer's can corrupt
|
|
78
|
+
#: terminal rendering inside the control pane / agent CLIs). MUST stay in lockstep
|
|
79
|
+
#: with the set ``_ENV_SHIM`` preserves: anything skipped here but not re-kept by the
|
|
80
|
+
#: shim would be lost from the inner's env entirely (neither snapshotted nor inherited).
|
|
81
|
+
_SNAPSHOT_SKIP_ENV = frozenset({"TMUX", "TMUX_PANE", "TERM"})
|
|
82
|
+
|
|
83
|
+
#: Env vars stripped from the ATTACH client only: dropping ``TMUX``/``TMUX_PANE`` lets
|
|
84
|
+
#: ``attach-session`` work from inside an existing tmux session (it otherwise refuses
|
|
85
|
+
#: to nest). ``TERM`` is KEPT — the attaching client needs the caller's terminal type
|
|
86
|
+
#: to render, so stripping it would break a normal ``--monitor`` attach.
|
|
87
|
+
_ATTACH_STRIP_ENV = frozenset({"TMUX", "TMUX_PANE"})
|
|
88
|
+
|
|
89
|
+
#: A tiny exec shim run as the pane's process: it REPLACES the pane's inherited
|
|
90
|
+
#: environment with the OUTER's ``0600`` snapshot, preserving only the tmux-owned
|
|
91
|
+
#: pane vars (``TERM``/``TMUX``/``TMUX_PANE``), then ``execvp``s the real inner argv.
|
|
92
|
+
#: Replacing (not ``update``-ing) is load-bearing: a new tmux session inherits the
|
|
93
|
+
#: tmux **server's** env, which can be stale (a server started hours ago in another
|
|
94
|
+
#: project). An additive ``update`` would leave any stale ``SYNCMILL_*``/auth/PATH
|
|
95
|
+
#: var that the outer client lacks, silently changing the inner run's config — so we
|
|
96
|
+
#: clear first, giving the in-tmux INNER EXACTLY the outer's environment (PATH,
|
|
97
|
+
#: ``SYNCMILL_*`` config, agent auth) plus the pane's own terminal vars. Secrets ride
|
|
98
|
+
#: the file, never the tmux command-line argv, and the launch argv stays tiny (a
|
|
99
|
+
#: large env can't blow the exec arg limit).
|
|
100
|
+
#:
|
|
101
|
+
#: The shim SELF-UNLINKS the snapshot, in a ``finally`` before parsing it (the
|
|
102
|
+
#: ``session.py`` recipe). The file holds the outer's entire environment —
|
|
103
|
+
#: provider API keys, tokens, everything — and the run dir outlives the shim on
|
|
104
|
+
#: two paths: a detach before the run finishes and a failed attach both keep it
|
|
105
|
+
#: so the inner can still write ``result.json``. Left in place, that plaintext
|
|
106
|
+
#: dump sits in the temp dir until some LATER monitored run's stale sweep finds
|
|
107
|
+
#: it a day old, which may be never. The parent cannot delete it instead: the
|
|
108
|
+
#: detached pane's shim runs asynchronously after ``new-session`` returns.
|
|
109
|
+
_ENV_SHIM = (
|
|
110
|
+
"import os,sys,json\n"
|
|
111
|
+
"keep={k:os.environ[k] for k in ('TERM','TMUX','TMUX_PANE') if k in os.environ}\n"
|
|
112
|
+
"p=os.path.join(sys.argv[1],'env')\n"
|
|
113
|
+
"try:\n"
|
|
114
|
+
" data=open(p).read()\n"
|
|
115
|
+
"finally:\n"
|
|
116
|
+
" try: os.unlink(p)\n"
|
|
117
|
+
" except OSError: pass\n"
|
|
118
|
+
"snap=json.loads(data)\n"
|
|
119
|
+
"os.environ.clear(); os.environ.update(snap); os.environ.update(keep)\n"
|
|
120
|
+
"os.execvp(sys.argv[2], sys.argv[2:])\n"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class MonitorLaunchError(RuntimeError):
|
|
125
|
+
"""The OUTER tmux session couldn't be created. The CLI catches this and falls
|
|
126
|
+
back to a normal (unmonitored) run — a tmux hiccup must never abort the task."""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def inner_dir() -> str | None:
|
|
130
|
+
"""The run dir iff THIS process is the in-tmux INNER orchestration, else None.
|
|
131
|
+
|
|
132
|
+
The single source of truth for "am I the inner process?", read at call time
|
|
133
|
+
(never frozen at import). Requires BOTH the env var the OUTER set AND the
|
|
134
|
+
sentinel file it wrote — belt-and-suspenders against a stray exported
|
|
135
|
+
``SYNCMILL_MONITOR_DIR``. Both the CLI's outer-launch gate and
|
|
136
|
+
:func:`build_monitor` route through here so they can never disagree (a drift
|
|
137
|
+
would silently run the inner with no monitor → no ``result.json`` → the outer
|
|
138
|
+
reports the wrong exit code).
|
|
139
|
+
"""
|
|
140
|
+
run_dir = os.environ.get(INNER_ENV)
|
|
141
|
+
if not run_dir:
|
|
142
|
+
return None
|
|
143
|
+
if not (Path(run_dir) / SENTINEL_NAME).is_file():
|
|
144
|
+
return None
|
|
145
|
+
return run_dir
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def tmux_available() -> bool:
|
|
149
|
+
"""True if a ``tmux`` binary is on PATH — the only liveness check, and a pure
|
|
150
|
+
PATH lookup (no probe). Patched in tests so the suite never depends on a real
|
|
151
|
+
tmux being installed (it is on the dev box, absent in CI)."""
|
|
152
|
+
return shutil.which("tmux") is not None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _tmux(*args: str) -> subprocess.CompletedProcess[str]:
|
|
156
|
+
"""Run a tmux CONTROL command, capturing output. Never raises on tmux's own
|
|
157
|
+
non-zero exit — callers inspect ``returncode``. tmux-missing is guarded
|
|
158
|
+
upstream by :func:`tmux_available`, so a bare launch failure here is
|
|
159
|
+
exceptional and surfaces as ``FileNotFoundError`` only if that guard was
|
|
160
|
+
skipped."""
|
|
161
|
+
return subprocess.run(["tmux", *args], capture_output=True, text=True)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _tmux_attach(session: str) -> int:
|
|
165
|
+
"""Attach the real terminal to ``session`` — FOREGROUND, blocks until the user
|
|
166
|
+
detaches (Ctrl-b d) or the session ends; returns tmux's exit code.
|
|
167
|
+
|
|
168
|
+
Must inherit all three std streams (NO ``capture_output``, NO ``stdin=PIPE``)
|
|
169
|
+
or tmux can't draw the UI and hangs. Runs with ``TMUX``/``TMUX_PANE`` UNSET so
|
|
170
|
+
``attach-session`` works even when ``--monitor`` is invoked from INSIDE an
|
|
171
|
+
existing tmux session (it otherwise refuses to nest). The caller inspects the
|
|
172
|
+
return code so a real attach failure isn't mistaken for a clean detach.
|
|
173
|
+
"""
|
|
174
|
+
env = {k: v for k, v in os.environ.items() if k not in _ATTACH_STRIP_ENV}
|
|
175
|
+
return subprocess.run(
|
|
176
|
+
["tmux", "attach-session", "-t", session], env=env
|
|
177
|
+
).returncode
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _result_to_json(result: TaskResult) -> dict:
|
|
181
|
+
"""The single result.json contract (inner writes / outer reads). Flattens the
|
|
182
|
+
gate verdict to a string + bool — never the ``GateReport`` object — and keeps
|
|
183
|
+
only what the OUTER needs to reproduce the CLI's result display."""
|
|
184
|
+
return {
|
|
185
|
+
"status": str(result.status),
|
|
186
|
+
"agent_id": result.agent_id,
|
|
187
|
+
"output": result.output,
|
|
188
|
+
"exit_code": result.exit_code,
|
|
189
|
+
"duration_seconds": result.duration_seconds,
|
|
190
|
+
"files_changed": list(result.files_changed),
|
|
191
|
+
"error": result.error,
|
|
192
|
+
"gate_summary": result.gate.summary() if result.gate is not None else None,
|
|
193
|
+
"gate_passed": result.gate.passed if result.gate is not None else None,
|
|
194
|
+
# Already a plain str (or None) — no flattening needed. Carried so the
|
|
195
|
+
# OUTER can honor `--emit-patch` from result.json after a monitored run
|
|
196
|
+
# (json escapes any surrogateescape'd non-UTF-8 bytes losslessly).
|
|
197
|
+
"patch": result.patch,
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def write_result(run_dir: str | Path, result: TaskResult) -> None:
|
|
202
|
+
"""Atomically write the inner's result to ``<run_dir>/result.json`` (a
|
|
203
|
+
UNIQUE temp in the same dir — the report mkstemp recipe, not a fixed
|
|
204
|
+
``.tmp`` sibling two concurrent writers would interleave into — then
|
|
205
|
+
``os.replace``), so the OUTER never reads a half-written file."""
|
|
206
|
+
path = Path(run_dir) / _RESULT_FILE
|
|
207
|
+
fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
|
|
208
|
+
try:
|
|
209
|
+
with os.fdopen(fd, "w") as handle:
|
|
210
|
+
handle.write(json.dumps(_result_to_json(result)))
|
|
211
|
+
handle.flush()
|
|
212
|
+
os.fsync(handle.fileno())
|
|
213
|
+
os.replace(tmp, path)
|
|
214
|
+
except BaseException:
|
|
215
|
+
with contextlib.suppress(OSError):
|
|
216
|
+
os.unlink(tmp)
|
|
217
|
+
raise
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def read_result(run_dir: str | Path) -> dict | None:
|
|
221
|
+
"""Read ``<run_dir>/result.json``; None if absent or unreadable (the inner
|
|
222
|
+
hasn't finished, or never wrote it)."""
|
|
223
|
+
try:
|
|
224
|
+
return json.loads((Path(run_dir) / _RESULT_FILE).read_text())
|
|
225
|
+
except (OSError, ValueError):
|
|
226
|
+
return None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def render_result(data: dict) -> int:
|
|
230
|
+
"""Echo a result.json dict the way the CLI prints a live result, and return the
|
|
231
|
+
process exit code (0 completed, 1 failed). Shared by the OUTER (after detach)
|
|
232
|
+
and the CLI's normal path, so a monitored and unmonitored run print identically.
|
|
233
|
+
|
|
234
|
+
result.json is read back without schema validation, so its one-line fields
|
|
235
|
+
go through ``printable_name``; only output and error stay multi-line.
|
|
236
|
+
"""
|
|
237
|
+
if data.get("status") == str(TaskStatus.COMPLETED):
|
|
238
|
+
agent = printable_name(str(data.get("agent_id")))
|
|
239
|
+
duration = printable_name(str(data.get("duration_seconds")))
|
|
240
|
+
secho(f"[{agent}] Completed in {duration}s", fg="green")
|
|
241
|
+
if data.get("gate_summary") is not None:
|
|
242
|
+
summary = printable_name(str(data["gate_summary"]))
|
|
243
|
+
secho(f"[syncmill] Quality gate: passed ({summary})", fg="green")
|
|
244
|
+
if data.get("files_changed"):
|
|
245
|
+
names = ", ".join(printable_name(str(f)) for f in data["files_changed"])
|
|
246
|
+
echo(f"[syncmill] Files changed: {names}")
|
|
247
|
+
echo()
|
|
248
|
+
echo(printable_text(str(data.get("output") or "")))
|
|
249
|
+
return 0
|
|
250
|
+
secho("[syncmill] Failed", fg="red")
|
|
251
|
+
if data.get("error"):
|
|
252
|
+
echo(printable_text(str(data["error"])))
|
|
253
|
+
return 1
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def render_task_result(result: TaskResult) -> int:
|
|
257
|
+
"""Render a live ``TaskResult`` through the same path as a monitored run's
|
|
258
|
+
``result.json`` (:func:`render_result`), so the monitored and unmonitored CLI
|
|
259
|
+
displays are byte-identical — the user sees the same thing either way."""
|
|
260
|
+
return render_result(_result_to_json(result))
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def emit_patch_file(path: str | Path, data: dict) -> bool:
|
|
264
|
+
"""Write the winner's collected ``patch`` from a result.json-shaped dict to
|
|
265
|
+
``path`` (the ``run --emit-patch`` recovery path). Lives beside
|
|
266
|
+
:func:`render_result` for the same reason: it consumes the one result
|
|
267
|
+
contract, shared by the CLI's normal path and the OUTER's monitored path so
|
|
268
|
+
the two behave identically.
|
|
269
|
+
|
|
270
|
+
Returns ``False`` only when a patch existed but could not be written (the
|
|
271
|
+
one failure of the user's explicit export request — callers turn it into a
|
|
272
|
+
non-zero exit); the nothing-to-emit cases (failed run, no diff, non-git
|
|
273
|
+
workspace) print a notice and return ``True``. Encodes with
|
|
274
|
+
``surrogateescape`` to round-trip ``capture_changes``'s byte-exact decode.
|
|
275
|
+
"""
|
|
276
|
+
if data.get("status") != str(TaskStatus.COMPLETED):
|
|
277
|
+
secho(
|
|
278
|
+
"[syncmill] --emit-patch: run did not complete — no patch written",
|
|
279
|
+
fg="yellow",
|
|
280
|
+
)
|
|
281
|
+
return True
|
|
282
|
+
patch = data.get("patch")
|
|
283
|
+
if not patch:
|
|
284
|
+
secho(
|
|
285
|
+
"[syncmill] --emit-patch: no diff to emit (the winner changed no "
|
|
286
|
+
"files, or its workspace was not a git worktree)",
|
|
287
|
+
fg="yellow",
|
|
288
|
+
)
|
|
289
|
+
return True
|
|
290
|
+
try:
|
|
291
|
+
Path(path).write_text(patch, encoding="utf-8", errors="surrogateescape")
|
|
292
|
+
except OSError as exc:
|
|
293
|
+
secho(
|
|
294
|
+
f"[syncmill] --emit-patch: failed to write {printable_name(str(path))}: {exc}",
|
|
295
|
+
fg="red",
|
|
296
|
+
)
|
|
297
|
+
return False
|
|
298
|
+
command = printable_command("git", "apply", args=[path]) # copy/paste-safe with spaces
|
|
299
|
+
secho(
|
|
300
|
+
f"[syncmill] Patch written: {printable_name(str(path))} — apply with: {command}",
|
|
301
|
+
fg="green",
|
|
302
|
+
)
|
|
303
|
+
return True
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ── INNER monitor: tee output to per-agent logs + live tmux panes ──────────
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@runtime_checkable
|
|
310
|
+
class Monitor(Protocol):
|
|
311
|
+
"""Live observability of a concurrent run, threaded through the strategy layer
|
|
312
|
+
(like the gate/breaker). A concurrent strategy announces each :meth:`phase`,
|
|
313
|
+
wraps each fan-out in :meth:`streaming` (which both opens the output seam and
|
|
314
|
+
ensures a live pane exists per agent about to run), reports each agent's final
|
|
315
|
+
:meth:`status`, and the Supervisor calls :meth:`finish` once with the winner.
|
|
316
|
+
A ``None`` monitor (the default / sequential strategies) means no monitoring.
|
|
317
|
+
"""
|
|
318
|
+
|
|
319
|
+
def phase(self, name: str) -> None: ...
|
|
320
|
+
|
|
321
|
+
def streaming(self, agent_ids: list[str]) -> contextlib.AbstractContextManager: ...
|
|
322
|
+
|
|
323
|
+
def status(self, agent_id: str, status: TaskStatus) -> None: ...
|
|
324
|
+
|
|
325
|
+
def finish(self, result: TaskResult) -> None: ...
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _safe(agent_id: str) -> str:
|
|
329
|
+
"""A filename-safe form of an agent id (the real ids — claude/codex/kimi-code —
|
|
330
|
+
are already safe; this only guards against a path separator sneaking in)."""
|
|
331
|
+
return agent_id.replace("/", "_").replace(os.sep, "_")
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
#: Bytes per rotated log segment (two segments per agent → 2× this retained).
|
|
335
|
+
#: Bounds a runaway agent from filling the run dir while keeping the most
|
|
336
|
+
#: recent output for the tmux pane; small enough that rotation is cheap.
|
|
337
|
+
_LOG_SEGMENT_BYTES = 256 * 1024
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class FileSink:
|
|
341
|
+
"""An :class:`~syncmill.agents.OutputSink` that appends each agent's streamed
|
|
342
|
+
chunks to ``<run_dir>/agent-<id>.log`` via a single non-blocking ``os.write``.
|
|
343
|
+
|
|
344
|
+
No Python buffering and no ``fsync``: ``feed`` runs inside the agent's drain
|
|
345
|
+
loop on the shared event loop, so it must not block (a slow/NFS/full-disk dir
|
|
346
|
+
must not stall every other agent's drain *and* memtomem's I/O) and must never
|
|
347
|
+
raise out — a dropped write loses a log chunk, never the run. The fd is opened
|
|
348
|
+
once per agent in ``O_APPEND`` mode and reused, so a council member's output
|
|
349
|
+
across propose/critique/synthesize accumulates in one log.
|
|
350
|
+
|
|
351
|
+
The log is *bounded* (the rooms ``RoundLogSink`` rotation): two rotated
|
|
352
|
+
segments of at most :data:`_LOG_SEGMENT_BYTES` each, rolled by a cheap
|
|
353
|
+
``os.replace`` of the active segment onto its ``.1`` sibling. The pane's
|
|
354
|
+
``tail -F`` follows by name, so rotation is transparent to the live view.
|
|
355
|
+
Unlike the rooms sink this one never creates the run dir (a missing dir
|
|
356
|
+
stays a silent no-op — the monitor owns the dir's lifecycle, not the sink).
|
|
357
|
+
"""
|
|
358
|
+
|
|
359
|
+
def __init__(self, run_dir: Path):
|
|
360
|
+
self._run_dir = run_dir
|
|
361
|
+
self._fds: dict[str, int] = {}
|
|
362
|
+
self._written: dict[str, int] = {}
|
|
363
|
+
|
|
364
|
+
def path_for(self, agent_id: str) -> Path:
|
|
365
|
+
return self._run_dir / f"agent-{_safe(agent_id)}.log"
|
|
366
|
+
|
|
367
|
+
def _fd(self, agent_id: str) -> int | None:
|
|
368
|
+
fd = self._fds.get(agent_id)
|
|
369
|
+
if fd is not None:
|
|
370
|
+
return fd
|
|
371
|
+
try:
|
|
372
|
+
fd = os.open(
|
|
373
|
+
str(self.path_for(agent_id)),
|
|
374
|
+
os.O_WRONLY | os.O_APPEND | os.O_CREAT,
|
|
375
|
+
0o644,
|
|
376
|
+
)
|
|
377
|
+
except OSError:
|
|
378
|
+
return None
|
|
379
|
+
self._fds[agent_id] = fd
|
|
380
|
+
try:
|
|
381
|
+
self._written[agent_id] = os.fstat(fd).st_size
|
|
382
|
+
except OSError:
|
|
383
|
+
self._written[agent_id] = 0
|
|
384
|
+
return fd
|
|
385
|
+
|
|
386
|
+
def _rotate(self, agent_id: str) -> int | None:
|
|
387
|
+
"""Roll the active segment onto its ``.1`` sibling and reopen empty.
|
|
388
|
+
|
|
389
|
+
Returns ``None`` when the roll fails: the active segment is already at
|
|
390
|
+
the cap, so reopening it and continuing to append would grow the log
|
|
391
|
+
past the bound. Dropping the chunk (best-effort) keeps the bound; the
|
|
392
|
+
next feed retries the rotate.
|
|
393
|
+
"""
|
|
394
|
+
old = self._fds.pop(agent_id, None)
|
|
395
|
+
if old is not None:
|
|
396
|
+
with contextlib.suppress(OSError):
|
|
397
|
+
os.close(old)
|
|
398
|
+
path = self.path_for(agent_id)
|
|
399
|
+
try:
|
|
400
|
+
os.replace(str(path), str(path.with_suffix(path.suffix + ".1")))
|
|
401
|
+
except OSError:
|
|
402
|
+
return None # active segment stays full; the bound holds
|
|
403
|
+
self._written[agent_id] = 0
|
|
404
|
+
return self._fd(agent_id)
|
|
405
|
+
|
|
406
|
+
def feed(self, agent_id: str, chunk: bytes) -> None:
|
|
407
|
+
fd = self._fd(agent_id)
|
|
408
|
+
if fd is None:
|
|
409
|
+
return
|
|
410
|
+
# A single chunk larger than a whole segment would blow the bound even
|
|
411
|
+
# right after a rotate, so keep only its most-recent tail — the segment
|
|
412
|
+
# cap holds regardless of how the tee sizes its writes.
|
|
413
|
+
if len(chunk) > _LOG_SEGMENT_BYTES:
|
|
414
|
+
chunk = chunk[-_LOG_SEGMENT_BYTES:]
|
|
415
|
+
if self._written.get(agent_id, 0) + len(chunk) > _LOG_SEGMENT_BYTES:
|
|
416
|
+
fd = self._rotate(agent_id)
|
|
417
|
+
if fd is None:
|
|
418
|
+
return
|
|
419
|
+
try:
|
|
420
|
+
written = os.write(fd, chunk)
|
|
421
|
+
except OSError:
|
|
422
|
+
return # best-effort: a failed log write never breaks the run
|
|
423
|
+
# Account for the bytes actually written; a short write must not let the
|
|
424
|
+
# tracked size drift below the real file size and defeat the cap.
|
|
425
|
+
self._written[agent_id] = self._written.get(agent_id, 0) + written
|
|
426
|
+
|
|
427
|
+
def close(self) -> None:
|
|
428
|
+
for fd in self._fds.values():
|
|
429
|
+
with contextlib.suppress(OSError):
|
|
430
|
+
os.close(fd)
|
|
431
|
+
self._fds.clear()
|
|
432
|
+
self._written.clear()
|
|
433
|
+
|
|
434
|
+
# Context-manager protocol so standalone users (outside TmuxFileMonitor,
|
|
435
|
+
# whose finish() is the only in-module close site) can't leak the
|
|
436
|
+
# per-agent O_APPEND fds when the monitored run aborts early.
|
|
437
|
+
def __enter__(self) -> "FileSink":
|
|
438
|
+
return self
|
|
439
|
+
|
|
440
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
441
|
+
self.close()
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class TmuxFileMonitor:
|
|
445
|
+
"""The INNER-side :class:`Monitor`: tees agent output to per-agent logs and
|
|
446
|
+
drives the tmux panes from inside the session (``$TMUX`` is inherited).
|
|
447
|
+
|
|
448
|
+
Panes are created LAZILY and inner-driven — one ``tail -F`` pane the first time
|
|
449
|
+
an agent actually runs (via :meth:`streaming`), so the live panes track the
|
|
450
|
+
real runnable set (after capability re-ordering + the breaker's OPEN filter)
|
|
451
|
+
with zero outer/inner drift and no blank panes for skipped agents. Phase
|
|
452
|
+
headers and per-agent outcomes are appended to ``status.log`` (the planner's
|
|
453
|
+
sync phase, which can't stream, is status-only there). On :meth:`finish` it
|
|
454
|
+
writes ``result.json`` and keeps the control pane's result on screen
|
|
455
|
+
(``remain-on-exit``) while the ``tail`` panes hold the session open until the
|
|
456
|
+
user detaches.
|
|
457
|
+
"""
|
|
458
|
+
|
|
459
|
+
def __init__(self, run_dir: Path, config: MonitorConfig):
|
|
460
|
+
self._run_dir = run_dir
|
|
461
|
+
self._config = config # reserved (session_prefix etc. are outer-only today)
|
|
462
|
+
self._sink = FileSink(run_dir)
|
|
463
|
+
self._status_path = run_dir / "status.log"
|
|
464
|
+
self._seen: set[str] = set() # agents that already have a pane attempt
|
|
465
|
+
self._in_tmux = bool(os.environ.get("TMUX"))
|
|
466
|
+
|
|
467
|
+
# -- strategy-facing API --------------------------------------------------
|
|
468
|
+
|
|
469
|
+
def phase(self, name: str) -> None:
|
|
470
|
+
self._log_status(f"── phase: {name} ──")
|
|
471
|
+
|
|
472
|
+
def streaming(self, agent_ids: list[str]) -> contextlib.AbstractContextManager:
|
|
473
|
+
for agent_id in agent_ids:
|
|
474
|
+
self._ensure_pane(agent_id)
|
|
475
|
+
return stream_to(self._sink)
|
|
476
|
+
|
|
477
|
+
def status(self, agent_id: str, status: TaskStatus) -> None:
|
|
478
|
+
self._log_status(f"[{status}] {agent_id}")
|
|
479
|
+
|
|
480
|
+
def finish(self, result: TaskResult) -> None:
|
|
481
|
+
# Best-effort: a failed result.json write (full disk, vanished run dir) must
|
|
482
|
+
# not break the Supervisor's cleanup — and the sink fds always get closed.
|
|
483
|
+
try:
|
|
484
|
+
write_result(self._run_dir, result)
|
|
485
|
+
except OSError:
|
|
486
|
+
pass
|
|
487
|
+
finally:
|
|
488
|
+
self._sink.close()
|
|
489
|
+
if not self._in_tmux:
|
|
490
|
+
return
|
|
491
|
+
# Keep THIS (control) pane visible after the inner process exits so the
|
|
492
|
+
# printed result stays on screen; the tail panes keep the session alive
|
|
493
|
+
# until the user detaches (the auto-attach UX), at which point the OUTER
|
|
494
|
+
# reads result.json and tears the session down.
|
|
495
|
+
with contextlib.suppress(Exception):
|
|
496
|
+
_tmux("set-option", "-p", "remain-on-exit", "on")
|
|
497
|
+
_tmux("select-pane", "-T", "syncmill — run complete · detach: Ctrl-b d")
|
|
498
|
+
|
|
499
|
+
# -- internals ------------------------------------------------------------
|
|
500
|
+
|
|
501
|
+
def _log_status(self, line: str) -> None:
|
|
502
|
+
with contextlib.suppress(OSError):
|
|
503
|
+
with self._status_path.open("a") as fh:
|
|
504
|
+
fh.write(line + "\n")
|
|
505
|
+
|
|
506
|
+
def _ensure_pane(self, agent_id: str) -> None:
|
|
507
|
+
if agent_id in self._seen or not self._in_tmux:
|
|
508
|
+
return
|
|
509
|
+
self._seen.add(agent_id) # record the attempt up front; never retry per phase
|
|
510
|
+
log = self._sink.path_for(agent_id)
|
|
511
|
+
# Seed the log so the file exists (tail opens cleanly) and the pane shows a
|
|
512
|
+
# header immediately rather than an empty box.
|
|
513
|
+
self._sink.feed(agent_id, f"══ {agent_id} ══\n".encode())
|
|
514
|
+
# Pane setup is best-effort: a tmux launch failure (fork EAGAIN, PATH
|
|
515
|
+
# mutation) must NEVER abort the run — it would surface inside the
|
|
516
|
+
# strategy's `with _monitor_streaming` block on the main thread.
|
|
517
|
+
with contextlib.suppress(Exception):
|
|
518
|
+
res = _tmux(
|
|
519
|
+
"split-window", "-d", "-P", "-F", "#{pane_id}",
|
|
520
|
+
"tail", "-n", "+1", "-F", str(log),
|
|
521
|
+
)
|
|
522
|
+
if res.returncode != 0:
|
|
523
|
+
return
|
|
524
|
+
pane_id = res.stdout.strip()
|
|
525
|
+
if pane_id:
|
|
526
|
+
_tmux("select-pane", "-t", pane_id, "-T", agent_id)
|
|
527
|
+
_tmux("select-layout", "tiled")
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def build_monitor(config: MonitorConfig, *, monitor_dir: str | None) -> Monitor | None:
|
|
531
|
+
"""The live monitor when ``monitor_dir`` is set (this is the in-tmux INNER
|
|
532
|
+
process), else ``None`` — so normal runs and the OUTER launcher never build
|
|
533
|
+
one, and monitoring is off out of the box.
|
|
534
|
+
|
|
535
|
+
Pure in its inputs (no env read — the caller passes ``inner_dir()``), mirroring
|
|
536
|
+
``build_gate``/``build_breaker``. ``config.monitor.enabled`` gates the OUTER
|
|
537
|
+
tmux launch in the CLI, not this.
|
|
538
|
+
"""
|
|
539
|
+
if not monitor_dir:
|
|
540
|
+
return None
|
|
541
|
+
return TmuxFileMonitor(Path(monitor_dir), config)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
# ── OUTER launcher: build the tmux session, run the inner, attach ──────────
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def launch_monitored_run(
|
|
548
|
+
inner_argv: list[str],
|
|
549
|
+
*,
|
|
550
|
+
session_prefix: str = "syncmill",
|
|
551
|
+
emit_patch: str | None = None,
|
|
552
|
+
report_runs_dir: str | Path | None = None,
|
|
553
|
+
) -> int:
|
|
554
|
+
"""Run ``inner_argv`` (a reconstructed ``syncmill run …``) inside tmux, attach
|
|
555
|
+
live, and return the run's exit code.
|
|
556
|
+
|
|
557
|
+
Builds a detached session whose control pane runs ``inner_argv`` through a tiny
|
|
558
|
+
``sys.executable`` exec shim (:data:`_ENV_SHIM`) that loads the outer env from a
|
|
559
|
+
``0600`` snapshot and ``execvp``s it — so the inner runs with the outer's full
|
|
560
|
+
environment (no secrets in argv) regardless of the tmux server's stale env, and
|
|
561
|
+
argv is passed DIRECTLY (no shell re-parse → no injection, works under fish/csh).
|
|
562
|
+
Then it ``tmux attach``es (foreground, with ``TMUX`` unset so it works from inside
|
|
563
|
+
tmux too). After attach returns it reads ``result.json``: present → tear the
|
|
564
|
+
session down, render, return the code; absent → either a clean detach-before-
|
|
565
|
+
completion (the run continues; print re-attach info, return 0) or a failed attach
|
|
566
|
+
(non-zero tmux rc → surface it, return non-zero).
|
|
567
|
+
|
|
568
|
+
``emit_patch`` is the ``--emit-patch`` destination: the flag is deliberately
|
|
569
|
+
NOT forwarded to the inner (``_inner_argv`` omits it, like ``--monitor``) —
|
|
570
|
+
the OUTER honors it here from result.json's ``patch`` field, so the file is
|
|
571
|
+
written relative to the user's own invocation, not inside the tmux pane.
|
|
572
|
+
|
|
573
|
+
``report_runs_dir`` is the OUTER's view of ``[integration.report] runs_dir``
|
|
574
|
+
(``None`` when reporting is off for the OUTER): after rendering the result it
|
|
575
|
+
prints the run-envelope's path — gated on the envelope actually EXISTING,
|
|
576
|
+
because the OUTER can't know whether the INNER's config had reporting enabled
|
|
577
|
+
(toml discovery is cwd-relative and the pane's cwd may differ). A plain path
|
|
578
|
+
in, the literal filename here — no ``report`` import (acyclic).
|
|
579
|
+
|
|
580
|
+
Raises :class:`MonitorLaunchError` if the session can't be created, so the CLI
|
|
581
|
+
can fall back to a normal run rather than failing the task.
|
|
582
|
+
"""
|
|
583
|
+
_sweep_stale_run_dirs() # bound the detach-early tempdir leak (best-effort)
|
|
584
|
+
try:
|
|
585
|
+
run_dir = Path(tempfile.mkdtemp(prefix="syncmill-monitor-"))
|
|
586
|
+
except OSError as exc:
|
|
587
|
+
# mkdtemp itself can fail (missing/unwritable TMPDIR) BEFORE run_dir exists —
|
|
588
|
+
# convert to MonitorLaunchError so the CLI falls back to a normal unmonitored
|
|
589
|
+
# run rather than crashing the task (the monitor is a convenience, never
|
|
590
|
+
# required). This sits OUTSIDE the setup/launch try below precisely because
|
|
591
|
+
# that block's except rmtrees run_dir, which has not been created yet here.
|
|
592
|
+
raise MonitorLaunchError(str(exc)) from exc
|
|
593
|
+
session = f"{session_prefix}-{uuid4().hex[:8]}"
|
|
594
|
+
|
|
595
|
+
# Set up + launch under one guard so ANY setup failure (disk full on the
|
|
596
|
+
# sentinel/snapshot write — OSError; a non-UTF8 env var — ValueError; the
|
|
597
|
+
# exec/arg-limit on new-session — OSError) becomes a MonitorLaunchError the CLI
|
|
598
|
+
# falls back on, rather than crashing the task. The pane runs:
|
|
599
|
+
# <outer python> -c <shim> <run_dir> <inner argv…> — sys.executable is an
|
|
600
|
+
# absolute interpreter path (resolves regardless of the pane's PATH); the shim
|
|
601
|
+
# restores the outer env (incl. PATH) and execs the real `syncmill`.
|
|
602
|
+
# The run's stable id — the OUTER mints it because it never builds a Supervisor
|
|
603
|
+
# (the resolver's "fresh" path, performed here on the inner's behalf); the inner's
|
|
604
|
+
# `resolve_run_id` picks it up from the snapshot env, gated on `inner_dir()`.
|
|
605
|
+
run_id = str(uuid4())
|
|
606
|
+
try:
|
|
607
|
+
(run_dir / SENTINEL_NAME).write_text("") # mark the inner process
|
|
608
|
+
# outer env the shim restores (0600, secrets safe) + the minted run_id
|
|
609
|
+
_write_env_snapshot(run_dir, run_id)
|
|
610
|
+
shim_cmd = [sys.executable, "-c", _ENV_SHIM, str(run_dir), *inner_argv]
|
|
611
|
+
created = _tmux(
|
|
612
|
+
"new-session", "-d", "-s", session, "-x", "220", "-y", "50", *shim_cmd
|
|
613
|
+
)
|
|
614
|
+
except (OSError, ValueError) as exc:
|
|
615
|
+
shutil.rmtree(run_dir, ignore_errors=True)
|
|
616
|
+
raise MonitorLaunchError(str(exc)) from exc
|
|
617
|
+
if created.returncode != 0:
|
|
618
|
+
shutil.rmtree(run_dir, ignore_errors=True)
|
|
619
|
+
raise command_error(MonitorLaunchError, "tmux new-session failed", created.stderr)
|
|
620
|
+
|
|
621
|
+
# Visible pane titles (best-effort; harmless if the user's tmux overrides it).
|
|
622
|
+
_tmux("set-option", "-t", session, "pane-border-status", "top")
|
|
623
|
+
_tmux("set-option", "-t", session, "pane-border-format", " #{pane_title} ")
|
|
624
|
+
_tmux("select-pane", "-t", session, "-T", "syncmill (control)")
|
|
625
|
+
|
|
626
|
+
attach_rc = _tmux_attach(session) # FOREGROUND — blocks until the user detaches
|
|
627
|
+
|
|
628
|
+
data = _await_result(run_dir)
|
|
629
|
+
if data is not None:
|
|
630
|
+
_tmux("kill-session", "-t", session)
|
|
631
|
+
code = render_result(data)
|
|
632
|
+
if emit_patch is not None and not emit_patch_file(emit_patch, data):
|
|
633
|
+
code = code or 1 # explicit export failed — never exit 0
|
|
634
|
+
if report_runs_dir is not None:
|
|
635
|
+
# The INNER (the real Supervisor) wrote the envelope under the
|
|
636
|
+
# OUTER-minted run_id; existence is the only truthful enablement gate.
|
|
637
|
+
# Best-effort like every report surface: expanduser raises RuntimeError
|
|
638
|
+
# with no HOME, exists() can raise OSError — neither may abort the
|
|
639
|
+
# monitored run's exit path or skip the run-dir cleanup below.
|
|
640
|
+
try:
|
|
641
|
+
envelope = (
|
|
642
|
+
Path(report_runs_dir).expanduser() / run_id / "envelope.json"
|
|
643
|
+
)
|
|
644
|
+
if envelope.exists():
|
|
645
|
+
echo(f"[syncmill] report: {printable_name(str(envelope))}")
|
|
646
|
+
except (OSError, RuntimeError, ValueError):
|
|
647
|
+
secho(
|
|
648
|
+
"[syncmill] report path unavailable — continuing", fg="yellow"
|
|
649
|
+
)
|
|
650
|
+
shutil.rmtree(run_dir, ignore_errors=True)
|
|
651
|
+
return code
|
|
652
|
+
|
|
653
|
+
# No result yet — the run continues (or attach failed); either way there is no
|
|
654
|
+
# patch to export yet. Point at result.json's `patch` field so a detach-early
|
|
655
|
+
# user can still extract it once the inner finishes. The hint must be a
|
|
656
|
+
# byte-preserving extractor (NOT `jq -r`, which replaces the JSON-escaped
|
|
657
|
+
# surrogateescape bytes of a non-UTF-8 patch with U+FFFD, silently corrupting
|
|
658
|
+
# it) — mirror emit_patch_file's surrogateescape write exactly.
|
|
659
|
+
if emit_patch is not None:
|
|
660
|
+
source, dest = str(run_dir / _RESULT_FILE), emit_patch
|
|
661
|
+
source_arg, dest_arg = shell_arg(source), shell_arg(dest)
|
|
662
|
+
if source_arg and dest_arg:
|
|
663
|
+
extractor = (
|
|
664
|
+
"python3 -c 'import json,sys; "
|
|
665
|
+
'open(sys.argv[2],"w",encoding="utf-8",errors="surrogateescape")'
|
|
666
|
+
'.write(json.load(open(sys.argv[1]))["patch"] or "")\' '
|
|
667
|
+
f"{source_arg} {dest_arg}"
|
|
668
|
+
)
|
|
669
|
+
secho(
|
|
670
|
+
"[syncmill] --emit-patch: no result yet — once finished, extract with: "
|
|
671
|
+
f"{extractor}",
|
|
672
|
+
fg="yellow",
|
|
673
|
+
)
|
|
674
|
+
else:
|
|
675
|
+
secho(
|
|
676
|
+
"[syncmill] --emit-patch: no result yet — once finished, write the "
|
|
677
|
+
f"`patch` field of {printable_name(source)} to {printable_name(dest)} "
|
|
678
|
+
f"({NO_COMMAND})",
|
|
679
|
+
fg="yellow",
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
# Distinguish a failed attach from a clean detach-before-finish. The session
|
|
683
|
+
# name carries the configured prefix and the result path the temp root.
|
|
684
|
+
reattach = printable_command("tmux", "attach", options=["-t", session])
|
|
685
|
+
shown_result = printable_name(str(run_dir / _RESULT_FILE))
|
|
686
|
+
if attach_rc != 0:
|
|
687
|
+
secho(
|
|
688
|
+
f"[syncmill] could not attach to the live monitor (tmux exit {attach_rc}) "
|
|
689
|
+
f"— the run continues in tmux.\n"
|
|
690
|
+
f" re-attach: {reattach}\n"
|
|
691
|
+
f" result: {shown_result}",
|
|
692
|
+
fg="red",
|
|
693
|
+
)
|
|
694
|
+
return 1
|
|
695
|
+
secho(
|
|
696
|
+
"[syncmill] detached before the run finished — it continues in tmux.\n"
|
|
697
|
+
f" re-attach: {reattach}\n"
|
|
698
|
+
f" result: {shown_result}",
|
|
699
|
+
fg="yellow",
|
|
700
|
+
)
|
|
701
|
+
return 0
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
#: A monitor run dir older than this is assumed orphaned (a detach-before-completion
|
|
705
|
+
#: that nobody reaped) and swept on the next launch — far longer than any real run,
|
|
706
|
+
#: so a still-attached/running inner's dir is never touched.
|
|
707
|
+
_STALE_RUN_DIR_SECONDS = 24 * 3600
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _sweep_stale_run_dirs() -> None:
|
|
711
|
+
"""Remove orphaned ``syncmill-monitor-*`` dirs from prior detach-early runs.
|
|
712
|
+
|
|
713
|
+
The detach-before-completion path deliberately leaves its run dir so the still-
|
|
714
|
+
running inner can finish writing ``result.json``; nothing else reaps it. This
|
|
715
|
+
age-bounded sweep keeps that from accumulating in ``/tmp`` without a GC registry
|
|
716
|
+
(and without risking a live run — the threshold dwarfs any real run). Fully
|
|
717
|
+
best-effort: any error is ignored.
|
|
718
|
+
|
|
719
|
+
Requires the inner ``SENTINEL_NAME`` file before removing anything — the prefix
|
|
720
|
+
match alone would rmtree any same-prefixed dir syncmill didn't create, so the
|
|
721
|
+
sentinel (written into every real run dir at launch) is the proof of ownership.
|
|
722
|
+
"""
|
|
723
|
+
tmp = Path(tempfile.gettempdir())
|
|
724
|
+
try:
|
|
725
|
+
entries = list(tmp.glob("syncmill-monitor-*"))
|
|
726
|
+
except OSError:
|
|
727
|
+
return
|
|
728
|
+
for entry in entries:
|
|
729
|
+
with contextlib.suppress(OSError):
|
|
730
|
+
if (
|
|
731
|
+
entry.is_dir()
|
|
732
|
+
and (entry / SENTINEL_NAME).is_file()
|
|
733
|
+
and time.time() - entry.stat().st_mtime > _STALE_RUN_DIR_SECONDS
|
|
734
|
+
):
|
|
735
|
+
shutil.rmtree(entry, ignore_errors=True)
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _write_env_snapshot(run_dir: Path, run_id: str) -> None:
|
|
739
|
+
"""Write the OUTER process's environment to ``<run_dir>/env`` (mode ``0600``).
|
|
740
|
+
|
|
741
|
+
The exec shim (:data:`_ENV_SHIM`) loads this so the in-tmux INNER runs with the
|
|
742
|
+
outer's environment — load-bearing because a new tmux session inherits the
|
|
743
|
+
(possibly stale) tmux **server** env, not the outer client's, so a venv
|
|
744
|
+
``syncmill``, the user's PATH, their ``SYNCMILL_*`` config, and agent auth would
|
|
745
|
+
otherwise be invisible in the pane. A FILE (not ``-e`` argv) keeps secrets out
|
|
746
|
+
of the tmux command-line and the launch argv tiny. The tmux-owned vars (TERM,
|
|
747
|
+
TMUX, TMUX_PANE) are excluded so the pane keeps its own; ``SYNCMILL_MONITOR_DIR``
|
|
748
|
+
is added so the inner detects itself, and ``SYNCMILL_RUN_ID`` carries the
|
|
749
|
+
OUTER-minted run_id to the inner's ``resolve_run_id`` (assigned AFTER the
|
|
750
|
+
comprehension, so a stale exported ``SYNCMILL_RUN_ID`` in the outer's own env
|
|
751
|
+
is overwritten, never forwarded).
|
|
752
|
+
"""
|
|
753
|
+
snapshot = {
|
|
754
|
+
k: v for k, v in os.environ.items() if k not in _SNAPSHOT_SKIP_ENV
|
|
755
|
+
}
|
|
756
|
+
snapshot[INNER_ENV] = str(run_dir)
|
|
757
|
+
snapshot[RUN_ID_ENV] = run_id
|
|
758
|
+
fd = os.open(
|
|
759
|
+
str(run_dir / "env"), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600
|
|
760
|
+
)
|
|
761
|
+
with os.fdopen(fd, "w") as fh:
|
|
762
|
+
json.dump(snapshot, fh)
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def _await_result(run_dir: Path) -> dict | None:
|
|
766
|
+
"""Poll for the inner's ``result.json`` for a short grace window, then give up
|
|
767
|
+
(the user detached before it finished)."""
|
|
768
|
+
deadline = time.monotonic() + _RESULT_GRACE_SECONDS
|
|
769
|
+
while True:
|
|
770
|
+
data = read_result(run_dir)
|
|
771
|
+
if data is not None:
|
|
772
|
+
return data
|
|
773
|
+
if time.monotonic() >= deadline:
|
|
774
|
+
return None
|
|
775
|
+
time.sleep(_RESULT_POLL_INTERVAL)
|