syncmill 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syncmill/__init__.py +11 -0
- syncmill/agents.py +2064 -0
- syncmill/artifact_io.py +201 -0
- syncmill/artifacts.py +1946 -0
- syncmill/board.py +2060 -0
- syncmill/breaker.py +327 -0
- syncmill/cli.py +4986 -0
- syncmill/command_errors.py +32 -0
- syncmill/config.py +1118 -0
- syncmill/dashboard.py +935 -0
- syncmill/gates.py +179 -0
- syncmill/gateway.py +234 -0
- syncmill/git_env.py +99 -0
- syncmill/isolated_session.py +431 -0
- syncmill/matcher.py +127 -0
- syncmill/memory.py +1067 -0
- syncmill/memory_backend.py +1907 -0
- syncmill/memory_migration.py +962 -0
- syncmill/model_routing.py +482 -0
- syncmill/models.py +244 -0
- syncmill/monitor.py +775 -0
- syncmill/phoenix_exporter.py +339 -0
- syncmill/planning.py +512 -0
- syncmill/platform_support.py +14 -0
- syncmill/preflight.py +1129 -0
- syncmill/project.py +1067 -0
- syncmill/report.py +502 -0
- syncmill/review_candidates.py +107 -0
- syncmill/room_compare.py +2380 -0
- syncmill/room_raw.py +484 -0
- syncmill/room_viewer.py +604 -0
- syncmill/rooms.py +4044 -0
- syncmill/session.py +444 -0
- syncmill/strategies/__init__.py +82 -0
- syncmill/strategies/base.py +910 -0
- syncmill/strategies/compete.py +233 -0
- syncmill/strategies/council.py +429 -0
- syncmill/strategies/decompose.py +570 -0
- syncmill/strategies/pipeline.py +221 -0
- syncmill/strategies/route.py +130 -0
- syncmill/supervisor.py +1423 -0
- syncmill/teamfile.py +571 -0
- syncmill/telemetry_hook.py +231 -0
- syncmill/terminal.py +245 -0
- syncmill/tracer.py +548 -0
- syncmill/tui.py +3537 -0
- syncmill/tui_collaboration.py +499 -0
- syncmill/tui_jobs.py +1407 -0
- syncmill/tui_render.py +156 -0
- syncmill/workflow.py +3746 -0
- syncmill/worktree.py +1045 -0
- syncmill-0.2.0.dist-info/METADATA +853 -0
- syncmill-0.2.0.dist-info/RECORD +56 -0
- syncmill-0.2.0.dist-info/WHEEL +4 -0
- syncmill-0.2.0.dist-info/entry_points.txt +2 -0
- syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
syncmill/supervisor.py
ADDED
|
@@ -0,0 +1,1423 @@
|
|
|
1
|
+
"""The Supervisor — the sole mediator between agents and memory.
|
|
2
|
+
|
|
3
|
+
Agents don't know about each other or about memory; the Supervisor wires them
|
|
4
|
+
to a ``Strategy`` (the orchestration topology — see ``syncmill.strategies``) and
|
|
5
|
+
to ``SyncmillMemory``. Strategies were extracted into the ``strategies`` package
|
|
6
|
+
(#12) once the four of them made this module unwieldy; the Supervisor keeps only
|
|
7
|
+
the run lifecycle, strategy selection, and the capability re-ordering it applies
|
|
8
|
+
before dispatch.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
import logging
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from uuid import uuid4
|
|
19
|
+
|
|
20
|
+
from syncmill.platform_support import require_supported_platform
|
|
21
|
+
from syncmill.agents import (
|
|
22
|
+
AgentRegistry,
|
|
23
|
+
AgentRunner,
|
|
24
|
+
capture_tool_calls,
|
|
25
|
+
model_routing_scope,
|
|
26
|
+
provider_session_scope,
|
|
27
|
+
)
|
|
28
|
+
from syncmill.artifacts import ApplyLockBusy, ArtifactStore, _cross_process_lock, printable_name
|
|
29
|
+
from syncmill.board import (
|
|
30
|
+
Board,
|
|
31
|
+
BoardError,
|
|
32
|
+
WorkState,
|
|
33
|
+
build_board,
|
|
34
|
+
parse_board_update,
|
|
35
|
+
render_item_prompt,
|
|
36
|
+
strip_board_block,
|
|
37
|
+
)
|
|
38
|
+
from syncmill.breaker import CircuitBreaker, build_breaker
|
|
39
|
+
from syncmill.config import SyncmillConfig
|
|
40
|
+
from syncmill.gates import build_gate
|
|
41
|
+
from syncmill.matcher import CapabilityMatcher, build_matcher
|
|
42
|
+
from syncmill.memory import SyncmillMemory, run_async, run_memory_bounded
|
|
43
|
+
from syncmill.model_routing import (
|
|
44
|
+
AllUnmappedError,
|
|
45
|
+
ClassifierError,
|
|
46
|
+
ModelRoutingDecision,
|
|
47
|
+
build_decision,
|
|
48
|
+
inherit_decision,
|
|
49
|
+
)
|
|
50
|
+
from syncmill.models import Task, TaskResult, TaskStatus
|
|
51
|
+
from syncmill.monitor import build_monitor, inner_dir
|
|
52
|
+
from syncmill.preflight import run_preflight
|
|
53
|
+
from syncmill.planning import build_control_plan, build_run_plan
|
|
54
|
+
from syncmill.project import project_id
|
|
55
|
+
from syncmill.report import (
|
|
56
|
+
RunReporter,
|
|
57
|
+
build_reporter,
|
|
58
|
+
patch_envelope_board,
|
|
59
|
+
resolve_run_id,
|
|
60
|
+
sweep_runs_dir,
|
|
61
|
+
)
|
|
62
|
+
from syncmill.strategies import Strategy, _make_strategy
|
|
63
|
+
from syncmill.tracer import build_tracer, flush_trace, run_span
|
|
64
|
+
from syncmill.worktree import (
|
|
65
|
+
PatchCaptureError,
|
|
66
|
+
WorktreePool,
|
|
67
|
+
WorkspaceError,
|
|
68
|
+
capture_repository_snapshot,
|
|
69
|
+
)
|
|
70
|
+
from syncmill.workflow import WorkflowCoordinator, WorkflowRecord, WorkflowStore
|
|
71
|
+
|
|
72
|
+
logger = logging.getLogger(__name__)
|
|
73
|
+
|
|
74
|
+
_SCRATCH_GC_BATCH_SIZE = 100
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _LeaseHeartbeat:
|
|
78
|
+
"""Renew a real Board claim from a connection owned by this thread."""
|
|
79
|
+
|
|
80
|
+
def __init__(
|
|
81
|
+
self, board: Board, item_id: str, run_id: str, interval: float | None = None
|
|
82
|
+
):
|
|
83
|
+
self._board = board
|
|
84
|
+
self._item_id = item_id
|
|
85
|
+
self._run_id = run_id
|
|
86
|
+
default_interval = max(0.25, min(60.0, board.lease_timeout_seconds / 3))
|
|
87
|
+
self._interval = default_interval if interval is None else max(0.001, interval)
|
|
88
|
+
self._stop = threading.Event()
|
|
89
|
+
self._renewed = threading.Event()
|
|
90
|
+
self._started = False
|
|
91
|
+
self._warning: str | None = None
|
|
92
|
+
self._thread = threading.Thread(
|
|
93
|
+
target=self._run,
|
|
94
|
+
name=f"syncmill-lease-{item_id[:8]}",
|
|
95
|
+
daemon=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def start(self) -> None:
|
|
99
|
+
self._thread.start()
|
|
100
|
+
self._started = True
|
|
101
|
+
|
|
102
|
+
def stop(self) -> None:
|
|
103
|
+
self._stop.set()
|
|
104
|
+
# A never-started thread can't be joined (RuntimeError). `start()` is inside
|
|
105
|
+
# the caller's crash guard, so the failed-to-start path reaches this stop().
|
|
106
|
+
if not self._started:
|
|
107
|
+
return
|
|
108
|
+
self._thread.join(timeout=max(1.0, min(5.0, self._interval + 0.5)))
|
|
109
|
+
|
|
110
|
+
def wait_for_renewal(self, timeout: float) -> bool:
|
|
111
|
+
return self._renewed.wait(timeout)
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def warning(self) -> str | None:
|
|
115
|
+
"""Heartbeat health: why renewal stopped, or ``None``.
|
|
116
|
+
|
|
117
|
+
Set when the claim was gone on a renewal (the item's state or lease owner
|
|
118
|
+
changed under us) or when renewal itself broke. This is NOT a verdict on the
|
|
119
|
+
run's board write — a renewal that merely broke transiently can still end
|
|
120
|
+
with the fenced write landing. ``apply_update``'s ``applied`` decides that;
|
|
121
|
+
this only supplies the reason when it comes back False.
|
|
122
|
+
"""
|
|
123
|
+
return self._warning
|
|
124
|
+
|
|
125
|
+
#: Consecutive renewal failures tolerated before the heartbeat gives up. A
|
|
126
|
+
#: single transient sqlite error (a concurrent BEGIN IMMEDIATE held past
|
|
127
|
+
#: busy_timeout) must not silently kill the heartbeat for the rest of a long
|
|
128
|
+
#: run — the lease would expire under a HEALTHY run, stale recovery would
|
|
129
|
+
#: reclaim it, and a second runner would redo (and fence-drop) the work.
|
|
130
|
+
#: Retries reconnect (a broken connection stays broken) and are paced by
|
|
131
|
+
#: ``_RETRY_DELAY_SECONDS``, so persistent breakage still surfaces within
|
|
132
|
+
#: well under one lease timeout.
|
|
133
|
+
_MAX_RENEWAL_FAILURES = 5
|
|
134
|
+
_RETRY_DELAY_SECONDS = 5.0
|
|
135
|
+
|
|
136
|
+
def _record_failure(self, exc: Exception, failures: int) -> None:
|
|
137
|
+
logger.warning(
|
|
138
|
+
"board lease renewal failed (attempt %d/%d)",
|
|
139
|
+
failures,
|
|
140
|
+
self._MAX_RENEWAL_FAILURES,
|
|
141
|
+
exc_info=True,
|
|
142
|
+
)
|
|
143
|
+
# Provisional: a later successful renewal CLEARS it (the lease survived
|
|
144
|
+
# — no false alarm), while a stop() before recovery leaves it standing
|
|
145
|
+
# so the caller still learns renewal was broken when the run ended.
|
|
146
|
+
self._warning = (
|
|
147
|
+
f"board lease renewal failed for item {printable_name(str(self._item_id))} "
|
|
148
|
+
f"({printable_name(f'{type(exc).__name__}: {exc}')}) — the lease may "
|
|
149
|
+
f"have expired"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def _run(self) -> None:
|
|
153
|
+
conn: Board | None = None
|
|
154
|
+
failures = 0
|
|
155
|
+
try:
|
|
156
|
+
# Prime the connection eagerly (the pre-retry behavior): a board
|
|
157
|
+
# that is broken from the start surfaces on the first tick's fast
|
|
158
|
+
# retry schedule instead of a full interval later.
|
|
159
|
+
try:
|
|
160
|
+
conn = self._board.new_connection()
|
|
161
|
+
except Exception as exc:
|
|
162
|
+
failures = 1
|
|
163
|
+
self._record_failure(exc, failures)
|
|
164
|
+
while True:
|
|
165
|
+
delay = (
|
|
166
|
+
min(self._RETRY_DELAY_SECONDS, self._interval)
|
|
167
|
+
if failures
|
|
168
|
+
else self._interval
|
|
169
|
+
)
|
|
170
|
+
if self._stop.wait(delay):
|
|
171
|
+
return
|
|
172
|
+
try:
|
|
173
|
+
if conn is None:
|
|
174
|
+
conn = self._board.new_connection()
|
|
175
|
+
renewed = conn.renew_claim(self._item_id, self._run_id)
|
|
176
|
+
except Exception as exc:
|
|
177
|
+
failures += 1
|
|
178
|
+
self._record_failure(exc, failures)
|
|
179
|
+
# The connection may be what broke — rebuild it on retry.
|
|
180
|
+
if conn is not None:
|
|
181
|
+
with contextlib.suppress(Exception):
|
|
182
|
+
conn.close()
|
|
183
|
+
conn = None
|
|
184
|
+
if failures >= self._MAX_RENEWAL_FAILURES:
|
|
185
|
+
return
|
|
186
|
+
continue
|
|
187
|
+
if not renewed:
|
|
188
|
+
# The renewal CAS matched no row: the item left `running` or its
|
|
189
|
+
# lease owner changed. Stale recovery + re-dispatch is the usual
|
|
190
|
+
# cause, but a manual transition does it too — so name what was
|
|
191
|
+
# observed, not a mechanism that may not have happened.
|
|
192
|
+
logger.warning(
|
|
193
|
+
"board: lost the lease on item %s (run %s) — its state or "
|
|
194
|
+
"lease owner changed while this run was working",
|
|
195
|
+
self._item_id, self._run_id,
|
|
196
|
+
)
|
|
197
|
+
self._warning = (
|
|
198
|
+
f"lost the board lease on item {printable_name(str(self._item_id))} "
|
|
199
|
+
f"— its state or "
|
|
200
|
+
f"lease owner changed while this run was working"
|
|
201
|
+
)
|
|
202
|
+
return
|
|
203
|
+
failures = 0
|
|
204
|
+
self._warning = None # a successful renewal clears provisional trouble
|
|
205
|
+
self._renewed.set()
|
|
206
|
+
finally:
|
|
207
|
+
if conn is not None:
|
|
208
|
+
conn.close()
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _matched_order(
|
|
212
|
+
agents: list[AgentRunner], task: Task, matcher: CapabilityMatcher | None
|
|
213
|
+
) -> list[AgentRunner]:
|
|
214
|
+
"""Re-order agents by capability fit for this task; identity when no matcher.
|
|
215
|
+
|
|
216
|
+
Capability matching (#9) layered OVER priority: a matching agent moves ahead
|
|
217
|
+
of raw ``agent_priority``, and equal-fit agents keep priority order (the
|
|
218
|
+
matcher's sort is stable). Returns a permutation — it never drops an agent, so
|
|
219
|
+
Route's fallback chain / Pipeline's stages / Compete's competitors / Council's
|
|
220
|
+
members / Decompose's workers stay intact and the ``if not agents`` guard still
|
|
221
|
+
covers the empty case.
|
|
222
|
+
|
|
223
|
+
Applied ONCE in the Supervisor (not threaded into ``Strategy.execute`` like
|
|
224
|
+
the gate/breaker) because re-ordering has no topology-specific meaning: every
|
|
225
|
+
strategy consumes the candidate list as a priority sequence, so re-ordering at
|
|
226
|
+
the source makes all five capability-aware with no strategy change.
|
|
227
|
+
"""
|
|
228
|
+
if matcher is None:
|
|
229
|
+
return list(agents)
|
|
230
|
+
return matcher.order(agents, task)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _crash_result(task: Task, run_id: str | None = None) -> TaskResult:
|
|
234
|
+
"""A FAILED result standing in for an orchestration that raised before
|
|
235
|
+
producing one — so a monitored run's OUTER reads a real failure (and a
|
|
236
|
+
non-zero exit) instead of mistaking the crash for a clean detach. Carries
|
|
237
|
+
``run_id`` like the success path, so consumers (the CLI's `run: <id>`
|
|
238
|
+
line) can still name the run whose artifacts to inspect."""
|
|
239
|
+
return TaskResult(
|
|
240
|
+
task_id=task.id,
|
|
241
|
+
agent_id="none",
|
|
242
|
+
status=TaskStatus.FAILED,
|
|
243
|
+
output="",
|
|
244
|
+
exit_code=1,
|
|
245
|
+
duration_seconds=0.0,
|
|
246
|
+
error="orchestration crashed before producing a result",
|
|
247
|
+
run_id=run_id,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _preflight_block_result(
|
|
252
|
+
task: Task, reasons: tuple[str, ...], run_id: str | None = None
|
|
253
|
+
) -> TaskResult:
|
|
254
|
+
rendered = ", ".join(reasons) or "policy rejected the run"
|
|
255
|
+
return TaskResult(
|
|
256
|
+
task_id=task.id,
|
|
257
|
+
agent_id="preflight",
|
|
258
|
+
status=TaskStatus.FAILED,
|
|
259
|
+
output="",
|
|
260
|
+
exit_code=1,
|
|
261
|
+
duration_seconds=0.0,
|
|
262
|
+
error=f"strict toolgraph preflight blocked execution: {rendered}",
|
|
263
|
+
run_id=run_id,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _degraded_sources(mem: SyncmillMemory, breaker: CircuitBreaker | None) -> list[str]:
|
|
268
|
+
"""The envelope's ``degraded`` array: which best-effort subsystems degraded
|
|
269
|
+
during this run. ``mem`` is the RUN's memory (the board-scoped view on a
|
|
270
|
+
board run), not necessarily ``self._memory``. Open vocabulary — v1 knows
|
|
271
|
+
memory and the breaker (the two carriers of a ``degraded`` flag)."""
|
|
272
|
+
sources = []
|
|
273
|
+
if mem.degraded:
|
|
274
|
+
sources.append("memory")
|
|
275
|
+
if breaker is not None and breaker.degraded:
|
|
276
|
+
sources.append("breaker")
|
|
277
|
+
return sources
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _append_trace_degradation(degraded: list[str], trace: dict | None) -> None:
|
|
281
|
+
"""Add file and streaming trace failures to the open degraded vocabulary."""
|
|
282
|
+
if trace is None:
|
|
283
|
+
return
|
|
284
|
+
if trace.get("status") == "error":
|
|
285
|
+
degraded.append("trace")
|
|
286
|
+
stream = trace.get("stream")
|
|
287
|
+
if isinstance(stream, dict) and stream.get("status") == "error":
|
|
288
|
+
degraded.append("phoenix")
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class Supervisor:
|
|
292
|
+
"""Central orchestrator. The only component that touches both agents and memory."""
|
|
293
|
+
|
|
294
|
+
def __init__(self, config: SyncmillConfig):
|
|
295
|
+
require_supported_platform()
|
|
296
|
+
self._config = config
|
|
297
|
+
self._registry = AgentRegistry(config)
|
|
298
|
+
self._memory = SyncmillMemory(config.memory)
|
|
299
|
+
# Built once (like memory) because it carries cross-run state loaded from
|
|
300
|
+
# disk and a `degraded` flag the CLI surfaces; None when disabled, so the
|
|
301
|
+
# default path never touches the filesystem. (The gate, being stateless,
|
|
302
|
+
# is built per-run inside `run`.)
|
|
303
|
+
self._breaker = build_breaker(config.breaker)
|
|
304
|
+
# Capability matcher (#9) — None when no agent declares a capability (the
|
|
305
|
+
# default), so ordering stays at raw priority. Stateless/config-derived,
|
|
306
|
+
# so it's built once here; the known agent ids surface a config typo.
|
|
307
|
+
self._matcher = build_matcher(config.capabilities, config.agent_priority)
|
|
308
|
+
|
|
309
|
+
# The work-board (Phase 1) is LOAD-BEARING and built LAZILY — NOT here, so
|
|
310
|
+
# an enabled-but-corrupt board can never break a plain `run` (which builds a
|
|
311
|
+
# Supervisor but never touches `run_item`). `_require_board` opens it on
|
|
312
|
+
# first board use; `close()` tears it down.
|
|
313
|
+
self._board: Board | None = None
|
|
314
|
+
# Native workflows are lazy like the board: a plain run must not create
|
|
315
|
+
# workflow state or open SQLite merely because the feature exists.
|
|
316
|
+
self._workflow: WorkflowCoordinator | None = None
|
|
317
|
+
|
|
318
|
+
self._strategy: Strategy = _make_strategy(config.default_strategy)
|
|
319
|
+
# Where the last run's envelope landed (None when reporting is disabled
|
|
320
|
+
# or the write degraded) — the CLI's read seam for the report-path line,
|
|
321
|
+
# like `supervisor.memory` / `supervisor.breaker`.
|
|
322
|
+
self.last_report_path: Path | None = None
|
|
323
|
+
# The last run's advisory-preflight warnings — the CLI surfaces them as
|
|
324
|
+
# yellow notices (the memory.degraded pattern). Reset per run.
|
|
325
|
+
self.preflight_warnings: list[str] = []
|
|
326
|
+
self.last_preflight_blocked = False
|
|
327
|
+
self.last_run_id: str | None = None
|
|
328
|
+
self.last_artifact_path: Path | None = None
|
|
329
|
+
self.recovery_workspace: Path | None = None
|
|
330
|
+
# The last run's model-routing decision (None when routing was off/
|
|
331
|
+
# disabled for the run) + the classifier-degrade warning — the CLI's
|
|
332
|
+
# read seams for the compact banner and the yellow degrade notice.
|
|
333
|
+
self.last_routing_decision: ModelRoutingDecision | None = None
|
|
334
|
+
self.last_routing_warning: str | None = None
|
|
335
|
+
# Set by `run_item` when a board run's work was discarded (its lease was
|
|
336
|
+
# lost / its fenced write dropped) — the CLI's read seam for that notice.
|
|
337
|
+
# Reset per `run_item`, not per `run`: a plain run never touches the board.
|
|
338
|
+
self.last_board_warning: str | None = None
|
|
339
|
+
|
|
340
|
+
def _finalize_early_failure(
|
|
341
|
+
self, failed: TaskResult, reporter: RunReporter | None
|
|
342
|
+
) -> TaskResult:
|
|
343
|
+
"""Close the observability surfaces for a failure that aborts a run before
|
|
344
|
+
the strategy starts, then hand the result back.
|
|
345
|
+
|
|
346
|
+
Both surfaces are observability, so both are guarded: a report or monitor
|
|
347
|
+
failure must never replace the real cause with an unrelated traceback. The
|
|
348
|
+
monitor is the load-bearing half — a monitored INNER that returns without
|
|
349
|
+
writing result.json leaves the OUTER reporting an attach error instead of
|
|
350
|
+
this failure (the same discipline as the routing- and strict-preflight-
|
|
351
|
+
failure paths).
|
|
352
|
+
"""
|
|
353
|
+
if reporter is not None:
|
|
354
|
+
try:
|
|
355
|
+
self.last_report_path = reporter.finish(failed, degraded=[])
|
|
356
|
+
except Exception:
|
|
357
|
+
logger.warning(
|
|
358
|
+
"early-failure report finalization failed", exc_info=True
|
|
359
|
+
)
|
|
360
|
+
monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
|
|
361
|
+
if monitor is not None:
|
|
362
|
+
try:
|
|
363
|
+
monitor.finish(failed)
|
|
364
|
+
except Exception:
|
|
365
|
+
logger.warning(
|
|
366
|
+
"early-failure monitor finalization failed", exc_info=True
|
|
367
|
+
)
|
|
368
|
+
return failed
|
|
369
|
+
|
|
370
|
+
def run(
|
|
371
|
+
self,
|
|
372
|
+
task: Task,
|
|
373
|
+
agent_id: str | None = None,
|
|
374
|
+
exclude: set[str] | None = None,
|
|
375
|
+
memory: SyncmillMemory | None = None,
|
|
376
|
+
run_id: str | None = None,
|
|
377
|
+
resume_session_id: str | None = None,
|
|
378
|
+
parent_run_id: str | None = None,
|
|
379
|
+
seed_patch: bytes | None = None,
|
|
380
|
+
fork_session: bool = False,
|
|
381
|
+
difficulty: str | None = None,
|
|
382
|
+
inherited_routing: dict | None = None,
|
|
383
|
+
enable_routing: bool = True,
|
|
384
|
+
) -> TaskResult:
|
|
385
|
+
"""Execute a task through the orchestration pipeline.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
task: The task to execute.
|
|
389
|
+
agent_id: If specified, force this agent only (no fallback).
|
|
390
|
+
exclude: Agent ids to drop from the priority fallback — e.g. run one
|
|
391
|
+
agent interactively yourself while syncmill orchestrates the
|
|
392
|
+
rest. Forcing an excluded agent via ``agent_id`` is an error.
|
|
393
|
+
memory: Override the memory used for THIS run (default ``self._memory``).
|
|
394
|
+
``run_item`` passes a board-scoped view so a board run's session
|
|
395
|
+
prompt-log AND every strategy-internal share/search stay off the
|
|
396
|
+
global ``shared`` namespace (decision #6). Plain ``run`` passes
|
|
397
|
+
``None`` → behavior-identical to before.
|
|
398
|
+
run_id: The run's stable identity (the run-envelope join key),
|
|
399
|
+
resolved through the single ``resolve_run_id`` seam: ``run_item``
|
|
400
|
+
passes its board lease/scope id (one identity, no double
|
|
401
|
+
issuance); a monitored inner picks up the OUTER-minted id from
|
|
402
|
+
the snapshot env; a direct run mints a fresh uuid4.
|
|
403
|
+
difficulty: The ``--difficulty`` request (``auto|low|medium|high``)
|
|
404
|
+
for model routing; ``None`` defers to ``config.model_routing``.
|
|
405
|
+
inherited_routing: A continuation's parent ``plan.model_routing``
|
|
406
|
+
block (selector affinity). Only consulted when
|
|
407
|
+
``parent_run_id`` marks this run as a continuation — a legacy
|
|
408
|
+
parent (``None``) then resumes with no model flag even under
|
|
409
|
+
``mode=observe``.
|
|
410
|
+
enable_routing: ``run_item`` passes ``False`` — board runs are a
|
|
411
|
+
PR-2 surface, and an observe-mode config must not classify the
|
|
412
|
+
enriched board prompt (raw ``item.task`` is the contract input).
|
|
413
|
+
"""
|
|
414
|
+
# The one identity every artifact of this run shares — the log, and (when
|
|
415
|
+
# [integration.report] is enabled) the run-envelope under runs_dir/{run_id}.
|
|
416
|
+
run_id = resolve_run_id(run_id)
|
|
417
|
+
self.last_run_id = run_id
|
|
418
|
+
self.last_artifact_path = None
|
|
419
|
+
self.recovery_workspace = None
|
|
420
|
+
logger.info("run %s: task %s", run_id, task.id)
|
|
421
|
+
# Reset up front so an early argument-error return can't leave a PRIOR
|
|
422
|
+
# run's report path / preflight warnings dangling on this Supervisor.
|
|
423
|
+
self.last_report_path = None
|
|
424
|
+
self.preflight_warnings = []
|
|
425
|
+
self.last_preflight_blocked = False
|
|
426
|
+
self.last_routing_decision = None
|
|
427
|
+
self.last_routing_warning = None
|
|
428
|
+
# Use `mem` for EVERY memory touchpoint below — register, session start, the
|
|
429
|
+
# strategy's `memory=`, the success share, and session end — so a board run
|
|
430
|
+
# (which passes a scoped view) can't leak the enriched prompt through any of
|
|
431
|
+
# them via `self._memory`.
|
|
432
|
+
mem = memory or self._memory
|
|
433
|
+
exclude = exclude or set()
|
|
434
|
+
# Determine which agents to use
|
|
435
|
+
if agent_id:
|
|
436
|
+
if agent_id in exclude:
|
|
437
|
+
return TaskResult(
|
|
438
|
+
task_id=task.id,
|
|
439
|
+
agent_id=agent_id,
|
|
440
|
+
status=TaskStatus.FAILED,
|
|
441
|
+
output="",
|
|
442
|
+
exit_code=-1,
|
|
443
|
+
duration_seconds=0.0,
|
|
444
|
+
error=f"Agent '{agent_id}' is both forced (agent_id) and excluded",
|
|
445
|
+
)
|
|
446
|
+
agent = self._registry.get(agent_id)
|
|
447
|
+
if agent is None:
|
|
448
|
+
return TaskResult(
|
|
449
|
+
task_id=task.id,
|
|
450
|
+
agent_id=agent_id,
|
|
451
|
+
status=TaskStatus.FAILED,
|
|
452
|
+
output="",
|
|
453
|
+
exit_code=-1,
|
|
454
|
+
duration_seconds=0.0,
|
|
455
|
+
error=f"Agent '{agent_id}' not found or not enabled",
|
|
456
|
+
)
|
|
457
|
+
agents = [agent]
|
|
458
|
+
else:
|
|
459
|
+
agents = [
|
|
460
|
+
a for a in self._registry.available() if a.agent_id not in exclude
|
|
461
|
+
]
|
|
462
|
+
# Capability matching (#9): re-order the candidate list by fit for
|
|
463
|
+
# this task BEFORE the strategy runs, so every strategy sees a
|
|
464
|
+
# capability-preferred priority sequence. No-op (identity) when
|
|
465
|
+
# no matcher is configured or the task carries no hint. Not applied on
|
|
466
|
+
# the forced-`agent_id` path above — forcing one agent has no ordering
|
|
467
|
+
# to influence. Re-orders before the breaker's per-strategy OPEN-agent
|
|
468
|
+
# filter (preference first, then availability), and never empties the
|
|
469
|
+
# pool (it's a permutation), so the guard below still holds.
|
|
470
|
+
agents = _matched_order(agents, task, self._matcher)
|
|
471
|
+
|
|
472
|
+
if not agents:
|
|
473
|
+
error = "No agents available"
|
|
474
|
+
if exclude:
|
|
475
|
+
error += f" (excluded: {', '.join(sorted(exclude))})"
|
|
476
|
+
return TaskResult(
|
|
477
|
+
task_id=task.id,
|
|
478
|
+
agent_id="none",
|
|
479
|
+
status=TaskStatus.FAILED,
|
|
480
|
+
output="",
|
|
481
|
+
exit_code=-1,
|
|
482
|
+
duration_seconds=0.0,
|
|
483
|
+
error=error,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
# Difficulty-based model routing: ONE immutable decision per top-level
|
|
487
|
+
# run, computed right after the lineup is final (post-matcher) and
|
|
488
|
+
# before any side effect. A continuation (parent_run_id set) NEVER
|
|
489
|
+
# classifies — it inherits the parent's effective selector policy
|
|
490
|
+
# (selector affinity), a legacy parent meaning "no model flag" even
|
|
491
|
+
# under mode=observe. Failures here are argument/metadata errors, not
|
|
492
|
+
# runs (no reporter/memory/worktree has been touched yet).
|
|
493
|
+
routing: ModelRoutingDecision | None = None
|
|
494
|
+
if enable_routing:
|
|
495
|
+
try:
|
|
496
|
+
if parent_run_id is not None:
|
|
497
|
+
routing = inherit_decision(
|
|
498
|
+
inherited_routing, agent_id=agents[0].agent_id
|
|
499
|
+
)
|
|
500
|
+
else:
|
|
501
|
+
routing = build_decision(
|
|
502
|
+
self._config.model_routing,
|
|
503
|
+
prompt=task.prompt,
|
|
504
|
+
lineup=[agent.agent_id for agent in agents],
|
|
505
|
+
requested=difficulty,
|
|
506
|
+
)
|
|
507
|
+
except ClassifierError as exc:
|
|
508
|
+
# Observe-only path: nothing would be applied, so a classifier
|
|
509
|
+
# bug degrades (warn + no decision) instead of failing the run.
|
|
510
|
+
self.last_routing_warning = (
|
|
511
|
+
f"model routing classifier failed ({exc}); "
|
|
512
|
+
"continuing without a recommendation"
|
|
513
|
+
)
|
|
514
|
+
logger.warning("run %s: %s", run_id, self.last_routing_warning)
|
|
515
|
+
except (AllUnmappedError, ValueError) as exc:
|
|
516
|
+
# An applied tier with no mapping anywhere, or a corrupt
|
|
517
|
+
# inherited block — both fail closed before any provider call.
|
|
518
|
+
failed = TaskResult(
|
|
519
|
+
task_id=task.id,
|
|
520
|
+
agent_id="routing",
|
|
521
|
+
status=TaskStatus.FAILED,
|
|
522
|
+
output="",
|
|
523
|
+
exit_code=-1,
|
|
524
|
+
duration_seconds=0.0,
|
|
525
|
+
error=str(exc),
|
|
526
|
+
)
|
|
527
|
+
# A monitored INNER must still write result.json (the strict-
|
|
528
|
+
# preflight discipline) — without it the OUTER's detach path
|
|
529
|
+
# reports an attach error instead of this routing failure.
|
|
530
|
+
monitor = build_monitor(
|
|
531
|
+
self._config.monitor, monitor_dir=inner_dir()
|
|
532
|
+
)
|
|
533
|
+
if monitor is not None:
|
|
534
|
+
try:
|
|
535
|
+
monitor.finish(failed)
|
|
536
|
+
except Exception:
|
|
537
|
+
logger.warning(
|
|
538
|
+
"routing-failure monitor finalization failed",
|
|
539
|
+
exc_info=True,
|
|
540
|
+
)
|
|
541
|
+
return failed
|
|
542
|
+
self.last_routing_decision = routing
|
|
543
|
+
|
|
544
|
+
# One pure projection feeds both durable run artifacts and, when opted
|
|
545
|
+
# in, Toolgraph's body-free control-plan preflight. Building it before
|
|
546
|
+
# reporting/workspaces means the advisory producer can inspect the
|
|
547
|
+
# declared upper bounds without seeing prompts or execution state.
|
|
548
|
+
run_plan = build_run_plan(
|
|
549
|
+
self._config,
|
|
550
|
+
[agent.agent_id for agent in agents],
|
|
551
|
+
model_routing=routing.as_dict() if routing is not None else None,
|
|
552
|
+
).as_dict()
|
|
553
|
+
control_plan: dict | None = None
|
|
554
|
+
# BOTH flags, the way `WorkflowCoordinator` reads them. `enabled=false`
|
|
555
|
+
# is the documented rollback, and the config validator deliberately
|
|
556
|
+
# stops requiring a tool surface once it is off — so a config that turns
|
|
557
|
+
# the integration off without also clearing `control_plan_enabled`
|
|
558
|
+
# reaches projection with no candidate tools and raises before dispatch.
|
|
559
|
+
# The rollback has to be one flag, not two that must agree.
|
|
560
|
+
toolgraph_cfg = self._config.integration.toolgraph
|
|
561
|
+
if toolgraph_cfg.enabled and toolgraph_cfg.control_plan_enabled:
|
|
562
|
+
control_plan = build_control_plan(
|
|
563
|
+
self._config,
|
|
564
|
+
[agent.agent_id for agent in agents],
|
|
565
|
+
run_id=run_id,
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
# The run-report envelope writer (S3) — None when [integration.report] is
|
|
569
|
+
# disabled (the default), so the disabled path touches no filesystem. Runs
|
|
570
|
+
# start once a candidate lineup exists (the early returns above are
|
|
571
|
+
# argument errors, not runs). Best-effort throughout: start() writes a
|
|
572
|
+
# schema-valid skeleton (a SIGKILL'd run still leaves an honest "failed"
|
|
573
|
+
# artifact); finish() in the finally below rewrites it with the truth.
|
|
574
|
+
reporter = build_reporter(self._config.integration.report)
|
|
575
|
+
if reporter is not None:
|
|
576
|
+
try:
|
|
577
|
+
reporter.start(
|
|
578
|
+
run_id=run_id,
|
|
579
|
+
task_id=str(task.id),
|
|
580
|
+
strategy=self._config.default_strategy,
|
|
581
|
+
team=self._config.applied_team,
|
|
582
|
+
agents=[a.agent_id for a in agents],
|
|
583
|
+
)
|
|
584
|
+
except Exception:
|
|
585
|
+
# RunReporter.start has its own degrade net; this outer guard is
|
|
586
|
+
# belt-and-suspenders (the monitor.finish discipline) so NOTHING
|
|
587
|
+
# report-shaped can ever abort a run.
|
|
588
|
+
logger.warning("report start failed; continuing run", exc_info=True)
|
|
589
|
+
|
|
590
|
+
# Advisory preflight (S5) — run ONCE for the lead matched agent, BEFORE the
|
|
591
|
+
# strategy and OUTSIDE the try/finally below (so it can never affect
|
|
592
|
+
# cleanup). None when disabled; never raises — every failure degrades to a
|
|
593
|
+
# warning and the run continues. The artifact lands in the run dir when
|
|
594
|
+
# reporting is enabled; otherwise it runs (its identity warning is the
|
|
595
|
+
# tier-1 value) and the artifact is discarded.
|
|
596
|
+
preflight_out_dir: Path | None = None
|
|
597
|
+
if reporter is not None:
|
|
598
|
+
try:
|
|
599
|
+
preflight_out_dir = (
|
|
600
|
+
Path(self._config.integration.report.runs_dir).expanduser() / run_id
|
|
601
|
+
)
|
|
602
|
+
except RuntimeError: # unresolvable ~ — fall back to no artifact copy
|
|
603
|
+
preflight_out_dir = None
|
|
604
|
+
preflight = run_preflight(
|
|
605
|
+
self._config.integration.toolgraph,
|
|
606
|
+
run_id=run_id,
|
|
607
|
+
agent_ids=[a.agent_id for a in agents],
|
|
608
|
+
out_dir=preflight_out_dir,
|
|
609
|
+
control_plan=control_plan,
|
|
610
|
+
)
|
|
611
|
+
self.preflight_warnings = list(preflight.warnings) if preflight else []
|
|
612
|
+
|
|
613
|
+
# P3.1 strict enforcement is a normal failed run, not an exception. It
|
|
614
|
+
# stops before memory registration, worktree creation, or any agent
|
|
615
|
+
# subprocess, but still completes every observability surface so monitor
|
|
616
|
+
# outers and report consumers never see a stranded skeleton.
|
|
617
|
+
if preflight is not None and preflight.should_block:
|
|
618
|
+
self.last_preflight_blocked = True
|
|
619
|
+
result = _preflight_block_result(task, preflight.block_reasons, run_id)
|
|
620
|
+
monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
|
|
621
|
+
now = time.time()
|
|
622
|
+
tracer = build_tracer(
|
|
623
|
+
self._config.integration.trace,
|
|
624
|
+
run_id=run_id,
|
|
625
|
+
fallback_dir=self._config.integration.report.runs_dir,
|
|
626
|
+
run_start=now,
|
|
627
|
+
)
|
|
628
|
+
trace_block: dict | None = None
|
|
629
|
+
if tracer is not None:
|
|
630
|
+
try:
|
|
631
|
+
tracer.record(
|
|
632
|
+
run_span(
|
|
633
|
+
strategy=self._config.default_strategy,
|
|
634
|
+
status="failed",
|
|
635
|
+
start=now,
|
|
636
|
+
end=now,
|
|
637
|
+
)
|
|
638
|
+
)
|
|
639
|
+
except Exception:
|
|
640
|
+
logger.warning("strict preflight trace record failed", exc_info=True)
|
|
641
|
+
trace_block = flush_trace(tracer)
|
|
642
|
+
if reporter is not None:
|
|
643
|
+
try:
|
|
644
|
+
degraded = _degraded_sources(mem, self._breaker)
|
|
645
|
+
_append_trace_degradation(degraded, trace_block)
|
|
646
|
+
self.last_report_path = reporter.finish(
|
|
647
|
+
result,
|
|
648
|
+
degraded=degraded,
|
|
649
|
+
trace=trace_block,
|
|
650
|
+
preflight=preflight.block,
|
|
651
|
+
)
|
|
652
|
+
except Exception:
|
|
653
|
+
logger.warning("strict preflight report finalization failed", exc_info=True)
|
|
654
|
+
if monitor is not None:
|
|
655
|
+
try:
|
|
656
|
+
monitor.finish(result)
|
|
657
|
+
except Exception:
|
|
658
|
+
logger.warning("strict preflight monitor finalization failed", exc_info=True)
|
|
659
|
+
return result
|
|
660
|
+
|
|
661
|
+
# Register agents in memory (best-effort)
|
|
662
|
+
for agent in agents:
|
|
663
|
+
run_memory_bounded(
|
|
664
|
+
mem.register_agent(agent.agent_id),
|
|
665
|
+
memory=mem, label="register_agent",
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
# The task's worktree pool — the strategy acquires from it (one worktree
|
|
669
|
+
# per agent, or a single shared one for a pipeline); the Supervisor owns
|
|
670
|
+
# its lifetime and tears every worktree down in the finally below.
|
|
671
|
+
repo_root = Path(self._config.repo_root).resolve()
|
|
672
|
+
# `supports_snapshots` is a hardcoded True on the real pool, so the
|
|
673
|
+
# `else` below is UNREACHABLE in production — deliberately. It is a test
|
|
674
|
+
# seam: the supervisor suite swaps `WorktreePool` for a two-argument
|
|
675
|
+
# factory returning a FakeWorktreePool, and the getattr default routes
|
|
676
|
+
# those runs past snapshot capture. Keep the probe (removing it would
|
|
677
|
+
# silently send every one of those tests through real snapshot capture),
|
|
678
|
+
# but read it for what it is — the fallback builds a pool against the
|
|
679
|
+
# RESOLVED USER REPO, the only path in the orchestration core whose
|
|
680
|
+
# `git worktree add` / `branch -D` would touch it.
|
|
681
|
+
supports_snapshot = bool(
|
|
682
|
+
getattr(WorktreePool, "supports_snapshots", False)
|
|
683
|
+
)
|
|
684
|
+
snapshot = None
|
|
685
|
+
if supports_snapshot:
|
|
686
|
+
try:
|
|
687
|
+
snapshot = capture_repository_snapshot(
|
|
688
|
+
repo_root,
|
|
689
|
+
run_id,
|
|
690
|
+
Path(self._config.workspace.root),
|
|
691
|
+
# A run's workspace is disposable, so claim it for the sweep
|
|
692
|
+
# before the clone starts — an interrupt in there would
|
|
693
|
+
# otherwise leave a directory nothing can ever reclaim.
|
|
694
|
+
sweepable=True,
|
|
695
|
+
)
|
|
696
|
+
except WorkspaceError as exc:
|
|
697
|
+
failed = TaskResult(
|
|
698
|
+
task_id=task.id,
|
|
699
|
+
agent_id="workspace",
|
|
700
|
+
status=TaskStatus.FAILED,
|
|
701
|
+
output="",
|
|
702
|
+
exit_code=1,
|
|
703
|
+
duration_seconds=0.0,
|
|
704
|
+
error=str(exc),
|
|
705
|
+
run_id=run_id,
|
|
706
|
+
)
|
|
707
|
+
return self._finalize_early_failure(failed, reporter)
|
|
708
|
+
repo_root = snapshot.source_root
|
|
709
|
+
pool = WorktreePool(
|
|
710
|
+
repo_root, str(task.id), snapshot=snapshot, seed_patch=seed_patch
|
|
711
|
+
)
|
|
712
|
+
else:
|
|
713
|
+
pool = WorktreePool(repo_root, str(task.id))
|
|
714
|
+
|
|
715
|
+
# From here the pool owns a real workspace on disk (a control clone under
|
|
716
|
+
# workspace.root), so every path out of this setup block must tear it
|
|
717
|
+
# down. The strategy's own try/finally starts further below, after several
|
|
718
|
+
# constructions that can raise (the artifact store, the monitor's tmux
|
|
719
|
+
# setup, the tracer's directory creation) — without this guard any of them
|
|
720
|
+
# would leak the whole workspace permanently.
|
|
721
|
+
try:
|
|
722
|
+
artifact_store: ArtifactStore | None = None
|
|
723
|
+
if snapshot is not None:
|
|
724
|
+
try:
|
|
725
|
+
artifact_store = ArtifactStore(
|
|
726
|
+
Path(self._config.runs.root),
|
|
727
|
+
project_id(repo_root, self._config.project.id),
|
|
728
|
+
)
|
|
729
|
+
running_artifact = artifact_store.begin(
|
|
730
|
+
run_id=run_id,
|
|
731
|
+
task=task,
|
|
732
|
+
repo_root=repo_root,
|
|
733
|
+
snapshot_commit=snapshot.commit,
|
|
734
|
+
source_fingerprint=snapshot.source_fingerprint,
|
|
735
|
+
strategy=self._config.default_strategy,
|
|
736
|
+
agents=[agent.agent_id for agent in agents],
|
|
737
|
+
plan=run_plan,
|
|
738
|
+
parent_run_id=parent_run_id,
|
|
739
|
+
)
|
|
740
|
+
self.last_artifact_path = running_artifact.path
|
|
741
|
+
except Exception as exc:
|
|
742
|
+
# No agent has run yet, so there is nothing to recover: the
|
|
743
|
+
# workspace holds only the empty control clone. Retaining it
|
|
744
|
+
# would promise the user a recovery workspace with no work in
|
|
745
|
+
# it AND leak it forever (a retained pool is never swept).
|
|
746
|
+
pool.cleanup_all()
|
|
747
|
+
failed = TaskResult(
|
|
748
|
+
task_id=task.id,
|
|
749
|
+
agent_id="artifact",
|
|
750
|
+
status=TaskStatus.FAILED,
|
|
751
|
+
output="",
|
|
752
|
+
exit_code=1,
|
|
753
|
+
duration_seconds=0.0,
|
|
754
|
+
error=f"failed to create mandatory run artifact: {exc}",
|
|
755
|
+
run_id=run_id,
|
|
756
|
+
)
|
|
757
|
+
return self._finalize_early_failure(failed, reporter)
|
|
758
|
+
|
|
759
|
+
# The quality gate (#10), built per-run from config like the pool — None
|
|
760
|
+
# when no commands are configured, so gating is off and behavior is
|
|
761
|
+
# unchanged. The strategy decides what a verdict means for its topology.
|
|
762
|
+
gate = build_gate(self._config.gate)
|
|
763
|
+
|
|
764
|
+
# The live tmux monitor (#16) — non-None only when this is the in-tmux
|
|
765
|
+
# INNER process (inner_dir() set), so normal runs are unaffected. The
|
|
766
|
+
# concurrent strategies tee per-agent output to it; finish() writes the
|
|
767
|
+
# result.json the OUTER launcher reads back. Built per run like the gate.
|
|
768
|
+
monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
|
|
769
|
+
|
|
770
|
+
# The trace recorder (S3′) — None when [integration.trace] is disabled
|
|
771
|
+
# (the default): no spans, no filesystem, byte-identical behavior. The
|
|
772
|
+
# strategies append completed spans retrospectively; the finally below
|
|
773
|
+
# records the run root span and flushes the OTLP artifact. fallback_dir
|
|
774
|
+
# is the report runs_dir, so dir="" lands the trace beside the envelope.
|
|
775
|
+
run_start = time.time()
|
|
776
|
+
tracer = build_tracer(
|
|
777
|
+
self._config.integration.trace,
|
|
778
|
+
run_id=run_id,
|
|
779
|
+
fallback_dir=self._config.integration.report.runs_dir,
|
|
780
|
+
run_start=run_start,
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
# Start memory session (best-effort; None if memory is unavailable)
|
|
784
|
+
session_id = run_memory_bounded(
|
|
785
|
+
mem.start_task_session(agents[0].agent_id, task),
|
|
786
|
+
memory=mem, label="start_task_session",
|
|
787
|
+
)
|
|
788
|
+
except BaseException:
|
|
789
|
+
# Setup failed outside the run's own lifecycle try/finally. Clean the
|
|
790
|
+
# workspace here so it can't leak, then let the failure propagate
|
|
791
|
+
# unchanged — this guard adds cleanup, never swallowing.
|
|
792
|
+
pool.cleanup_all()
|
|
793
|
+
raise
|
|
794
|
+
|
|
795
|
+
result: TaskResult | None = None
|
|
796
|
+
try:
|
|
797
|
+
# Retention for a trace-only run: with reporting disabled nobody has
|
|
798
|
+
# swept {runs_dir} (RunReporter.start owns it otherwise), yet the
|
|
799
|
+
# tracer's default dir IS that tree — so its run dirs would pile up
|
|
800
|
+
# forever. Placed FIRST inside this try, not before it: an interrupt
|
|
801
|
+
# mid-sweep must still enter finalization (the trace flush, and the
|
|
802
|
+
# normal recovery-retention handling of the workspace) rather than
|
|
803
|
+
# escaping it. A custom trace.dir is user-owned and never swept.
|
|
804
|
+
trace_cfg = self._config.integration.trace
|
|
805
|
+
if reporter is None and tracer is not None and not trace_cfg.dir:
|
|
806
|
+
sweep_runs_dir(
|
|
807
|
+
self._config.integration.report.runs_dir,
|
|
808
|
+
self._config.integration.report.keep_last,
|
|
809
|
+
run_id,
|
|
810
|
+
)
|
|
811
|
+
# Each acquired worktree starts clean at HEAD; the strategy diffs the
|
|
812
|
+
# one that produced its result (via _collect_files_changed) so
|
|
813
|
+
# files_changed reflects only that agent's work.
|
|
814
|
+
seed_sessions = (
|
|
815
|
+
{agents[0].agent_id: resume_session_id}
|
|
816
|
+
if resume_session_id and len(agents) == 1
|
|
817
|
+
else None
|
|
818
|
+
)
|
|
819
|
+
# Off-path invariant: no decision → no scope at all (nullcontext),
|
|
820
|
+
# so a disabled run's ContextVar surface is byte-identical to before.
|
|
821
|
+
routing_cm = (
|
|
822
|
+
model_routing_scope(routing.effective_selectors)
|
|
823
|
+
if routing is not None
|
|
824
|
+
else contextlib.nullcontext()
|
|
825
|
+
)
|
|
826
|
+
with routing_cm, provider_session_scope(
|
|
827
|
+
self._config.conversations.enabled, seed_sessions, fork_session
|
|
828
|
+
):
|
|
829
|
+
with capture_tool_calls(
|
|
830
|
+
tracer is not None
|
|
831
|
+
and self._config.integration.trace.tool_calls_enabled,
|
|
832
|
+
self._config.integration.trace.tool_name_map,
|
|
833
|
+
):
|
|
834
|
+
result = self._strategy.execute(
|
|
835
|
+
task=task,
|
|
836
|
+
agents=agents,
|
|
837
|
+
memory=mem,
|
|
838
|
+
pool=pool,
|
|
839
|
+
config=self._config,
|
|
840
|
+
gate=gate,
|
|
841
|
+
breaker=self._breaker,
|
|
842
|
+
monitor=monitor,
|
|
843
|
+
tracer=tracer,
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
result.run_id = run_id
|
|
847
|
+
# A successful coding result is not disposable telemetry. Persist it
|
|
848
|
+
# before memory sharing and before any worktree is removed.
|
|
849
|
+
if snapshot is not None:
|
|
850
|
+
try:
|
|
851
|
+
if artifact_store is None:
|
|
852
|
+
raise RuntimeError("run artifact store was not initialized")
|
|
853
|
+
artifact = artifact_store.save(
|
|
854
|
+
run_id=run_id,
|
|
855
|
+
task=task,
|
|
856
|
+
result=result,
|
|
857
|
+
repo_root=repo_root,
|
|
858
|
+
snapshot_commit=snapshot.commit,
|
|
859
|
+
source_fingerprint=snapshot.source_fingerprint,
|
|
860
|
+
strategy=self._config.default_strategy,
|
|
861
|
+
parent_run_id=parent_run_id,
|
|
862
|
+
plan=run_plan,
|
|
863
|
+
)
|
|
864
|
+
result.artifact_path = str(artifact.path)
|
|
865
|
+
self.last_artifact_path = artifact.path
|
|
866
|
+
# Auto keep_last retention (#118): the one point a new run
|
|
867
|
+
# dir was just added, so pruning here bounds the store on
|
|
868
|
+
# every producer path. prune() only removes TERMINAL runs
|
|
869
|
+
# (never pending/applying/recovery_required) under the
|
|
870
|
+
# apply lock; belt-and-suspenders guard — retention must
|
|
871
|
+
# never fail a healthy, already-persisted run.
|
|
872
|
+
try:
|
|
873
|
+
# protect=run_id: `created_at` ranks by run START, so a
|
|
874
|
+
# long run finishing after `keep_last` newer runs began
|
|
875
|
+
# would otherwise prune the artifact it just saved.
|
|
876
|
+
# blocking=False: this is the tail of a finished run,
|
|
877
|
+
# still holding every worktree and the memory session,
|
|
878
|
+
# and the same lock covers `apply --verify`'s clone and
|
|
879
|
+
# gate commands. Housekeeping never waits behind that —
|
|
880
|
+
# a skipped sweep is repeated by the next run.
|
|
881
|
+
artifact_store.prune(
|
|
882
|
+
self._config.runs.keep_last, protect=run_id,
|
|
883
|
+
blocking=False,
|
|
884
|
+
)
|
|
885
|
+
except ApplyLockBusy:
|
|
886
|
+
logger.info(
|
|
887
|
+
"run-artifact retention sweep skipped: another "
|
|
888
|
+
"process holds the apply lock"
|
|
889
|
+
)
|
|
890
|
+
except Exception:
|
|
891
|
+
logger.warning(
|
|
892
|
+
"run-artifact retention sweep failed", exc_info=True
|
|
893
|
+
)
|
|
894
|
+
except Exception as exc:
|
|
895
|
+
pool.retain()
|
|
896
|
+
self.recovery_workspace = pool.recovery_path
|
|
897
|
+
if artifact_store is not None:
|
|
898
|
+
try:
|
|
899
|
+
artifact_store.recovery_required(run_id, str(exc))
|
|
900
|
+
except Exception:
|
|
901
|
+
logger.warning(
|
|
902
|
+
"could not mark artifact save failure as recovery-required",
|
|
903
|
+
exc_info=True,
|
|
904
|
+
)
|
|
905
|
+
result.status = TaskStatus.FAILED
|
|
906
|
+
result.exit_code = result.exit_code or 1
|
|
907
|
+
result.error = (
|
|
908
|
+
f"failed to persist mandatory run artifact: {exc}; "
|
|
909
|
+
f"recovery workspace: {pool.recovery_path}"
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
# Share successful results to memory
|
|
913
|
+
if result.status == TaskStatus.COMPLETED:
|
|
914
|
+
run_memory_bounded(
|
|
915
|
+
mem.share_result(task, result),
|
|
916
|
+
memory=mem, label="share_result",
|
|
917
|
+
)
|
|
918
|
+
|
|
919
|
+
return result
|
|
920
|
+
except PatchCaptureError as exc:
|
|
921
|
+
pool.retain()
|
|
922
|
+
self.recovery_workspace = pool.recovery_path
|
|
923
|
+
if artifact_store is not None:
|
|
924
|
+
try:
|
|
925
|
+
artifact_store.recovery_required(run_id, str(exc), blocking=False)
|
|
926
|
+
except Exception:
|
|
927
|
+
logger.warning("could not mark patch failure recovery-required", exc_info=True)
|
|
928
|
+
result = _crash_result(task, run_id)
|
|
929
|
+
result.error = f"{exc}; recovery workspace: {pool.recovery_path}"
|
|
930
|
+
return result
|
|
931
|
+
except BaseException as exc:
|
|
932
|
+
if artifact_store is not None:
|
|
933
|
+
try:
|
|
934
|
+
# retain() FIRST: it is a local suppress-guarded file write and
|
|
935
|
+
# exempts the workspace from the finally's cleanup_all().
|
|
936
|
+
# recovery_required() takes the blocking cross-process apply
|
|
937
|
+
# lock — if it raises (or a second Ctrl-C lands in it), the
|
|
938
|
+
# workspace must already be retained, or the finally would
|
|
939
|
+
# delete the crashed run's worktrees out from under recovery.
|
|
940
|
+
pool.retain()
|
|
941
|
+
self.recovery_workspace = pool.recovery_path
|
|
942
|
+
# Non-blocking: a Ctrl-C handler must never hang behind
|
|
943
|
+
# another process's long apply/verify holding the lock. The
|
|
944
|
+
# workspace is already retained above; an unmarked manifest
|
|
945
|
+
# is later reconciled by `syncmill recover`.
|
|
946
|
+
artifact_store.recovery_required(run_id, str(exc), blocking=False)
|
|
947
|
+
except ApplyLockBusy:
|
|
948
|
+
logger.warning(
|
|
949
|
+
"apply lock busy; run %s not marked recovery-required "
|
|
950
|
+
"(workspace retained — run `syncmill recover %s`)",
|
|
951
|
+
run_id,
|
|
952
|
+
run_id,
|
|
953
|
+
)
|
|
954
|
+
except Exception:
|
|
955
|
+
logger.warning("could not mark run recovery-required", exc_info=True)
|
|
956
|
+
raise
|
|
957
|
+
finally:
|
|
958
|
+
# Hand the final result to the monitor first (writes result.json for the
|
|
959
|
+
# OUTER, keeps the control pane's result on screen), then end the memory
|
|
960
|
+
# session and tear every worktree down. If the strategy RAISED (result is
|
|
961
|
+
# None), still finalize the monitor with a synthetic FAILED result so the
|
|
962
|
+
# OUTER reads a real failure + non-zero exit code rather than mistaking the
|
|
963
|
+
# crash for a clean detach ("exit 0"). Monitor finalization is
|
|
964
|
+
# observability — guard it so a failure (e.g. result.json write error) can
|
|
965
|
+
# NEVER skip the memory-session close or worktree cleanup below.
|
|
966
|
+
# The envelope first — BEFORE monitor.finish (whose result.json write
|
|
967
|
+
# unblocks the OUTER, whose report-path print is existence-gated: the
|
|
968
|
+
# final envelope must already be on disk) and BEFORE pool.cleanup_all()
|
|
969
|
+
# (the spec's ordering; no worktree race — patch/files_changed were
|
|
970
|
+
# already captured by _collect_files_changed). Reporting is
|
|
971
|
+
# observability — guard it like the monitor so a failure can NEVER
|
|
972
|
+
# skip the memory-session close or worktree cleanup below.
|
|
973
|
+
# The trace flush comes first — the reporter's finish digests the
|
|
974
|
+
# artifact into the envelope's trace block, so the file must already
|
|
975
|
+
# be on disk. Tracing is observability: both steps are guarded so a
|
|
976
|
+
# tracer failure can NEVER skip the envelope, the memory-session
|
|
977
|
+
# close, or the worktree cleanup below.
|
|
978
|
+
trace_block: dict | None = None
|
|
979
|
+
if tracer is not None:
|
|
980
|
+
try:
|
|
981
|
+
final = result if result is not None else _crash_result(task, run_id)
|
|
982
|
+
tracer.record(
|
|
983
|
+
run_span(
|
|
984
|
+
strategy=self._config.default_strategy,
|
|
985
|
+
status=(
|
|
986
|
+
"completed"
|
|
987
|
+
if final.status == TaskStatus.COMPLETED
|
|
988
|
+
else "failed"
|
|
989
|
+
),
|
|
990
|
+
start=run_start,
|
|
991
|
+
end=time.time(),
|
|
992
|
+
)
|
|
993
|
+
)
|
|
994
|
+
except Exception:
|
|
995
|
+
logger.warning(
|
|
996
|
+
"trace run-span record failed; continuing cleanup",
|
|
997
|
+
exc_info=True,
|
|
998
|
+
)
|
|
999
|
+
trace_block = flush_trace(tracer) # never raises
|
|
1000
|
+
if reporter is not None:
|
|
1001
|
+
try:
|
|
1002
|
+
degraded = _degraded_sources(mem, self._breaker)
|
|
1003
|
+
_append_trace_degradation(degraded, trace_block)
|
|
1004
|
+
self.last_report_path = reporter.finish(
|
|
1005
|
+
result if result is not None else _crash_result(task, run_id),
|
|
1006
|
+
degraded=degraded,
|
|
1007
|
+
trace=trace_block,
|
|
1008
|
+
preflight=preflight.block if preflight is not None else None,
|
|
1009
|
+
)
|
|
1010
|
+
except Exception:
|
|
1011
|
+
logger.warning(
|
|
1012
|
+
"report finalization failed; continuing cleanup",
|
|
1013
|
+
exc_info=True,
|
|
1014
|
+
)
|
|
1015
|
+
if monitor is not None:
|
|
1016
|
+
try:
|
|
1017
|
+
monitor.finish(result if result is not None else _crash_result(task, run_id))
|
|
1018
|
+
except Exception:
|
|
1019
|
+
logger.warning(
|
|
1020
|
+
"monitor finalization failed; continuing cleanup",
|
|
1021
|
+
exc_info=True,
|
|
1022
|
+
)
|
|
1023
|
+
# Ending the memory session is best-effort like everything else in
|
|
1024
|
+
# this finalizer: guard it so a memtomem hiccup can never strand the
|
|
1025
|
+
# worktrees. Cleanup is the last, unconditional step.
|
|
1026
|
+
try:
|
|
1027
|
+
# A crash gets the same synthesized result the reporter and the
|
|
1028
|
+
# monitor above are given: the session was opened either way,
|
|
1029
|
+
# and leaving it open on the one path that ended badly is how a
|
|
1030
|
+
# long-lived process (`board run --loop`, the TUI) accumulates
|
|
1031
|
+
# them.
|
|
1032
|
+
run_memory_bounded(
|
|
1033
|
+
mem.end_task_session(
|
|
1034
|
+
session_id,
|
|
1035
|
+
result if result is not None else _crash_result(task, run_id),
|
|
1036
|
+
),
|
|
1037
|
+
memory=mem, label="end_task_session",
|
|
1038
|
+
)
|
|
1039
|
+
except Exception:
|
|
1040
|
+
logger.warning(
|
|
1041
|
+
"memory session close failed; continuing cleanup",
|
|
1042
|
+
exc_info=True,
|
|
1043
|
+
)
|
|
1044
|
+
pool.cleanup_all()
|
|
1045
|
+
|
|
1046
|
+
# ── Board run lifecycle (Phase 1) ────────────────────────────────────────
|
|
1047
|
+
|
|
1048
|
+
def _require_board(self) -> Board:
|
|
1049
|
+
"""The board, built lazily on first use. Raises ``BoardError`` if the board
|
|
1050
|
+
is disabled (so a board CLI command run without ``SYNCMILL_BOARD__ENABLED``
|
|
1051
|
+
gets a clear message) — or if the db is corrupt (``build_board`` →
|
|
1052
|
+
``Board.__init__`` raises). Stored on ``self._board`` so it's opened once
|
|
1053
|
+
per Supervisor and closed in ``close()``."""
|
|
1054
|
+
if self._board is None:
|
|
1055
|
+
self._board = build_board(self._config.board)
|
|
1056
|
+
if self._board is None:
|
|
1057
|
+
raise BoardError(
|
|
1058
|
+
"the work-board is disabled — enable it with "
|
|
1059
|
+
"SYNCMILL_BOARD__ENABLED=true (or [board] enabled=true in syncmill.toml)"
|
|
1060
|
+
)
|
|
1061
|
+
return self._board
|
|
1062
|
+
|
|
1063
|
+
def run_item(
|
|
1064
|
+
self,
|
|
1065
|
+
item_id: str | None = None,
|
|
1066
|
+
agent_id: str | None = None,
|
|
1067
|
+
exclude: set[str] | None = None,
|
|
1068
|
+
goal: str | None = None,
|
|
1069
|
+
) -> TaskResult | None:
|
|
1070
|
+
"""Claim a board item, run it through the normal pipeline, and record the
|
|
1071
|
+
outcome back onto the board. The board's run-lifecycle wrapper around
|
|
1072
|
+
:meth:`run` — the ``Strategy`` protocol is unchanged.
|
|
1073
|
+
|
|
1074
|
+
Returns the run's ``TaskResult``, or ``None`` when there is nothing to
|
|
1075
|
+
dispatch (an unclaimable id, or no runnable item) so a ``--loop`` caller
|
|
1076
|
+
stops. Flow: recover stale claims → atomically claim (``claim`` for an
|
|
1077
|
+
explicit id, else ``next_runnable``) → render the item DETERMINISTICALLY
|
|
1078
|
+
into the prompt → ``run`` it with a board-scoped memory (no global-``shared``
|
|
1079
|
+
leak) → parse the agent's optional board block → ``apply_update`` (advisory:
|
|
1080
|
+
an invalid agent ``done`` clamps to ``review``) → write ONE sanitized
|
|
1081
|
+
summary to global ``shared``. A genuine orchestration CRASH transitions the
|
|
1082
|
+
item ``→ blocked`` and re-raises (a FAILED result is NOT a crash —
|
|
1083
|
+
``apply_update`` records it).
|
|
1084
|
+
|
|
1085
|
+
``--strategy`` is intentionally NOT a parameter: the board CLI mutates
|
|
1086
|
+
``config.default_strategy`` BEFORE constructing the Supervisor, exactly as
|
|
1087
|
+
plain ``run`` selects its strategy in ``__init__``. A ``goal`` scopes the
|
|
1088
|
+
``next_runnable`` claim to that goal (the ``board run --goal`` filter,
|
|
1089
|
+
Phase 4); the claim-by-id path ignores it — the CLI makes ``--id`` and
|
|
1090
|
+
``--goal`` mutually exclusive."""
|
|
1091
|
+
board = self._require_board()
|
|
1092
|
+
self.last_board_warning = None
|
|
1093
|
+
board.recover_stale_running()
|
|
1094
|
+
run_id = str(uuid4())
|
|
1095
|
+
try:
|
|
1096
|
+
# Side-effect-free view construction validates the FULL backend scope
|
|
1097
|
+
# before any load-bearing board mutation. A missing backend remains
|
|
1098
|
+
# advisory and still returns a degraded view.
|
|
1099
|
+
board_memory = self._memory.scoped(run_id)
|
|
1100
|
+
except ValueError as exc:
|
|
1101
|
+
raise BoardError(str(exc)) from exc
|
|
1102
|
+
item = (
|
|
1103
|
+
board.claim(item_id, run_id)
|
|
1104
|
+
if item_id is not None
|
|
1105
|
+
else board.next_runnable(run_id, goal=goal)
|
|
1106
|
+
)
|
|
1107
|
+
if item is None:
|
|
1108
|
+
# Nothing to dispatch — housekeeping can afford to run NOW (no item
|
|
1109
|
+
# is waiting behind it). The scratch-namespace sweep used to run
|
|
1110
|
+
# BEFORE every claim, where a contended purge stalled dispatch by up
|
|
1111
|
+
# to its full budget per iteration; it now runs only when the claim
|
|
1112
|
+
# drained (here) or after an item's outcome is recorded (below).
|
|
1113
|
+
self._gc_board_run_namespaces(board)
|
|
1114
|
+
return None # unclaimable id / nothing runnable → caller's loop stops
|
|
1115
|
+
|
|
1116
|
+
heartbeat = (
|
|
1117
|
+
_LeaseHeartbeat(board, item.id, run_id)
|
|
1118
|
+
if isinstance(board, Board)
|
|
1119
|
+
else None
|
|
1120
|
+
)
|
|
1121
|
+
try:
|
|
1122
|
+
if heartbeat is not None:
|
|
1123
|
+
# Started INSIDE the crash guard: the claim already succeeded, so a
|
|
1124
|
+
# failure spawning the thread has to strand-proof the item like any
|
|
1125
|
+
# other crash instead of leaving it `running` until stale recovery
|
|
1126
|
+
# reclaims it a lease-timeout later.
|
|
1127
|
+
heartbeat.start()
|
|
1128
|
+
# Active team decisions for this item's goal are injected DETERMINISTICALLY
|
|
1129
|
+
# (in full, never via top-k recall) so the agent can't re-litigate them
|
|
1130
|
+
# (Phase 2, D1). Best-effort: a decisions read failure degrades to none
|
|
1131
|
+
# rather than blocking the item — decision injection is advisory prompt
|
|
1132
|
+
# enrichment (like memory context), not part of the load-bearing dispatch.
|
|
1133
|
+
try:
|
|
1134
|
+
decisions = [d.text for d in board.active_decisions(goal=item.goal)]
|
|
1135
|
+
except BoardError:
|
|
1136
|
+
logger.warning(
|
|
1137
|
+
"board: failed to load decisions for item %s — running without",
|
|
1138
|
+
item.id, exc_info=True,
|
|
1139
|
+
)
|
|
1140
|
+
decisions = []
|
|
1141
|
+
# The prevalidated board-scoped view keeps enriched writes out of global
|
|
1142
|
+
# memory and UNION-READs shared cross-run context (D4).
|
|
1143
|
+
task = Task(prompt=render_item_prompt(item, active_decisions=decisions))
|
|
1144
|
+
# Pass the lease/scope run_id through — lease, `board-run:{run_id}`
|
|
1145
|
+
# memory scope, and the run's own identity are ONE id (no double
|
|
1146
|
+
# issuance; the run-envelope contract's single-resolver rule).
|
|
1147
|
+
# Board runs are a PR-2 model-routing surface: routing is disabled
|
|
1148
|
+
# here so an observe-mode config can't classify the ENRICHED board
|
|
1149
|
+
# prompt (the contract input is the raw item.task, §4.1).
|
|
1150
|
+
result = self.run(
|
|
1151
|
+
task,
|
|
1152
|
+
agent_id=agent_id,
|
|
1153
|
+
exclude=exclude,
|
|
1154
|
+
memory=board_memory,
|
|
1155
|
+
run_id=run_id,
|
|
1156
|
+
enable_routing=False,
|
|
1157
|
+
)
|
|
1158
|
+
# The agent is finished. Stop renewing before the final board write,
|
|
1159
|
+
# which intentionally moves the item out of ``running``.
|
|
1160
|
+
lease_warning: str | None = None
|
|
1161
|
+
if heartbeat is not None:
|
|
1162
|
+
heartbeat.stop()
|
|
1163
|
+
lease_warning = heartbeat.warning
|
|
1164
|
+
heartbeat = None
|
|
1165
|
+
parsed = parse_board_update(result.output)
|
|
1166
|
+
# Pass run_id so apply_update is lease-fenced: if this item was reclaimed
|
|
1167
|
+
# + re-dispatched while we ran (a slow run past the lease, with a second
|
|
1168
|
+
# concurrent `board run`), the stale write is dropped, never clobbering
|
|
1169
|
+
# the new owner or reverting a verified state.
|
|
1170
|
+
_, applied = board.apply_update(
|
|
1171
|
+
item.id, parsed, result, actor=result.agent_id, run_id=run_id
|
|
1172
|
+
)
|
|
1173
|
+
if not applied:
|
|
1174
|
+
# The fence dropped this run's write: the item is no longer `running`
|
|
1175
|
+
# under our lease, so everything below (the envelope board block
|
|
1176
|
+
# aside) is deliberately suppressed and the agent's work is discarded.
|
|
1177
|
+
# Surface it — the run otherwise exits 0 looking like a normal success,
|
|
1178
|
+
# and the operator never learns the work was thrown away.
|
|
1179
|
+
# `last_board_warning` is the CLI seam (`last_report_path` /
|
|
1180
|
+
# `preflight_warnings` precedent).
|
|
1181
|
+
self.last_board_warning = lease_warning or (
|
|
1182
|
+
f"board update for item {printable_name(str(item.id))} was dropped "
|
|
1183
|
+
f"— this run no "
|
|
1184
|
+
f"longer owns the lease (its state or lease owner changed)"
|
|
1185
|
+
)
|
|
1186
|
+
logger.warning("%s", self.last_board_warning)
|
|
1187
|
+
# Deliberately NOT set when `applied` is True, even if the heartbeat
|
|
1188
|
+
# reported a failure: a renewal that merely timed out behind a writer
|
|
1189
|
+
# (with no recovery in between) still ends with the fenced write landing,
|
|
1190
|
+
# and claiming that result "was NOT recorded" would contradict the board.
|
|
1191
|
+
# `applied` is the authoritative signal; the heartbeat's own trouble is
|
|
1192
|
+
# already logged where it happened.
|
|
1193
|
+
# Best-effort: stamp the finished envelope's `board` block (run()'s
|
|
1194
|
+
# own finally already finalized it — this is a read-modify-write).
|
|
1195
|
+
# Deliberately NOT gated on the lease fence: the fence protects the
|
|
1196
|
+
# board/shared system of record, while the envelope records what THIS
|
|
1197
|
+
# run worked on — its own identity, true either way. The enabled check
|
|
1198
|
+
# keeps the disabled path at zero filesystem touches.
|
|
1199
|
+
if self._config.integration.report.enabled:
|
|
1200
|
+
patch_envelope_board(
|
|
1201
|
+
self._config.integration.report.runs_dir,
|
|
1202
|
+
run_id,
|
|
1203
|
+
item_id=item.id,
|
|
1204
|
+
goal=item.goal or None,
|
|
1205
|
+
)
|
|
1206
|
+
# The ONE board-derived write to GLOBAL shared (cross-link tag), built
|
|
1207
|
+
# from the sanitized output — never the enriched prompt. Gated on the
|
|
1208
|
+
# SAME lease fence as the board write: if this run lost its lease (the
|
|
1209
|
+
# item's state or lease owner changed while we ran) apply_update dropped
|
|
1210
|
+
# the board update, so we must NOT publish this superseded run's output
|
|
1211
|
+
# to global `shared` either — otherwise the `board-result:{item_id}`
|
|
1212
|
+
# cross-link would carry discarded work beside the new owner's summary,
|
|
1213
|
+
# breaking the "single sanitized summary" / system-of-record contract.
|
|
1214
|
+
if applied:
|
|
1215
|
+
run_memory_bounded(
|
|
1216
|
+
self._memory.share_board_result(
|
|
1217
|
+
item.id, item.task, strip_board_block(result.output)
|
|
1218
|
+
),
|
|
1219
|
+
memory=self._memory, label="share_board_result",
|
|
1220
|
+
)
|
|
1221
|
+
# Housekeeping AFTER the item's outcome is on the board — never on
|
|
1222
|
+
# the claim path (dispatch latency) and never on the crash path
|
|
1223
|
+
# (a Ctrl-C must not wait out a contended purge).
|
|
1224
|
+
self._gc_board_run_namespaces(board, exclude_run_id=run_id)
|
|
1225
|
+
return result
|
|
1226
|
+
except BaseException as exc:
|
|
1227
|
+
# A genuine crash (in run / parse / apply_update / share) — NOT a FAILED
|
|
1228
|
+
# result, which apply_update already recorded. Strand-proof the item by
|
|
1229
|
+
# moving it OUT of running, then re-raise.
|
|
1230
|
+
self._block_on_crash(board, item.id, run_id, exc)
|
|
1231
|
+
raise
|
|
1232
|
+
finally:
|
|
1233
|
+
if heartbeat is not None:
|
|
1234
|
+
heartbeat.stop()
|
|
1235
|
+
# A concurrent runner may have reclaimed this lease and purged its
|
|
1236
|
+
# scope while it was still finishing. Re-arm the durable tombstone
|
|
1237
|
+
# after all scoped-memory activity so a later sweep catches any
|
|
1238
|
+
# recreation. Best-effort: cleanup bookkeeping never masks a result.
|
|
1239
|
+
try:
|
|
1240
|
+
board.reactivate_board_run_namespace(run_id)
|
|
1241
|
+
except Exception:
|
|
1242
|
+
logger.warning(
|
|
1243
|
+
"board: failed to reactivate scratch ledger for run %s",
|
|
1244
|
+
run_id,
|
|
1245
|
+
exc_info=True,
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
def _gc_board_run_namespaces(
|
|
1249
|
+
self, board: Board, *, exclude_run_id: str | None = None
|
|
1250
|
+
) -> None:
|
|
1251
|
+
"""Best-effort cleanup of board-owned, non-live scratch namespaces.
|
|
1252
|
+
|
|
1253
|
+
Board SQLite now owns discovery and liveness in one snapshot: claims
|
|
1254
|
+
commit their lease and ledger row atomically. Only successful per-run
|
|
1255
|
+
backend calls are tombstoned; failures are scheduled for backoff retry
|
|
1256
|
+
(``record_board_run_purge_failures``) so one poisoned namespace can't
|
|
1257
|
+
head-of-line-block the sweep forever. A cross-process advisory lock
|
|
1258
|
+
(non-blocking, keyed by the board db) lets concurrent ``--loop`` workers
|
|
1259
|
+
SKIP a sweep another worker is already running instead of queueing on
|
|
1260
|
+
the same namespace locks to redo its work. ``exclude_run_id`` keeps the
|
|
1261
|
+
just-finished run's own namespace out of its post-run sweep (the finally
|
|
1262
|
+
re-arms its ledger row right after; purging it here would only mint a
|
|
1263
|
+
no-op re-purge next sweep).
|
|
1264
|
+
"""
|
|
1265
|
+
try:
|
|
1266
|
+
db_path = getattr(board, "db_path", None)
|
|
1267
|
+
gc_lock = (
|
|
1268
|
+
_cross_process_lock(
|
|
1269
|
+
Path(f"{db_path}.gc-lock"), blocking=False
|
|
1270
|
+
)
|
|
1271
|
+
if db_path is not None
|
|
1272
|
+
else contextlib.nullcontext()
|
|
1273
|
+
)
|
|
1274
|
+
try:
|
|
1275
|
+
with gc_lock:
|
|
1276
|
+
self._gc_board_run_namespaces_locked(
|
|
1277
|
+
board, exclude_run_id=exclude_run_id
|
|
1278
|
+
)
|
|
1279
|
+
except ApplyLockBusy:
|
|
1280
|
+
logger.debug(
|
|
1281
|
+
"board-run namespace GC already running in another process"
|
|
1282
|
+
)
|
|
1283
|
+
except Exception:
|
|
1284
|
+
logger.warning(
|
|
1285
|
+
"board-run namespace GC failed; continuing dispatch",
|
|
1286
|
+
exc_info=True,
|
|
1287
|
+
)
|
|
1288
|
+
|
|
1289
|
+
def _gc_board_run_namespaces_locked(
|
|
1290
|
+
self, board: Board, *, exclude_run_id: str | None
|
|
1291
|
+
) -> None:
|
|
1292
|
+
candidates = [
|
|
1293
|
+
rid
|
|
1294
|
+
for rid in board.collectible_board_run_ids(limit=_SCRATCH_GC_BATCH_SIZE)
|
|
1295
|
+
if rid != exclude_run_id
|
|
1296
|
+
]
|
|
1297
|
+
if not candidates:
|
|
1298
|
+
return
|
|
1299
|
+
# purge_board_runs owns a cooperative deadline, including readiness.
|
|
1300
|
+
# Do not wrap it in another timeout: cancelling its receipt collection
|
|
1301
|
+
# turns a partial purge into None and bypasses both ledger and backoff.
|
|
1302
|
+
result = run_async(self._memory.purge_board_runs(candidates))
|
|
1303
|
+
if result is None:
|
|
1304
|
+
# Backend unavailable: no per-namespace verdict was reached — don't
|
|
1305
|
+
# penalize rows for an outage that would delay collection after the
|
|
1306
|
+
# backend recovers.
|
|
1307
|
+
return
|
|
1308
|
+
marked = board.mark_board_run_namespaces_purged(result.successful)
|
|
1309
|
+
if marked:
|
|
1310
|
+
logger.info("board: purged %d stale board-run namespace(s)", marked)
|
|
1311
|
+
failed = [rid for rid in candidates if rid in result.failed]
|
|
1312
|
+
if failed and hasattr(board, "record_board_run_purge_failures"):
|
|
1313
|
+
quarantined = board.record_board_run_purge_failures(failed)
|
|
1314
|
+
for rid in quarantined:
|
|
1315
|
+
logger.warning(
|
|
1316
|
+
"board: scratch namespace for run %s failed to purge %d "
|
|
1317
|
+
"times in a row — quarantined to a slow retry schedule; "
|
|
1318
|
+
"its store residue likely needs operator attention "
|
|
1319
|
+
"(symlinked source, unmounted index root, or legacy day "
|
|
1320
|
+
"file)",
|
|
1321
|
+
rid,
|
|
1322
|
+
Board.PURGE_QUARANTINE_ATTEMPTS,
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
def _block_on_crash(
|
|
1326
|
+
self, board: Board, item_id: str, run_id: str, exc: BaseException
|
|
1327
|
+
) -> None:
|
|
1328
|
+
"""Best-effort, LEASE-FENCED: move a crashed run's item to ``blocked`` so it
|
|
1329
|
+
isn't stranded in ``running`` — but ONLY while this run still owns the claim.
|
|
1330
|
+
Passing ``run_id`` fences the transition exactly like ``apply_update``: if the
|
|
1331
|
+
item was reclaimed + re-dispatched while we ran (a slow run past its lease with
|
|
1332
|
+
a second concurrent ``board run``), or already left ``running`` (a crash AFTER
|
|
1333
|
+
``apply_update`` resolved it — e.g. Ctrl-C during the shared-memory write), the
|
|
1334
|
+
fenced write matches no rows and is dropped, so a stale crashed run never
|
|
1335
|
+
clobbers the new owner or reverts a verified state. Never raises — a board
|
|
1336
|
+
failure here must not mask the original crash."""
|
|
1337
|
+
try:
|
|
1338
|
+
note = (
|
|
1339
|
+
"orchestration cancelled by operator (SIGINT)"
|
|
1340
|
+
if isinstance(exc, KeyboardInterrupt)
|
|
1341
|
+
else f"orchestration crashed: {type(exc).__name__}: {exc}"[:500]
|
|
1342
|
+
)
|
|
1343
|
+
board.transition(
|
|
1344
|
+
item_id, WorkState.BLOCKED.value, actor="system",
|
|
1345
|
+
note=note,
|
|
1346
|
+
# Also onto the ROW, not just the audit note: `board show` reads
|
|
1347
|
+
# `blocked_reason`, and a crashed item is exactly the one an operator
|
|
1348
|
+
# opens first.
|
|
1349
|
+
blocked_reason=note,
|
|
1350
|
+
human_required=True, run_id=run_id,
|
|
1351
|
+
)
|
|
1352
|
+
except Exception:
|
|
1353
|
+
logger.warning(
|
|
1354
|
+
"failed to mark item %s blocked after a crash", item_id, exc_info=True
|
|
1355
|
+
)
|
|
1356
|
+
|
|
1357
|
+
def close(self) -> None:
|
|
1358
|
+
# The board close is in a `finally`: a RuntimeError out of the memory close
|
|
1359
|
+
# (a closed/foreign event loop) would otherwise skip it and leak the SQLite
|
|
1360
|
+
# connection — and in a long-lived process (`tui`, `board run --loop`) that
|
|
1361
|
+
# accumulates.
|
|
1362
|
+
try:
|
|
1363
|
+
# Bounded (cooperatively — see run_memory_bounded): an
|
|
1364
|
+
# await-shaped hang in the backend close must not hang shutdown.
|
|
1365
|
+
run_memory_bounded(self._memory.close(), label="memory close")
|
|
1366
|
+
finally:
|
|
1367
|
+
if self._board is not None:
|
|
1368
|
+
self._board.close()
|
|
1369
|
+
|
|
1370
|
+
def _require_workflow(self) -> WorkflowCoordinator:
|
|
1371
|
+
if self._workflow is None:
|
|
1372
|
+
self._workflow = WorkflowCoordinator(self._config, self._registry)
|
|
1373
|
+
return self._workflow
|
|
1374
|
+
|
|
1375
|
+
def plan_workflow(
|
|
1376
|
+
self, goal: str, planner_agent: str | None = None
|
|
1377
|
+
) -> WorkflowRecord:
|
|
1378
|
+
"""Plan and persist a bounded declarative workflow without executing it."""
|
|
1379
|
+
return self._require_workflow().plan(goal, planner_agent=planner_agent)
|
|
1380
|
+
|
|
1381
|
+
def run_workflow(
|
|
1382
|
+
self,
|
|
1383
|
+
workflow_id: str,
|
|
1384
|
+
*,
|
|
1385
|
+
resume: bool = False,
|
|
1386
|
+
owner: str | None = None,
|
|
1387
|
+
job_id: str | None = None,
|
|
1388
|
+
) -> WorkflowRecord:
|
|
1389
|
+
"""Execute or resume a previously approved native workflow."""
|
|
1390
|
+
return self._require_workflow().run(
|
|
1391
|
+
workflow_id, resume=resume, owner=owner, job_id=job_id
|
|
1392
|
+
)
|
|
1393
|
+
|
|
1394
|
+
@property
|
|
1395
|
+
def workflow_store(self) -> WorkflowStore:
|
|
1396
|
+
return self._require_workflow().store
|
|
1397
|
+
|
|
1398
|
+
@property
|
|
1399
|
+
def workflow_preflight_warnings(self) -> list[str]:
|
|
1400
|
+
"""Advisory preflight warnings from the last workflow operation.
|
|
1401
|
+
|
|
1402
|
+
Mirrors ``preflight_warnings`` for the workflow path so the CLI has one
|
|
1403
|
+
place to read them from; empty when no workflow ran in this process.
|
|
1404
|
+
"""
|
|
1405
|
+
if self._workflow is None:
|
|
1406
|
+
return []
|
|
1407
|
+
return list(self._workflow.preflight_warnings)
|
|
1408
|
+
|
|
1409
|
+
@property
|
|
1410
|
+
def registry(self) -> AgentRegistry:
|
|
1411
|
+
return self._registry
|
|
1412
|
+
|
|
1413
|
+
@property
|
|
1414
|
+
def memory(self) -> SyncmillMemory:
|
|
1415
|
+
return self._memory
|
|
1416
|
+
|
|
1417
|
+
@property
|
|
1418
|
+
def breaker(self) -> CircuitBreaker | None:
|
|
1419
|
+
return self._breaker
|
|
1420
|
+
|
|
1421
|
+
@property
|
|
1422
|
+
def matcher(self) -> CapabilityMatcher | None:
|
|
1423
|
+
return self._matcher
|