syncmill 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. syncmill/__init__.py +11 -0
  2. syncmill/agents.py +2064 -0
  3. syncmill/artifact_io.py +201 -0
  4. syncmill/artifacts.py +1946 -0
  5. syncmill/board.py +2060 -0
  6. syncmill/breaker.py +327 -0
  7. syncmill/cli.py +4986 -0
  8. syncmill/command_errors.py +32 -0
  9. syncmill/config.py +1118 -0
  10. syncmill/dashboard.py +935 -0
  11. syncmill/gates.py +179 -0
  12. syncmill/gateway.py +234 -0
  13. syncmill/git_env.py +99 -0
  14. syncmill/isolated_session.py +431 -0
  15. syncmill/matcher.py +127 -0
  16. syncmill/memory.py +1067 -0
  17. syncmill/memory_backend.py +1907 -0
  18. syncmill/memory_migration.py +962 -0
  19. syncmill/model_routing.py +482 -0
  20. syncmill/models.py +244 -0
  21. syncmill/monitor.py +775 -0
  22. syncmill/phoenix_exporter.py +339 -0
  23. syncmill/planning.py +512 -0
  24. syncmill/platform_support.py +14 -0
  25. syncmill/preflight.py +1129 -0
  26. syncmill/project.py +1067 -0
  27. syncmill/report.py +502 -0
  28. syncmill/review_candidates.py +107 -0
  29. syncmill/room_compare.py +2380 -0
  30. syncmill/room_raw.py +484 -0
  31. syncmill/room_viewer.py +604 -0
  32. syncmill/rooms.py +4044 -0
  33. syncmill/session.py +444 -0
  34. syncmill/strategies/__init__.py +82 -0
  35. syncmill/strategies/base.py +910 -0
  36. syncmill/strategies/compete.py +233 -0
  37. syncmill/strategies/council.py +429 -0
  38. syncmill/strategies/decompose.py +570 -0
  39. syncmill/strategies/pipeline.py +221 -0
  40. syncmill/strategies/route.py +130 -0
  41. syncmill/supervisor.py +1423 -0
  42. syncmill/teamfile.py +571 -0
  43. syncmill/telemetry_hook.py +231 -0
  44. syncmill/terminal.py +245 -0
  45. syncmill/tracer.py +548 -0
  46. syncmill/tui.py +3537 -0
  47. syncmill/tui_collaboration.py +499 -0
  48. syncmill/tui_jobs.py +1407 -0
  49. syncmill/tui_render.py +156 -0
  50. syncmill/workflow.py +3746 -0
  51. syncmill/worktree.py +1045 -0
  52. syncmill-0.2.0.dist-info/METADATA +853 -0
  53. syncmill-0.2.0.dist-info/RECORD +56 -0
  54. syncmill-0.2.0.dist-info/WHEEL +4 -0
  55. syncmill-0.2.0.dist-info/entry_points.txt +2 -0
  56. syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
syncmill/supervisor.py ADDED
@@ -0,0 +1,1423 @@
1
+ """The Supervisor — the sole mediator between agents and memory.
2
+
3
+ Agents don't know about each other or about memory; the Supervisor wires them
4
+ to a ``Strategy`` (the orchestration topology — see ``syncmill.strategies``) and
5
+ to ``SyncmillMemory``. Strategies were extracted into the ``strategies`` package
6
+ (#12) once the four of them made this module unwieldy; the Supervisor keeps only
7
+ the run lifecycle, strategy selection, and the capability re-ordering it applies
8
+ before dispatch.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import contextlib
14
+ import logging
15
+ import threading
16
+ import time
17
+ from pathlib import Path
18
+ from uuid import uuid4
19
+
20
+ from syncmill.platform_support import require_supported_platform
21
+ from syncmill.agents import (
22
+ AgentRegistry,
23
+ AgentRunner,
24
+ capture_tool_calls,
25
+ model_routing_scope,
26
+ provider_session_scope,
27
+ )
28
+ from syncmill.artifacts import ApplyLockBusy, ArtifactStore, _cross_process_lock, printable_name
29
+ from syncmill.board import (
30
+ Board,
31
+ BoardError,
32
+ WorkState,
33
+ build_board,
34
+ parse_board_update,
35
+ render_item_prompt,
36
+ strip_board_block,
37
+ )
38
+ from syncmill.breaker import CircuitBreaker, build_breaker
39
+ from syncmill.config import SyncmillConfig
40
+ from syncmill.gates import build_gate
41
+ from syncmill.matcher import CapabilityMatcher, build_matcher
42
+ from syncmill.memory import SyncmillMemory, run_async, run_memory_bounded
43
+ from syncmill.model_routing import (
44
+ AllUnmappedError,
45
+ ClassifierError,
46
+ ModelRoutingDecision,
47
+ build_decision,
48
+ inherit_decision,
49
+ )
50
+ from syncmill.models import Task, TaskResult, TaskStatus
51
+ from syncmill.monitor import build_monitor, inner_dir
52
+ from syncmill.preflight import run_preflight
53
+ from syncmill.planning import build_control_plan, build_run_plan
54
+ from syncmill.project import project_id
55
+ from syncmill.report import (
56
+ RunReporter,
57
+ build_reporter,
58
+ patch_envelope_board,
59
+ resolve_run_id,
60
+ sweep_runs_dir,
61
+ )
62
+ from syncmill.strategies import Strategy, _make_strategy
63
+ from syncmill.tracer import build_tracer, flush_trace, run_span
64
+ from syncmill.worktree import (
65
+ PatchCaptureError,
66
+ WorktreePool,
67
+ WorkspaceError,
68
+ capture_repository_snapshot,
69
+ )
70
+ from syncmill.workflow import WorkflowCoordinator, WorkflowRecord, WorkflowStore
71
+
72
+ logger = logging.getLogger(__name__)
73
+
74
+ _SCRATCH_GC_BATCH_SIZE = 100
75
+
76
+
77
+ class _LeaseHeartbeat:
78
+ """Renew a real Board claim from a connection owned by this thread."""
79
+
80
+ def __init__(
81
+ self, board: Board, item_id: str, run_id: str, interval: float | None = None
82
+ ):
83
+ self._board = board
84
+ self._item_id = item_id
85
+ self._run_id = run_id
86
+ default_interval = max(0.25, min(60.0, board.lease_timeout_seconds / 3))
87
+ self._interval = default_interval if interval is None else max(0.001, interval)
88
+ self._stop = threading.Event()
89
+ self._renewed = threading.Event()
90
+ self._started = False
91
+ self._warning: str | None = None
92
+ self._thread = threading.Thread(
93
+ target=self._run,
94
+ name=f"syncmill-lease-{item_id[:8]}",
95
+ daemon=True,
96
+ )
97
+
98
+ def start(self) -> None:
99
+ self._thread.start()
100
+ self._started = True
101
+
102
+ def stop(self) -> None:
103
+ self._stop.set()
104
+ # A never-started thread can't be joined (RuntimeError). `start()` is inside
105
+ # the caller's crash guard, so the failed-to-start path reaches this stop().
106
+ if not self._started:
107
+ return
108
+ self._thread.join(timeout=max(1.0, min(5.0, self._interval + 0.5)))
109
+
110
+ def wait_for_renewal(self, timeout: float) -> bool:
111
+ return self._renewed.wait(timeout)
112
+
113
+ @property
114
+ def warning(self) -> str | None:
115
+ """Heartbeat health: why renewal stopped, or ``None``.
116
+
117
+ Set when the claim was gone on a renewal (the item's state or lease owner
118
+ changed under us) or when renewal itself broke. This is NOT a verdict on the
119
+ run's board write — a renewal that merely broke transiently can still end
120
+ with the fenced write landing. ``apply_update``'s ``applied`` decides that;
121
+ this only supplies the reason when it comes back False.
122
+ """
123
+ return self._warning
124
+
125
+ #: Consecutive renewal failures tolerated before the heartbeat gives up. A
126
+ #: single transient sqlite error (a concurrent BEGIN IMMEDIATE held past
127
+ #: busy_timeout) must not silently kill the heartbeat for the rest of a long
128
+ #: run — the lease would expire under a HEALTHY run, stale recovery would
129
+ #: reclaim it, and a second runner would redo (and fence-drop) the work.
130
+ #: Retries reconnect (a broken connection stays broken) and are paced by
131
+ #: ``_RETRY_DELAY_SECONDS``, so persistent breakage still surfaces within
132
+ #: well under one lease timeout.
133
+ _MAX_RENEWAL_FAILURES = 5
134
+ _RETRY_DELAY_SECONDS = 5.0
135
+
136
+ def _record_failure(self, exc: Exception, failures: int) -> None:
137
+ logger.warning(
138
+ "board lease renewal failed (attempt %d/%d)",
139
+ failures,
140
+ self._MAX_RENEWAL_FAILURES,
141
+ exc_info=True,
142
+ )
143
+ # Provisional: a later successful renewal CLEARS it (the lease survived
144
+ # — no false alarm), while a stop() before recovery leaves it standing
145
+ # so the caller still learns renewal was broken when the run ended.
146
+ self._warning = (
147
+ f"board lease renewal failed for item {printable_name(str(self._item_id))} "
148
+ f"({printable_name(f'{type(exc).__name__}: {exc}')}) — the lease may "
149
+ f"have expired"
150
+ )
151
+
152
+ def _run(self) -> None:
153
+ conn: Board | None = None
154
+ failures = 0
155
+ try:
156
+ # Prime the connection eagerly (the pre-retry behavior): a board
157
+ # that is broken from the start surfaces on the first tick's fast
158
+ # retry schedule instead of a full interval later.
159
+ try:
160
+ conn = self._board.new_connection()
161
+ except Exception as exc:
162
+ failures = 1
163
+ self._record_failure(exc, failures)
164
+ while True:
165
+ delay = (
166
+ min(self._RETRY_DELAY_SECONDS, self._interval)
167
+ if failures
168
+ else self._interval
169
+ )
170
+ if self._stop.wait(delay):
171
+ return
172
+ try:
173
+ if conn is None:
174
+ conn = self._board.new_connection()
175
+ renewed = conn.renew_claim(self._item_id, self._run_id)
176
+ except Exception as exc:
177
+ failures += 1
178
+ self._record_failure(exc, failures)
179
+ # The connection may be what broke — rebuild it on retry.
180
+ if conn is not None:
181
+ with contextlib.suppress(Exception):
182
+ conn.close()
183
+ conn = None
184
+ if failures >= self._MAX_RENEWAL_FAILURES:
185
+ return
186
+ continue
187
+ if not renewed:
188
+ # The renewal CAS matched no row: the item left `running` or its
189
+ # lease owner changed. Stale recovery + re-dispatch is the usual
190
+ # cause, but a manual transition does it too — so name what was
191
+ # observed, not a mechanism that may not have happened.
192
+ logger.warning(
193
+ "board: lost the lease on item %s (run %s) — its state or "
194
+ "lease owner changed while this run was working",
195
+ self._item_id, self._run_id,
196
+ )
197
+ self._warning = (
198
+ f"lost the board lease on item {printable_name(str(self._item_id))} "
199
+ f"— its state or "
200
+ f"lease owner changed while this run was working"
201
+ )
202
+ return
203
+ failures = 0
204
+ self._warning = None # a successful renewal clears provisional trouble
205
+ self._renewed.set()
206
+ finally:
207
+ if conn is not None:
208
+ conn.close()
209
+
210
+
211
+ def _matched_order(
212
+ agents: list[AgentRunner], task: Task, matcher: CapabilityMatcher | None
213
+ ) -> list[AgentRunner]:
214
+ """Re-order agents by capability fit for this task; identity when no matcher.
215
+
216
+ Capability matching (#9) layered OVER priority: a matching agent moves ahead
217
+ of raw ``agent_priority``, and equal-fit agents keep priority order (the
218
+ matcher's sort is stable). Returns a permutation — it never drops an agent, so
219
+ Route's fallback chain / Pipeline's stages / Compete's competitors / Council's
220
+ members / Decompose's workers stay intact and the ``if not agents`` guard still
221
+ covers the empty case.
222
+
223
+ Applied ONCE in the Supervisor (not threaded into ``Strategy.execute`` like
224
+ the gate/breaker) because re-ordering has no topology-specific meaning: every
225
+ strategy consumes the candidate list as a priority sequence, so re-ordering at
226
+ the source makes all five capability-aware with no strategy change.
227
+ """
228
+ if matcher is None:
229
+ return list(agents)
230
+ return matcher.order(agents, task)
231
+
232
+
233
+ def _crash_result(task: Task, run_id: str | None = None) -> TaskResult:
234
+ """A FAILED result standing in for an orchestration that raised before
235
+ producing one — so a monitored run's OUTER reads a real failure (and a
236
+ non-zero exit) instead of mistaking the crash for a clean detach. Carries
237
+ ``run_id`` like the success path, so consumers (the CLI's `run: <id>`
238
+ line) can still name the run whose artifacts to inspect."""
239
+ return TaskResult(
240
+ task_id=task.id,
241
+ agent_id="none",
242
+ status=TaskStatus.FAILED,
243
+ output="",
244
+ exit_code=1,
245
+ duration_seconds=0.0,
246
+ error="orchestration crashed before producing a result",
247
+ run_id=run_id,
248
+ )
249
+
250
+
251
+ def _preflight_block_result(
252
+ task: Task, reasons: tuple[str, ...], run_id: str | None = None
253
+ ) -> TaskResult:
254
+ rendered = ", ".join(reasons) or "policy rejected the run"
255
+ return TaskResult(
256
+ task_id=task.id,
257
+ agent_id="preflight",
258
+ status=TaskStatus.FAILED,
259
+ output="",
260
+ exit_code=1,
261
+ duration_seconds=0.0,
262
+ error=f"strict toolgraph preflight blocked execution: {rendered}",
263
+ run_id=run_id,
264
+ )
265
+
266
+
267
+ def _degraded_sources(mem: SyncmillMemory, breaker: CircuitBreaker | None) -> list[str]:
268
+ """The envelope's ``degraded`` array: which best-effort subsystems degraded
269
+ during this run. ``mem`` is the RUN's memory (the board-scoped view on a
270
+ board run), not necessarily ``self._memory``. Open vocabulary — v1 knows
271
+ memory and the breaker (the two carriers of a ``degraded`` flag)."""
272
+ sources = []
273
+ if mem.degraded:
274
+ sources.append("memory")
275
+ if breaker is not None and breaker.degraded:
276
+ sources.append("breaker")
277
+ return sources
278
+
279
+
280
+ def _append_trace_degradation(degraded: list[str], trace: dict | None) -> None:
281
+ """Add file and streaming trace failures to the open degraded vocabulary."""
282
+ if trace is None:
283
+ return
284
+ if trace.get("status") == "error":
285
+ degraded.append("trace")
286
+ stream = trace.get("stream")
287
+ if isinstance(stream, dict) and stream.get("status") == "error":
288
+ degraded.append("phoenix")
289
+
290
+
291
+ class Supervisor:
292
+ """Central orchestrator. The only component that touches both agents and memory."""
293
+
294
+ def __init__(self, config: SyncmillConfig):
295
+ require_supported_platform()
296
+ self._config = config
297
+ self._registry = AgentRegistry(config)
298
+ self._memory = SyncmillMemory(config.memory)
299
+ # Built once (like memory) because it carries cross-run state loaded from
300
+ # disk and a `degraded` flag the CLI surfaces; None when disabled, so the
301
+ # default path never touches the filesystem. (The gate, being stateless,
302
+ # is built per-run inside `run`.)
303
+ self._breaker = build_breaker(config.breaker)
304
+ # Capability matcher (#9) — None when no agent declares a capability (the
305
+ # default), so ordering stays at raw priority. Stateless/config-derived,
306
+ # so it's built once here; the known agent ids surface a config typo.
307
+ self._matcher = build_matcher(config.capabilities, config.agent_priority)
308
+
309
+ # The work-board (Phase 1) is LOAD-BEARING and built LAZILY — NOT here, so
310
+ # an enabled-but-corrupt board can never break a plain `run` (which builds a
311
+ # Supervisor but never touches `run_item`). `_require_board` opens it on
312
+ # first board use; `close()` tears it down.
313
+ self._board: Board | None = None
314
+ # Native workflows are lazy like the board: a plain run must not create
315
+ # workflow state or open SQLite merely because the feature exists.
316
+ self._workflow: WorkflowCoordinator | None = None
317
+
318
+ self._strategy: Strategy = _make_strategy(config.default_strategy)
319
+ # Where the last run's envelope landed (None when reporting is disabled
320
+ # or the write degraded) — the CLI's read seam for the report-path line,
321
+ # like `supervisor.memory` / `supervisor.breaker`.
322
+ self.last_report_path: Path | None = None
323
+ # The last run's advisory-preflight warnings — the CLI surfaces them as
324
+ # yellow notices (the memory.degraded pattern). Reset per run.
325
+ self.preflight_warnings: list[str] = []
326
+ self.last_preflight_blocked = False
327
+ self.last_run_id: str | None = None
328
+ self.last_artifact_path: Path | None = None
329
+ self.recovery_workspace: Path | None = None
330
+ # The last run's model-routing decision (None when routing was off/
331
+ # disabled for the run) + the classifier-degrade warning — the CLI's
332
+ # read seams for the compact banner and the yellow degrade notice.
333
+ self.last_routing_decision: ModelRoutingDecision | None = None
334
+ self.last_routing_warning: str | None = None
335
+ # Set by `run_item` when a board run's work was discarded (its lease was
336
+ # lost / its fenced write dropped) — the CLI's read seam for that notice.
337
+ # Reset per `run_item`, not per `run`: a plain run never touches the board.
338
+ self.last_board_warning: str | None = None
339
+
340
+ def _finalize_early_failure(
341
+ self, failed: TaskResult, reporter: RunReporter | None
342
+ ) -> TaskResult:
343
+ """Close the observability surfaces for a failure that aborts a run before
344
+ the strategy starts, then hand the result back.
345
+
346
+ Both surfaces are observability, so both are guarded: a report or monitor
347
+ failure must never replace the real cause with an unrelated traceback. The
348
+ monitor is the load-bearing half — a monitored INNER that returns without
349
+ writing result.json leaves the OUTER reporting an attach error instead of
350
+ this failure (the same discipline as the routing- and strict-preflight-
351
+ failure paths).
352
+ """
353
+ if reporter is not None:
354
+ try:
355
+ self.last_report_path = reporter.finish(failed, degraded=[])
356
+ except Exception:
357
+ logger.warning(
358
+ "early-failure report finalization failed", exc_info=True
359
+ )
360
+ monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
361
+ if monitor is not None:
362
+ try:
363
+ monitor.finish(failed)
364
+ except Exception:
365
+ logger.warning(
366
+ "early-failure monitor finalization failed", exc_info=True
367
+ )
368
+ return failed
369
+
370
+ def run(
371
+ self,
372
+ task: Task,
373
+ agent_id: str | None = None,
374
+ exclude: set[str] | None = None,
375
+ memory: SyncmillMemory | None = None,
376
+ run_id: str | None = None,
377
+ resume_session_id: str | None = None,
378
+ parent_run_id: str | None = None,
379
+ seed_patch: bytes | None = None,
380
+ fork_session: bool = False,
381
+ difficulty: str | None = None,
382
+ inherited_routing: dict | None = None,
383
+ enable_routing: bool = True,
384
+ ) -> TaskResult:
385
+ """Execute a task through the orchestration pipeline.
386
+
387
+ Args:
388
+ task: The task to execute.
389
+ agent_id: If specified, force this agent only (no fallback).
390
+ exclude: Agent ids to drop from the priority fallback — e.g. run one
391
+ agent interactively yourself while syncmill orchestrates the
392
+ rest. Forcing an excluded agent via ``agent_id`` is an error.
393
+ memory: Override the memory used for THIS run (default ``self._memory``).
394
+ ``run_item`` passes a board-scoped view so a board run's session
395
+ prompt-log AND every strategy-internal share/search stay off the
396
+ global ``shared`` namespace (decision #6). Plain ``run`` passes
397
+ ``None`` → behavior-identical to before.
398
+ run_id: The run's stable identity (the run-envelope join key),
399
+ resolved through the single ``resolve_run_id`` seam: ``run_item``
400
+ passes its board lease/scope id (one identity, no double
401
+ issuance); a monitored inner picks up the OUTER-minted id from
402
+ the snapshot env; a direct run mints a fresh uuid4.
403
+ difficulty: The ``--difficulty`` request (``auto|low|medium|high``)
404
+ for model routing; ``None`` defers to ``config.model_routing``.
405
+ inherited_routing: A continuation's parent ``plan.model_routing``
406
+ block (selector affinity). Only consulted when
407
+ ``parent_run_id`` marks this run as a continuation — a legacy
408
+ parent (``None``) then resumes with no model flag even under
409
+ ``mode=observe``.
410
+ enable_routing: ``run_item`` passes ``False`` — board runs are a
411
+ PR-2 surface, and an observe-mode config must not classify the
412
+ enriched board prompt (raw ``item.task`` is the contract input).
413
+ """
414
+ # The one identity every artifact of this run shares — the log, and (when
415
+ # [integration.report] is enabled) the run-envelope under runs_dir/{run_id}.
416
+ run_id = resolve_run_id(run_id)
417
+ self.last_run_id = run_id
418
+ self.last_artifact_path = None
419
+ self.recovery_workspace = None
420
+ logger.info("run %s: task %s", run_id, task.id)
421
+ # Reset up front so an early argument-error return can't leave a PRIOR
422
+ # run's report path / preflight warnings dangling on this Supervisor.
423
+ self.last_report_path = None
424
+ self.preflight_warnings = []
425
+ self.last_preflight_blocked = False
426
+ self.last_routing_decision = None
427
+ self.last_routing_warning = None
428
+ # Use `mem` for EVERY memory touchpoint below — register, session start, the
429
+ # strategy's `memory=`, the success share, and session end — so a board run
430
+ # (which passes a scoped view) can't leak the enriched prompt through any of
431
+ # them via `self._memory`.
432
+ mem = memory or self._memory
433
+ exclude = exclude or set()
434
+ # Determine which agents to use
435
+ if agent_id:
436
+ if agent_id in exclude:
437
+ return TaskResult(
438
+ task_id=task.id,
439
+ agent_id=agent_id,
440
+ status=TaskStatus.FAILED,
441
+ output="",
442
+ exit_code=-1,
443
+ duration_seconds=0.0,
444
+ error=f"Agent '{agent_id}' is both forced (agent_id) and excluded",
445
+ )
446
+ agent = self._registry.get(agent_id)
447
+ if agent is None:
448
+ return TaskResult(
449
+ task_id=task.id,
450
+ agent_id=agent_id,
451
+ status=TaskStatus.FAILED,
452
+ output="",
453
+ exit_code=-1,
454
+ duration_seconds=0.0,
455
+ error=f"Agent '{agent_id}' not found or not enabled",
456
+ )
457
+ agents = [agent]
458
+ else:
459
+ agents = [
460
+ a for a in self._registry.available() if a.agent_id not in exclude
461
+ ]
462
+ # Capability matching (#9): re-order the candidate list by fit for
463
+ # this task BEFORE the strategy runs, so every strategy sees a
464
+ # capability-preferred priority sequence. No-op (identity) when
465
+ # no matcher is configured or the task carries no hint. Not applied on
466
+ # the forced-`agent_id` path above — forcing one agent has no ordering
467
+ # to influence. Re-orders before the breaker's per-strategy OPEN-agent
468
+ # filter (preference first, then availability), and never empties the
469
+ # pool (it's a permutation), so the guard below still holds.
470
+ agents = _matched_order(agents, task, self._matcher)
471
+
472
+ if not agents:
473
+ error = "No agents available"
474
+ if exclude:
475
+ error += f" (excluded: {', '.join(sorted(exclude))})"
476
+ return TaskResult(
477
+ task_id=task.id,
478
+ agent_id="none",
479
+ status=TaskStatus.FAILED,
480
+ output="",
481
+ exit_code=-1,
482
+ duration_seconds=0.0,
483
+ error=error,
484
+ )
485
+
486
+ # Difficulty-based model routing: ONE immutable decision per top-level
487
+ # run, computed right after the lineup is final (post-matcher) and
488
+ # before any side effect. A continuation (parent_run_id set) NEVER
489
+ # classifies — it inherits the parent's effective selector policy
490
+ # (selector affinity), a legacy parent meaning "no model flag" even
491
+ # under mode=observe. Failures here are argument/metadata errors, not
492
+ # runs (no reporter/memory/worktree has been touched yet).
493
+ routing: ModelRoutingDecision | None = None
494
+ if enable_routing:
495
+ try:
496
+ if parent_run_id is not None:
497
+ routing = inherit_decision(
498
+ inherited_routing, agent_id=agents[0].agent_id
499
+ )
500
+ else:
501
+ routing = build_decision(
502
+ self._config.model_routing,
503
+ prompt=task.prompt,
504
+ lineup=[agent.agent_id for agent in agents],
505
+ requested=difficulty,
506
+ )
507
+ except ClassifierError as exc:
508
+ # Observe-only path: nothing would be applied, so a classifier
509
+ # bug degrades (warn + no decision) instead of failing the run.
510
+ self.last_routing_warning = (
511
+ f"model routing classifier failed ({exc}); "
512
+ "continuing without a recommendation"
513
+ )
514
+ logger.warning("run %s: %s", run_id, self.last_routing_warning)
515
+ except (AllUnmappedError, ValueError) as exc:
516
+ # An applied tier with no mapping anywhere, or a corrupt
517
+ # inherited block — both fail closed before any provider call.
518
+ failed = TaskResult(
519
+ task_id=task.id,
520
+ agent_id="routing",
521
+ status=TaskStatus.FAILED,
522
+ output="",
523
+ exit_code=-1,
524
+ duration_seconds=0.0,
525
+ error=str(exc),
526
+ )
527
+ # A monitored INNER must still write result.json (the strict-
528
+ # preflight discipline) — without it the OUTER's detach path
529
+ # reports an attach error instead of this routing failure.
530
+ monitor = build_monitor(
531
+ self._config.monitor, monitor_dir=inner_dir()
532
+ )
533
+ if monitor is not None:
534
+ try:
535
+ monitor.finish(failed)
536
+ except Exception:
537
+ logger.warning(
538
+ "routing-failure monitor finalization failed",
539
+ exc_info=True,
540
+ )
541
+ return failed
542
+ self.last_routing_decision = routing
543
+
544
+ # One pure projection feeds both durable run artifacts and, when opted
545
+ # in, Toolgraph's body-free control-plan preflight. Building it before
546
+ # reporting/workspaces means the advisory producer can inspect the
547
+ # declared upper bounds without seeing prompts or execution state.
548
+ run_plan = build_run_plan(
549
+ self._config,
550
+ [agent.agent_id for agent in agents],
551
+ model_routing=routing.as_dict() if routing is not None else None,
552
+ ).as_dict()
553
+ control_plan: dict | None = None
554
+ # BOTH flags, the way `WorkflowCoordinator` reads them. `enabled=false`
555
+ # is the documented rollback, and the config validator deliberately
556
+ # stops requiring a tool surface once it is off — so a config that turns
557
+ # the integration off without also clearing `control_plan_enabled`
558
+ # reaches projection with no candidate tools and raises before dispatch.
559
+ # The rollback has to be one flag, not two that must agree.
560
+ toolgraph_cfg = self._config.integration.toolgraph
561
+ if toolgraph_cfg.enabled and toolgraph_cfg.control_plan_enabled:
562
+ control_plan = build_control_plan(
563
+ self._config,
564
+ [agent.agent_id for agent in agents],
565
+ run_id=run_id,
566
+ )
567
+
568
+ # The run-report envelope writer (S3) — None when [integration.report] is
569
+ # disabled (the default), so the disabled path touches no filesystem. Runs
570
+ # start once a candidate lineup exists (the early returns above are
571
+ # argument errors, not runs). Best-effort throughout: start() writes a
572
+ # schema-valid skeleton (a SIGKILL'd run still leaves an honest "failed"
573
+ # artifact); finish() in the finally below rewrites it with the truth.
574
+ reporter = build_reporter(self._config.integration.report)
575
+ if reporter is not None:
576
+ try:
577
+ reporter.start(
578
+ run_id=run_id,
579
+ task_id=str(task.id),
580
+ strategy=self._config.default_strategy,
581
+ team=self._config.applied_team,
582
+ agents=[a.agent_id for a in agents],
583
+ )
584
+ except Exception:
585
+ # RunReporter.start has its own degrade net; this outer guard is
586
+ # belt-and-suspenders (the monitor.finish discipline) so NOTHING
587
+ # report-shaped can ever abort a run.
588
+ logger.warning("report start failed; continuing run", exc_info=True)
589
+
590
+ # Advisory preflight (S5) — run ONCE for the lead matched agent, BEFORE the
591
+ # strategy and OUTSIDE the try/finally below (so it can never affect
592
+ # cleanup). None when disabled; never raises — every failure degrades to a
593
+ # warning and the run continues. The artifact lands in the run dir when
594
+ # reporting is enabled; otherwise it runs (its identity warning is the
595
+ # tier-1 value) and the artifact is discarded.
596
+ preflight_out_dir: Path | None = None
597
+ if reporter is not None:
598
+ try:
599
+ preflight_out_dir = (
600
+ Path(self._config.integration.report.runs_dir).expanduser() / run_id
601
+ )
602
+ except RuntimeError: # unresolvable ~ — fall back to no artifact copy
603
+ preflight_out_dir = None
604
+ preflight = run_preflight(
605
+ self._config.integration.toolgraph,
606
+ run_id=run_id,
607
+ agent_ids=[a.agent_id for a in agents],
608
+ out_dir=preflight_out_dir,
609
+ control_plan=control_plan,
610
+ )
611
+ self.preflight_warnings = list(preflight.warnings) if preflight else []
612
+
613
+ # P3.1 strict enforcement is a normal failed run, not an exception. It
614
+ # stops before memory registration, worktree creation, or any agent
615
+ # subprocess, but still completes every observability surface so monitor
616
+ # outers and report consumers never see a stranded skeleton.
617
+ if preflight is not None and preflight.should_block:
618
+ self.last_preflight_blocked = True
619
+ result = _preflight_block_result(task, preflight.block_reasons, run_id)
620
+ monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
621
+ now = time.time()
622
+ tracer = build_tracer(
623
+ self._config.integration.trace,
624
+ run_id=run_id,
625
+ fallback_dir=self._config.integration.report.runs_dir,
626
+ run_start=now,
627
+ )
628
+ trace_block: dict | None = None
629
+ if tracer is not None:
630
+ try:
631
+ tracer.record(
632
+ run_span(
633
+ strategy=self._config.default_strategy,
634
+ status="failed",
635
+ start=now,
636
+ end=now,
637
+ )
638
+ )
639
+ except Exception:
640
+ logger.warning("strict preflight trace record failed", exc_info=True)
641
+ trace_block = flush_trace(tracer)
642
+ if reporter is not None:
643
+ try:
644
+ degraded = _degraded_sources(mem, self._breaker)
645
+ _append_trace_degradation(degraded, trace_block)
646
+ self.last_report_path = reporter.finish(
647
+ result,
648
+ degraded=degraded,
649
+ trace=trace_block,
650
+ preflight=preflight.block,
651
+ )
652
+ except Exception:
653
+ logger.warning("strict preflight report finalization failed", exc_info=True)
654
+ if monitor is not None:
655
+ try:
656
+ monitor.finish(result)
657
+ except Exception:
658
+ logger.warning("strict preflight monitor finalization failed", exc_info=True)
659
+ return result
660
+
661
+ # Register agents in memory (best-effort)
662
+ for agent in agents:
663
+ run_memory_bounded(
664
+ mem.register_agent(agent.agent_id),
665
+ memory=mem, label="register_agent",
666
+ )
667
+
668
+ # The task's worktree pool — the strategy acquires from it (one worktree
669
+ # per agent, or a single shared one for a pipeline); the Supervisor owns
670
+ # its lifetime and tears every worktree down in the finally below.
671
+ repo_root = Path(self._config.repo_root).resolve()
672
+ # `supports_snapshots` is a hardcoded True on the real pool, so the
673
+ # `else` below is UNREACHABLE in production — deliberately. It is a test
674
+ # seam: the supervisor suite swaps `WorktreePool` for a two-argument
675
+ # factory returning a FakeWorktreePool, and the getattr default routes
676
+ # those runs past snapshot capture. Keep the probe (removing it would
677
+ # silently send every one of those tests through real snapshot capture),
678
+ # but read it for what it is — the fallback builds a pool against the
679
+ # RESOLVED USER REPO, the only path in the orchestration core whose
680
+ # `git worktree add` / `branch -D` would touch it.
681
+ supports_snapshot = bool(
682
+ getattr(WorktreePool, "supports_snapshots", False)
683
+ )
684
+ snapshot = None
685
+ if supports_snapshot:
686
+ try:
687
+ snapshot = capture_repository_snapshot(
688
+ repo_root,
689
+ run_id,
690
+ Path(self._config.workspace.root),
691
+ # A run's workspace is disposable, so claim it for the sweep
692
+ # before the clone starts — an interrupt in there would
693
+ # otherwise leave a directory nothing can ever reclaim.
694
+ sweepable=True,
695
+ )
696
+ except WorkspaceError as exc:
697
+ failed = TaskResult(
698
+ task_id=task.id,
699
+ agent_id="workspace",
700
+ status=TaskStatus.FAILED,
701
+ output="",
702
+ exit_code=1,
703
+ duration_seconds=0.0,
704
+ error=str(exc),
705
+ run_id=run_id,
706
+ )
707
+ return self._finalize_early_failure(failed, reporter)
708
+ repo_root = snapshot.source_root
709
+ pool = WorktreePool(
710
+ repo_root, str(task.id), snapshot=snapshot, seed_patch=seed_patch
711
+ )
712
+ else:
713
+ pool = WorktreePool(repo_root, str(task.id))
714
+
715
+ # From here the pool owns a real workspace on disk (a control clone under
716
+ # workspace.root), so every path out of this setup block must tear it
717
+ # down. The strategy's own try/finally starts further below, after several
718
+ # constructions that can raise (the artifact store, the monitor's tmux
719
+ # setup, the tracer's directory creation) — without this guard any of them
720
+ # would leak the whole workspace permanently.
721
+ try:
722
+ artifact_store: ArtifactStore | None = None
723
+ if snapshot is not None:
724
+ try:
725
+ artifact_store = ArtifactStore(
726
+ Path(self._config.runs.root),
727
+ project_id(repo_root, self._config.project.id),
728
+ )
729
+ running_artifact = artifact_store.begin(
730
+ run_id=run_id,
731
+ task=task,
732
+ repo_root=repo_root,
733
+ snapshot_commit=snapshot.commit,
734
+ source_fingerprint=snapshot.source_fingerprint,
735
+ strategy=self._config.default_strategy,
736
+ agents=[agent.agent_id for agent in agents],
737
+ plan=run_plan,
738
+ parent_run_id=parent_run_id,
739
+ )
740
+ self.last_artifact_path = running_artifact.path
741
+ except Exception as exc:
742
+ # No agent has run yet, so there is nothing to recover: the
743
+ # workspace holds only the empty control clone. Retaining it
744
+ # would promise the user a recovery workspace with no work in
745
+ # it AND leak it forever (a retained pool is never swept).
746
+ pool.cleanup_all()
747
+ failed = TaskResult(
748
+ task_id=task.id,
749
+ agent_id="artifact",
750
+ status=TaskStatus.FAILED,
751
+ output="",
752
+ exit_code=1,
753
+ duration_seconds=0.0,
754
+ error=f"failed to create mandatory run artifact: {exc}",
755
+ run_id=run_id,
756
+ )
757
+ return self._finalize_early_failure(failed, reporter)
758
+
759
+ # The quality gate (#10), built per-run from config like the pool — None
760
+ # when no commands are configured, so gating is off and behavior is
761
+ # unchanged. The strategy decides what a verdict means for its topology.
762
+ gate = build_gate(self._config.gate)
763
+
764
+ # The live tmux monitor (#16) — non-None only when this is the in-tmux
765
+ # INNER process (inner_dir() set), so normal runs are unaffected. The
766
+ # concurrent strategies tee per-agent output to it; finish() writes the
767
+ # result.json the OUTER launcher reads back. Built per run like the gate.
768
+ monitor = build_monitor(self._config.monitor, monitor_dir=inner_dir())
769
+
770
+ # The trace recorder (S3′) — None when [integration.trace] is disabled
771
+ # (the default): no spans, no filesystem, byte-identical behavior. The
772
+ # strategies append completed spans retrospectively; the finally below
773
+ # records the run root span and flushes the OTLP artifact. fallback_dir
774
+ # is the report runs_dir, so dir="" lands the trace beside the envelope.
775
+ run_start = time.time()
776
+ tracer = build_tracer(
777
+ self._config.integration.trace,
778
+ run_id=run_id,
779
+ fallback_dir=self._config.integration.report.runs_dir,
780
+ run_start=run_start,
781
+ )
782
+
783
+ # Start memory session (best-effort; None if memory is unavailable)
784
+ session_id = run_memory_bounded(
785
+ mem.start_task_session(agents[0].agent_id, task),
786
+ memory=mem, label="start_task_session",
787
+ )
788
+ except BaseException:
789
+ # Setup failed outside the run's own lifecycle try/finally. Clean the
790
+ # workspace here so it can't leak, then let the failure propagate
791
+ # unchanged — this guard adds cleanup, never swallowing.
792
+ pool.cleanup_all()
793
+ raise
794
+
795
+ result: TaskResult | None = None
796
+ try:
797
+ # Retention for a trace-only run: with reporting disabled nobody has
798
+ # swept {runs_dir} (RunReporter.start owns it otherwise), yet the
799
+ # tracer's default dir IS that tree — so its run dirs would pile up
800
+ # forever. Placed FIRST inside this try, not before it: an interrupt
801
+ # mid-sweep must still enter finalization (the trace flush, and the
802
+ # normal recovery-retention handling of the workspace) rather than
803
+ # escaping it. A custom trace.dir is user-owned and never swept.
804
+ trace_cfg = self._config.integration.trace
805
+ if reporter is None and tracer is not None and not trace_cfg.dir:
806
+ sweep_runs_dir(
807
+ self._config.integration.report.runs_dir,
808
+ self._config.integration.report.keep_last,
809
+ run_id,
810
+ )
811
+ # Each acquired worktree starts clean at HEAD; the strategy diffs the
812
+ # one that produced its result (via _collect_files_changed) so
813
+ # files_changed reflects only that agent's work.
814
+ seed_sessions = (
815
+ {agents[0].agent_id: resume_session_id}
816
+ if resume_session_id and len(agents) == 1
817
+ else None
818
+ )
819
+ # Off-path invariant: no decision → no scope at all (nullcontext),
820
+ # so a disabled run's ContextVar surface is byte-identical to before.
821
+ routing_cm = (
822
+ model_routing_scope(routing.effective_selectors)
823
+ if routing is not None
824
+ else contextlib.nullcontext()
825
+ )
826
+ with routing_cm, provider_session_scope(
827
+ self._config.conversations.enabled, seed_sessions, fork_session
828
+ ):
829
+ with capture_tool_calls(
830
+ tracer is not None
831
+ and self._config.integration.trace.tool_calls_enabled,
832
+ self._config.integration.trace.tool_name_map,
833
+ ):
834
+ result = self._strategy.execute(
835
+ task=task,
836
+ agents=agents,
837
+ memory=mem,
838
+ pool=pool,
839
+ config=self._config,
840
+ gate=gate,
841
+ breaker=self._breaker,
842
+ monitor=monitor,
843
+ tracer=tracer,
844
+ )
845
+
846
+ result.run_id = run_id
847
+ # A successful coding result is not disposable telemetry. Persist it
848
+ # before memory sharing and before any worktree is removed.
849
+ if snapshot is not None:
850
+ try:
851
+ if artifact_store is None:
852
+ raise RuntimeError("run artifact store was not initialized")
853
+ artifact = artifact_store.save(
854
+ run_id=run_id,
855
+ task=task,
856
+ result=result,
857
+ repo_root=repo_root,
858
+ snapshot_commit=snapshot.commit,
859
+ source_fingerprint=snapshot.source_fingerprint,
860
+ strategy=self._config.default_strategy,
861
+ parent_run_id=parent_run_id,
862
+ plan=run_plan,
863
+ )
864
+ result.artifact_path = str(artifact.path)
865
+ self.last_artifact_path = artifact.path
866
+ # Auto keep_last retention (#118): the one point a new run
867
+ # dir was just added, so pruning here bounds the store on
868
+ # every producer path. prune() only removes TERMINAL runs
869
+ # (never pending/applying/recovery_required) under the
870
+ # apply lock; belt-and-suspenders guard — retention must
871
+ # never fail a healthy, already-persisted run.
872
+ try:
873
+ # protect=run_id: `created_at` ranks by run START, so a
874
+ # long run finishing after `keep_last` newer runs began
875
+ # would otherwise prune the artifact it just saved.
876
+ # blocking=False: this is the tail of a finished run,
877
+ # still holding every worktree and the memory session,
878
+ # and the same lock covers `apply --verify`'s clone and
879
+ # gate commands. Housekeeping never waits behind that —
880
+ # a skipped sweep is repeated by the next run.
881
+ artifact_store.prune(
882
+ self._config.runs.keep_last, protect=run_id,
883
+ blocking=False,
884
+ )
885
+ except ApplyLockBusy:
886
+ logger.info(
887
+ "run-artifact retention sweep skipped: another "
888
+ "process holds the apply lock"
889
+ )
890
+ except Exception:
891
+ logger.warning(
892
+ "run-artifact retention sweep failed", exc_info=True
893
+ )
894
+ except Exception as exc:
895
+ pool.retain()
896
+ self.recovery_workspace = pool.recovery_path
897
+ if artifact_store is not None:
898
+ try:
899
+ artifact_store.recovery_required(run_id, str(exc))
900
+ except Exception:
901
+ logger.warning(
902
+ "could not mark artifact save failure as recovery-required",
903
+ exc_info=True,
904
+ )
905
+ result.status = TaskStatus.FAILED
906
+ result.exit_code = result.exit_code or 1
907
+ result.error = (
908
+ f"failed to persist mandatory run artifact: {exc}; "
909
+ f"recovery workspace: {pool.recovery_path}"
910
+ )
911
+
912
+ # Share successful results to memory
913
+ if result.status == TaskStatus.COMPLETED:
914
+ run_memory_bounded(
915
+ mem.share_result(task, result),
916
+ memory=mem, label="share_result",
917
+ )
918
+
919
+ return result
920
+ except PatchCaptureError as exc:
921
+ pool.retain()
922
+ self.recovery_workspace = pool.recovery_path
923
+ if artifact_store is not None:
924
+ try:
925
+ artifact_store.recovery_required(run_id, str(exc), blocking=False)
926
+ except Exception:
927
+ logger.warning("could not mark patch failure recovery-required", exc_info=True)
928
+ result = _crash_result(task, run_id)
929
+ result.error = f"{exc}; recovery workspace: {pool.recovery_path}"
930
+ return result
931
+ except BaseException as exc:
932
+ if artifact_store is not None:
933
+ try:
934
+ # retain() FIRST: it is a local suppress-guarded file write and
935
+ # exempts the workspace from the finally's cleanup_all().
936
+ # recovery_required() takes the blocking cross-process apply
937
+ # lock — if it raises (or a second Ctrl-C lands in it), the
938
+ # workspace must already be retained, or the finally would
939
+ # delete the crashed run's worktrees out from under recovery.
940
+ pool.retain()
941
+ self.recovery_workspace = pool.recovery_path
942
+ # Non-blocking: a Ctrl-C handler must never hang behind
943
+ # another process's long apply/verify holding the lock. The
944
+ # workspace is already retained above; an unmarked manifest
945
+ # is later reconciled by `syncmill recover`.
946
+ artifact_store.recovery_required(run_id, str(exc), blocking=False)
947
+ except ApplyLockBusy:
948
+ logger.warning(
949
+ "apply lock busy; run %s not marked recovery-required "
950
+ "(workspace retained — run `syncmill recover %s`)",
951
+ run_id,
952
+ run_id,
953
+ )
954
+ except Exception:
955
+ logger.warning("could not mark run recovery-required", exc_info=True)
956
+ raise
957
+ finally:
958
+ # Hand the final result to the monitor first (writes result.json for the
959
+ # OUTER, keeps the control pane's result on screen), then end the memory
960
+ # session and tear every worktree down. If the strategy RAISED (result is
961
+ # None), still finalize the monitor with a synthetic FAILED result so the
962
+ # OUTER reads a real failure + non-zero exit code rather than mistaking the
963
+ # crash for a clean detach ("exit 0"). Monitor finalization is
964
+ # observability — guard it so a failure (e.g. result.json write error) can
965
+ # NEVER skip the memory-session close or worktree cleanup below.
966
+ # The envelope first — BEFORE monitor.finish (whose result.json write
967
+ # unblocks the OUTER, whose report-path print is existence-gated: the
968
+ # final envelope must already be on disk) and BEFORE pool.cleanup_all()
969
+ # (the spec's ordering; no worktree race — patch/files_changed were
970
+ # already captured by _collect_files_changed). Reporting is
971
+ # observability — guard it like the monitor so a failure can NEVER
972
+ # skip the memory-session close or worktree cleanup below.
973
+ # The trace flush comes first — the reporter's finish digests the
974
+ # artifact into the envelope's trace block, so the file must already
975
+ # be on disk. Tracing is observability: both steps are guarded so a
976
+ # tracer failure can NEVER skip the envelope, the memory-session
977
+ # close, or the worktree cleanup below.
978
+ trace_block: dict | None = None
979
+ if tracer is not None:
980
+ try:
981
+ final = result if result is not None else _crash_result(task, run_id)
982
+ tracer.record(
983
+ run_span(
984
+ strategy=self._config.default_strategy,
985
+ status=(
986
+ "completed"
987
+ if final.status == TaskStatus.COMPLETED
988
+ else "failed"
989
+ ),
990
+ start=run_start,
991
+ end=time.time(),
992
+ )
993
+ )
994
+ except Exception:
995
+ logger.warning(
996
+ "trace run-span record failed; continuing cleanup",
997
+ exc_info=True,
998
+ )
999
+ trace_block = flush_trace(tracer) # never raises
1000
+ if reporter is not None:
1001
+ try:
1002
+ degraded = _degraded_sources(mem, self._breaker)
1003
+ _append_trace_degradation(degraded, trace_block)
1004
+ self.last_report_path = reporter.finish(
1005
+ result if result is not None else _crash_result(task, run_id),
1006
+ degraded=degraded,
1007
+ trace=trace_block,
1008
+ preflight=preflight.block if preflight is not None else None,
1009
+ )
1010
+ except Exception:
1011
+ logger.warning(
1012
+ "report finalization failed; continuing cleanup",
1013
+ exc_info=True,
1014
+ )
1015
+ if monitor is not None:
1016
+ try:
1017
+ monitor.finish(result if result is not None else _crash_result(task, run_id))
1018
+ except Exception:
1019
+ logger.warning(
1020
+ "monitor finalization failed; continuing cleanup",
1021
+ exc_info=True,
1022
+ )
1023
+ # Ending the memory session is best-effort like everything else in
1024
+ # this finalizer: guard it so a memtomem hiccup can never strand the
1025
+ # worktrees. Cleanup is the last, unconditional step.
1026
+ try:
1027
+ # A crash gets the same synthesized result the reporter and the
1028
+ # monitor above are given: the session was opened either way,
1029
+ # and leaving it open on the one path that ended badly is how a
1030
+ # long-lived process (`board run --loop`, the TUI) accumulates
1031
+ # them.
1032
+ run_memory_bounded(
1033
+ mem.end_task_session(
1034
+ session_id,
1035
+ result if result is not None else _crash_result(task, run_id),
1036
+ ),
1037
+ memory=mem, label="end_task_session",
1038
+ )
1039
+ except Exception:
1040
+ logger.warning(
1041
+ "memory session close failed; continuing cleanup",
1042
+ exc_info=True,
1043
+ )
1044
+ pool.cleanup_all()
1045
+
1046
+ # ── Board run lifecycle (Phase 1) ────────────────────────────────────────
1047
+
1048
+ def _require_board(self) -> Board:
1049
+ """The board, built lazily on first use. Raises ``BoardError`` if the board
1050
+ is disabled (so a board CLI command run without ``SYNCMILL_BOARD__ENABLED``
1051
+ gets a clear message) — or if the db is corrupt (``build_board`` →
1052
+ ``Board.__init__`` raises). Stored on ``self._board`` so it's opened once
1053
+ per Supervisor and closed in ``close()``."""
1054
+ if self._board is None:
1055
+ self._board = build_board(self._config.board)
1056
+ if self._board is None:
1057
+ raise BoardError(
1058
+ "the work-board is disabled — enable it with "
1059
+ "SYNCMILL_BOARD__ENABLED=true (or [board] enabled=true in syncmill.toml)"
1060
+ )
1061
+ return self._board
1062
+
1063
+ def run_item(
1064
+ self,
1065
+ item_id: str | None = None,
1066
+ agent_id: str | None = None,
1067
+ exclude: set[str] | None = None,
1068
+ goal: str | None = None,
1069
+ ) -> TaskResult | None:
1070
+ """Claim a board item, run it through the normal pipeline, and record the
1071
+ outcome back onto the board. The board's run-lifecycle wrapper around
1072
+ :meth:`run` — the ``Strategy`` protocol is unchanged.
1073
+
1074
+ Returns the run's ``TaskResult``, or ``None`` when there is nothing to
1075
+ dispatch (an unclaimable id, or no runnable item) so a ``--loop`` caller
1076
+ stops. Flow: recover stale claims → atomically claim (``claim`` for an
1077
+ explicit id, else ``next_runnable``) → render the item DETERMINISTICALLY
1078
+ into the prompt → ``run`` it with a board-scoped memory (no global-``shared``
1079
+ leak) → parse the agent's optional board block → ``apply_update`` (advisory:
1080
+ an invalid agent ``done`` clamps to ``review``) → write ONE sanitized
1081
+ summary to global ``shared``. A genuine orchestration CRASH transitions the
1082
+ item ``→ blocked`` and re-raises (a FAILED result is NOT a crash —
1083
+ ``apply_update`` records it).
1084
+
1085
+ ``--strategy`` is intentionally NOT a parameter: the board CLI mutates
1086
+ ``config.default_strategy`` BEFORE constructing the Supervisor, exactly as
1087
+ plain ``run`` selects its strategy in ``__init__``. A ``goal`` scopes the
1088
+ ``next_runnable`` claim to that goal (the ``board run --goal`` filter,
1089
+ Phase 4); the claim-by-id path ignores it — the CLI makes ``--id`` and
1090
+ ``--goal`` mutually exclusive."""
1091
+ board = self._require_board()
1092
+ self.last_board_warning = None
1093
+ board.recover_stale_running()
1094
+ run_id = str(uuid4())
1095
+ try:
1096
+ # Side-effect-free view construction validates the FULL backend scope
1097
+ # before any load-bearing board mutation. A missing backend remains
1098
+ # advisory and still returns a degraded view.
1099
+ board_memory = self._memory.scoped(run_id)
1100
+ except ValueError as exc:
1101
+ raise BoardError(str(exc)) from exc
1102
+ item = (
1103
+ board.claim(item_id, run_id)
1104
+ if item_id is not None
1105
+ else board.next_runnable(run_id, goal=goal)
1106
+ )
1107
+ if item is None:
1108
+ # Nothing to dispatch — housekeeping can afford to run NOW (no item
1109
+ # is waiting behind it). The scratch-namespace sweep used to run
1110
+ # BEFORE every claim, where a contended purge stalled dispatch by up
1111
+ # to its full budget per iteration; it now runs only when the claim
1112
+ # drained (here) or after an item's outcome is recorded (below).
1113
+ self._gc_board_run_namespaces(board)
1114
+ return None # unclaimable id / nothing runnable → caller's loop stops
1115
+
1116
+ heartbeat = (
1117
+ _LeaseHeartbeat(board, item.id, run_id)
1118
+ if isinstance(board, Board)
1119
+ else None
1120
+ )
1121
+ try:
1122
+ if heartbeat is not None:
1123
+ # Started INSIDE the crash guard: the claim already succeeded, so a
1124
+ # failure spawning the thread has to strand-proof the item like any
1125
+ # other crash instead of leaving it `running` until stale recovery
1126
+ # reclaims it a lease-timeout later.
1127
+ heartbeat.start()
1128
+ # Active team decisions for this item's goal are injected DETERMINISTICALLY
1129
+ # (in full, never via top-k recall) so the agent can't re-litigate them
1130
+ # (Phase 2, D1). Best-effort: a decisions read failure degrades to none
1131
+ # rather than blocking the item — decision injection is advisory prompt
1132
+ # enrichment (like memory context), not part of the load-bearing dispatch.
1133
+ try:
1134
+ decisions = [d.text for d in board.active_decisions(goal=item.goal)]
1135
+ except BoardError:
1136
+ logger.warning(
1137
+ "board: failed to load decisions for item %s — running without",
1138
+ item.id, exc_info=True,
1139
+ )
1140
+ decisions = []
1141
+ # The prevalidated board-scoped view keeps enriched writes out of global
1142
+ # memory and UNION-READs shared cross-run context (D4).
1143
+ task = Task(prompt=render_item_prompt(item, active_decisions=decisions))
1144
+ # Pass the lease/scope run_id through — lease, `board-run:{run_id}`
1145
+ # memory scope, and the run's own identity are ONE id (no double
1146
+ # issuance; the run-envelope contract's single-resolver rule).
1147
+ # Board runs are a PR-2 model-routing surface: routing is disabled
1148
+ # here so an observe-mode config can't classify the ENRICHED board
1149
+ # prompt (the contract input is the raw item.task, §4.1).
1150
+ result = self.run(
1151
+ task,
1152
+ agent_id=agent_id,
1153
+ exclude=exclude,
1154
+ memory=board_memory,
1155
+ run_id=run_id,
1156
+ enable_routing=False,
1157
+ )
1158
+ # The agent is finished. Stop renewing before the final board write,
1159
+ # which intentionally moves the item out of ``running``.
1160
+ lease_warning: str | None = None
1161
+ if heartbeat is not None:
1162
+ heartbeat.stop()
1163
+ lease_warning = heartbeat.warning
1164
+ heartbeat = None
1165
+ parsed = parse_board_update(result.output)
1166
+ # Pass run_id so apply_update is lease-fenced: if this item was reclaimed
1167
+ # + re-dispatched while we ran (a slow run past the lease, with a second
1168
+ # concurrent `board run`), the stale write is dropped, never clobbering
1169
+ # the new owner or reverting a verified state.
1170
+ _, applied = board.apply_update(
1171
+ item.id, parsed, result, actor=result.agent_id, run_id=run_id
1172
+ )
1173
+ if not applied:
1174
+ # The fence dropped this run's write: the item is no longer `running`
1175
+ # under our lease, so everything below (the envelope board block
1176
+ # aside) is deliberately suppressed and the agent's work is discarded.
1177
+ # Surface it — the run otherwise exits 0 looking like a normal success,
1178
+ # and the operator never learns the work was thrown away.
1179
+ # `last_board_warning` is the CLI seam (`last_report_path` /
1180
+ # `preflight_warnings` precedent).
1181
+ self.last_board_warning = lease_warning or (
1182
+ f"board update for item {printable_name(str(item.id))} was dropped "
1183
+ f"— this run no "
1184
+ f"longer owns the lease (its state or lease owner changed)"
1185
+ )
1186
+ logger.warning("%s", self.last_board_warning)
1187
+ # Deliberately NOT set when `applied` is True, even if the heartbeat
1188
+ # reported a failure: a renewal that merely timed out behind a writer
1189
+ # (with no recovery in between) still ends with the fenced write landing,
1190
+ # and claiming that result "was NOT recorded" would contradict the board.
1191
+ # `applied` is the authoritative signal; the heartbeat's own trouble is
1192
+ # already logged where it happened.
1193
+ # Best-effort: stamp the finished envelope's `board` block (run()'s
1194
+ # own finally already finalized it — this is a read-modify-write).
1195
+ # Deliberately NOT gated on the lease fence: the fence protects the
1196
+ # board/shared system of record, while the envelope records what THIS
1197
+ # run worked on — its own identity, true either way. The enabled check
1198
+ # keeps the disabled path at zero filesystem touches.
1199
+ if self._config.integration.report.enabled:
1200
+ patch_envelope_board(
1201
+ self._config.integration.report.runs_dir,
1202
+ run_id,
1203
+ item_id=item.id,
1204
+ goal=item.goal or None,
1205
+ )
1206
+ # The ONE board-derived write to GLOBAL shared (cross-link tag), built
1207
+ # from the sanitized output — never the enriched prompt. Gated on the
1208
+ # SAME lease fence as the board write: if this run lost its lease (the
1209
+ # item's state or lease owner changed while we ran) apply_update dropped
1210
+ # the board update, so we must NOT publish this superseded run's output
1211
+ # to global `shared` either — otherwise the `board-result:{item_id}`
1212
+ # cross-link would carry discarded work beside the new owner's summary,
1213
+ # breaking the "single sanitized summary" / system-of-record contract.
1214
+ if applied:
1215
+ run_memory_bounded(
1216
+ self._memory.share_board_result(
1217
+ item.id, item.task, strip_board_block(result.output)
1218
+ ),
1219
+ memory=self._memory, label="share_board_result",
1220
+ )
1221
+ # Housekeeping AFTER the item's outcome is on the board — never on
1222
+ # the claim path (dispatch latency) and never on the crash path
1223
+ # (a Ctrl-C must not wait out a contended purge).
1224
+ self._gc_board_run_namespaces(board, exclude_run_id=run_id)
1225
+ return result
1226
+ except BaseException as exc:
1227
+ # A genuine crash (in run / parse / apply_update / share) — NOT a FAILED
1228
+ # result, which apply_update already recorded. Strand-proof the item by
1229
+ # moving it OUT of running, then re-raise.
1230
+ self._block_on_crash(board, item.id, run_id, exc)
1231
+ raise
1232
+ finally:
1233
+ if heartbeat is not None:
1234
+ heartbeat.stop()
1235
+ # A concurrent runner may have reclaimed this lease and purged its
1236
+ # scope while it was still finishing. Re-arm the durable tombstone
1237
+ # after all scoped-memory activity so a later sweep catches any
1238
+ # recreation. Best-effort: cleanup bookkeeping never masks a result.
1239
+ try:
1240
+ board.reactivate_board_run_namespace(run_id)
1241
+ except Exception:
1242
+ logger.warning(
1243
+ "board: failed to reactivate scratch ledger for run %s",
1244
+ run_id,
1245
+ exc_info=True,
1246
+ )
1247
+
1248
+ def _gc_board_run_namespaces(
1249
+ self, board: Board, *, exclude_run_id: str | None = None
1250
+ ) -> None:
1251
+ """Best-effort cleanup of board-owned, non-live scratch namespaces.
1252
+
1253
+ Board SQLite now owns discovery and liveness in one snapshot: claims
1254
+ commit their lease and ledger row atomically. Only successful per-run
1255
+ backend calls are tombstoned; failures are scheduled for backoff retry
1256
+ (``record_board_run_purge_failures``) so one poisoned namespace can't
1257
+ head-of-line-block the sweep forever. A cross-process advisory lock
1258
+ (non-blocking, keyed by the board db) lets concurrent ``--loop`` workers
1259
+ SKIP a sweep another worker is already running instead of queueing on
1260
+ the same namespace locks to redo its work. ``exclude_run_id`` keeps the
1261
+ just-finished run's own namespace out of its post-run sweep (the finally
1262
+ re-arms its ledger row right after; purging it here would only mint a
1263
+ no-op re-purge next sweep).
1264
+ """
1265
+ try:
1266
+ db_path = getattr(board, "db_path", None)
1267
+ gc_lock = (
1268
+ _cross_process_lock(
1269
+ Path(f"{db_path}.gc-lock"), blocking=False
1270
+ )
1271
+ if db_path is not None
1272
+ else contextlib.nullcontext()
1273
+ )
1274
+ try:
1275
+ with gc_lock:
1276
+ self._gc_board_run_namespaces_locked(
1277
+ board, exclude_run_id=exclude_run_id
1278
+ )
1279
+ except ApplyLockBusy:
1280
+ logger.debug(
1281
+ "board-run namespace GC already running in another process"
1282
+ )
1283
+ except Exception:
1284
+ logger.warning(
1285
+ "board-run namespace GC failed; continuing dispatch",
1286
+ exc_info=True,
1287
+ )
1288
+
1289
+ def _gc_board_run_namespaces_locked(
1290
+ self, board: Board, *, exclude_run_id: str | None
1291
+ ) -> None:
1292
+ candidates = [
1293
+ rid
1294
+ for rid in board.collectible_board_run_ids(limit=_SCRATCH_GC_BATCH_SIZE)
1295
+ if rid != exclude_run_id
1296
+ ]
1297
+ if not candidates:
1298
+ return
1299
+ # purge_board_runs owns a cooperative deadline, including readiness.
1300
+ # Do not wrap it in another timeout: cancelling its receipt collection
1301
+ # turns a partial purge into None and bypasses both ledger and backoff.
1302
+ result = run_async(self._memory.purge_board_runs(candidates))
1303
+ if result is None:
1304
+ # Backend unavailable: no per-namespace verdict was reached — don't
1305
+ # penalize rows for an outage that would delay collection after the
1306
+ # backend recovers.
1307
+ return
1308
+ marked = board.mark_board_run_namespaces_purged(result.successful)
1309
+ if marked:
1310
+ logger.info("board: purged %d stale board-run namespace(s)", marked)
1311
+ failed = [rid for rid in candidates if rid in result.failed]
1312
+ if failed and hasattr(board, "record_board_run_purge_failures"):
1313
+ quarantined = board.record_board_run_purge_failures(failed)
1314
+ for rid in quarantined:
1315
+ logger.warning(
1316
+ "board: scratch namespace for run %s failed to purge %d "
1317
+ "times in a row — quarantined to a slow retry schedule; "
1318
+ "its store residue likely needs operator attention "
1319
+ "(symlinked source, unmounted index root, or legacy day "
1320
+ "file)",
1321
+ rid,
1322
+ Board.PURGE_QUARANTINE_ATTEMPTS,
1323
+ )
1324
+
1325
+ def _block_on_crash(
1326
+ self, board: Board, item_id: str, run_id: str, exc: BaseException
1327
+ ) -> None:
1328
+ """Best-effort, LEASE-FENCED: move a crashed run's item to ``blocked`` so it
1329
+ isn't stranded in ``running`` — but ONLY while this run still owns the claim.
1330
+ Passing ``run_id`` fences the transition exactly like ``apply_update``: if the
1331
+ item was reclaimed + re-dispatched while we ran (a slow run past its lease with
1332
+ a second concurrent ``board run``), or already left ``running`` (a crash AFTER
1333
+ ``apply_update`` resolved it — e.g. Ctrl-C during the shared-memory write), the
1334
+ fenced write matches no rows and is dropped, so a stale crashed run never
1335
+ clobbers the new owner or reverts a verified state. Never raises — a board
1336
+ failure here must not mask the original crash."""
1337
+ try:
1338
+ note = (
1339
+ "orchestration cancelled by operator (SIGINT)"
1340
+ if isinstance(exc, KeyboardInterrupt)
1341
+ else f"orchestration crashed: {type(exc).__name__}: {exc}"[:500]
1342
+ )
1343
+ board.transition(
1344
+ item_id, WorkState.BLOCKED.value, actor="system",
1345
+ note=note,
1346
+ # Also onto the ROW, not just the audit note: `board show` reads
1347
+ # `blocked_reason`, and a crashed item is exactly the one an operator
1348
+ # opens first.
1349
+ blocked_reason=note,
1350
+ human_required=True, run_id=run_id,
1351
+ )
1352
+ except Exception:
1353
+ logger.warning(
1354
+ "failed to mark item %s blocked after a crash", item_id, exc_info=True
1355
+ )
1356
+
1357
+ def close(self) -> None:
1358
+ # The board close is in a `finally`: a RuntimeError out of the memory close
1359
+ # (a closed/foreign event loop) would otherwise skip it and leak the SQLite
1360
+ # connection — and in a long-lived process (`tui`, `board run --loop`) that
1361
+ # accumulates.
1362
+ try:
1363
+ # Bounded (cooperatively — see run_memory_bounded): an
1364
+ # await-shaped hang in the backend close must not hang shutdown.
1365
+ run_memory_bounded(self._memory.close(), label="memory close")
1366
+ finally:
1367
+ if self._board is not None:
1368
+ self._board.close()
1369
+
1370
+ def _require_workflow(self) -> WorkflowCoordinator:
1371
+ if self._workflow is None:
1372
+ self._workflow = WorkflowCoordinator(self._config, self._registry)
1373
+ return self._workflow
1374
+
1375
+ def plan_workflow(
1376
+ self, goal: str, planner_agent: str | None = None
1377
+ ) -> WorkflowRecord:
1378
+ """Plan and persist a bounded declarative workflow without executing it."""
1379
+ return self._require_workflow().plan(goal, planner_agent=planner_agent)
1380
+
1381
+ def run_workflow(
1382
+ self,
1383
+ workflow_id: str,
1384
+ *,
1385
+ resume: bool = False,
1386
+ owner: str | None = None,
1387
+ job_id: str | None = None,
1388
+ ) -> WorkflowRecord:
1389
+ """Execute or resume a previously approved native workflow."""
1390
+ return self._require_workflow().run(
1391
+ workflow_id, resume=resume, owner=owner, job_id=job_id
1392
+ )
1393
+
1394
+ @property
1395
+ def workflow_store(self) -> WorkflowStore:
1396
+ return self._require_workflow().store
1397
+
1398
+ @property
1399
+ def workflow_preflight_warnings(self) -> list[str]:
1400
+ """Advisory preflight warnings from the last workflow operation.
1401
+
1402
+ Mirrors ``preflight_warnings`` for the workflow path so the CLI has one
1403
+ place to read them from; empty when no workflow ran in this process.
1404
+ """
1405
+ if self._workflow is None:
1406
+ return []
1407
+ return list(self._workflow.preflight_warnings)
1408
+
1409
+ @property
1410
+ def registry(self) -> AgentRegistry:
1411
+ return self._registry
1412
+
1413
+ @property
1414
+ def memory(self) -> SyncmillMemory:
1415
+ return self._memory
1416
+
1417
+ @property
1418
+ def breaker(self) -> CircuitBreaker | None:
1419
+ return self._breaker
1420
+
1421
+ @property
1422
+ def matcher(self) -> CapabilityMatcher | None:
1423
+ return self._matcher