syncmill 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syncmill/__init__.py +11 -0
- syncmill/agents.py +2064 -0
- syncmill/artifact_io.py +201 -0
- syncmill/artifacts.py +1946 -0
- syncmill/board.py +2060 -0
- syncmill/breaker.py +327 -0
- syncmill/cli.py +4986 -0
- syncmill/command_errors.py +32 -0
- syncmill/config.py +1118 -0
- syncmill/dashboard.py +935 -0
- syncmill/gates.py +179 -0
- syncmill/gateway.py +234 -0
- syncmill/git_env.py +99 -0
- syncmill/isolated_session.py +431 -0
- syncmill/matcher.py +127 -0
- syncmill/memory.py +1067 -0
- syncmill/memory_backend.py +1907 -0
- syncmill/memory_migration.py +962 -0
- syncmill/model_routing.py +482 -0
- syncmill/models.py +244 -0
- syncmill/monitor.py +775 -0
- syncmill/phoenix_exporter.py +339 -0
- syncmill/planning.py +512 -0
- syncmill/platform_support.py +14 -0
- syncmill/preflight.py +1129 -0
- syncmill/project.py +1067 -0
- syncmill/report.py +502 -0
- syncmill/review_candidates.py +107 -0
- syncmill/room_compare.py +2380 -0
- syncmill/room_raw.py +484 -0
- syncmill/room_viewer.py +604 -0
- syncmill/rooms.py +4044 -0
- syncmill/session.py +444 -0
- syncmill/strategies/__init__.py +82 -0
- syncmill/strategies/base.py +910 -0
- syncmill/strategies/compete.py +233 -0
- syncmill/strategies/council.py +429 -0
- syncmill/strategies/decompose.py +570 -0
- syncmill/strategies/pipeline.py +221 -0
- syncmill/strategies/route.py +130 -0
- syncmill/supervisor.py +1423 -0
- syncmill/teamfile.py +571 -0
- syncmill/telemetry_hook.py +231 -0
- syncmill/terminal.py +245 -0
- syncmill/tracer.py +548 -0
- syncmill/tui.py +3537 -0
- syncmill/tui_collaboration.py +499 -0
- syncmill/tui_jobs.py +1407 -0
- syncmill/tui_render.py +156 -0
- syncmill/workflow.py +3746 -0
- syncmill/worktree.py +1045 -0
- syncmill-0.2.0.dist-info/METADATA +853 -0
- syncmill-0.2.0.dist-info/RECORD +56 -0
- syncmill-0.2.0.dist-info/WHEEL +4 -0
- syncmill-0.2.0.dist-info/entry_points.txt +2 -0
- syncmill-0.2.0.dist-info/licenses/LICENSE +191 -0
|
@@ -0,0 +1,910 @@
|
|
|
1
|
+
"""Strategy Protocol + the helpers shared across every strategy.
|
|
2
|
+
|
|
3
|
+
This is the foundation layer of the ``strategies`` package: the ``Strategy``
|
|
4
|
+
Protocol every strategy structurally satisfies, plus the cross-cutting helpers
|
|
5
|
+
(prompt enrichment, gate/breaker plumbing, failure builders, winner selection,
|
|
6
|
+
per-worktree diff collection) that more than one strategy reuses. The concrete
|
|
7
|
+
strategy modules
|
|
8
|
+
(``route``/``pipeline``/``compete``/``council``/``decompose``) import from here;
|
|
9
|
+
nothing here imports them, so the package stays acyclic. The Supervisor
|
|
10
|
+
sits *above* the strategy modules (it imports the registry), so diff collection
|
|
11
|
+
lives here — below the strategies — rather than in ``supervisor.py``, which
|
|
12
|
+
would create an import cycle.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import contextlib
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import select
|
|
22
|
+
import subprocess
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Protocol, runtime_checkable
|
|
25
|
+
|
|
26
|
+
import time
|
|
27
|
+
|
|
28
|
+
from syncmill.agents import (
|
|
29
|
+
AgentRunner,
|
|
30
|
+
_close_process_streams,
|
|
31
|
+
_kill_process_group,
|
|
32
|
+
provider_session_scope,
|
|
33
|
+
scrub_prompt_text,
|
|
34
|
+
)
|
|
35
|
+
from syncmill.breaker import CircuitBreaker
|
|
36
|
+
from syncmill.config import SyncmillConfig
|
|
37
|
+
from syncmill.gates import QualityGate
|
|
38
|
+
from syncmill.git_env import isolated_git_env
|
|
39
|
+
from syncmill.memory import SyncmillMemory
|
|
40
|
+
from syncmill.models import GateReport, Task, TaskResult, TaskStatus, ToolOutcome
|
|
41
|
+
from syncmill.monitor import Monitor
|
|
42
|
+
from syncmill.tracer import SpanRecord, Tracer
|
|
43
|
+
from syncmill.worktree import Worktree, WorktreePool, _run_git, capture_changes
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# ── Strategy Protocol ────────────────────────────────────────────────────
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@runtime_checkable
|
|
52
|
+
class Strategy(Protocol):
|
|
53
|
+
"""Orchestration strategy — determines how agents are selected and run.
|
|
54
|
+
|
|
55
|
+
A strategy is handed a ``WorktreePool`` rather than a single ``workdir``: it
|
|
56
|
+
owns its worktree *topology* (one isolated worktree per agent for concurrent
|
|
57
|
+
isolation, or a single shared one for a sequential pipeline), acquiring from
|
|
58
|
+
the pool as it runs. The Supervisor owns the pool's *lifetime* and tears
|
|
59
|
+
every acquired worktree down afterward. The strategy also sets
|
|
60
|
+
``files_changed`` on the result it returns (only it knows which worktree
|
|
61
|
+
produced that result) via ``_collect_files_changed``.
|
|
62
|
+
|
|
63
|
+
The optional ``gate`` (#10) is a post-finish quality check the strategy runs
|
|
64
|
+
against an agent's worktree before accepting its result; ``None`` means no
|
|
65
|
+
gate is configured and the strategy accepts a completed result as today.
|
|
66
|
+
|
|
67
|
+
The optional ``breaker`` (#8) is a reactive circuit breaker the strategy
|
|
68
|
+
consults to skip agents whose circuit is OPEN (too many recent failures) and
|
|
69
|
+
to record each agent's raw execution outcome; ``None`` means no breaker is
|
|
70
|
+
configured and every agent runs as today.
|
|
71
|
+
|
|
72
|
+
The optional ``monitor`` (#16) is a live tmux monitor the CONCURRENT strategies
|
|
73
|
+
drive — announcing each phase, opening the output-streaming seam around each
|
|
74
|
+
fan-out, and reporting per-agent status; the sequential strategies accept it
|
|
75
|
+
and ignore it (there is nothing concurrent to watch). ``None`` means no monitor,
|
|
76
|
+
so behavior is unchanged.
|
|
77
|
+
|
|
78
|
+
The optional ``tracer`` (S3′, completed by S6) is a retrospective span
|
|
79
|
+
recorder: a strategy appends completed span records right after each
|
|
80
|
+
attempt/fan-out returns (the same shared-loop boundary as
|
|
81
|
+
``_breaker_record``), via the ``_trace_*`` helpers and span builders below
|
|
82
|
+
— never from inside a fan-out, and never through the agents (agents stay
|
|
83
|
+
pure). Unlike ``monitor``, the SEQUENTIAL strategies use it too
|
|
84
|
+
(retrospective recording is sync-safe): all five strategies record at
|
|
85
|
+
their own attempt/phase boundaries. ``None`` means no tracing, so behavior
|
|
86
|
+
is unchanged.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def execute(
|
|
90
|
+
self,
|
|
91
|
+
task: Task,
|
|
92
|
+
agents: list[AgentRunner],
|
|
93
|
+
memory: SyncmillMemory,
|
|
94
|
+
pool: WorktreePool,
|
|
95
|
+
config: SyncmillConfig,
|
|
96
|
+
gate: QualityGate | None = None,
|
|
97
|
+
breaker: CircuitBreaker | None = None,
|
|
98
|
+
monitor: Monitor | None = None,
|
|
99
|
+
tracer: Tracer | None = None,
|
|
100
|
+
) -> TaskResult: ...
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
#: Byte cap on the shared-memory context a strategy may prepend, mirroring
|
|
104
|
+
#: ``rooms._ROOM_MEMORY_CONTEXT_MAX_BYTES``. Enrichment is best-effort: an
|
|
105
|
+
#: unbounded recall (a run that stored a tens-of-MB transcript) would otherwise
|
|
106
|
+
#: push the composed prompt past the platform's argv limit and turn every
|
|
107
|
+
#: agent launch into an E2BIG ``OSError``.
|
|
108
|
+
_MEMORY_CONTEXT_MAX_BYTES = 4_000
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _enrich_prompt(prompt: str, context: str) -> str:
|
|
112
|
+
"""Prepend shared-memory context to a task prompt, if any.
|
|
113
|
+
|
|
114
|
+
Shared by every strategy so the enrichment format stays identical across
|
|
115
|
+
them (RouteStrategy, PipelineStrategy, …).
|
|
116
|
+
|
|
117
|
+
The context is recalled agent output — untrusted text that goes straight
|
|
118
|
+
into an argv element — so it is bounded and scrubbed here, at the single
|
|
119
|
+
seam every strategy composes through. search_context coordinates with this
|
|
120
|
+
via its default _CONTEXT_MAX_BYTES (3,800 bytes). The 4,000-byte context-only
|
|
121
|
+
cap remains a backstop for other callers or larger custom budgets; neither
|
|
122
|
+
limit includes the wrapper or the original task prompt.
|
|
123
|
+
A NUL in it would make the launch raise ``ValueError`` and freeze the run as
|
|
124
|
+
``recovery_required`` without running an agent; ``rooms.py`` already scrubs
|
|
125
|
+
its own equivalent prefix. The TASK prompt is never touched: it is the
|
|
126
|
+
caller's own text and must reach the agent verbatim. Note that this is a
|
|
127
|
+
deliberate choice, not a safety claim — only `--prompt-file` is validated
|
|
128
|
+
at the CLI boundary, so a positional prompt carrying a NUL reaches the
|
|
129
|
+
agent layer, where the launch pre-check turns it into a FAILED result
|
|
130
|
+
rather than an escaped exception.
|
|
131
|
+
"""
|
|
132
|
+
if context:
|
|
133
|
+
# Normalize ALWAYS, not only when truncating. `surrogateescape` only
|
|
134
|
+
# round-trips the \udc80-\udcff range, so an arbitrary lone surrogate
|
|
135
|
+
# (\ud800) raised UnicodeEncodeError right here — inside a best-effort
|
|
136
|
+
# enrichment path that is supposed to degrade, never throw. `replace`
|
|
137
|
+
# on the ENCODE side cannot raise for any input.
|
|
138
|
+
raw = context.encode("utf-8", errors="replace")[:_MEMORY_CONTEXT_MAX_BYTES]
|
|
139
|
+
# `ignore` on the way back drops a codepoint the byte cut split, so the
|
|
140
|
+
# result is valid UTF-8 AND genuinely within the budget (a replacement
|
|
141
|
+
# character would push a boundary cut back over it).
|
|
142
|
+
context = scrub_prompt_text(raw.decode("utf-8", errors="ignore")).strip()
|
|
143
|
+
if context:
|
|
144
|
+
return f"Context from previous work:\n{context}\n\n---\n\nTask: {prompt}"
|
|
145
|
+
return prompt
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _collect_files_changed(result: TaskResult, worktree: Worktree) -> None:
|
|
149
|
+
"""Capture the completed winner before its mandatory artifact is persisted."""
|
|
150
|
+
if worktree.is_git and result.status == TaskStatus.COMPLETED:
|
|
151
|
+
changes = capture_changes(worktree.path, getattr(worktree, "base_commit", None))
|
|
152
|
+
result.files_changed = list(changes.files)
|
|
153
|
+
result.patch = changes.patch
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
#: Hang bound for the diff/patch helpers below (via worktree._run_git, which
|
|
157
|
+
#: also group-kills git's own descendants — a filter/fsmonitor child would
|
|
158
|
+
#: otherwise survive the timeout). The reader already degrades on rc!=0, so a
|
|
159
|
+
#: timeout's rc-124 lands in that net — the bound exists so a wedged git (stale
|
|
160
|
+
#: index.lock, a prompting filter) can't hang the post-fan-out path forever.
|
|
161
|
+
_GIT_TIMEOUT_SECONDS = 120.0
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
_EXCERPT_TRUNCATION_MARKER = "\n[... truncated for the judge prompt ...]"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _bounded_head(text: str, max_bytes: int) -> str:
|
|
168
|
+
"""Byte-bounded head of ``text`` with a truncation marker when cut."""
|
|
169
|
+
data = text.encode("utf-8", errors="replace")
|
|
170
|
+
if len(data) <= max_bytes:
|
|
171
|
+
return text
|
|
172
|
+
return (
|
|
173
|
+
data[:max_bytes].decode("utf-8", errors="replace")
|
|
174
|
+
+ _EXCERPT_TRUNCATION_MARKER
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _git_stdout_head(
|
|
179
|
+
cmd: list[str], workdir: Path, max_bytes: int
|
|
180
|
+
) -> tuple[bytes, bool] | None:
|
|
181
|
+
"""Read at most ``max_bytes`` of a git command's stdout, then stop it.
|
|
182
|
+
|
|
183
|
+
The judge prompt keeps only a head window, so materializing a multi-MB diff
|
|
184
|
+
just to slice it is pure peak memory — this reads the pipe in chunks and
|
|
185
|
+
group-kills git the moment the window is full (POSIX; elsewhere it degrades
|
|
186
|
+
to a bounded slice of a full ``_run_git`` capture). Returns
|
|
187
|
+
``(head, truncated)``, or ``None`` on any failure/timeout — best-effort,
|
|
188
|
+
like every diff helper here.
|
|
189
|
+
"""
|
|
190
|
+
if os.name != "posix":
|
|
191
|
+
proc = _run_git(
|
|
192
|
+
cmd, capture_output=True, cwd=str(workdir),
|
|
193
|
+
env=isolated_git_env(), timeout=_GIT_TIMEOUT_SECONDS,
|
|
194
|
+
)
|
|
195
|
+
if proc.returncode != 0:
|
|
196
|
+
return None
|
|
197
|
+
return proc.stdout[:max_bytes], len(proc.stdout) > max_bytes
|
|
198
|
+
deadline = time.monotonic() + _GIT_TIMEOUT_SECONDS
|
|
199
|
+
proc = subprocess.Popen(
|
|
200
|
+
cmd,
|
|
201
|
+
cwd=str(workdir),
|
|
202
|
+
env=isolated_git_env(),
|
|
203
|
+
stdout=subprocess.PIPE,
|
|
204
|
+
stderr=subprocess.DEVNULL,
|
|
205
|
+
start_new_session=True,
|
|
206
|
+
)
|
|
207
|
+
buf = bytearray()
|
|
208
|
+
limit = max_bytes + 1 # read one byte past the window to detect truncation
|
|
209
|
+
try:
|
|
210
|
+
try:
|
|
211
|
+
fd = proc.stdout.fileno()
|
|
212
|
+
while len(buf) < limit:
|
|
213
|
+
wait = deadline - time.monotonic()
|
|
214
|
+
if wait <= 0: # wedged git — degrade like a _run_git timeout
|
|
215
|
+
_kill_process_group(proc, owned=True)
|
|
216
|
+
proc.kill()
|
|
217
|
+
with contextlib.suppress(Exception):
|
|
218
|
+
proc.wait(timeout=5.0)
|
|
219
|
+
return None
|
|
220
|
+
ready, _, _ = select.select([fd], [], [], min(wait, 1.0))
|
|
221
|
+
if not ready:
|
|
222
|
+
continue
|
|
223
|
+
chunk = os.read(fd, min(65536, limit - len(buf)))
|
|
224
|
+
if not chunk:
|
|
225
|
+
break
|
|
226
|
+
buf += chunk
|
|
227
|
+
truncated = len(buf) > max_bytes
|
|
228
|
+
if truncated:
|
|
229
|
+
del buf[max_bytes:]
|
|
230
|
+
# The tail is never needed — stop git instead of draining it.
|
|
231
|
+
_kill_process_group(proc, owned=True)
|
|
232
|
+
proc.kill()
|
|
233
|
+
with contextlib.suppress(Exception):
|
|
234
|
+
proc.wait(timeout=5.0)
|
|
235
|
+
return bytes(buf), True
|
|
236
|
+
try:
|
|
237
|
+
rc = proc.wait(timeout=max(deadline - time.monotonic(), 5.0))
|
|
238
|
+
except subprocess.TimeoutExpired:
|
|
239
|
+
_kill_process_group(proc, owned=True)
|
|
240
|
+
proc.kill()
|
|
241
|
+
with contextlib.suppress(Exception):
|
|
242
|
+
proc.wait(timeout=5.0)
|
|
243
|
+
return None
|
|
244
|
+
if rc != 0:
|
|
245
|
+
return None
|
|
246
|
+
return bytes(buf), False
|
|
247
|
+
except BaseException:
|
|
248
|
+
with contextlib.suppress(Exception):
|
|
249
|
+
_kill_process_group(proc, owned=True)
|
|
250
|
+
proc.kill()
|
|
251
|
+
with contextlib.suppress(Exception):
|
|
252
|
+
proc.wait(timeout=5.0)
|
|
253
|
+
raise
|
|
254
|
+
finally:
|
|
255
|
+
_close_process_streams(proc)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _git_diff_excerpt(
|
|
259
|
+
workdir: Path, base_commit: str | None = None, *, max_bytes: int = 12_000
|
|
260
|
+
) -> str | None:
|
|
261
|
+
"""Truncation-aware diff excerpt for the judge prompt (#116).
|
|
262
|
+
|
|
263
|
+
Produces only what the prompt will keep: ``--stat`` (a quarter of the
|
|
264
|
+
budget) plus a byte-bounded textual diff head — never ``--binary``, never a
|
|
265
|
+
full patch materialized in memory. Still stages the tree (``git add -A``)
|
|
266
|
+
so untracked files enter the diff; the cost removed is the full-patch
|
|
267
|
+
materialization, not the index work. ``None`` on any failure (non-git dir,
|
|
268
|
+
wedged git) — the judge section degrades to report-only.
|
|
269
|
+
"""
|
|
270
|
+
base = base_commit or "HEAD"
|
|
271
|
+
try:
|
|
272
|
+
add = _run_git(
|
|
273
|
+
["git", "add", "-A"], capture_output=True, cwd=str(workdir),
|
|
274
|
+
env=isolated_git_env(), timeout=_GIT_TIMEOUT_SECONDS,
|
|
275
|
+
)
|
|
276
|
+
if add.returncode != 0:
|
|
277
|
+
return None
|
|
278
|
+
# --stat is ONE LINE PER CHANGED FILE, so it is unbounded in the
|
|
279
|
+
# changed-file count and has to stream under the same head window as
|
|
280
|
+
# the diff body — capturing it whole first would reintroduce exactly
|
|
281
|
+
# the peak-memory problem this helper exists to remove.
|
|
282
|
+
stat_budget = max(1, max_bytes // 4)
|
|
283
|
+
stat_head = _git_stdout_head(
|
|
284
|
+
["git", "diff", "--cached", "--no-color", "--stat", base],
|
|
285
|
+
workdir,
|
|
286
|
+
stat_budget,
|
|
287
|
+
)
|
|
288
|
+
if stat_head is None:
|
|
289
|
+
return None
|
|
290
|
+
stat_bytes, stat_truncated = stat_head
|
|
291
|
+
stat_text = stat_bytes.decode("utf-8", errors="replace")
|
|
292
|
+
if stat_truncated:
|
|
293
|
+
stat_text += _EXCERPT_TRUNCATION_MARKER + "\n"
|
|
294
|
+
diff_budget = max(1, max_bytes - len(stat_bytes))
|
|
295
|
+
head = _git_stdout_head(
|
|
296
|
+
["git", "diff", "--cached", "--no-color", base], workdir, diff_budget
|
|
297
|
+
)
|
|
298
|
+
if head is None:
|
|
299
|
+
return None
|
|
300
|
+
diff_bytes, diff_truncated = head
|
|
301
|
+
excerpt = stat_text + diff_bytes.decode("utf-8", errors="replace")
|
|
302
|
+
if diff_truncated:
|
|
303
|
+
excerpt += _EXCERPT_TRUNCATION_MARKER
|
|
304
|
+
return excerpt
|
|
305
|
+
except Exception:
|
|
306
|
+
return None
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _gate_accepts(
|
|
310
|
+
result: TaskResult, worktree: Worktree, gate: QualityGate | None
|
|
311
|
+
) -> bool:
|
|
312
|
+
"""Run the quality gate against a COMPLETED result's worktree and judge it.
|
|
313
|
+
|
|
314
|
+
Returns ``True`` when the result is accepted — no gate configured, or the gate
|
|
315
|
+
passed. On a gate failure it returns ``False`` and **downgrades the result in
|
|
316
|
+
place** to FAILED with a gate error, so the rejection is reported through the
|
|
317
|
+
existing failure plumbing (RouteStrategy's aggregate, the Compete aggregate,
|
|
318
|
+
the pipeline failure) rather than silently accepted. The ``GateReport`` is
|
|
319
|
+
attached to ``result.gate`` either way (pass or fail) — an annotation like
|
|
320
|
+
``files_changed``, so callers/CLI can surface the verdict.
|
|
321
|
+
|
|
322
|
+
Shared by every strategy so "passed the gate" means exactly one thing across
|
|
323
|
+
them. Call only on a COMPLETED result — a failed agent is never gated (there's
|
|
324
|
+
nothing trustworthy to check).
|
|
325
|
+
"""
|
|
326
|
+
if gate is None:
|
|
327
|
+
return True
|
|
328
|
+
report = gate.evaluate(worktree)
|
|
329
|
+
result.gate = report
|
|
330
|
+
if report.passed:
|
|
331
|
+
return True
|
|
332
|
+
result.status = TaskStatus.FAILED
|
|
333
|
+
result.error = f"quality gate failed: {report.summary()}"
|
|
334
|
+
return False
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _breaker_runnable(
|
|
338
|
+
agents: list[AgentRunner], breaker: CircuitBreaker | None
|
|
339
|
+
) -> list[AgentRunner]:
|
|
340
|
+
"""Agents whose circuit allows a run (CLOSED or HALF-OPEN); all of them when
|
|
341
|
+
no breaker is configured.
|
|
342
|
+
|
|
343
|
+
Skipping an OPEN agent is reactive back-off from its *recorded* past failures
|
|
344
|
+
— never a liveness probe (no ``health_check``). Shared by every strategy so
|
|
345
|
+
"is this agent currently allowed to run" means one thing across them.
|
|
346
|
+
"""
|
|
347
|
+
if breaker is None:
|
|
348
|
+
return list(agents)
|
|
349
|
+
runnable: list[AgentRunner] = []
|
|
350
|
+
for agent in agents:
|
|
351
|
+
if breaker.allows(agent.agent_id):
|
|
352
|
+
runnable.append(agent)
|
|
353
|
+
else:
|
|
354
|
+
logger.info(
|
|
355
|
+
"circuit breaker open for %s — skipping this run", agent.agent_id
|
|
356
|
+
)
|
|
357
|
+
return runnable
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _breaker_record(
|
|
361
|
+
breaker: CircuitBreaker | None, agent_id: str, status: TaskStatus
|
|
362
|
+
) -> None:
|
|
363
|
+
"""Record an agent's RAW execution outcome (no-op without a breaker).
|
|
364
|
+
|
|
365
|
+
Called immediately after the agent runs and **before** the quality gate, so
|
|
366
|
+
the breaker tracks *execution reliability* (did it run to completion?) — a
|
|
367
|
+
gate rejection is a quality verdict on a working agent and must not trip the
|
|
368
|
+
breaker. Shared by every strategy so recording stays identical across them.
|
|
369
|
+
"""
|
|
370
|
+
if breaker is not None:
|
|
371
|
+
breaker.record(agent_id, status)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _breaker_all_open_failure(
|
|
375
|
+
task: Task, agents: list[AgentRunner]
|
|
376
|
+
) -> TaskResult:
|
|
377
|
+
"""Failure returned when every candidate agent's circuit is OPEN.
|
|
378
|
+
|
|
379
|
+
Distinct from ``_no_stages_failure``/"no agents available" so the user knows
|
|
380
|
+
it's the breaker cooling down after repeated failures, not a config gap.
|
|
381
|
+
|
|
382
|
+
Failing fast here is deliberate, not the breaker "aborting" work: every agent
|
|
383
|
+
is within its cooldown after recent recorded failures, so honoring the
|
|
384
|
+
back-off means there is nothing to run this invocation — the same shape as the
|
|
385
|
+
quality gate failing the run when every output is rejected. (The *advisory*
|
|
386
|
+
property — never blocking work — applies to a DEGRADED breaker, whose
|
|
387
|
+
``allows`` returns True for everyone; an OPEN circuit is the back-off working
|
|
388
|
+
as designed.) The error tells the user how to recover: wait for a cooldown,
|
|
389
|
+
or disable the breaker.
|
|
390
|
+
"""
|
|
391
|
+
names = ", ".join(a.agent_id for a in agents)
|
|
392
|
+
return TaskResult(
|
|
393
|
+
task_id=task.id,
|
|
394
|
+
agent_id="none",
|
|
395
|
+
status=TaskStatus.FAILED,
|
|
396
|
+
output="",
|
|
397
|
+
exit_code=-1,
|
|
398
|
+
duration_seconds=0.0,
|
|
399
|
+
error=(
|
|
400
|
+
f"All agents are circuit-open (cooling down after repeated failures): "
|
|
401
|
+
f"{names}. Retry once a cooldown elapses, or disable the breaker."
|
|
402
|
+
),
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _monitor_phase(monitor: Monitor | None, name: str) -> None:
|
|
407
|
+
"""Announce a phase to the live monitor (no-op without one). Shared so every
|
|
408
|
+
concurrent strategy labels its phases through one seam."""
|
|
409
|
+
if monitor is not None:
|
|
410
|
+
monitor.phase(name)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _monitor_streaming(monitor: Monitor | None, agent_ids: list[str]):
|
|
414
|
+
"""Open the output-streaming seam around a fan-out for ``agent_ids`` (and ensure
|
|
415
|
+
each has a live pane); a plain no-op context when no monitor is configured.
|
|
416
|
+
|
|
417
|
+
Wrap a concurrent strategy's ``run_async(aexecute_*)`` in this: every agent
|
|
418
|
+
task the fan-out creates inherits the monitor's ``OutputSink`` (via the
|
|
419
|
+
``stream_to`` ContextVar), so its output tees to the agent's tmux pane live.
|
|
420
|
+
"""
|
|
421
|
+
if monitor is None:
|
|
422
|
+
return contextlib.nullcontext()
|
|
423
|
+
return monitor.streaming(agent_ids)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _monitor_status(
|
|
427
|
+
monitor: Monitor | None, agent_id: str, status: TaskStatus
|
|
428
|
+
) -> None:
|
|
429
|
+
"""Report an agent's final outcome to the live monitor (no-op without one)."""
|
|
430
|
+
if monitor is not None:
|
|
431
|
+
monitor.status(agent_id, status)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _trace_record(tracer: Tracer | None, span: SpanRecord) -> None:
|
|
435
|
+
"""Append one completed span to the tracer (no-op without one). Shared so
|
|
436
|
+
every strategy records through one seam — like ``_monitor_status``."""
|
|
437
|
+
if tracer is not None:
|
|
438
|
+
tracer.record(span)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _span_status(raw_status: TaskStatus, error: str | None) -> str:
|
|
442
|
+
"""Map a RAW (pre-gate) execution outcome to the span-status vocabulary.
|
|
443
|
+
|
|
444
|
+
``TaskStatus`` has no TIMEOUT member — a timed-out agent is FAILED with the
|
|
445
|
+
``"Timed out after {n}s"`` error (``agents._failure``), which is the only
|
|
446
|
+
timeout signal there is. ``cancelled`` exists in the contract vocabulary but
|
|
447
|
+
is unreachable here: a cancelled agent never becomes a ``TaskResult`` (the
|
|
448
|
+
cancellation re-raises out of the fan-out).
|
|
449
|
+
"""
|
|
450
|
+
if raw_status == TaskStatus.COMPLETED:
|
|
451
|
+
return "completed"
|
|
452
|
+
if error is not None and error.startswith("Timed out after"):
|
|
453
|
+
return "timeout"
|
|
454
|
+
return "failed"
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _attempt_span(
|
|
458
|
+
agent_id: str,
|
|
459
|
+
*,
|
|
460
|
+
node_id: str,
|
|
461
|
+
parent: str,
|
|
462
|
+
attempt: int,
|
|
463
|
+
raw_status: TaskStatus,
|
|
464
|
+
error: str | None,
|
|
465
|
+
exit_code: int,
|
|
466
|
+
end: float,
|
|
467
|
+
duration: float,
|
|
468
|
+
phase: str = "attempt",
|
|
469
|
+
name: str | None = None,
|
|
470
|
+
round_no: int | None = None,
|
|
471
|
+
links: tuple[str, ...] = (),
|
|
472
|
+
winner: bool = False,
|
|
473
|
+
files_changed_count: int | None = None,
|
|
474
|
+
worktree_slot: int | None = None,
|
|
475
|
+
) -> SpanRecord:
|
|
476
|
+
"""One agent execution as a span record — EVERY strategy's agent-run spans
|
|
477
|
+
(attempt/stage/propose/critique/synthesize/plan/subtask) flow through this
|
|
478
|
+
one builder so naming/attribute semantics can't drift between strategies.
|
|
479
|
+
|
|
480
|
+
``raw_status``/``error`` are the PRE-GATE snapshot — a gate rejection is a
|
|
481
|
+
quality verdict on a *working* agent, so the attempt span keeps the raw
|
|
482
|
+
``completed`` and the verdict rides the separate gate span (the same reason
|
|
483
|
+
``_breaker_record`` takes the raw status). The span ``start`` is approximated
|
|
484
|
+
as ``end − duration`` — safe because tracegraph never promotes timestamps to
|
|
485
|
+
causality (they are only a prerank tiebreak). The OTLP error message is a
|
|
486
|
+
bounded literal (``timeout`` | ``exit_code=N``), never raw agent output.
|
|
487
|
+
``syncmill.exit_code`` is emitted only for a real process exit (the ``-1``
|
|
488
|
+
no-clean-exit sentinel is noise, not an exit code).
|
|
489
|
+
|
|
490
|
+
Cross-strategy conventions (S6):
|
|
491
|
+
- ``name`` defaults to ``{phase}:{agent_id}`` — a diff-stable structural
|
|
492
|
+
label (only decompose's ``subtask:{i}:{agent}`` overrides it, since the
|
|
493
|
+
round-robin repeats agent ids and the label needs the index).
|
|
494
|
+
- ``attempt`` is the index embedded in the node id (route ``i``, pipeline
|
|
495
|
+
stage ``i``, decompose subtask ``i``) and ``0`` for index-free nodes
|
|
496
|
+
(compete attempts, propose/critique/synthesize/plan).
|
|
497
|
+
- ``round_no`` is the council critique round → ``syncmill.round``.
|
|
498
|
+
- ``links`` are the nodes whose results were PUBLISHED to ``shared`` at the
|
|
499
|
+
previous phase boundary — declared availability, never claimed
|
|
500
|
+
consumption: the cross-agent handoff is a relevance-ranked best-effort
|
|
501
|
+
search, so syncmill cannot know what was actually consumed (the design's
|
|
502
|
+
finding-6 approximation).
|
|
503
|
+
- ``worktree_slot`` only on spans whose phase concurrently acquired fresh
|
|
504
|
+
worktrees (compete attempts, council propose, decompose subtask/
|
|
505
|
+
synthesize) — sequential steps and reused-worktree phases omit it.
|
|
506
|
+
"""
|
|
507
|
+
status = _span_status(raw_status, error)
|
|
508
|
+
attrs: dict[str, str | int | bool] = {
|
|
509
|
+
"phase": phase,
|
|
510
|
+
"agent_id": agent_id,
|
|
511
|
+
"attempt": attempt,
|
|
512
|
+
"status": status,
|
|
513
|
+
}
|
|
514
|
+
if round_no is not None:
|
|
515
|
+
attrs["round"] = round_no
|
|
516
|
+
if exit_code >= 0:
|
|
517
|
+
attrs["exit_code"] = exit_code
|
|
518
|
+
if worktree_slot is not None:
|
|
519
|
+
attrs["worktree_slot"] = worktree_slot
|
|
520
|
+
if winner:
|
|
521
|
+
attrs["winner"] = True
|
|
522
|
+
if files_changed_count is not None:
|
|
523
|
+
attrs["files_changed_count"] = files_changed_count
|
|
524
|
+
return SpanRecord(
|
|
525
|
+
name=name if name is not None else f"{phase}:{agent_id}",
|
|
526
|
+
kind="AGENT",
|
|
527
|
+
node_id=node_id,
|
|
528
|
+
node_parent=parent,
|
|
529
|
+
links=links,
|
|
530
|
+
start=max(end - duration, 0.0),
|
|
531
|
+
end=end,
|
|
532
|
+
status=status,
|
|
533
|
+
error=None if status == "completed" else (
|
|
534
|
+
"timeout" if status == "timeout" else f"exit_code={exit_code}"
|
|
535
|
+
),
|
|
536
|
+
attrs=attrs,
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _trace_attempt(tracer: Tracer | None, span: SpanRecord, result: TaskResult) -> None:
|
|
541
|
+
"""Record one attempt and body-free observed MCP calls below it."""
|
|
542
|
+
_trace_record(tracer, span)
|
|
543
|
+
telemetry = result.tool_telemetry
|
|
544
|
+
if tracer is None or telemetry is None:
|
|
545
|
+
return
|
|
546
|
+
calls_by_sequence = {}
|
|
547
|
+
for call in telemetry.calls:
|
|
548
|
+
if call.sequence in calls_by_sequence:
|
|
549
|
+
logger.warning(
|
|
550
|
+
"ignored duplicate tool call sequence=%d; keeping first observed call",
|
|
551
|
+
call.sequence,
|
|
552
|
+
)
|
|
553
|
+
continue
|
|
554
|
+
calls_by_sequence[call.sequence] = call
|
|
555
|
+
|
|
556
|
+
# Parent-before-child recording is required by the streaming Phoenix sink. Sorting the
|
|
557
|
+
# de-duplicated projection also keeps the buffered file artifact deterministic when a
|
|
558
|
+
# controlled producer supplies otherwise valid calls out of order.
|
|
559
|
+
for call in sorted(calls_by_sequence.values(), key=lambda item: item.sequence):
|
|
560
|
+
tool_parent = span.node_id
|
|
561
|
+
if call.retry_of_sequence is not None:
|
|
562
|
+
prior = calls_by_sequence.get(call.retry_of_sequence)
|
|
563
|
+
valid_retry = (
|
|
564
|
+
telemetry.complete
|
|
565
|
+
and prior is not None
|
|
566
|
+
and prior.sequence < call.sequence
|
|
567
|
+
and prior.tool_key == call.tool_key
|
|
568
|
+
)
|
|
569
|
+
if valid_retry:
|
|
570
|
+
marker_id = f"{span.node_id}:retry:{call.sequence}"
|
|
571
|
+
_trace_record(
|
|
572
|
+
tracer,
|
|
573
|
+
SpanRecord(
|
|
574
|
+
name=f"retry:{call.tool_key}",
|
|
575
|
+
kind="CHAIN",
|
|
576
|
+
node_id=marker_id,
|
|
577
|
+
node_parent=f"{span.node_id}:tool:{prior.sequence}",
|
|
578
|
+
start=span.end,
|
|
579
|
+
end=span.end,
|
|
580
|
+
status="completed",
|
|
581
|
+
attrs={
|
|
582
|
+
"phase": "retry",
|
|
583
|
+
"agent_id": result.agent_id,
|
|
584
|
+
"status": "completed",
|
|
585
|
+
"tool.key": call.tool_key,
|
|
586
|
+
"tool.sequence": call.sequence,
|
|
587
|
+
},
|
|
588
|
+
),
|
|
589
|
+
)
|
|
590
|
+
tool_parent = marker_id
|
|
591
|
+
else:
|
|
592
|
+
logger.warning(
|
|
593
|
+
"ignored invalid explicit tool retry relation sequence=%d retry_of=%d; "
|
|
594
|
+
"recording call without retry causality",
|
|
595
|
+
call.sequence,
|
|
596
|
+
call.retry_of_sequence,
|
|
597
|
+
)
|
|
598
|
+
status = (
|
|
599
|
+
"completed"
|
|
600
|
+
if call.outcome == ToolOutcome.SUCCEEDED
|
|
601
|
+
else "failed"
|
|
602
|
+
if call.outcome == ToolOutcome.FAILED
|
|
603
|
+
else "unknown"
|
|
604
|
+
)
|
|
605
|
+
_trace_record(
|
|
606
|
+
tracer,
|
|
607
|
+
SpanRecord(
|
|
608
|
+
name=call.tool_key,
|
|
609
|
+
kind="TOOL",
|
|
610
|
+
node_id=f"{span.node_id}:tool:{call.sequence}",
|
|
611
|
+
node_parent=tool_parent,
|
|
612
|
+
start=span.end,
|
|
613
|
+
end=span.end,
|
|
614
|
+
status=status,
|
|
615
|
+
error="tool_failed" if status == "failed" else None,
|
|
616
|
+
attrs={
|
|
617
|
+
"phase": "tool",
|
|
618
|
+
"agent_id": result.agent_id,
|
|
619
|
+
"status": status,
|
|
620
|
+
"tool.key": call.tool_key,
|
|
621
|
+
"tool.sequence": call.sequence,
|
|
622
|
+
"telemetry.source": telemetry.source,
|
|
623
|
+
"telemetry.complete": telemetry.complete,
|
|
624
|
+
},
|
|
625
|
+
),
|
|
626
|
+
)
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def _gate_span(
|
|
630
|
+
agent_id: str,
|
|
631
|
+
*,
|
|
632
|
+
node_id: str,
|
|
633
|
+
parent: str,
|
|
634
|
+
report: GateReport,
|
|
635
|
+
end: float,
|
|
636
|
+
) -> SpanRecord:
|
|
637
|
+
"""One gate evaluation as a CHAIN span caused by its attempt. Emitted for
|
|
638
|
+
every candidate that was actually gated (``result.gate is not None``), pass
|
|
639
|
+
or fail — the verdict is the point. The error message is the bounded
|
|
640
|
+
``gate: <summary>`` form (check names/counts, never check output)."""
|
|
641
|
+
duration = sum(check.duration_seconds for check in report.checks)
|
|
642
|
+
status = "completed" if report.passed else "failed"
|
|
643
|
+
return SpanRecord(
|
|
644
|
+
name=f"gate:{agent_id}",
|
|
645
|
+
kind="CHAIN",
|
|
646
|
+
node_id=node_id,
|
|
647
|
+
node_parent=parent,
|
|
648
|
+
start=max(end - duration, 0.0),
|
|
649
|
+
end=end,
|
|
650
|
+
status=status,
|
|
651
|
+
error=None if report.passed else f"gate: {report.summary()}",
|
|
652
|
+
attrs={
|
|
653
|
+
"phase": "gate",
|
|
654
|
+
"agent_id": agent_id,
|
|
655
|
+
"status": status,
|
|
656
|
+
"gate.passed": report.passed,
|
|
657
|
+
},
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def _select_span(
|
|
662
|
+
*,
|
|
663
|
+
winner_node: str | None,
|
|
664
|
+
links: list[str],
|
|
665
|
+
start: float,
|
|
666
|
+
end: float,
|
|
667
|
+
) -> SpanRecord:
|
|
668
|
+
"""The convergence (``_select_winner``) as a span: parented on the winning
|
|
669
|
+
attempt (the promoted single cause — the win path stays a clean tree), or on
|
|
670
|
+
``run`` when nobody won; ``links`` carry the other raw-COMPLETED candidates
|
|
671
|
+
(fan-in: the pool the selection was made over — the T1-golden reading of
|
|
672
|
+
"examined", computable without changing lazy ``_select_winner``)."""
|
|
673
|
+
return SpanRecord(
|
|
674
|
+
name="select",
|
|
675
|
+
kind="CHAIN",
|
|
676
|
+
node_id="select",
|
|
677
|
+
node_parent=winner_node if winner_node is not None else "run",
|
|
678
|
+
links=tuple(links),
|
|
679
|
+
start=start,
|
|
680
|
+
end=end,
|
|
681
|
+
status="completed",
|
|
682
|
+
attrs={"phase": "select", "agent_id": "supervisor", "status": "completed"},
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _no_stages_failure(task: Task) -> TaskResult:
|
|
687
|
+
"""Defensive failure when a strategy is handed no agents to run."""
|
|
688
|
+
return TaskResult(
|
|
689
|
+
task_id=task.id,
|
|
690
|
+
agent_id="none",
|
|
691
|
+
status=TaskStatus.FAILED,
|
|
692
|
+
output="",
|
|
693
|
+
exit_code=-1,
|
|
694
|
+
duration_seconds=0.0,
|
|
695
|
+
error="No agents available",
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def _aggregate_failure(
|
|
700
|
+
task: Task, errors: list[TaskResult], *, duration_seconds: float | None = None
|
|
701
|
+
) -> TaskResult:
|
|
702
|
+
"""Combine all failure reasons into a single TaskResult.
|
|
703
|
+
|
|
704
|
+
``duration_seconds`` defaults to the *sum* of the attempts — correct for the
|
|
705
|
+
sequential RouteStrategy, whose agents run one after another. A concurrent
|
|
706
|
+
strategy (CompeteStrategy) overrides it with the *max*, since its agents ran
|
|
707
|
+
at once and the wall-clock is the slowest one, not their sum.
|
|
708
|
+
"""
|
|
709
|
+
lines = []
|
|
710
|
+
for e in errors:
|
|
711
|
+
reason = e.error or f"exit_code={e.exit_code}"
|
|
712
|
+
lines.append(f" [{e.agent_id}] {reason}")
|
|
713
|
+
combined_error = "All agents failed:\n" + "\n".join(lines)
|
|
714
|
+
|
|
715
|
+
if duration_seconds is None:
|
|
716
|
+
duration_seconds = sum(e.duration_seconds for e in errors)
|
|
717
|
+
|
|
718
|
+
return TaskResult(
|
|
719
|
+
task_id=task.id,
|
|
720
|
+
agent_id="none",
|
|
721
|
+
status=TaskStatus.FAILED,
|
|
722
|
+
output="",
|
|
723
|
+
exit_code=-1,
|
|
724
|
+
duration_seconds=duration_seconds,
|
|
725
|
+
error=combined_error,
|
|
726
|
+
)
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _select_winner(
|
|
730
|
+
worktrees: list[Worktree],
|
|
731
|
+
results: list[TaskResult],
|
|
732
|
+
gate: QualityGate | None = None,
|
|
733
|
+
*,
|
|
734
|
+
mode: str = "verified",
|
|
735
|
+
agents: list[AgentRunner] | None = None,
|
|
736
|
+
pool: WorktreePool | None = None,
|
|
737
|
+
config: SyncmillConfig | None = None,
|
|
738
|
+
tracer: Tracer | None = None,
|
|
739
|
+
candidate_node_prefix: str = "attempt",
|
|
740
|
+
trace_out: dict | None = None,
|
|
741
|
+
) -> tuple[TaskResult, Worktree] | None:
|
|
742
|
+
"""Select one COMPLETED candidate under ``mode`` and pair it with its worktree.
|
|
743
|
+
|
|
744
|
+
Deterministic: ``results`` (and the aligned ``worktrees``) arrive in the
|
|
745
|
+
caller-fixed priority order — gather preserves submission order, and the
|
|
746
|
+
strategy acquired worktrees in that same order — so selection iterates
|
|
747
|
+
priority, never gather's completion order and never ``duration_seconds``.
|
|
748
|
+
Identical inputs always pick the same winner. Failed/timed-out candidates are
|
|
749
|
+
never eligible; losing is observed by running, not probing (no
|
|
750
|
+
``health_check``).
|
|
751
|
+
|
|
752
|
+
Three modes, all reading the same priority-ordered candidate list:
|
|
753
|
+
``verified`` (the default) walks gate-passing candidates in priority order;
|
|
754
|
+
``priority`` checks only the first completed candidate; ``judge`` gates every
|
|
755
|
+
completer, lets an agent rank the verified pool, and degrades to verified
|
|
756
|
+
priority if judging fails.
|
|
757
|
+
|
|
758
|
+
This is the seam quality gates (#10) extend — the promised hook is now live.
|
|
759
|
+
With a ``gate``, each COMPLETED candidate it examines is verified against that
|
|
760
|
+
candidate's own worktree (via ``_gate_accepts``): a candidate that fails is
|
|
761
|
+
downgraded to FAILED with a gate error *in place* (so the caller's aggregate
|
|
762
|
+
failure names it) and is not eligible. Under ``verified`` the winner is then
|
|
763
|
+
the first candidate that both completed and passed, in priority order — the
|
|
764
|
+
ranking is "verified score (binary pass) with priority index as the final
|
|
765
|
+
deterministic tie-break", all passers scoring 1.0 so priority breaks the tie —
|
|
766
|
+
and evaluation is lazy: it stops at the first passer, so candidates after the
|
|
767
|
+
winner aren't gated (losers, discarded anyway). ``judge`` deliberately is not
|
|
768
|
+
lazy: it must gate every completer to know the verified pool it ranks. Without
|
|
769
|
+
a gate (``None``) both reduce to the original "first COMPLETED in priority
|
|
770
|
+
order".
|
|
771
|
+
|
|
772
|
+
Introduced by CompeteStrategy as its convergence seam and reused verbatim by
|
|
773
|
+
CouncilStrategy and DecomposeStrategy for synthesized candidates — the single,
|
|
774
|
+
shared definition of "which finished candidate wins".
|
|
775
|
+
"""
|
|
776
|
+
if mode not in {"verified", "priority", "judge"}:
|
|
777
|
+
# SelectionConfig's Literal makes this unreachable from config; a
|
|
778
|
+
# programmatic caller with a typo'd mode used to silently degrade to
|
|
779
|
+
# `verified` via an implicit `!=` — fail like _make_strategy instead.
|
|
780
|
+
raise ValueError(f"unknown selection mode: {mode!r}")
|
|
781
|
+
completed = [
|
|
782
|
+
(worktree, result)
|
|
783
|
+
for worktree, result in zip(worktrees, results)
|
|
784
|
+
if result.status == TaskStatus.COMPLETED
|
|
785
|
+
]
|
|
786
|
+
if mode == "priority":
|
|
787
|
+
if not completed:
|
|
788
|
+
return None
|
|
789
|
+
worktree, result = completed[0]
|
|
790
|
+
return (result, worktree) if _gate_accepts(result, worktree, gate) else None
|
|
791
|
+
|
|
792
|
+
if mode == "verified":
|
|
793
|
+
for worktree, result in completed:
|
|
794
|
+
if _gate_accepts(result, worktree, gate):
|
|
795
|
+
return result, worktree
|
|
796
|
+
return None
|
|
797
|
+
|
|
798
|
+
verified = [
|
|
799
|
+
(worktree, result)
|
|
800
|
+
for worktree, result in completed
|
|
801
|
+
if _gate_accepts(result, worktree, gate)
|
|
802
|
+
]
|
|
803
|
+
if not verified:
|
|
804
|
+
return None
|
|
805
|
+
if len(verified) == 1 or not agents or pool is None or config is None:
|
|
806
|
+
result_wt, result = verified[0]
|
|
807
|
+
return result, result_wt
|
|
808
|
+
|
|
809
|
+
judge_id = config.selection.judge_agent
|
|
810
|
+
judge = next((agent for agent in agents if agent.agent_id == judge_id), None)
|
|
811
|
+
if judge is None:
|
|
812
|
+
judge = agents[0]
|
|
813
|
+
candidate_ids = {result.agent_id for _, result in verified}
|
|
814
|
+
# Judging is a preference between candidates that ALREADY passed the gate, so
|
|
815
|
+
# every step of it is best-effort: acquiring the judge's worktree can raise
|
|
816
|
+
# WorkspaceError (disk pressure, a git failure) and executing the judge is a
|
|
817
|
+
# real subprocess. Letting either escape would unwind through the strategy
|
|
818
|
+
# into the Supervisor's `except BaseException`, marking a run that has a
|
|
819
|
+
# perfectly good verified winner as recovery-required. Any failure degrades to
|
|
820
|
+
# the same verified-priority fallback a bad verdict already uses.
|
|
821
|
+
try:
|
|
822
|
+
sections: list[str] = []
|
|
823
|
+
for worktree, result in verified:
|
|
824
|
+
# Bounded excerpt, never the full --binary patch (#116): the winner's
|
|
825
|
+
# single full patch is still produced by _collect_files_changed, so
|
|
826
|
+
# the judge no longer stages+diffs the winning tree a second time
|
|
827
|
+
# into a string that was sliced anyway.
|
|
828
|
+
excerpt = (
|
|
829
|
+
_git_diff_excerpt(
|
|
830
|
+
worktree.path, getattr(worktree, "base_commit", None),
|
|
831
|
+
max_bytes=12_000,
|
|
832
|
+
)
|
|
833
|
+
if worktree.is_git
|
|
834
|
+
else None
|
|
835
|
+
)
|
|
836
|
+
sections.append(
|
|
837
|
+
f"CANDIDATE {result.agent_id}\n"
|
|
838
|
+
f"REPORT:\n{_bounded_head(result.output, 6000)}\n"
|
|
839
|
+
f"PATCH:\n{excerpt or ''}"
|
|
840
|
+
)
|
|
841
|
+
judge_prompt = (
|
|
842
|
+
"Act as a read-only coding judge. Compare the verified candidates below "
|
|
843
|
+
"for correctness, scope, maintainability, and test evidence. Do not edit "
|
|
844
|
+
"files. End with exactly `WINNER: <agent-id>`.\n\n"
|
|
845
|
+
+ "\n\n---\n\n".join(sections)
|
|
846
|
+
)
|
|
847
|
+
judge_worktree = pool.acquire(f"{judge.agent_id}-judge")
|
|
848
|
+
# No provider session for the judge: its worktree is a throwaway path,
|
|
849
|
+
# so a session keyed on it would mint a fresh conversation per run —
|
|
850
|
+
# and on a continuation (seed set) the seeded PARENT conversation would
|
|
851
|
+
# be attached to the judge key, appending this read-only ranking prompt
|
|
852
|
+
# to the user's continued conversation history.
|
|
853
|
+
with provider_session_scope(enabled=False):
|
|
854
|
+
judge_start = time.time()
|
|
855
|
+
verdict = judge.execute(
|
|
856
|
+
judge_prompt,
|
|
857
|
+
judge_worktree.path,
|
|
858
|
+
config.agent_config(judge.agent_id).timeout_seconds,
|
|
859
|
+
)
|
|
860
|
+
judge_end = time.time()
|
|
861
|
+
except Exception:
|
|
862
|
+
logger.warning(
|
|
863
|
+
"LLM judge could not run; using verified priority", exc_info=True
|
|
864
|
+
)
|
|
865
|
+
worktree, result = verified[0]
|
|
866
|
+
return result, worktree
|
|
867
|
+
# The judge is a whole agent invocation — without a span the trace
|
|
868
|
+
# under-reports the run's agent calls by exactly the +1 the RunPlan
|
|
869
|
+
# promises. Parent = run (like any fan-out root); raw status, per the
|
|
870
|
+
# attempt-span convention. Causality can't ride timestamps (the trace
|
|
871
|
+
# contract), so the judged candidates feed the judge via `links` (the
|
|
872
|
+
# caller's node naming, `candidate_node_prefix`), and the judge node is
|
|
873
|
+
# handed back through `trace_out` so the caller's select span can link
|
|
874
|
+
# the verdict the selection depended on. With `trace_out` the emission is
|
|
875
|
+
# DEFERRED — the caller records it after its candidate spans land: this
|
|
876
|
+
# runs BEFORE the strategy has recorded those spans, and a streaming sink
|
|
877
|
+
# (PhoenixSpanSink) drops any span whose link targets it hasn't seen —
|
|
878
|
+
# which would take the select span down with it.
|
|
879
|
+
judge_node = f"judge:{judge.agent_id}"
|
|
880
|
+
judge_span = _attempt_span(
|
|
881
|
+
judge.agent_id,
|
|
882
|
+
node_id=judge_node,
|
|
883
|
+
parent="run",
|
|
884
|
+
attempt=0,
|
|
885
|
+
raw_status=verdict.status,
|
|
886
|
+
error=verdict.error,
|
|
887
|
+
exit_code=verdict.exit_code,
|
|
888
|
+
end=judge_end,
|
|
889
|
+
duration=verdict.duration_seconds or (judge_end - judge_start),
|
|
890
|
+
phase="judge",
|
|
891
|
+
links=tuple(
|
|
892
|
+
f"{candidate_node_prefix}:{result.agent_id}"
|
|
893
|
+
for _, result in verified
|
|
894
|
+
),
|
|
895
|
+
)
|
|
896
|
+
if tracer is not None and trace_out is not None:
|
|
897
|
+
trace_out["judge_node"] = judge_node
|
|
898
|
+
trace_out["judge_span"] = (judge_span, verdict)
|
|
899
|
+
else:
|
|
900
|
+
_trace_attempt(tracer, judge_span, verdict)
|
|
901
|
+
if verdict.status == TaskStatus.COMPLETED:
|
|
902
|
+
matches = re.findall(r"(?im)^WINNER:\s*([^\s]+)\s*$", verdict.output)
|
|
903
|
+
if matches and matches[-1] in candidate_ids:
|
|
904
|
+
chosen = matches[-1]
|
|
905
|
+
for worktree, result in verified:
|
|
906
|
+
if result.agent_id == chosen:
|
|
907
|
+
return result, worktree
|
|
908
|
+
logger.warning("LLM judge returned no valid winner; using verified priority")
|
|
909
|
+
worktree, result = verified[0]
|
|
910
|
+
return result, worktree
|