agent-postmortem 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_postmortem/__init__.py +9 -0
- agent_postmortem/argerrors.py +160 -0
- agent_postmortem/claims.py +43 -0
- agent_postmortem/cli.py +2278 -0
- agent_postmortem/commands.py +1543 -0
- agent_postmortem/config.py +435 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/journal.jsonl +7 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000.jsonl +4 -0
- agent_postmortem/detectors/__init__.py +98 -0
- agent_postmortem/detectors/attempt_failed.py +261 -0
- agent_postmortem/detectors/base.py +319 -0
- agent_postmortem/detectors/bash_backgrounded.py +72 -0
- agent_postmortem/detectors/bash_timeout.py +94 -0
- agent_postmortem/detectors/claim_contradiction.py +251 -0
- agent_postmortem/detectors/classifier_unavailable.py +73 -0
- agent_postmortem/detectors/codex_unavailable.py +102 -0
- agent_postmortem/detectors/codex_write_denied.py +100 -0
- agent_postmortem/detectors/git_cwd_mismatch.py +74 -0
- agent_postmortem/detectors/mislabeled_kill.py +164 -0
- agent_postmortem/detectors/missing_artifact.py +159 -0
- agent_postmortem/detectors/permission_denied.py +99 -0
- agent_postmortem/detectors/silent_retry.py +163 -0
- agent_postmortem/detectors/structured_output_failure.py +91 -0
- agent_postmortem/detectors/tool_json_error.py +77 -0
- agent_postmortem/detectors/wasted_time.py +125 -0
- agent_postmortem/detectors/workflow_failed.py +62 -0
- agent_postmortem/disclosure.py +931 -0
- agent_postmortem/hooks/model_allocation_guard.mjs +248 -0
- agent_postmortem/hooks_cli.py +1269 -0
- agent_postmortem/ledger.py +624 -0
- agent_postmortem/markers.py +505 -0
- agent_postmortem/models.py +815 -0
- agent_postmortem/observe.py +792 -0
- agent_postmortem/reader.py +4611 -0
- agent_postmortem/redact.py +1981 -0
- agent_postmortem/report.py +3010 -0
- agent_postmortem/rescue.py +2096 -0
- agent_postmortem/returns.py +499 -0
- agent_postmortem/rules.py +627 -0
- agent_postmortem/severity.py +275 -0
- agent_postmortem/state.py +1261 -0
- agent_postmortem/trend.py +391 -0
- agent_postmortem/watch.py +973 -0
- agent_postmortem-0.3.4.dist-info/METADATA +264 -0
- agent_postmortem-0.3.4.dist-info/RECORD +56 -0
- agent_postmortem-0.3.4.dist-info/WHEEL +4 -0
- agent_postmortem-0.3.4.dist-info/entry_points.txt +3 -0
- agent_postmortem-0.3.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,973 @@
|
|
|
1
|
+
"""`agent-postmortem watch`: a read-only follower for one workflow run
|
|
2
|
+
(docs/DESIGN.md 4.12; .reviews/proposals/2026-09-v0.2-guided-rescue.md
|
|
3
|
+
section 7).
|
|
4
|
+
|
|
5
|
+
Designed to be the command line a Claude Code Monitor runs: it prints one
|
|
6
|
+
START line first, then one line per event a person would act on (KILL, FAILED,
|
|
7
|
+
OK, QUIET) and nothing otherwise, then a terminal DONE or LIMIT line and exits.
|
|
8
|
+
|
|
9
|
+
Polling never re-reads a file it does not have to: the journal is read by
|
|
10
|
+
appended bytes only (`_read_new_journal_lines` keeps a byte offset and
|
|
11
|
+
restarts from 0 when the file is smaller than that offset -- a truncated or
|
|
12
|
+
replaced journal, which resets every other piece of derived state too and
|
|
13
|
+
prints one RESET line); a transcript is only ever stat'd for its mtime
|
|
14
|
+
(QUIET), or fully parsed once, the moment a KILL or FAILED needs its last
|
|
15
|
+
line or its silence span -- never polled incrementally. A partial trailing
|
|
16
|
+
journal line growing past 1 MB, or a complete line that is not a JSON
|
|
17
|
+
object, is malformed evidence (MALFORMED); a KILL/FAILED/OK event this poll
|
|
18
|
+
cannot yet resolve (the rate limit, or a transcript not on disk yet) is
|
|
19
|
+
retried on later polls, bounded at 12, rather than dropped.
|
|
20
|
+
|
|
21
|
+
DONE (docs/DESIGN.md 4.12, N8b K5.4) is eligible at once from a lifecycle terminal
|
|
22
|
+
of completed, failed, or stopped (an accepted notification, a TaskStop, or a
|
|
23
|
+
synchronous launch result), or after `--done-grace` once every journal key is
|
|
24
|
+
resolved. The lifecycle is evaluated from watch's own in-memory occurrences and
|
|
25
|
+
a parent index refreshed only when the parent transcript's stat signature
|
|
26
|
+
changes; a RESET starts a new journal epoch whose watermark hides the ended
|
|
27
|
+
epoch's generations.
|
|
28
|
+
|
|
29
|
+
`_now` and `_sleep` are module attributes so a test can monkeypatch both: the
|
|
30
|
+
suite never sleeps for a real second, and `_sleep` doubles as the point
|
|
31
|
+
where a scripted test appends the next bytes of a growing journal or
|
|
32
|
+
transcript.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import json
|
|
38
|
+
import sys
|
|
39
|
+
import time
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from types import SimpleNamespace
|
|
43
|
+
|
|
44
|
+
from . import detectors, disclosure, ledger, reader, report
|
|
45
|
+
from .detectors import attempt_failed as _attempt_failed
|
|
46
|
+
from .detectors.base import final_event, meaningful_event_before
|
|
47
|
+
from .markers import KILL_SENTINEL_PREFIX
|
|
48
|
+
|
|
49
|
+
DEFAULT_INTERVAL = 5.0
|
|
50
|
+
DEFAULT_MAX_HOURS = 6.0
|
|
51
|
+
DEFAULT_QUIET_AFTER = 300.0
|
|
52
|
+
# A workflow that runs its phases one after another has a moment between
|
|
53
|
+
# two phases where every key seen so far is resolved. DONE waits this long
|
|
54
|
+
# with no new `started` before it is printed, so that gap is not mistaken
|
|
55
|
+
# for the end of the run.
|
|
56
|
+
DEFAULT_DONE_GRACE = 30.0
|
|
57
|
+
|
|
58
|
+
# How long `watch` waits, printing nothing, for a run directory (and its
|
|
59
|
+
# journal.jsonl) that does not exist yet -- the launch hook fires before the
|
|
60
|
+
# Workflow tool has created either (docs/DESIGN.md 7.2's D8).
|
|
61
|
+
_JOURNAL_WAIT_SECONDS = 60.0
|
|
62
|
+
|
|
63
|
+
# At most one line per agent per this many seconds, except DONE and LIMIT.
|
|
64
|
+
_RATE_LIMIT_SECONDS = 60.0
|
|
65
|
+
|
|
66
|
+
# docs/DESIGN.md 4.12 (release finding F9): a partial (no trailing newline
|
|
67
|
+
# yet) journal line is withheld and reread on the next poll, but a line
|
|
68
|
+
# that keeps growing forever with no newline (a stuck or malformed writer)
|
|
69
|
+
# must not be reread and rechecked without bound.
|
|
70
|
+
_MAX_PARTIAL_LINE_BYTES = 1_048_576
|
|
71
|
+
|
|
72
|
+
# docs/DESIGN.md 4.12 (release finding F10): a KILL/FAILED/OK event suppressed
|
|
73
|
+
# by the rate limit, or (KILL only) by a transcript that does not exist
|
|
74
|
+
# yet, is retried on the next poll rather than dropped -- but only for this
|
|
75
|
+
# many polls, after which it is force-emitted (KILL, with a degraded
|
|
76
|
+
# message) or emitted as-is (FAILED, OK) and marked handled either way, so a
|
|
77
|
+
# transcript that never appears can never wedge the loop forever.
|
|
78
|
+
_MAX_PENDING_POLLS = 12
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class JournalLineTooLong(Exception):
|
|
82
|
+
"""docs/DESIGN.md 4.12 (release finding F9): the journal's unterminated
|
|
83
|
+
tail (from the last read offset to end of file, with no newline yet)
|
|
84
|
+
has grown past _MAX_PARTIAL_LINE_BYTES."""
|
|
85
|
+
|
|
86
|
+
# Every event word is padded to this width so the message text lines up the
|
|
87
|
+
# same way across KILL/FAILED/QUIET/DONE/LIMIT (and so KILL's own second,
|
|
88
|
+
# indented line lines up under it): "KILL".ljust(7) == "KILL ", "FAILED"
|
|
89
|
+
# ljust(7) == "FAILED ", and so on.
|
|
90
|
+
_LABEL_WIDTH = 7
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _now() -> float:
|
|
94
|
+
"""Epoch seconds (docs/DESIGN.md 4.12): the same domain as a file's own
|
|
95
|
+
mtime, so QUIET's 'now - mtime' needs no unit conversion. A test
|
|
96
|
+
monkeypatches this to a controllable float instead of the real clock."""
|
|
97
|
+
return time.time()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _sleep(seconds: float) -> None:
|
|
101
|
+
"""A test monkeypatches this to advance a fake clock (and grow the
|
|
102
|
+
journal/transcripts a poll is about to see) instead of actually
|
|
103
|
+
sleeping."""
|
|
104
|
+
time.sleep(seconds)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# --- run resolution -------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def find_run_dir(run_id_or_path: str, root: Path) -> Path | None:
|
|
111
|
+
"""A single matching workflow run directory, or None when nothing (yet)
|
|
112
|
+
matches -- never raises, since the caller's job is to keep polling until
|
|
113
|
+
the 60s journal-wait budget runs out (docs/DESIGN.md 4.12). Only
|
|
114
|
+
workflow runs are considered: a session carries no journal.jsonl, so
|
|
115
|
+
`watch` has nothing to follow for one."""
|
|
116
|
+
direct = Path(run_id_or_path)
|
|
117
|
+
if direct.is_dir():
|
|
118
|
+
return direct
|
|
119
|
+
try:
|
|
120
|
+
candidates = [
|
|
121
|
+
ref
|
|
122
|
+
for ref in reader.discover_runs(root, kinds=frozenset({"workflow"}))
|
|
123
|
+
if ref.run_id.startswith(run_id_or_path)
|
|
124
|
+
]
|
|
125
|
+
except OSError:
|
|
126
|
+
return None
|
|
127
|
+
if len(candidates) == 1:
|
|
128
|
+
return candidates[0].path
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _wait_for_journal(
|
|
133
|
+
run_id_or_path: str, root: Path, start: float, interval: float, err,
|
|
134
|
+
) -> Path | None:
|
|
135
|
+
while True:
|
|
136
|
+
run_dir = find_run_dir(run_id_or_path, root)
|
|
137
|
+
if run_dir is not None and (run_dir / "journal.jsonl").exists():
|
|
138
|
+
return run_dir
|
|
139
|
+
remaining = _JOURNAL_WAIT_SECONDS - (_now() - start)
|
|
140
|
+
if remaining <= 0:
|
|
141
|
+
print(
|
|
142
|
+
"no journal appeared within 60 s: watch follows a Workflow run while it is "
|
|
143
|
+
"running (the plugin hands you this command when one starts); "
|
|
144
|
+
"agent-postmortem scan lists past runs",
|
|
145
|
+
file=err,
|
|
146
|
+
)
|
|
147
|
+
return None
|
|
148
|
+
# docs/DESIGN.md 4.12 (release finding F11): capped to whatever is
|
|
149
|
+
# left of the 60s budget, so a large --interval cannot overshoot it
|
|
150
|
+
# (--interval 300 must still give up within 60s, not 300s).
|
|
151
|
+
_sleep(min(interval, remaining))
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --- appended-bytes journal reading ----------------------------------------
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _read_new_journal_lines(path: Path, offset: int) -> tuple[list[str], int, bool]:
|
|
158
|
+
"""The complete non-blank lines appended to path since `offset` bytes,
|
|
159
|
+
the new offset, and whether the file was found truncated or replaced
|
|
160
|
+
(release finding F8). A file smaller than `offset` has been truncated
|
|
161
|
+
or replaced (a fresh run reusing the same path): read from 0 instead of
|
|
162
|
+
raising or silently missing its early lines, and report `truncated`
|
|
163
|
+
True so the caller can reset every piece of state it derived from the
|
|
164
|
+
old content before processing the replacement. A trailing line with no
|
|
165
|
+
newline yet (a write still in flight) is withheld -- the offset does not
|
|
166
|
+
advance past it -- so the next call re-reads it once it is complete,
|
|
167
|
+
unless it has already grown past _MAX_PARTIAL_LINE_BYTES with still no
|
|
168
|
+
newline, which raises JournalLineTooLong (release finding F9) instead
|
|
169
|
+
of being reread and rechecked in full, without bound, forever."""
|
|
170
|
+
try:
|
|
171
|
+
size = path.stat().st_size
|
|
172
|
+
except OSError:
|
|
173
|
+
return [], offset, False
|
|
174
|
+
truncated = size < offset
|
|
175
|
+
if truncated:
|
|
176
|
+
offset = 0
|
|
177
|
+
if size <= offset:
|
|
178
|
+
return [], offset, truncated
|
|
179
|
+
with path.open("rb") as handle:
|
|
180
|
+
handle.seek(offset)
|
|
181
|
+
chunk = handle.read()
|
|
182
|
+
last_newline = chunk.rfind(b"\n")
|
|
183
|
+
# The bytes after the last newline (the whole chunk when there is none
|
|
184
|
+
# at all) are the as-yet-unterminated next line -- checked regardless
|
|
185
|
+
# of how many complete lines share this same read.
|
|
186
|
+
if len(chunk) - (last_newline + 1) > _MAX_PARTIAL_LINE_BYTES:
|
|
187
|
+
raise JournalLineTooLong()
|
|
188
|
+
if last_newline == -1:
|
|
189
|
+
return [], offset, truncated
|
|
190
|
+
usable = chunk[: last_newline + 1]
|
|
191
|
+
new_offset = offset + len(usable)
|
|
192
|
+
text = usable.decode("utf-8", errors="replace")
|
|
193
|
+
lines = [line for line in text.split("\n") if line.strip()]
|
|
194
|
+
return lines, new_offset, truncated
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# --- transcript helpers: full, one-shot parses, never polled ---------------
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class _EventsHolder:
|
|
201
|
+
"""The minimal duck-typed shape detectors.base.meaningful_event_before
|
|
202
|
+
needs (attempt.events) -- built directly from a one-shot
|
|
203
|
+
reader._read_transcript parse, never a full reader.Attempt."""
|
|
204
|
+
|
|
205
|
+
__slots__ = ("events",)
|
|
206
|
+
|
|
207
|
+
def __init__(self, events: list) -> None:
|
|
208
|
+
self.events = events
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _transcript_path(run_dir: Path, agent_id: str) -> Path:
|
|
212
|
+
return run_dir / f"agent-{agent_id}.jsonl"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _last_user_index(events: list) -> int | None:
|
|
216
|
+
last = None
|
|
217
|
+
for index, event in enumerate(events):
|
|
218
|
+
if event.kind == "user":
|
|
219
|
+
last = index
|
|
220
|
+
return last
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _parse_kill(path: Path) -> tuple[bool, float | None, int, int, bool, int]:
|
|
224
|
+
"""(is_kill, silence_seconds, paths_written, commits, paths_partial, commits_without_id) for
|
|
225
|
+
one attempt's transcript, from a single full parse (`paths_partial`, N8 fix 1 S-m5: the
|
|
226
|
+
attempt's ledger is partial, so `paths_written` is a lower bound; `commits_without_id`,
|
|
227
|
+
G4b: completed calls with a commit command that reported no id): is_kill is whether its
|
|
228
|
+
last user event is the kill sentinel; silence_seconds is computed the same way the
|
|
229
|
+
interrupt machinery does (reader._build_interrupt): the gap from the
|
|
230
|
+
last meaningful event before the sentinel (detectors.base.
|
|
231
|
+
meaningful_event_before) to the sentinel itself, None when a timestamp
|
|
232
|
+
is missing or the delta would be negative."""
|
|
233
|
+
diagnostics: list[str] = []
|
|
234
|
+
state = ledger.TranscriptState()
|
|
235
|
+
events, _version, _session_id, _total, _bad, tool_calls, _marker_hits = reader._read_transcript(
|
|
236
|
+
path, diagnostics, state=state,
|
|
237
|
+
)
|
|
238
|
+
kill_index = _last_user_index(events)
|
|
239
|
+
if kill_index is None:
|
|
240
|
+
return False, None, 0, 0, False, 0
|
|
241
|
+
kill_event = events[kill_index]
|
|
242
|
+
if not kill_event.text.startswith(KILL_SENTINEL_PREFIX):
|
|
243
|
+
return False, None, 0, 0, False, 0
|
|
244
|
+
|
|
245
|
+
preceding = meaningful_event_before(_EventsHolder(events), kill_index)
|
|
246
|
+
silence: float | None = None
|
|
247
|
+
if preceding is not None and preceding.ts is not None and kill_event.ts is not None:
|
|
248
|
+
delta = (kill_event.ts - preceding.ts).total_seconds()
|
|
249
|
+
if delta >= 0:
|
|
250
|
+
silence = delta
|
|
251
|
+
|
|
252
|
+
facts = reader._build_attempt_facts(tool_calls, "killed", kill_event.ts, path, False)
|
|
253
|
+
paths = len(facts.paths_written) if facts is not None else 0
|
|
254
|
+
commits = len(facts.commits) if facts is not None else 0
|
|
255
|
+
partial = ledger.build_ledger(tool_calls, path.name, state).completeness == "partial"
|
|
256
|
+
without_id = report._commit_commands_without_id([SimpleNamespace(facts=facts)])
|
|
257
|
+
return True, silence, paths, commits, partial, without_id
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _cause_phrase_for_failed(path: Path) -> tuple[str, str | None]:
|
|
261
|
+
"""(phrase, reset_time) from the same last-transcript-line rule
|
|
262
|
+
detectors/attempt_failed.py uses (docs/DESIGN.md 4.3, 4.13.2): 'cause not
|
|
263
|
+
recorded' when the last entry carries none of the four runtime-produced
|
|
264
|
+
markers attempt_failed itself recognizes."""
|
|
265
|
+
diagnostics: list[str] = []
|
|
266
|
+
events, _version, _session_id, _total, _bad, _tool_calls, marker_hits = (
|
|
267
|
+
reader._read_transcript(path, diagnostics)
|
|
268
|
+
)
|
|
269
|
+
last_event = final_event(events)
|
|
270
|
+
if last_event is None:
|
|
271
|
+
return "cause not recorded", None
|
|
272
|
+
# _cause_from_last_event only reads attempt.marker_hits; a bare
|
|
273
|
+
# namespace stands in for the real Attempt this one-off transcript was
|
|
274
|
+
# never otherwise built into.
|
|
275
|
+
pseudo_attempt = SimpleNamespace(marker_hits=marker_hits)
|
|
276
|
+
if last_event.kind == "user" and _attempt_failed._anchored_on_event(
|
|
277
|
+
pseudo_attempt, "kill_sentinel", last_event
|
|
278
|
+
):
|
|
279
|
+
return "interrupted before finishing", None
|
|
280
|
+
if last_event.kind == "assistant":
|
|
281
|
+
matched = _attempt_failed._cause_from_last_event(pseudo_attempt, last_event)
|
|
282
|
+
if matched is not None:
|
|
283
|
+
_cause, phrase, reset_time = matched
|
|
284
|
+
return phrase, reset_time
|
|
285
|
+
return "cause not recorded", None
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
# --- line formatting --------------------------------------------------
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _line(word: str, rest: str) -> str:
|
|
292
|
+
return f"{word:<{_LABEL_WIDTH}}{rest}"
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _rate_limited(key: str, now: float, last_print: dict[str, float]) -> bool:
|
|
296
|
+
last = last_print.get(key)
|
|
297
|
+
return last is not None and (now - last) < _RATE_LIMIT_SECONDS
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@dataclass
|
|
301
|
+
class _PendingKill:
|
|
302
|
+
"""A detected supersede transition not yet resolved to a printed KILL
|
|
303
|
+
line (release finding F10): retried on every poll until it either
|
|
304
|
+
prints, is conclusively decided not to be a kill, or hits
|
|
305
|
+
_MAX_PENDING_POLLS."""
|
|
306
|
+
|
|
307
|
+
previous: object
|
|
308
|
+
current: object
|
|
309
|
+
polls: int = 0
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@dataclass
|
|
313
|
+
class _PendingFailed:
|
|
314
|
+
"""A journal 'failed' occurrence not yet resolved to a printed FAILED
|
|
315
|
+
line (release finding F10), retried the same way as _PendingKill."""
|
|
316
|
+
|
|
317
|
+
occurrence: object
|
|
318
|
+
polls: int = 0
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
@dataclass
|
|
322
|
+
class _PendingOk:
|
|
323
|
+
"""A journal 'result' occurrence not yet resolved to a printed OK line
|
|
324
|
+
(docs/DESIGN.md 4.12, N8b K5.2), retried the same way as _PendingFailed."""
|
|
325
|
+
|
|
326
|
+
occurrence: object
|
|
327
|
+
polls: int = 0
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
# The lifecycle terminals that make DONE eligible at once (docs/DESIGN.md 4.12): the
|
|
331
|
+
# terminals a parent notification, a TaskStop, or a synchronous launch result can give.
|
|
332
|
+
_IMMEDIATE_TERMINALS = frozenset({"completed", "failed", "stopped"})
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _parent_signature(path: Path) -> tuple | None:
|
|
336
|
+
"""(inode, size, mtime_ns, ctime_ns) of the parent transcript, None when it cannot be
|
|
337
|
+
statted. The parent index is refreshed only when this changes (docs/DESIGN.md 4.12)."""
|
|
338
|
+
try:
|
|
339
|
+
stat = path.stat()
|
|
340
|
+
except OSError:
|
|
341
|
+
return None
|
|
342
|
+
return (stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# --- agent names (N3b P14, C-1, C-5, C-6) -----------------------------------
|
|
346
|
+
|
|
347
|
+
WATCH_DIAGNOSTIC = f"agent-postmortem watch: {disclosure.UNAVAILABLE_DIAGNOSTIC}"
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class _AgentNames:
|
|
351
|
+
"""The agent named on a watch line or in an event's `agent` field is the
|
|
352
|
+
event attempt's own stable alias `a:<alias>` (P14), the one `run` shows
|
|
353
|
+
for the same attempt. The agents known at start register in one call; an
|
|
354
|
+
agent first named by a later event registers then (one locked
|
|
355
|
+
registration per new agent, the book's slow path). If a registration
|
|
356
|
+
fails the book degrades: that event and every later one name agents
|
|
357
|
+
positionally (`agent-N`, numbered within this watch) while the lines
|
|
358
|
+
already printed stand (C-1). The diagnostic goes to stderr once, as the
|
|
359
|
+
one fixed line, in text and --json mode alike (C-6); stdout stays clean.
|
|
360
|
+
|
|
361
|
+
An attempt whose agentId is missing, null or not a canonical id (C-5) is
|
|
362
|
+
named by a positional `agent-N`, never by its label."""
|
|
363
|
+
|
|
364
|
+
def __init__(self, err) -> None:
|
|
365
|
+
self._err = err
|
|
366
|
+
self._diagnosed = False
|
|
367
|
+
self._book: disclosure.AliasBook | None = None
|
|
368
|
+
|
|
369
|
+
def _check(self) -> None:
|
|
370
|
+
if self._book is not None and not self._book.available and not self._diagnosed:
|
|
371
|
+
self._diagnosed = True
|
|
372
|
+
print(WATCH_DIAGNOSTIC, file=self._err)
|
|
373
|
+
|
|
374
|
+
def start(self, occurrences) -> None:
|
|
375
|
+
known: list[tuple[str, str]] = []
|
|
376
|
+
for occurrence in occurrences:
|
|
377
|
+
agent_id = occurrence.agent_id
|
|
378
|
+
if disclosure.is_canonical_id("agent", agent_id) and ("agent", agent_id) not in known:
|
|
379
|
+
known.append(("agent", agent_id))
|
|
380
|
+
self._book = disclosure.register(known) if known else disclosure.lazy_book()
|
|
381
|
+
self._check()
|
|
382
|
+
|
|
383
|
+
def started(self) -> bool:
|
|
384
|
+
return self._book is not None
|
|
385
|
+
|
|
386
|
+
def name(self, occurrence) -> str:
|
|
387
|
+
assert self._book is not None
|
|
388
|
+
agent_id = occurrence.agent_id
|
|
389
|
+
if disclosure.is_canonical_id("agent", agent_id):
|
|
390
|
+
alias = self._book.alias("agent", agent_id)
|
|
391
|
+
else:
|
|
392
|
+
# Not canonical: never an alias and never the label. The book
|
|
393
|
+
# numbers it positionally, in step with every other positional
|
|
394
|
+
# name of this watch.
|
|
395
|
+
alias = self._book.alias("agent", f"no-identity:{occurrence.key}:{occurrence.ordinal}")
|
|
396
|
+
self._check()
|
|
397
|
+
return disclosure.rendered_form("agent", alias)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
# --- the poll loop ----------------------------------------------------
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def watch_run(
|
|
404
|
+
run_id_or_path: str,
|
|
405
|
+
root: Path,
|
|
406
|
+
*,
|
|
407
|
+
interval: float = DEFAULT_INTERVAL,
|
|
408
|
+
max_hours: float = DEFAULT_MAX_HOURS,
|
|
409
|
+
quiet_after: float = DEFAULT_QUIET_AFTER,
|
|
410
|
+
done_grace: float = DEFAULT_DONE_GRACE,
|
|
411
|
+
json_mode: bool = False,
|
|
412
|
+
out=None,
|
|
413
|
+
err=None,
|
|
414
|
+
) -> int:
|
|
415
|
+
out = sys.stdout if out is None else out
|
|
416
|
+
err = sys.stderr if err is None else err
|
|
417
|
+
|
|
418
|
+
start = _now()
|
|
419
|
+
run_dir = _wait_for_journal(run_id_or_path, root, start, interval, err)
|
|
420
|
+
if run_dir is None:
|
|
421
|
+
return 2
|
|
422
|
+
journal_path = run_dir / "journal.jsonl"
|
|
423
|
+
# N3b C2: a directory name is printed as a run id only when it passes the
|
|
424
|
+
# reader's validator; otherwise the replacement `run` itself uses.
|
|
425
|
+
run_id = reader.valid_workflow_run_id(run_dir.name)
|
|
426
|
+
|
|
427
|
+
names = _AgentNames(err)
|
|
428
|
+
offset = 0
|
|
429
|
+
full_journal: list[dict] = []
|
|
430
|
+
last_print: dict[str, float] = {}
|
|
431
|
+
handled_kill: set[tuple[str, int]] = set()
|
|
432
|
+
handled_fail: set[tuple[str, int]] = set()
|
|
433
|
+
pending_kill: dict[tuple[str, int], _PendingKill] = {}
|
|
434
|
+
pending_fail: dict[tuple[str, int], _PendingFailed] = {}
|
|
435
|
+
handled_ok: set[tuple[str, int]] = set()
|
|
436
|
+
pending_ok: dict[tuple[str, int], _PendingOk] = {}
|
|
437
|
+
quiet_threshold: dict[tuple[str, int], float] = {}
|
|
438
|
+
resolved_since: float | None = None
|
|
439
|
+
started_seen = 0
|
|
440
|
+
next_line_no = 0
|
|
441
|
+
malformed = False
|
|
442
|
+
start_printed = False
|
|
443
|
+
# docs/DESIGN.md 4.12 (N8b K5.4): the parent index and the stat signature it was last
|
|
444
|
+
# cleanly refreshed at, the newest parent generation launch line known after the previous
|
|
445
|
+
# poll, and the watermark a RESET leaves behind for the epoch it ended.
|
|
446
|
+
parent_path = reader._parent_transcript_path_for_workflow_dir(run_dir)
|
|
447
|
+
parent_index = reader.ParentIndex(None, [], [], []) if parent_path is None else None
|
|
448
|
+
parent_signature: tuple | None = None
|
|
449
|
+
known_newest_launch = 0
|
|
450
|
+
epoch_watermark = 0
|
|
451
|
+
|
|
452
|
+
while True:
|
|
453
|
+
try:
|
|
454
|
+
new_lines, offset, truncated = _read_new_journal_lines(journal_path, offset)
|
|
455
|
+
except JournalLineTooLong:
|
|
456
|
+
# docs/DESIGN.md 4.12 (release finding F9): an immediate, final
|
|
457
|
+
# verdict -- not gated on every key being resolved, unlike the
|
|
458
|
+
# malformed-complete-line case below. A first poll that ends here prints
|
|
459
|
+
# no START: the journal was never parsed, so its counts are unknown and
|
|
460
|
+
# the exit is final (docs/DESIGN.md 4.12).
|
|
461
|
+
if json_mode:
|
|
462
|
+
payload = {"event": "MALFORMED", "reason": "line longer than 1 MB"}
|
|
463
|
+
print(json.dumps(payload), file=out)
|
|
464
|
+
else:
|
|
465
|
+
print("MALFORMED journal line longer than 1 MB", file=out)
|
|
466
|
+
return 4
|
|
467
|
+
|
|
468
|
+
if truncated:
|
|
469
|
+
# docs/DESIGN.md 4.12 (release finding F8): the journal shrank
|
|
470
|
+
# (truncated or replaced by a fresh run reusing the same path)
|
|
471
|
+
# -- every piece of state derived from the old content is reset
|
|
472
|
+
# before the replacement content is processed at all, so no
|
|
473
|
+
# event from the old journal can combine with the new one into
|
|
474
|
+
# a false KILL or DONE.
|
|
475
|
+
full_journal = []
|
|
476
|
+
last_print = {}
|
|
477
|
+
handled_kill = set()
|
|
478
|
+
handled_fail = set()
|
|
479
|
+
pending_kill = {}
|
|
480
|
+
pending_fail = {}
|
|
481
|
+
handled_ok = set()
|
|
482
|
+
pending_ok = {}
|
|
483
|
+
quiet_threshold = {}
|
|
484
|
+
resolved_since = None
|
|
485
|
+
started_seen = 0
|
|
486
|
+
next_line_no = 0
|
|
487
|
+
malformed = False
|
|
488
|
+
# K5.4: a new journal epoch. Generations launched up to the newest line known
|
|
489
|
+
# before this reset belong to the ended epoch and cannot finish the replacement.
|
|
490
|
+
epoch_watermark = known_newest_launch
|
|
491
|
+
if json_mode:
|
|
492
|
+
print(json.dumps({"event": "RESET"}), file=out)
|
|
493
|
+
else:
|
|
494
|
+
print(_line("RESET", "journal was truncated or replaced; state cleared."), file=out)
|
|
495
|
+
|
|
496
|
+
malformed_notice: int | None = None
|
|
497
|
+
for raw_line in new_lines:
|
|
498
|
+
next_line_no += 1
|
|
499
|
+
try:
|
|
500
|
+
entry = json.loads(raw_line)
|
|
501
|
+
except (json.JSONDecodeError, ValueError, RecursionError):
|
|
502
|
+
entry = None
|
|
503
|
+
if not isinstance(entry, dict):
|
|
504
|
+
# docs/DESIGN.md 4.12 (release finding F9): a complete line
|
|
505
|
+
# that is not a JSON object is malformed evidence -- printed
|
|
506
|
+
# once (after START, K5.1), and DONE can never be printed
|
|
507
|
+
# again for this run.
|
|
508
|
+
if not malformed:
|
|
509
|
+
malformed = True
|
|
510
|
+
malformed_notice = next_line_no
|
|
511
|
+
continue
|
|
512
|
+
full_journal.append(entry)
|
|
513
|
+
|
|
514
|
+
occurrences, _diagnostics = reader._run_journal_state_machine(full_journal)
|
|
515
|
+
if not names.started():
|
|
516
|
+
names.start(occurrences)
|
|
517
|
+
|
|
518
|
+
# docs/DESIGN.md 4.12 (K5.1): START is the first byte of output, once, ahead of the
|
|
519
|
+
# MALFORMED line this very poll parsed and ahead of every other event.
|
|
520
|
+
if not start_printed:
|
|
521
|
+
_emit_start(run_id, occurrences, json_mode, out)
|
|
522
|
+
start_printed = True
|
|
523
|
+
if malformed_notice is not None:
|
|
524
|
+
if json_mode:
|
|
525
|
+
payload = {"event": "MALFORMED", "line": malformed_notice, "reason": "not JSON"}
|
|
526
|
+
print(json.dumps(payload), file=out)
|
|
527
|
+
else:
|
|
528
|
+
print(f"MALFORMED journal line {malformed_notice} is not JSON", file=out)
|
|
529
|
+
|
|
530
|
+
now = _now()
|
|
531
|
+
|
|
532
|
+
# KILL: a second (or later) 'started' occurrence for a key, whose
|
|
533
|
+
# immediately preceding occurrence's own transcript ends in the kill
|
|
534
|
+
# sentinel (docs/DESIGN.md 4.12). Newly detected transitions join
|
|
535
|
+
# pending_kill rather than being resolved inline (release finding
|
|
536
|
+
# F10): an event is marked handled only once it is actually
|
|
537
|
+
# emitted, decided not to be a kill, or forced past the poll cap.
|
|
538
|
+
previous_by_key: dict[str, object] = {}
|
|
539
|
+
for occurrence in occurrences:
|
|
540
|
+
previous = previous_by_key.get(occurrence.key)
|
|
541
|
+
if previous is not None and occurrence.ordinal == previous.ordinal + 1:
|
|
542
|
+
handle = (occurrence.key, previous.ordinal)
|
|
543
|
+
if handle not in handled_kill and handle not in pending_kill:
|
|
544
|
+
pending_kill[handle] = _PendingKill(previous=previous, current=occurrence)
|
|
545
|
+
previous_by_key[occurrence.key] = occurrence
|
|
546
|
+
|
|
547
|
+
for handle in list(pending_kill.keys()):
|
|
548
|
+
pending = pending_kill[handle]
|
|
549
|
+
status = _try_emit_kill(
|
|
550
|
+
run_dir, run_id, names, pending.previous, pending.current, now, last_print,
|
|
551
|
+
json_mode, out,
|
|
552
|
+
)
|
|
553
|
+
if status == "retry":
|
|
554
|
+
pending.polls += 1
|
|
555
|
+
if pending.polls < _MAX_PENDING_POLLS:
|
|
556
|
+
continue
|
|
557
|
+
_emit_kill_fallback(
|
|
558
|
+
run_id, names, pending.previous, pending.current, json_mode, out,
|
|
559
|
+
)
|
|
560
|
+
handled_kill.add(handle)
|
|
561
|
+
del pending_kill[handle]
|
|
562
|
+
|
|
563
|
+
# FAILED: a journal 'failed' event. Same handled-only-after-
|
|
564
|
+
# emission, bounded-retry treatment as KILL above (release finding
|
|
565
|
+
# F10).
|
|
566
|
+
for occurrence in occurrences:
|
|
567
|
+
if occurrence.closed_by != "failed":
|
|
568
|
+
continue
|
|
569
|
+
handle = (occurrence.key, occurrence.ordinal)
|
|
570
|
+
if handle not in handled_fail and handle not in pending_fail:
|
|
571
|
+
pending_fail[handle] = _PendingFailed(occurrence=occurrence)
|
|
572
|
+
|
|
573
|
+
for handle in list(pending_fail.keys()):
|
|
574
|
+
pending = pending_fail[handle]
|
|
575
|
+
status = _try_emit_failed(
|
|
576
|
+
run_dir, names, pending.occurrence, now, last_print, json_mode, out,
|
|
577
|
+
)
|
|
578
|
+
if status == "retry":
|
|
579
|
+
pending.polls += 1
|
|
580
|
+
if pending.polls < _MAX_PENDING_POLLS:
|
|
581
|
+
continue
|
|
582
|
+
_force_emit_failed(
|
|
583
|
+
run_dir, names, pending.occurrence, now, last_print, json_mode, out,
|
|
584
|
+
)
|
|
585
|
+
handled_fail.add(handle)
|
|
586
|
+
del pending_fail[handle]
|
|
587
|
+
|
|
588
|
+
# OK: a journal 'result' closed an attempt. Same handled-only-after-emission,
|
|
589
|
+
# bounded-retry treatment as FAILED (docs/DESIGN.md 4.12, N8b K5.2).
|
|
590
|
+
for occurrence in occurrences:
|
|
591
|
+
if occurrence.closed_by != "completed":
|
|
592
|
+
continue
|
|
593
|
+
handle = (occurrence.key, occurrence.ordinal)
|
|
594
|
+
if handle not in handled_ok and handle not in pending_ok:
|
|
595
|
+
pending_ok[handle] = _PendingOk(occurrence=occurrence)
|
|
596
|
+
|
|
597
|
+
for handle in list(pending_ok.keys()):
|
|
598
|
+
pending = pending_ok[handle]
|
|
599
|
+
status = _try_emit_ok(names, pending.occurrence, now, last_print, json_mode, out)
|
|
600
|
+
if status == "retry":
|
|
601
|
+
pending.polls += 1
|
|
602
|
+
if pending.polls < _MAX_PENDING_POLLS:
|
|
603
|
+
continue
|
|
604
|
+
_force_emit_ok(names, pending.occurrence, now, last_print, json_mode, out)
|
|
605
|
+
handled_ok.add(handle)
|
|
606
|
+
del pending_ok[handle]
|
|
607
|
+
|
|
608
|
+
# DONE (docs/DESIGN.md 4.12, N8b K5.4). The parent index is refreshed only when the
|
|
609
|
+
# parent transcript's stat signature changed since the last clean refresh, and the
|
|
610
|
+
# signature is not remembered while the index still owes a full recheck (a transient
|
|
611
|
+
# read failure is retried on the next poll though the file did not change). The
|
|
612
|
+
# lifecycle table is evaluated from this poll's own occurrences, never from a second
|
|
613
|
+
# journal read.
|
|
614
|
+
if parent_path is not None:
|
|
615
|
+
signature = _parent_signature(parent_path)
|
|
616
|
+
if (
|
|
617
|
+
parent_index is None or signature != parent_signature
|
|
618
|
+
or parent_index.pending_full_recheck
|
|
619
|
+
):
|
|
620
|
+
parent_index = reader.ParentIndex.for_path(parent_path)
|
|
621
|
+
parent_signature = None if parent_index.pending_full_recheck else signature
|
|
622
|
+
journal_resolved = reader.journal_resolved_from_occurrences(occurrences)
|
|
623
|
+
lifecycle = reader.lifecycle_from_journal_state(
|
|
624
|
+
run_dir, run_id, journal_resolved, parent_index, after_launch_line=epoch_watermark,
|
|
625
|
+
)
|
|
626
|
+
known_newest_launch = max(
|
|
627
|
+
(generation.launch_line for generation in parent_index.generations_for(run_id)),
|
|
628
|
+
default=0,
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
# Eligible at once from terminal evidence the journal did not give, or after the grace
|
|
632
|
+
# period once every key is resolved (the gap between two sequential phases is not the
|
|
633
|
+
# end of the run). A malformed journal (release finding F9) makes DONE impossible:
|
|
634
|
+
# LIMIT is printed instead, at the point DONE would have been. DONE waits for every
|
|
635
|
+
# already-detected KILL, FAILED, and OK transition to be emitted or forced.
|
|
636
|
+
if len(occurrences) != started_seen:
|
|
637
|
+
started_seen = len(occurrences)
|
|
638
|
+
resolved_since = None
|
|
639
|
+
if journal_resolved:
|
|
640
|
+
if resolved_since is None:
|
|
641
|
+
resolved_since = now
|
|
642
|
+
else:
|
|
643
|
+
resolved_since = None
|
|
644
|
+
eligible = lifecycle.terminal in _IMMEDIATE_TERMINALS or (
|
|
645
|
+
lifecycle.terminal == "journal_resolved" and resolved_since is not None
|
|
646
|
+
and now - resolved_since >= done_grace
|
|
647
|
+
)
|
|
648
|
+
if eligible and not (pending_kill or pending_fail or pending_ok):
|
|
649
|
+
if malformed:
|
|
650
|
+
return _emit_malformed_limit(json_mode, out)
|
|
651
|
+
return _emit_done(run_dir, json_mode, out)
|
|
652
|
+
|
|
653
|
+
# QUIET: the currently open attempt of every key, if any.
|
|
654
|
+
last_by_key: dict[str, object] = {}
|
|
655
|
+
for occurrence in occurrences:
|
|
656
|
+
last_by_key[occurrence.key] = occurrence
|
|
657
|
+
for key, occurrence in last_by_key.items():
|
|
658
|
+
if occurrence.closed_by is not None:
|
|
659
|
+
continue
|
|
660
|
+
_maybe_emit_quiet(
|
|
661
|
+
run_dir, names, key, occurrence, now, last_print, quiet_threshold,
|
|
662
|
+
quiet_after, json_mode, out,
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
if now - start >= max_hours * 3600:
|
|
666
|
+
_emit_limit(max_hours, json_mode, out)
|
|
667
|
+
return 4
|
|
668
|
+
|
|
669
|
+
# docs/DESIGN.md 4.12 (release finding F11): capped to whatever is
|
|
670
|
+
# left of the --max-hours deadline, so a large --interval cannot
|
|
671
|
+
# overshoot it either.
|
|
672
|
+
remaining = max_hours * 3600 - (now - start)
|
|
673
|
+
_sleep(min(interval, remaining))
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _paths_words(paths: int, partial: bool) -> str:
|
|
677
|
+
return f"{paths}+ paths" if partial else f"{paths} {report._plural(paths, 'path')}"
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def _commit_words(commits: int, without_id: int) -> str:
|
|
681
|
+
"""docs/DESIGN.md 4.12 (G4b): `reported <c> commit(s)`, `reported <c>+ commits`, or
|
|
682
|
+
`commit count unknown`, by the COMMITS cell's rule (4.7)."""
|
|
683
|
+
words = report._commits_words(commits, without_id)
|
|
684
|
+
return words if without_id > 0 and not commits else f"reported {words}"
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _emit_kill_line(
|
|
688
|
+
run_id: str, names: _AgentNames, previous, current, silence: float | None, paths: int,
|
|
689
|
+
commits: int, json_mode: bool, out, *, paths_partial: bool = False,
|
|
690
|
+
commits_without_id: int = 0,
|
|
691
|
+
) -> None:
|
|
692
|
+
agent = names.name(previous)
|
|
693
|
+
silence_int = None if silence is None else round(silence)
|
|
694
|
+
silence_str = "unknown" if silence_int is None else str(silence_int)
|
|
695
|
+
|
|
696
|
+
if json_mode:
|
|
697
|
+
payload = {
|
|
698
|
+
"event": "KILL",
|
|
699
|
+
"agent": agent,
|
|
700
|
+
"attempt": previous.ordinal,
|
|
701
|
+
"silence_seconds": silence_int,
|
|
702
|
+
"restarted_as": current.ordinal,
|
|
703
|
+
"paths_written": paths,
|
|
704
|
+
"commits": commits,
|
|
705
|
+
"run_id": run_id,
|
|
706
|
+
}
|
|
707
|
+
print(json.dumps(payload), file=out)
|
|
708
|
+
else:
|
|
709
|
+
print(
|
|
710
|
+
_line(
|
|
711
|
+
"KILL",
|
|
712
|
+
f"{agent} attempt {previous.ordinal} after {silence_str}s of silence; "
|
|
713
|
+
f"restarted as attempt {current.ordinal}.",
|
|
714
|
+
),
|
|
715
|
+
file=out,
|
|
716
|
+
)
|
|
717
|
+
print(
|
|
718
|
+
_line(
|
|
719
|
+
"",
|
|
720
|
+
f"attempt {previous.ordinal} wrote {_paths_words(paths, paths_partial)}, "
|
|
721
|
+
f"{_commit_words(commits, commits_without_id)}. "
|
|
722
|
+
f"-> agent-postmortem rescue plan {run_id}",
|
|
723
|
+
),
|
|
724
|
+
file=out,
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def _try_emit_kill(
|
|
729
|
+
run_dir: Path, run_id: str, names: _AgentNames, previous, current, now: float,
|
|
730
|
+
last_print: dict[str, float], json_mode: bool, out,
|
|
731
|
+
) -> str:
|
|
732
|
+
"""Attempt to resolve one supersede transition to a printed KILL line
|
|
733
|
+
(release finding F10). Returns 'emitted' (printed, mark handled),
|
|
734
|
+
'not_a_kill' (conclusively decided -- the earlier transcript exists and
|
|
735
|
+
does not end in the kill sentinel -- mark handled, never print, never
|
|
736
|
+
retry again), or 'retry' (suppressed by the rate limit, or the earlier
|
|
737
|
+
transcript does not exist yet -- try again next poll, up to
|
|
738
|
+
_MAX_PENDING_POLLS)."""
|
|
739
|
+
if not previous.agent_id:
|
|
740
|
+
return "not_a_kill"
|
|
741
|
+
transcript_path = _transcript_path(run_dir, previous.agent_id)
|
|
742
|
+
if not transcript_path.exists():
|
|
743
|
+
return "retry"
|
|
744
|
+
is_kill, silence, paths, commits, partial, without_id = _parse_kill(transcript_path)
|
|
745
|
+
if not is_kill:
|
|
746
|
+
return "not_a_kill"
|
|
747
|
+
if _rate_limited(previous.key, now, last_print):
|
|
748
|
+
return "retry"
|
|
749
|
+
|
|
750
|
+
_emit_kill_line(
|
|
751
|
+
run_id, names, previous, current, silence, paths, commits, json_mode, out,
|
|
752
|
+
paths_partial=partial, commits_without_id=without_id,
|
|
753
|
+
)
|
|
754
|
+
last_print[previous.key] = now
|
|
755
|
+
return "emitted"
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def _emit_kill_fallback(
|
|
759
|
+
run_id: str, names: _AgentNames, previous, current, json_mode: bool, out,
|
|
760
|
+
) -> None:
|
|
761
|
+
"""docs/DESIGN.md 4.12 (release finding F10): previous.polls reached
|
|
762
|
+
_MAX_PENDING_POLLS with the transition still unresolved (the earlier
|
|
763
|
+
transcript never appeared, or the rate limit never cleared) -- printed
|
|
764
|
+
anyway, without the transcript-derived details, so a killed attempt can
|
|
765
|
+
never be silently dropped forever."""
|
|
766
|
+
agent = names.name(previous)
|
|
767
|
+
if json_mode:
|
|
768
|
+
payload = {
|
|
769
|
+
"event": "KILL",
|
|
770
|
+
"agent": agent,
|
|
771
|
+
"attempt": previous.ordinal,
|
|
772
|
+
"silence_seconds": None,
|
|
773
|
+
"restarted_as": current.ordinal,
|
|
774
|
+
"paths_written": None,
|
|
775
|
+
"commits": None,
|
|
776
|
+
"run_id": run_id,
|
|
777
|
+
"details_unavailable": True,
|
|
778
|
+
}
|
|
779
|
+
print(json.dumps(payload), file=out)
|
|
780
|
+
else:
|
|
781
|
+
print(
|
|
782
|
+
_line(
|
|
783
|
+
"KILL",
|
|
784
|
+
f"{agent} attempt {previous.ordinal} (details unavailable); "
|
|
785
|
+
f"restarted as attempt {current.ordinal}.",
|
|
786
|
+
),
|
|
787
|
+
file=out,
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _compute_and_emit_failed(
|
|
792
|
+
run_dir: Path, names: _AgentNames, occurrence, json_mode: bool, out,
|
|
793
|
+
) -> None:
|
|
794
|
+
phrase, reset_time = "cause not recorded", None
|
|
795
|
+
if occurrence.agent_id:
|
|
796
|
+
transcript_path = _transcript_path(run_dir, occurrence.agent_id)
|
|
797
|
+
if transcript_path.exists():
|
|
798
|
+
phrase, reset_time = _cause_phrase_for_failed(transcript_path)
|
|
799
|
+
|
|
800
|
+
agent = names.name(occurrence)
|
|
801
|
+
if json_mode:
|
|
802
|
+
payload = {
|
|
803
|
+
"event": "FAILED", "agent": agent, "attempt": occurrence.ordinal, "cause": phrase,
|
|
804
|
+
"reset_time": reset_time,
|
|
805
|
+
}
|
|
806
|
+
print(json.dumps(payload), file=out)
|
|
807
|
+
else:
|
|
808
|
+
print(_line("FAILED", f"{agent} attempt {occurrence.ordinal}: {phrase}."), file=out)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _try_emit_failed(
|
|
812
|
+
run_dir: Path, names: _AgentNames, occurrence, now: float, last_print: dict[str, float],
|
|
813
|
+
json_mode: bool, out,
|
|
814
|
+
) -> str:
|
|
815
|
+
"""docs/DESIGN.md 4.12 (release finding F10): 'emitted' or 'retry'
|
|
816
|
+
(suppressed by the rate limit only -- a missing transcript already
|
|
817
|
+
degrades gracefully to 'cause not recorded' rather than blocking
|
|
818
|
+
emission, so it is never a reason to retry)."""
|
|
819
|
+
if _rate_limited(occurrence.key, now, last_print):
|
|
820
|
+
return "retry"
|
|
821
|
+
_compute_and_emit_failed(run_dir, names, occurrence, json_mode, out)
|
|
822
|
+
last_print[occurrence.key] = now
|
|
823
|
+
return "emitted"
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def _force_emit_failed(
|
|
827
|
+
run_dir: Path, names: _AgentNames, occurrence, now: float, last_print: dict[str, float],
|
|
828
|
+
json_mode: bool, out,
|
|
829
|
+
) -> None:
|
|
830
|
+
"""release finding F10: occurrence.polls reached _MAX_PENDING_POLLS
|
|
831
|
+
while only ever suppressed by the rate limit -- printed anyway, bypassing
|
|
832
|
+
the limit, so a FAILED event can never be dropped forever either."""
|
|
833
|
+
_compute_and_emit_failed(run_dir, names, occurrence, json_mode, out)
|
|
834
|
+
last_print[occurrence.key] = now
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def _emit_start(run_id: str, occurrences: list, json_mode: bool, out) -> None:
|
|
838
|
+
"""docs/DESIGN.md 4.12 (N8b K5.1): once, first. `agents` is the number of distinct journal
|
|
839
|
+
keys started so far and `attempts` the number of started occurrences."""
|
|
840
|
+
agents = len({occurrence.key for occurrence in occurrences})
|
|
841
|
+
attempts = len(occurrences)
|
|
842
|
+
if json_mode:
|
|
843
|
+
payload = {"event": "START", "run": run_id, "agents": agents, "attempts": attempts}
|
|
844
|
+
print(json.dumps(payload), file=out)
|
|
845
|
+
else:
|
|
846
|
+
print(
|
|
847
|
+
_line("START", f"{run_id}: {agents} agents started so far, {attempts} attempts."),
|
|
848
|
+
file=out,
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _emit_ok_line(names: _AgentNames, occurrence, json_mode: bool, out) -> None:
|
|
853
|
+
agent = names.name(occurrence)
|
|
854
|
+
if json_mode:
|
|
855
|
+
payload = {"event": "OK", "agent": agent, "attempt": occurrence.ordinal}
|
|
856
|
+
print(json.dumps(payload), file=out)
|
|
857
|
+
else:
|
|
858
|
+
print(_line("OK", f"{agent} attempt {occurrence.ordinal} returned a result."), file=out)
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
def _try_emit_ok(
|
|
862
|
+
names: _AgentNames, occurrence, now: float, last_print: dict[str, float], json_mode: bool,
|
|
863
|
+
out,
|
|
864
|
+
) -> str:
|
|
865
|
+
"""docs/DESIGN.md 4.12 (N8b K5.2): 'emitted' or 'retry' (suppressed by the per-agent rate
|
|
866
|
+
limit, the only reason an OK waits)."""
|
|
867
|
+
if _rate_limited(occurrence.key, now, last_print):
|
|
868
|
+
return "retry"
|
|
869
|
+
_emit_ok_line(names, occurrence, json_mode, out)
|
|
870
|
+
last_print[occurrence.key] = now
|
|
871
|
+
return "emitted"
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _force_emit_ok(
|
|
875
|
+
names: _AgentNames, occurrence, now: float, last_print: dict[str, float], json_mode: bool,
|
|
876
|
+
out,
|
|
877
|
+
) -> None:
|
|
878
|
+
"""The bounded retry ran out: printed through the rate limit, so an OK is never dropped."""
|
|
879
|
+
_emit_ok_line(names, occurrence, json_mode, out)
|
|
880
|
+
last_print[occurrence.key] = now
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def _emit_malformed_limit(json_mode: bool, out) -> int:
|
|
884
|
+
"""docs/DESIGN.md 4.12 (release finding F9): printed at the exact point
|
|
885
|
+
DONE would otherwise have been printed -- every key is resolved -- but a
|
|
886
|
+
malformed journal line means the run's own final state can no longer be
|
|
887
|
+
trusted, so this replaces DONE and exits 4 instead of 0/3."""
|
|
888
|
+
if json_mode:
|
|
889
|
+
print(json.dumps({"event": "LIMIT", "reason": "malformed"}), file=out)
|
|
890
|
+
else:
|
|
891
|
+
print(_line("LIMIT", "journal has malformed lines; run state unknown."), file=out)
|
|
892
|
+
return 4
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def _maybe_emit_quiet(
|
|
896
|
+
run_dir: Path, names: _AgentNames, key: str, occurrence, now: float,
|
|
897
|
+
last_print: dict[str, float], quiet_threshold: dict[tuple[str, int], float],
|
|
898
|
+
quiet_after: float, json_mode: bool, out,
|
|
899
|
+
) -> None:
|
|
900
|
+
if not occurrence.agent_id:
|
|
901
|
+
return
|
|
902
|
+
transcript_path = _transcript_path(run_dir, occurrence.agent_id)
|
|
903
|
+
try:
|
|
904
|
+
mtime = transcript_path.stat().st_mtime
|
|
905
|
+
except OSError:
|
|
906
|
+
return
|
|
907
|
+
silence = now - mtime
|
|
908
|
+
if silence < 0:
|
|
909
|
+
return
|
|
910
|
+
threshold_key = (occurrence.key, occurrence.ordinal)
|
|
911
|
+
threshold = quiet_threshold.get(threshold_key, quiet_after)
|
|
912
|
+
if silence < threshold:
|
|
913
|
+
return
|
|
914
|
+
if _rate_limited(key, now, last_print):
|
|
915
|
+
return
|
|
916
|
+
|
|
917
|
+
agent = names.name(occurrence)
|
|
918
|
+
silence_int = round(silence)
|
|
919
|
+
if json_mode:
|
|
920
|
+
payload = {
|
|
921
|
+
"event": "QUIET", "agent": agent, "attempt": occurrence.ordinal,
|
|
922
|
+
"silence_seconds": silence_int,
|
|
923
|
+
}
|
|
924
|
+
print(json.dumps(payload), file=out)
|
|
925
|
+
else:
|
|
926
|
+
rest = f"{agent} attempt {occurrence.ordinal} has produced nothing for {silence_int}s."
|
|
927
|
+
print(_line("QUIET", rest), file=out)
|
|
928
|
+
last_print[key] = now
|
|
929
|
+
quiet_threshold[threshold_key] = threshold * 2
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def _emit_done(run_dir: Path, json_mode: bool, out) -> int:
|
|
933
|
+
run = reader.load_run(run_dir)
|
|
934
|
+
findings = detectors.run_detectors(run)
|
|
935
|
+
verdict = report.verdict_of(run, findings)
|
|
936
|
+
agents = len(run.agents)
|
|
937
|
+
attempts = len(run.attempts)
|
|
938
|
+
|
|
939
|
+
if json_mode:
|
|
940
|
+
payload = {
|
|
941
|
+
"event": "DONE", "agents": agents, "attempts": attempts, "verdict": verdict,
|
|
942
|
+
"run_id": run.run_id, "next_command": f"agent-postmortem run {run.run_id}",
|
|
943
|
+
}
|
|
944
|
+
print(json.dumps(payload), file=out)
|
|
945
|
+
else:
|
|
946
|
+
print(
|
|
947
|
+
_line(
|
|
948
|
+
"DONE",
|
|
949
|
+
f"{agents} agents, {attempts} attempts, {verdict}. "
|
|
950
|
+
f"-> agent-postmortem run {run.run_id}",
|
|
951
|
+
),
|
|
952
|
+
file=out,
|
|
953
|
+
)
|
|
954
|
+
# docs/DESIGN.md 4.12, 4.13.11 (N5c-2), ordered: a FAILED verdict is 3, ahead of a run
|
|
955
|
+
# whose lifecycle terminal is `unknown`, which cannot be called a clean DONE and is 4.
|
|
956
|
+
if verdict.startswith("FAILED"):
|
|
957
|
+
return 3
|
|
958
|
+
if run.lifecycle is not None and run.lifecycle.terminal == "unknown":
|
|
959
|
+
return 4
|
|
960
|
+
return 0
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
def _format_hours(max_hours: float) -> str:
|
|
964
|
+
if float(max_hours).is_integer():
|
|
965
|
+
return str(int(max_hours))
|
|
966
|
+
return str(max_hours)
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def _emit_limit(max_hours: float, json_mode: bool, out) -> None:
|
|
970
|
+
if json_mode:
|
|
971
|
+
print(json.dumps({"event": "LIMIT", "hours": max_hours}), file=out)
|
|
972
|
+
else:
|
|
973
|
+
print(_line("LIMIT", f"{_format_hours(max_hours)}h reached; run not finished."), file=out)
|