agent-postmortem 0.3.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. agent_postmortem/__init__.py +9 -0
  2. agent_postmortem/argerrors.py +160 -0
  3. agent_postmortem/claims.py +43 -0
  4. agent_postmortem/cli.py +2278 -0
  5. agent_postmortem/commands.py +1543 -0
  6. agent_postmortem/config.py +435 -0
  7. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.jsonl +5 -0
  8. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.meta.json +8 -0
  9. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.jsonl +2 -0
  10. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.meta.json +8 -0
  11. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.jsonl +5 -0
  12. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.meta.json +8 -0
  13. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.jsonl +2 -0
  14. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.meta.json +8 -0
  15. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/journal.jsonl +7 -0
  16. agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000.jsonl +4 -0
  17. agent_postmortem/detectors/__init__.py +98 -0
  18. agent_postmortem/detectors/attempt_failed.py +261 -0
  19. agent_postmortem/detectors/base.py +319 -0
  20. agent_postmortem/detectors/bash_backgrounded.py +72 -0
  21. agent_postmortem/detectors/bash_timeout.py +94 -0
  22. agent_postmortem/detectors/claim_contradiction.py +251 -0
  23. agent_postmortem/detectors/classifier_unavailable.py +73 -0
  24. agent_postmortem/detectors/codex_unavailable.py +102 -0
  25. agent_postmortem/detectors/codex_write_denied.py +100 -0
  26. agent_postmortem/detectors/git_cwd_mismatch.py +74 -0
  27. agent_postmortem/detectors/mislabeled_kill.py +164 -0
  28. agent_postmortem/detectors/missing_artifact.py +159 -0
  29. agent_postmortem/detectors/permission_denied.py +99 -0
  30. agent_postmortem/detectors/silent_retry.py +163 -0
  31. agent_postmortem/detectors/structured_output_failure.py +91 -0
  32. agent_postmortem/detectors/tool_json_error.py +77 -0
  33. agent_postmortem/detectors/wasted_time.py +125 -0
  34. agent_postmortem/detectors/workflow_failed.py +62 -0
  35. agent_postmortem/disclosure.py +931 -0
  36. agent_postmortem/hooks/model_allocation_guard.mjs +248 -0
  37. agent_postmortem/hooks_cli.py +1269 -0
  38. agent_postmortem/ledger.py +624 -0
  39. agent_postmortem/markers.py +505 -0
  40. agent_postmortem/models.py +815 -0
  41. agent_postmortem/observe.py +792 -0
  42. agent_postmortem/reader.py +4611 -0
  43. agent_postmortem/redact.py +1981 -0
  44. agent_postmortem/report.py +3010 -0
  45. agent_postmortem/rescue.py +2096 -0
  46. agent_postmortem/returns.py +499 -0
  47. agent_postmortem/rules.py +627 -0
  48. agent_postmortem/severity.py +275 -0
  49. agent_postmortem/state.py +1261 -0
  50. agent_postmortem/trend.py +391 -0
  51. agent_postmortem/watch.py +973 -0
  52. agent_postmortem-0.3.4.dist-info/METADATA +264 -0
  53. agent_postmortem-0.3.4.dist-info/RECORD +56 -0
  54. agent_postmortem-0.3.4.dist-info/WHEEL +4 -0
  55. agent_postmortem-0.3.4.dist-info/entry_points.txt +3 -0
  56. agent_postmortem-0.3.4.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,973 @@
1
+ """`agent-postmortem watch`: a read-only follower for one workflow run
2
+ (docs/DESIGN.md 4.12; .reviews/proposals/2026-09-v0.2-guided-rescue.md
3
+ section 7).
4
+
5
+ Designed to be the command line a Claude Code Monitor runs: it prints one
6
+ START line first, then one line per event a person would act on (KILL, FAILED,
7
+ OK, QUIET) and nothing otherwise, then a terminal DONE or LIMIT line and exits.
8
+
9
+ Polling never re-reads a file it does not have to: the journal is read by
10
+ appended bytes only (`_read_new_journal_lines` keeps a byte offset and
11
+ restarts from 0 when the file is smaller than that offset -- a truncated or
12
+ replaced journal, which resets every other piece of derived state too and
13
+ prints one RESET line); a transcript is only ever stat'd for its mtime
14
+ (QUIET), or fully parsed once, the moment a KILL or FAILED needs its last
15
+ line or its silence span -- never polled incrementally. A partial trailing
16
+ journal line growing past 1 MB, or a complete line that is not a JSON
17
+ object, is malformed evidence (MALFORMED); a KILL/FAILED/OK event this poll
18
+ cannot yet resolve (the rate limit, or a transcript not on disk yet) is
19
+ retried on later polls, bounded at 12, rather than dropped.
20
+
21
+ DONE (docs/DESIGN.md 4.12, N8b K5.4) is eligible at once from a lifecycle terminal
22
+ of completed, failed, or stopped (an accepted notification, a TaskStop, or a
23
+ synchronous launch result), or after `--done-grace` once every journal key is
24
+ resolved. The lifecycle is evaluated from watch's own in-memory occurrences and
25
+ a parent index refreshed only when the parent transcript's stat signature
26
+ changes; a RESET starts a new journal epoch whose watermark hides the ended
27
+ epoch's generations.
28
+
29
+ `_now` and `_sleep` are module attributes so a test can monkeypatch both: the
30
+ suite never sleeps for a real second, and `_sleep` doubles as the point
31
+ where a scripted test appends the next bytes of a growing journal or
32
+ transcript.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import json
38
+ import sys
39
+ import time
40
+ from dataclasses import dataclass
41
+ from pathlib import Path
42
+ from types import SimpleNamespace
43
+
44
+ from . import detectors, disclosure, ledger, reader, report
45
+ from .detectors import attempt_failed as _attempt_failed
46
+ from .detectors.base import final_event, meaningful_event_before
47
+ from .markers import KILL_SENTINEL_PREFIX
48
+
49
+ DEFAULT_INTERVAL = 5.0
50
+ DEFAULT_MAX_HOURS = 6.0
51
+ DEFAULT_QUIET_AFTER = 300.0
52
+ # A workflow that runs its phases one after another has a moment between
53
+ # two phases where every key seen so far is resolved. DONE waits this long
54
+ # with no new `started` before it is printed, so that gap is not mistaken
55
+ # for the end of the run.
56
+ DEFAULT_DONE_GRACE = 30.0
57
+
58
+ # How long `watch` waits, printing nothing, for a run directory (and its
59
+ # journal.jsonl) that does not exist yet -- the launch hook fires before the
60
+ # Workflow tool has created either (docs/DESIGN.md 7.2's D8).
61
+ _JOURNAL_WAIT_SECONDS = 60.0
62
+
63
+ # At most one line per agent per this many seconds, except DONE and LIMIT.
64
+ _RATE_LIMIT_SECONDS = 60.0
65
+
66
+ # docs/DESIGN.md 4.12 (release finding F9): a partial (no trailing newline
67
+ # yet) journal line is withheld and reread on the next poll, but a line
68
+ # that keeps growing forever with no newline (a stuck or malformed writer)
69
+ # must not be reread and rechecked without bound.
70
+ _MAX_PARTIAL_LINE_BYTES = 1_048_576
71
+
72
+ # docs/DESIGN.md 4.12 (release finding F10): a KILL/FAILED/OK event suppressed
73
+ # by the rate limit, or (KILL only) by a transcript that does not exist
74
+ # yet, is retried on the next poll rather than dropped -- but only for this
75
+ # many polls, after which it is force-emitted (KILL, with a degraded
76
+ # message) or emitted as-is (FAILED, OK) and marked handled either way, so a
77
+ # transcript that never appears can never wedge the loop forever.
78
+ _MAX_PENDING_POLLS = 12
79
+
80
+
81
+ class JournalLineTooLong(Exception):
82
+ """docs/DESIGN.md 4.12 (release finding F9): the journal's unterminated
83
+ tail (from the last read offset to end of file, with no newline yet)
84
+ has grown past _MAX_PARTIAL_LINE_BYTES."""
85
+
86
+ # Every event word is padded to this width so the message text lines up the
87
+ # same way across KILL/FAILED/QUIET/DONE/LIMIT (and so KILL's own second,
88
+ # indented line lines up under it): "KILL".ljust(7) == "KILL ", "FAILED"
89
+ # ljust(7) == "FAILED ", and so on.
90
+ _LABEL_WIDTH = 7
91
+
92
+
93
+ def _now() -> float:
94
+ """Epoch seconds (docs/DESIGN.md 4.12): the same domain as a file's own
95
+ mtime, so QUIET's 'now - mtime' needs no unit conversion. A test
96
+ monkeypatches this to a controllable float instead of the real clock."""
97
+ return time.time()
98
+
99
+
100
+ def _sleep(seconds: float) -> None:
101
+ """A test monkeypatches this to advance a fake clock (and grow the
102
+ journal/transcripts a poll is about to see) instead of actually
103
+ sleeping."""
104
+ time.sleep(seconds)
105
+
106
+
107
+ # --- run resolution -------------------------------------------------------
108
+
109
+
110
+ def find_run_dir(run_id_or_path: str, root: Path) -> Path | None:
111
+ """A single matching workflow run directory, or None when nothing (yet)
112
+ matches -- never raises, since the caller's job is to keep polling until
113
+ the 60s journal-wait budget runs out (docs/DESIGN.md 4.12). Only
114
+ workflow runs are considered: a session carries no journal.jsonl, so
115
+ `watch` has nothing to follow for one."""
116
+ direct = Path(run_id_or_path)
117
+ if direct.is_dir():
118
+ return direct
119
+ try:
120
+ candidates = [
121
+ ref
122
+ for ref in reader.discover_runs(root, kinds=frozenset({"workflow"}))
123
+ if ref.run_id.startswith(run_id_or_path)
124
+ ]
125
+ except OSError:
126
+ return None
127
+ if len(candidates) == 1:
128
+ return candidates[0].path
129
+ return None
130
+
131
+
132
+ def _wait_for_journal(
133
+ run_id_or_path: str, root: Path, start: float, interval: float, err,
134
+ ) -> Path | None:
135
+ while True:
136
+ run_dir = find_run_dir(run_id_or_path, root)
137
+ if run_dir is not None and (run_dir / "journal.jsonl").exists():
138
+ return run_dir
139
+ remaining = _JOURNAL_WAIT_SECONDS - (_now() - start)
140
+ if remaining <= 0:
141
+ print(
142
+ "no journal appeared within 60 s: watch follows a Workflow run while it is "
143
+ "running (the plugin hands you this command when one starts); "
144
+ "agent-postmortem scan lists past runs",
145
+ file=err,
146
+ )
147
+ return None
148
+ # docs/DESIGN.md 4.12 (release finding F11): capped to whatever is
149
+ # left of the 60s budget, so a large --interval cannot overshoot it
150
+ # (--interval 300 must still give up within 60s, not 300s).
151
+ _sleep(min(interval, remaining))
152
+
153
+
154
+ # --- appended-bytes journal reading ----------------------------------------
155
+
156
+
157
+ def _read_new_journal_lines(path: Path, offset: int) -> tuple[list[str], int, bool]:
158
+ """The complete non-blank lines appended to path since `offset` bytes,
159
+ the new offset, and whether the file was found truncated or replaced
160
+ (release finding F8). A file smaller than `offset` has been truncated
161
+ or replaced (a fresh run reusing the same path): read from 0 instead of
162
+ raising or silently missing its early lines, and report `truncated`
163
+ True so the caller can reset every piece of state it derived from the
164
+ old content before processing the replacement. A trailing line with no
165
+ newline yet (a write still in flight) is withheld -- the offset does not
166
+ advance past it -- so the next call re-reads it once it is complete,
167
+ unless it has already grown past _MAX_PARTIAL_LINE_BYTES with still no
168
+ newline, which raises JournalLineTooLong (release finding F9) instead
169
+ of being reread and rechecked in full, without bound, forever."""
170
+ try:
171
+ size = path.stat().st_size
172
+ except OSError:
173
+ return [], offset, False
174
+ truncated = size < offset
175
+ if truncated:
176
+ offset = 0
177
+ if size <= offset:
178
+ return [], offset, truncated
179
+ with path.open("rb") as handle:
180
+ handle.seek(offset)
181
+ chunk = handle.read()
182
+ last_newline = chunk.rfind(b"\n")
183
+ # The bytes after the last newline (the whole chunk when there is none
184
+ # at all) are the as-yet-unterminated next line -- checked regardless
185
+ # of how many complete lines share this same read.
186
+ if len(chunk) - (last_newline + 1) > _MAX_PARTIAL_LINE_BYTES:
187
+ raise JournalLineTooLong()
188
+ if last_newline == -1:
189
+ return [], offset, truncated
190
+ usable = chunk[: last_newline + 1]
191
+ new_offset = offset + len(usable)
192
+ text = usable.decode("utf-8", errors="replace")
193
+ lines = [line for line in text.split("\n") if line.strip()]
194
+ return lines, new_offset, truncated
195
+
196
+
197
+ # --- transcript helpers: full, one-shot parses, never polled ---------------
198
+
199
+
200
+ class _EventsHolder:
201
+ """The minimal duck-typed shape detectors.base.meaningful_event_before
202
+ needs (attempt.events) -- built directly from a one-shot
203
+ reader._read_transcript parse, never a full reader.Attempt."""
204
+
205
+ __slots__ = ("events",)
206
+
207
+ def __init__(self, events: list) -> None:
208
+ self.events = events
209
+
210
+
211
+ def _transcript_path(run_dir: Path, agent_id: str) -> Path:
212
+ return run_dir / f"agent-{agent_id}.jsonl"
213
+
214
+
215
+ def _last_user_index(events: list) -> int | None:
216
+ last = None
217
+ for index, event in enumerate(events):
218
+ if event.kind == "user":
219
+ last = index
220
+ return last
221
+
222
+
223
+ def _parse_kill(path: Path) -> tuple[bool, float | None, int, int, bool, int]:
224
+ """(is_kill, silence_seconds, paths_written, commits, paths_partial, commits_without_id) for
225
+ one attempt's transcript, from a single full parse (`paths_partial`, N8 fix 1 S-m5: the
226
+ attempt's ledger is partial, so `paths_written` is a lower bound; `commits_without_id`,
227
+ G4b: completed calls with a commit command that reported no id): is_kill is whether its
228
+ last user event is the kill sentinel; silence_seconds is computed the same way the
229
+ interrupt machinery does (reader._build_interrupt): the gap from the
230
+ last meaningful event before the sentinel (detectors.base.
231
+ meaningful_event_before) to the sentinel itself, None when a timestamp
232
+ is missing or the delta would be negative."""
233
+ diagnostics: list[str] = []
234
+ state = ledger.TranscriptState()
235
+ events, _version, _session_id, _total, _bad, tool_calls, _marker_hits = reader._read_transcript(
236
+ path, diagnostics, state=state,
237
+ )
238
+ kill_index = _last_user_index(events)
239
+ if kill_index is None:
240
+ return False, None, 0, 0, False, 0
241
+ kill_event = events[kill_index]
242
+ if not kill_event.text.startswith(KILL_SENTINEL_PREFIX):
243
+ return False, None, 0, 0, False, 0
244
+
245
+ preceding = meaningful_event_before(_EventsHolder(events), kill_index)
246
+ silence: float | None = None
247
+ if preceding is not None and preceding.ts is not None and kill_event.ts is not None:
248
+ delta = (kill_event.ts - preceding.ts).total_seconds()
249
+ if delta >= 0:
250
+ silence = delta
251
+
252
+ facts = reader._build_attempt_facts(tool_calls, "killed", kill_event.ts, path, False)
253
+ paths = len(facts.paths_written) if facts is not None else 0
254
+ commits = len(facts.commits) if facts is not None else 0
255
+ partial = ledger.build_ledger(tool_calls, path.name, state).completeness == "partial"
256
+ without_id = report._commit_commands_without_id([SimpleNamespace(facts=facts)])
257
+ return True, silence, paths, commits, partial, without_id
258
+
259
+
260
+ def _cause_phrase_for_failed(path: Path) -> tuple[str, str | None]:
261
+ """(phrase, reset_time) from the same last-transcript-line rule
262
+ detectors/attempt_failed.py uses (docs/DESIGN.md 4.3, 4.13.2): 'cause not
263
+ recorded' when the last entry carries none of the four runtime-produced
264
+ markers attempt_failed itself recognizes."""
265
+ diagnostics: list[str] = []
266
+ events, _version, _session_id, _total, _bad, _tool_calls, marker_hits = (
267
+ reader._read_transcript(path, diagnostics)
268
+ )
269
+ last_event = final_event(events)
270
+ if last_event is None:
271
+ return "cause not recorded", None
272
+ # _cause_from_last_event only reads attempt.marker_hits; a bare
273
+ # namespace stands in for the real Attempt this one-off transcript was
274
+ # never otherwise built into.
275
+ pseudo_attempt = SimpleNamespace(marker_hits=marker_hits)
276
+ if last_event.kind == "user" and _attempt_failed._anchored_on_event(
277
+ pseudo_attempt, "kill_sentinel", last_event
278
+ ):
279
+ return "interrupted before finishing", None
280
+ if last_event.kind == "assistant":
281
+ matched = _attempt_failed._cause_from_last_event(pseudo_attempt, last_event)
282
+ if matched is not None:
283
+ _cause, phrase, reset_time = matched
284
+ return phrase, reset_time
285
+ return "cause not recorded", None
286
+
287
+
288
+ # --- line formatting --------------------------------------------------
289
+
290
+
291
+ def _line(word: str, rest: str) -> str:
292
+ return f"{word:<{_LABEL_WIDTH}}{rest}"
293
+
294
+
295
+ def _rate_limited(key: str, now: float, last_print: dict[str, float]) -> bool:
296
+ last = last_print.get(key)
297
+ return last is not None and (now - last) < _RATE_LIMIT_SECONDS
298
+
299
+
300
+ @dataclass
301
+ class _PendingKill:
302
+ """A detected supersede transition not yet resolved to a printed KILL
303
+ line (release finding F10): retried on every poll until it either
304
+ prints, is conclusively decided not to be a kill, or hits
305
+ _MAX_PENDING_POLLS."""
306
+
307
+ previous: object
308
+ current: object
309
+ polls: int = 0
310
+
311
+
312
+ @dataclass
313
+ class _PendingFailed:
314
+ """A journal 'failed' occurrence not yet resolved to a printed FAILED
315
+ line (release finding F10), retried the same way as _PendingKill."""
316
+
317
+ occurrence: object
318
+ polls: int = 0
319
+
320
+
321
+ @dataclass
322
+ class _PendingOk:
323
+ """A journal 'result' occurrence not yet resolved to a printed OK line
324
+ (docs/DESIGN.md 4.12, N8b K5.2), retried the same way as _PendingFailed."""
325
+
326
+ occurrence: object
327
+ polls: int = 0
328
+
329
+
330
+ # The lifecycle terminals that make DONE eligible at once (docs/DESIGN.md 4.12): the
331
+ # terminals a parent notification, a TaskStop, or a synchronous launch result can give.
332
+ _IMMEDIATE_TERMINALS = frozenset({"completed", "failed", "stopped"})
333
+
334
+
335
+ def _parent_signature(path: Path) -> tuple | None:
336
+ """(inode, size, mtime_ns, ctime_ns) of the parent transcript, None when it cannot be
337
+ statted. The parent index is refreshed only when this changes (docs/DESIGN.md 4.12)."""
338
+ try:
339
+ stat = path.stat()
340
+ except OSError:
341
+ return None
342
+ return (stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)
343
+
344
+
345
+ # --- agent names (N3b P14, C-1, C-5, C-6) -----------------------------------
346
+
347
+ WATCH_DIAGNOSTIC = f"agent-postmortem watch: {disclosure.UNAVAILABLE_DIAGNOSTIC}"
348
+
349
+
350
+ class _AgentNames:
351
+ """The agent named on a watch line or in an event's `agent` field is the
352
+ event attempt's own stable alias `a:<alias>` (P14), the one `run` shows
353
+ for the same attempt. The agents known at start register in one call; an
354
+ agent first named by a later event registers then (one locked
355
+ registration per new agent, the book's slow path). If a registration
356
+ fails the book degrades: that event and every later one name agents
357
+ positionally (`agent-N`, numbered within this watch) while the lines
358
+ already printed stand (C-1). The diagnostic goes to stderr once, as the
359
+ one fixed line, in text and --json mode alike (C-6); stdout stays clean.
360
+
361
+ An attempt whose agentId is missing, null or not a canonical id (C-5) is
362
+ named by a positional `agent-N`, never by its label."""
363
+
364
+ def __init__(self, err) -> None:
365
+ self._err = err
366
+ self._diagnosed = False
367
+ self._book: disclosure.AliasBook | None = None
368
+
369
+ def _check(self) -> None:
370
+ if self._book is not None and not self._book.available and not self._diagnosed:
371
+ self._diagnosed = True
372
+ print(WATCH_DIAGNOSTIC, file=self._err)
373
+
374
+ def start(self, occurrences) -> None:
375
+ known: list[tuple[str, str]] = []
376
+ for occurrence in occurrences:
377
+ agent_id = occurrence.agent_id
378
+ if disclosure.is_canonical_id("agent", agent_id) and ("agent", agent_id) not in known:
379
+ known.append(("agent", agent_id))
380
+ self._book = disclosure.register(known) if known else disclosure.lazy_book()
381
+ self._check()
382
+
383
+ def started(self) -> bool:
384
+ return self._book is not None
385
+
386
+ def name(self, occurrence) -> str:
387
+ assert self._book is not None
388
+ agent_id = occurrence.agent_id
389
+ if disclosure.is_canonical_id("agent", agent_id):
390
+ alias = self._book.alias("agent", agent_id)
391
+ else:
392
+ # Not canonical: never an alias and never the label. The book
393
+ # numbers it positionally, in step with every other positional
394
+ # name of this watch.
395
+ alias = self._book.alias("agent", f"no-identity:{occurrence.key}:{occurrence.ordinal}")
396
+ self._check()
397
+ return disclosure.rendered_form("agent", alias)
398
+
399
+
400
+ # --- the poll loop ----------------------------------------------------
401
+
402
+
403
+ def watch_run(
404
+ run_id_or_path: str,
405
+ root: Path,
406
+ *,
407
+ interval: float = DEFAULT_INTERVAL,
408
+ max_hours: float = DEFAULT_MAX_HOURS,
409
+ quiet_after: float = DEFAULT_QUIET_AFTER,
410
+ done_grace: float = DEFAULT_DONE_GRACE,
411
+ json_mode: bool = False,
412
+ out=None,
413
+ err=None,
414
+ ) -> int:
415
+ out = sys.stdout if out is None else out
416
+ err = sys.stderr if err is None else err
417
+
418
+ start = _now()
419
+ run_dir = _wait_for_journal(run_id_or_path, root, start, interval, err)
420
+ if run_dir is None:
421
+ return 2
422
+ journal_path = run_dir / "journal.jsonl"
423
+ # N3b C2: a directory name is printed as a run id only when it passes the
424
+ # reader's validator; otherwise the replacement `run` itself uses.
425
+ run_id = reader.valid_workflow_run_id(run_dir.name)
426
+
427
+ names = _AgentNames(err)
428
+ offset = 0
429
+ full_journal: list[dict] = []
430
+ last_print: dict[str, float] = {}
431
+ handled_kill: set[tuple[str, int]] = set()
432
+ handled_fail: set[tuple[str, int]] = set()
433
+ pending_kill: dict[tuple[str, int], _PendingKill] = {}
434
+ pending_fail: dict[tuple[str, int], _PendingFailed] = {}
435
+ handled_ok: set[tuple[str, int]] = set()
436
+ pending_ok: dict[tuple[str, int], _PendingOk] = {}
437
+ quiet_threshold: dict[tuple[str, int], float] = {}
438
+ resolved_since: float | None = None
439
+ started_seen = 0
440
+ next_line_no = 0
441
+ malformed = False
442
+ start_printed = False
443
+ # docs/DESIGN.md 4.12 (N8b K5.4): the parent index and the stat signature it was last
444
+ # cleanly refreshed at, the newest parent generation launch line known after the previous
445
+ # poll, and the watermark a RESET leaves behind for the epoch it ended.
446
+ parent_path = reader._parent_transcript_path_for_workflow_dir(run_dir)
447
+ parent_index = reader.ParentIndex(None, [], [], []) if parent_path is None else None
448
+ parent_signature: tuple | None = None
449
+ known_newest_launch = 0
450
+ epoch_watermark = 0
451
+
452
+ while True:
453
+ try:
454
+ new_lines, offset, truncated = _read_new_journal_lines(journal_path, offset)
455
+ except JournalLineTooLong:
456
+ # docs/DESIGN.md 4.12 (release finding F9): an immediate, final
457
+ # verdict -- not gated on every key being resolved, unlike the
458
+ # malformed-complete-line case below. A first poll that ends here prints
459
+ # no START: the journal was never parsed, so its counts are unknown and
460
+ # the exit is final (docs/DESIGN.md 4.12).
461
+ if json_mode:
462
+ payload = {"event": "MALFORMED", "reason": "line longer than 1 MB"}
463
+ print(json.dumps(payload), file=out)
464
+ else:
465
+ print("MALFORMED journal line longer than 1 MB", file=out)
466
+ return 4
467
+
468
+ if truncated:
469
+ # docs/DESIGN.md 4.12 (release finding F8): the journal shrank
470
+ # (truncated or replaced by a fresh run reusing the same path)
471
+ # -- every piece of state derived from the old content is reset
472
+ # before the replacement content is processed at all, so no
473
+ # event from the old journal can combine with the new one into
474
+ # a false KILL or DONE.
475
+ full_journal = []
476
+ last_print = {}
477
+ handled_kill = set()
478
+ handled_fail = set()
479
+ pending_kill = {}
480
+ pending_fail = {}
481
+ handled_ok = set()
482
+ pending_ok = {}
483
+ quiet_threshold = {}
484
+ resolved_since = None
485
+ started_seen = 0
486
+ next_line_no = 0
487
+ malformed = False
488
+ # K5.4: a new journal epoch. Generations launched up to the newest line known
489
+ # before this reset belong to the ended epoch and cannot finish the replacement.
490
+ epoch_watermark = known_newest_launch
491
+ if json_mode:
492
+ print(json.dumps({"event": "RESET"}), file=out)
493
+ else:
494
+ print(_line("RESET", "journal was truncated or replaced; state cleared."), file=out)
495
+
496
+ malformed_notice: int | None = None
497
+ for raw_line in new_lines:
498
+ next_line_no += 1
499
+ try:
500
+ entry = json.loads(raw_line)
501
+ except (json.JSONDecodeError, ValueError, RecursionError):
502
+ entry = None
503
+ if not isinstance(entry, dict):
504
+ # docs/DESIGN.md 4.12 (release finding F9): a complete line
505
+ # that is not a JSON object is malformed evidence -- printed
506
+ # once (after START, K5.1), and DONE can never be printed
507
+ # again for this run.
508
+ if not malformed:
509
+ malformed = True
510
+ malformed_notice = next_line_no
511
+ continue
512
+ full_journal.append(entry)
513
+
514
+ occurrences, _diagnostics = reader._run_journal_state_machine(full_journal)
515
+ if not names.started():
516
+ names.start(occurrences)
517
+
518
+ # docs/DESIGN.md 4.12 (K5.1): START is the first byte of output, once, ahead of the
519
+ # MALFORMED line this very poll parsed and ahead of every other event.
520
+ if not start_printed:
521
+ _emit_start(run_id, occurrences, json_mode, out)
522
+ start_printed = True
523
+ if malformed_notice is not None:
524
+ if json_mode:
525
+ payload = {"event": "MALFORMED", "line": malformed_notice, "reason": "not JSON"}
526
+ print(json.dumps(payload), file=out)
527
+ else:
528
+ print(f"MALFORMED journal line {malformed_notice} is not JSON", file=out)
529
+
530
+ now = _now()
531
+
532
+ # KILL: a second (or later) 'started' occurrence for a key, whose
533
+ # immediately preceding occurrence's own transcript ends in the kill
534
+ # sentinel (docs/DESIGN.md 4.12). Newly detected transitions join
535
+ # pending_kill rather than being resolved inline (release finding
536
+ # F10): an event is marked handled only once it is actually
537
+ # emitted, decided not to be a kill, or forced past the poll cap.
538
+ previous_by_key: dict[str, object] = {}
539
+ for occurrence in occurrences:
540
+ previous = previous_by_key.get(occurrence.key)
541
+ if previous is not None and occurrence.ordinal == previous.ordinal + 1:
542
+ handle = (occurrence.key, previous.ordinal)
543
+ if handle not in handled_kill and handle not in pending_kill:
544
+ pending_kill[handle] = _PendingKill(previous=previous, current=occurrence)
545
+ previous_by_key[occurrence.key] = occurrence
546
+
547
+ for handle in list(pending_kill.keys()):
548
+ pending = pending_kill[handle]
549
+ status = _try_emit_kill(
550
+ run_dir, run_id, names, pending.previous, pending.current, now, last_print,
551
+ json_mode, out,
552
+ )
553
+ if status == "retry":
554
+ pending.polls += 1
555
+ if pending.polls < _MAX_PENDING_POLLS:
556
+ continue
557
+ _emit_kill_fallback(
558
+ run_id, names, pending.previous, pending.current, json_mode, out,
559
+ )
560
+ handled_kill.add(handle)
561
+ del pending_kill[handle]
562
+
563
+ # FAILED: a journal 'failed' event. Same handled-only-after-
564
+ # emission, bounded-retry treatment as KILL above (release finding
565
+ # F10).
566
+ for occurrence in occurrences:
567
+ if occurrence.closed_by != "failed":
568
+ continue
569
+ handle = (occurrence.key, occurrence.ordinal)
570
+ if handle not in handled_fail and handle not in pending_fail:
571
+ pending_fail[handle] = _PendingFailed(occurrence=occurrence)
572
+
573
+ for handle in list(pending_fail.keys()):
574
+ pending = pending_fail[handle]
575
+ status = _try_emit_failed(
576
+ run_dir, names, pending.occurrence, now, last_print, json_mode, out,
577
+ )
578
+ if status == "retry":
579
+ pending.polls += 1
580
+ if pending.polls < _MAX_PENDING_POLLS:
581
+ continue
582
+ _force_emit_failed(
583
+ run_dir, names, pending.occurrence, now, last_print, json_mode, out,
584
+ )
585
+ handled_fail.add(handle)
586
+ del pending_fail[handle]
587
+
588
+ # OK: a journal 'result' closed an attempt. Same handled-only-after-emission,
589
+ # bounded-retry treatment as FAILED (docs/DESIGN.md 4.12, N8b K5.2).
590
+ for occurrence in occurrences:
591
+ if occurrence.closed_by != "completed":
592
+ continue
593
+ handle = (occurrence.key, occurrence.ordinal)
594
+ if handle not in handled_ok and handle not in pending_ok:
595
+ pending_ok[handle] = _PendingOk(occurrence=occurrence)
596
+
597
+ for handle in list(pending_ok.keys()):
598
+ pending = pending_ok[handle]
599
+ status = _try_emit_ok(names, pending.occurrence, now, last_print, json_mode, out)
600
+ if status == "retry":
601
+ pending.polls += 1
602
+ if pending.polls < _MAX_PENDING_POLLS:
603
+ continue
604
+ _force_emit_ok(names, pending.occurrence, now, last_print, json_mode, out)
605
+ handled_ok.add(handle)
606
+ del pending_ok[handle]
607
+
608
+ # DONE (docs/DESIGN.md 4.12, N8b K5.4). The parent index is refreshed only when the
609
+ # parent transcript's stat signature changed since the last clean refresh, and the
610
+ # signature is not remembered while the index still owes a full recheck (a transient
611
+ # read failure is retried on the next poll though the file did not change). The
612
+ # lifecycle table is evaluated from this poll's own occurrences, never from a second
613
+ # journal read.
614
+ if parent_path is not None:
615
+ signature = _parent_signature(parent_path)
616
+ if (
617
+ parent_index is None or signature != parent_signature
618
+ or parent_index.pending_full_recheck
619
+ ):
620
+ parent_index = reader.ParentIndex.for_path(parent_path)
621
+ parent_signature = None if parent_index.pending_full_recheck else signature
622
+ journal_resolved = reader.journal_resolved_from_occurrences(occurrences)
623
+ lifecycle = reader.lifecycle_from_journal_state(
624
+ run_dir, run_id, journal_resolved, parent_index, after_launch_line=epoch_watermark,
625
+ )
626
+ known_newest_launch = max(
627
+ (generation.launch_line for generation in parent_index.generations_for(run_id)),
628
+ default=0,
629
+ )
630
+
631
+ # Eligible at once from terminal evidence the journal did not give, or after the grace
632
+ # period once every key is resolved (the gap between two sequential phases is not the
633
+ # end of the run). A malformed journal (release finding F9) makes DONE impossible:
634
+ # LIMIT is printed instead, at the point DONE would have been. DONE waits for every
635
+ # already-detected KILL, FAILED, and OK transition to be emitted or forced.
636
+ if len(occurrences) != started_seen:
637
+ started_seen = len(occurrences)
638
+ resolved_since = None
639
+ if journal_resolved:
640
+ if resolved_since is None:
641
+ resolved_since = now
642
+ else:
643
+ resolved_since = None
644
+ eligible = lifecycle.terminal in _IMMEDIATE_TERMINALS or (
645
+ lifecycle.terminal == "journal_resolved" and resolved_since is not None
646
+ and now - resolved_since >= done_grace
647
+ )
648
+ if eligible and not (pending_kill or pending_fail or pending_ok):
649
+ if malformed:
650
+ return _emit_malformed_limit(json_mode, out)
651
+ return _emit_done(run_dir, json_mode, out)
652
+
653
+ # QUIET: the currently open attempt of every key, if any.
654
+ last_by_key: dict[str, object] = {}
655
+ for occurrence in occurrences:
656
+ last_by_key[occurrence.key] = occurrence
657
+ for key, occurrence in last_by_key.items():
658
+ if occurrence.closed_by is not None:
659
+ continue
660
+ _maybe_emit_quiet(
661
+ run_dir, names, key, occurrence, now, last_print, quiet_threshold,
662
+ quiet_after, json_mode, out,
663
+ )
664
+
665
+ if now - start >= max_hours * 3600:
666
+ _emit_limit(max_hours, json_mode, out)
667
+ return 4
668
+
669
+ # docs/DESIGN.md 4.12 (release finding F11): capped to whatever is
670
+ # left of the --max-hours deadline, so a large --interval cannot
671
+ # overshoot it either.
672
+ remaining = max_hours * 3600 - (now - start)
673
+ _sleep(min(interval, remaining))
674
+
675
+
676
+ def _paths_words(paths: int, partial: bool) -> str:
677
+ return f"{paths}+ paths" if partial else f"{paths} {report._plural(paths, 'path')}"
678
+
679
+
680
+ def _commit_words(commits: int, without_id: int) -> str:
681
+ """docs/DESIGN.md 4.12 (G4b): `reported <c> commit(s)`, `reported <c>+ commits`, or
682
+ `commit count unknown`, by the COMMITS cell's rule (4.7)."""
683
+ words = report._commits_words(commits, without_id)
684
+ return words if without_id > 0 and not commits else f"reported {words}"
685
+
686
+
687
+ def _emit_kill_line(
688
+ run_id: str, names: _AgentNames, previous, current, silence: float | None, paths: int,
689
+ commits: int, json_mode: bool, out, *, paths_partial: bool = False,
690
+ commits_without_id: int = 0,
691
+ ) -> None:
692
+ agent = names.name(previous)
693
+ silence_int = None if silence is None else round(silence)
694
+ silence_str = "unknown" if silence_int is None else str(silence_int)
695
+
696
+ if json_mode:
697
+ payload = {
698
+ "event": "KILL",
699
+ "agent": agent,
700
+ "attempt": previous.ordinal,
701
+ "silence_seconds": silence_int,
702
+ "restarted_as": current.ordinal,
703
+ "paths_written": paths,
704
+ "commits": commits,
705
+ "run_id": run_id,
706
+ }
707
+ print(json.dumps(payload), file=out)
708
+ else:
709
+ print(
710
+ _line(
711
+ "KILL",
712
+ f"{agent} attempt {previous.ordinal} after {silence_str}s of silence; "
713
+ f"restarted as attempt {current.ordinal}.",
714
+ ),
715
+ file=out,
716
+ )
717
+ print(
718
+ _line(
719
+ "",
720
+ f"attempt {previous.ordinal} wrote {_paths_words(paths, paths_partial)}, "
721
+ f"{_commit_words(commits, commits_without_id)}. "
722
+ f"-> agent-postmortem rescue plan {run_id}",
723
+ ),
724
+ file=out,
725
+ )
726
+
727
+
728
+ def _try_emit_kill(
729
+ run_dir: Path, run_id: str, names: _AgentNames, previous, current, now: float,
730
+ last_print: dict[str, float], json_mode: bool, out,
731
+ ) -> str:
732
+ """Attempt to resolve one supersede transition to a printed KILL line
733
+ (release finding F10). Returns 'emitted' (printed, mark handled),
734
+ 'not_a_kill' (conclusively decided -- the earlier transcript exists and
735
+ does not end in the kill sentinel -- mark handled, never print, never
736
+ retry again), or 'retry' (suppressed by the rate limit, or the earlier
737
+ transcript does not exist yet -- try again next poll, up to
738
+ _MAX_PENDING_POLLS)."""
739
+ if not previous.agent_id:
740
+ return "not_a_kill"
741
+ transcript_path = _transcript_path(run_dir, previous.agent_id)
742
+ if not transcript_path.exists():
743
+ return "retry"
744
+ is_kill, silence, paths, commits, partial, without_id = _parse_kill(transcript_path)
745
+ if not is_kill:
746
+ return "not_a_kill"
747
+ if _rate_limited(previous.key, now, last_print):
748
+ return "retry"
749
+
750
+ _emit_kill_line(
751
+ run_id, names, previous, current, silence, paths, commits, json_mode, out,
752
+ paths_partial=partial, commits_without_id=without_id,
753
+ )
754
+ last_print[previous.key] = now
755
+ return "emitted"
756
+
757
+
758
+ def _emit_kill_fallback(
759
+ run_id: str, names: _AgentNames, previous, current, json_mode: bool, out,
760
+ ) -> None:
761
+ """docs/DESIGN.md 4.12 (release finding F10): previous.polls reached
762
+ _MAX_PENDING_POLLS with the transition still unresolved (the earlier
763
+ transcript never appeared, or the rate limit never cleared) -- printed
764
+ anyway, without the transcript-derived details, so a killed attempt can
765
+ never be silently dropped forever."""
766
+ agent = names.name(previous)
767
+ if json_mode:
768
+ payload = {
769
+ "event": "KILL",
770
+ "agent": agent,
771
+ "attempt": previous.ordinal,
772
+ "silence_seconds": None,
773
+ "restarted_as": current.ordinal,
774
+ "paths_written": None,
775
+ "commits": None,
776
+ "run_id": run_id,
777
+ "details_unavailable": True,
778
+ }
779
+ print(json.dumps(payload), file=out)
780
+ else:
781
+ print(
782
+ _line(
783
+ "KILL",
784
+ f"{agent} attempt {previous.ordinal} (details unavailable); "
785
+ f"restarted as attempt {current.ordinal}.",
786
+ ),
787
+ file=out,
788
+ )
789
+
790
+
791
+ def _compute_and_emit_failed(
792
+ run_dir: Path, names: _AgentNames, occurrence, json_mode: bool, out,
793
+ ) -> None:
794
+ phrase, reset_time = "cause not recorded", None
795
+ if occurrence.agent_id:
796
+ transcript_path = _transcript_path(run_dir, occurrence.agent_id)
797
+ if transcript_path.exists():
798
+ phrase, reset_time = _cause_phrase_for_failed(transcript_path)
799
+
800
+ agent = names.name(occurrence)
801
+ if json_mode:
802
+ payload = {
803
+ "event": "FAILED", "agent": agent, "attempt": occurrence.ordinal, "cause": phrase,
804
+ "reset_time": reset_time,
805
+ }
806
+ print(json.dumps(payload), file=out)
807
+ else:
808
+ print(_line("FAILED", f"{agent} attempt {occurrence.ordinal}: {phrase}."), file=out)
809
+
810
+
811
+ def _try_emit_failed(
812
+ run_dir: Path, names: _AgentNames, occurrence, now: float, last_print: dict[str, float],
813
+ json_mode: bool, out,
814
+ ) -> str:
815
+ """docs/DESIGN.md 4.12 (release finding F10): 'emitted' or 'retry'
816
+ (suppressed by the rate limit only -- a missing transcript already
817
+ degrades gracefully to 'cause not recorded' rather than blocking
818
+ emission, so it is never a reason to retry)."""
819
+ if _rate_limited(occurrence.key, now, last_print):
820
+ return "retry"
821
+ _compute_and_emit_failed(run_dir, names, occurrence, json_mode, out)
822
+ last_print[occurrence.key] = now
823
+ return "emitted"
824
+
825
+
826
+ def _force_emit_failed(
827
+ run_dir: Path, names: _AgentNames, occurrence, now: float, last_print: dict[str, float],
828
+ json_mode: bool, out,
829
+ ) -> None:
830
+ """release finding F10: occurrence.polls reached _MAX_PENDING_POLLS
831
+ while only ever suppressed by the rate limit -- printed anyway, bypassing
832
+ the limit, so a FAILED event can never be dropped forever either."""
833
+ _compute_and_emit_failed(run_dir, names, occurrence, json_mode, out)
834
+ last_print[occurrence.key] = now
835
+
836
+
837
+ def _emit_start(run_id: str, occurrences: list, json_mode: bool, out) -> None:
838
+ """docs/DESIGN.md 4.12 (N8b K5.1): once, first. `agents` is the number of distinct journal
839
+ keys started so far and `attempts` the number of started occurrences."""
840
+ agents = len({occurrence.key for occurrence in occurrences})
841
+ attempts = len(occurrences)
842
+ if json_mode:
843
+ payload = {"event": "START", "run": run_id, "agents": agents, "attempts": attempts}
844
+ print(json.dumps(payload), file=out)
845
+ else:
846
+ print(
847
+ _line("START", f"{run_id}: {agents} agents started so far, {attempts} attempts."),
848
+ file=out,
849
+ )
850
+
851
+
852
+ def _emit_ok_line(names: _AgentNames, occurrence, json_mode: bool, out) -> None:
853
+ agent = names.name(occurrence)
854
+ if json_mode:
855
+ payload = {"event": "OK", "agent": agent, "attempt": occurrence.ordinal}
856
+ print(json.dumps(payload), file=out)
857
+ else:
858
+ print(_line("OK", f"{agent} attempt {occurrence.ordinal} returned a result."), file=out)
859
+
860
+
861
+ def _try_emit_ok(
862
+ names: _AgentNames, occurrence, now: float, last_print: dict[str, float], json_mode: bool,
863
+ out,
864
+ ) -> str:
865
+ """docs/DESIGN.md 4.12 (N8b K5.2): 'emitted' or 'retry' (suppressed by the per-agent rate
866
+ limit, the only reason an OK waits)."""
867
+ if _rate_limited(occurrence.key, now, last_print):
868
+ return "retry"
869
+ _emit_ok_line(names, occurrence, json_mode, out)
870
+ last_print[occurrence.key] = now
871
+ return "emitted"
872
+
873
+
874
+ def _force_emit_ok(
875
+ names: _AgentNames, occurrence, now: float, last_print: dict[str, float], json_mode: bool,
876
+ out,
877
+ ) -> None:
878
+ """The bounded retry ran out: printed through the rate limit, so an OK is never dropped."""
879
+ _emit_ok_line(names, occurrence, json_mode, out)
880
+ last_print[occurrence.key] = now
881
+
882
+
883
+ def _emit_malformed_limit(json_mode: bool, out) -> int:
884
+ """docs/DESIGN.md 4.12 (release finding F9): printed at the exact point
885
+ DONE would otherwise have been printed -- every key is resolved -- but a
886
+ malformed journal line means the run's own final state can no longer be
887
+ trusted, so this replaces DONE and exits 4 instead of 0/3."""
888
+ if json_mode:
889
+ print(json.dumps({"event": "LIMIT", "reason": "malformed"}), file=out)
890
+ else:
891
+ print(_line("LIMIT", "journal has malformed lines; run state unknown."), file=out)
892
+ return 4
893
+
894
+
895
+ def _maybe_emit_quiet(
896
+ run_dir: Path, names: _AgentNames, key: str, occurrence, now: float,
897
+ last_print: dict[str, float], quiet_threshold: dict[tuple[str, int], float],
898
+ quiet_after: float, json_mode: bool, out,
899
+ ) -> None:
900
+ if not occurrence.agent_id:
901
+ return
902
+ transcript_path = _transcript_path(run_dir, occurrence.agent_id)
903
+ try:
904
+ mtime = transcript_path.stat().st_mtime
905
+ except OSError:
906
+ return
907
+ silence = now - mtime
908
+ if silence < 0:
909
+ return
910
+ threshold_key = (occurrence.key, occurrence.ordinal)
911
+ threshold = quiet_threshold.get(threshold_key, quiet_after)
912
+ if silence < threshold:
913
+ return
914
+ if _rate_limited(key, now, last_print):
915
+ return
916
+
917
+ agent = names.name(occurrence)
918
+ silence_int = round(silence)
919
+ if json_mode:
920
+ payload = {
921
+ "event": "QUIET", "agent": agent, "attempt": occurrence.ordinal,
922
+ "silence_seconds": silence_int,
923
+ }
924
+ print(json.dumps(payload), file=out)
925
+ else:
926
+ rest = f"{agent} attempt {occurrence.ordinal} has produced nothing for {silence_int}s."
927
+ print(_line("QUIET", rest), file=out)
928
+ last_print[key] = now
929
+ quiet_threshold[threshold_key] = threshold * 2
930
+
931
+
932
+ def _emit_done(run_dir: Path, json_mode: bool, out) -> int:
933
+ run = reader.load_run(run_dir)
934
+ findings = detectors.run_detectors(run)
935
+ verdict = report.verdict_of(run, findings)
936
+ agents = len(run.agents)
937
+ attempts = len(run.attempts)
938
+
939
+ if json_mode:
940
+ payload = {
941
+ "event": "DONE", "agents": agents, "attempts": attempts, "verdict": verdict,
942
+ "run_id": run.run_id, "next_command": f"agent-postmortem run {run.run_id}",
943
+ }
944
+ print(json.dumps(payload), file=out)
945
+ else:
946
+ print(
947
+ _line(
948
+ "DONE",
949
+ f"{agents} agents, {attempts} attempts, {verdict}. "
950
+ f"-> agent-postmortem run {run.run_id}",
951
+ ),
952
+ file=out,
953
+ )
954
+ # docs/DESIGN.md 4.12, 4.13.11 (N5c-2), ordered: a FAILED verdict is 3, ahead of a run
955
+ # whose lifecycle terminal is `unknown`, which cannot be called a clean DONE and is 4.
956
+ if verdict.startswith("FAILED"):
957
+ return 3
958
+ if run.lifecycle is not None and run.lifecycle.terminal == "unknown":
959
+ return 4
960
+ return 0
961
+
962
+
963
+ def _format_hours(max_hours: float) -> str:
964
+ if float(max_hours).is_integer():
965
+ return str(int(max_hours))
966
+ return str(max_hours)
967
+
968
+
969
+ def _emit_limit(max_hours: float, json_mode: bool, out) -> None:
970
+ if json_mode:
971
+ print(json.dumps({"event": "LIMIT", "hours": max_hours}), file=out)
972
+ else:
973
+ print(_line("LIMIT", f"{_format_hours(max_hours)}h reached; run not finished."), file=out)