agent-postmortem 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_postmortem/__init__.py +9 -0
- agent_postmortem/argerrors.py +160 -0
- agent_postmortem/claims.py +43 -0
- agent_postmortem/cli.py +2278 -0
- agent_postmortem/commands.py +1543 -0
- agent_postmortem/config.py +435 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/journal.jsonl +7 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000.jsonl +4 -0
- agent_postmortem/detectors/__init__.py +98 -0
- agent_postmortem/detectors/attempt_failed.py +261 -0
- agent_postmortem/detectors/base.py +319 -0
- agent_postmortem/detectors/bash_backgrounded.py +72 -0
- agent_postmortem/detectors/bash_timeout.py +94 -0
- agent_postmortem/detectors/claim_contradiction.py +251 -0
- agent_postmortem/detectors/classifier_unavailable.py +73 -0
- agent_postmortem/detectors/codex_unavailable.py +102 -0
- agent_postmortem/detectors/codex_write_denied.py +100 -0
- agent_postmortem/detectors/git_cwd_mismatch.py +74 -0
- agent_postmortem/detectors/mislabeled_kill.py +164 -0
- agent_postmortem/detectors/missing_artifact.py +159 -0
- agent_postmortem/detectors/permission_denied.py +99 -0
- agent_postmortem/detectors/silent_retry.py +163 -0
- agent_postmortem/detectors/structured_output_failure.py +91 -0
- agent_postmortem/detectors/tool_json_error.py +77 -0
- agent_postmortem/detectors/wasted_time.py +125 -0
- agent_postmortem/detectors/workflow_failed.py +62 -0
- agent_postmortem/disclosure.py +931 -0
- agent_postmortem/hooks/model_allocation_guard.mjs +248 -0
- agent_postmortem/hooks_cli.py +1269 -0
- agent_postmortem/ledger.py +624 -0
- agent_postmortem/markers.py +505 -0
- agent_postmortem/models.py +815 -0
- agent_postmortem/observe.py +792 -0
- agent_postmortem/reader.py +4611 -0
- agent_postmortem/redact.py +1981 -0
- agent_postmortem/report.py +3010 -0
- agent_postmortem/rescue.py +2096 -0
- agent_postmortem/returns.py +499 -0
- agent_postmortem/rules.py +627 -0
- agent_postmortem/severity.py +275 -0
- agent_postmortem/state.py +1261 -0
- agent_postmortem/trend.py +391 -0
- agent_postmortem/watch.py +973 -0
- agent_postmortem-0.3.4.dist-info/METADATA +264 -0
- agent_postmortem-0.3.4.dist-info/RECORD +56 -0
- agent_postmortem-0.3.4.dist-info/WHEEL +4 -0
- agent_postmortem-0.3.4.dist-info/entry_points.txt +3 -0
- agent_postmortem-0.3.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""attempt_failed: names the cause of a workflow attempt the journal closed
|
|
2
|
+
with a `failed` event (.reviews/proposals/2026-09-v0.2-guided-rescue.md
|
|
3
|
+
section 4.2). The cause is read from exactly one place: the attempt's own
|
|
4
|
+
last transcript entry (not counting trailing attachment entries), never a
|
|
5
|
+
wider window and never a user-authored prompt -- a fixture whose prompt
|
|
6
|
+
merely mentions "API Error" must still come back unattributed
|
|
7
|
+
(tests/fixtures/runs/attempt_failed_causes's af-decoy key, docs/DESIGN.md 4.3's disposition N6).
|
|
8
|
+
docs/DESIGN.md 4.13.2 (v0.3 N1): the four runtime-cause markers and the kill sentinel are read as
|
|
9
|
+
anchored, not-in-payload MarkerHits on the attempt's own last event, never
|
|
10
|
+
Event.text.startswith directly -- the registry's own permitted shapes
|
|
11
|
+
already restrict these ids to the attempt's final transcript entry (the
|
|
12
|
+
four causes) or a user entry (the sentinel), so an anchored hit there is, by
|
|
13
|
+
construction, on the last event whenever the last event itself carries one.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
from .. import severity as severity_rules
|
|
21
|
+
from ..markers import USAGE_LIMIT_SESSION, USAGE_LIMIT_WEEKLY
|
|
22
|
+
from ..models import Attempt, Event, Finding, Run
|
|
23
|
+
from .base import (
|
|
24
|
+
KINDS_BOTH,
|
|
25
|
+
Detector,
|
|
26
|
+
anchored_marker_hits,
|
|
27
|
+
evidence_line,
|
|
28
|
+
final_event,
|
|
29
|
+
make_finding_id,
|
|
30
|
+
relative_path,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
ID = "attempt_failed"
|
|
34
|
+
SEVERITY = "major" # the highest row; the real severity comes from the `recovered` input
|
|
35
|
+
RULE_ID_USAGE_LIMIT = "R-USAGE-LIMIT"
|
|
36
|
+
|
|
37
|
+
_RESET_TIME_LIMIT = 40
|
|
38
|
+
# F3: a reset time is kept only from characters in this set (stripped of
|
|
39
|
+
# anything else) -- an untrusted remainder past the marker should never let
|
|
40
|
+
# an arbitrary character ride into the title or exported metrics.
|
|
41
|
+
# What a reset time may contain: digits, letters, ":", ".", ",", " ", and
|
|
42
|
+
# the "/", "_", "(", ")" a zone name such as "(America/Los_Angeles)" needs.
|
|
43
|
+
_RESET_TIME_DISALLOWED_RE = re.compile(r"[^0-9A-Za-z:., /_()-]")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _clean_reset_time(remainder: str) -> str:
|
|
47
|
+
return _RESET_TIME_DISALLOWED_RE.sub("", remainder).strip()[:_RESET_TIME_LIMIT]
|
|
48
|
+
|
|
49
|
+
# (canonical marker id, title phrase, retry_now) -- one row per
|
|
50
|
+
# runtime-produced marker a `failed` attempt's last transcript entry can
|
|
51
|
+
# carry (docs/DESIGN.md 4.3, 4.13.2). Order does not matter: the four ids
|
|
52
|
+
# are mutually exclusive by construction (each anchors on a distinct,
|
|
53
|
+
# non-overlapping prefix of the same final entry).
|
|
54
|
+
_CAUSE_TABLE: tuple[tuple[str, str, str], ...] = (
|
|
55
|
+
("usage_limit_session", "session usage limit reached", "no"),
|
|
56
|
+
("usage_limit_weekly", "weekly usage limit reached", "no"),
|
|
57
|
+
("machine_sleep", "the machine went to sleep", "yes"),
|
|
58
|
+
("api_error", "a server error occurred", "yes"),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# The two usage-limit markers alone carry a reset time after the fixed
|
|
62
|
+
# prefix; machine_sleep and api_error do not.
|
|
63
|
+
_RESET_TIME_FRAGMENTS = {
|
|
64
|
+
"usage_limit_session": USAGE_LIMIT_SESSION[0],
|
|
65
|
+
"usage_limit_weekly": USAGE_LIMIT_WEEKLY[0],
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _anchored_on_event(attempt: Attempt, marker: str, event: Event) -> bool:
|
|
70
|
+
"""Whether attempt carries an anchored, not-in-payload MarkerHit for
|
|
71
|
+
marker whose event_line is event's own line (docs/DESIGN.md 4.13.2)."""
|
|
72
|
+
return any(hit.event_line == event.line_no for hit in anchored_marker_hits(attempt, marker))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _cause_from_last_event(attempt: Attempt, event: Event) -> tuple[str, str, str | None] | None:
|
|
76
|
+
"""(cause, title phrase, reset_time) from the caller's own last
|
|
77
|
+
transcript entry, checked only there, never anywhere else (docs/
|
|
78
|
+
DESIGN.md 4.3): the registry's final_assistant_entry shape already
|
|
79
|
+
restricts usage_limit_session/weekly, machine_sleep, and api_error to
|
|
80
|
+
exactly this entry, so an anchored hit for one of them is never
|
|
81
|
+
misattributed to a marker merely quoted inside ordinary prose elsewhere.
|
|
82
|
+
Returns None when none of the four ids anchor there."""
|
|
83
|
+
for marker, phrase, _retry in _CAUSE_TABLE:
|
|
84
|
+
if not _anchored_on_event(attempt, marker, event):
|
|
85
|
+
continue
|
|
86
|
+
reset_time = None
|
|
87
|
+
fragment = _RESET_TIME_FRAGMENTS.get(marker)
|
|
88
|
+
# docs/DESIGN.md 4.13.2: the anchored check this loop just passed is
|
|
89
|
+
# computed on the flattened text after leading whitespace is
|
|
90
|
+
# stripped; the reset-time extraction must read the same stripped
|
|
91
|
+
# text, not the raw excerpt, so the two can never disagree about
|
|
92
|
+
# where the fragment starts.
|
|
93
|
+
stripped_text = event.text.lstrip(" \t\r\n")
|
|
94
|
+
if fragment is not None and stripped_text.startswith(fragment):
|
|
95
|
+
remainder = stripped_text[len(fragment):]
|
|
96
|
+
reset_time = _clean_reset_time(remainder) or None
|
|
97
|
+
if reset_time:
|
|
98
|
+
phrase = f"{phrase}, resets {reset_time}"
|
|
99
|
+
return marker, phrase, reset_time
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _retry_now_for(cause: str) -> str:
|
|
104
|
+
"""'no' for both usage limits, 'yes' for machine_sleep and api_error,
|
|
105
|
+
'unknown' for interrupted and unattributed (docs/DESIGN.md 4.3)."""
|
|
106
|
+
for table_cause, _phrase, retry_now in _CAUSE_TABLE:
|
|
107
|
+
if table_cause == cause:
|
|
108
|
+
return retry_now
|
|
109
|
+
return "unknown"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# F4: the three tails docs/DESIGN.md 4.3 distinguishes, keyed by
|
|
113
|
+
# (a later attempt exists, one of them completed).
|
|
114
|
+
_TAIL_NOT_RETRIED = "the key was not retried"
|
|
115
|
+
_TAIL_RETRIED_INCOMPLETE = "retried, no later attempt completed"
|
|
116
|
+
_TAIL_RETRIED_COMPLETED = "a later attempt completed"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _retry_outcome(run: Run, attempt: Attempt) -> tuple[str, str]:
|
|
120
|
+
"""(tail phrase, severity) for attempt's title (docs/DESIGN.md 4.3, 4.13.4).
|
|
121
|
+
The severity comes from the `recovered` input; the tail reads the same input,
|
|
122
|
+
so the two cannot disagree: no later attempt for the key at all ("the key was
|
|
123
|
+
not retried"); a later attempt exists but none was closed with a result
|
|
124
|
+
("retried, no later attempt completed" -- a `failed, started, failed` and a
|
|
125
|
+
`failed, started` (still open) sequence both land here); a later attempt was
|
|
126
|
+
closed with a result ("a later attempt completed"). "Later" means a higher
|
|
127
|
+
ordinal of the same key."""
|
|
128
|
+
recovered = severity_rules.recovered(run, attempt, include_self=False)
|
|
129
|
+
severity = severity_rules.resolve(run, ID, recovered=recovered)
|
|
130
|
+
has_later = attempt.key is not None and any(
|
|
131
|
+
sibling.key == attempt.key and sibling.ordinal > attempt.ordinal
|
|
132
|
+
for sibling in run.attempts
|
|
133
|
+
)
|
|
134
|
+
if not has_later:
|
|
135
|
+
return _TAIL_NOT_RETRIED, severity
|
|
136
|
+
if recovered == "yes":
|
|
137
|
+
return _TAIL_RETRIED_COMPLETED, severity
|
|
138
|
+
return _TAIL_RETRIED_INCOMPLETE, severity
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class AttemptFailed:
|
|
142
|
+
id = ID
|
|
143
|
+
severity_default = SEVERITY
|
|
144
|
+
kinds = KINDS_BOTH
|
|
145
|
+
|
|
146
|
+
def run(self, run: Run) -> list[Finding]:
|
|
147
|
+
findings: list[Finding] = []
|
|
148
|
+
|
|
149
|
+
if run.kind == "session":
|
|
150
|
+
return self._session_findings(run)
|
|
151
|
+
|
|
152
|
+
for attempt in run.attempts:
|
|
153
|
+
if attempt.outcome != "failed":
|
|
154
|
+
continue
|
|
155
|
+
|
|
156
|
+
# docs/DESIGN.md 4.13.2: trailing attachment entries do not count as the
|
|
157
|
+
# attempt's final entry (the reader's marker shape skips them the same way).
|
|
158
|
+
last_event = final_event(attempt.events)
|
|
159
|
+
cause = "unattributed"
|
|
160
|
+
phrase: str | None = None
|
|
161
|
+
reset_time: str | None = None
|
|
162
|
+
|
|
163
|
+
if last_event is not None:
|
|
164
|
+
if last_event.kind == "user" and _anchored_on_event(
|
|
165
|
+
attempt, "kill_sentinel", last_event
|
|
166
|
+
):
|
|
167
|
+
cause = "interrupted"
|
|
168
|
+
phrase = "interrupted before finishing"
|
|
169
|
+
elif last_event.kind == "assistant":
|
|
170
|
+
matched = _cause_from_last_event(attempt, last_event)
|
|
171
|
+
if matched is not None:
|
|
172
|
+
cause, phrase, reset_time = matched
|
|
173
|
+
|
|
174
|
+
retry_now = _retry_now_for(cause)
|
|
175
|
+
tail, severity = _retry_outcome(run, attempt)
|
|
176
|
+
|
|
177
|
+
if cause == "unattributed":
|
|
178
|
+
title = (
|
|
179
|
+
f"{attempt.label} attempt {attempt.ordinal} failed, "
|
|
180
|
+
f"cause not recorded in the transcript; {tail}"
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
title = f"{attempt.label} attempt {attempt.ordinal} failed: {phrase}; {tail}"
|
|
184
|
+
|
|
185
|
+
evidence = [evidence_line(run, last_event)] if last_event is not None else []
|
|
186
|
+
rule_id = (
|
|
187
|
+
RULE_ID_USAGE_LIMIT
|
|
188
|
+
if cause in ("usage_limit_session", "usage_limit_weekly")
|
|
189
|
+
else None
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
findings.append(
|
|
193
|
+
Finding(
|
|
194
|
+
detector=ID,
|
|
195
|
+
severity=severity,
|
|
196
|
+
title=title,
|
|
197
|
+
run_id=run.run_id,
|
|
198
|
+
agent_id=attempt.agent_id,
|
|
199
|
+
evidence=evidence,
|
|
200
|
+
metrics={"cause": cause, "retry_now": retry_now, "reset_time": reset_time},
|
|
201
|
+
rule_id=rule_id,
|
|
202
|
+
finding_id=make_finding_id(
|
|
203
|
+
run.run_id, ID, attempt.agent_id, evidence[0] if evidence else None
|
|
204
|
+
),
|
|
205
|
+
)
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
return findings
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _session_findings(self, run: Run) -> list[Finding]:
|
|
212
|
+
"""docs/DESIGN.md 4.3 (G1c, v0.3.1): one major finding per session subagent whose
|
|
213
|
+
`return_state` is `reported_failed`; `reported_killed` adds none. The cause names what
|
|
214
|
+
decided it (a synchronous launch result or a task notification), the retry advice is
|
|
215
|
+
unknown, and the severity is the table's row for `recovered=unknown`."""
|
|
216
|
+
findings: list[Finding] = []
|
|
217
|
+
for attempt in run.attempts:
|
|
218
|
+
if attempt.return_state != "reported_failed" or attempt.return_evidence is None:
|
|
219
|
+
continue
|
|
220
|
+
source, path, line = attempt.return_evidence
|
|
221
|
+
cause = (
|
|
222
|
+
"launch_result_failed" if source == "launch_result" else "task_notification_failed"
|
|
223
|
+
)
|
|
224
|
+
phrase = _SESSION_CAUSE_PHRASES[cause]
|
|
225
|
+
event = next(
|
|
226
|
+
(
|
|
227
|
+
candidate
|
|
228
|
+
for holder in (run.main, *run.attempts) if holder is not None
|
|
229
|
+
for candidate in holder.events
|
|
230
|
+
if candidate.path == path and candidate.line_no == line
|
|
231
|
+
),
|
|
232
|
+
None,
|
|
233
|
+
)
|
|
234
|
+
pointer = (
|
|
235
|
+
evidence_line(run, event) if event is not None
|
|
236
|
+
else f"{relative_path(run, path)}:{line}"
|
|
237
|
+
)
|
|
238
|
+
severity = severity_rules.resolve(run, ID, recovered="unknown")
|
|
239
|
+
title = f"{attempt.label or 'subagent'} failed: {phrase}"
|
|
240
|
+
findings.append(
|
|
241
|
+
Finding(
|
|
242
|
+
detector=ID,
|
|
243
|
+
severity=severity,
|
|
244
|
+
title=title,
|
|
245
|
+
run_id=run.run_id,
|
|
246
|
+
agent_id=attempt.agent_id,
|
|
247
|
+
evidence=[pointer],
|
|
248
|
+
metrics={"cause": cause, "retry_now": "unknown", "reset_time": None},
|
|
249
|
+
rule_id=None,
|
|
250
|
+
finding_id=make_finding_id(run.run_id, ID, attempt.agent_id, pointer),
|
|
251
|
+
)
|
|
252
|
+
)
|
|
253
|
+
return findings
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
_SESSION_CAUSE_PHRASES = {
|
|
257
|
+
"launch_result_failed": "its launch result reported failure",
|
|
258
|
+
"task_notification_failed": "its task notification reported failure",
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
DETECTOR: Detector = AttemptFailed()
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""The contract every detector module must satisfy, plus the small pieces of
|
|
2
|
+
logic every detector needs and none of them owns: formatting an evidence
|
|
3
|
+
line, deciding whether an event is a Bash call, and summing an attempt's
|
|
4
|
+
wall-clock span. These used to be copy-pasted into every detector module;
|
|
5
|
+
a fix to the evidence-format contract (docs/DESIGN.md 4.1) had as many call
|
|
6
|
+
sites to update as there were detectors, and nothing caught a site that was
|
|
7
|
+
missed. A detector module lives in this package and exports exactly one
|
|
8
|
+
module-level instance named DETECTOR. detectors/__init__.py finds it by
|
|
9
|
+
iterating the package; there is no registry list to edit by hand.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
from collections.abc import Sequence
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Protocol
|
|
19
|
+
|
|
20
|
+
from ..models import Attempt, Event, Finding, MarkerHit, Run
|
|
21
|
+
|
|
22
|
+
# docs/DESIGN.md 4.4: how close a parent-transcript TaskStop or kill sentinel
|
|
23
|
+
# has to land to a child's own kill timestamp to count as evidence of that
|
|
24
|
+
# cause, rather than coincidence. 30s comfortably covers both of the two
|
|
25
|
+
# sources of slop between two files written by two different processes: the
|
|
26
|
+
# runtime buffers transcript writes rather than fsyncing per line, and
|
|
27
|
+
# timestamps are only as fine-grained as the clock the two writers share.
|
|
28
|
+
# Kept as two named constants, not one, because 4.4 evaluates them against
|
|
29
|
+
# different things (a TaskStop's timestamp vs. a second file's own kill
|
|
30
|
+
# sentinel) and a future revision may need to tune them independently.
|
|
31
|
+
PARENT_STOP_WINDOW_SECONDS = 30
|
|
32
|
+
PARENT_SENTINEL_WINDOW_SECONDS = 30
|
|
33
|
+
|
|
34
|
+
# docs/DESIGN.md 4.3: which run kinds a detector applies to. silent_retry is
|
|
35
|
+
# workflow-only (a session's subagents are never retried in a way v0.1 can
|
|
36
|
+
# observe -- 4.3's own note); every other detector applies to both kinds.
|
|
37
|
+
KINDS_WORKFLOW_ONLY: frozenset[str] = frozenset({"workflow"})
|
|
38
|
+
KINDS_BOTH: frozenset[str] = frozenset({"workflow", "session"})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Detector(Protocol):
|
|
42
|
+
id: str
|
|
43
|
+
severity_default: str
|
|
44
|
+
kinds: frozenset[str] # run kinds this detector applies to (4.3)
|
|
45
|
+
|
|
46
|
+
def run(self, run: Run) -> list[Finding]:
|
|
47
|
+
"""Inspect one Run and return zero or more Findings."""
|
|
48
|
+
...
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def tooling_attempts(run: Run) -> list[Attempt]:
|
|
52
|
+
"""The attempts a tooling detector must scan (docs/DESIGN.md 4.3): a
|
|
53
|
+
session's own main thread first (it is where a session's -p invocation
|
|
54
|
+
and its own tool calls live, and it is excluded from run.attempts by
|
|
55
|
+
design -- see models.Run), then every child attempt. A workflow run has
|
|
56
|
+
no main, so this is just run.attempts for one.
|
|
57
|
+
|
|
58
|
+
The seven tooling detectors (bash_backgrounded, bash_timeout,
|
|
59
|
+
tool_json_error, structured_output_failure, codex_write_denied,
|
|
60
|
+
classifier_unavailable, git_cwd_mismatch) all use this instead of
|
|
61
|
+
iterating run.attempts directly, so a marker in a session's main thread
|
|
62
|
+
is never silently invisible to them. mislabeled_kill and missing_artifact
|
|
63
|
+
stay on run.attempts only: a session's main-thread interrupt is not
|
|
64
|
+
attributable (4.3's own documented limit) and a session has no
|
|
65
|
+
'first user event' of the kind missing_artifact looks for."""
|
|
66
|
+
attempts = [run.main] if run.main is not None else []
|
|
67
|
+
attempts.extend(run.attempts)
|
|
68
|
+
return attempts
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def relative_path(run: Run, path: Path) -> str:
|
|
72
|
+
"""path relative to run.path, or just its name if it is not underneath
|
|
73
|
+
(for example a transcript from a fixture loaded by a bare, non-standard
|
|
74
|
+
path) or is run.path itself -- a session's cause_evidence (docs/DESIGN.md
|
|
75
|
+
4.4) can point at the main transcript, and for a session run.path is
|
|
76
|
+
that same file, so a literal relative_to would collapse to '.', not a
|
|
77
|
+
path a reader could use at all."""
|
|
78
|
+
try:
|
|
79
|
+
relative = path.relative_to(run.path)
|
|
80
|
+
except ValueError:
|
|
81
|
+
return path.name
|
|
82
|
+
return str(relative) if str(relative) != "." else path.name
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def evidence_line(run: Run, event: Event, limit: int = 160) -> str:
|
|
86
|
+
"""'relative/path:line: excerpt' (<=160 chars), the evidence format
|
|
87
|
+
docs/DESIGN.md 4.1 requires of every Finding."""
|
|
88
|
+
excerpt = " ".join(event.text.split())[:limit]
|
|
89
|
+
return f"{relative_path(run, event.path)}:{event.line_no}: {excerpt}"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def make_finding_id(
|
|
93
|
+
run_id: str, detector: str, agent_id: str | None, first_evidence: str | None,
|
|
94
|
+
) -> str:
|
|
95
|
+
"""sha256(run_id, detector, agent_id, first evidence path:line)[:12]
|
|
96
|
+
(docs/DESIGN.md 4.1). first_evidence is a full 'path:line: excerpt'
|
|
97
|
+
evidence line, or None; only its 'path:line' prefix goes into the hash,
|
|
98
|
+
so a later change to excerpt wording or truncation does not change a
|
|
99
|
+
finding's identity. Every detector module calls this to fill
|
|
100
|
+
Finding.finding_id; nothing else computes one."""
|
|
101
|
+
path_line = (first_evidence or "").split(": ", 1)[0]
|
|
102
|
+
payload = "\x1f".join((run_id, detector, agent_id or "", path_line))
|
|
103
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:12]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def single_agent_id(agent_ids) -> str | None:
|
|
107
|
+
"""The agent id when every hit of a run-wide tooling finding belongs to
|
|
108
|
+
the one agent, else None (docs/DESIGN.md 4.3, 4.7): a finding that counts
|
|
109
|
+
hits across several agents is not any one agent's story."""
|
|
110
|
+
distinct = set(agent_ids)
|
|
111
|
+
return distinct.pop() if len(distinct) == 1 else None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def anchored_marker_hits(attempt: Attempt, marker: str) -> list[MarkerHit]:
|
|
115
|
+
"""Every MarkerHit on attempt with this canonical id that is anchored
|
|
116
|
+
and not in_payload (docs/DESIGN.md 4.13.2): the only hits a detector may
|
|
117
|
+
ever treat as evidence. A detector never reads Event.markers directly
|
|
118
|
+
(kept only for compatibility); it reads Attempt.marker_hits through this
|
|
119
|
+
helper instead."""
|
|
120
|
+
return [
|
|
121
|
+
hit for hit in attempt.marker_hits
|
|
122
|
+
if hit.marker == marker and hit.anchored and not hit.in_payload
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def event_for_marker_hit(attempt: Attempt, hit: MarkerHit) -> Event | None:
|
|
127
|
+
"""The Event a MarkerHit was built from (docs/DESIGN.md 4.13.2): matched
|
|
128
|
+
by (line_no, tool_use_id) for a tool_result shape -- a single user entry
|
|
129
|
+
can carry more than one tool_result block on the same physical line, so
|
|
130
|
+
line_no alone cannot disambiguate them -- or by line_no plus kind for a
|
|
131
|
+
whole-entry shape (kill_sentinel, the final-assistant-entry markers,
|
|
132
|
+
sandbox_write_denied's assistant form), where the first matching 'user'
|
|
133
|
+
or 'assistant' event is the one the hit's own entry-level flattening
|
|
134
|
+
rule reads from (its first text block)."""
|
|
135
|
+
if hit.tool_use_id is not None:
|
|
136
|
+
for event in attempt.events:
|
|
137
|
+
if event.line_no == hit.event_line and event.tool_use_id == hit.tool_use_id:
|
|
138
|
+
return event
|
|
139
|
+
return None
|
|
140
|
+
for event in attempt.events:
|
|
141
|
+
if event.line_no == hit.event_line and event.kind in ("user", "assistant"):
|
|
142
|
+
return event
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def is_bash_event(event: Event) -> bool:
|
|
147
|
+
"""A tool_result the reader resolved to the Bash tool. A generic
|
|
148
|
+
substring match on an attachment's text (a hook result, not necessarily
|
|
149
|
+
tied to Bash at all) is not reliable evidence of a Bash call."""
|
|
150
|
+
return event.kind == "tool_result" and event.tool_name == "Bash"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
_COMMAND_EXCERPT_LIMIT = 160
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def preceding_bash_command(attempt: Attempt, index: int) -> str:
|
|
157
|
+
"""The command text of the Bash tool_use immediately before
|
|
158
|
+
attempt.events[index], or "" if none is found. Shared by
|
|
159
|
+
bash_backgrounded (which quotes it, redacted, in a finding's metrics)
|
|
160
|
+
and codex_write_denied (docs/DESIGN.md 4.3's D-C: a marker only counts
|
|
161
|
+
on a Bash tool_result whose own command mentions "codex")."""
|
|
162
|
+
for event in reversed(attempt.events[:index]):
|
|
163
|
+
if event.kind == "tool_use" and event.tool_name == "Bash":
|
|
164
|
+
try:
|
|
165
|
+
data = json.loads(event.text)
|
|
166
|
+
except (json.JSONDecodeError, TypeError, ValueError, RecursionError):
|
|
167
|
+
return event.text[:_COMMAND_EXCERPT_LIMIT]
|
|
168
|
+
command = data.get("command") if isinstance(data, dict) else None
|
|
169
|
+
return command if isinstance(command, str) else event.text[:_COMMAND_EXCERPT_LIMIT]
|
|
170
|
+
return ""
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def attempt_seconds(attempt: Attempt) -> float:
|
|
174
|
+
"""An attempt's wall-clock span, or 0.0 if either endpoint is unknown."""
|
|
175
|
+
if attempt.first_ts is None or attempt.last_ts is None:
|
|
176
|
+
return 0.0
|
|
177
|
+
return max((attempt.last_ts - attempt.first_ts).total_seconds(), 0.0)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# docs/DESIGN.md 4.3 (FA5): the outcomes of an attempt that ended without a result.
|
|
181
|
+
NO_RESULT_OUTCOMES = frozenset({"killed", "failed", "superseded"})
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def wasted_attempts(attempts: list[Attempt]) -> list[Attempt]:
|
|
185
|
+
"""The attempts of ONE logical agent whose wall-clock is wasted (docs/DESIGN.md
|
|
186
|
+
4.3, FA5), each once: every attempt but the last (a retry, whatever its
|
|
187
|
+
outcome) plus the last one when its outcome is killed, failed, or superseded.
|
|
188
|
+
An open final attempt that may still be running is not counted (its outcome
|
|
189
|
+
is completed or unknown)."""
|
|
190
|
+
ordered = sorted(attempts, key=lambda attempt: attempt.ordinal)
|
|
191
|
+
if not ordered:
|
|
192
|
+
return []
|
|
193
|
+
final = ordered[-1]
|
|
194
|
+
wasted = ordered[:-1]
|
|
195
|
+
if final.outcome in NO_RESULT_OUTCOMES:
|
|
196
|
+
wasted.append(final)
|
|
197
|
+
return wasted
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def run_wasted_attempts(run: Run) -> list[Attempt]:
|
|
201
|
+
"""wasted_attempts over every logical agent of the run, in agent order."""
|
|
202
|
+
return [attempt for agent in run.agents for attempt in wasted_attempts(agent.attempts)]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def format_duration(seconds: float) -> str:
|
|
206
|
+
"""2h13m for an hour or more, 41m for a minute or more, else 58s."""
|
|
207
|
+
total = max(int(round(seconds)), 0)
|
|
208
|
+
hours, remainder = divmod(total, 3600)
|
|
209
|
+
minutes, secs = divmod(remainder, 60)
|
|
210
|
+
if hours:
|
|
211
|
+
return f"{hours}h{minutes:02d}m"
|
|
212
|
+
if minutes:
|
|
213
|
+
return f"{minutes}m"
|
|
214
|
+
return f"{secs}s"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def final_event(events: Sequence[Event]) -> Event | None:
|
|
218
|
+
"""The last event whose raw_type is not exactly "attachment" (docs/DESIGN.md
|
|
219
|
+
4.13.2). Shared by attempt_failed and watch so both read one cause source."""
|
|
220
|
+
return next((ev for ev in reversed(events) if ev.raw_type != "attachment"), None)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def meaningful_event_before(attempt: Attempt, index: int) -> Event | None:
|
|
224
|
+
"""The last event before attempt.events[index] worth calling "preceding"
|
|
225
|
+
(docs/DESIGN.md 4.5): a reader.py-and-detector-shared notion of evidence
|
|
226
|
+
quality, not just reader.py's own interrupt-building.
|
|
227
|
+
|
|
228
|
+
Walks backward skipping 'thinking' and 'attachment' events and any event
|
|
229
|
+
whose text is empty once harness noise has been stripped (reader.py
|
|
230
|
+
strips <system-reminder>, <total_tokens>, and [harness: ...] spans before
|
|
231
|
+
an Event's text is set, so this is a plain emptiness check here, not a
|
|
232
|
+
second strip). A 'compaction' event stops the walk outright and this
|
|
233
|
+
returns None: a context reset makes whatever came before it not
|
|
234
|
+
"preceding" in any sense a caller can use (silence across a compaction is
|
|
235
|
+
not silence, and evidence from before one is not evidence of what
|
|
236
|
+
happened after)."""
|
|
237
|
+
for i in range(index - 1, -1, -1):
|
|
238
|
+
event = attempt.events[i]
|
|
239
|
+
if event.kind == "compaction":
|
|
240
|
+
return None
|
|
241
|
+
if event.kind in ("thinking", "attachment"):
|
|
242
|
+
continue
|
|
243
|
+
if not event.text.strip():
|
|
244
|
+
continue
|
|
245
|
+
return event
|
|
246
|
+
return None
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _task_stop_matches(attempt: Attempt, task_stop, kill_ts) -> bool:
|
|
250
|
+
"""docs/DESIGN.md 4.4 clause 1: named directly, or landing within
|
|
251
|
+
PARENT_STOP_WINDOW_SECONDS before the kill (a TaskStop with no timestamp
|
|
252
|
+
of its own can still match by name, just never by window)."""
|
|
253
|
+
if attempt.agent_id in task_stop.ids:
|
|
254
|
+
return True
|
|
255
|
+
if task_stop.ts is None or kill_ts is None:
|
|
256
|
+
return False
|
|
257
|
+
delta = (kill_ts - task_stop.ts).total_seconds()
|
|
258
|
+
return 0 <= delta <= PARENT_STOP_WINDOW_SECONDS
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _task_stop_evidence(parent_index, task_stop) -> Event:
|
|
262
|
+
return Event(
|
|
263
|
+
path=parent_index.path, line_no=task_stop.line_no, ts=task_stop.ts,
|
|
264
|
+
kind="tool_use", raw_type="assistant", tool_name="TaskStop",
|
|
265
|
+
text=task_stop.text, exit_code=None, duration_ms=None, is_error=False,
|
|
266
|
+
usage={}, markers=frozenset(),
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _kill_sentinel_evidence(parent_index, sentinel) -> Event:
|
|
271
|
+
return Event(
|
|
272
|
+
path=parent_index.path, line_no=sentinel.line_no, ts=sentinel.ts,
|
|
273
|
+
kind="user", raw_type="user", tool_name=None,
|
|
274
|
+
text=sentinel.text, exit_code=None, duration_ms=None, is_error=False,
|
|
275
|
+
usage={}, markers=frozenset(),
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def attribute_interrupt(run: Run, attempt: Attempt, parent_index) -> tuple[str, Event | None]:
|
|
280
|
+
"""cause and cause_evidence for attempt.interrupt, evaluated against
|
|
281
|
+
parent_index (a reader.ParentIndex; duck-typed here rather than imported,
|
|
282
|
+
since reader.py imports this module) in the exact order docs/DESIGN.md
|
|
283
|
+
4.4 specifies, first match wins:
|
|
284
|
+
|
|
285
|
+
1. parent_stop: a parent TaskStop names this attempt (docs/DESIGN.md 4.2
|
|
286
|
+
flattens every string value of a TaskStop's input into `ids`, so an
|
|
287
|
+
"agent_id" or a "task_id" field both work the same way here), or any
|
|
288
|
+
TaskStop lands within PARENT_STOP_WINDOW_SECONDS before the kill.
|
|
289
|
+
2. human: a parent kill sentinel lands within
|
|
290
|
+
PARENT_SENTINEL_WINDOW_SECONDS of the kill, either side (the owner's
|
|
291
|
+
Escape and the child's own kill timestamp are two different files'
|
|
292
|
+
clocks, not a strict before/after).
|
|
293
|
+
3. runtime: workflow kind only, and this attempt's key has a later
|
|
294
|
+
attempt -- the restart itself is affirmative evidence the runtime
|
|
295
|
+
killed and replaced it, never conjecture.
|
|
296
|
+
4. unattributed: everything else.
|
|
297
|
+
|
|
298
|
+
Returns (cause, cause_evidence); cause_evidence is only ever set for
|
|
299
|
+
parent_stop and human (docs/DESIGN.md 4.1's Interrupt.cause_evidence)."""
|
|
300
|
+
interrupt = attempt.interrupt
|
|
301
|
+
kill_ts = interrupt.kill_ts if interrupt is not None else None
|
|
302
|
+
|
|
303
|
+
for task_stop in parent_index.task_stops:
|
|
304
|
+
if _task_stop_matches(attempt, task_stop, kill_ts):
|
|
305
|
+
return "parent_stop", _task_stop_evidence(parent_index, task_stop)
|
|
306
|
+
|
|
307
|
+
if kill_ts is not None:
|
|
308
|
+
for sentinel in parent_index.kill_sentinels:
|
|
309
|
+
if sentinel.ts is None:
|
|
310
|
+
continue
|
|
311
|
+
if abs((kill_ts - sentinel.ts).total_seconds()) <= PARENT_SENTINEL_WINDOW_SECONDS:
|
|
312
|
+
return "human", _kill_sentinel_evidence(parent_index, sentinel)
|
|
313
|
+
|
|
314
|
+
if run.kind == "workflow" and attempt.key is not None:
|
|
315
|
+
siblings = run.attempts
|
|
316
|
+
if any(s.ordinal > attempt.ordinal for s in siblings if s.key == attempt.key):
|
|
317
|
+
return "runtime", None
|
|
318
|
+
|
|
319
|
+
return "unattributed", None
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Detects a Bash command the runtime moved to the background after its timeout.
|
|
2
|
+
|
|
3
|
+
DESIGN.md 4.3: minor severity, fires on every anchored, not-in-payload
|
|
4
|
+
'bash_backgrounded' MarkerHit (docs/DESIGN.md 4.13.2: the registry's own
|
|
5
|
+
predicate already requires both BASH_BACKGROUNDED fragments, line1-anchored,
|
|
6
|
+
on a tool_result paired to Bash). metrics: count only (docs/DESIGN.md 5.3's
|
|
7
|
+
D3: the evidence lines already show the marker line the detector fired on;
|
|
8
|
+
a finding's metrics never carries a second copy of transcript-derived free
|
|
9
|
+
text alongside them). rule_id: R-NO-LONG-BASH. Never consults
|
|
10
|
+
Event.read_only_output (docs/DESIGN.md 4.3: a timed-out `cat` is still a
|
|
11
|
+
timed-out command).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from .. import severity as severity_rules
|
|
17
|
+
from ..models import Attempt, Event, Finding, Run
|
|
18
|
+
from .base import (
|
|
19
|
+
KINDS_BOTH,
|
|
20
|
+
Detector,
|
|
21
|
+
anchored_marker_hits,
|
|
22
|
+
event_for_marker_hit,
|
|
23
|
+
evidence_line,
|
|
24
|
+
make_finding_id,
|
|
25
|
+
single_agent_id,
|
|
26
|
+
tooling_attempts,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
ID = "bash_backgrounded"
|
|
30
|
+
SEVERITY = "minor"
|
|
31
|
+
RULE_ID = "R-NO-LONG-BASH"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class BashBackgrounded:
|
|
35
|
+
id = ID
|
|
36
|
+
severity_default = SEVERITY
|
|
37
|
+
kinds = KINDS_BOTH
|
|
38
|
+
|
|
39
|
+
def run(self, run: Run) -> list[Finding]:
|
|
40
|
+
hits: list[tuple[Attempt, Event]] = []
|
|
41
|
+
for attempt in tooling_attempts(run):
|
|
42
|
+
for hit in anchored_marker_hits(attempt, ID):
|
|
43
|
+
event = event_for_marker_hit(attempt, hit)
|
|
44
|
+
if event is not None:
|
|
45
|
+
hits.append((attempt, event))
|
|
46
|
+
if not hits:
|
|
47
|
+
return []
|
|
48
|
+
|
|
49
|
+
count = len(hits)
|
|
50
|
+
evidence = [evidence_line(run, event) for _, event in hits]
|
|
51
|
+
plural = "s" if count != 1 else ""
|
|
52
|
+
title = f"{count} Bash command{plural} timed out and moved to the background"
|
|
53
|
+
|
|
54
|
+
agent_id = single_agent_id(attempt.agent_id for attempt, _ in hits)
|
|
55
|
+
return [
|
|
56
|
+
Finding(
|
|
57
|
+
detector=ID,
|
|
58
|
+
severity=severity_rules.resolve(run, ID),
|
|
59
|
+
title=title,
|
|
60
|
+
run_id=run.run_id,
|
|
61
|
+
agent_id=agent_id,
|
|
62
|
+
evidence=evidence,
|
|
63
|
+
metrics={"count": count},
|
|
64
|
+
rule_id=RULE_ID,
|
|
65
|
+
finding_id=make_finding_id(
|
|
66
|
+
run.run_id, ID, agent_id, evidence[0] if evidence else None
|
|
67
|
+
),
|
|
68
|
+
)
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
DETECTOR: Detector = BashBackgrounded()
|