agent-postmortem 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_postmortem/__init__.py +9 -0
- agent_postmortem/argerrors.py +160 -0
- agent_postmortem/claims.py +43 -0
- agent_postmortem/cli.py +2278 -0
- agent_postmortem/commands.py +1543 -0
- agent_postmortem/config.py +435 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/journal.jsonl +7 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000.jsonl +4 -0
- agent_postmortem/detectors/__init__.py +98 -0
- agent_postmortem/detectors/attempt_failed.py +261 -0
- agent_postmortem/detectors/base.py +319 -0
- agent_postmortem/detectors/bash_backgrounded.py +72 -0
- agent_postmortem/detectors/bash_timeout.py +94 -0
- agent_postmortem/detectors/claim_contradiction.py +251 -0
- agent_postmortem/detectors/classifier_unavailable.py +73 -0
- agent_postmortem/detectors/codex_unavailable.py +102 -0
- agent_postmortem/detectors/codex_write_denied.py +100 -0
- agent_postmortem/detectors/git_cwd_mismatch.py +74 -0
- agent_postmortem/detectors/mislabeled_kill.py +164 -0
- agent_postmortem/detectors/missing_artifact.py +159 -0
- agent_postmortem/detectors/permission_denied.py +99 -0
- agent_postmortem/detectors/silent_retry.py +163 -0
- agent_postmortem/detectors/structured_output_failure.py +91 -0
- agent_postmortem/detectors/tool_json_error.py +77 -0
- agent_postmortem/detectors/wasted_time.py +125 -0
- agent_postmortem/detectors/workflow_failed.py +62 -0
- agent_postmortem/disclosure.py +931 -0
- agent_postmortem/hooks/model_allocation_guard.mjs +248 -0
- agent_postmortem/hooks_cli.py +1269 -0
- agent_postmortem/ledger.py +624 -0
- agent_postmortem/markers.py +505 -0
- agent_postmortem/models.py +815 -0
- agent_postmortem/observe.py +792 -0
- agent_postmortem/reader.py +4611 -0
- agent_postmortem/redact.py +1981 -0
- agent_postmortem/report.py +3010 -0
- agent_postmortem/rescue.py +2096 -0
- agent_postmortem/returns.py +499 -0
- agent_postmortem/rules.py +627 -0
- agent_postmortem/severity.py +275 -0
- agent_postmortem/state.py +1261 -0
- agent_postmortem/trend.py +391 -0
- agent_postmortem/watch.py +973 -0
- agent_postmortem-0.3.4.dist-info/METADATA +264 -0
- agent_postmortem-0.3.4.dist-info/RECORD +56 -0
- agent_postmortem-0.3.4.dist-info/WHEEL +4 -0
- agent_postmortem-0.3.4.dist-info/entry_points.txt +3 -0
- agent_postmortem-0.3.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Detects a Bash command that ran outside any git repository.
|
|
2
|
+
|
|
3
|
+
DESIGN.md 4.3: minor severity, fires on every anchored, not-in-payload
|
|
4
|
+
'git_not_repo' MarkerHit (docs/DESIGN.md 4.13.2: the registry's own shape
|
|
5
|
+
restricts this id to a tool_result paired to Bash). metrics: count.
|
|
6
|
+
rule_id: R-GIT-CWD. Skips a read_only_output event (docs/DESIGN.md 4.3's
|
|
7
|
+
self-reference rule): a read-only Bash call (cat, sed, git show/diff/log/
|
|
8
|
+
grep/blame/cat-file, and the rest) quoting the marker from a file on disk
|
|
9
|
+
is not a command that actually ran outside a git repository.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from .. import severity as severity_rules
|
|
15
|
+
from ..models import Event, Finding, Run
|
|
16
|
+
from .base import (
|
|
17
|
+
KINDS_BOTH,
|
|
18
|
+
Detector,
|
|
19
|
+
anchored_marker_hits,
|
|
20
|
+
event_for_marker_hit,
|
|
21
|
+
evidence_line,
|
|
22
|
+
make_finding_id,
|
|
23
|
+
single_agent_id,
|
|
24
|
+
tooling_attempts,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
ID = "git_cwd_mismatch"
|
|
28
|
+
SEVERITY = "minor"
|
|
29
|
+
RULE_ID = "R-GIT-CWD"
|
|
30
|
+
_REGISTRY_MARKER = "git_not_repo"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class GitCwdMismatch:
|
|
34
|
+
id = ID
|
|
35
|
+
severity_default = SEVERITY
|
|
36
|
+
kinds = KINDS_BOTH
|
|
37
|
+
|
|
38
|
+
def run(self, run: Run) -> list[Finding]:
|
|
39
|
+
hits: list[Event] = []
|
|
40
|
+
hit_agents: list[str] = []
|
|
41
|
+
for attempt in tooling_attempts(run):
|
|
42
|
+
for hit in anchored_marker_hits(attempt, _REGISTRY_MARKER):
|
|
43
|
+
event = event_for_marker_hit(attempt, hit)
|
|
44
|
+
if event is None or event.read_only_output:
|
|
45
|
+
continue
|
|
46
|
+
hits.append(event)
|
|
47
|
+
hit_agents.append(attempt.agent_id)
|
|
48
|
+
if not hits:
|
|
49
|
+
return []
|
|
50
|
+
agent_id = single_agent_id(hit_agents)
|
|
51
|
+
|
|
52
|
+
count = len(hits)
|
|
53
|
+
plural = "s" if count != 1 else ""
|
|
54
|
+
title = f"{count} Bash command{plural} ran outside a git repository"
|
|
55
|
+
evidence = [evidence_line(run, event) for event in hits]
|
|
56
|
+
|
|
57
|
+
return [
|
|
58
|
+
Finding(
|
|
59
|
+
detector=ID,
|
|
60
|
+
severity=severity_rules.resolve(run, ID),
|
|
61
|
+
title=title,
|
|
62
|
+
run_id=run.run_id,
|
|
63
|
+
agent_id=agent_id,
|
|
64
|
+
evidence=evidence,
|
|
65
|
+
metrics={"count": count},
|
|
66
|
+
rule_id=RULE_ID,
|
|
67
|
+
finding_id=make_finding_id(
|
|
68
|
+
run.run_id, ID, agent_id, evidence[0] if evidence else None
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
DETECTOR: Detector = GitCwdMismatch()
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""mislabeled_kill: an attempt's transcript ends with the runtime's kill
|
|
2
|
+
sentinel, which Claude Code logs as "[Request interrupted by user]" even
|
|
3
|
+
when no human pressed anything. reader.py sets Attempt.interrupt for every
|
|
4
|
+
such attempt and attributes its cause against the parent transcript (docs/
|
|
5
|
+
DESIGN.md 4.4); this detector turns each attributed interrupt into a
|
|
6
|
+
Finding whose severity, title, and rule_id follow that cause. The main
|
|
7
|
+
attempt of a session run is never evaluated here: run.attempts is child
|
|
8
|
+
attempts only and never includes it (docs/DESIGN.md 4.3).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from .. import severity as severity_rules
|
|
14
|
+
from ..models import Attempt, Event, Finding, Run
|
|
15
|
+
from .base import KINDS_BOTH, Detector, anchored_marker_hits, evidence_line, make_finding_id
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _kill_event(attempt: Attempt) -> Event | None:
|
|
19
|
+
"""The kill sentinel event: the last 'user'-kind event in the attempt.
|
|
20
|
+
attempt.interrupt being set already guarantees one exists; this just
|
|
21
|
+
re-locates it for evidence purposes (Interrupt itself only carries its
|
|
22
|
+
timestamp, not the Event)."""
|
|
23
|
+
last_user_index = None
|
|
24
|
+
for index, event in enumerate(attempt.events):
|
|
25
|
+
if event.kind == "user":
|
|
26
|
+
last_user_index = index
|
|
27
|
+
return attempt.events[last_user_index] if last_user_index is not None else None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _backgrounded_precursor_line(attempt: Attempt) -> int | None:
|
|
31
|
+
"""docs/DESIGN.md 4.4, 4.13.8: the transcript line of an anchored
|
|
32
|
+
bash_backgrounded tool_result (4.13.2) when it is the attempt's last tool
|
|
33
|
+
activity: no `tool_use` and no later `tool_result` event follows it in
|
|
34
|
+
the attempt (parallel calls can return out of order, so a later result of
|
|
35
|
+
another call also disqualifies it). None otherwise."""
|
|
36
|
+
last_tool = None
|
|
37
|
+
for index, event in enumerate(attempt.events):
|
|
38
|
+
if event.kind in ("tool_use", "tool_result"):
|
|
39
|
+
last_tool = index
|
|
40
|
+
if last_tool is None:
|
|
41
|
+
return None
|
|
42
|
+
last = attempt.events[last_tool]
|
|
43
|
+
if last.kind != "tool_result" or last.tool_use_id is None:
|
|
44
|
+
return None
|
|
45
|
+
for hit in anchored_marker_hits(attempt, "bash_backgrounded"):
|
|
46
|
+
if hit.event_line == last.line_no and hit.tool_use_id == last.tool_use_id:
|
|
47
|
+
return hit.event_line
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _severity_and_title(run: Run, attempt: Attempt) -> tuple[str, str]:
|
|
52
|
+
"""Severity and title for attempt.interrupt.cause. The severity is the row
|
|
53
|
+
of docs/DESIGN.md 4.3's table for (cause, silence class, position); the
|
|
54
|
+
title keeps the four forms 4.4 specifies. "Consistent with a runtime kill"
|
|
55
|
+
is added to an unattributed interrupt after a long silence (the one 300 s
|
|
56
|
+
rule)."""
|
|
57
|
+
interrupt = attempt.interrupt
|
|
58
|
+
label = attempt.label
|
|
59
|
+
cause = interrupt.cause
|
|
60
|
+
silence = interrupt.silence_seconds
|
|
61
|
+
silence_class = severity_rules.silence_class(silence)
|
|
62
|
+
severity = severity_rules.resolve(
|
|
63
|
+
run, "mislabeled_kill", cause=cause, silence=silence_class,
|
|
64
|
+
position=severity_rules.position(run, attempt),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if cause == "parent_stop":
|
|
68
|
+
return severity, f"{label} stopped by the parent session"
|
|
69
|
+
|
|
70
|
+
if cause == "human":
|
|
71
|
+
return severity, f"{label} interrupted together with the parent session"
|
|
72
|
+
|
|
73
|
+
# The precursor clause only states what was observed just before the
|
|
74
|
+
# kill; it never says the backgrounding caused it (4.13.8).
|
|
75
|
+
precursor_line = _backgrounded_precursor_line(attempt)
|
|
76
|
+
precursor = (
|
|
77
|
+
f", after a Bash command was moved to the background (line {precursor_line})"
|
|
78
|
+
if precursor_line is not None else ""
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if cause == "runtime":
|
|
82
|
+
silence_text = (
|
|
83
|
+
"unknown silence" if silence is None else f"{int(round(silence))}s of silence"
|
|
84
|
+
)
|
|
85
|
+
title = (
|
|
86
|
+
f"{label} attempt {attempt.ordinal} was killed by the runtime after "
|
|
87
|
+
f"{silence_text}{precursor}, logged as a user interrupt"
|
|
88
|
+
)
|
|
89
|
+
return severity, title
|
|
90
|
+
|
|
91
|
+
# unattributed
|
|
92
|
+
if silence is None:
|
|
93
|
+
return severity, f"{label} interrupted with silence unknown, cause unattributed"
|
|
94
|
+
base = (
|
|
95
|
+
f"{label} interrupted after {int(round(silence))}s of silence{precursor}, "
|
|
96
|
+
"cause unattributed"
|
|
97
|
+
)
|
|
98
|
+
if silence_class == "long":
|
|
99
|
+
return severity, f"{base}, consistent with a runtime kill"
|
|
100
|
+
return severity, base
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _rule_id(cause: str, severity: str) -> str | None:
|
|
104
|
+
"""R-MONITOR-RETRIES only when the cause is runtime, or when it is
|
|
105
|
+
unattributed and major (docs/DESIGN.md 4.3): the two cases where the
|
|
106
|
+
evidence itself points at the runtime, not at the parent session or a
|
|
107
|
+
human."""
|
|
108
|
+
if cause == "runtime":
|
|
109
|
+
return "R-MONITOR-RETRIES"
|
|
110
|
+
if cause == "unattributed" and severity == "major":
|
|
111
|
+
return "R-MONITOR-RETRIES"
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class MislabeledKill:
|
|
116
|
+
id = "mislabeled_kill"
|
|
117
|
+
severity_default = "major" # the highest row; the real severity comes from the table
|
|
118
|
+
kinds = KINDS_BOTH
|
|
119
|
+
|
|
120
|
+
def run(self, run: Run) -> list[Finding]:
|
|
121
|
+
findings: list[Finding] = []
|
|
122
|
+
|
|
123
|
+
for attempt in run.attempts:
|
|
124
|
+
interrupt = attempt.interrupt
|
|
125
|
+
if interrupt is None:
|
|
126
|
+
continue
|
|
127
|
+
kill_event = _kill_event(attempt)
|
|
128
|
+
if kill_event is None:
|
|
129
|
+
continue
|
|
130
|
+
|
|
131
|
+
severity, title = _severity_and_title(run, attempt)
|
|
132
|
+
rule_id = _rule_id(interrupt.cause, severity)
|
|
133
|
+
|
|
134
|
+
evidence = []
|
|
135
|
+
if interrupt.preceding is not None:
|
|
136
|
+
evidence.append(evidence_line(run, interrupt.preceding))
|
|
137
|
+
evidence.append(evidence_line(run, kill_event))
|
|
138
|
+
if interrupt.cause_evidence is not None:
|
|
139
|
+
evidence.append(evidence_line(run, interrupt.cause_evidence))
|
|
140
|
+
|
|
141
|
+
metrics = {
|
|
142
|
+
"silence_seconds": interrupt.silence_seconds,
|
|
143
|
+
"cause": interrupt.cause,
|
|
144
|
+
"events": len(attempt.events),
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
findings.append(Finding(
|
|
148
|
+
detector=self.id,
|
|
149
|
+
severity=severity,
|
|
150
|
+
title=title,
|
|
151
|
+
run_id=run.run_id,
|
|
152
|
+
agent_id=attempt.agent_id,
|
|
153
|
+
evidence=evidence,
|
|
154
|
+
metrics=metrics,
|
|
155
|
+
rule_id=rule_id,
|
|
156
|
+
finding_id=make_finding_id(
|
|
157
|
+
run.run_id, self.id, attempt.agent_id, evidence[0] if evidence else None
|
|
158
|
+
),
|
|
159
|
+
))
|
|
160
|
+
|
|
161
|
+
return findings
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
DETECTOR: Detector = MislabeledKill()
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""Detects an attempt told to write a file that never showed up.
|
|
2
|
+
|
|
3
|
+
DESIGN.md 4.3, 4.13.4: info severity (D2), fires when an attempt's first user
|
|
4
|
+
event names an absolute path it must write (a write/save/append/create/persist
|
|
5
|
+
verb followed within 80 chars by a path ending .md/.json/.txt/.jsonl) and the
|
|
6
|
+
path is not present at scan time. This is the only detector that touches
|
|
7
|
+
anything outside the run directory, with claim_contradiction: one
|
|
8
|
+
severity.absence() (an os.stat) on the named path and nothing else. The
|
|
9
|
+
check is `observed_later`: the file may have existed when the run ended and
|
|
10
|
+
been moved or removed since, and the finding's title says so. The `verified`
|
|
11
|
+
metric records whether the path is under the home directory or /tmp.
|
|
12
|
+
rule_id: R-ASSERT-ARTIFACT.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
import tempfile
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from .. import severity as severity_rules
|
|
22
|
+
from ..models import Attempt, Event, Finding, Run, TitleParts
|
|
23
|
+
from .base import KINDS_BOTH, Detector, evidence_line, make_finding_id
|
|
24
|
+
|
|
25
|
+
ID = "missing_artifact"
|
|
26
|
+
SEVERITY = "info" # the highest (only) row of docs/DESIGN.md 4.3
|
|
27
|
+
RULE_ID = "R-ASSERT-ARTIFACT"
|
|
28
|
+
_MAX_GAP = 80
|
|
29
|
+
|
|
30
|
+
# A write/save/append/create/persist verb, then up to _MAX_GAP arbitrary
|
|
31
|
+
# characters, then an absolute path ending in one of the artifact
|
|
32
|
+
# extensions. The path itself is not length-limited, only its start. The
|
|
33
|
+
# trailing (?!...) boundary keeps a longer filename that merely contains one
|
|
34
|
+
# of the extensions (report.md.backup) from being truncated into a shorter
|
|
35
|
+
# path that does not actually exist as its own file (report.md).
|
|
36
|
+
_ARTIFACT_PATH_RE = re.compile(
|
|
37
|
+
r"""\b(?:write|save|append|create|persist)\b.{0,80}?(/[^\s,;:"'()]+\.(?:md|json|txt|jsonl))(?![A-Za-z0-9_.-])""",
|
|
38
|
+
re.IGNORECASE | re.DOTALL,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# A negation immediately before the verb ("Do not write ..."). Checked in a
|
|
42
|
+
# short fixed window before the verb rather than a variable-length regex
|
|
43
|
+
# lookbehind, which Python's re module does not support.
|
|
44
|
+
_NEGATION_RE = re.compile(
|
|
45
|
+
r"\b(?:do\s+not|does\s+not|don't|never|must\s+not|should\s+not|won't|cannot|can't)\s*$",
|
|
46
|
+
re.IGNORECASE,
|
|
47
|
+
)
|
|
48
|
+
_NEGATION_WINDOW = 20
|
|
49
|
+
|
|
50
|
+
# /tmp is kept literally (it is the conventional POSIX temp root real
|
|
51
|
+
# prompts name, and what DESIGN.md's own fixtures use) and supplemented with
|
|
52
|
+
# tempfile.gettempdir(), since on some machines and on any non-POSIX system
|
|
53
|
+
# the OS temp root is elsewhere. gettempdir() is resolved lazily, inside
|
|
54
|
+
# _allowed_roots() below, not here at import time: some sandboxes have no
|
|
55
|
+
# discoverable temp directory at all, and gettempdir() raises in that case
|
|
56
|
+
# (docs/DESIGN.md 4.3). Since detectors/__init__.py imports every detector
|
|
57
|
+
# module eagerly, a module-level call here would crash importing
|
|
58
|
+
# agent_postmortem itself -- including `agent-postmortem demo`, which needs
|
|
59
|
+
# no filesystem access at all.
|
|
60
|
+
_STATIC_ALLOWED_ROOTS = (Path.home(), Path("/tmp"))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _allowed_roots() -> tuple[Path, ...]:
|
|
64
|
+
"""Path.home() and /tmp, plus tempfile.gettempdir() when it can be
|
|
65
|
+
resolved. A gettempdir() failure is tolerated as simply 'no extra temp
|
|
66
|
+
root' rather than propagated -- it is not evidence that a path is
|
|
67
|
+
untrusted, only that this one extra check could not run."""
|
|
68
|
+
try:
|
|
69
|
+
extra = Path(tempfile.gettempdir())
|
|
70
|
+
except OSError:
|
|
71
|
+
return _STATIC_ALLOWED_ROOTS
|
|
72
|
+
return _STATIC_ALLOWED_ROOTS + (extra,)
|
|
73
|
+
|
|
74
|
+
def _first_user_event(attempt: Attempt) -> Event | None:
|
|
75
|
+
for event in attempt.events:
|
|
76
|
+
if event.kind == "user":
|
|
77
|
+
return event
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _is_under_allowed_root(path: Path) -> bool:
|
|
82
|
+
try:
|
|
83
|
+
resolved = path.resolve()
|
|
84
|
+
except OSError:
|
|
85
|
+
resolved = path
|
|
86
|
+
for root in _allowed_roots():
|
|
87
|
+
try:
|
|
88
|
+
resolved.relative_to(root.resolve())
|
|
89
|
+
return True
|
|
90
|
+
except (OSError, ValueError):
|
|
91
|
+
continue
|
|
92
|
+
return False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
_ABSENT_TEMPLATE = (
|
|
96
|
+
"{path} is absent now; the tool cannot tell whether it existed when the run ended"
|
|
97
|
+
)
|
|
98
|
+
_UNCHECKED_TEMPLATE = (
|
|
99
|
+
"{path} could not be checked now; the tool cannot tell whether it existed "
|
|
100
|
+
"when the run ended"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _title_parts(path: str, state: str) -> TitleParts:
|
|
105
|
+
"""D2 (docs/DESIGN.md 4.13.4): a check made now says nothing about the end of the run. The
|
|
106
|
+
title is a fixed template and the path (F fix 7, W7-2): a renderer formats it once and never
|
|
107
|
+
scans the result."""
|
|
108
|
+
template = _ABSENT_TEMPLATE if state == "absent_now" else _UNCHECKED_TEMPLATE
|
|
109
|
+
return TitleParts(template, path)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _title(path: str, state: str) -> str:
|
|
113
|
+
"""The title with the real path (what `raw` prints)."""
|
|
114
|
+
return _title_parts(path, state).raw()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class MissingArtifact:
|
|
118
|
+
id = ID
|
|
119
|
+
severity_default = SEVERITY
|
|
120
|
+
kinds = KINDS_BOTH
|
|
121
|
+
|
|
122
|
+
def run(self, run: Run) -> list[Finding]:
|
|
123
|
+
findings: list[Finding] = []
|
|
124
|
+
for attempt in run.attempts:
|
|
125
|
+
event = _first_user_event(attempt)
|
|
126
|
+
if event is None:
|
|
127
|
+
continue
|
|
128
|
+
for match in _ARTIFACT_PATH_RE.finditer(event.text):
|
|
129
|
+
preceding = event.text[max(0, match.start() - _NEGATION_WINDOW):match.start()]
|
|
130
|
+
if _NEGATION_RE.search(preceding):
|
|
131
|
+
continue
|
|
132
|
+
path_str = match.group(1)
|
|
133
|
+
state = severity_rules.absence(path_str)
|
|
134
|
+
if state == "present_now":
|
|
135
|
+
continue
|
|
136
|
+
verified = _is_under_allowed_root(Path(path_str))
|
|
137
|
+
severity = severity_rules.resolve(
|
|
138
|
+
run, ID, obligation="prompt_path", absence=state,
|
|
139
|
+
)
|
|
140
|
+
evidence = [evidence_line(run, event)]
|
|
141
|
+
parts = _title_parts(path_str, state)
|
|
142
|
+
findings.append(
|
|
143
|
+
Finding(
|
|
144
|
+
detector=ID,
|
|
145
|
+
severity=severity,
|
|
146
|
+
title=parts.raw(),
|
|
147
|
+
title_parts=parts,
|
|
148
|
+
run_id=run.run_id,
|
|
149
|
+
agent_id=attempt.agent_id,
|
|
150
|
+
evidence=evidence,
|
|
151
|
+
metrics={"paths": [path_str], "verified": verified},
|
|
152
|
+
rule_id=RULE_ID,
|
|
153
|
+
finding_id=make_finding_id(run.run_id, ID, attempt.agent_id, evidence[0]),
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
return findings
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
DETECTOR: Detector = MissingArtifact()
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Detects tool calls the runtime refused (docs/DESIGN.md 4.3, 4.13.2, 4.13.4; G7, v0.3.1).
|
|
2
|
+
|
|
3
|
+
Fires on every anchored, not-in-payload `permission_denied` MarkerHit: an `is_error` tool result
|
|
4
|
+
whose text starts with one of the runtime's four denial sentences (markers.denial_kind). One
|
|
5
|
+
finding per attempt that holds at least one, the main thread of a session included (its finding
|
|
6
|
+
carries no agent id and backs the run story), minor at every level.
|
|
7
|
+
|
|
8
|
+
The title counts the denials by kind, zero counts omitted. Evidence is up to 5 `file:line`
|
|
9
|
+
pointers and nothing else: the permission-rule sentence carries the denied command, so no excerpt
|
|
10
|
+
of any denial reaches a finding. `read_only_output` is not consulted: the text is the runtime's
|
|
11
|
+
own reply to the call, not output a command printed.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from .. import markers
|
|
17
|
+
from .. import severity as severity_rules
|
|
18
|
+
from ..models import Attempt, Event, Finding, Run
|
|
19
|
+
from .base import (
|
|
20
|
+
KINDS_BOTH,
|
|
21
|
+
Detector,
|
|
22
|
+
anchored_marker_hits,
|
|
23
|
+
event_for_marker_hit,
|
|
24
|
+
make_finding_id,
|
|
25
|
+
relative_path,
|
|
26
|
+
tooling_attempts,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
ID = "permission_denied"
|
|
30
|
+
SEVERITY = "minor"
|
|
31
|
+
MAX_EVIDENCE = 5
|
|
32
|
+
|
|
33
|
+
# (kind, metrics key, title label), in the title's fixed order
|
|
34
|
+
_KINDS = (
|
|
35
|
+
(markers.DENIAL_AUTO_MODE, "auto_mode", "auto mode"),
|
|
36
|
+
(markers.DENIAL_USER, "user", "user"),
|
|
37
|
+
(markers.DENIAL_PERMISSION_RULE, "permission_rule", "permission rule"),
|
|
38
|
+
(markers.DENIAL_SAFETY_CHECK, "safety_check", "safety check"),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _attempt_denials(attempt: Attempt) -> list[tuple[Event, str]]:
|
|
43
|
+
"""(event, kind) for every anchored denial of this attempt, in transcript order."""
|
|
44
|
+
ordered = sorted(
|
|
45
|
+
anchored_marker_hits(attempt, ID), key=lambda hit: (hit.event_line, hit.tool_use_id or ""),
|
|
46
|
+
)
|
|
47
|
+
found: list[tuple[Event, str]] = []
|
|
48
|
+
for hit in ordered:
|
|
49
|
+
event = event_for_marker_hit(attempt, hit)
|
|
50
|
+
if event is None:
|
|
51
|
+
continue
|
|
52
|
+
kind = markers.denial_prefix_kind(event.text)
|
|
53
|
+
if kind is None: # cannot be classified: not claimed
|
|
54
|
+
continue
|
|
55
|
+
found.append((event, kind))
|
|
56
|
+
return found
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class PermissionDenied:
|
|
60
|
+
id = ID
|
|
61
|
+
severity_default = SEVERITY
|
|
62
|
+
kinds = KINDS_BOTH
|
|
63
|
+
|
|
64
|
+
def run(self, run: Run) -> list[Finding]:
|
|
65
|
+
findings: list[Finding] = []
|
|
66
|
+
for attempt in tooling_attempts(run):
|
|
67
|
+
denials = _attempt_denials(attempt)
|
|
68
|
+
if not denials:
|
|
69
|
+
continue
|
|
70
|
+
counts = {key: 0 for _kind, key, _label in _KINDS}
|
|
71
|
+
for _event, kind in denials:
|
|
72
|
+
for known, key, _label in _KINDS:
|
|
73
|
+
if kind == known:
|
|
74
|
+
counts[key] += 1
|
|
75
|
+
total = len(denials)
|
|
76
|
+
parts = [f"{label} {counts[key]}" for _k, key, label in _KINDS if counts[key]]
|
|
77
|
+
title = (
|
|
78
|
+
f"{total} tool {'call' if total == 1 else 'calls'} denied ({', '.join(parts)})"
|
|
79
|
+
)
|
|
80
|
+
evidence = [
|
|
81
|
+
f"{relative_path(run, event.path)}:{event.line_no}"
|
|
82
|
+
for event, _kind in denials[:MAX_EVIDENCE]
|
|
83
|
+
]
|
|
84
|
+
agent_id = None if attempt is run.main else attempt.agent_id
|
|
85
|
+
findings.append(Finding(
|
|
86
|
+
detector=ID,
|
|
87
|
+
severity=severity_rules.resolve(run, ID),
|
|
88
|
+
title=title,
|
|
89
|
+
run_id=run.run_id,
|
|
90
|
+
agent_id=agent_id,
|
|
91
|
+
evidence=evidence,
|
|
92
|
+
metrics={"count": total, **counts},
|
|
93
|
+
rule_id=None,
|
|
94
|
+
finding_id=make_finding_id(run.run_id, ID, agent_id, evidence[0]),
|
|
95
|
+
))
|
|
96
|
+
return findings
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
DETECTOR: Detector = PermissionDenied()
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""silent_retry: a logical agent (a journal key) was restarted more than once.
|
|
2
|
+
|
|
3
|
+
The Workflow runtime silently discards a killed attempt and starts a fresh
|
|
4
|
+
one under the same key; nothing in the journal calls this out as a failure.
|
|
5
|
+
Every attempt before the last one for a key was thrown away, so its
|
|
6
|
+
wall-clock time is wasted work. Fires once per key that shows more than one
|
|
7
|
+
attempt, as a single Finding carrying every rule the retry backs in
|
|
8
|
+
rule_ids: R-RETRY-SAFE always, plus R-CAP-CONCURRENCY too when two or more
|
|
9
|
+
logical agents in the run started their first attempt within 5 seconds of
|
|
10
|
+
each other (simultaneous launches are the observed trigger for paired
|
|
11
|
+
stalls and kills) -- never two separate Findings for the same retry.
|
|
12
|
+
|
|
13
|
+
Evidence never quotes a non-final attempt's own prompt (that text is
|
|
14
|
+
user-authored and, for a retried key, is often the exact same prompt
|
|
15
|
+
repeated verbatim on every attempt -- a privacy residual, not useful
|
|
16
|
+
evidence). For each non-final attempt, evidence points at the kill sentinel
|
|
17
|
+
line that ended it, when present (a runtime marker, docs/DESIGN.md section
|
|
18
|
+
3, never something the agent or user wrote); an attempt whose transcript is
|
|
19
|
+
missing or lacks a sentinel is simply omitted. Then, for the final attempt,
|
|
20
|
+
one plain pointer line names where it started, with no excerpt at all:
|
|
21
|
+
"<transcript>:<line>: attempt <ordinal> started". See docs/DESIGN.md
|
|
22
|
+
section 4.3.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from .. import severity as severity_rules
|
|
28
|
+
from ..markers import KILL_SENTINEL_PREFIX
|
|
29
|
+
from ..models import Attempt, Event, Finding, Run
|
|
30
|
+
from .base import (
|
|
31
|
+
KINDS_WORKFLOW_ONLY,
|
|
32
|
+
Detector,
|
|
33
|
+
attempt_seconds,
|
|
34
|
+
evidence_line,
|
|
35
|
+
format_duration,
|
|
36
|
+
make_finding_id,
|
|
37
|
+
relative_path,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
_BURST_WINDOW_SECONDS = 5.0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _first_event(attempt: Attempt) -> Event | None:
|
|
44
|
+
return attempt.events[0] if attempt.events else None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _kill_sentinel_event(attempt: Attempt) -> Event | None:
|
|
48
|
+
"""The attempt's last user event, if it starts with the kill sentinel
|
|
49
|
+
(docs/DESIGN.md section 3): the runtime's own marker, never the
|
|
50
|
+
attempt's own prompt text, so it is safe to quote as evidence. None when
|
|
51
|
+
the attempt has no events (a missing transcript) or its last user event
|
|
52
|
+
is not the sentinel."""
|
|
53
|
+
last_user: Event | None = None
|
|
54
|
+
for event in attempt.events:
|
|
55
|
+
if event.kind == "user":
|
|
56
|
+
last_user = event
|
|
57
|
+
if last_user is not None and last_user.text.startswith(KILL_SENTINEL_PREFIX):
|
|
58
|
+
return last_user
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _final_attempt_pointer(run: Run, attempt: Attempt) -> str | None:
|
|
63
|
+
"""A plain path:line pointer at the final attempt's own start, with no
|
|
64
|
+
excerpt of its prompt (docs/DESIGN.md 4.5): a completed attempt's first
|
|
65
|
+
line is user-authored prompt text -- often the very same prompt every
|
|
66
|
+
other attempt of this key also carries -- so quoting it here would be
|
|
67
|
+
the same privacy residual this detector exists to avoid. None when the
|
|
68
|
+
attempt has no events (a missing transcript)."""
|
|
69
|
+
first_event = _first_event(attempt)
|
|
70
|
+
if first_event is None:
|
|
71
|
+
return None
|
|
72
|
+
path = relative_path(run, first_event.path)
|
|
73
|
+
return f"{path}:{first_event.line_no}: attempt {attempt.ordinal} started"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _run_has_burst(run: Run) -> bool:
|
|
77
|
+
"""True when 2+ logical agents started their first attempt within 5s."""
|
|
78
|
+
starts = []
|
|
79
|
+
for agent in run.agents:
|
|
80
|
+
attempts = sorted(agent.attempts, key=lambda a: a.ordinal)
|
|
81
|
+
if attempts and attempts[0].first_ts is not None:
|
|
82
|
+
starts.append(attempts[0].first_ts)
|
|
83
|
+
starts.sort()
|
|
84
|
+
return any(
|
|
85
|
+
(later - earlier).total_seconds() <= _BURST_WINDOW_SECONDS
|
|
86
|
+
for earlier, later in zip(starts, starts[1:])
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class SilentRetry:
|
|
91
|
+
id = "silent_retry"
|
|
92
|
+
severity_default = "major"
|
|
93
|
+
kinds = KINDS_WORKFLOW_ONLY
|
|
94
|
+
|
|
95
|
+
def run(self, run: Run) -> list[Finding]:
|
|
96
|
+
findings: list[Finding] = []
|
|
97
|
+
burst = _run_has_burst(run)
|
|
98
|
+
work_risk: str | None = None # run-level (4.13.4), read once and only when needed
|
|
99
|
+
|
|
100
|
+
for agent in run.agents:
|
|
101
|
+
attempts = sorted(agent.attempts, key=lambda a: a.ordinal)
|
|
102
|
+
if len(attempts) <= 1:
|
|
103
|
+
continue
|
|
104
|
+
|
|
105
|
+
non_final = attempts[:-1]
|
|
106
|
+
final = attempts[-1]
|
|
107
|
+
wasted_seconds = sum(attempt_seconds(attempt) for attempt in non_final)
|
|
108
|
+
|
|
109
|
+
evidence: list[str] = []
|
|
110
|
+
for attempt in non_final:
|
|
111
|
+
sentinel = _kill_sentinel_event(attempt)
|
|
112
|
+
if sentinel is not None:
|
|
113
|
+
evidence.append(evidence_line(run, sentinel))
|
|
114
|
+
final_pointer = _final_attempt_pointer(run, final)
|
|
115
|
+
if final_pointer is not None:
|
|
116
|
+
evidence.append(final_pointer)
|
|
117
|
+
|
|
118
|
+
metrics = {
|
|
119
|
+
"attempts": len(attempts),
|
|
120
|
+
"keys": 1,
|
|
121
|
+
"wasted_seconds": wasted_seconds,
|
|
122
|
+
"burst": burst,
|
|
123
|
+
}
|
|
124
|
+
# docs/DESIGN.md 4.3: N attempts is N-1 restarts (2 attempts is
|
|
125
|
+
# one restart, not two) -- the honesty rule (section 2) means the
|
|
126
|
+
# title must count the thing it names, not the length of the
|
|
127
|
+
# list it was counted from.
|
|
128
|
+
restarts = len(attempts) - 1
|
|
129
|
+
restart_word = "time" if restarts == 1 else "times"
|
|
130
|
+
title = (
|
|
131
|
+
f"{agent.label} restarted {restarts} {restart_word}, "
|
|
132
|
+
f"{format_duration(wasted_seconds)} wasted"
|
|
133
|
+
)
|
|
134
|
+
agent_id = final.agent_id
|
|
135
|
+
if work_risk is None:
|
|
136
|
+
work_risk = severity_rules.work_risk(run)
|
|
137
|
+
severity = severity_rules.resolve(
|
|
138
|
+
run, self.id,
|
|
139
|
+
recovered=severity_rules.recovered(run, attempts[0], include_self=False),
|
|
140
|
+
work_risk=work_risk,
|
|
141
|
+
)
|
|
142
|
+
rule_ids = ("R-RETRY-SAFE", "R-CAP-CONCURRENCY") if burst else ("R-RETRY-SAFE",)
|
|
143
|
+
finding_id = make_finding_id(
|
|
144
|
+
run.run_id, self.id, agent_id, evidence[0] if evidence else None
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
findings.append(Finding(
|
|
148
|
+
detector=self.id,
|
|
149
|
+
severity=severity,
|
|
150
|
+
title=title,
|
|
151
|
+
run_id=run.run_id,
|
|
152
|
+
agent_id=agent_id,
|
|
153
|
+
evidence=evidence,
|
|
154
|
+
metrics=metrics,
|
|
155
|
+
rule_id=rule_ids[0],
|
|
156
|
+
rule_ids=rule_ids,
|
|
157
|
+
finding_id=finding_id,
|
|
158
|
+
))
|
|
159
|
+
|
|
160
|
+
return findings
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
DETECTOR: Detector = SilentRetry()
|