agent-postmortem 0.3.4__py3-none-any.whl

agent_postmortem/detectors/structured_output_failure.py

@@ -0,0 +1,91 @@
+"""Detects a StructuredOutput call whose payload failed schema validation.
+
+DESIGN.md 4.3, 4.13.4: severity comes from the `recovered` input (major unless
+the attempt or a later attempt completed with a result); fires on every
+anchored, not-in-payload 'structured_output_mismatch' MarkerHit (docs/DESIGN.md
+4.13.2: the
+registry's own shape restricts this id to a tool_result paired to
+StructuredOutput). metrics: count. A 'bytes' metric measuring the recovered
+payload is not included: reader.py truncates every event's text to a
+400-char excerpt before any detector sees it, so there is no way to recover
+the original payload size here, only the length of the (already bounded)
+marker text, which is not the same thing. rule_id: R-SCALAR-SCHEMA. Also
+skips a read_only_output event (docs/DESIGN.md 4.3's self-reference rule),
+though a StructuredOutput call is never itself read-only in practice.
+"""
+
+from __future__ import annotations
+
+from .. import severity as severity_rules
+from ..models import Event, Finding, Run
+from .base import (
+    KINDS_BOTH,
+    Detector,
+    anchored_marker_hits,
+    event_for_marker_hit,
+    evidence_line,
+    make_finding_id,
+    single_agent_id,
+    tooling_attempts,
+)
+
+ID = "structured_output_failure"
+SEVERITY = "major"
+RULE_ID = "R-SCALAR-SCHEMA"
+_REGISTRY_MARKER = "structured_output_mismatch"
+
+
+class StructuredOutputFailure:
+    id = ID
+    severity_default = SEVERITY
+    kinds = KINDS_BOTH
+
+    def run(self, run: Run) -> list[Finding]:
+        hits: list[Event] = []
+        hit_agents: list[str | None] = []
+        levels: list[str] = []
+        for attempt in tooling_attempts(run):
+            attempt_hits = 0
+            for hit in anchored_marker_hits(attempt, _REGISTRY_MARKER):
+                event = event_for_marker_hit(attempt, hit)
+                if event is None or event.read_only_output:
+                    continue
+                hits.append(event)
+                attempt_hits += 1
+                hit_agents.append(attempt.agent_id)
+            if attempt_hits:
+                # Per attempt: did a later attempt, or the attempt itself, complete
+                # with a result (4.13.4)? The finding takes the highest of them.
+                levels.append(severity_rules.resolve(
+                    run, ID,
+                    recovered=severity_rules.recovered(run, attempt, include_self=True),
+                ))
+        if not hits:
+            return []
+
+        # docs/DESIGN.md 4.3, 4.7: attributed to an agent only when every hit is in it.
+        agent_id = single_agent_id(hit_agents)
+        count = len(hits)
+        plural = "s" if count != 1 else ""
+        title = f"{count} StructuredOutput call{plural} failed schema validation"
+        evidence = [evidence_line(run, event) for event in hits]
+        metrics = {"count": count}
+
+        return [
+            Finding(
+                detector=ID,
+                severity=severity_rules.worst(levels),
+                title=title,
+                run_id=run.run_id,
+                agent_id=agent_id,
+                evidence=evidence,
+                metrics=metrics,
+                rule_id=RULE_ID,
+                finding_id=make_finding_id(
+                    run.run_id, ID, agent_id, evidence[0] if evidence else None
+                ),
+            )
+        ]
+
+
+DETECTOR: Detector = StructuredOutputFailure()

agent_postmortem/detectors/tool_json_error.py

@@ -0,0 +1,77 @@
+"""Detects a tool call whose input failed the runtime's own JSON validation.
+
+DESIGN.md 4.3: minor severity, fires on every anchored, not-in-payload
+'tool_input_invalid' MarkerHit (docs/DESIGN.md 4.13.2: the registry's own
+shape already excludes a StructuredOutput call, which gets its own
+'structured_output_mismatch' id instead). This is a runtime defect an agent
+cannot route around, so there is no rule_id; it goes into the bug report
+instead. metrics: count, tools (the distinct tool names that hit it). Skips
+a read_only_output event (docs/DESIGN.md 4.3's self-reference rule): a
+Read/Grep/cat/sed/etc. call can only ever be quoting a marker string that
+already exists on disk, never reporting a tool-input failure that actually
+happened to this agent.
+"""
+
+from __future__ import annotations
+
+from .. import severity as severity_rules
+from ..models import Event, Finding, Run
+from .base import (
+    KINDS_BOTH,
+    Detector,
+    anchored_marker_hits,
+    event_for_marker_hit,
+    evidence_line,
+    make_finding_id,
+    single_agent_id,
+    tooling_attempts,
+)
+
+ID = "tool_json_error"
+SEVERITY = "minor"
+_REGISTRY_MARKER = "tool_input_invalid"
+
+
+class ToolJsonError:
+    id = ID
+    severity_default = SEVERITY
+    kinds = KINDS_BOTH
+
+    def run(self, run: Run) -> list[Finding]:
+        hits: list[Event] = []
+        hit_agents: list[str] = []
+        for attempt in tooling_attempts(run):
+            for hit in anchored_marker_hits(attempt, _REGISTRY_MARKER):
+                event = event_for_marker_hit(attempt, hit)
+                if event is None or event.read_only_output:
+                    continue
+                hits.append(event)
+                hit_agents.append(attempt.agent_id)
+        if not hits:
+            return []
+        agent_id = single_agent_id(hit_agents)
+
+        count = len(hits)
+        tools = sorted({event.tool_name for event in hits if event.tool_name})
+        plural = "s" if count != 1 else ""
+        title = f"{count} tool call{plural} failed input validation"
+        evidence = [evidence_line(run, event) for event in hits]
+
+        return [
+            Finding(
+                detector=ID,
+                severity=severity_rules.resolve(run, ID),
+                title=title,
+                run_id=run.run_id,
+                agent_id=agent_id,
+                evidence=evidence,
+                metrics={"count": count, "tools": tools},
+                rule_id=None,
+                finding_id=make_finding_id(
+                    run.run_id, ID, agent_id, evidence[0] if evidence else None
+                ),
+            )
+        ]
+
+
+DETECTOR: Detector = ToolJsonError()

agent_postmortem/detectors/wasted_time.py

@@ -0,0 +1,125 @@
+"""wasted_time: one info-level summary Finding per run of wall-clock burned.
+
+Always fires, even on a clean run with nothing to report (wasted_seconds and
+wasted_tokens are then zero), so the headline number this tool exists to
+produce is always present. wasted_seconds and wasted_tokens sum the attempts that
+returned no result (docs/DESIGN.md 4.3, FA5): every non-final attempt of each
+logical agent (the ones a retry threw away) plus every final attempt whose
+outcome is killed, failed, or superseded; run_seconds and attempt_seconds
+cover the whole run.
+
+A session run has no retries to observe (docs/DESIGN.md 4.7), so its
+metrics are a different, smaller set: run_seconds (the same whole-run span),
+child_seconds (the sum of every subagent's own wall-clock span, not just
+non-final ones -- there is no "non-final" for a session), child_count
+(how many subagents ran), and wasted_seconds (the same rule: a killed or
+failed subagent's span). See docs/DESIGN.md section 4.3.
+"""
+
+from __future__ import annotations
+
+from .. import severity as severity_rules
+from ..models import Attempt, Finding, Run
+from .base import (
+    KINDS_BOTH,
+    Detector,
+    attempt_seconds,
+    format_duration,
+    make_finding_id,
+    run_wasted_attempts,
+)
+
+
+def _sum_usage(attempts: list[Attempt]) -> int:
+    total = 0
+    for attempt in attempts:
+        for value in attempt.usage.values():
+            if isinstance(value, (int, float)) and not isinstance(value, bool):
+                total += value
+    return int(total)
+
+
+def _non_final_attempts(run: Run) -> list[Attempt]:
+    non_final: list[Attempt] = []
+    for agent in run.agents:
+        attempts = sorted(agent.attempts, key=lambda a: a.ordinal)
+        non_final.extend(attempts[:-1])
+    return non_final
+
+
+def _run_seconds(run: Run) -> float:
+    if run.started_at is None or run.ended_at is None:
+        return 0.0
+    return max((run.ended_at - run.started_at).total_seconds(), 0.0)
+
+
+def _finding(run: Run, title: str, metrics: dict) -> Finding:
+    return Finding(
+        detector="wasted_time",
+        severity=severity_rules.resolve(run, "wasted_time"),
+        title=title,
+        run_id=run.run_id,
+        agent_id=None,
+        evidence=[],
+        metrics=metrics,
+        rule_id=None,
+        finding_id=make_finding_id(run.run_id, "wasted_time", None, None),
+    )
+
+
+def _workflow_finding(run: Run) -> Finding:
+    non_final = _non_final_attempts(run)
+    wasted = run_wasted_attempts(run)
+    run_seconds = _run_seconds(run)
+    total_attempt_seconds = sum(attempt_seconds(attempt) for attempt in run.attempts)
+    wasted_seconds = sum(attempt_seconds(attempt) for attempt in wasted)
+    wasted_tokens = _sum_usage(wasted)
+    final_extra = len(wasted) - len(non_final)
+
+    title = (
+        f"run spent {format_duration(run_seconds)} wall-clock; "
+        f"{format_duration(wasted_seconds)} wasted across {len(non_final)} retried attempts"
+    )
+    if final_extra:
+        noun = "attempt" if final_extra == 1 else "attempts"
+        title += f" and {final_extra} final {noun} that ended without a result"
+    metrics = {
+        "run_seconds": run_seconds,
+        "attempt_seconds": total_attempt_seconds,
+        "wasted_seconds": wasted_seconds,
+        "wasted_tokens": wasted_tokens,
+    }
+    return _finding(run, title, metrics)
+
+
+def _session_finding(run: Run) -> Finding:
+    run_seconds = _run_seconds(run)
+    child_seconds = sum(attempt_seconds(attempt) for attempt in run.attempts)
+    child_count = len(run.attempts)
+    wasted_seconds = sum(attempt_seconds(attempt) for attempt in run_wasted_attempts(run))
+
+    title = (
+        f"session ran {format_duration(run_seconds)}; {child_count} subagents "
+        f"used {format_duration(child_seconds)} of agent time"
+    )
+    metrics = {
+        "run_seconds": run_seconds,
+        "child_seconds": child_seconds,
+        "child_count": child_count,
+        "wasted_seconds": wasted_seconds,
+    }
+    return _finding(run, title, metrics)
+
+
+class WastedTime:
+    id = "wasted_time"
+    severity_default = "info"
+    kinds = KINDS_BOTH
+
+    def run(self, run: Run) -> list[Finding]:
+        if run.kind == "session":
+            return [_session_finding(run)]
+        return [_workflow_finding(run)]
+
+
+DETECTOR: Detector = WastedTime()

agent_postmortem/detectors/workflow_failed.py

@@ -0,0 +1,62 @@
+"""The harness itself reported this workflow run failed.
+
+docs/DESIGN.md 4.13.3/4.13.4 (v0.3 N2): fires once per run when Run.lifecycle
+.terminal == 'failed' -- an accepted terminal notification, TaskStop, or
+synchronous launch result whose value is 'failed' (4.13.3), independent of
+whatever the run's own journal says (a journal can report every attempt
+'completed' while the harness that launched the workflow still saw it fail,
+docs/DESIGN.md's field_failed_generation_completed_attempt case). major
+severity (4.13.4), workflow only. Evidence is the parent transcript's own
+accepted notification/TaskStop/launch-result line, `path:line`, never an
+excerpt of it (the line itself is a structured handshake, not prose worth
+quoting). No rule_id: section 9's rules library names none for this
+detector.
+"""
+
+from __future__ import annotations
+
+from .. import reader
+from .. import severity as severity_rules
+from ..models import Finding, Run
+from .base import KINDS_WORKFLOW_ONLY, Detector, make_finding_id, relative_path
+
+ID = "workflow_failed"
+SEVERITY = "major"
+
+
+class WorkflowFailed:
+    id = ID
+    severity_default = SEVERITY
+    kinds = KINDS_WORKFLOW_ONLY
+
+    def run(self, run: Run) -> list[Finding]:
+        lifecycle = run.lifecycle
+        if lifecycle is None or lifecycle.terminal != "failed":
+            return []
+
+        parent_path = reader.parent_transcript_path(run)
+        line = reader.lifecycle_terminal_line(run)
+        evidence = (
+            [f"{relative_path(run, parent_path)}:{line}"]
+            if parent_path is not None and line is not None
+            else []
+        )
+
+        return [
+            Finding(
+                detector=ID,
+                severity=severity_rules.resolve(run, ID, terminal=lifecycle.terminal),
+                title="the harness reported this workflow failed",
+                run_id=run.run_id,
+                agent_id=None,
+                evidence=evidence,
+                metrics={},
+                rule_id=None,
+                finding_id=make_finding_id(
+                    run.run_id, ID, None, evidence[0] if evidence else None
+                ),
+            )
+        ]
+
+
+DETECTOR: Detector = WorkflowFailed()