agentdebugx 0.1.0__py3-none-any.whl

agentdebug/analyzers.py

@@ -0,0 +1,152 @@
+"""Trajectory analyzers."""
+
+from __future__ import annotations
+
+from typing import Iterable, List, Optional, Tuple
+
+from agentdebug.models import (
+    AgentEvent,
+    AgentTrajectory,
+    DiagnosticReport,
+    EventType,
+    FailureFinding,
+    FailureMode,
+)
+from agentdebug.taxonomy import SEED_FAILURE_MODES
+
+
+class HeuristicAnalyzer:
+    """A deterministic baseline analyzer.
+
+    This is intentionally simple: it gives projects an immediate local signal
+    while future LLM judges, constraint synthesis, and learned attribution
+    models share the same output schema.
+    """
+
+    def analyze(self, trajectory: AgentTrajectory) -> DiagnosticReport:
+        findings = [finding for event in trajectory.events for finding in self._event_findings(event)]
+        root = self._select_root_cause(findings)
+        suggestions = self._dedupe(
+            finding.suggestion for finding in findings if finding.suggestion is not None
+        )
+
+        report = DiagnosticReport(
+            trace_id=trajectory.trace_id,
+            task_id=trajectory.task_id,
+            findings=findings,
+            suggestions=suggestions,
+            metadata={'analyzer': self.__class__.__name__},
+        )
+        if root is not None:
+            report.root_cause_event_id = root.event_id
+            report.root_cause_agent = root.agent_name
+            report.root_cause_step_index = root.step_index
+            report.summary = (
+                f'Likely root cause: {root.failure_mode.name}'
+                f' in {root.agent_name or "unknown agent"}'
+                f' at step {root.step_index}.'
+            )
+        return report
+
+    def _event_findings(self, event: AgentEvent) -> List[FailureFinding]:
+        text = self._event_text(event)
+        matched = self._match_failure_mode(event, text)
+        if matched is None:
+            return []
+
+        failure_mode, confidence, evidence = matched
+        return [
+            FailureFinding(
+                failure_mode=failure_mode,
+                event_id=event.event_id,
+                agent_name=event.agent_name,
+                step_index=event.step_index,
+                confidence=confidence,
+                evidence=evidence,
+                suggestion=self._suggestion(failure_mode),
+                metadata={'event_type': self._event_type_value(event.event_type)},
+            )
+        ]
+
+    def _match_failure_mode(
+        self, event: AgentEvent, text: str
+    ) -> Optional[Tuple[FailureMode, float, List[str]]]:
+        lowered = text.lower()
+        if event.error:
+            if event.event_type in {EventType.TOOL_CALL, EventType.TOOL_RESULT}:
+                return self._result('system.tool_execution_error', 0.86, [event.error])
+            return self._result('system.environment_error', 0.72, [event.error])
+
+        if self._contains(lowered, ['context length', 'rate limit', 'token limit']):
+            return self._result('system.llm_limit', 0.82, ['limit signal in event payload'])
+        if self._contains(lowered, ['json', 'schema', 'malformed', 'parse error']):
+            return self._result('action.format_error', 0.78, ['format/schema signal in event payload'])
+        if self._contains(lowered, ['missing parameter', 'invalid parameter', 'argument']):
+            return self._result('action.parameter_error', 0.76, ['parameter signal in event payload'])
+        if self._contains(lowered, ['unknown tool', 'wrong tool', 'tool mismatch']):
+            return self._result('action.wrong_tool', 0.8, ['tool-selection signal in event payload'])
+        if self._contains(lowered, ['loop', 'repeated', 'no progress', 'max steps']):
+            return self._result('planning.inefficient_plan', 0.67, ['loop/progress signal in event payload'])
+        if self._contains(lowered, ['policy violation', 'constraint ignored', 'must not']):
+            return self._result('planning.constraint_ignorance', 0.74, ['constraint signal in event payload'])
+        if self._contains(lowered, ['premature', 'early termination', 'not complete']):
+            return self._result('verification.premature_stop', 0.71, ['termination signal in event payload'])
+        if self._contains(lowered, ['handoff', 'lost context', 'missing context']):
+            return self._result('multiagent.handoff_loss', 0.7, ['handoff/context signal in event payload'])
+        if self._contains(lowered, ['screenshot', 'ocr', 'visual', 'ui element']):
+            return self._result('multimodal.perception_error', 0.62, ['multimodal perception signal in event payload'])
+
+        status = str(event.metadata.get('status', '')).lower()
+        if status in {'failed', 'failure', 'error'}:
+            return self._result('system.environment_error', 0.6, ['metadata status marks event as failed'])
+        return None
+
+    def _result(
+        self, mode_id: str, confidence: float, evidence: List[str]
+    ) -> Tuple[FailureMode, float, List[str]]:
+        return SEED_FAILURE_MODES[mode_id], confidence, evidence
+
+    def _select_root_cause(self, findings: List[FailureFinding]) -> Optional[FailureFinding]:
+        if not findings:
+            return None
+        return sorted(
+            findings,
+            key=lambda finding: (
+                finding.step_index is None,
+                finding.step_index if finding.step_index is not None else 10**9,
+                -finding.confidence,
+            ),
+        )[0]
+
+    def _event_text(self, event: AgentEvent) -> str:
+        parts = [
+            self._event_type_value(event.event_type),
+            event.module or '',
+            str(event.input or ''),
+            str(event.output or ''),
+            event.error or '',
+            str(event.metadata),
+        ]
+        return '\n'.join(parts)
+
+    def _suggestion(self, failure_mode: FailureMode) -> Optional[str]:
+        if not failure_mode.suggestion_templates:
+            return None
+        return str(failure_mode.suggestion_templates[0])
+
+    def _contains(self, text: str, needles: Iterable[str]) -> bool:
+        return any(needle in text for needle in needles)
+
+    def _dedupe(self, values: Iterable[str]) -> List[str]:
+        seen = set()
+        result = []
+        for value in values:
+            if value in seen:
+                continue
+            seen.add(value)
+            result.append(value)
+        return result
+
+    def _event_type_value(self, event_type: EventType) -> str:
+        value = getattr(event_type, 'value', event_type)
+        return str(value)

agentdebug/attribution.py

@@ -0,0 +1,230 @@
+"""Failure attribution.
+
+v0.1 ships two backends, both behind the same :class:`Attributor` protocol:
+
+* :class:`HeuristicAttributor` — uses the earliest finding (with confidence
+  tie-break) from a ``DiagnosticReport`` to derive blame. Zero-cost; always
+  available; matches what :class:`agentdebug.analyzers.HeuristicAnalyzer` does
+  internally today.
+* :class:`AllAtOnceAttributor` — feeds the full trajectory + findings to an
+  LLM and asks for a single blame hypothesis. Uses the Who&When "All-at-Once"
+  method from arXiv:2505.00212.
+
+Both produce an :class:`AttributionResult` carrying a list of :class:`Blame`
+hypotheses with confidence, rationale, and source attribution. Honest UX: we
+always return ranked hypotheses, never single-point claims.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Protocol
+
+from agentdebug.llm import LLMClient, extract_json_block
+from agentdebug.models import AgentTrajectory, FailureFinding, new_id
+
+LOG = logging.getLogger('agentdebug.attribution')
+
+
+@dataclass
+class Blame:
+    span_id: Optional[str]
+    step_index: Optional[int]
+    agent_name: Optional[str]
+    confidence: float
+    rationale: str
+    evidence: List[str] = field(default_factory=list)
+    sources: List[str] = field(default_factory=list)
+
+
+@dataclass
+class AttributionResult:
+    method: str
+    hypotheses: List[Blame]
+    elapsed_ms: int = 0
+    raw: Dict[str, Any] = field(default_factory=dict)
+
+
+class Attributor(Protocol):
+    id: str
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> AttributionResult:
+        ...
+
+
+class HeuristicAttributor:
+    """Cheap, model-free fallback that picks the earliest highest-confidence finding."""
+
+    id = 'heuristic'
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> AttributionResult:
+        if not findings:
+            return AttributionResult(method=self.id, hypotheses=[])
+        ranked = sorted(
+            findings,
+            key=lambda f: (
+                f.step_index is None,
+                f.step_index if f.step_index is not None else 10**9,
+                -f.confidence,
+            ),
+        )
+        primary = ranked[0]
+        return AttributionResult(
+            method=self.id,
+            hypotheses=[
+                Blame(
+                    span_id=primary.event_id,
+                    step_index=primary.step_index,
+                    agent_name=primary.agent_name,
+                    confidence=primary.confidence,
+                    rationale=(
+                        f'Earliest finding with non-trivial confidence: '
+                        f'{primary.failure_mode.name}'
+                    ),
+                    evidence=list(primary.evidence),
+                    sources=[self.id],
+                )
+            ],
+        )
+
+
+_ATTR_SYSTEM_PROMPT = """You are AgentDebugX-Attributor. Given a failed agent
+trajectory and the list of step-level failure findings already produced by a
+judge, identify the single MOST RESPONSIBLE step (the decisive root cause).
+
+Respond ONLY with a JSON object matching this schema (no prose, no markdown):
+
+{
+  "span_id": "<event_id from the input or null>",
+  "step_index": <int or null>,
+  "agent_name": "<agent_name from the input or null>",
+  "confidence": <float between 0 and 1>,
+  "rationale": "<one or two sentences justifying the choice>",
+  "evidence": ["<short quoted evidence>", ...]
+}
+
+If the trajectory does not appear to have failed, return all fields as null and
+confidence 0.
+"""
+
+
+class AllAtOnceAttributor:
+    """LLM-based attributor mirroring Who&When's All-at-Once method."""
+
+    id = 'all_at_once'
+
+    def __init__(
+        self,
+        llm: LLMClient,
+        *,
+        fallback: Optional[Attributor] = None,
+        max_findings: int = 20,
+        max_tokens: int = 2048,
+    ) -> None:
+        self.llm = llm
+        self.fallback: Attributor = fallback or HeuristicAttributor()
+        self.max_findings = max_findings
+        self.max_tokens = max_tokens
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> AttributionResult:
+        prompt_user = self._render_prompt(trajectory, findings[: self.max_findings])
+        messages = [
+            {'role': 'system', 'content': _ATTR_SYSTEM_PROMPT},
+            {'role': 'user', 'content': prompt_user},
+        ]
+        try:
+            result = self.llm.complete(messages=messages, max_tokens=self.max_tokens)
+        except Exception as exc:  # pragma: no cover - defensive
+            LOG.warning('LLM attribution failed; falling back: %s', exc)
+            return self.fallback.attribute(trajectory, findings)
+        parsed = extract_json_block(result.text)
+        if not parsed:
+            LOG.info('LLM attributor returned no JSON; falling back')
+            return self.fallback.attribute(trajectory, findings)
+        blame = Blame(
+            span_id=self._coerce_str(parsed.get('span_id')),
+            step_index=self._coerce_int(parsed.get('step_index')),
+            agent_name=self._coerce_str(parsed.get('agent_name')),
+            confidence=self._coerce_float(parsed.get('confidence'), default=0.0),
+            rationale=str(parsed.get('rationale') or ''),
+            evidence=self._coerce_str_list(parsed.get('evidence')),
+            sources=[self.id],
+        )
+        return AttributionResult(
+            method=self.id,
+            hypotheses=[blame],
+            raw={'finding_count': len(findings)},
+        )
+
+    def _render_prompt(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> str:
+        findings_doc = '\n'.join(self._render_finding(f) for f in findings) or '(none)'
+        events_doc = '\n'.join(
+            f'event_id={e.event_id} step={e.step_index} agent={e.agent_name} '
+            f'type={getattr(e.event_type, "value", e.event_type)} '
+            f'output={str(e.output)[:200]} error={str(e.error)[:200]}'
+            for e in trajectory.events
+        )
+        return (
+            f'GOAL: {trajectory.goal!r}\n'
+            f'FRAMEWORK: {trajectory.framework!r}\n\n'
+            f'JUDGE FINDINGS:\n{findings_doc}\n\n'
+            f'EVENTS:\n{events_doc}\n'
+        )
+
+    def _render_finding(self, finding: FailureFinding) -> str:
+        return (
+            f'- mode={finding.failure_mode.mode_id} '
+            f'agent={finding.agent_name} step={finding.step_index} '
+            f'confidence={finding.confidence:.2f} '
+            f'evidence={"; ".join(finding.evidence)[:200]}'
+        )
+
+    @staticmethod
+    def _coerce_str(value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        return str(value)
+
+    @staticmethod
+    def _coerce_int(value: Any) -> Optional[int]:
+        if value is None:
+            return None
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _coerce_float(value: Any, *, default: float) -> float:
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_str_list(value: Any) -> List[str]:
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return [str(v) for v in value]
+        return [str(value)]
+
+
+__all__ = ['Attributor', 'Blame', 'AttributionResult', 'HeuristicAttributor', 'AllAtOnceAttributor']

agentdebug/cli.py

@@ -0,0 +1,272 @@
+"""Command line entry points.
+
+v0.1 commands:
+
+* ``agentdebug analyze`` — run an analyzer on a trajectory JSON file.
+* ``agentdebug list``    — list trace IDs in a store.
+* ``agentdebug show``    — pretty-print a trajectory by ID.
+* ``agentdebug judge``   — run :class:`LLMJudgeAnalyzer` against a stored trajectory.
+* ``agentdebug doctor``  — report which adapters/integrations are available.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from typing import Optional, Sequence
+
+from agentdebug.analyzers import HeuristicAnalyzer
+from agentdebug.attribution import AllAtOnceAttributor, HeuristicAttributor
+from agentdebug.models import DiagnosticReport, model_to_json, trajectory_from_json
+from agentdebug.recovery import FixProposal, ReflexionSuggestion
+from agentdebug.storage import JsonlTraceStore, SQLiteTraceStore, TraceStore
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser(prog='agentdebug')
+    sub = parser.add_subparsers(dest='command', required=True)
+
+    p_analyze = sub.add_parser('analyze', help='Analyze a trajectory JSON file')
+    p_analyze.add_argument('trajectory', help='Path to an AgentTrajectory JSON file')
+    p_analyze.add_argument('--out', help='Optional output path for the report')
+    p_analyze.add_argument(
+        '--suggest',
+        action='store_true',
+        help='Also emit Reflexion-style retry suggestions for each finding',
+    )
+
+    p_list = sub.add_parser('list', help='List trace IDs in a store')
+    _add_store_args(p_list)
+
+    p_show = sub.add_parser('show', help='Print a stored trajectory as JSON')
+    _add_store_args(p_show)
+    p_show.add_argument('trace_id', help='Trace ID to print')
+
+    p_judge = sub.add_parser(
+        'judge',
+        help='Run the LLM judge against a trajectory JSON file or stored trace',
+    )
+    p_judge.add_argument(
+        'target',
+        help='Path to a trajectory JSON file, or a trace_id when --store is set',
+    )
+    _add_store_args(p_judge, required=False)
+    p_judge.add_argument(
+        '--model',
+        default=os.environ.get('AGENTDEBUG_LLM_MODEL', 'gemini-3-flash'),
+    )
+    p_judge.add_argument('--base-url', dest='base_url')
+    p_judge.add_argument('--api-key', dest='api_key')
+    p_judge.add_argument('--out', help='Optional output path for the report')
+    p_judge.add_argument(
+        '--attribute',
+        action='store_true',
+        help='Also run AllAtOnceAttributor on the judge findings',
+    )
+
+    sub.add_parser('doctor', help='Report adapter and integration availability')
+
+    p_serve = sub.add_parser('serve', help='Run the local FastAPI dashboard')
+    _add_store_args(p_serve, required=True)
+    p_serve.add_argument('--host', default='127.0.0.1')
+    p_serve.add_argument('--port', type=int, default=7777)
+
+    args = parser.parse_args(argv)
+    if args.command == 'analyze':
+        return _cmd_analyze(args)
+    if args.command == 'list':
+        return _cmd_list(args)
+    if args.command == 'show':
+        return _cmd_show(args)
+    if args.command == 'judge':
+        return _cmd_judge(args)
+    if args.command == 'doctor':
+        return _cmd_doctor()
+    if args.command == 'serve':
+        return _cmd_serve(args)
+    return 1
+
+
+# ---------------- subcommands ----------------
+
+
+def _cmd_analyze(args: argparse.Namespace) -> int:
+    trajectory_path = Path(args.trajectory)
+    trajectory = trajectory_from_json(trajectory_path.read_text(encoding='utf-8'))
+    report = HeuristicAnalyzer().analyze(trajectory)
+    rendered = model_to_json(report, indent=2)
+    if args.suggest:
+        proposals = ReflexionSuggestion().suggest(trajectory, report)
+        rendered = _augment_with_suggestions(rendered, report, proposals)
+    _emit(rendered, args.out)
+    return 0
+
+
+def _cmd_list(args: argparse.Namespace) -> int:
+    store = _resolve_store(args)
+    if store is None:
+        print('No store configured. Use --store-sqlite or --store-jsonl.', file=sys.stderr)
+        return 2
+    for trace_id in store.list_traces():
+        print(trace_id)
+    return 0
+
+
+def _cmd_show(args: argparse.Namespace) -> int:
+    store = _resolve_store(args)
+    if store is None:
+        print('No store configured. Use --store-sqlite or --store-jsonl.', file=sys.stderr)
+        return 2
+    trajectory = store.load_trajectory(args.trace_id)
+    if trajectory is None:
+        print(f'Unknown trace_id: {args.trace_id}', file=sys.stderr)
+        return 3
+    print(model_to_json(trajectory, indent=2))
+    return 0
+
+
+def _cmd_judge(args: argparse.Namespace) -> int:
+    from agentdebug.judges import LLMJudgeAnalyzer
+    from agentdebug.llm import OpenAICompatClient
+
+    # Load trajectory: file path first, fall back to store lookup.
+    trajectory_path = Path(args.target)
+    if trajectory_path.exists():
+        trajectory = trajectory_from_json(
+            trajectory_path.read_text(encoding='utf-8')
+        )
+    else:
+        store = _resolve_store(args)
+        if store is None:
+            print(
+                f'Could not find {args.target!r} on disk and no store configured.',
+                file=sys.stderr,
+            )
+            return 2
+        loaded = store.load_trajectory(args.target)
+        if loaded is None:
+            print(f'Unknown trace_id: {args.target}', file=sys.stderr)
+            return 3
+        trajectory = loaded
+
+    base_url = args.base_url or os.environ.get('AGENTDEBUG_LLM_BASE_URL')
+    api_key = args.api_key or os.environ.get('AGENTDEBUG_LLM_API_KEY')
+    if not base_url or not api_key:
+        print(
+            'LLM judge requires --base-url and --api-key (or '
+            'AGENTDEBUG_LLM_BASE_URL / AGENTDEBUG_LLM_API_KEY).',
+            file=sys.stderr,
+        )
+        return 4
+
+    llm = OpenAICompatClient(base_url=base_url, api_key=api_key, model=args.model)
+    report = LLMJudgeAnalyzer(llm=llm).analyze(trajectory)
+    rendered = model_to_json(report, indent=2)
+    if args.attribute:
+        blame = AllAtOnceAttributor(llm=llm).attribute(trajectory, report.findings)
+        rendered = _augment_with_blame(rendered, blame)
+    _emit(rendered, args.out)
+    return 0
+
+
+def _cmd_serve(args: argparse.Namespace) -> int:
+    store = _resolve_store(args)
+    if store is None:
+        print(
+            'serve requires --store-sqlite or --store-jsonl.', file=sys.stderr
+        )
+        return 2
+    try:
+        from agentdebug.ui import serve
+
+        serve(store, host=args.host, port=args.port)
+    except ImportError as exc:
+        print(str(exc), file=sys.stderr)
+        return 5
+    return 0
+
+
+def _cmd_doctor() -> int:
+    statuses = []
+    try:
+        from agentdebug.adapters.langgraph import LangGraphAdapter
+
+        statuses.append(LangGraphAdapter().instrument(_dummy_debugger()))
+    except Exception as exc:  # pragma: no cover - defensive
+        statuses.append(_status('langgraph', False, str(exc)))
+    try:
+        from agentdebug.adapters.otel import OTelExportAdapter
+
+        statuses.append(OTelExportAdapter().instrument(_dummy_debugger()))
+    except Exception as exc:  # pragma: no cover - defensive
+        statuses.append(_status('otel', False, str(exc)))
+    from agentdebug.adapters.raw import RawLoopAdapter
+
+    statuses.append(RawLoopAdapter().instrument(_dummy_debugger()))
+    for s in statuses:
+        flag = '✓' if s.implemented else '✗'
+        print(f'  {flag}  {s.framework:<10} {s.notes}')
+    return 0
+
+
+# ---------------- helpers ----------------
+
+
+def _add_store_args(p: argparse.ArgumentParser, *, required: bool = False) -> None:
+    group = p.add_mutually_exclusive_group(required=required)
+    group.add_argument('--store-sqlite', help='Path to a SQLite trace store')
+    group.add_argument('--store-jsonl', help='Path to a JSONL trace store')
+
+
+def _resolve_store(args: argparse.Namespace) -> Optional[TraceStore]:
+    if getattr(args, 'store_sqlite', None):
+        return SQLiteTraceStore(args.store_sqlite)
+    if getattr(args, 'store_jsonl', None):
+        return JsonlTraceStore(args.store_jsonl)
+    return None
+
+
+def _emit(rendered: str, out_path: Optional[str]) -> None:
+    if out_path is None:
+        print(rendered)
+        return
+    out = Path(out_path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(rendered + '\n', encoding='utf-8')
+
+
+def _augment_with_suggestions(
+    rendered: str, report: DiagnosticReport, proposals: list[FixProposal]
+) -> str:
+    proposals_block = '\n'.join(
+        f'-- proposal {p.proposal_id} ({p.recoverer_id}) --\n{p.suggestion_text}'
+        for p in proposals
+    )
+    return rendered + (
+        '\n\n# === Reflexion suggestions ===\n' + proposals_block
+        if proposals_block
+        else ''
+    )
+
+
+def _augment_with_blame(rendered: str, blame_result: object) -> str:
+    return rendered + '\n\n# === Attribution ===\n' + repr(blame_result)
+
+
+def _status(framework: str, implemented: bool, notes: str) -> object:
+    from agentdebug.adapters.base import AdapterStatus
+
+    return AdapterStatus(framework=framework, implemented=implemented, notes=notes)
+
+
+def _dummy_debugger() -> object:
+    from agentdebug.recorder import AgentDebug
+    from agentdebug.storage import JsonlTraceStore
+
+    return AgentDebug(store=JsonlTraceStore('.agentdebug/_doctor.jsonl'))
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())