agentdebugx 0.1.0__py3-none-any.whl

agentdebug/models.py

@@ -0,0 +1,169 @@
+"""Core data models for portable agent debugging traces."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Optional, cast
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict, Field
+
+JsonDict = Dict[str, Any]
+
+
+def utc_now() -> datetime:
+    """Return a timezone-aware UTC timestamp."""
+    return datetime.now(timezone.utc)
+
+
+def new_id(prefix: str) -> str:
+    """Create a short stable identifier for trace objects."""
+    return f'{prefix}_{uuid4().hex}'
+
+
+class EventType(str, Enum):
+    RUN_START = 'run.start'
+    RUN_END = 'run.end'
+    AGENT_STEP = 'agent.step'
+    LLM_CALL = 'llm.call'
+    LLM_RESPONSE = 'llm.response'
+    TOOL_CALL = 'tool.call'
+    TOOL_RESULT = 'tool.result'
+    MEMORY_READ = 'memory.read'
+    MEMORY_WRITE = 'memory.write'
+    REFLECTION = 'reflection'
+    PLAN = 'plan'
+    HANDOFF = 'handoff'
+    GUARDRAIL = 'guardrail'
+    OBSERVATION = 'observation'
+    ERROR = 'error'
+    HUMAN_FEEDBACK = 'human.feedback'
+
+
+class Modality(str, Enum):
+    TEXT = 'text'
+    IMAGE = 'image'
+    AUDIO = 'audio'
+    VIDEO = 'video'
+    UI = 'ui'
+    FILE = 'file'
+    TOOL = 'tool'
+    STATE = 'state'
+    OTHER = 'other'
+
+
+class Artifact(BaseModel):
+    """A payload linked to an event, including multimodal evidence."""
+
+    model_config = ConfigDict(use_enum_values=True)
+
+    uri: str
+    modality: Modality = Modality.OTHER
+    media_type: Optional[str] = None
+    description: Optional[str] = None
+    metadata: JsonDict = Field(default_factory=dict)
+
+
+class AgentEvent(BaseModel):
+    """A normalized event emitted by an agentic system."""
+
+    model_config = ConfigDict(use_enum_values=True)
+
+    event_id: str = Field(default_factory=lambda: new_id('evt'))
+    trace_id: str
+    parent_event_id: Optional[str] = None
+    agent_name: str = 'agent'
+    event_type: EventType = EventType.AGENT_STEP
+    module: Optional[str] = None
+    step_index: Optional[int] = None
+    timestamp: datetime = Field(default_factory=utc_now)
+    input: Any = None
+    output: Any = None
+    error: Optional[str] = None
+    duration_ms: Optional[float] = None
+    metadata: JsonDict = Field(default_factory=dict)
+    artifacts: List[Artifact] = Field(default_factory=list)
+
+
+class AgentTrajectory(BaseModel):
+    """Portable trajectory IR for single-agent and multi-agent runs."""
+
+    trace_id: str = Field(default_factory=lambda: new_id('trace'))
+    task_id: Optional[str] = None
+    goal: Optional[str] = None
+    framework: Optional[str] = None
+    started_at: datetime = Field(default_factory=utc_now)
+    ended_at: Optional[datetime] = None
+    metadata: JsonDict = Field(default_factory=dict)
+    events: List[AgentEvent] = Field(default_factory=list)
+
+    def add_event(self, event: AgentEvent) -> AgentEvent:
+        self.events.append(event)
+        return event
+
+
+class FailureMode(BaseModel):
+    """A seed or generated taxonomy node."""
+
+    mode_id: str
+    name: str
+    family: str
+    description: str
+    signals: List[str] = Field(default_factory=list)
+    suggestion_templates: List[str] = Field(default_factory=list)
+    source: Optional[str] = None
+
+
+class FailureFinding(BaseModel):
+    """A localized failure diagnosis for one event or trajectory region."""
+
+    finding_id: str = Field(default_factory=lambda: new_id('finding'))
+    failure_mode: FailureMode
+    event_id: Optional[str] = None
+    agent_name: Optional[str] = None
+    step_index: Optional[int] = None
+    confidence: float = 0.0
+    evidence: List[str] = Field(default_factory=list)
+    suggestion: Optional[str] = None
+    metadata: JsonDict = Field(default_factory=dict)
+
+
+class DiagnosticReport(BaseModel):
+    """Structured output from an analyzer."""
+
+    report_id: str = Field(default_factory=lambda: new_id('report'))
+    trace_id: str
+    task_id: Optional[str] = None
+    generated_at: datetime = Field(default_factory=utc_now)
+    root_cause_event_id: Optional[str] = None
+    root_cause_agent: Optional[str] = None
+    root_cause_step_index: Optional[int] = None
+    findings: List[FailureFinding] = Field(default_factory=list)
+    summary: str = 'No failure was detected.'
+    suggestions: List[str] = Field(default_factory=list)
+    metadata: JsonDict = Field(default_factory=dict)
+
+
+def model_to_json(model: BaseModel, indent: Optional[int] = None) -> str:
+    """Serialize a Pydantic model across v1 and v2 runtimes."""
+    dumper = getattr(model, 'model_dump_json', None)
+    if callable(dumper):
+        return str(dumper(indent=indent))
+    return str(model.json(indent=indent))
+
+
+def trajectory_from_json(payload: str) -> AgentTrajectory:
+    """Parse an AgentTrajectory across Pydantic v1 and v2 runtimes."""
+    validator = getattr(AgentTrajectory, 'model_validate_json', None)
+    if callable(validator):
+        return cast(AgentTrajectory, validator(payload))
+    return AgentTrajectory.parse_raw(payload)
+
+
+def report_from_json(payload: str) -> DiagnosticReport:
+    """Parse a DiagnosticReport across Pydantic v1 and v2 runtimes."""
+    validator = getattr(DiagnosticReport, 'model_validate_json', None)
+    if callable(validator):
+        return cast(DiagnosticReport, validator(payload))
+    return DiagnosticReport.parse_raw(payload)

agentdebug/recorder.py

@@ -0,0 +1,183 @@
+"""High-level recording API."""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Any, Literal, Optional, Type, Union
+
+from agentdebug.analyzers import HeuristicAnalyzer
+from agentdebug.models import AgentEvent, AgentTrajectory, DiagnosticReport, EventType, utc_now
+from agentdebug.storage import JsonlTraceStore, TraceStore
+
+
+class AgentDebug:
+    """Main entry point for embedding AgentDebugX in an agent system."""
+
+    def __init__(
+        self,
+        store: Optional[TraceStore] = None,
+        analyzer: Optional[HeuristicAnalyzer] = None,
+    ) -> None:
+        self.store = store or JsonlTraceStore()
+        self.analyzer = analyzer or HeuristicAnalyzer()
+
+    def start_trace(
+        self,
+        goal: Optional[str] = None,
+        task_id: Optional[str] = None,
+        framework: Optional[str] = None,
+        trace_id: Optional[str] = None,
+        **metadata: Any,
+    ) -> AgentTrajectory:
+        trajectory = AgentTrajectory(
+            goal=goal,
+            task_id=task_id,
+            framework=framework,
+            metadata=metadata,
+        )
+        if trace_id is not None:
+            trajectory.trace_id = trace_id
+        self.record_event(
+            trajectory,
+            event_type=EventType.RUN_START,
+            agent_name='system',
+            output={'goal': goal, 'framework': framework},
+        )
+        return trajectory
+
+    def record_event(
+        self,
+        trajectory: AgentTrajectory,
+        event_type: Union[EventType, str] = EventType.AGENT_STEP,
+        agent_name: str = 'agent',
+        module: Optional[str] = None,
+        step_index: Optional[int] = None,
+        parent_event_id: Optional[str] = None,
+        input: Any = None,
+        output: Any = None,
+        error: Optional[str] = None,
+        duration_ms: Optional[float] = None,
+        **metadata: Any,
+    ) -> AgentEvent:
+        if isinstance(event_type, str):
+            event_type = EventType(event_type)
+        event = AgentEvent(
+            trace_id=trajectory.trace_id,
+            parent_event_id=parent_event_id,
+            agent_name=agent_name,
+            event_type=event_type,
+            module=module,
+            step_index=step_index,
+            input=input,
+            output=output,
+            error=error,
+            duration_ms=duration_ms,
+            metadata=metadata,
+        )
+        return trajectory.add_event(event)
+
+    def finish_trace(
+        self,
+        trajectory: AgentTrajectory,
+        success: bool,
+        output: Any = None,
+        error: Optional[str] = None,
+        **metadata: Any,
+    ) -> AgentTrajectory:
+        trajectory.ended_at = utc_now()
+        self.record_event(
+            trajectory,
+            event_type=EventType.RUN_END if success else EventType.ERROR,
+            agent_name='system',
+            output=output,
+            error=error,
+            success=success,
+            **metadata,
+        )
+        self.store.save_trajectory(trajectory)
+        return trajectory
+
+    def analyze(self, trajectory: AgentTrajectory) -> DiagnosticReport:
+        report = self.analyzer.analyze(trajectory)
+        save_report = getattr(self.store, 'save_report', None)
+        if callable(save_report):
+            save_report(report)
+        return report
+
+    def analyze_trace(self, trace_id: str) -> DiagnosticReport:
+        trajectory = self.store.load_trajectory(trace_id)
+        if trajectory is None:
+            raise KeyError(f'Unknown trace_id: {trace_id}')
+        return self.analyze(trajectory)
+
+    def trace(
+        self,
+        goal: Optional[str] = None,
+        task_id: Optional[str] = None,
+        framework: Optional[str] = None,
+        **metadata: Any,
+    ) -> 'TraceSession':
+        trajectory = self.start_trace(
+            goal=goal,
+            task_id=task_id,
+            framework=framework,
+            **metadata,
+        )
+        return TraceSession(debugger=self, trajectory=trajectory)
+
+
+class TraceSession:
+    """Context manager around a trajectory."""
+
+    def __init__(self, debugger: AgentDebug, trajectory: AgentTrajectory) -> None:
+        self.debugger = debugger
+        self.trajectory = trajectory
+
+    def __enter__(self) -> 'TraceSession':
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Literal[False]:
+        if exc_value is None:
+            self.debugger.finish_trace(self.trajectory, success=True)
+            return False
+        self.debugger.finish_trace(
+            self.trajectory,
+            success=False,
+            error=f'{exc_type.__name__ if exc_type else "Error"}: {exc_value}',
+        )
+        return False
+
+    def record(
+        self,
+        event_type: Union[EventType, str] = EventType.AGENT_STEP,
+        agent_name: str = 'agent',
+        module: Optional[str] = None,
+        step_index: Optional[int] = None,
+        parent_event_id: Optional[str] = None,
+        input: Any = None,
+        output: Any = None,
+        error: Optional[str] = None,
+        duration_ms: Optional[float] = None,
+        **metadata: Any,
+    ) -> AgentEvent:
+        return self.debugger.record_event(
+            self.trajectory,
+            event_type=event_type,
+            agent_name=agent_name,
+            module=module,
+            step_index=step_index,
+            parent_event_id=parent_event_id,
+            input=input,
+            output=output,
+            error=error,
+            duration_ms=duration_ms,
+            **metadata,
+        )
+
+    def analyze(self) -> DiagnosticReport:
+        return self.debugger.analyze(self.trajectory)

agentdebug/recovery.py

@@ -0,0 +1,113 @@
+"""Lightweight recovery suggestions.
+
+v0.1 ships ``ReflexionSuggestion`` — a *suggest-only* recovery generator that
+produces a structured retry-prompt artifact based on Reflexion (Shinn et al.,
+NeurIPS 2023, arXiv:2303.11366). Heavier strategies (Self-Refine loop, CRITIC,
+Saga rollback, MCTS) are deferred per the roadmap and will land behind the same
+:class:`Recoverer` protocol.
+
+By design, **nothing here re-executes the agent** — recovery proposals are
+artifacts to be surfaced (CLI/UI/PR comment) or fed back into the next run.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Protocol
+
+from agentdebug.models import (
+    AgentTrajectory,
+    DiagnosticReport,
+    FailureFinding,
+    new_id,
+)
+
+
+@dataclass
+class FixProposal:
+    proposal_id: str
+    recoverer_id: str
+    target_event_id: Optional[str]
+    summary: str
+    rationale: str
+    confidence: float
+    suggestion_text: str
+    side_effects: List[str] = field(default_factory=list)
+    requires_human_approval: bool = False
+
+
+class Recoverer(Protocol):
+    id: str
+
+    def suggest(
+        self,
+        trajectory: AgentTrajectory,
+        report: DiagnosticReport,
+    ) -> List[FixProposal]:
+        ...
+
+
+class ReflexionSuggestion:
+    """Emit a Reflexion-style retry reflection per finding.
+
+    The output is purely textual — it can be appended to the agent's next
+    system prompt, written to a project ``MANUAL.md``, or surfaced in the
+    Console. There is no auto-apply.
+    """
+
+    id = 'reflexion'
+
+    def suggest(
+        self,
+        trajectory: AgentTrajectory,
+        report: DiagnosticReport,
+    ) -> List[FixProposal]:
+        if not report.findings:
+            return []
+        proposals: List[FixProposal] = []
+        for finding in report.findings:
+            proposals.append(self._build_proposal(trajectory, finding))
+        return proposals
+
+    def _build_proposal(
+        self, trajectory: AgentTrajectory, finding: FailureFinding
+    ) -> FixProposal:
+        goal = trajectory.goal or '(no goal recorded)'
+        framework = trajectory.framework or '(framework not declared)'
+        evidence_block = '\n'.join(f'  - {e}' for e in finding.evidence) or '  (none)'
+        suggestion_template = (
+            finding.suggestion
+            or (finding.failure_mode.suggestion_templates[0]
+                if finding.failure_mode.suggestion_templates
+                else 'Inspect the offending step and constrain the agent at that point.')
+        )
+        reflection = (
+            f'Task: {goal}\n'
+            f'Framework: {framework}\n'
+            f'Observed failure mode: {finding.failure_mode.mode_id} '
+            f'({finding.failure_mode.name})\n'
+            f'Located at agent={finding.agent_name}, step={finding.step_index}, '
+            f'event_id={finding.event_id}\n'
+            f'Evidence:\n{evidence_block}\n'
+            f'Next time, do the following:\n  {suggestion_template}\n'
+        )
+        return FixProposal(
+            proposal_id=new_id('fix'),
+            recoverer_id=self.id,
+            target_event_id=finding.event_id,
+            summary=(
+                f'Reflexion retry hint for {finding.failure_mode.mode_id} '
+                f'at step {finding.step_index}'
+            ),
+            rationale=(
+                'Reflexion (Shinn et al., NeurIPS 2023) converts a failure '
+                'into a verbal hint appended to next attempt.'
+            ),
+            confidence=min(0.9, max(0.1, finding.confidence)),
+            suggestion_text=reflection,
+            side_effects=['memory.write'],
+            requires_human_approval=False,
+        )
+
+
+__all__ = ['Recoverer', 'FixProposal', 'ReflexionSuggestion']

agentdebug/storage.py

@@ -0,0 +1,167 @@
+"""Persistence backends for traces and diagnostic reports."""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import List, Optional, Protocol
+
+from agentdebug.models import (
+    AgentTrajectory,
+    DiagnosticReport,
+    model_to_json,
+    report_from_json,
+    trajectory_from_json,
+    utc_now,
+)
+
+
+class TraceStore(Protocol):
+    def save_trajectory(self, trajectory: AgentTrajectory) -> None:
+        ...
+
+    def load_trajectory(self, trace_id: str) -> Optional[AgentTrajectory]:
+        ...
+
+    def list_traces(self) -> List[str]:
+        ...
+
+
+class JsonlTraceStore:
+    """Append-only local store for quick adoption and reproducible examples."""
+
+    def __init__(self, path: str = '.agentdebug/traces.jsonl') -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def save_trajectory(self, trajectory: AgentTrajectory) -> None:
+        with self.path.open('a', encoding='utf-8') as handle:
+            handle.write(model_to_json(trajectory))
+            handle.write('\n')
+
+    def load_trajectory(self, trace_id: str) -> Optional[AgentTrajectory]:
+        if not self.path.exists():
+            return None
+        match = None
+        with self.path.open('r', encoding='utf-8') as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                candidate = trajectory_from_json(line)
+                if candidate.trace_id == trace_id:
+                    match = candidate
+        return match
+
+    def list_traces(self) -> List[str]:
+        if not self.path.exists():
+            return []
+        trace_ids = []
+        with self.path.open('r', encoding='utf-8') as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                trace_ids.append(trajectory_from_json(line).trace_id)
+        return trace_ids
+
+
+class SQLiteTraceStore:
+    """Small embedded error database for local development and CI artifacts."""
+
+    def __init__(self, path: str = '.agentdebug/agentdebug.sqlite') -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+
+    def save_trajectory(self, trajectory: AgentTrajectory) -> None:
+        payload = model_to_json(trajectory)
+        with self._connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO trajectories(trace_id, task_id, framework, updated_at, payload_json)
+                VALUES (?, ?, ?, ?, ?)
+                ON CONFLICT(trace_id) DO UPDATE SET
+                    task_id=excluded.task_id,
+                    framework=excluded.framework,
+                    updated_at=excluded.updated_at,
+                    payload_json=excluded.payload_json
+                """,
+                (
+                    trajectory.trace_id,
+                    trajectory.task_id,
+                    trajectory.framework,
+                    utc_now().isoformat(),
+                    payload,
+                ),
+            )
+
+    def load_trajectory(self, trace_id: str) -> Optional[AgentTrajectory]:
+        with self._connect() as conn:
+            row = conn.execute(
+                'SELECT payload_json FROM trajectories WHERE trace_id = ?',
+                (trace_id,),
+            ).fetchone()
+        if row is None:
+            return None
+        return trajectory_from_json(str(row[0]))
+
+    def list_traces(self) -> List[str]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                'SELECT trace_id FROM trajectories ORDER BY updated_at DESC'
+            ).fetchall()
+        return [str(row[0]) for row in rows]
+
+    def save_report(self, report: DiagnosticReport) -> None:
+        with self._connect() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO diagnostic_reports(
+                    report_id, trace_id, generated_at, payload_json
+                )
+                VALUES (?, ?, ?, ?)
+                """,
+                (
+                    report.report_id,
+                    report.trace_id,
+                    report.generated_at.isoformat(),
+                    model_to_json(report),
+                ),
+            )
+
+    def list_reports(self, trace_id: Optional[str] = None) -> List[DiagnosticReport]:
+        query = 'SELECT payload_json FROM diagnostic_reports'
+        params: tuple[str, ...] = ()
+        if trace_id is not None:
+            query += ' WHERE trace_id = ?'
+            params = (trace_id,)
+        query += ' ORDER BY generated_at DESC'
+        with self._connect() as conn:
+            rows = conn.execute(query, params).fetchall()
+        return [report_from_json(str(row[0])) for row in rows]
+
+    def _connect(self) -> sqlite3.Connection:
+        return sqlite3.connect(self.path)
+
+    def _init_db(self) -> None:
+        with self._connect() as conn:
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS trajectories (
+                    trace_id TEXT PRIMARY KEY,
+                    task_id TEXT,
+                    framework TEXT,
+                    updated_at TEXT NOT NULL,
+                    payload_json TEXT NOT NULL
+                )
+                """
+            )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS diagnostic_reports (
+                    report_id TEXT PRIMARY KEY,
+                    trace_id TEXT NOT NULL,
+                    generated_at TEXT NOT NULL,
+                    payload_json TEXT NOT NULL
+                )
+                """
+            )