agentdebugx 0.1.0__py3-none-any.whl

agentdebug/events.py

@@ -0,0 +1,114 @@
+"""In-process pub/sub event bus for live trace introspection.
+
+The bus is intentionally simple: synchronous fan-out, bounded queue per
+subscriber, and never raises out of a publish call. Detectors that need to run
+on a streaming trace subscribe here; offline detectors continue to walk the
+``AgentTrajectory`` directly.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Callable, DefaultDict, Dict, List, Optional
+from uuid import uuid4
+
+from agentdebug.models import AgentEvent, AgentTrajectory, DiagnosticReport
+
+LOG = logging.getLogger('agentdebug.events')
+
+EventHandler = Callable[['BusEvent'], None]
+
+
+@dataclass(frozen=True)
+class BusEvent:
+    """Wrapper passed to bus subscribers.
+
+    The ``kind`` field is one of: ``trace.start``, ``trace.event``,
+    ``trace.end``, ``analysis.report``.
+    """
+
+    kind: str
+    trajectory: Optional[AgentTrajectory] = None
+    event: Optional[AgentEvent] = None
+    report: Optional[DiagnosticReport] = None
+    payload: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EventSubscription:
+    """Returned by :meth:`EventBus.subscribe`; call ``.unsubscribe()`` to detach."""
+
+    id: str
+    kind: str
+    handler: EventHandler
+    _bus: 'EventBus'
+
+    def unsubscribe(self) -> None:
+        self._bus._remove(self)
+
+
+class EventBus:
+    """A tiny synchronous pub/sub bus.
+
+    The bus never raises during ``publish`` — handler exceptions are logged and
+    a per-handler error counter is incremented. Misbehaving handlers are
+    auto-detached after ``error_budget`` consecutive failures so a broken
+    detector cannot wedge a live agent.
+    """
+
+    KIND_ANY = '*'
+
+    def __init__(self, *, error_budget: int = 5) -> None:
+        self._subs: DefaultDict[str, List[EventSubscription]] = defaultdict(list)
+        self._error_counts: Dict[str, int] = {}
+        self._error_budget = error_budget
+
+    def subscribe(self, kind: str, handler: EventHandler) -> EventSubscription:
+        sub = EventSubscription(
+            id=f'sub_{uuid4().hex}',
+            kind=kind,
+            handler=handler,
+            _bus=self,
+        )
+        self._subs[kind].append(sub)
+        self._error_counts[sub.id] = 0
+        return sub
+
+    def publish(self, evt: BusEvent) -> None:
+        # Fan-out to exact-kind subscribers AND wildcard subscribers.
+        for sub in list(self._subs.get(evt.kind, ())):
+            self._invoke(sub, evt)
+        for sub in list(self._subs.get(self.KIND_ANY, ())):
+            self._invoke(sub, evt)
+
+    def subscribers(self, kind: str) -> int:
+        return len(self._subs.get(kind, ()))
+
+    def _invoke(self, sub: EventSubscription, evt: BusEvent) -> None:
+        try:
+            sub.handler(evt)
+            self._error_counts[sub.id] = 0
+        except Exception as exc:  # pragma: no cover - defensive only
+            LOG.warning(
+                'event handler %s raised on kind=%s: %s', sub.id, evt.kind, exc
+            )
+            self._error_counts[sub.id] = self._error_counts.get(sub.id, 0) + 1
+            if self._error_counts[sub.id] >= self._error_budget:
+                LOG.warning(
+                    'detaching subscription %s after %d failures',
+                    sub.id,
+                    self._error_budget,
+                )
+                self._remove(sub)
+
+    def _remove(self, sub: EventSubscription) -> None:
+        bucket = self._subs.get(sub.kind, [])
+        self._subs[sub.kind] = [s for s in bucket if s.id != sub.id]
+        self._error_counts.pop(sub.id, None)
+
+
+# A process-global default bus. Tracers that don't take an explicit bus argument
+# use this one; tests construct their own.
+DEFAULT_BUS = EventBus()

agentdebug/instrumentation.py

@@ -0,0 +1,57 @@
+"""Small helpers for instrumenting plain Python functions."""
+
+from __future__ import annotations
+
+import time
+from functools import wraps
+from typing import Any, Callable, Optional, TypeVar, cast
+
+from agentdebug.models import EventType
+from agentdebug.recorder import TraceSession
+
+F = TypeVar('F', bound=Callable[..., Any])
+
+
+def traced_tool(
+    session: TraceSession,
+    agent_name: str = 'tool',
+    tool_name: Optional[str] = None,
+) -> Callable[[F], F]:
+    """Record a function invocation as tool.call/tool.result events."""
+
+    def decorator(func: F) -> F:
+        name = tool_name or func.__name__
+
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            started = time.perf_counter()
+            call_event = session.record(
+                EventType.TOOL_CALL,
+                agent_name=agent_name,
+                input={'tool': name, 'args': repr(args), 'kwargs': repr(kwargs)},
+            )
+            try:
+                result = func(*args, **kwargs)
+            except Exception as exc:
+                session.record(
+                    EventType.TOOL_RESULT,
+                    agent_name=agent_name,
+                    parent_event_id=call_event.event_id,
+                    error=str(exc),
+                    duration_ms=(time.perf_counter() - started) * 1000,
+                    tool=name,
+                )
+                raise
+            session.record(
+                EventType.TOOL_RESULT,
+                agent_name=agent_name,
+                parent_event_id=call_event.event_id,
+                output=result,
+                duration_ms=(time.perf_counter() - started) * 1000,
+                tool=name,
+            )
+            return result
+
+        return cast(F, wrapper)
+
+    return decorator

agentdebug/judges.py

@@ -0,0 +1,258 @@
+"""LLM-based judge analyzer.
+
+Walks an ``AgentTrajectory`` and asks an LLM to label step-level failures
+against the 19 seed modes shipped in :mod:`agentdebug.taxonomy`. The judge is
+intentionally schemed to the existing taxonomy so the rest of the pipeline
+(``DiagnosticReport``, recovery, attribution) doesn't need a parallel label
+space.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, Sequence
+
+from agentdebug.llm import LLMClient, extract_json_block
+from agentdebug.models import (
+    AgentEvent,
+    AgentTrajectory,
+    DiagnosticReport,
+    EventType,
+    FailureFinding,
+    FailureMode,
+    new_id,
+)
+from agentdebug.taxonomy import SEED_FAILURE_MODES
+
+LOG = logging.getLogger('agentdebug.judges')
+
+_SYSTEM_PROMPT = """You are AgentDebugX-Judge, an expert at diagnosing failures
+in LLM agent trajectories.
+
+You will be given:
+* the agent's goal,
+* a list of ALLOWED failure mode codes with descriptions,
+* the chronological events of one agent run.
+
+Your job: identify each step where a failure occurred and label it with ONE of
+the allowed failure mode codes. Be conservative — only flag steps where the
+evidence in the event payload supports the label. If the trajectory contains no
+failure, return an empty findings list.
+
+Respond ONLY with a JSON object matching this schema (no prose, no markdown):
+
+{
+  "findings": [
+    {
+      "event_id": "<event_id from the input>",
+      "step_index": <int or null>,
+      "agent_name": "<agent_name from the input>",
+      "failure_mode_id": "<one of the allowed codes>",
+      "confidence": <float between 0 and 1>,
+      "evidence": ["<short quote or summary of the supporting payload>"]
+    }
+  ],
+  "summary": "<one-sentence diagnosis or 'No failure detected.'>"
+}
+"""
+
+
+class LLMJudgeAnalyzer:
+    """LLM-as-judge analyzer schemed to the 19 seed failure modes."""
+
+    def __init__(
+        self,
+        llm: LLMClient,
+        *,
+        max_events_per_call: int = 80,
+        max_evidence_chars: int = 300,
+        max_tokens: int = 4096,
+    ) -> None:
+        self.llm = llm
+        self.max_events_per_call = max_events_per_call
+        self.max_evidence_chars = max_evidence_chars
+        # NOTE: thinking models (Gemini 2.x/3.x, o-series) spend a substantial
+        # fraction of `max_tokens` on reasoning tokens before any text is
+        # emitted. 4096 is the safe default; bump higher for long traces.
+        self.max_tokens = max_tokens
+
+    def analyze(self, trajectory: AgentTrajectory) -> DiagnosticReport:
+        events = trajectory.events
+        findings: List[FailureFinding] = []
+        summary_parts: List[str] = []
+        for chunk in self._chunk_events(events):
+            findings_chunk, summary = self._judge_chunk(trajectory, chunk)
+            findings.extend(findings_chunk)
+            if summary:
+                summary_parts.append(summary)
+        root = self._select_root(findings)
+        report = DiagnosticReport(
+            trace_id=trajectory.trace_id,
+            task_id=trajectory.task_id,
+            findings=findings,
+            suggestions=self._collect_suggestions(findings),
+            metadata={'analyzer': self.__class__.__name__, 'model': self.llm.model},
+        )
+        report.summary = (
+            ' '.join(s for s in summary_parts if s)
+            or (
+                f'Likely root cause: {root.failure_mode.name}'
+                f' in {root.agent_name or "unknown agent"}'
+                f' at step {root.step_index}.'
+                if root
+                else 'No failure was detected.'
+            )
+        )
+        if root is not None:
+            report.root_cause_event_id = root.event_id
+            report.root_cause_agent = root.agent_name
+            report.root_cause_step_index = root.step_index
+        return report
+
+    def _chunk_events(self, events: Sequence[AgentEvent]) -> List[List[AgentEvent]]:
+        if not events:
+            return []
+        return [
+            list(events[i : i + self.max_events_per_call])
+            for i in range(0, len(events), self.max_events_per_call)
+        ]
+
+    def _judge_chunk(
+        self, trajectory: AgentTrajectory, chunk: List[AgentEvent]
+    ) -> tuple[List[FailureFinding], str]:
+        user = self._render_user_prompt(trajectory, chunk)
+        messages = [
+            {'role': 'system', 'content': _SYSTEM_PROMPT},
+            {'role': 'user', 'content': user},
+        ]
+        result = self.llm.complete(messages=messages, max_tokens=self.max_tokens)
+        parsed = extract_json_block(result.text)
+        if not parsed:
+            LOG.warning('LLM judge returned no JSON; raw=%r', result.text[:300])
+            return [], ''
+        raw_findings = parsed.get('findings') or []
+        summary = str(parsed.get('summary') or '')
+        findings: List[FailureFinding] = []
+        for raw in raw_findings:
+            if not isinstance(raw, dict):
+                continue
+            mode_id = str(raw.get('failure_mode_id') or '')
+            failure_mode = SEED_FAILURE_MODES.get(mode_id)
+            if failure_mode is None:
+                LOG.debug('skipping unknown failure_mode_id from judge: %s', mode_id)
+                continue
+            findings.append(
+                FailureFinding(
+                    finding_id=new_id('finding'),
+                    failure_mode=failure_mode,
+                    event_id=self._coerce_str(raw.get('event_id')),
+                    agent_name=self._coerce_str(raw.get('agent_name')),
+                    step_index=self._coerce_int(raw.get('step_index')),
+                    confidence=self._coerce_float(raw.get('confidence'), default=0.5),
+                    evidence=self._coerce_str_list(raw.get('evidence')),
+                    suggestion=self._suggestion(failure_mode),
+                    metadata={'source': 'llm_judge'},
+                )
+            )
+        return findings, summary
+
+    def _render_user_prompt(
+        self, trajectory: AgentTrajectory, chunk: List[AgentEvent]
+    ) -> str:
+        modes_doc = '\n'.join(
+            f'- {mode_id}: {mode.description}'
+            for mode_id, mode in SEED_FAILURE_MODES.items()
+        )
+        events_doc = '\n'.join(self._render_event(evt) for evt in chunk)
+        return (
+            f'GOAL: {trajectory.goal!r}\n'
+            f'FRAMEWORK: {trajectory.framework!r}\n\n'
+            f'ALLOWED FAILURE MODES:\n{modes_doc}\n\n'
+            f'EVENTS:\n{events_doc}\n'
+        )
+
+    def _render_event(self, event: AgentEvent) -> str:
+        def shorten(value: Any) -> str:
+            text = '' if value is None else str(value)
+            if len(text) > self.max_evidence_chars:
+                text = text[: self.max_evidence_chars] + '…'
+            return text
+
+        return (
+            f'event_id={event.event_id} '
+            f'type={self._event_type_value(event.event_type)} '
+            f'agent={event.agent_name} '
+            f'module={event.module} step={event.step_index} '
+            f'input={shorten(event.input)} '
+            f'output={shorten(event.output)} '
+            f'error={shorten(event.error)} '
+            f'metadata={shorten(event.metadata)}'
+        )
+
+    def _select_root(
+        self, findings: List[FailureFinding]
+    ) -> Optional[FailureFinding]:
+        if not findings:
+            return None
+        return sorted(
+            findings,
+            key=lambda finding: (
+                finding.step_index is None,
+                finding.step_index if finding.step_index is not None else 10**9,
+                -finding.confidence,
+            ),
+        )[0]
+
+    def _collect_suggestions(self, findings: List[FailureFinding]) -> List[str]:
+        seen = set()
+        out: List[str] = []
+        for f in findings:
+            if f.suggestion and f.suggestion not in seen:
+                seen.add(f.suggestion)
+                out.append(f.suggestion)
+        return out
+
+    def _suggestion(self, failure_mode: FailureMode) -> Optional[str]:
+        if not failure_mode.suggestion_templates:
+            return None
+        return str(failure_mode.suggestion_templates[0])
+
+    @staticmethod
+    def _coerce_str(value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        return str(value)
+
+    @staticmethod
+    def _coerce_int(value: Any) -> Optional[int]:
+        if value is None:
+            return None
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _coerce_float(value: Any, *, default: float) -> float:
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_str_list(value: Any) -> List[str]:
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return [str(v) for v in value]
+        return [str(value)]
+
+    @staticmethod
+    def _event_type_value(event_type: EventType) -> str:
+        value = getattr(event_type, 'value', event_type)
+        if isinstance(value, str):
+            return value
+        return str(value)
+
+
+__all__ = ['LLMJudgeAnalyzer']

agentdebug/llm.py

@@ -0,0 +1,165 @@
+"""Thin LLM client abstraction.
+
+AgentDebugX needs an LLM for the judge analyzer and for the All-at-Once
+attributor. We use an OpenAI-compatible chat-completions interface so users can
+point us at OpenAI, Anthropic via LiteLLM, the Gemini endpoint they hand us, or
+a local vLLM/Ollama deployment.
+
+The implementation deliberately avoids depending on the ``openai`` Python SDK:
+a single ``httpx`` POST keeps the install lightweight and lets users target any
+``/v1/chat/completions``-compatible URL.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Protocol
+
+import httpx
+
+LOG = logging.getLogger('agentdebug.llm')
+
+
+@dataclass
+class CompletionResult:
+    text: str
+    raw: Dict[str, Any]
+
+
+class LLMClient(Protocol):
+    model: str
+
+    def complete(
+        self,
+        messages: List[Dict[str, Any]],
+        *,
+        response_format: Optional[Dict[str, Any]] = None,
+        temperature: float = 0.0,
+        max_tokens: int = 2048,
+        timeout: float = 60.0,
+    ) -> CompletionResult:
+        ...
+
+
+class OpenAICompatClient:
+    """OpenAI-compatible chat completions client.
+
+    Works against:
+
+    * OpenAI (``base_url='https://api.openai.com/v1'``)
+    * LiteLLM proxy (any ``/v1`` URL)
+    * The Gemini gateway used in this repo:
+      ``https://compliant-wagner-simulations-coaches.trycloudflare.com/v1``
+      with ``model='gemini-3-flash'``
+    * vLLM / Ollama with their OpenAI-compat servers
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        api_key: str,
+        model: str,
+        default_max_tokens: int = 2048,
+        timeout: float = 60.0,
+    ) -> None:
+        self.base_url = base_url.rstrip('/')
+        self.api_key = api_key
+        self.model = model
+        self.default_max_tokens = default_max_tokens
+        self.timeout = timeout
+
+    @classmethod
+    def from_env(
+        cls,
+        *,
+        env_prefix: str = 'AGENTDEBUG_LLM',
+        model: Optional[str] = None,
+    ) -> 'OpenAICompatClient':
+        """Construct from environment variables.
+
+        Reads ``<PREFIX>_BASE_URL``, ``<PREFIX>_API_KEY``, ``<PREFIX>_MODEL``.
+        """
+        base_url = os.environ[f'{env_prefix}_BASE_URL']
+        api_key = os.environ[f'{env_prefix}_API_KEY']
+        model_id: str = (
+            model if model is not None
+            else os.environ.get(f'{env_prefix}_MODEL', 'gpt-4o-mini')
+        )
+        return cls(base_url=base_url, api_key=api_key, model=model_id)
+
+    def complete(
+        self,
+        messages: List[Dict[str, Any]],
+        *,
+        response_format: Optional[Dict[str, Any]] = None,
+        temperature: float = 0.0,
+        max_tokens: Optional[int] = None,
+        timeout: Optional[float] = None,
+    ) -> CompletionResult:
+        body: Dict[str, Any] = {
+            'model': self.model,
+            'messages': messages,
+            'temperature': temperature,
+            'max_tokens': max_tokens or self.default_max_tokens,
+        }
+        if response_format is not None:
+            body['response_format'] = response_format
+        url = f'{self.base_url}/chat/completions'
+        headers = {
+            'Authorization': f'Bearer {self.api_key}',
+            'Content-Type': 'application/json',
+        }
+        resp = httpx.post(
+            url, headers=headers, json=body, timeout=timeout or self.timeout
+        )
+        resp.raise_for_status()
+        data: Dict[str, Any] = resp.json()
+        choice = (data.get('choices') or [{}])[0]
+        message = choice.get('message') or {}
+        text = message.get('content') or ''
+        if not text:
+            LOG.warning(
+                'empty content from %s (model=%s, finish_reason=%s, usage=%s)',
+                url,
+                self.model,
+                choice.get('finish_reason'),
+                data.get('usage'),
+            )
+        return CompletionResult(text=text, raw=data)
+
+
+def extract_json_block(text: str) -> Optional[Dict[str, Any]]:
+    """Extract the first top-level JSON object from a possibly-fenced response."""
+    if not text:
+        return None
+    # Strip code fences if present.
+    cleaned = text.strip()
+    if cleaned.startswith('```'):
+        # remove ``` or ```json prefix and trailing ```
+        cleaned = cleaned.split('\n', 1)[1] if '\n' in cleaned else cleaned[3:]
+        if cleaned.endswith('```'):
+            cleaned = cleaned[: -len('```')]
+        cleaned = cleaned.strip()
+    # Try strict first.
+    try:
+        parsed = json.loads(cleaned)
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        pass
+    # Fallback: greedy slice between the first { and the last }.
+    start = cleaned.find('{')
+    end = cleaned.rfind('}')
+    if start == -1 or end == -1 or end <= start:
+        return None
+    try:
+        parsed = json.loads(cleaned[start : end + 1])
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        return None
+    return None