agentdebugx 0.2.9__tar.gz → 0.2.11__tar.gz

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentdebugx
-Version: 0.2.9
+Version: 0.2.11
 Summary: Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`.
 License: MIT
 License-File: LICENSE

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/docs/23_status_v0_2.md

@@ -23,6 +23,7 @@ the forward-looking plan; this doc is the rear-view mirror.
 | Attribution | `agentdebug.attribution.BinarySearchAttributor` | ✅ **new 0.2.3** | oracle-LLM logarithmic convergence + fallback + render elision |
 | Attribution | `agentdebug.attribution.CounterfactualAttributor` | ✅ **new 0.2.7** | scripted-rescue-prob ranking + candidate selection priority (findings → errors → tail) + dual fallback (no candidates / silent LLM) |
 | Attribution | `agentdebug.attribution.SBFLAttributor` (Tarantula / Ochiai / DStar) | ✅ **new 0.2.8** | signature design + ranking on synthetic corpus + 3 formulas + dual fallback + edge cases. Model-free: zero LLM cost at inference. |
+| Attribution | `agentdebug.attribution.EnsembleAttributor` (+ `AttributionBudget`) | ✅ **new 0.2.10** | Borda + Bayesian merge, weighted backends, walltime/backend-count budget, swallows backend exceptions, falls back when all empty, honest per-source provenance |
 | Recovery | `agentdebug.recovery.ReflexionSuggestion` | ✅ stable | per-finding + empty |
 | Recovery | `agentdebug.recovery.CriticRecoverer` + `VerifierSpec` registry | ✅ **new 0.2.3** | 5 family-matched verifier templates; dedup + custom-override |
 | Recovery | `agentdebug.recovery.SelfRefineLoop` | ✅ **new 0.2.9** | scripted-LLM happy + silent-LLM safe + multi-iter; per-finding generator→critic→refiner |
@@ -30,6 +31,8 @@ the forward-looking plan; this doc is the rear-view mirror.
 | DeepDebug | `agentdebug.deep.DeepDebugAnalyzer` | ✅ stable | full loop + silent LLM |
 | Cascade view | `agentdebug.traceback.format_traceback` | ✅ stable | cascade + step-order + ANSI + empty |
 | Detectors | `agentdebug.detectors.RepeatedToolCall / RepeatedState / StepCountLimit` | ✅ **new 0.2.2** | threshold + window + budget |
+| Detectors | `agentdebug.detectors.TopicDriftDetector` (embedding cosine vs goal) | ✅ **new 0.2.11** | stub-embedder ranking + no-goal short-circuit + embedder-raises safe + threshold boundary |
+| LLM client | `agentdebug.llm.OpenAICompatClient.embed()` (+ `EmbeddingClient` Protocol) | ✅ **new 0.2.11** | mocked-httpx POST /v1/embeddings + empty-input short-circuit |
 | Hub bundle | `agentdebug.hub.Bundle / pack_bundle / unpack_bundle` | ✅ stable | round-trip |
 | Hub scrubber | `agentdebug.hub.Scrubber` | ✅ stable | 12 redactions + idempotent |
 | Hub backends | `LocalHubBackend`, `GitHubBackend`, `HuggingFaceBackend` | ✅ stable | local-bare-git + local |
@@ -48,13 +51,11 @@ across 32 source files.
 
 | Doc | Component | Why deferred | Realistic ship |
 |---|---|---|---|
-| [06_detectors.md](./06_detectors.md) | `trajectory_perplexity` (TrajAD) | needs token-level LM perplexity API or embedding model + baseline calibration | v0.3 |
-| [06_detectors.md](./06_detectors.md) | `topic_drift` (embedding cosine) | needs embedding client; consider reusing `OpenAICompatClient` `/embeddings` | v0.3 |
+| [06_detectors.md](./06_detectors.md) | `trajectory_perplexity` (TrajAD) | needs token-level LM perplexity API; v0.3 |
 | [06_detectors.md](./06_detectors.md) | LTL spec monitors | requires user-supplied spec or LLM-synthesized monitors; gated on RV research | v1.2 |
 | [07_attribution.md](./07_attribution.md) | `CounterfactualAttributor` — *real* replay variant | true re-rollout requires framework-specific replay surface; the v0.2.7 LLM-simulated variant ships now, the real-replay variant is gated on adapter support (LangGraph checkpointer / OpenHands rewind) | v0.4 |
 | [07_attribution.md](./07_attribution.md) | `SBFLAttributor` — *corpus* | shipped in 0.2.8 (`tarantula`/`ochiai`/`dstar`); awaiting paired-trace adoption to gather a useful corpus in production | corpus tooling deferred to v0.4 |
 | [07_attribution.md](./07_attribution.md) | `DeltaDebugAttributor` (Zeller) | same replay constraint | v0.3 |
-| [07_attribution.md](./07_attribution.md) | `EnsembleAttributor` | trivial once Counterfactual lands; awaits Counterfactual | v0.3 |
 | [08_recovery.md](./08_recovery.md) | `LangGraphRewind` | depends on LangGraph checkpointer; ships when we have a real LangGraph user | v0.3 |
 | [08_recovery.md](./08_recovery.md) | `SagaRollback` | needs compensation registry on tool definitions; new schema | v0.3 |
 | [08_recovery.md](./08_recovery.md) | `MCTSBranchExploration` (LATS) | heavy; v2 feature | v2.0 |

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentdebugx"
-version = "0.2.9"
+version = "0.2.11"
 description = "Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`."
 authors = ["ULab @ UIUC <ulab@illinois.edu>"]
 license = "MIT"

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/src/agentdebug/__init__.py

@@ -11,11 +11,13 @@ See ``docs/14_api_reference.md`` for the full public surface and
 from agentdebug.analyzers import HeuristicAnalyzer
 from agentdebug.attribution import (
     AllAtOnceAttributor,
+    AttributionBudget,
     AttributionResult,
     Attributor,
     BinarySearchAttributor,
     Blame,
     CounterfactualAttributor,
+    EnsembleAttributor,
     HeuristicAttributor,
     SBFLAttributor,
     StepByStepAttributor,
@@ -26,6 +28,7 @@ from agentdebug.detectors import (
     RepeatedStateDetector,
     RepeatedToolCallDetector,
     StepCountLimitDetector,
+    TopicDriftDetector,
     default_detectors,
     run_detectors,
 )
@@ -65,6 +68,7 @@ __all__ = [
     'Attributor',
     'Blame',
     'BusEvent',
+    'AttributionBudget',
     'AutoManualRules',
     'BinarySearchAttributor',
     'CascadeFrame',
@@ -73,11 +77,13 @@ __all__ = [
     'DEFAULT_VERIFIERS',
     'Detector',
     'DetectorConfig',
+    'EnsembleAttributor',
     'RepeatedStateDetector',
     'RepeatedToolCallDetector',
     'SBFLAttributor',
     'SelfRefineLoop',
     'StepByStepAttributor',
+    'TopicDriftDetector',
     'StepCountLimitDetector',
     'VerifierSpec',
     'build_cascade',
@@ -104,4 +110,4 @@ __all__ = [
     'get_failure_mode',
 ]
 
-__version__ = '0.2.9'
+__version__ = '0.2.11'

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/src/agentdebug/attribution.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Protocol, cast
+from typing import Any, Dict, List, Optional, Protocol, Tuple, cast
 
 
 # Forward decl so BinarySearchAttributor.attribute can reference _EllipsisEvent
@@ -936,8 +936,217 @@ def _suspiciousness(
     return float((ef ** dstar_exponent) / denom)
 
 
+@dataclass
+class AttributionBudget:
+    """Optional cap on ensemble cost."""
+
+    max_backends: Optional[int] = None     # short-circuit after N successful backends
+    max_seconds: Optional[float] = None    # walltime cap; remaining backends skipped
+
+    def exceeded(
+        self, *, completed: int, elapsed_s: float,
+    ) -> bool:
+        if self.max_backends is not None and completed >= self.max_backends:
+            return True
+        if self.max_seconds is not None and elapsed_s >= self.max_seconds:
+            return True
+        return False
+
+
+class EnsembleAttributor:
+    """Compose any subset of attributors into a single ranked result.
+
+    Two merge strategies:
+
+      * ``"borda"`` (default) — each backend's ranked list contributes
+        Borda points (top hypothesis = N, next = N-1, …). Steps are
+        ranked by total points × backend weight; ties broken by primary
+        confidence × weight.
+      * ``"bayesian"`` — combine confidences as 1 - ∏(1 - w_i × c_i)
+        across backends that nominated the same (event_id, step_index).
+        Treats each backend as an independent noisy classifier; good when
+        backends are heterogeneous and confidences are well-calibrated.
+
+    Provenance is HONEST: every output ``Blame.sources`` lists every
+    backend that contributed; the rationale aggregates source-by-source
+    so the UI can show which backends agreed.
+
+    Pair with ``AttributionBudget`` to cap wall-clock or backend count
+    when running expensive backends (e.g., BinarySearch + Counterfactual)
+    alongside cheap ones (Heuristic + SBFL).
+    """
+
+    id = 'ensemble'
+
+    def __init__(
+        self,
+        backends: List[Attributor],
+        *,
+        weights: Optional[Dict[str, float]] = None,
+        merge: str = 'borda',
+        top_k: int = 5,
+        budget: Optional[AttributionBudget] = None,
+        fallback: Optional[Attributor] = None,
+    ) -> None:
+        if not backends:
+            raise ValueError('EnsembleAttributor requires at least one backend')
+        if merge not in {'borda', 'bayesian'}:
+            raise ValueError(
+                f"Unknown merge strategy {merge!r}; use 'borda' or 'bayesian'"
+            )
+        self.backends = list(backends)
+        self.weights = weights or {b.id: 1.0 for b in backends}
+        self.merge = merge
+        self.top_k = top_k
+        self.budget = budget
+        self.fallback: Attributor = fallback or HeuristicAttributor()
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> AttributionResult:
+        import time as _time
+
+        per_backend: List[Tuple[str, AttributionResult]] = []
+        started = _time.perf_counter()
+        for backend in self.backends:
+            if self.budget and self.budget.exceeded(
+                completed=len(per_backend),
+                elapsed_s=_time.perf_counter() - started,
+            ):
+                LOG.debug('ensemble budget hit; stopping early')
+                break
+            try:
+                result = backend.attribute(trajectory, findings)
+            except Exception as exc:  # pragma: no cover - defensive
+                LOG.warning('ensemble backend %s raised: %s', backend.id, exc)
+                continue
+            per_backend.append((backend.id, result))
+
+        # Any backend produced hypotheses? Otherwise fall back.
+        if not any(r.hypotheses for _id, r in per_backend):
+            return self.fallback.attribute(trajectory, findings)
+
+        if self.merge == 'borda':
+            merged = self._merge_borda(per_backend)
+        else:
+            merged = self._merge_bayesian(per_backend)
+
+        merged = merged[: self.top_k]
+        elapsed_ms = int((_time.perf_counter() - started) * 1000)
+        return AttributionResult(
+            method=self.id,
+            hypotheses=merged,
+            elapsed_ms=elapsed_ms,
+            raw={
+                'merge': self.merge,
+                'backends_run': [bid for bid, _ in per_backend],
+                'weights': dict(self.weights),
+                'budget_exceeded': bool(
+                    self.budget
+                    and self.budget.exceeded(
+                        completed=len(per_backend),
+                        elapsed_s=_time.perf_counter() - started,
+                    )
+                ),
+            },
+        )
+
+    # ---- merge strategies ----
+
+    def _merge_borda(
+        self, per_backend: List[Tuple[str, AttributionResult]],
+    ) -> List[Blame]:
+        # Aggregator keyed by canonical step identity.
+        agg: Dict[Tuple[Optional[str], Optional[int]], _MergeRow] = {}
+        for backend_id, result in per_backend:
+            w = self.weights.get(backend_id, 1.0)
+            n = len(result.hypotheses)
+            for rank, h in enumerate(result.hypotheses):
+                key = (h.span_id, h.step_index)
+                row = agg.setdefault(key, _MergeRow(span_id=h.span_id,
+                                                   step_index=h.step_index))
+                # Borda points: top = N, next = N-1, … last = 1
+                row.borda_points += (n - rank) * w
+                row.weighted_conf_sum += w * h.confidence
+                row.weight_sum += w
+                row.sources.add(backend_id)
+                if h.agent_name and not row.agent_name:
+                    row.agent_name = h.agent_name
+                if h.rationale and backend_id not in row.rationales:
+                    row.rationales[backend_id] = h.rationale
+                row.evidence.extend(h.evidence)
+        ranked = sorted(
+            agg.values(),
+            key=lambda r: (-r.borda_points, -r.weighted_conf_sum),
+        )
+        return [self._row_to_blame(r) for r in ranked]
+
+    def _merge_bayesian(
+        self, per_backend: List[Tuple[str, AttributionResult]],
+    ) -> List[Blame]:
+        agg: Dict[Tuple[Optional[str], Optional[int]], _MergeRow] = {}
+        for backend_id, result in per_backend:
+            w = self.weights.get(backend_id, 1.0)
+            for h in result.hypotheses:
+                key = (h.span_id, h.step_index)
+                row = agg.setdefault(key, _MergeRow(span_id=h.span_id,
+                                                   step_index=h.step_index))
+                # P(no source supports) *= (1 - w*c)
+                effective = max(0.0, min(1.0, w * h.confidence))
+                row.bayesian_not_pos *= (1.0 - effective)
+                row.weighted_conf_sum += w * h.confidence
+                row.weight_sum += w
+                row.sources.add(backend_id)
+                if h.agent_name and not row.agent_name:
+                    row.agent_name = h.agent_name
+                if h.rationale and backend_id not in row.rationales:
+                    row.rationales[backend_id] = h.rationale
+                row.evidence.extend(h.evidence)
+        ranked = sorted(
+            agg.values(),
+            key=lambda r: (-(1.0 - r.bayesian_not_pos), -r.weighted_conf_sum),
+        )
+        return [self._row_to_blame(r) for r in ranked]
+
+    def _row_to_blame(self, row: '_MergeRow') -> Blame:
+        if self.merge == 'borda':
+            # Confidence = weighted average of backend confidences (cap at 1).
+            conf = min(1.0, row.weighted_conf_sum / max(row.weight_sum, 1e-9))
+        else:
+            conf = max(0.0, min(1.0, 1.0 - row.bayesian_not_pos))
+        rationale = '; '.join(
+            f'{bid}: {txt[:140]}' for bid, txt in row.rationales.items()
+        ) or 'ensemble agreement'
+        return Blame(
+            span_id=row.span_id,
+            step_index=row.step_index,
+            agent_name=row.agent_name,
+            confidence=conf,
+            rationale=rationale,
+            evidence=list(dict.fromkeys(row.evidence)),  # dedupe preserve order
+            sources=sorted(row.sources),
+        )
+
+
+@dataclass
+class _MergeRow:
+    span_id: Optional[str]
+    step_index: Optional[int]
+    agent_name: Optional[str] = None
+    borda_points: float = 0.0
+    bayesian_not_pos: float = 1.0
+    weighted_conf_sum: float = 0.0
+    weight_sum: float = 0.0
+    sources: set[str] = field(default_factory=set)
+    rationales: Dict[str, str] = field(default_factory=dict)
+    evidence: List[str] = field(default_factory=list)
+
+
 __all__ = [
-    'Attributor', 'Blame', 'AttributionResult',
-    'HeuristicAttributor', 'AllAtOnceAttributor', 'StepByStepAttributor',
-    'BinarySearchAttributor', 'CounterfactualAttributor', 'SBFLAttributor',
+    'AllAtOnceAttributor', 'AttributionBudget', 'AttributionResult',
+    'Attributor', 'BinarySearchAttributor', 'Blame',
+    'CounterfactualAttributor', 'EnsembleAttributor', 'HeuristicAttributor',
+    'SBFLAttributor', 'StepByStepAttributor',
 ]

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/src/agentdebug/detectors.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 import logging
 from dataclasses import dataclass
-from typing import List, Optional, Protocol
+from typing import Any, List, Optional, Protocol
 
 from agentdebug.models import (
     AgentEvent,
@@ -273,12 +273,106 @@ def _suggestion(mode: FailureMode) -> Optional[str]:
     return None
 
 
+class TopicDriftDetector:
+    """Embedding-based anomaly detector for goal drift.
+
+    Embed the trajectory's goal once; embed each user-facing event payload
+    (LLM_RESPONSE / PLAN / OBSERVATION outputs); flag any step whose cosine
+    similarity with the goal drops below ``threshold``.
+
+    Maps to ``FM-2.3 task_derailment`` (MAST) / ``planning.inefficient_plan``.
+    Closes the anomaly family from doc 06 alongside the existing
+    RepeatedToolCallDetector / RepeatedStateDetector.
+
+    Skipped silently if the embedding client raises or returns no vectors —
+    the rest of the detector pipeline is unaffected.
+    """
+
+    id = 'topic_drift'
+
+    def __init__(
+        self,
+        embedding_client: Any,
+        *,
+        threshold: float = 0.35,
+        max_events: int = 60,
+    ) -> None:
+        # embedding_client is duck-typed to EmbeddingClient to avoid an
+        # import cycle (detectors.py is imported from agentdebug/__init__.py).
+        self.embedding_client = embedding_client
+        self.threshold = threshold
+        self.max_events = max_events
+
+    def detect(self, trajectory: AgentTrajectory) -> List[FailureFinding]:
+        if not trajectory.goal:
+            return []
+        contextful = [
+            e for e in trajectory.events
+            if e.event_type in {
+                EventType.LLM_RESPONSE, EventType.PLAN, EventType.OBSERVATION,
+                EventType.LLM_RESPONSE.value, EventType.PLAN.value,
+                EventType.OBSERVATION.value,
+            }
+            and e.output is not None and str(e.output).strip()
+        ]
+        if not contextful:
+            return []
+        contextful = contextful[-self.max_events:]
+        texts = [trajectory.goal] + [str(e.output)[:1000] for e in contextful]
+        try:
+            vectors = self.embedding_client.embed(texts)
+        except Exception as exc:  # pragma: no cover - defensive
+            LOG.warning('topic_drift detector embed() failed: %s', exc)
+            return []
+        if not vectors or len(vectors) != len(texts):
+            return []
+        goal_vec = vectors[0]
+        findings: List[FailureFinding] = []
+        mode = SEED_FAILURE_MODES['planning.inefficient_plan']
+        for evt, evt_vec in zip(contextful, vectors[1:]):
+            sim = _cosine(goal_vec, evt_vec)
+            if sim >= self.threshold:
+                continue
+            findings.append(FailureFinding(
+                finding_id=new_id('finding'),
+                failure_mode=mode,
+                event_id=evt.event_id,
+                agent_name=evt.agent_name,
+                step_index=evt.step_index,
+                # Confidence proportional to how far below threshold we drifted.
+                confidence=min(0.95, 0.4 + (self.threshold - sim)),
+                evidence=[
+                    f'goal/output cosine={sim:.3f} < threshold={self.threshold:.2f}',
+                ],
+                suggestion=_suggestion(mode),
+                metadata={
+                    'source': self.id,
+                    'cosine_to_goal': round(sim, 4),
+                    'threshold': self.threshold,
+                },
+            ))
+        return findings
+
+
+def _cosine(a: List[float], b: List[float]) -> float:
+    import math
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(y * y for y in b))
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
 __all__ = [
     'Detector',
     'DetectorConfig',
     'RepeatedStateDetector',
     'RepeatedToolCallDetector',
     'StepCountLimitDetector',
+    'TopicDriftDetector',
     'default_detectors',
     'run_detectors',
 ]

{agentdebugx-0.2.9 → agentdebugx-0.2.11}/src/agentdebug/llm.py

@@ -44,6 +44,25 @@ class LLMClient(Protocol):
         ...
 
 
+class EmbeddingClient(Protocol):
+    """Subprotocol for clients that also expose ``/v1/embeddings``.
+
+    Kept separate from :class:`LLMClient` so detectors can declare a
+    narrower dependency and tests can stub embeddings without faking
+    a chat client.
+    """
+
+    embedding_model: str
+
+    def embed(
+        self,
+        texts: List[str],
+        *,
+        timeout: float = 60.0,
+    ) -> List[List[float]]:
+        ...
+
+
 class OpenAICompatClient:
     """OpenAI-compatible chat completions client.
 
@@ -63,12 +82,16 @@ class OpenAICompatClient:
         base_url: str,
         api_key: str,
         model: str,
+        embedding_model: str = 'text-embedding-3-small',
         default_max_tokens: int = 2048,
         timeout: float = 60.0,
     ) -> None:
         self.base_url = base_url.rstrip('/')
         self.api_key = api_key
         self.model = model
+        # Embeddings hit a separate endpoint with a separate model id; default
+        # to OpenAI's small embedding model since the gateway is OpenAI-compat.
+        self.embedding_model = embedding_model
         self.default_max_tokens = default_max_tokens
         self.timeout = timeout
 
@@ -132,6 +155,40 @@ class OpenAICompatClient:
         return CompletionResult(text=text, raw=data)
 
 
+    def embed(
+        self,
+        texts: List[str],
+        *,
+        timeout: Optional[float] = None,
+    ) -> List[List[float]]:
+        """OpenAI-compatible ``/v1/embeddings`` POST.
+
+        Returns a list of vectors (one per input text) in the same order.
+        Empty ``texts`` short-circuits to ``[]`` to save a network round-trip.
+        """
+        if not texts:
+            return []
+        url = f'{self.base_url}/embeddings'
+        headers = {
+            'Authorization': f'Bearer {self.api_key}',
+            'Content-Type': 'application/json',
+        }
+        body = {'model': self.embedding_model, 'input': list(texts)}
+        resp = httpx.post(
+            url, headers=headers, json=body, timeout=timeout or self.timeout
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        rows = data.get('data') or []
+        out: List[List[float]] = []
+        for row in rows:
+            vec = row.get('embedding')
+            if not isinstance(vec, list):
+                continue
+            out.append([float(v) for v in vec])
+        return out
+
+
 def extract_json_block(text: str) -> Optional[Dict[str, Any]]:
     """Extract the first top-level JSON object from a possibly-fenced response."""
     if not text: