agentdebugx 0.2.7__tar.gz → 0.2.8__tar.gz

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentdebugx
-Version: 0.2.7
+Version: 0.2.8
 Summary: Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`.
 License: MIT
 License-File: LICENSE

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/docs/22_industry_track_paper_eval_plan.md

@@ -37,14 +37,14 @@ Examples worth emulating:
 
 For AgentDebugX, the paper should therefore be framed as:
 
-> A deployment-oriented debugging layer for agentic NLP systems, evaluated on
+> A deployment-oriented debugging framework for agentic NLP systems, evaluated on
 > whether it makes failures observable, attributable, shareable, and easier for
 > humans to fix.
 
 The central claim should not be "we beat every attributor." A stronger claim
 for the Industry Track is:
 
-> AgentDebugX provides the missing operating layer between raw agent traces and
+> AgentDebugX provides the missing operational bridge between raw agent traces and
 > actionable debugging workflows: aligned native/error traces, taxonomy-backed
 > reports, Error Hub bundles, and cost-aware analysis profiles.
 

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/docs/23_status_v0_2.md

@@ -22,6 +22,7 @@ the forward-looking plan; this doc is the rear-view mirror.
 | Attribution | `agentdebug.attribution.StepByStepAttributor` | ✅ **new 0.2.2** | scripted-LLM + fallback |
 | Attribution | `agentdebug.attribution.BinarySearchAttributor` | ✅ **new 0.2.3** | oracle-LLM logarithmic convergence + fallback + render elision |
 | Attribution | `agentdebug.attribution.CounterfactualAttributor` | ✅ **new 0.2.7** | scripted-rescue-prob ranking + candidate selection priority (findings → errors → tail) + dual fallback (no candidates / silent LLM) |
+| Attribution | `agentdebug.attribution.SBFLAttributor` (Tarantula / Ochiai / DStar) | ✅ **new 0.2.8** | signature design + ranking on synthetic corpus + 3 formulas + dual fallback + edge cases. Model-free: zero LLM cost at inference. |
 | Recovery | `agentdebug.recovery.ReflexionSuggestion` | ✅ stable | per-finding + empty |
 | Recovery | `agentdebug.recovery.CriticRecoverer` + `VerifierSpec` registry | ✅ **new 0.2.3** | 5 family-matched verifier templates; dedup + custom-override |
 | DeepDebug | `agentdebug.deep.DeepDebugAnalyzer` | ✅ stable | full loop + silent LLM |
@@ -49,7 +50,7 @@ across 32 source files.
 | [06_detectors.md](./06_detectors.md) | `topic_drift` (embedding cosine) | needs embedding client; consider reusing `OpenAICompatClient` `/embeddings` | v0.3 |
 | [06_detectors.md](./06_detectors.md) | LTL spec monitors | requires user-supplied spec or LLM-synthesized monitors; gated on RV research | v1.2 |
 | [07_attribution.md](./07_attribution.md) | `CounterfactualAttributor` — *real* replay variant | true re-rollout requires framework-specific replay surface; the v0.2.7 LLM-simulated variant ships now, the real-replay variant is gated on adapter support (LangGraph checkpointer / OpenHands rewind) | v0.4 |
-| [07_attribution.md](./07_attribution.md) | `SBFLAttributor` (Tarantula/Ochiai) | needs corpus of passing + failing traces of same task; gated on Hub adoption | v0.4 |
+| [07_attribution.md](./07_attribution.md) | `SBFLAttributor` — *corpus* | shipped in 0.2.8 (`tarantula`/`ochiai`/`dstar`); awaiting paired-trace adoption to gather a useful corpus in production | corpus tooling deferred to v0.4 |
 | [07_attribution.md](./07_attribution.md) | `DeltaDebugAttributor` (Zeller) | same replay constraint | v0.3 |
 | [07_attribution.md](./07_attribution.md) | `EnsembleAttributor` | trivial once Counterfactual lands; awaits Counterfactual | v0.3 |
 | [08_recovery.md](./08_recovery.md) | `SelfRefineLoop` | small but needs a generator-critic-refiner orchestration | v0.3 |

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentdebugx"
-version = "0.2.7"
+version = "0.2.8"
 description = "Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`."
 authors = ["ULab @ UIUC <ulab@illinois.edu>"]
 license = "MIT"

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/src/agentdebug/__init__.py

@@ -17,6 +17,7 @@ from agentdebug.attribution import (
     Blame,
     CounterfactualAttributor,
     HeuristicAttributor,
+    SBFLAttributor,
     StepByStepAttributor,
 )
 from agentdebug.detectors import (
@@ -71,6 +72,7 @@ __all__ = [
     'DetectorConfig',
     'RepeatedStateDetector',
     'RepeatedToolCallDetector',
+    'SBFLAttributor',
     'StepByStepAttributor',
     'StepCountLimitDetector',
     'VerifierSpec',
@@ -98,4 +100,4 @@ __all__ = [
     'get_failure_mode',
 ]
 
-__version__ = '0.2.7'
+__version__ = '0.2.8'

{agentdebugx-0.2.7 → agentdebugx-0.2.8}/src/agentdebug/attribution.py

@@ -751,8 +751,193 @@ class CounterfactualAttributor:
             return default
 
 
+class SBFLAttributor:
+    """Spectrum-based fault localization (Tarantula / Ochiai / DStar).
+
+    Model-free. Costs ZERO LLM calls at inference time. The price: it needs
+    a corpus of *other* traces of the same task (a mix of passing and
+    failing). For each step in the failing trace under attribution, compute
+    suspiciousness from how often that step's signature appears in failing
+    vs passing traces of the same task.
+
+    Signature design — the cross-trace identity of a "step":
+
+      (agent_name, event_type, module, normalized_io_hash)
+
+    The normalized I/O hash collapses whitespace + truncates to 120 chars
+    so semantically-equivalent step executions match across runs (a
+    `search` call with the same query produces the same signature even if
+    the wrapping prompt differs trace-to-trace).
+
+    Suspiciousness formulas:
+
+      Tarantula = (ef / nf) / (ef / nf + ep / np)
+      Ochiai    = ef / sqrt((ef + ep) * (nf))
+      DStar*    = ef^* / (ep + (nf - ef))     [* = exponent, default 2]
+
+    where for the candidate step:
+      ef = # failing corpus traces containing this signature
+      ep = # passing corpus traces containing this signature
+      nf = total failing traces in the corpus
+      np = total passing traces in the corpus
+
+    Reference: Jones & Harrold (Tarantula, ICSE 2002); Abreu et al.
+    (Ochiai, TR 2007); Wong et al. (DStar, IEEE TR 2014).
+    """
+
+    id = 'sbfl'
+
+    def __init__(
+        self,
+        *,
+        passing_corpus: List[AgentTrajectory],
+        failing_corpus: List[AgentTrajectory],
+        formula: str = 'ochiai',
+        dstar_exponent: float = 2.0,
+        top_k: int = 5,
+        fallback: Optional[Attributor] = None,
+    ) -> None:
+        if formula not in {'tarantula', 'ochiai', 'dstar'}:
+            raise ValueError(
+                f"Unknown SBFL formula {formula!r}; "
+                "use 'tarantula', 'ochiai', or 'dstar'"
+            )
+        self.passing_corpus = list(passing_corpus)
+        self.failing_corpus = list(failing_corpus)
+        self.formula = formula
+        self.dstar_exponent = dstar_exponent
+        self.top_k = top_k
+        self.fallback: Attributor = fallback or HeuristicAttributor()
+        # Precompute signature presence per trace so attribution is O(N) over
+        # the candidate's events × O(corpus_size).
+        self._sigs_in_failing: List[set[str]] = [
+            _signatures_for(t) for t in self.failing_corpus
+        ]
+        self._sigs_in_passing: List[set[str]] = [
+            _signatures_for(t) for t in self.passing_corpus
+        ]
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: List[FailureFinding],
+    ) -> AttributionResult:
+        nf = len(self._sigs_in_failing)
+        np = len(self._sigs_in_passing)
+        if nf == 0 and np == 0:
+            return self.fallback.attribute(trajectory, findings)
+
+        scored: List[tuple[AgentEvent, float, int, int]] = []
+        for evt in trajectory.events:
+            sig = _signature_for(evt)
+            if not sig:
+                continue
+            ef = sum(1 for s in self._sigs_in_failing if sig in s)
+            ep = sum(1 for s in self._sigs_in_passing if sig in s)
+            score = _suspiciousness(
+                ef=ef, ep=ep, nf=max(nf, 1), np=max(np, 1),
+                formula=self.formula, dstar_exponent=self.dstar_exponent,
+            )
+            scored.append((evt, score, ef, ep))
+
+        if not scored:
+            return self.fallback.attribute(trajectory, findings)
+
+        # Sort by suspiciousness desc, tie-break by step index asc (earlier wins).
+        scored.sort(key=lambda r: (-r[1], r[0].step_index or 10**9))
+        hypotheses: List[Blame] = []
+        for evt, score, ef, ep in scored[: self.top_k]:
+            if score <= 0.0:
+                continue  # nothing suspicious about this step
+            hypotheses.append(Blame(
+                span_id=evt.event_id,
+                step_index=evt.step_index,
+                agent_name=evt.agent_name,
+                confidence=min(1.0, max(0.0, score)),
+                rationale=(
+                    f'{self.formula} suspiciousness={score:.3f} '
+                    f'(ef={ef}/{nf}, ep={ep}/{np})'
+                ),
+                evidence=[
+                    f'signature occurs in {ef}/{nf} failing and {ep}/{np} '
+                    f'passing corpus traces',
+                ],
+                sources=[self.id],
+            ))
+        if not hypotheses:
+            return self.fallback.attribute(trajectory, findings)
+        return AttributionResult(
+            method=self.id,
+            hypotheses=hypotheses,
+            raw={
+                'formula': self.formula,
+                'corpus_size': {'failing': nf, 'passing': np},
+                'scored_events': len(scored),
+            },
+        )
+
+
+def _signature_for(event: AgentEvent) -> Optional[str]:
+    """Cross-trace identity of a step. Returns None for events that should
+    not contribute to suspiciousness:
+
+      * ``run.start`` / ``run.end`` — trajectory lifecycle markers.
+      * ``error`` events emitted by ``agent_name='system'`` — these are the
+        synthetic markers from ``AgentDebug.finish_trace(success=False)`` and
+        are present in every failed trace; they would saturate suspiciousness
+        without revealing anything about the cause.
+    """
+    et = getattr(event.event_type, 'value', event.event_type)
+    if et in {'run.start', 'run.end'}:
+        return None
+    if et == 'error' and (event.agent_name or '') == 'system':
+        return None
+    payload = ' '.join(str(event.output or event.input or '').split())[:120]
+    return '|'.join([
+        str(et),
+        str(event.agent_name or ''),
+        str(event.module or ''),
+        payload,
+    ])
+
+
+def _signatures_for(trajectory: AgentTrajectory) -> set[str]:
+    out: set[str] = set()
+    for evt in trajectory.events:
+        sig = _signature_for(evt)
+        if sig:
+            out.add(sig)
+    return out
+
+
+def _suspiciousness(
+    *, ef: int, ep: int, nf: int, np: int,
+    formula: str, dstar_exponent: float,
+) -> float:
+    import math
+    if formula == 'tarantula':
+        # Defined as 0 when the step never appears in failing traces.
+        if ef == 0:
+            return 0.0
+        # Defensive: nf>=1 by caller.
+        f_rate = ef / nf
+        p_rate = (ep / np) if np > 0 else 0.0
+        denom = f_rate + p_rate
+        return f_rate / denom if denom > 0 else 0.0
+    if formula == 'ochiai':
+        denom = math.sqrt((ef + ep) * nf)
+        return (ef / denom) if denom > 0 else 0.0
+    # dstar
+    if ep == 0 and (nf - ef) == 0:
+        return float('inf') if ef > 0 else 0.0
+    denom = ep + (nf - ef)
+    if denom == 0:
+        return 0.0
+    return float((ef ** dstar_exponent) / denom)
+
+
 __all__ = [
     'Attributor', 'Blame', 'AttributionResult',
     'HeuristicAttributor', 'AllAtOnceAttributor', 'StepByStepAttributor',
-    'BinarySearchAttributor', 'CounterfactualAttributor',
+    'BinarySearchAttributor', 'CounterfactualAttributor', 'SBFLAttributor',
 ]