drift-detection 0.1.0__py3-none-any.whl

drift/__init__.py

@@ -0,0 +1,28 @@
+"""Drift — Statistical anomaly detection for AI agent workflows.
+
+Catch silent failures, hallucination drift, and off-script behavior
+in your LangChain, CrewAI, and custom AI agents.
+
+Quickstart:
+    from drift import DriftGuard
+    from drift.callbacks.langchain import DriftCallbackHandler
+
+    guard = DriftGuard()
+    handler = DriftCallbackHandler(guard)
+    agent.run("your query", callbacks=[handler])
+    guard.report()
+"""
+
+from drift.core import DriftGuard
+from drift.models import AgentEvent, Anomaly, AnomalyType, EventType, Severity
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DriftGuard",
+    "AgentEvent",
+    "Anomaly",
+    "AnomalyType",
+    "EventType",
+    "Severity",
+]

drift/callbacks/__init__.py

@@ -0,0 +1 @@
+"""Framework callback integrations for Drift."""

drift/callbacks/langchain.py

@@ -0,0 +1,243 @@
+"""LangChain callback handler for Drift.
+
+Integrates with LangChain's callback system to automatically capture
+agent events and feed them into the DriftGuard detection engine.
+
+Usage:
+    from drift import DriftGuard
+    from drift.callbacks.langchain import DriftCallbackHandler
+    
+    guard = DriftGuard()
+    handler = DriftCallbackHandler(guard)
+    
+    # Use with any LangChain agent/chain:
+    agent.run("your query", callbacks=[handler])
+    
+    # Check results:
+    guard.report()
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Optional
+from uuid import UUID
+
+from drift.core import DriftGuard
+from drift.models import AgentEvent, EventType
+
+try:
+    from langchain_core.callbacks import BaseCallbackHandler
+except ImportError:
+    raise ImportError(
+        "LangChain integration requires langchain-core. "
+        "Install it with: pip install driftguard[langchain]"
+    )
+
+
+class DriftCallbackHandler(BaseCallbackHandler):
+    """LangChain callback handler that feeds events into DriftGuard.
+    
+    Captures LLM calls, tool calls, and chain executions with timing
+    data and feeds them into the anomaly detection pipeline.
+    """
+
+    def __init__(self, guard: DriftGuard):
+        self.guard = guard
+        self._start_times: dict[str, float] = {}
+        self._event_names: dict[str, str] = {}
+        self._event_inputs: dict[str, dict[str, Any]] = {}
+
+    def _run_key(self, run_id: UUID) -> str:
+        return str(run_id)
+
+    # ---- LLM events ----
+
+    def on_llm_start(
+        self,
+        serialized: dict[str, Any],
+        prompts: list[str],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        self._start_times[key] = time.time()
+
+        name = serialized.get("id", ["unknown"])
+        if isinstance(name, list):
+            name = name[-1] if name else "unknown"
+        self._event_names[key] = str(name)
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.LLM_START,
+            name=str(name),
+            run_id=key,
+            parent_id=self._run_key(parent_run_id) if parent_run_id else None,
+            inputs={"prompts": prompts[:1]},  # Only first prompt to save memory
+        ))
+
+    def on_llm_end(
+        self,
+        response: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        start = self._start_times.pop(key, None)
+        latency_ms = (time.time() - start) * 1000 if start else None
+        name = self._event_names.pop(key, "unknown")
+
+        # Extract output text and token usage
+        output_text = ""
+        token_count = None
+        input_tokens = None
+        output_tokens = None
+
+        if hasattr(response, "generations") and response.generations:
+            gen = response.generations[0]
+            if gen:
+                output_text = gen[0].text if hasattr(gen[0], "text") else str(gen[0])
+
+        if hasattr(response, "llm_output") and response.llm_output:
+            usage = response.llm_output.get("token_usage", {})
+            input_tokens = usage.get("prompt_tokens")
+            output_tokens = usage.get("completion_tokens")
+            token_count = usage.get("total_tokens")
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.LLM_END,
+            name=name,
+            run_id=key,
+            parent_id=self._run_key(parent_run_id) if parent_run_id else None,
+            latency_ms=latency_ms,
+            output_text=output_text,
+            token_count=token_count,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+        ))
+
+    def on_llm_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        start = self._start_times.pop(key, None)
+        latency_ms = (time.time() - start) * 1000 if start else None
+        name = self._event_names.pop(key, "unknown")
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.ERROR,
+            name=name,
+            run_id=key,
+            latency_ms=latency_ms,
+            error=str(error),
+        ))
+
+    # ---- Tool events ----
+
+    def on_tool_start(
+        self,
+        serialized: dict[str, Any],
+        input_str: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        self._start_times[key] = time.time()
+
+        name = serialized.get("name", "unknown_tool")
+        self._event_names[key] = name
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.TOOL_START,
+            name=name,
+            run_id=key,
+            parent_id=self._run_key(parent_run_id) if parent_run_id else None,
+            inputs={"input": input_str[:500]},  # Truncate large inputs
+        ))
+
+    def on_tool_end(
+        self,
+        output: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        start = self._start_times.pop(key, None)
+        latency_ms = (time.time() - start) * 1000 if start else None
+        name = self._event_names.pop(key, "unknown_tool")
+
+        output_text = str(output)[:2000] if output else ""
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.TOOL_END,
+            name=name,
+            run_id=key,
+            parent_id=self._run_key(parent_run_id) if parent_run_id else None,
+            latency_ms=latency_ms,
+            output_text=output_text,
+        ))
+
+    def on_tool_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        start = self._start_times.pop(key, None)
+        latency_ms = (time.time() - start) * 1000 if start else None
+        name = self._event_names.pop(key, "unknown_tool")
+
+        self.guard.ingest(AgentEvent(
+            event_type=EventType.ERROR,
+            name=name,
+            run_id=key,
+            latency_ms=latency_ms,
+            error=str(error),
+        ))
+
+    # ---- Chain events ----
+
+    def on_chain_start(
+        self,
+        serialized: dict[str, Any],
+        inputs: dict[str, Any],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        self._start_times[key] = time.time()
+
+        name = serialized.get("id", ["unknown"])
+        if isinstance(name, list):
+            name = name[-1] if name else "unknown"
+        self._event_names[key] = str(name)
+
+    def on_chain_end(
+        self,
+        outputs: dict[str, Any],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> None:
+        key = self._run_key(run_id)
+        self._start_times.pop(key, None)
+        self._event_names.pop(key, None)

drift/core.py

@@ -0,0 +1,190 @@
+"""Core Drift engine — orchestrates detectors and manages the event stream."""
+
+from __future__ import annotations
+
+import threading
+from typing import Callable, Optional
+
+from drift.detectors import BaseDetector
+from drift.detectors.latency import LatencyDetector
+from drift.detectors.output_drift import OutputDriftDetector
+from drift.detectors.sequence import SequenceDetector
+from drift.models import AgentEvent, Anomaly, Severity
+
+
+# Type alias for anomaly callbacks
+AnomalyCallback = Callable[[Anomaly], None]
+
+
+class DriftGuard:
+    """Main entry point for Drift anomaly detection.
+    
+    Manages a set of detectors, ingests agent events, collects anomalies,
+    and optionally fires callbacks when anomalies are detected.
+    
+    Usage:
+        guard = DriftGuard()
+        
+        # Use with LangChain:
+        from drift.callbacks.langchain import DriftCallbackHandler
+        handler = DriftCallbackHandler(guard)
+        agent.run("query", callbacks=[handler])
+        
+        # Or ingest events directly:
+        guard.ingest(event)
+        
+        # Check results:
+        guard.report()
+    """
+
+    def __init__(
+        self,
+        detectors: list[BaseDetector] | None = None,
+        on_anomaly: AnomalyCallback | None = None,
+        min_severity: Severity = Severity.LOW,
+    ):
+        """Initialize DriftGuard.
+        
+        Args:
+            detectors: Custom list of detectors. If None, uses all defaults.
+            on_anomaly: Callback fired for each anomaly (e.g., log, alert, raise).
+            min_severity: Minimum severity to record/report. 
+        """
+        if detectors is None:
+            self.detectors: list[BaseDetector] = [
+                LatencyDetector(),
+                SequenceDetector(),
+                OutputDriftDetector(),
+            ]
+        else:
+            self.detectors = detectors
+
+        self.on_anomaly = on_anomaly
+        self.min_severity = min_severity
+        self._anomalies: list[Anomaly] = []
+        self._events: list[AgentEvent] = []
+        self._lock = threading.Lock()
+
+    def ingest(self, event: AgentEvent) -> list[Anomaly]:
+        """Process an event through all detectors.
+        
+        Args:
+            event: The agent event to analyze.
+            
+        Returns:
+            List of anomalies detected from this event.
+        """
+        new_anomalies: list[Anomaly] = []
+
+        with self._lock:
+            self._events.append(event)
+
+            for detector in self.detectors:
+                try:
+                    detected = detector.ingest(event)
+                    for anomaly in detected:
+                        if self._severity_value(anomaly.severity) >= self._severity_value(
+                            self.min_severity
+                        ):
+                            new_anomalies.append(anomaly)
+                            self._anomalies.append(anomaly)
+                except Exception as e:
+                    # Detectors should never crash the agent
+                    import sys
+                    print(
+                        f"[drift] Detector {detector.name!r} error: {e}",
+                        file=sys.stderr,
+                    )
+
+        # Fire callbacks outside the lock
+        if self.on_anomaly:
+            for anomaly in new_anomalies:
+                try:
+                    self.on_anomaly(anomaly)
+                except Exception:
+                    pass
+
+        return new_anomalies
+
+    @property
+    def anomalies(self) -> list[Anomaly]:
+        """All anomalies detected so far."""
+        return list(self._anomalies)
+
+    @property
+    def events(self) -> list[AgentEvent]:
+        """All events ingested so far."""
+        return list(self._events)
+
+    @property
+    def anomaly_count(self) -> int:
+        return len(self._anomalies)
+
+    @property
+    def event_count(self) -> int:
+        return len(self._events)
+
+    def report(self, verbose: bool = False) -> str:
+        """Generate a human-readable anomaly report.
+        
+        Args:
+            verbose: If True, include statistical context for each anomaly.
+            
+        Returns:
+            Formatted report string (also printed to stdout).
+        """
+        lines: list[str] = []
+        lines.append("")
+        lines.append("=" * 60)
+        lines.append("  DRIFT ANOMALY REPORT")
+        lines.append("=" * 60)
+        lines.append(f"  Events processed: {self.event_count}")
+        lines.append(f"  Anomalies found:  {self.anomaly_count}")
+
+        if not self._anomalies:
+            lines.append("")
+            lines.append("  ✓ No anomalies detected.")
+        else:
+            # Group by severity
+            by_severity: dict[Severity, list[Anomaly]] = {}
+            for a in self._anomalies:
+                by_severity.setdefault(a.severity, []).append(a)
+
+            for sev in [Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM, Severity.LOW]:
+                group = by_severity.get(sev, [])
+                if not group:
+                    continue
+                lines.append("")
+                lines.append(f"  [{sev.value.upper()}] ({len(group)})")
+                for a in group:
+                    lines.append(f"    • {a.anomaly_type.value}: {a.message}")
+                    if verbose and a.z_score is not None:
+                        lines.append(f"      z-score: {a.z_score:.2f}")
+                    if verbose and a.expected_range is not None:
+                        lo, hi = a.expected_range
+                        lines.append(f"      expected range: [{lo:.1f}, {hi:.1f}]")
+
+        lines.append("")
+        lines.append("=" * 60)
+        lines.append("")
+
+        report_text = "\n".join(lines)
+        print(report_text)
+        return report_text
+
+    def reset(self) -> None:
+        """Reset all state — detectors, events, anomalies."""
+        with self._lock:
+            for detector in self.detectors:
+                detector.reset()
+            self._anomalies.clear()
+            self._events.clear()
+
+    @staticmethod
+    def _severity_value(severity: Severity) -> int:
+        return {
+            Severity.LOW: 0,
+            Severity.MEDIUM: 1,
+            Severity.HIGH: 2,
+            Severity.CRITICAL: 3,
+        }[severity]

drift/detectors/__init__.py

@@ -0,0 +1,34 @@
+"""Base detector interface."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from drift.models import AgentEvent, Anomaly
+
+
+class BaseDetector(ABC):
+    """Abstract base class for all anomaly detectors."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable detector name."""
+        ...
+
+    @abstractmethod
+    def ingest(self, event: AgentEvent) -> list[Anomaly]:
+        """Process an event and return any anomalies detected.
+        
+        Args:
+            event: The agent event to analyze.
+            
+        Returns:
+            List of anomalies (empty if none detected).
+        """
+        ...
+
+    @abstractmethod
+    def reset(self) -> None:
+        """Reset detector state (clear baselines, history, etc.)."""
+        ...

drift/detectors/latency.py

@@ -0,0 +1,148 @@
+"""Latency and token count anomaly detection via statistical process control.
+
+Uses a rolling window to compute mean and standard deviation, then flags
+events whose latency or token count falls outside a configurable z-score
+threshold. This is the simplest and most immediately useful detector —
+it catches hung API calls, runaway generation, and upstream provider issues.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from drift.detectors import BaseDetector
+from drift.models import AgentEvent, Anomaly, AnomalyType, EventType, Severity
+
+
+@dataclass
+class LatencyDetectorConfig:
+    """Configuration for the latency/token detector."""
+    window_size: int = 50          # Rolling window for baseline stats
+    z_threshold: float = 3.0       # Standard deviations before flagging
+    min_samples: int = 5           # Minimum events before detection activates
+    critical_z: float = 5.0        # Z-score threshold for CRITICAL severity
+    track_tokens: bool = True      # Also monitor token counts
+
+
+class LatencyDetector(BaseDetector):
+    """Detects anomalous latency and token counts using rolling z-scores.
+    
+    Maintains per-tool and per-model baselines, so a slow tool won't
+    pollute the baseline for a fast one.
+    """
+
+    def __init__(self, config: LatencyDetectorConfig | None = None):
+        self.config = config or LatencyDetectorConfig()
+        # Keyed by event name (tool name or model name)
+        self._latency_windows: dict[str, list[float]] = defaultdict(list)
+        self._token_windows: dict[str, list[float]] = defaultdict(list)
+
+    @property
+    def name(self) -> str:
+        return "latency_detector"
+
+    def ingest(self, event: AgentEvent) -> list[Anomaly]:
+        anomalies: list[Anomaly] = []
+
+        # Only analyze completion events (they have latency data)
+        if event.event_type not in (EventType.LLM_END, EventType.TOOL_END):
+            return anomalies
+
+        key = event.name or "unknown"
+
+        # --- Latency check ---
+        if event.latency_ms is not None:
+            window = self._latency_windows[key]
+            anomaly = self._check_value(
+                value=event.latency_ms,
+                window=window,
+                metric_name="latency",
+                unit="ms",
+                anomaly_type=AnomalyType.LATENCY_SPIKE,
+                event=event,
+            )
+            if anomaly:
+                anomalies.append(anomaly)
+
+            # Update window
+            window.append(event.latency_ms)
+            if len(window) > self.config.window_size:
+                window.pop(0)
+
+        # --- Token count check ---
+        if self.config.track_tokens and event.token_count is not None:
+            window = self._token_windows[key]
+            anomaly = self._check_value(
+                value=float(event.token_count),
+                window=window,
+                metric_name="token_count",
+                unit="tokens",
+                anomaly_type=AnomalyType.TOKEN_ANOMALY,
+                event=event,
+            )
+            if anomaly:
+                anomalies.append(anomaly)
+
+            window.append(float(event.token_count))
+            if len(window) > self.config.window_size:
+                window.pop(0)
+
+        return anomalies
+
+    def _check_value(
+        self,
+        value: float,
+        window: list[float],
+        metric_name: str,
+        unit: str,
+        anomaly_type: AnomalyType,
+        event: AgentEvent,
+    ) -> Anomaly | None:
+        """Check a single metric against its rolling baseline."""
+        if len(window) < self.config.min_samples:
+            return None
+
+        arr = np.array(window)
+        mean = float(np.mean(arr))
+        std = float(np.std(arr))
+
+        if std < 1e-9:  # Constant values — can't compute z-score meaningfully
+            return None
+
+        z = (value - mean) / std
+
+        if abs(z) < self.config.z_threshold:
+            return None
+
+        # Determine severity
+        if abs(z) >= self.config.critical_z:
+            severity = Severity.CRITICAL
+        elif abs(z) >= self.config.z_threshold + 1:
+            severity = Severity.HIGH
+        else:
+            severity = Severity.MEDIUM
+
+        direction = "above" if z > 0 else "below"
+
+        return Anomaly(
+            anomaly_type=anomaly_type,
+            severity=severity,
+            message=(
+                f"{event.name!r} {metric_name} is {value:.1f}{unit}, "
+                f"{abs(z):.1f}σ {direction} mean ({mean:.1f}{unit} ± {std:.1f})"
+            ),
+            event=event,
+            detector_name=self.name,
+            observed_value=value,
+            expected_range=(mean - self.config.z_threshold * std, 
+                          mean + self.config.z_threshold * std),
+            z_score=z,
+            context={"window_size": len(window), "mean": mean, "std": std},
+        )
+
+    def reset(self) -> None:
+        self._latency_windows.clear()
+        self._token_windows.clear()

drift/detectors/output_drift.py

@@ -0,0 +1,213 @@
+"""Output drift detection.
+
+Tracks statistical properties of agent outputs over time and flags when
+the distribution shifts significantly. This catches hallucination drift,
+prompt injection effects, and gradual quality degradation.
+
+Uses lightweight heuristics (output length, vocabulary diversity, 
+structural patterns) rather than embeddings for zero-dependency operation.
+Embedding-based drift detection can be added as an optional enhancement.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+
+import numpy as np
+
+from drift.detectors import BaseDetector
+from drift.models import AgentEvent, Anomaly, AnomalyType, EventType, Severity
+
+
+@dataclass
+class OutputDriftConfig:
+    """Configuration for the output drift detector."""
+    window_size: int = 30            # Rolling window for baseline
+    min_samples: int = 8             # Minimum samples before detection
+    length_z_threshold: float = 3.0  # Z-score for output length anomaly
+    vocab_z_threshold: float = 3.0   # Z-score for vocabulary diversity anomaly
+    structure_change: bool = True    # Track structural pattern changes
+
+
+class OutputDriftDetector(BaseDetector):
+    """Detects drift in agent output characteristics.
+    
+    Tracks multiple lightweight signals per tool/model:
+    - Output length distribution
+    - Vocabulary diversity (unique words / total words)
+    - Structural patterns (presence of code blocks, JSON, lists, etc.)
+    
+    All signals are zero-dependency (no embedding models required).
+    """
+
+    def __init__(self, config: OutputDriftConfig | None = None):
+        self.config = config or OutputDriftConfig()
+        self._length_windows: dict[str, list[float]] = defaultdict(list)
+        self._vocab_windows: dict[str, list[float]] = defaultdict(list)
+        self._structure_counts: dict[str, dict[str, int]] = defaultdict(
+            lambda: defaultdict(int)
+        )
+        self._total_by_key: dict[str, int] = defaultdict(int)
+
+    @property
+    def name(self) -> str:
+        return "output_drift_detector"
+
+    def ingest(self, event: AgentEvent) -> list[Anomaly]:
+        anomalies: list[Anomaly] = []
+
+        if event.event_type not in (EventType.LLM_END, EventType.TOOL_END):
+            return anomalies
+
+        if not event.output_text:
+            return anomalies
+
+        key = event.name or "unknown"
+        text = event.output_text
+
+        # --- Length anomaly ---
+        length = float(len(text))
+        length_anomaly = self._check_zscore(
+            value=length,
+            window=self._length_windows[key],
+            metric="output_length",
+            unit="chars",
+            event=event,
+            z_threshold=self.config.length_z_threshold,
+        )
+        if length_anomaly:
+            anomalies.append(length_anomaly)
+        self._update_window(self._length_windows[key], length)
+
+        # --- Vocabulary diversity ---
+        words = text.lower().split()
+        if len(words) > 5:
+            diversity = len(set(words)) / len(words)
+            vocab_anomaly = self._check_zscore(
+                value=diversity,
+                window=self._vocab_windows[key],
+                metric="vocab_diversity",
+                unit="ratio",
+                event=event,
+                z_threshold=self.config.vocab_z_threshold,
+            )
+            if vocab_anomaly:
+                anomalies.append(vocab_anomaly)
+            self._update_window(self._vocab_windows[key], diversity)
+
+        # --- Structural pattern change ---
+        if self.config.structure_change:
+            structure = self._extract_structure(text)
+            struct_anomaly = self._check_structure(key, structure, event)
+            if struct_anomaly:
+                anomalies.append(struct_anomaly)
+
+        return anomalies
+
+    def _check_zscore(
+        self,
+        value: float,
+        window: list[float],
+        metric: str,
+        unit: str,
+        event: AgentEvent,
+        z_threshold: float,
+    ) -> Anomaly | None:
+        """Generic z-score check against a rolling window."""
+        if len(window) < self.config.min_samples:
+            return None
+
+        arr = np.array(window)
+        mean = float(np.mean(arr))
+        std = float(np.std(arr))
+
+        if std < 1e-9:
+            return None
+
+        z = (value - mean) / std
+        if abs(z) < z_threshold:
+            return None
+
+        severity = Severity.HIGH if abs(z) > z_threshold + 2 else Severity.MEDIUM
+        direction = "above" if z > 0 else "below"
+
+        return Anomaly(
+            anomaly_type=AnomalyType.OUTPUT_DRIFT,
+            severity=severity,
+            message=(
+                f"{event.name!r} {metric} is {value:.1f} {unit}, "
+                f"{abs(z):.1f}σ {direction} baseline ({mean:.1f} ± {std:.1f})"
+            ),
+            event=event,
+            detector_name=self.name,
+            observed_value=value,
+            expected_range=(mean - z_threshold * std, mean + z_threshold * std),
+            z_score=z,
+        )
+
+    def _extract_structure(self, text: str) -> str:
+        """Classify the structural pattern of an output."""
+        patterns: list[str] = []
+        if re.search(r"```", text):
+            patterns.append("code_block")
+        if re.search(r"[{\[].*[}\]]", text, re.DOTALL):
+            patterns.append("json_like")
+        if re.search(r"^\s*[-*]\s", text, re.MULTILINE):
+            patterns.append("bullet_list")
+        if re.search(r"^\s*\d+\.\s", text, re.MULTILINE):
+            patterns.append("numbered_list")
+        if re.search(r"^#+\s", text, re.MULTILINE):
+            patterns.append("markdown_headers")
+        if len(text.strip()) == 0:
+            patterns.append("empty")
+        return "|".join(sorted(patterns)) if patterns else "plain_text"
+
+    def _check_structure(
+        self, key: str, structure: str, event: AgentEvent
+    ) -> Anomaly | None:
+        """Flag if the output structure is novel for this tool/model."""
+        self._total_by_key[key] += 1
+        counts = self._structure_counts[key]
+
+        total = self._total_by_key[key]
+        if total < self.config.min_samples:
+            counts[structure] += 1
+            return None
+
+        # Check if this structure has been seen before
+        if structure not in counts:
+            anomaly = Anomaly(
+                anomaly_type=AnomalyType.OUTPUT_DRIFT,
+                severity=Severity.MEDIUM,
+                message=(
+                    f"{event.name!r} produced a novel output structure: {structure!r} "
+                    f"(previously seen: {list(counts.keys())})"
+                ),
+                event=event,
+                detector_name=self.name,
+                context={
+                    "novel_structure": structure,
+                    "known_structures": dict(counts),
+                    "total_observations": total,
+                },
+            )
+            counts[structure] += 1
+            return anomaly
+
+        counts[structure] += 1
+        return None
+
+    def _update_window(self, window: list[float], value: float) -> None:
+        """Append value and trim to window size."""
+        window.append(value)
+        if len(window) > self.config.window_size:
+            window.pop(0)
+
+    def reset(self) -> None:
+        self._length_windows.clear()
+        self._vocab_windows.clear()
+        self._structure_counts.clear()
+        self._total_by_key.clear()

drift/detectors/sequence.py

@@ -0,0 +1,153 @@
+"""Action sequence anomaly detection.
+
+Builds a transition probability matrix from observed tool/LLM call sequences
+and flags when the agent takes a path that has never or rarely been seen.
+This catches agents going "off-script" — calling tools in unexpected orders,
+skipping required steps, or entering novel execution paths.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+
+from drift.detectors import BaseDetector
+from drift.models import AgentEvent, Anomaly, AnomalyType, EventType, Severity
+
+
+@dataclass
+class SequenceDetectorConfig:
+    """Configuration for the sequence anomaly detector."""
+    min_observations: int = 10       # Min transitions before detection activates
+    novel_severity: Severity = Severity.HIGH    # Severity for never-seen transitions
+    rare_threshold: float = 0.02     # Transitions below this probability are flagged
+    rare_severity: Severity = Severity.MEDIUM   # Severity for rare transitions
+
+
+class SequenceDetector(BaseDetector):
+    """Detects anomalous action sequences by tracking transition probabilities.
+    
+    Maintains a first-order Markov transition matrix over tool/LLM call names.
+    When an agent takes a transition that's never been observed or is very rare
+    relative to the baseline, it fires an anomaly.
+    
+    Example: If your agent always calls search → parse → respond, and suddenly
+    calls search → delete → respond, the search→delete transition gets flagged.
+    """
+
+    def __init__(self, config: SequenceDetectorConfig | None = None):
+        self.config = config or SequenceDetectorConfig()
+        # transition_counts[from_action][to_action] = count
+        self._transition_counts: dict[str, dict[str, int]] = defaultdict(
+            lambda: defaultdict(int)
+        )
+        self._total_from: dict[str, int] = defaultdict(int)
+        self._last_action: str | None = None
+        self._total_transitions: int = 0
+
+    @property
+    def name(self) -> str:
+        return "sequence_detector"
+
+    def ingest(self, event: AgentEvent) -> list[Anomaly]:
+        anomalies: list[Anomaly] = []
+
+        # Only track start events (they represent the agent's decision to act)
+        if event.event_type not in (
+            EventType.TOOL_START, EventType.LLM_START, EventType.CHAIN_START
+        ):
+            return anomalies
+
+        current_action = event.name or "unknown"
+
+        if self._last_action is not None and self._total_transitions >= self.config.min_observations:
+            anomaly = self._check_transition(self._last_action, current_action, event)
+            if anomaly:
+                anomalies.append(anomaly)
+
+        # Update transition matrix
+        if self._last_action is not None:
+            self._transition_counts[self._last_action][current_action] += 1
+            self._total_from[self._last_action] += 1
+            self._total_transitions += 1
+
+        self._last_action = current_action
+        return anomalies
+
+    def _check_transition(
+        self, from_action: str, to_action: str, event: AgentEvent
+    ) -> Anomaly | None:
+        """Check if a transition is anomalous."""
+        from_counts = self._transition_counts.get(from_action)
+
+        # Case 1: We've never seen ANY transition from this action
+        if from_counts is None or self._total_from.get(from_action, 0) == 0:
+            return None  # Can't evaluate — not enough data for this source
+
+        total_from = self._total_from[from_action]
+        to_count = from_counts.get(to_action, 0)
+
+        # Case 2: Never-seen transition from a known source
+        if to_count == 0:
+            seen_targets = list(from_counts.keys())
+            return Anomaly(
+                anomaly_type=AnomalyType.SEQUENCE_ANOMALY,
+                severity=self.config.novel_severity,
+                message=(
+                    f"Novel transition: {from_action!r} → {to_action!r} "
+                    f"(never observed; known transitions from {from_action!r}: "
+                    f"{seen_targets})"
+                ),
+                event=event,
+                detector_name=self.name,
+                observed_value=0.0,
+                context={
+                    "from_action": from_action,
+                    "to_action": to_action,
+                    "known_transitions": dict(from_counts),
+                    "total_from_count": total_from,
+                },
+            )
+
+        # Case 3: Rare transition
+        probability = to_count / total_from
+        if probability < self.config.rare_threshold:
+            return Anomaly(
+                anomaly_type=AnomalyType.SEQUENCE_ANOMALY,
+                severity=self.config.rare_severity,
+                message=(
+                    f"Rare transition: {from_action!r} → {to_action!r} "
+                    f"(p={probability:.3f}, seen {to_count}/{total_from} times)"
+                ),
+                event=event,
+                detector_name=self.name,
+                observed_value=probability,
+                expected_range=(self.config.rare_threshold, 1.0),
+                context={
+                    "from_action": from_action,
+                    "to_action": to_action,
+                    "probability": probability,
+                    "count": to_count,
+                    "total_from_count": total_from,
+                },
+            )
+
+        return None
+
+    def get_transition_matrix(self) -> dict[str, dict[str, float]]:
+        """Return the current transition probability matrix (for debugging/viz)."""
+        matrix: dict[str, dict[str, float]] = {}
+        for from_action, targets in self._transition_counts.items():
+            total = self._total_from[from_action]
+            if total > 0:
+                matrix[from_action] = {
+                    to_action: count / total
+                    for to_action, count in targets.items()
+                }
+        return matrix
+
+    def reset(self) -> None:
+        self._transition_counts.clear()
+        self._total_from.clear()
+        self._last_action = None
+        self._total_transitions = 0

drift/models.py

@@ -0,0 +1,101 @@
+"""Data models for Drift agent events and anomalies."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Optional
+
+
+class EventType(Enum):
+    """Types of agent events we track."""
+    LLM_START = "llm_start"
+    LLM_END = "llm_end"
+    TOOL_START = "tool_start"
+    TOOL_END = "tool_end"
+    CHAIN_START = "chain_start"
+    CHAIN_END = "chain_end"
+    ERROR = "error"
+
+
+class AnomalyType(Enum):
+    """Categories of detected anomalies."""
+    LATENCY_SPIKE = "latency_spike"
+    TOKEN_ANOMALY = "token_anomaly"
+    SEQUENCE_ANOMALY = "sequence_anomaly"
+    OUTPUT_DRIFT = "output_drift"
+    ERROR_RATE = "error_rate"
+    COST_SPIKE = "cost_spike"
+
+
+class Severity(Enum):
+    """Anomaly severity levels."""
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+@dataclass
+class AgentEvent:
+    """A single event in an agent's execution trace."""
+    event_type: EventType
+    timestamp: float = field(default_factory=time.time)
+    event_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
+    run_id: Optional[str] = None
+
+    # What happened
+    name: str = ""  # tool name, llm model, chain name
+    inputs: Optional[dict[str, Any]] = None
+    outputs: Optional[dict[str, Any]] = None
+
+    # Measurements
+    latency_ms: Optional[float] = None
+    token_count: Optional[int] = None
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    cost_usd: Optional[float] = None
+
+    # Raw content for drift detection
+    output_text: Optional[str] = None
+    error: Optional[str] = None
+
+    # Parent event for nesting
+    parent_id: Optional[str] = None
+
+    def __repr__(self) -> str:
+        parts = [f"{self.event_type.value}({self.name!r}"]
+        if self.latency_ms is not None:
+            parts.append(f", latency={self.latency_ms:.0f}ms")
+        if self.token_count is not None:
+            parts.append(f", tokens={self.token_count}")
+        if self.error:
+            parts.append(f", error={self.error!r}")
+        return "".join(parts) + ")"
+
+
+@dataclass
+class Anomaly:
+    """A detected anomaly in agent behavior."""
+    anomaly_type: AnomalyType
+    severity: Severity
+    message: str
+    timestamp: float = field(default_factory=time.time)
+
+    # What triggered it
+    event: Optional[AgentEvent] = None
+    detector_name: str = ""
+
+    # Statistical context
+    observed_value: Optional[float] = None
+    expected_range: Optional[tuple[float, float]] = None
+    z_score: Optional[float] = None
+
+    # The raw data window that informed the detection
+    context: Optional[dict[str, Any]] = None
+
+    def __repr__(self) -> str:
+        sev = self.severity.value.upper()
+        return f"[{sev}] {self.anomaly_type.value}: {self.message}"

drift_detection-0.1.0.dist-info/METADATA

@@ -0,0 +1,181 @@
+Metadata-Version: 2.4
+Name: drift-detection
+Version: 0.1.0
+Summary: Statistical anomaly detection for AI agent workflows
+Project-URL: Homepage, https://github.com/dombinic/Drift
+Project-URL: Repository, https://github.com/dombinic/Drift
+Project-URL: Issues, https://github.com/dombinic/Drift/issues
+Author: Dominic
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,anomaly-detection,langchain,llm,monitoring,observability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.24.0
+Provides-Extra: all
+Requires-Dist: langchain-core>=0.1.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.1.0; extra == 'langchain'
+Description-Content-Type: text/markdown
+
+# Drift
+
+**Statistical anomaly detection for AI agent workflows.**
+
+Catch silent failures, hallucination drift, and off-script behavior before they corrupt your data.
+
+---
+
+Your AI agents are failing silently. A tool call takes 10x longer than usual. The agent calls `delete_file` when it's never done that before. Output quality gradually degrades over hundreds of runs. Traditional monitoring tools weren't built for non-deterministic systems — Drift is.
+
+## What it does
+
+Drift hooks into your agent's execution and applies statistical anomaly detection to the event stream:
+
+- **Latency & token SPC** — Flags when a tool call or LLM response takes significantly longer or uses significantly more tokens than its rolling baseline. Catches hung API calls, runaway generation, and upstream provider issues.
+
+- **Sequence anomaly detection** — Builds a transition matrix of tool-call sequences and flags when the agent takes a path that's never or rarely been seen. Catches agents going off-script, skipping required steps, or entering novel execution paths.
+
+- **Output drift detection** — Tracks output length, vocabulary diversity, and structural patterns over time. Flags when outputs shift significantly from baseline. Catches hallucination drift, prompt injection effects, and gradual quality degradation.
+
+## Install
+
+```bash
+pip install driftguard
+```
+
+With LangChain support:
+```bash
+pip install driftguard[langchain]
+```
+
+## Quickstart
+
+### With LangChain
+
+```python
+from drift import DriftGuard
+from drift.callbacks.langchain import DriftCallbackHandler
+
+guard = DriftGuard(on_anomaly=lambda a: print(f"🚨 {a}"))
+handler = DriftCallbackHandler(guard)
+
+# Use with any LangChain agent, chain, or LLM
+agent.run("your query", callbacks=[handler])
+
+# See what happened
+guard.report()
+```
+
+### Standalone (no framework required)
+
+```python
+from drift import DriftGuard, AgentEvent, EventType
+
+guard = DriftGuard(on_anomaly=lambda a: print(f"🚨 {a}"))
+
+# Feed events from any source
+guard.ingest(AgentEvent(
+    event_type=EventType.TOOL_END,
+    name="search_web",
+    latency_ms=150.0,
+    token_count=85,
+    output_text="Found 3 results for query...",
+))
+
+guard.report()
+```
+
+### Run the demo
+
+```bash
+python examples/demo.py
+```
+
+This simulates normal agent operation, builds baselines, then injects latency spikes, sequence anomalies, and output drift — showing each detector catching real failure modes.
+
+## Architecture
+
+```
+drift/
+├── core.py              # DriftGuard engine — orchestrates detectors
+├── models.py            # AgentEvent, Anomaly, Severity data models
+├── detectors/
+│   ├── latency.py       # Statistical process control on latency/tokens
+│   ├── sequence.py      # Action transition probability anomalies
+│   └── output_drift.py  # Output distribution shift detection
+└── callbacks/
+    └── langchain.py     # LangChain callback integration
+```
+
+**Design principles:**
+
+1. **Zero-overhead default** — Detectors use numpy for fast rolling statistics. No embedding models, no external services, no network calls.
+2. **Per-tool baselines** — Each tool and model gets its own statistical baseline, so a slow tool won't pollute the baseline for a fast one.
+3. **Framework-agnostic core** — The detection engine works with raw `AgentEvent` objects. Framework integrations (LangChain, CrewAI, etc.) are thin adapters that translate framework callbacks into events.
+4. **Non-blocking** — Drift never throws exceptions that would crash your agent. Detector errors are caught and logged to stderr.
+
+## Detectors
+
+| Detector | What it catches | Method |
+|----------|----------------|--------|
+| `LatencyDetector` | Hung calls, slow APIs, runaway generation | Rolling z-score on latency and token counts |
+| `SequenceDetector` | Off-script behavior, unexpected tool calls | First-order Markov transition probabilities |
+| `OutputDriftDetector` | Hallucination drift, prompt injection, quality degradation | Output length, vocab diversity, structural pattern tracking |
+
+## Configuration
+
+Each detector is independently configurable:
+
+```python
+from drift import DriftGuard
+from drift.detectors.latency import LatencyDetector, LatencyDetectorConfig
+from drift.detectors.sequence import SequenceDetector, SequenceDetectorConfig
+
+guard = DriftGuard(detectors=[
+    LatencyDetector(LatencyDetectorConfig(
+        window_size=100,     # Longer baseline window
+        z_threshold=2.5,     # More sensitive
+        min_samples=10,      # Require more data before alerting
+    )),
+    SequenceDetector(SequenceDetectorConfig(
+        min_observations=20, # Require more transitions before flagging
+    )),
+])
+```
+
+## Roadmap
+
+- [ ] CrewAI callback handler
+- [ ] OpenAI Agents SDK integration
+- [ ] Slack / PagerDuty alerting
+- [ ] Persistent baselines (save/load detector state)
+- [ ] Embedding-based output drift (optional dependency)
+- [ ] Web dashboard
+- [ ] Cost anomaly detection (track spend per run)
+
+## Contributing
+
+Issues and PRs welcome. Run tests with:
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT

drift_detection-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+drift/__init__.py,sha256=Yyh-NYOmurrocCTKgQe0kcTptJ6J_At38yQvKr7L7ws,689
+drift/core.py,sha256=F9H6LXOiU5luUhGVnK7LPhcwDWe3FLxlB-HTfocV1ms,6250
+drift/models.py,sha256=J8xpHWDxW8Qw6TrrP9WFsOt4y_2RiLhXTcIetOxVIYo,2820
+drift/callbacks/__init__.py,sha256=5qGB2nKy3iZXKMf5yDhit0TOwaq_HgLGjPJrDaL3Mts,49
+drift/callbacks/langchain.py,sha256=e-VnXf9_3LaJFDtDb-iF5SbME9hi1U52P4pggAWpYXU,7212
+drift/detectors/__init__.py,sha256=Dpmtfg911Z2o-CJ2O8KTBgZPukI6dgf9IKA-iYOJQds,805
+drift/detectors/latency.py,sha256=JgZx-oDZ4KIMUggPTVeOn2nwK5MxbdEZapWQ2BUry6k,5047
+drift/detectors/output_drift.py,sha256=uzaHoRNYubQOHr5N5Rb7ETvIgUOEm3GUiYzZ8NY26iw,7461
+drift/detectors/sequence.py,sha256=0CuBQEX4DPR4sCet4RFGRw3p_y9Me_38r_fPRSALhk4,6112
+drift_detection-0.1.0.dist-info/METADATA,sha256=3Fc8FXTWEClYk1lTiUOw_UXC7gPqgqMeBju5L0BZsrA,6488
+drift_detection-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+drift_detection-0.1.0.dist-info/licenses/LICENSE,sha256=ZbH4VA0-TPt-OXg5a8JPLpKNj6zzckRf4sZx-rxomjE,1065
+drift_detection-0.1.0.dist-info/RECORD,,

drift_detection-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

drift_detection-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 dombinic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.