spanforge 2.0.0__py3-none-any.whl

spanforge/auto.py

@@ -0,0 +1,181 @@
+"""spanforge.auto — Automatic integration discovery and patching.
+
+Call :func:`setup` to automatically detect and patch all SpanForge-supported
+LLM libraries that are installed in the current environment.  This eliminates
+the need to manually import each integration module.
+
+Usage \u2014 fastest path to value::
+
+    import spanforge.auto
+    spanforge.auto.setup()  # patches everything installed
+
+Or call explicitly for programmatic control::
+
+    from spanforge.auto import setup
+    patched = setup(verbose=True)
+    # patched = {"openai", "anthropic"}
+
+Note
+----
+:func:`setup` is **not** called automatically on import.  You must call it
+explicitly so that importing :mod:`spanforge` never silently monkey-patches
+third-party libraries without your consent.
+
+Supported libraries (patched when installed):
+    * **openai** — :mod:`spanforge.integrations.openai`
+    * **anthropic** — :mod:`spanforge.integrations.anthropic`
+    * **groq** — :mod:`spanforge.integrations.groq`
+    * **ollama** — :mod:`spanforge.integrations.ollama`
+    * **together** — :mod:`spanforge.integrations.together`
+
+Callback-based integrations (register manually):
+    * **LangChain** — use :class:`~spanforge.integrations.langchain.LLMSchemaCallbackHandler`
+    * **LlamaIndex** — use :class:`~spanforge.integrations.llamaindex.LLMSchemaEventHandler`
+    * **CrewAI** — use :func:`~spanforge.integrations.crewai.patch`
+
+Security note
+-------------
+Monkey-patching is only applied when the target library is already installed.
+The patching flag ``_spanforge_patched`` prevents double-patching.  Each
+integration is wrapped in a ``try/except`` so a broken integration never
+prevents the others from loading.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import threading
+import warnings
+
+__all__ = ["setup", "teardown", "patched_integrations"]
+
+# Internal registry of successfully patched integrations (module name → patch fn).
+_PATCHED: set[str] = set()
+_PATCHED_LOCK = threading.Lock()
+
+# Map of library import name → (integration module path, patch fn name, unpatch fn name)
+_INTEGRATIONS: list[tuple[str, str, str, str]] = [
+    ("openai", "spanforge.integrations.openai", "patch", "unpatch"),
+    ("anthropic", "spanforge.integrations.anthropic", "patch", "unpatch"),
+    ("groq", "spanforge.integrations.groq", "patch", "unpatch"),
+    ("ollama", "spanforge.integrations.ollama", "patch", "unpatch"),
+    ("together", "spanforge.integrations.together", "patch", "unpatch"),
+]
+
+
+def _try_patch_integration(lib_name: str, integration_module: str, patch_fn: str, verbose: bool) -> bool:
+    """Attempt to patch one integration; returns True if newly patched."""
+    try:
+        mod = importlib.import_module(integration_module)
+        getattr(mod, patch_fn)()
+        _PATCHED.add(lib_name)
+        if verbose:
+            print(f"  {lib_name}: patched \u2713")
+        return True
+    except Exception as exc:
+        warnings.warn(
+            f"spanforge.auto: failed to patch {lib_name!r}: {exc}",
+            UserWarning,
+            stacklevel=3,
+        )
+        if verbose:
+            print(f"  {lib_name}: patch failed — {exc}")
+        return False
+
+
+def setup(*, verbose: bool = False) -> set[str]:
+    """Detect and patch all installed SpanForge-supported LLM libraries.
+
+    Iterates over supported integrations and calls their ``patch()`` function
+    if the underlying library is installed.  Already-patched integrations are
+    skipped silently (idempotent).
+
+    Args:
+        verbose: When ``True``, print a status line for each integration
+                 attempted.
+
+    Returns:
+        Set of library names that were newly patched in this call (does not
+        include libraries already patched in previous calls).
+
+    Example::
+
+        from spanforge.auto import setup
+        patched = setup(verbose=True)
+        # openai patched ✓
+        # anthropic not installed, skipped
+
+    Note:
+        Callback-based integrations (LangChain, LlamaIndex, CrewAI) are not
+        auto-patched because they require manual handler registration.  See
+        their respective integration guides.
+    """
+    newly_patched: set[str] = set()
+
+    for lib_name, integration_module, patch_fn, _unpatch_fn in _INTEGRATIONS:
+        if lib_name in _PATCHED:
+            if verbose:
+                print(f"  {lib_name}: already patched, skipped")
+            continue
+
+        if importlib.util.find_spec(lib_name) is None:
+            if verbose:
+                print(f"  {lib_name}: not installed, skipped")
+            continue
+
+        if _try_patch_integration(lib_name, integration_module, patch_fn, verbose):
+            newly_patched.add(lib_name)
+
+    return newly_patched
+
+
+def teardown(*, verbose: bool = False) -> set[str]:
+    """Unpatch all auto-patched integrations and reset the auto-patch registry.
+
+    Calls ``unpatch()`` on every integration that was patched via
+    :func:`setup`.  Safe to call even if :func:`setup` was never called.
+
+    Args:
+        verbose: When ``True``, print a status line for each integration.
+
+    Returns:
+        Set of library names that were unpatched.
+    """
+    unpatched: set[str] = set()
+
+    for lib_name, integration_module, _patch_fn, unpatch_fn in _INTEGRATIONS:
+        with _PATCHED_LOCK:
+            if lib_name not in _PATCHED:
+                continue
+        try:
+            mod = importlib.import_module(integration_module)
+            getattr(mod, unpatch_fn)()
+            with _PATCHED_LOCK:
+                _PATCHED.discard(lib_name)
+            unpatched.add(lib_name)
+            if verbose:
+                print(f"  {lib_name}: unpatched \u2713")
+        except Exception as exc:
+            warnings.warn(
+                f"spanforge.auto: failed to unpatch {lib_name!r}: {exc}",
+                UserWarning,
+                stacklevel=2,
+            )
+
+    return unpatched
+
+
+def patched_integrations() -> set[str]:
+    """Return the set of library names currently patched via :func:`setup`.
+
+    Returns:
+        Snapshot of the currently patched integration names.
+    """
+    with _PATCHED_LOCK:
+        return set(_PATCHED)
+
+
+# NOTE: setup() is NOT called automatically on import.
+# Call spanforge.auto.setup() explicitly to patch installed integrations.
+# This is intentional: importing spanforge should never monkey-patch
+# third-party libraries without explicit user consent.

spanforge/baseline.py

@@ -0,0 +1,336 @@
+"""spanforge.baseline — Behavioural baseline construction for drift detection.
+
+:class:`BehaviouralBaseline` captures the statistical summary of an agent's
+typical behaviour over an initial traffic window (default: up to 1 000 events
+or 24 hours).  The baseline is serialisable to JSON so it can be persisted and
+reloaded across restarts.
+
+Usage::
+
+    from spanforge.baseline import BehaviouralBaseline
+    from spanforge.stream import iter_file
+
+    events = list(iter_file("events.jsonl"))
+    baseline = BehaviouralBaseline.from_events(events)
+    baseline.save("baseline.json")
+
+    # — on restart —
+    baseline = BehaviouralBaseline.load("baseline.json")
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import pathlib
+import statistics
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Iterable
+
+if TYPE_CHECKING:
+    from spanforge.event import Event
+
+__all__ = ["BehaviouralBaseline", "DistributionStats"]
+
+
+# ---------------------------------------------------------------------------
+# Statistical helpers
+# ---------------------------------------------------------------------------
+
+
+def _percentile(sorted_data: list[float], pct: float) -> float:
+    """Return the *pct*-th percentile of an already-sorted list."""
+    if not sorted_data:
+        return 0.0
+    if len(sorted_data) == 1:
+        return float(sorted_data[0])
+    idx = (pct / 100.0) * (len(sorted_data) - 1)
+    lo = int(idx)
+    hi = lo + 1
+    if hi >= len(sorted_data):
+        return float(sorted_data[-1])
+    frac = idx - lo
+    return sorted_data[lo] * (1.0 - frac) + sorted_data[hi] * frac
+
+
+def _event_type_str(event: "Event") -> str:
+    et = event.event_type
+    return et.value if hasattr(et, "value") else str(et)
+
+
+# ---------------------------------------------------------------------------
+# Value object
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class DistributionStats:
+    """Mean, standard deviation, and percentiles for a numeric metric.
+
+    Attributes:
+        mean:         Arithmetic mean of the sample population.
+        stddev:       Sample standard deviation (0.0 when fewer than 2 samples).
+        p50:          50th percentile (median).
+        p95:          95th percentile.
+        p99:          99th percentile.
+        sample_count: Number of observations used to compute the statistics.
+    """
+
+    mean: float
+    stddev: float
+    p50: float
+    p95: float
+    p99: float
+    sample_count: int
+
+    # ------------------------------------------------------------------
+    # Factory
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def from_samples(cls, samples: list[float]) -> "DistributionStats":
+        """Build a :class:`DistributionStats` from a list of observations."""
+        if not samples:
+            return cls(mean=0.0, stddev=0.0, p50=0.0, p95=0.0, p99=0.0, sample_count=0)
+        s = sorted(samples)
+        mean = statistics.mean(s)
+        stddev = statistics.stdev(s) if len(s) >= 2 else 0.0
+        return cls(
+            mean=mean,
+            stddev=stddev,
+            p50=_percentile(s, 50),
+            p95=_percentile(s, 95),
+            p99=_percentile(s, 99),
+            sample_count=len(s),
+        )
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "mean": self.mean,
+            "stddev": self.stddev,
+            "p50": self.p50,
+            "p95": self.p95,
+            "p99": self.p99,
+            "sample_count": self.sample_count,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "DistributionStats":
+        return cls(
+            mean=float(d["mean"]),
+            stddev=float(d["stddev"]),
+            p50=float(d["p50"]),
+            p95=float(d["p95"]),
+            p99=float(d["p99"]),
+            sample_count=int(d["sample_count"]),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Baseline
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class BehaviouralBaseline:
+    """Statistical summary of an agent's typical behaviour.
+
+    Built from an initial traffic window and used by :class:`~spanforge.drift.DriftDetector`
+    to detect statistically significant deviations at runtime.
+
+    Attributes:
+        tokens:                  Token count distribution across all LLM spans.
+        confidence_by_type:      Per-decision-type confidence score distributions.
+        latency_by_operation:    Per-operation latency distributions (milliseconds).
+        tool_rate_per_hour:      Observed tool invocation rate per tool name (calls/h).
+        decision_rate_per_hour:  Observed decision rate per decision type (decisions/h).
+        event_count:             Number of events consumed to build this baseline.
+        window_seconds:          Duration of the baseline traffic window in seconds.
+        recorded_at:             ISO 8601 UTC timestamp when the baseline was created.
+    """
+
+    tokens: DistributionStats
+    confidence_by_type: dict[str, DistributionStats] = field(default_factory=dict)
+    latency_by_operation: dict[str, DistributionStats] = field(default_factory=dict)
+    tool_rate_per_hour: dict[str, float] = field(default_factory=dict)
+    decision_rate_per_hour: dict[str, float] = field(default_factory=dict)
+    event_count: int = 0
+    window_seconds: float = 86400.0
+    recorded_at: str = ""
+
+    # ------------------------------------------------------------------
+    # Factory
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def from_events(
+        cls,
+        events: Iterable["Event"],
+        max_events: int = 1000,
+        window_seconds: float = 86400.0,
+    ) -> "BehaviouralBaseline":
+        """Build a baseline from a stream of events.
+
+        Consumes at most *max_events* events from *events* (or the whole
+        iterable, whichever comes first) and computes statistical distributions
+        for the following metric groups:
+
+        - **Tokens** — total token count from ``llm.trace.span.completed``
+          payloads that contain a ``token_usage`` dict.
+        - **Confidence** — per-decision-type score from ``confidence.sample``
+          events.
+        - **Latency** — per-operation latency from ``llm.trace.span.completed``,
+          ``tool_call.*``, and ``latency.sample`` events.
+        - **Tool invocation rates** — calls per hour from ``tool_call.*`` events.
+        - **Decision rates** — decisions per hour from ``decision.made`` events.
+
+        Args:
+            events:         Source iterable of :class:`~spanforge.event.Event`.
+            max_events:     Upper bound on events consumed (default 1 000).
+            window_seconds: Denominator for rate calculations (default 86 400 s = 24 h).
+
+        Returns:
+            A fully-populated :class:`BehaviouralBaseline`.
+        """
+        token_samples: list[float] = []
+        confidence_samples: dict[str, list[float]] = {}
+        latency_samples: dict[str, list[float]] = {}
+        tool_counts: dict[str, int] = {}
+        decision_counts: dict[str, int] = {}
+
+        count = 0
+        for event in events:
+            if count >= max_events:
+                break
+            count += 1
+            etype = _event_type_str(event)
+            payload = event.payload
+
+            # LLM span events — tokens + latency
+            if etype in ("llm.trace.span.completed", "llm.trace.span.failed"):
+                tu = payload.get("token_usage")
+                if tu:
+                    total = int(tu.get("total_tokens", 0) or 0)
+                    if total > 0:
+                        token_samples.append(float(total))
+                dur = payload.get("duration_ms")
+                if dur is not None:
+                    op = str(payload.get("operation", "unknown"))
+                    latency_samples.setdefault(op, []).append(float(dur))
+                    if op == "tool_call":
+                        tool_counts[op] = tool_counts.get(op, 0) + 1
+
+            # Confidence namespace events
+            elif etype == "confidence.sample":
+                dtype = str(payload.get("decision_type", "unknown"))
+                score = payload.get("score")
+                if score is not None:
+                    confidence_samples.setdefault(dtype, []).append(float(score))
+
+            # Decision namespace events
+            elif etype == "decision.made":
+                dtype = str(payload.get("decision_type", "unknown"))
+                decision_counts[dtype] = decision_counts.get(dtype, 0) + 1
+
+            # Tool call namespace events
+            elif etype.startswith("tool_call."):
+                tool_name = str(payload.get("tool_name", "unknown"))
+                tool_counts[tool_name] = tool_counts.get(tool_name, 0) + 1
+                lat = payload.get("latency_ms")
+                if lat is not None:
+                    latency_samples.setdefault(tool_name, []).append(float(lat))
+
+            # Latency namespace events
+            elif etype == "latency.sample":
+                op = str(payload.get("operation", "unknown"))
+                lat = payload.get("latency_ms")
+                if lat is not None:
+                    latency_samples.setdefault(op, []).append(float(lat))
+
+        hours = (window_seconds / 3600.0) if window_seconds > 0 else 1.0
+
+        return cls(
+            tokens=DistributionStats.from_samples(token_samples),
+            confidence_by_type={
+                dt: DistributionStats.from_samples(samples)
+                for dt, samples in confidence_samples.items()
+            },
+            latency_by_operation={
+                op: DistributionStats.from_samples(samples)
+                for op, samples in latency_samples.items()
+            },
+            tool_rate_per_hour={
+                op: cnt / hours for op, cnt in tool_counts.items()
+            },
+            decision_rate_per_hour={
+                dt: cnt / hours for dt, cnt in decision_counts.items()
+            },
+            event_count=count,
+            window_seconds=window_seconds,
+            recorded_at=datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z",
+        )
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "tokens": self.tokens.to_dict(),
+            "confidence_by_type": {
+                k: v.to_dict() for k, v in self.confidence_by_type.items()
+            },
+            "latency_by_operation": {
+                k: v.to_dict() for k, v in self.latency_by_operation.items()
+            },
+            "tool_rate_per_hour": dict(self.tool_rate_per_hour),
+            "decision_rate_per_hour": dict(self.decision_rate_per_hour),
+            "event_count": self.event_count,
+            "window_seconds": self.window_seconds,
+            "recorded_at": self.recorded_at,
+        }
+
+    def to_json(self) -> str:
+        """Serialise to a compact JSON string (keys sorted)."""
+        return json.dumps(self.to_dict(), sort_keys=True, indent=2)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "BehaviouralBaseline":
+        return cls(
+            tokens=DistributionStats.from_dict(d["tokens"]),
+            confidence_by_type={
+                k: DistributionStats.from_dict(v)
+                for k, v in d.get("confidence_by_type", {}).items()
+            },
+            latency_by_operation={
+                k: DistributionStats.from_dict(v)
+                for k, v in d.get("latency_by_operation", {}).items()
+            },
+            tool_rate_per_hour={
+                k: float(v) for k, v in d.get("tool_rate_per_hour", {}).items()
+            },
+            decision_rate_per_hour={
+                k: float(v) for k, v in d.get("decision_rate_per_hour", {}).items()
+            },
+            event_count=int(d.get("event_count", 0)),
+            window_seconds=float(d.get("window_seconds", 86400.0)),
+            recorded_at=str(d.get("recorded_at", "")),
+        )
+
+    @classmethod
+    def from_json(cls, s: str) -> "BehaviouralBaseline":
+        """Deserialise from a JSON string produced by :meth:`to_json`."""
+        return cls.from_dict(json.loads(s))
+
+    def save(self, path: str | pathlib.Path) -> None:
+        """Write the baseline to *path* as UTF-8 JSON."""
+        pathlib.Path(path).write_text(self.to_json(), encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: str | pathlib.Path) -> "BehaviouralBaseline":
+        """Load a baseline previously saved with :meth:`save`."""
+        return cls.from_json(pathlib.Path(path).read_text(encoding="utf-8"))