driftvane 0.1.0__tar.gz

driftvane-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+*.egg
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.tox/
+.mypy_cache/
+.venv/
+venv/
+env/
+.idea/
+.vscode/
+.DS_Store

driftvane-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mukunda Rao Katta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

driftvane-0.1.0/PKG-INFO

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: driftvane
+Version: 0.1.0
+Summary: Compose drift detectors (embedding, retrieval, response, latency) into one report. Library-only, no server, no UI.
+Project-URL: Homepage, https://github.com/MukundaKatta/driftvane
+Project-URL: Issues, https://github.com/MukundaKatta/driftvane/issues
+Project-URL: Source, https://github.com/MukundaKatta/driftvane
+Author-email: Mukunda Rao Katta <mukunda.vjcs6@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,drift,embedding-drift,evals,llm,mlops,monitoring,rag,retrieval-drift
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Provides-Extra: dev
+Requires-Dist: pandas>=2.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: external-response
+Requires-Dist: context-drift-detector-py>=0.1; extra == 'external-response'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+# driftvane
+
+[![CI](https://github.com/MukundaKatta/driftvane/actions/workflows/ci.yml/badge.svg)](https://github.com/MukundaKatta/driftvane/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/pypi/pyversions/driftvane.svg)](https://pypi.org/project/driftvane/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+**Compose drift detectors for RAG and agent systems.**
+
+Most drift libraries are either tabular-only (Evidently, DataDrift) or are
+platforms that want you to ship telemetry to their backend (Phoenix, Arize).
+`driftvane` is a small Python library that lets you wire up multiple drift
+signals — embedding, retrieval, response, latency — into one report. No
+server, no UI, no telemetry. Plug it into a Lambda or Glue job, get a
+`pandas.DataFrame` or a JSON dict back.
+
+## Install
+
+```bash
+pip install driftvane
+# optional
+pip install "driftvane[pandas]"            # to_pandas()
+pip install "driftvane[external-response]" # delegate response scoring to context-drift-detector-py
+```
+
+## Quickstart
+
+```python
+import numpy as np
+from driftvane import (
+    DriftReport,
+    EmbeddingDrift,
+    RetrievalDrift,
+    ResponseDrift,
+    LatencyDrift,
+)
+from driftvane.detectors.response import Triple
+
+ref_emb = np.load("reference_query_embeddings.npy")  # (n, 768)
+cur_emb = np.load("current_query_embeddings.npy")
+
+report = DriftReport.from_signals([
+    EmbeddingDrift(threshold=0.1).compute(ref_emb, cur_emb),
+    RetrievalDrift(k=10, threshold=0.3).compute(ref_top_k, cur_top_k),
+    ResponseDrift(threshold=0.15).compute(ref_triples, cur_triples),
+    LatencyDrift(p_threshold=0.01).compute(ref_latencies, cur_latencies),
+])
+
+if report.any_drifted():
+    print(report.to_pandas())
+```
+
+Or fail a CI job when retrieval moves too much:
+
+```python
+from driftvane import DriftAlert
+
+try:
+    report.alert_if({"retrieval_jaccard_at_10": 0.2})
+except DriftAlert as e:
+    sys.exit(f"drift gate failed: {e}")
+```
+
+## Detectors
+
+| Detector | Input | Statistic | Notes |
+|---|---|---|---|
+| `EmbeddingDrift` | two `(n, d)` arrays | MMD with RBF kernel, median-heuristic sigma | numpy-only, O(n²) — subsample for n > a few thousand |
+| `RetrievalDrift` | paired top-k id lists | 1 − mean Jaccard@k; reports RBO too | aligned queries required |
+| `ResponseDrift` | `(intent, context, answer)` triples | shift in mean answer-to-context grounding | uses `context-drift-detector-py` if installed |
+| `LatencyDrift` | two 1-D arrays of floats | Kolmogorov–Smirnov D + asymptotic p-value | scipy-free |
+
+Each detector returns a `DriftSignal(name, value, threshold, drifted, metadata)`.
+`DriftReport` collects them.
+
+## What it does NOT do
+
+- No server. No UI. No telemetry shipping.
+- No tabular feature drift — use [DataDrift](https://github.com/MukundaKatta/DataDrift)
+  for KS/PSI on classical features.
+- No live trace ingestion or OTel collection — point this at parquet/numpy
+  arrays you already have.
+- No causal root-cause analysis. It tells you *that* drift is there, not why.
+- No model retraining triggers — emit your own when `report.any_drifted()`.
+
+## Why not Phoenix / Arize / Evidently / Ragas?
+
+| | driftvane | Phoenix | Arize | Evidently | Ragas |
+|---|---|---|---|---|---|
+| Library-only (no server) | ✓ | ✗ | ✗ | partial | ✓ |
+| RAG-shaped detectors | ✓ | ✓ | ✓ | ✗ | ✓ |
+| Embedding MMD out of the box | ✓ | partial | ✓ | ✗ | ✗ |
+| Retrieval rank-shift | ✓ | ✗ | partial | ✗ | ✗ |
+| Run inside a 5s Lambda | ✓ | ✗ | ✗ | ✓ | partial |
+| numpy-only core deps | ✓ | ✗ | ✗ | ✗ | ✗ |
+
+## Status
+
+v0.1 — alpha. The four detectors above work and have tests. Public API may
+change before v1.0. Issues and PRs welcome.

driftvane-0.1.0/README.md

@@ -0,0 +1,99 @@
+# driftvane
+
+[![CI](https://github.com/MukundaKatta/driftvane/actions/workflows/ci.yml/badge.svg)](https://github.com/MukundaKatta/driftvane/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/pypi/pyversions/driftvane.svg)](https://pypi.org/project/driftvane/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+**Compose drift detectors for RAG and agent systems.**
+
+Most drift libraries are either tabular-only (Evidently, DataDrift) or are
+platforms that want you to ship telemetry to their backend (Phoenix, Arize).
+`driftvane` is a small Python library that lets you wire up multiple drift
+signals — embedding, retrieval, response, latency — into one report. No
+server, no UI, no telemetry. Plug it into a Lambda or Glue job, get a
+`pandas.DataFrame` or a JSON dict back.
+
+## Install
+
+```bash
+pip install driftvane
+# optional
+pip install "driftvane[pandas]"            # to_pandas()
+pip install "driftvane[external-response]" # delegate response scoring to context-drift-detector-py
+```
+
+## Quickstart
+
+```python
+import numpy as np
+from driftvane import (
+    DriftReport,
+    EmbeddingDrift,
+    RetrievalDrift,
+    ResponseDrift,
+    LatencyDrift,
+)
+from driftvane.detectors.response import Triple
+
+ref_emb = np.load("reference_query_embeddings.npy")  # (n, 768)
+cur_emb = np.load("current_query_embeddings.npy")
+
+report = DriftReport.from_signals([
+    EmbeddingDrift(threshold=0.1).compute(ref_emb, cur_emb),
+    RetrievalDrift(k=10, threshold=0.3).compute(ref_top_k, cur_top_k),
+    ResponseDrift(threshold=0.15).compute(ref_triples, cur_triples),
+    LatencyDrift(p_threshold=0.01).compute(ref_latencies, cur_latencies),
+])
+
+if report.any_drifted():
+    print(report.to_pandas())
+```
+
+Or fail a CI job when retrieval moves too much:
+
+```python
+from driftvane import DriftAlert
+
+try:
+    report.alert_if({"retrieval_jaccard_at_10": 0.2})
+except DriftAlert as e:
+    sys.exit(f"drift gate failed: {e}")
+```
+
+## Detectors
+
+| Detector | Input | Statistic | Notes |
+|---|---|---|---|
+| `EmbeddingDrift` | two `(n, d)` arrays | MMD with RBF kernel, median-heuristic sigma | numpy-only, O(n²) — subsample for n > a few thousand |
+| `RetrievalDrift` | paired top-k id lists | 1 − mean Jaccard@k; reports RBO too | aligned queries required |
+| `ResponseDrift` | `(intent, context, answer)` triples | shift in mean answer-to-context grounding | uses `context-drift-detector-py` if installed |
+| `LatencyDrift` | two 1-D arrays of floats | Kolmogorov–Smirnov D + asymptotic p-value | scipy-free |
+
+Each detector returns a `DriftSignal(name, value, threshold, drifted, metadata)`.
+`DriftReport` collects them.
+
+## What it does NOT do
+
+- No server. No UI. No telemetry shipping.
+- No tabular feature drift — use [DataDrift](https://github.com/MukundaKatta/DataDrift)
+  for KS/PSI on classical features.
+- No live trace ingestion or OTel collection — point this at parquet/numpy
+  arrays you already have.
+- No causal root-cause analysis. It tells you *that* drift is there, not why.
+- No model retraining triggers — emit your own when `report.any_drifted()`.
+
+## Why not Phoenix / Arize / Evidently / Ragas?
+
+| | driftvane | Phoenix | Arize | Evidently | Ragas |
+|---|---|---|---|---|---|
+| Library-only (no server) | ✓ | ✗ | ✗ | partial | ✓ |
+| RAG-shaped detectors | ✓ | ✓ | ✓ | ✗ | ✓ |
+| Embedding MMD out of the box | ✓ | partial | ✓ | ✗ | ✗ |
+| Retrieval rank-shift | ✓ | ✗ | partial | ✗ | ✗ |
+| Run inside a 5s Lambda | ✓ | ✗ | ✗ | ✓ | partial |
+| numpy-only core deps | ✓ | ✗ | ✗ | ✗ | ✗ |
+
+## Status
+
+v0.1 — alpha. The four detectors above work and have tests. Public API may
+change before v1.0. Issues and PRs welcome.

driftvane-0.1.0/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
+
+[project]
+name = "driftvane"
+version = "0.1.0"
+description = "Compose drift detectors (embedding, retrieval, response, latency) into one report. Library-only, no server, no UI."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Mukunda Rao Katta", email = "mukunda.vjcs6@gmail.com" }]
+keywords = [
+  "ai",
+  "llm",
+  "rag",
+  "agents",
+  "drift",
+  "embedding-drift",
+  "retrieval-drift",
+  "monitoring",
+  "evals",
+  "mlops",
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+  "numpy>=1.24",
+]
+
+[project.optional-dependencies]
+pandas = ["pandas>=2.0"]
+external-response = ["context-drift-detector-py>=0.1"]
+dev = ["pytest>=8.0", "pandas>=2.0", "ruff>=0.4"]
+
+[project.urls]
+Homepage = "https://github.com/MukundaKatta/driftvane"
+Issues = "https://github.com/MukundaKatta/driftvane/issues"
+Source = "https://github.com/MukundaKatta/driftvane"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/driftvane"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/src", "/tests", "/README.md", "/LICENSE"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]

driftvane-0.1.0/src/driftvane/__init__.py

@@ -0,0 +1,25 @@
+"""driftvane — compose drift detectors for RAG and agent systems.
+
+A small library that lets you wire up multiple drift signals (embedding,
+retrieval, response, latency) into one DriftReport. No server, no UI.
+"""
+
+from driftvane.detector import DriftAlert, DriftSignal
+from driftvane.detectors.embedding import EmbeddingDrift
+from driftvane.detectors.latency import LatencyDrift
+from driftvane.detectors.response import ResponseDrift
+from driftvane.detectors.retrieval import RetrievalDrift
+from driftvane.report import DriftReport
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DriftAlert",
+    "DriftReport",
+    "DriftSignal",
+    "EmbeddingDrift",
+    "LatencyDrift",
+    "ResponseDrift",
+    "RetrievalDrift",
+    "__version__",
+]

driftvane-0.1.0/src/driftvane/detector.py

@@ -0,0 +1,42 @@
+"""Core types: DriftSignal, DriftAlert."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class DriftSignal:
+    """One detector's verdict.
+
+    name: stable identifier, e.g. "embedding_mmd", "retrieval_jaccard_at_10"
+    value: the raw statistic
+    threshold: the configured threshold; None means "report only, don't flag"
+    drifted: True when value exceeds threshold
+    metadata: detector-specific extras (sample sizes, kernel sigma, etc.)
+    """
+
+    name: str
+    value: float
+    threshold: float | None = None
+    drifted: bool = False
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "value": self.value,
+            "threshold": self.threshold,
+            "drifted": self.drifted,
+            "metadata": self.metadata,
+        }
+
+
+class DriftAlert(Exception):
+    """Raised by DriftReport.alert_if when a threshold is breached."""
+
+    def __init__(self, breaches: list[DriftSignal]):
+        self.breaches = breaches
+        names = ", ".join(f"{s.name}={s.value:.4f}>{s.threshold}" for s in breaches)
+        super().__init__(f"drift detected: {names}")

driftvane-0.1.0/src/driftvane/detectors/__init__.py

@@ -0,0 +1,6 @@
+from driftvane.detectors.embedding import EmbeddingDrift
+from driftvane.detectors.latency import LatencyDrift
+from driftvane.detectors.response import ResponseDrift
+from driftvane.detectors.retrieval import RetrievalDrift
+
+__all__ = ["EmbeddingDrift", "LatencyDrift", "ResponseDrift", "RetrievalDrift"]

driftvane-0.1.0/src/driftvane/detectors/embedding.py

@@ -0,0 +1,112 @@
+"""EmbeddingDrift — Maximum Mean Discrepancy with RBF kernel.
+
+MMD is a kernel two-sample test. It tests whether two batches of embeddings
+were drawn from the same distribution. MMD^2 is zero when the distributions
+match and grows with the distance between them.
+
+We compute the squared MMD with the RBF (Gaussian) kernel:
+    k(x, y) = exp(-||x - y||^2 / (2 * sigma^2))
+    MMD^2 = E[k(X, X')] + E[k(Y, Y')] - 2 E[k(X, Y)]
+
+When sigma is None we use the median heuristic on the merged sample, which
+is the standard default and removes the main hyperparameter footgun.
+
+Cost is O(n^2) memory and time, so call this with batches up to a few
+thousand vectors. For larger sets, subsample first.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from driftvane.detector import DriftSignal
+
+
+def _pairwise_sq_dists(a: np.ndarray, b: np.ndarray) -> np.ndarray:
+    """Squared Euclidean distance matrix, shape (len(a), len(b))."""
+    a2 = np.sum(a * a, axis=1)[:, None]
+    b2 = np.sum(b * b, axis=1)[None, :]
+    return np.maximum(a2 + b2 - 2.0 * a @ b.T, 0.0)
+
+
+def _median_heuristic_sigma(x: np.ndarray, y: np.ndarray) -> float:
+    """Median pairwise distance on the merged sample. Robust default for sigma."""
+    z = np.concatenate([x, y], axis=0)
+    # subsample to keep this cheap on big inputs
+    if len(z) > 1000:
+        rng = np.random.default_rng(0)
+        idx = rng.choice(len(z), size=1000, replace=False)
+        z = z[idx]
+    d2 = _pairwise_sq_dists(z, z)
+    iu = np.triu_indices_from(d2, k=1)
+    median_sq = float(np.median(d2[iu]))
+    # sigma is the bandwidth, not sigma^2; floor to avoid div-by-zero
+    return max(np.sqrt(median_sq / 2.0), 1e-8)
+
+
+def mmd_rbf(x: np.ndarray, y: np.ndarray, sigma: float | None = None) -> tuple[float, float]:
+    """Compute MMD^2 between two batches with RBF kernel.
+
+    Returns (mmd_squared, sigma_used).
+    """
+    if sigma is None:
+        sigma = _median_heuristic_sigma(x, y)
+    gamma = 1.0 / (2.0 * sigma * sigma)
+
+    kxx = np.exp(-gamma * _pairwise_sq_dists(x, x))
+    kyy = np.exp(-gamma * _pairwise_sq_dists(y, y))
+    kxy = np.exp(-gamma * _pairwise_sq_dists(x, y))
+
+    mmd2 = float(kxx.mean() + kyy.mean() - 2.0 * kxy.mean())
+    # numerical noise can push the value slightly negative; clamp at 0
+    return max(mmd2, 0.0), sigma
+
+
+class EmbeddingDrift:
+    """Detect distribution shift between two batches of embedding vectors.
+
+        ed = EmbeddingDrift(threshold=0.1)
+        signal = ed.compute(reference=ref_emb, current=cur_emb)
+    """
+
+    def __init__(
+        self,
+        method: str = "mmd",
+        sigma: float | None = None,
+        threshold: float | None = None,
+        name: str = "embedding_mmd",
+    ) -> None:
+        if method != "mmd":
+            raise ValueError(f"unknown method: {method!r}; only 'mmd' is supported")
+        self.method = method
+        self.sigma = sigma
+        self.threshold = threshold
+        self.name = name
+
+    def compute(self, reference: np.ndarray, current: np.ndarray) -> DriftSignal:
+        ref = np.asarray(reference, dtype=np.float64)
+        cur = np.asarray(current, dtype=np.float64)
+        if ref.ndim != 2 or cur.ndim != 2:
+            raise ValueError("reference and current must be 2-D (n_samples, n_dims)")
+        if ref.shape[1] != cur.shape[1]:
+            raise ValueError(
+                f"dim mismatch: reference has {ref.shape[1]}, current has {cur.shape[1]}"
+            )
+        if len(ref) < 2 or len(cur) < 2:
+            raise ValueError("need at least 2 samples in each set")
+
+        value, sigma_used = mmd_rbf(ref, cur, sigma=self.sigma)
+        drifted = self.threshold is not None and value > self.threshold
+        return DriftSignal(
+            name=self.name,
+            value=value,
+            threshold=self.threshold,
+            drifted=drifted,
+            metadata={
+                "n_ref": int(ref.shape[0]),
+                "n_cur": int(cur.shape[0]),
+                "dim": int(ref.shape[1]),
+                "sigma": float(sigma_used),
+                "method": self.method,
+            },
+        )

driftvane-0.1.0/src/driftvane/detectors/latency.py

@@ -0,0 +1,90 @@
+"""LatencyDrift — Kolmogorov-Smirnov two-sample test on latency arrays.
+
+KS compares the empirical CDFs of two samples. The statistic is the maximum
+absolute difference between the CDFs and is bounded in [0, 1]. It is robust
+to scale and doesn't assume any particular distribution, which matches how
+real LLM latency tails behave.
+
+We compute the KS statistic from sorted arrays without scipy so the install
+stays light. For an approximate p-value we use the standard asymptotic form
+sqrt(-0.5 * ln(alpha/2) * (n1+n2)/(n1*n2)).
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+
+import numpy as np
+
+from driftvane.detector import DriftSignal
+
+
+def ks_2samp(x: Sequence[float], y: Sequence[float]) -> tuple[float, float]:
+    """Return (D, approx_p_value). Numpy-only two-sample KS."""
+    a = np.sort(np.asarray(x, dtype=np.float64))
+    b = np.sort(np.asarray(y, dtype=np.float64))
+    n1, n2 = len(a), len(b)
+    if n1 == 0 or n2 == 0:
+        raise ValueError("both arrays must be non-empty")
+    all_v = np.concatenate([a, b])
+    cdf_a = np.searchsorted(a, all_v, side="right") / n1
+    cdf_b = np.searchsorted(b, all_v, side="right") / n2
+    d = float(np.max(np.abs(cdf_a - cdf_b)))
+
+    if d == 0.0:
+        # asymptotic series degenerates at d=0; the null is trivially consistent
+        return 0.0, 1.0
+
+    en = math.sqrt(n1 * n2 / (n1 + n2))
+    # asymptotic two-sided p-value (Smirnov)
+    lam = (en + 0.12 + 0.11 / en) * d
+    p = 2.0 * sum(((-1) ** (k - 1)) * math.exp(-2.0 * lam * lam * k * k) for k in range(1, 101))
+    p = max(0.0, min(1.0, p))
+    return d, p
+
+
+class LatencyDrift:
+    """Detect distribution shift in latency (or any 1-D numeric array).
+
+        ld = LatencyDrift(threshold=0.2)        # threshold on KS statistic
+        signal = ld.compute(reference=ref_lat, current=cur_lat)
+
+    Or threshold on p-value:
+
+        ld = LatencyDrift(p_threshold=0.01)
+    """
+
+    def __init__(
+        self,
+        threshold: float | None = None,
+        p_threshold: float | None = None,
+        name: str = "latency_ks",
+    ) -> None:
+        if threshold is not None and p_threshold is not None:
+            raise ValueError("set either threshold or p_threshold, not both")
+        self.threshold = threshold
+        self.p_threshold = p_threshold
+        self.name = name
+
+    def compute(self, reference: Sequence[float], current: Sequence[float]) -> DriftSignal:
+        d, p = ks_2samp(reference, current)
+        if self.p_threshold is not None:
+            drifted = p < self.p_threshold
+        else:
+            drifted = self.threshold is not None and d > self.threshold
+
+        return DriftSignal(
+            name=self.name,
+            value=d,
+            threshold=self.threshold,
+            drifted=drifted,
+            metadata={
+                "n_ref": len(reference),
+                "n_cur": len(current),
+                "ks_p_value": p,
+                "p_threshold": self.p_threshold,
+                "median_ref": float(np.median(reference)),
+                "median_cur": float(np.median(current)),
+            },
+        )

driftvane-0.1.0/src/driftvane/detectors/response.py

@@ -0,0 +1,128 @@
+"""ResponseDrift — answer-vs-context grounding drift across batches.
+
+For each (intent, context, answer) triple, compute Jaccard overlap of token
+sets between the answer and the context. Then compare the *distribution* of
+those scores between reference and current batches.
+
+The drift value is the absolute difference of the mean grounding scores. A
+shrinking mean answer-to-context overlap is the signal you want to catch:
+the model is wandering off the retrieved context.
+
+If `context-drift-detector-py` is installed we delegate per-triple scoring
+to it for compatibility with that library's signal definitions; otherwise
+we use the inline tokenizer below. Either way, the aggregation is ours.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from driftvane.detector import DriftSignal
+
+
+@dataclass(frozen=True)
+class Triple:
+    intent: str
+    context: str | list[str]
+    answer: str
+
+
+_WORD_RE = re.compile(r"[a-z0-9]+")
+
+
+def _tokens(text: str) -> set[str]:
+    return set(_WORD_RE.findall(text.lower()))
+
+
+def _flatten_context(ctx: str | Iterable[str]) -> str:
+    if isinstance(ctx, str):
+        return ctx
+    return " ".join(ctx)
+
+
+def _grounding_score(triple: Triple) -> float:
+    """answer ∩ context / answer (recall-style; 1.0 = fully grounded)."""
+    ans = _tokens(triple.answer)
+    if not ans:
+        return 1.0
+    ctx = _tokens(_flatten_context(triple.context))
+    return len(ans & ctx) / len(ans)
+
+
+def _try_load_external_scorer():
+    try:
+        from context_drift_detector import detect  # type: ignore
+    except ImportError:
+        return None
+
+    def _score(triple: Triple) -> float:
+        ctx = triple.context if isinstance(triple.context, list) else [triple.context]
+        result = detect(triple.intent, ctx, triple.answer)
+        # context-drift-detector-py exposes signals dict with answer_to_context
+        return float(result.signals.get("answer_to_context", _grounding_score(triple)))
+
+    return _score
+
+
+class ResponseDrift:
+    """Detect drift in how well answers stay grounded in retrieved context.
+
+        rsp = ResponseDrift(threshold=0.15)
+        signal = rsp.compute(
+            reference=[Triple("...", "...", "..."), ...],
+            current=[Triple("...", "...", "..."), ...],
+        )
+
+    Pass `use_external=False` to force the inline tokenizer even when
+    context-drift-detector-py is installed.
+    """
+
+    def __init__(
+        self,
+        threshold: float | None = None,
+        name: str = "response_grounding_shift",
+        use_external: bool = True,
+    ) -> None:
+        self.threshold = threshold
+        self.name = name
+        self.use_external = use_external
+        self._scorer = _try_load_external_scorer() if use_external else None
+
+    def compute(
+        self,
+        reference: Iterable[Triple | dict],
+        current: Iterable[Triple | dict],
+    ) -> DriftSignal:
+        ref = [t if isinstance(t, Triple) else Triple(**t) for t in reference]
+        cur = [t if isinstance(t, Triple) else Triple(**t) for t in current]
+        if not ref or not cur:
+            raise ValueError("need at least 1 triple in each batch")
+
+        score = self._scorer or _grounding_score
+        ref_scores = [score(t) for t in ref]
+        cur_scores = [score(t) for t in cur]
+
+        mean_ref = sum(ref_scores) / len(ref_scores)
+        mean_cur = sum(cur_scores) / len(cur_scores)
+        # we care about *worsening* grounding, so use signed shift but report
+        # absolute as the drift value
+        signed_shift = mean_cur - mean_ref
+        drift_value = abs(signed_shift)
+        drifted = self.threshold is not None and drift_value > self.threshold
+
+        return DriftSignal(
+            name=self.name,
+            value=drift_value,
+            threshold=self.threshold,
+            drifted=drifted,
+            metadata={
+                "n_ref": len(ref),
+                "n_cur": len(cur),
+                "mean_ref_grounding": mean_ref,
+                "mean_cur_grounding": mean_cur,
+                "signed_shift": signed_shift,
+                "scorer": "external" if self._scorer else "inline_jaccard",
+            },
+        )

driftvane-0.1.0/src/driftvane/detectors/retrieval.py

@@ -0,0 +1,113 @@
+"""RetrievalDrift — measure shift in retriever output for the same queries.
+
+Inputs are paired top-k document-id lists: for each query, the reference
+retriever produced one ranked list and the current retriever produced another.
+Drift = how much the top-k sets and rank order have moved.
+
+Two metrics:
+  * mean_jaccard_at_k: average Jaccard overlap of the top-k sets (1.0 = identical)
+  * mean_rbo: rank-biased overlap, weights early positions more (1.0 = identical)
+
+The reported drift value is 1 - mean_jaccard_at_k so that "more drift = larger
+value" matches the convention in the other detectors.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from driftvane.detector import DriftSignal
+
+
+def _jaccard(a: set[Any], b: set[Any]) -> float:
+    if not a and not b:
+        return 1.0
+    return len(a & b) / len(a | b)
+
+
+def _rbo(ref: Sequence[Any], cur: Sequence[Any], p: float = 0.9) -> float:
+    """Rank-biased overlap. Weighted overlap of the two prefix sets at each depth.
+
+    p controls how top-heavy the weighting is; p=0.9 puts ~86% of weight on the
+    top 10. See Webber, Moffat, Zobel 2010.
+    """
+    depth = max(len(ref), len(cur))
+    if depth == 0:
+        return 1.0
+    seen_ref: set[Any] = set()
+    seen_cur: set[Any] = set()
+    weighted_sum = 0.0
+    weight_total = 0.0
+    for i in range(depth):
+        if i < len(ref):
+            seen_ref.add(ref[i])
+        if i < len(cur):
+            seen_cur.add(cur[i])
+        agreement = len(seen_ref & seen_cur) / (i + 1)
+        w = p**i
+        weighted_sum += agreement * w
+        weight_total += w
+    return weighted_sum / weight_total if weight_total > 0 else 1.0
+
+
+class RetrievalDrift:
+    """Detect retrieval drift across paired query→top-k results.
+
+        rd = RetrievalDrift(k=10, threshold=0.3)
+        signal = rd.compute(
+            reference=[["doc_1", "doc_2", ...], ...],
+            current=[["doc_1", "doc_3", ...], ...],
+        )
+    """
+
+    def __init__(
+        self,
+        k: int = 10,
+        threshold: float | None = None,
+        name: str | None = None,
+    ) -> None:
+        if k < 1:
+            raise ValueError("k must be >= 1")
+        self.k = k
+        self.threshold = threshold
+        self.name = name or f"retrieval_jaccard_at_{k}"
+
+    def compute(
+        self,
+        reference: Sequence[Sequence[Any]],
+        current: Sequence[Sequence[Any]],
+    ) -> DriftSignal:
+        if len(reference) != len(current):
+            raise ValueError(
+                f"reference and current must have the same number of queries; "
+                f"got {len(reference)} vs {len(current)}"
+            )
+        if not reference:
+            raise ValueError("need at least 1 query")
+
+        jaccards: list[float] = []
+        rbos: list[float] = []
+        for ref_list, cur_list in zip(reference, current, strict=True):
+            ref_top = list(ref_list[: self.k])
+            cur_top = list(cur_list[: self.k])
+            jaccards.append(_jaccard(set(ref_top), set(cur_top)))
+            rbos.append(_rbo(ref_top, cur_top))
+
+        mean_jaccard = sum(jaccards) / len(jaccards)
+        mean_rbo = sum(rbos) / len(rbos)
+        drift_value = 1.0 - mean_jaccard
+        drifted = self.threshold is not None and drift_value > self.threshold
+
+        return DriftSignal(
+            name=self.name,
+            value=drift_value,
+            threshold=self.threshold,
+            drifted=drifted,
+            metadata={
+                "n_queries": len(reference),
+                "k": self.k,
+                "mean_jaccard_at_k": mean_jaccard,
+                "mean_rbo": mean_rbo,
+            },
+        )

driftvane-0.1.0/src/driftvane/report.py

@@ -0,0 +1,99 @@
+"""DriftReport — collect signals from multiple detectors."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from driftvane.detector import DriftAlert, DriftSignal
+
+
+class DriftReport:
+    """A bag of DriftSignals with output helpers.
+
+    Build it incrementally:
+
+        report = DriftReport()
+        report.add(EmbeddingDrift().compute(ref_emb, cur_emb))
+        report.add(LatencyDrift().compute(ref_lat, cur_lat))
+
+    Or in one shot:
+
+        report = DriftReport.from_signals([
+            EmbeddingDrift().compute(ref_emb, cur_emb),
+            LatencyDrift().compute(ref_lat, cur_lat),
+        ])
+    """
+
+    def __init__(self) -> None:
+        self._signals: list[DriftSignal] = []
+
+    @classmethod
+    def from_signals(cls, signals: list[DriftSignal]) -> DriftReport:
+        r = cls()
+        for s in signals:
+            r.add(s)
+        return r
+
+    def add(self, signal: DriftSignal) -> DriftReport:
+        self._signals.append(signal)
+        return self
+
+    @property
+    def signals(self) -> list[DriftSignal]:
+        return list(self._signals)
+
+    def get(self, name: str) -> DriftSignal | None:
+        for s in self._signals:
+            if s.name == name:
+                return s
+        return None
+
+    def any_drifted(self) -> bool:
+        return any(s.drifted for s in self._signals)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "signals": [s.to_dict() for s in self._signals],
+            "any_drifted": self.any_drifted(),
+        }
+
+    def to_pandas(self):
+        # imported lazily so pandas isn't required for non-DataFrame users
+        import pandas as pd
+
+        if not self._signals:
+            return pd.DataFrame(columns=["name", "value", "threshold", "drifted"])
+        return pd.DataFrame(
+            [
+                {
+                    "name": s.name,
+                    "value": s.value,
+                    "threshold": s.threshold,
+                    "drifted": s.drifted,
+                    **{f"meta_{k}": v for k, v in s.metadata.items()},
+                }
+                for s in self._signals
+            ]
+        )
+
+    def alert_if(self, thresholds: dict[str, float]) -> None:
+        """Raise DriftAlert if any of the given signals exceeds its threshold.
+
+        Overrides the threshold each signal was computed with. Use this when the
+        report is being evaluated against a different policy than the detector
+        was constructed with (e.g. CI vs. prod).
+        """
+        breaches = []
+        for s in self._signals:
+            if s.name in thresholds and s.value > thresholds[s.name]:
+                breaches.append(
+                    DriftSignal(
+                        name=s.name,
+                        value=s.value,
+                        threshold=thresholds[s.name],
+                        drifted=True,
+                        metadata=s.metadata,
+                    )
+                )
+        if breaches:
+            raise DriftAlert(breaches)

driftvane-0.1.0/tests/test_embedding.py

@@ -0,0 +1,73 @@
+import numpy as np
+import pytest
+
+from driftvane import EmbeddingDrift
+
+
+def _gen(n: int, dim: int, mean: float = 0.0, scale: float = 1.0, seed: int = 0):
+    rng = np.random.default_rng(seed)
+    return rng.normal(loc=mean, scale=scale, size=(n, dim))
+
+
+def test_no_drift_for_same_distribution():
+    a = _gen(200, 16, seed=1)
+    b = _gen(200, 16, seed=2)
+    sig = EmbeddingDrift().compute(a, b)
+    # MMD between two N(0,1) samples of the same shape should be small
+    assert sig.value < 0.05
+    assert sig.metadata["n_ref"] == 200
+    assert sig.metadata["dim"] == 16
+
+
+def test_drift_for_shifted_mean():
+    a = _gen(200, 16, mean=0.0, seed=1)
+    b = _gen(200, 16, mean=2.0, seed=2)
+    sig = EmbeddingDrift().compute(a, b)
+    assert sig.value > 0.1
+
+
+def test_threshold_flags_drift():
+    a = _gen(200, 16, mean=0.0, seed=1)
+    b = _gen(200, 16, mean=2.0, seed=2)
+    sig = EmbeddingDrift(threshold=0.1).compute(a, b)
+    assert sig.drifted is True
+
+
+def test_threshold_passes_when_under():
+    a = _gen(200, 16, seed=1)
+    b = _gen(200, 16, seed=2)
+    sig = EmbeddingDrift(threshold=0.5).compute(a, b)
+    assert sig.drifted is False
+
+
+def test_dim_mismatch_raises():
+    a = _gen(50, 8)
+    b = _gen(50, 16)
+    with pytest.raises(ValueError, match="dim mismatch"):
+        EmbeddingDrift().compute(a, b)
+
+
+def test_one_sample_raises():
+    a = _gen(1, 8)
+    b = _gen(50, 8)
+    with pytest.raises(ValueError, match="at least 2 samples"):
+        EmbeddingDrift().compute(a, b)
+
+
+def test_unknown_method_raises():
+    with pytest.raises(ValueError, match="unknown method"):
+        EmbeddingDrift(method="kld")
+
+
+def test_explicit_sigma_used():
+    a = _gen(50, 8)
+    b = _gen(50, 8)
+    sig = EmbeddingDrift(sigma=2.5).compute(a, b)
+    assert sig.metadata["sigma"] == 2.5
+
+
+def test_value_is_non_negative():
+    a = _gen(200, 16, seed=1)
+    b = _gen(200, 16, seed=1)
+    sig = EmbeddingDrift().compute(a, b)
+    assert sig.value >= 0.0

driftvane-0.1.0/tests/test_latency.py

@@ -0,0 +1,63 @@
+import numpy as np
+import pytest
+
+from driftvane import LatencyDrift
+from driftvane.detectors.latency import ks_2samp
+
+
+def test_identical_samples_have_zero_ks():
+    rng = np.random.default_rng(0)
+    a = rng.normal(size=500)
+    sig = LatencyDrift().compute(a, a)
+    assert sig.value == 0.0
+    assert sig.metadata["ks_p_value"] == pytest.approx(1.0, abs=0.05)
+
+
+def test_shifted_distribution_has_high_ks():
+    rng = np.random.default_rng(0)
+    a = rng.normal(loc=0.0, size=500)
+    b = rng.normal(loc=2.0, size=500)
+    sig = LatencyDrift().compute(a, b)
+    assert sig.value > 0.5
+    assert sig.metadata["ks_p_value"] < 0.001
+
+
+def test_threshold_flags_on_d_statistic():
+    rng = np.random.default_rng(0)
+    a = rng.normal(loc=0.0, size=200)
+    b = rng.normal(loc=2.0, size=200)
+    sig = LatencyDrift(threshold=0.3).compute(a, b)
+    assert sig.drifted is True
+
+
+def test_p_threshold_flags():
+    rng = np.random.default_rng(0)
+    a = rng.normal(loc=0.0, size=200)
+    b = rng.normal(loc=2.0, size=200)
+    sig = LatencyDrift(p_threshold=0.01).compute(a, b)
+    assert sig.drifted is True
+
+
+def test_cant_set_both_thresholds():
+    with pytest.raises(ValueError, match="not both"):
+        LatencyDrift(threshold=0.1, p_threshold=0.01)
+
+
+def test_empty_raises():
+    with pytest.raises(ValueError, match="non-empty"):
+        ks_2samp([], [1.0, 2.0])
+
+
+def test_ks_returns_in_unit_interval():
+    rng = np.random.default_rng(0)
+    a = rng.normal(size=100)
+    b = rng.normal(size=100)
+    d, p = ks_2samp(a, b)
+    assert 0.0 <= d <= 1.0
+    assert 0.0 <= p <= 1.0
+
+
+def test_median_metadata_present():
+    sig = LatencyDrift().compute([1.0, 2.0, 3.0, 4.0], [10.0, 20.0, 30.0, 40.0])
+    assert sig.metadata["median_ref"] == pytest.approx(2.5)
+    assert sig.metadata["median_cur"] == pytest.approx(25.0)

driftvane-0.1.0/tests/test_report.py

@@ -0,0 +1,61 @@
+import pytest
+
+from driftvane import DriftAlert, DriftReport, DriftSignal
+
+
+def _sig(
+    name: str,
+    value: float,
+    threshold: float | None = None,
+    drifted: bool = False,
+) -> DriftSignal:
+    return DriftSignal(name=name, value=value, threshold=threshold, drifted=drifted)
+
+
+def test_empty_report_is_not_drifted():
+    r = DriftReport()
+    assert r.signals == []
+    assert r.any_drifted() is False
+    assert r.to_dict() == {"signals": [], "any_drifted": False}
+
+
+def test_add_chains():
+    r = DriftReport()
+    out = r.add(_sig("a", 0.1)).add(_sig("b", 0.2))
+    assert out is r
+    assert [s.name for s in r.signals] == ["a", "b"]
+
+
+def test_from_signals():
+    r = DriftReport.from_signals([_sig("a", 0.1), _sig("b", 0.2, threshold=0.1, drifted=True)])
+    assert r.any_drifted() is True
+    assert r.get("a").value == 0.1
+    assert r.get("missing") is None
+
+
+def test_to_pandas_includes_metadata_columns():
+    r = DriftReport().add(
+        DriftSignal(name="x", value=0.5, metadata={"sigma": 1.5, "n": 100})
+    )
+    df = r.to_pandas()
+    assert list(df.columns) == ["name", "value", "threshold", "drifted", "meta_sigma", "meta_n"]
+    assert df.iloc[0]["meta_sigma"] == 1.5
+
+
+def test_alert_if_raises_on_breach():
+    r = DriftReport().add(_sig("emb", 0.3)).add(_sig("lat", 0.05))
+    with pytest.raises(DriftAlert) as exc:
+        r.alert_if({"emb": 0.2, "lat": 0.1})
+    assert len(exc.value.breaches) == 1
+    assert exc.value.breaches[0].name == "emb"
+
+
+def test_alert_if_no_breach():
+    r = DriftReport().add(_sig("emb", 0.05))
+    # should not raise
+    r.alert_if({"emb": 0.2})
+
+
+def test_alert_if_ignores_unlisted_signals():
+    r = DriftReport().add(_sig("emb", 999.0))
+    r.alert_if({"latency": 0.1})  # emb has no threshold here, should not raise

driftvane-0.1.0/tests/test_response.py

@@ -0,0 +1,67 @@
+import pytest
+
+from driftvane import ResponseDrift
+from driftvane.detectors.response import Triple
+
+
+def test_grounded_answers_have_no_drift():
+    triples = [
+        Triple(
+            intent="What is the capital of France?",
+            context="Paris is the capital of France.",
+            answer="Paris is the capital of France.",
+        ),
+        Triple(
+            intent="What is 2+2?",
+            context="Two plus two equals four.",
+            answer="Two plus two equals four.",
+        ),
+    ]
+    sig = ResponseDrift(use_external=False).compute(triples, triples)
+    assert sig.value == pytest.approx(0.0)
+
+
+def test_ungrounded_current_drifts_negative():
+    grounded = [
+        Triple(
+            intent="capital of France",
+            context="Paris is the capital of France.",
+            answer="Paris is the capital of France.",
+        )
+    ]
+    ungrounded = [
+        Triple(
+            intent="capital of France",
+            context="Paris is the capital of France.",
+            answer="Wombats live in Australia.",
+        )
+    ]
+    sig = ResponseDrift(use_external=False).compute(grounded, ungrounded)
+    assert sig.metadata["signed_shift"] < 0
+    assert sig.value > 0
+
+
+def test_threshold_flags():
+    a = [Triple("q", "the answer is forty two", "the answer is forty two")]
+    b = [Triple("q", "the answer is forty two", "completely unrelated text")]
+    sig = ResponseDrift(threshold=0.3, use_external=False).compute(a, b)
+    assert sig.drifted is True
+
+
+def test_accepts_dicts_and_list_context():
+    a = [{"intent": "q", "context": ["fact a", "fact b"], "answer": "fact a"}]
+    b = [{"intent": "q", "context": ["fact a"], "answer": "fact b fact a"}]
+    sig = ResponseDrift(use_external=False).compute(a, b)
+    assert sig.metadata["n_ref"] == 1
+    assert sig.metadata["n_cur"] == 1
+
+
+def test_empty_raises():
+    with pytest.raises(ValueError, match="at least 1 triple"):
+        ResponseDrift(use_external=False).compute([], [])
+
+
+def test_inline_scorer_used_when_external_disabled():
+    triples = [Triple("q", "ctx", "ctx")]
+    sig = ResponseDrift(use_external=False).compute(triples, triples)
+    assert sig.metadata["scorer"] == "inline_jaccard"

driftvane-0.1.0/tests/test_retrieval.py

@@ -0,0 +1,60 @@
+import pytest
+
+from driftvane import RetrievalDrift
+
+
+def test_identical_rankings_have_zero_drift():
+    ref = [["a", "b", "c", "d"], ["x", "y", "z"]]
+    cur = [["a", "b", "c", "d"], ["x", "y", "z"]]
+    sig = RetrievalDrift(k=4).compute(ref, cur)
+    assert sig.value == 0.0
+    assert sig.metadata["mean_jaccard_at_k"] == 1.0
+    assert sig.metadata["mean_rbo"] == 1.0
+
+
+def test_disjoint_rankings_have_max_drift():
+    ref = [["a", "b", "c"]]
+    cur = [["x", "y", "z"]]
+    sig = RetrievalDrift(k=3).compute(ref, cur)
+    assert sig.value == 1.0
+    assert sig.metadata["mean_jaccard_at_k"] == 0.0
+
+
+def test_partial_overlap():
+    ref = [["a", "b", "c", "d"]]
+    cur = [["a", "b", "x", "y"]]
+    sig = RetrievalDrift(k=4).compute(ref, cur)
+    # Jaccard of {a,b,c,d} vs {a,b,x,y} = 2/6 = 1/3
+    assert sig.metadata["mean_jaccard_at_k"] == pytest.approx(1 / 3)
+
+
+def test_threshold_flags():
+    ref = [["a", "b", "c"]]
+    cur = [["a", "x", "y"]]
+    sig = RetrievalDrift(k=3, threshold=0.5).compute(ref, cur)
+    # 1 - (1/5) = 0.8 > 0.5
+    assert sig.drifted is True
+
+
+def test_mismatched_lengths_raises():
+    with pytest.raises(ValueError, match="same number of queries"):
+        RetrievalDrift().compute([["a"]], [["a"], ["b"]])
+
+
+def test_empty_input_raises():
+    with pytest.raises(ValueError, match="at least 1 query"):
+        RetrievalDrift().compute([], [])
+
+
+def test_k_must_be_positive():
+    with pytest.raises(ValueError, match="k must be"):
+        RetrievalDrift(k=0)
+
+
+def test_rbo_weights_top_positions_more():
+    # Same set, different order: Jaccard sees no drift, RBO does
+    ref = [["a", "b", "c", "d", "e"]]
+    cur = [["e", "d", "c", "b", "a"]]
+    sig = RetrievalDrift(k=5).compute(ref, cur)
+    assert sig.metadata["mean_jaccard_at_k"] == 1.0
+    assert sig.metadata["mean_rbo"] < 1.0