dataeval 0.61.0__py3-none-any.whl

dataeval/_internal/detectors/drift/base.py

@@ -0,0 +1,265 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from abc import ABC, abstractmethod
+from functools import wraps
+from random import random
+from typing import Callable, Dict, Literal, Optional, Tuple, Union
+
+import numpy as np
+
+
+def update_x_ref(fn):
+    @wraps(fn)
+    def _(self, x, *args, **kwargs):
+        output = fn(self, x, *args, **kwargs)
+
+        # update reference dataset
+        if self.update_x_ref is not None:
+            self._x_ref = self.update_x_ref(self.x_ref, x, self.n)
+
+        # used for reservoir sampling
+        self.n += len(x)
+        return output
+
+    return _
+
+
+def preprocess_x(fn):
+    @wraps(fn)
+    def _(self, x, *args, **kwargs):
+        if self._x_refcount == 0:
+            self._x = self._preprocess(x)
+        self._x_refcount += 1
+        output = fn(self, self._x, *args, **kwargs)
+        self._x_refcount -= 1
+        if self._x_refcount == 0:
+            del self._x
+        return output
+
+    return _
+
+
+class UpdateStrategy(ABC):
+    def __init__(self, n: int):
+        self.n = n
+
+    @abstractmethod
+    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+        """Abstract implementation of update strategy"""
+
+
+class LastSeenUpdate(UpdateStrategy):
+    """
+    Updates reference dataset for drift detector using last seen method.
+
+    Parameters
+    ----------
+    n : int
+        Update with last n instances seen by the detector.
+    """
+
+    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+        x_updated = np.concatenate([x_ref, x], axis=0)
+        return x_updated[-self.n :]
+
+
+class ReservoirSamplingUpdate(UpdateStrategy):
+    """
+    Updates reference dataset for drift detector using reservoir sampling method.
+
+    Parameters
+    ----------
+    n : int
+        Update with reservoir sampling of size n.
+    """
+
+    def __call__(self, x_ref: np.ndarray, x: np.ndarray, count: int) -> np.ndarray:
+        if x.shape[0] + count <= self.n:
+            return np.concatenate([x_ref, x], axis=0)
+
+        n_ref = x_ref.shape[0]
+        output_size = min(self.n, n_ref + x.shape[0])
+        shape = (output_size,) + x.shape[1:]
+        x_reservoir = np.zeros(shape, dtype=x_ref.dtype)
+        x_reservoir[:n_ref] = x_ref
+        for item in x:
+            count += 1
+            if n_ref < self.n:
+                x_reservoir[n_ref, :] = item
+                n_ref += 1
+            else:
+                r = int(random() * count)
+                if r < self.n:
+                    x_reservoir[r, :] = item
+        return x_reservoir
+
+
+class BaseDrift:
+    """Generic drift detector component handling preprocessing of data and correction"""
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        correction: Literal["bonferroni", "fdr"] = "bonferroni",
+    ) -> None:
+        # Type checking
+        if preprocess_fn is not None and not isinstance(preprocess_fn, Callable):
+            raise ValueError("`preprocess_fn` is not a valid Callable.")
+        if update_x_ref is not None and not isinstance(update_x_ref, UpdateStrategy):
+            raise ValueError("`update_x_ref` is not a valid ReferenceUpdate class.")
+        if correction not in ["bonferroni", "fdr"]:
+            raise ValueError("`correction` must be `bonferroni` or `fdr`.")
+
+        self._x_ref = x_ref
+        self.x_ref_preprocessed = x_ref_preprocessed
+
+        # Other attributes
+        self.p_val = p_val
+        self.update_x_ref = update_x_ref
+        self.preprocess_fn = preprocess_fn
+        self.correction = correction
+        self.n = len(x_ref)
+
+        # Ref counter for preprocessed x
+        self._x_refcount = 0
+
+    @property
+    def x_ref(self) -> np.ndarray:
+        if not self.x_ref_preprocessed:
+            self.x_ref_preprocessed = True
+            if self.preprocess_fn is not None:
+                self._x_ref = self.preprocess_fn(self._x_ref)
+
+        return self._x_ref
+
+    def _preprocess(self, x: np.ndarray) -> np.ndarray:
+        """Data preprocessing before computing the drift scores."""
+        if self.preprocess_fn is not None:
+            x = self.preprocess_fn(x)
+        return x
+
+
+class BaseUnivariateDrift(BaseDrift):
+    """
+    Generic drift detector component which serves as a base class for methods using
+    univariate tests. If n_features > 1, a multivariate correction is applied such
+    that the false positive rate is upper bounded by the specified p-value, with
+    equality in the case of independent features.
+    """
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        correction: Literal["bonferroni", "fdr"] = "bonferroni",
+        n_features: Optional[int] = None,
+    ) -> None:
+        super().__init__(
+            x_ref,
+            p_val,
+            x_ref_preprocessed,
+            update_x_ref,
+            preprocess_fn,
+            correction,
+        )
+
+        self._n_features = n_features
+
+    @property
+    def n_features(self) -> int:
+        # lazy process n_features as needed
+        if not isinstance(self._n_features, int):
+            # compute number of features for the univariate tests
+            if not isinstance(self.preprocess_fn, Callable) or self.x_ref_preprocessed:
+                # infer features from preprocessed reference data
+                self._n_features = self.x_ref.reshape(self.x_ref.shape[0], -1).shape[-1]
+            else:
+                # infer number of features after applying preprocessing step
+                x = self.preprocess_fn(self.x_ref[0:1])
+                self._n_features = x.reshape(x.shape[0], -1).shape[-1]
+
+        return self._n_features
+
+    @preprocess_x
+    @abstractmethod
+    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        """Abstract method to calculate feature score after preprocessing"""
+
+    def _apply_correction(self, p_vals: np.ndarray) -> Tuple[int, float]:
+        if self.correction == "bonferroni":
+            threshold = self.p_val / self.n_features
+            drift_pred = int((p_vals < threshold).any())
+            return drift_pred, threshold
+        elif self.correction == "fdr":
+            n = p_vals.shape[0]
+            i = np.arange(n) + 1
+            p_sorted = np.sort(p_vals)
+            q_threshold = self.p_val * i / n
+            below_threshold = p_sorted < q_threshold
+            try:
+                idx_threshold = int(np.where(below_threshold)[0].max())
+            except ValueError:  # sorted p-values not below thresholds
+                return int(below_threshold.any()), q_threshold.min()
+            return int(below_threshold.any()), q_threshold[idx_threshold]
+        else:
+            raise ValueError("`correction` needs to be either `bonferroni` or `fdr`.")
+
+    @preprocess_x
+    @update_x_ref
+    def predict(
+        self,
+        x: np.ndarray,
+        drift_type: Literal["batch", "feature"] = "batch",
+    ) -> Dict[str, Union[int, float, np.ndarray]]:
+        """
+        Predict whether a batch of data has drifted from the reference data and update
+        reference data using specified update strategy.
+
+        Parameters
+        ----------
+        x : np.ndarray
+            Batch of instances.
+        drift_type : Literal["batch", "feature"], default "batch"
+            Predict drift at the 'feature' or 'batch' level. For 'batch', the test
+            statistics for each feature are aggregated using the Bonferroni or False
+            Discovery Rate correction (if n_features>1).
+
+        Returns
+        -------
+        Dictionary containing the drift prediction and optionally the feature level
+                p-values, threshold after multivariate correction if needed and test
+                statistics.
+        """
+        # compute drift scores
+        p_vals, dist = self.score(x)
+
+        # TODO: return both feature-level and batch-level drift predictions by default
+        # values below p-value threshold are drift
+        if drift_type == "feature":
+            drift_pred = (p_vals < self.p_val).astype(int)
+            threshold = self.p_val
+        elif drift_type == "batch":
+            drift_pred, threshold = self._apply_correction(p_vals)
+        else:
+            raise ValueError("`drift_type` needs to be either `feature` or `batch`.")
+
+        # populate drift dict
+        return {
+            "is_drift": drift_pred,
+            "p_val": p_vals,
+            "threshold": threshold,
+            "distance": dist,
+        }

dataeval/_internal/detectors/drift/cvm.py

@@ -0,0 +1,97 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable, Literal, Optional, Tuple
+
+import numpy as np
+from scipy.stats import cramervonmises_2samp
+
+from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
+
+
+class DriftCVM(BaseUnivariateDrift):
+    """
+    Cramér-von Mises (CVM) data drift detector, which tests for any change in the
+    distribution of continuous univariate data. For multivariate data, a separate
+    CVM test is applied to each feature, and the obtained p-values are aggregated
+    via the Bonferroni or False Discovery Rate (FDR) corrections.
+
+    Parameters
+    ----------
+    x_ref : np.ndarray
+        Data used as reference distribution.
+    p_val : float, default 0.05
+        p-value used for significance of the statistical test for each feature.
+        If the FDR correction method is used, this corresponds to the acceptable
+        q-value.
+    x_ref_preprocessed : bool, default False
+        Whether the given reference data `x_ref` has been preprocessed yet. If
+        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed at
+        prediction time. If `x_ref_preprocessed=False`, the reference data will also
+        be preprocessed.
+    update_x_ref : Optional[UpdateStrategy], default None
+        Reference data can optionally be updated using an UpdateStrategy class. Update
+        using the last n instances seen by the detector with
+        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
+        or via reservoir sampling with
+        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
+    preprocess_fn : Optional[Callable[[np.ndarray], np.ndarray]], default None
+        Function to preprocess the data before computing the data drift metrics.
+        Typically a dimensionality reduction technique.
+    correction : Literal["bonferroni", "fdr"], default "bonferroni"
+        Correction type for multivariate data. Either 'bonferroni' or 'fdr' (False
+        Discovery Rate).
+    n_features
+        Number of features used in the statistical test. No need to pass it if no
+        preprocessing takes place. In case of a preprocessing step, this can also
+        be inferred automatically but could be more expensive to compute.
+    """
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        correction: Literal["bonferroni", "fdr"] = "bonferroni",
+        n_features: Optional[int] = None,
+    ) -> None:
+        super().__init__(
+            x_ref=x_ref,
+            p_val=p_val,
+            x_ref_preprocessed=x_ref_preprocessed,
+            update_x_ref=update_x_ref,
+            preprocess_fn=preprocess_fn,
+            correction=correction,
+            n_features=n_features,
+        )
+
+    @preprocess_x
+    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Performs the two-sample Cramér-von Mises test(s), computing the p-value and
+        test statistic per feature.
+
+        Parameters
+        ----------
+        x
+            Batch of instances.
+
+        Returns
+        -------
+        Feature level p-values and CVM statistics.
+        """
+        x = x.reshape(x.shape[0], -1)
+        x_ref = self.x_ref.reshape(self.x_ref.shape[0], -1)
+        p_val = np.zeros(self.n_features, dtype=np.float32)
+        dist = np.zeros_like(p_val)
+        for f in range(self.n_features):
+            result = cramervonmises_2samp(x_ref[:, f], x[:, f], method="auto")
+            p_val[f], dist[f] = result.pvalue, result.statistic
+        return p_val, dist

dataeval/_internal/detectors/drift/ks.py

@@ -0,0 +1,100 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable, Literal, Optional, Tuple
+
+import numpy as np
+from scipy.stats import ks_2samp
+
+from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
+
+
+class DriftKS(BaseUnivariateDrift):
+    """
+    Kolmogorov-Smirnov (K-S) data drift detector with Bonferroni or False Discovery
+    Rate (FDR) correction for multivariate data.
+
+    Parameters
+    ----------
+    x_ref : np.ndarray
+        Data used as reference distribution.
+    p_val : float, default 0.05
+        p-value used for significance of the statistical test for each feature.
+        If the FDR correction method is used, this corresponds to the acceptable
+        q-value.
+    x_ref_preprocessed : bool, default False
+        Whether the given reference data `x_ref` has been preprocessed yet. If
+        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed at
+        prediction time. If `x_ref_preprocessed=False`, the reference data will also
+        be preprocessed.
+    update_x_ref : Optional[UpdateStrategy], default None
+        Reference data can optionally be updated using an UpdateStrategy class. Update
+        using the last n instances seen by the detector with
+        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
+        or via reservoir sampling with
+        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
+    preprocess_fn : Optional[Callable[[np.ndarray], np.ndarray]], default None
+        Function to preprocess the data before computing the data drift metrics.
+        Typically a dimensionality reduction technique.
+    correction : Literal["bonferroni", "fdr"], default "bonferroni"
+        Correction type for multivariate data. Either 'bonferroni' or 'fdr' (False
+        Discovery Rate).
+    alternative : Literal["two-sided", "less", "greater"], default "two-sided"
+        Defines the alternative hypothesis. Options are 'two-sided', 'less' or
+        'greater'.
+    n_features
+        Number of features used in the statistical test. No need to pass it if no
+        preprocessing takes place. In case of a preprocessing step, this can also
+        be inferred automatically but could be more expensive to compute.
+    """
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        correction: Literal["bonferroni", "fdr"] = "bonferroni",
+        alternative: Literal["two-sided", "less", "greater"] = "two-sided",
+        n_features: Optional[int] = None,
+    ) -> None:
+        super().__init__(
+            x_ref=x_ref,
+            p_val=p_val,
+            x_ref_preprocessed=x_ref_preprocessed,
+            update_x_ref=update_x_ref,
+            preprocess_fn=preprocess_fn,
+            correction=correction,
+            n_features=n_features,
+        )
+
+        # Other attributes
+        self.alternative = alternative
+
+    @preprocess_x
+    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Compute K-S scores and statistics per feature.
+
+        Parameters
+        ----------
+        x
+            Batch of instances.
+
+        Returns
+        -------
+        Feature level p-values and K-S statistics.
+        """
+        x = x.reshape(x.shape[0], -1)
+        x_ref = self.x_ref.reshape(self.x_ref.shape[0], -1)
+        p_val = np.zeros(self.n_features, dtype=np.float32)
+        dist = np.zeros_like(p_val)
+        for f in range(self.n_features):
+            dist[f], p_val[f] = ks_2samp(x_ref[:, f], x[:, f], alternative=self.alternative, method="exact")
+        return p_val, dist

dataeval/_internal/detectors/drift/mmd.py

@@ -0,0 +1,166 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable, Dict, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from .base import BaseDrift, UpdateStrategy, preprocess_x, update_x_ref
+from .torch import GaussianRBF, get_device, mmd2_from_kernel_matrix
+
+
+class DriftMMD(BaseDrift):
+    """
+    Maximum Mean Discrepancy (MMD) data drift detector using a permutation test.
+
+    Parameters
+    ----------
+    x_ref : np.ndarray
+        Data used as reference distribution.
+    p_val : float, default 0.05
+        p-value used for the significance of the permutation test.
+    x_ref_preprocessed : bool, default False
+        Whether the given reference data `x_ref` has been preprocessed yet. If
+        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed
+        at prediction time. If `x_ref_preprocessed=False`, the reference data
+        will also be preprocessed.
+    preprocess_at_init : bool, default True
+        Whether to preprocess the reference data when the detector is instantiated.
+        Otherwise, the reference data will be preprocessed at prediction time. Only
+        applies if `x_ref_preprocessed=False`.
+    update_x_ref : Optional[UpdateStrategy], default None
+        Reference data can optionally be updated using an UpdateStrategy class. Update
+        using the last n instances seen by the detector with
+        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
+        or via reservoir sampling with
+        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
+    preprocess_fn : Optional[Callable], default None
+        Function to preprocess the data before computing the data drift metrics.
+    kernel : Callable, default :py:class:`dataeval.detectors.GaussianRBF`
+        Kernel used for the MMD computation, defaults to Gaussian RBF kernel.
+    sigma : Optional[np.ndarray], default None
+        Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple
+        bandwidth values as an array. The kernel evaluation is then averaged over
+        those bandwidths.
+    configure_kernel_from_x_ref : bool, default True
+        Whether to already configure the kernel bandwidth from the reference data.
+    n_permutations : int, default 100
+        Number of permutations used in the permutation test.
+    device : Optional[str], default None
+        Device type used. The default None uses the GPU and falls back on CPU.
+        Can be specified by passing either 'cuda', 'gpu' or 'cpu'.
+    """
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        kernel: Callable = GaussianRBF,
+        sigma: Optional[np.ndarray] = None,
+        configure_kernel_from_x_ref: bool = True,
+        n_permutations: int = 100,
+        device: Optional[str] = None,
+    ) -> None:
+        super().__init__(x_ref, p_val, x_ref_preprocessed, update_x_ref, preprocess_fn)
+
+        self.infer_sigma = configure_kernel_from_x_ref
+        if configure_kernel_from_x_ref and isinstance(sigma, np.ndarray):
+            self.infer_sigma = False
+
+        self.n_permutations = n_permutations  # nb of iterations through permutation test
+
+        # set device
+        self.device = get_device(device)
+
+        # initialize kernel
+        sigma_tensor = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, np.ndarray) else None
+        self.kernel = kernel(sigma_tensor).to(self.device) if kernel == GaussianRBF else kernel
+
+        # compute kernel matrix for the reference data
+        if self.infer_sigma or isinstance(sigma_tensor, torch.Tensor):
+            x = torch.from_numpy(self.x_ref).to(self.device)
+            self.k_xx = self.kernel(x, x, infer_sigma=self.infer_sigma)
+            self.infer_sigma = False
+        else:
+            self.k_xx, self.infer_sigma = None, True
+
+    def _kernel_matrix(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+        """Compute and return full kernel matrix between arrays x and y."""
+        k_xy = self.kernel(x, y, self.infer_sigma)
+        k_xx = self.k_xx if self.k_xx is not None and self.update_x_ref is None else self.kernel(x, x)
+        k_yy = self.kernel(y, y)
+        kernel_mat = torch.cat([torch.cat([k_xx, k_xy], 1), torch.cat([k_xy.T, k_yy], 1)], 0)
+        return kernel_mat
+
+    @preprocess_x
+    def score(self, x: np.ndarray) -> Tuple[float, float, float]:
+        """
+        Compute the p-value resulting from a permutation test using the maximum mean
+        discrepancy as a distance measure between the reference data and the data to
+        be tested.
+
+        Parameters
+        ----------
+        x
+            Batch of instances.
+
+        Returns
+        -------
+        p-value obtained from the permutation test, the MMD^2 between the reference and
+        test set, and the MMD^2 threshold above which drift is flagged.
+        """
+        x_ref = torch.from_numpy(self.x_ref).to(self.device)
+        n = x.shape[0]
+        kernel_mat = self._kernel_matrix(x_ref, torch.from_numpy(x).to(self.device))
+        kernel_mat = kernel_mat - torch.diag(kernel_mat.diag())  # zero diagonal
+        mmd2 = mmd2_from_kernel_matrix(kernel_mat, n, permute=False, zero_diag=False)
+        mmd2_permuted = torch.Tensor(
+            [mmd2_from_kernel_matrix(kernel_mat, n, permute=True, zero_diag=False) for _ in range(self.n_permutations)]
+        )
+        mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu()
+        p_val = (mmd2 <= mmd2_permuted).float().mean()
+        # compute distance threshold
+        idx_threshold = int(self.p_val * len(mmd2_permuted))
+        distance_threshold = torch.sort(mmd2_permuted, descending=True).values[idx_threshold]
+        return p_val.numpy().item(), mmd2.numpy().item(), distance_threshold.numpy()
+
+    @preprocess_x
+    @update_x_ref
+    def predict(
+        self,
+        x: np.ndarray,
+    ) -> Dict[str, Union[int, float]]:
+        """
+        Predict whether a batch of data has drifted from the reference data and then
+        updates reference data using specified strategy.
+
+        Parameters
+        ----------
+        x
+            Batch of instances.
+
+        Returns
+        -------
+        Dictionary containing the drift prediction, p-value, threshold and MMD metric.
+        """
+        # compute drift scores
+        p_val, dist, distance_threshold = self.score(x)
+        drift_pred = int(p_val < self.p_val)
+
+        # populate drift dict
+        return {
+            "is_drift": drift_pred,
+            "p_val": p_val,
+            "threshold": self.p_val,
+            "distance": dist,
+            "distance_threshold": distance_threshold,
+        }