dataeval 0.61.0__py3-none-any.whl

dataeval/_internal/detectors/drift/torch.py

@@ -0,0 +1,310 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from functools import partial
+from typing import Callable, Optional, Type, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+def get_device(device: Optional[Union[str, torch.device]] = None) -> torch.device:
+    """
+    Instantiates a PyTorch device object.
+
+    Parameters
+    ----------
+    device
+        Either `None`, a str ('gpu' or 'cpu') indicating the device to choose, or an
+        already instantiated device object. If `None`, the GPU is selected if it is
+        detected, otherwise the CPU is used as a fallback.
+
+    Returns
+    -------
+    The instantiated device object.
+    """
+    if isinstance(device, torch.device):  # Already a torch device
+        return device
+    else:  # Instantiate device
+        if device is None or device.lower() in ["gpu", "cuda"]:
+            torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        else:
+            torch_device = torch.device("cpu")
+    return torch_device
+
+
+def mmd2_from_kernel_matrix(
+    kernel_mat: torch.Tensor, m: int, permute: bool = False, zero_diag: bool = True
+) -> torch.Tensor:
+    """
+    Compute maximum mean discrepancy (MMD^2) between 2 samples x and y from the
+    full kernel matrix between the samples.
+
+    Parameters
+    ----------
+    kernel_mat
+        Kernel matrix between samples x and y.
+    m
+        Number of instances in y.
+    permute
+        Whether to permute the row indices. Used for permutation tests.
+    zero_diag
+        Whether to zero out the diagonal of the kernel matrix.
+
+    Returns
+    -------
+    MMD^2 between the samples from the kernel matrix.
+    """
+    n = kernel_mat.shape[0] - m
+    if zero_diag:
+        kernel_mat = kernel_mat - torch.diag(kernel_mat.diag())
+    if permute:
+        idx = torch.randperm(kernel_mat.shape[0])
+        kernel_mat = kernel_mat[idx][:, idx]
+    k_xx, k_yy, k_xy = kernel_mat[:-m, :-m], kernel_mat[-m:, -m:], kernel_mat[-m:, :-m]
+    c_xx, c_yy = 1 / (n * (n - 1)), 1 / (m * (m - 1))
+    mmd2 = c_xx * k_xx.sum() + c_yy * k_yy.sum() - 2.0 * k_xy.mean()
+    return mmd2
+
+
+def predict_batch(
+    x: Union[np.ndarray, torch.Tensor],
+    model: Union[Callable, nn.Module, nn.Sequential],
+    device: Optional[torch.device] = None,
+    batch_size: int = int(1e10),
+    preprocess_fn: Optional[Callable] = None,
+    dtype: Union[Type[np.generic], torch.dtype] = np.float32,
+) -> Union[np.ndarray, torch.Tensor, tuple]:
+    """
+    Make batch predictions on a model.
+
+    Parameters
+    ----------
+    x
+        Batch of instances.
+    model
+        PyTorch model.
+    device
+        Device type used. The default None tries to use the GPU and falls back on CPU.
+        Can be specified by passing either torch.device('cuda') or torch.device('cpu').
+    batch_size
+        Batch size used during prediction.
+    preprocess_fn
+        Optional preprocessing function for each batch.
+    dtype
+        Model output type, e.g. np.float32 or torch.float32.
+
+    Returns
+    -------
+    Numpy array, torch tensor or tuples of those with model outputs.
+    """
+    device = get_device(device)
+    if isinstance(x, np.ndarray):
+        x = torch.from_numpy(x)
+    n = len(x)
+    n_minibatch = int(np.ceil(n / batch_size))
+    return_np = not isinstance(dtype, torch.dtype)
+    preds = []
+    with torch.no_grad():
+        for i in range(n_minibatch):
+            istart, istop = i * batch_size, min((i + 1) * batch_size, n)
+            x_batch = x[istart:istop]
+            if isinstance(preprocess_fn, Callable):
+                x_batch = preprocess_fn(x_batch)
+            preds_tmp = model(x_batch.to(device))
+            if isinstance(preds_tmp, (list, tuple)):
+                if len(preds) == 0:  # init tuple with lists to store predictions
+                    preds = tuple([] for _ in range(len(preds_tmp)))
+                for j, p in enumerate(preds_tmp):
+                    if isinstance(p, torch.Tensor):
+                        p = p.cpu()
+                    preds[j].append(p if not return_np or isinstance(p, np.ndarray) else p.numpy())
+            elif isinstance(preds_tmp, (np.ndarray, torch.Tensor)):
+                if isinstance(preds_tmp, torch.Tensor):
+                    preds_tmp = preds_tmp.cpu()
+                if isinstance(preds, tuple):
+                    preds = list(preds)
+                preds.append(
+                    preds_tmp
+                    if not return_np or isinstance(preds_tmp, np.ndarray)  # type: ignore
+                    else preds_tmp.numpy()
+                )
+            else:
+                raise TypeError(
+                    f"Model output type {type(preds_tmp)} not supported. The model \
+                    output type needs to be one of list, tuple, np.ndarray or \
+                    torch.Tensor."
+                )
+    concat = partial(np.concatenate, axis=0) if return_np else partial(torch.cat, dim=0)
+    out: Union[tuple, np.ndarray, torch.Tensor] = (
+        tuple(concat(p) for p in preds) if isinstance(preds, tuple) else concat(preds)  # type: ignore
+    )
+    return out
+
+
+def preprocess_drift(
+    x: np.ndarray,
+    model: nn.Module,
+    device: Optional[torch.device] = None,
+    preprocess_batch_fn: Optional[Callable] = None,
+    batch_size: int = int(1e10),
+    dtype: Union[Type[np.generic], torch.dtype] = np.float32,
+) -> Union[np.ndarray, torch.Tensor, tuple]:
+    """
+    Prediction function used for preprocessing step of drift detector.
+
+    Parameters
+    ----------
+    x
+        Batch of instances.
+    model
+        Model used for preprocessing.
+    device
+        Device type used. The default None tries to use the GPU and falls back on CPU.
+        Can be specified by passing either torch.device('cuda') or torch.device('cpu').
+    preprocess_batch_fn
+        Optional batch preprocessing function. For example to convert a list of objects
+        to a batch which can be processed by the PyTorch model.
+    batch_size
+        Batch size used during prediction.
+    dtype
+        Model output type, e.g. np.float32 or torch.float32.
+
+    Returns
+    -------
+    Numpy array or torch tensor with predictions.
+    """
+    return predict_batch(
+        x,
+        model,
+        device=device,
+        batch_size=batch_size,
+        preprocess_fn=preprocess_batch_fn,
+        dtype=dtype,
+    )
+
+
+@torch.jit.script
+def squared_pairwise_distance(
+    x: torch.Tensor, y: torch.Tensor, a_min: float = 1e-30
+) -> torch.Tensor:  # pragma: no cover - torch.jit.script code is compiled and copied
+    """
+    PyTorch pairwise squared Euclidean distance between samples x and y.
+
+    Parameters
+    ----------
+    x
+        Batch of instances of shape [Nx, features].
+    y
+        Batch of instances of shape [Ny, features].
+    a_min
+        Lower bound to clip distance values.
+    Returns
+    -------
+    Pairwise squared Euclidean distance [Nx, Ny].
+    """
+    x2 = x.pow(2).sum(dim=-1, keepdim=True)
+    y2 = y.pow(2).sum(dim=-1, keepdim=True)
+    dist = torch.addmm(y2.transpose(-2, -1), x, y.transpose(-2, -1), alpha=-2).add_(x2)
+    return dist.clamp_min_(a_min)
+
+
+def sigma_median(x: torch.Tensor, y: torch.Tensor, dist: torch.Tensor) -> torch.Tensor:
+    """
+    Bandwidth estimation using the median heuristic :cite:t:`Gretton2012`.
+
+    Parameters
+    ----------
+    x
+        Tensor of instances with dimension [Nx, features].
+    y
+        Tensor of instances with dimension [Ny, features].
+    dist
+        Tensor with dimensions [Nx, Ny], containing the pairwise distances
+        between `x` and `y`.
+
+    Returns
+    -------
+    The computed bandwidth, `sigma`.
+    """
+    n = min(x.shape[0], y.shape[0])
+    n = n if (x[:n] == y[:n]).all() and x.shape == y.shape else 0
+    n_median = n + (np.prod(dist.shape) - n) // 2 - 1
+    sigma = (0.5 * dist.flatten().sort().values[int(n_median)].unsqueeze(dim=-1)) ** 0.5
+    return sigma
+
+
+class GaussianRBF(nn.Module):
+    """
+    Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2). A forward pass
+    takes a batch of instances x [Nx, features] and y [Ny, features] and returns
+    the kernel matrix [Nx, Ny].
+
+    Parameters
+    ----------
+    sigma : Optional[torch.Tensor], default None
+        Bandwidth used for the kernel. Needn't be specified if being inferred or
+        trained. Can pass multiple values to eval kernel with and then average.
+    init_sigma_fn : Optional[Callable], default None
+        Function used to compute the bandwidth `sigma`. Used when `sigma` is to be
+        inferred. The function's signature should take in the tensors `x`, `y` and
+        `dist` and return `sigma`. If `None`, it is set to
+        :func:`~dataeval._internal.detectors.drift.torch.sigma_median`.
+    trainable : bool, default False
+        Whether or not to track gradients w.r.t. `sigma` to allow it to be trained.
+    """
+
+    def __init__(
+        self,
+        sigma: Optional[torch.Tensor] = None,
+        init_sigma_fn: Optional[Callable] = None,
+        trainable: bool = False,
+    ) -> None:
+        super().__init__()
+        init_sigma_fn = sigma_median if init_sigma_fn is None else init_sigma_fn
+        self.config = {
+            "sigma": sigma,
+            "trainable": trainable,
+            "init_sigma_fn": init_sigma_fn,
+        }
+        if sigma is None:
+            self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable)
+            self.init_required = True
+        else:
+            sigma = sigma.reshape(-1)  # [Ns,]
+            self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable)
+            self.init_required = False
+        self.init_sigma_fn = init_sigma_fn
+        self.trainable = trainable
+
+    @property
+    def sigma(self) -> torch.Tensor:
+        return self.log_sigma.exp()
+
+    def forward(
+        self,
+        x: Union[np.ndarray, torch.Tensor],
+        y: Union[np.ndarray, torch.Tensor],
+        infer_sigma: bool = False,
+    ) -> torch.Tensor:
+        x, y = torch.as_tensor(x), torch.as_tensor(y)
+        dist = squared_pairwise_distance(x.flatten(1), y.flatten(1))  # [Nx, Ny]
+
+        if infer_sigma or self.init_required:
+            if self.trainable and infer_sigma:
+                raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value")
+            sigma = self.init_sigma_fn(x, y, dist)
+            with torch.no_grad():
+                self.log_sigma.copy_(sigma.log().clone())
+            self.init_required = False
+
+        gamma = 1.0 / (2.0 * self.sigma**2)  # [Ns,]
+        # TODO: do matrix multiplication after all?
+        kernel_mat = torch.exp(-torch.cat([(g * dist)[None, :, :] for g in gamma], dim=0))  # [Ns, Nx, Ny]
+        return kernel_mat.mean(dim=0)  # [Nx, Ny]

dataeval/_internal/detectors/drift/uncertainty.py

@@ -0,0 +1,149 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from functools import partial
+from typing import Callable, Dict, Literal, Optional, Union
+
+import numpy as np
+from scipy.special import softmax
+from scipy.stats import entropy
+
+from .base import UpdateStrategy
+from .ks import DriftKS
+from .torch import get_device, preprocess_drift
+
+
+def classifier_uncertainty(
+    x: np.ndarray,
+    model_fn: Callable,
+    preds_type: Literal["probs", "logits"] = "probs",
+) -> np.ndarray:
+    """
+    Evaluate model_fn on x and transform predictions to prediction uncertainties.
+
+    Parameters
+    ----------
+    x
+        Batch of instances.
+    model_fn
+        Function that evaluates a classification model on x in a single call (contains
+        batching logic if necessary).
+    preds_type
+        Type of prediction output by the model. Options are 'probs' (in [0,1]) or
+        'logits' (in [-inf,inf]).
+
+    Returns
+    -------
+    A scalar indication of uncertainty of the model on each instance in x.
+    """
+
+    preds = model_fn(x)
+
+    if preds_type == "probs":
+        if np.abs(1 - np.sum(preds, axis=-1)).mean() > 1e-6:
+            raise ValueError("Probabilities across labels should sum to 1")
+        probs = preds
+    elif preds_type == "logits":
+        probs = softmax(preds, axis=-1)
+    else:
+        raise NotImplementedError("Only prediction types 'probs' and 'logits' supported.")
+
+    uncertainties = entropy(probs, axis=-1)
+    return uncertainties[:, None]  # Detectors expect N x d  # type: ignore
+
+
+class DriftUncertainty:
+    """
+    Test for a change in the number of instances falling into regions on which the
+    model is uncertain. Performs a K-S test on prediction entropies.
+
+    Parameters
+    ----------
+    x_ref : np.ndarray
+        Data used as reference distribution. Should be disjoint from the data the
+        model was trained on for accurate p-values.
+    model : Callable
+        Classification model outputting class probabilities (or logits)
+    p_val : float, default 0.05
+        p-value used for the significance of the test.
+    x_ref_preprocessed : bool, default False
+        Whether the given reference data `x_ref` has been preprocessed yet. If
+        `x_ref_preprocessed=True`, only the test data `x` will be preprocessed at
+        prediction time. If `x_ref_preprocessed=False`, the reference data will
+        also be preprocessed.
+    update_x_ref : Optional[UpdateStrategy], default None
+        Reference data can optionally be updated using an UpdateStrategy class. Update
+        using the last n instances seen by the detector with
+        :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
+        or via reservoir sampling with
+        :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
+    preds_type : Literal["probs", "logits"], default "logits"
+        Type of prediction output by the model. Options are 'probs' (in [0,1]) or
+        'logits' (in [-inf,inf]).
+    batch_size : int, default 32
+        Batch size used to evaluate model. Only relevant when backend has been
+        specified for batch prediction.
+    preprocess_batch_fn : Optional[Callable], default None
+        Optional batch preprocessing function. For example to convert a list of
+        objects to a batch which can be processed by the model.
+    device : Optional[str], default None
+        Device type used. The default None tries to use the GPU and falls back on
+        CPU if needed. Can be specified by passing either 'cuda', 'gpu' or 'cpu'.
+    input_shape : Optional[tuple], default None
+        Shape of input data.
+    """
+
+    def __init__(
+        self,
+        x_ref: np.ndarray,
+        model: Callable,
+        p_val: float = 0.05,
+        x_ref_preprocessed: bool = False,
+        update_x_ref: Optional[UpdateStrategy] = None,
+        preds_type: Literal["probs", "logits"] = "probs",
+        batch_size: int = 32,
+        preprocess_batch_fn: Optional[Callable] = None,
+        device: Optional[str] = None,
+    ) -> None:
+        def model_fn(x: np.ndarray) -> np.ndarray:
+            return preprocess_drift(
+                x,
+                model,  # type: ignore
+                batch_size=batch_size,
+                preprocess_batch_fn=preprocess_batch_fn,
+                device=get_device(device),
+            )
+
+        preprocess_fn = partial(
+            classifier_uncertainty,
+            model_fn=model_fn,
+            preds_type=preds_type,
+        )
+
+        self._detector = DriftKS(
+            x_ref=x_ref,
+            p_val=p_val,
+            x_ref_preprocessed=x_ref_preprocessed,
+            update_x_ref=update_x_ref,
+            preprocess_fn=preprocess_fn,
+        )
+
+    def predict(self, x: np.ndarray) -> Dict[str, Union[int, float, np.ndarray]]:
+        """
+        Predict whether a batch of data has drifted from the reference data.
+
+        Parameters
+        ----------
+        x
+            Batch of instances.
+
+        Returns
+        -------
+        Dictionary containing the drift prediction, p-value, and threshold statistics.
+        """
+        return self._detector.predict(x)

dataeval/_internal/detectors/duplicates.py

@@ -0,0 +1,49 @@
+from typing import Dict, List, Literal
+
+import numpy as np
+
+from dataeval._internal.flags import ImageHash
+from dataeval._internal.metrics.stats import ImageStats
+
+
+class Duplicates:
+    """
+    Finds the duplicate images in a dataset using xxhash for exact duplicates
+    and pchash for near duplicates
+    """
+
+    def __init__(
+        self,
+        images: np.ndarray,
+    ):
+        self.stats = ImageStats(ImageHash.ALL)
+        self.images = images
+
+    def _get_duplicates(self) -> dict:
+        exact = {}
+        near = {}
+        for i, value in enumerate(self.results["xxhash"]):
+            exact.setdefault(value, []).append(i)
+        for i, value in enumerate(self.results["pchash"]):
+            near.setdefault(value, []).append(i)
+        exact = [v for v in exact.values() if len(v) > 1]
+        near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+
+        return {
+            "exact": sorted(exact),
+            "near": sorted(near),
+        }
+
+    def evaluate(self) -> Dict[Literal["exact", "near"], List[int]]:
+        """
+        Returns duplicate image indices for both exact matches and near matches
+
+        Returns
+        -------
+        Dict[Literal["exact", "near"], List[int]]
+            Dictionary of exact and near match indices
+        """
+        self.stats.reset()
+        self.stats.update(self.images)
+        self.results = self.stats.compute()
+        return self._get_duplicates()

dataeval/_internal/detectors/linter.py

@@ -0,0 +1,78 @@
+from typing import Literal, Optional, Sequence, Union
+
+import numpy as np
+
+from dataeval._internal.flags import ImageProperty, ImageVisuals, LinterFlags
+from dataeval._internal.metrics.stats import ImageStats
+
+
+def _get_outlier_mask(
+    values: np.ndarray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
+) -> np.ndarray:
+    if method == "zscore":
+        threshold = threshold if threshold else 3.0
+        std = np.std(values)
+        abs_diff = np.abs(values - np.mean(values))
+        return (abs_diff / std) > threshold
+    elif method == "modzscore":
+        threshold = threshold if threshold else 3.5
+        abs_diff = np.abs(values - np.median(values))
+        med_abs_diff = np.median(abs_diff)
+        mod_z_score = 0.6745 * abs_diff / med_abs_diff
+        return mod_z_score > threshold
+    elif method == "iqr":
+        threshold = threshold if threshold else 1.5
+        qrt = np.percentile(values, q=(25, 75), method="midpoint")
+        iqr = (qrt[1] - qrt[0]) * threshold
+        return (values < (qrt[0] - iqr)) | (values > (qrt[1] + iqr))
+    else:
+        raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
+
+
+class Linter:
+    """
+    Calculates statistical outliers of a dataset using various statistical
+    tests applied to each image
+    """
+
+    def __init__(
+        self,
+        images: np.ndarray,
+        flags: Optional[Union[LinterFlags, Sequence[LinterFlags]]] = None,
+    ):
+        flags = flags if flags is not None else (ImageProperty.ALL, ImageVisuals.ALL)
+        self.stats = ImageStats(flags)
+        self.images = images
+
+    def _get_outliers(
+        self,
+        outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
+        outlier_threshold: Optional[float] = None,
+    ) -> dict:
+        flagged_images = {}
+
+        for stat, values in self.results.items():
+            if not isinstance(values, np.ndarray):
+                continue
+
+            if values.ndim == 1 and np.std(values) != 0:
+                mask = _get_outlier_mask(values, outlier_method, outlier_threshold)
+                indices = np.flatnonzero(mask)
+                for i, value in zip(indices, values[mask]):
+                    flagged_images.setdefault(i, {}).update({stat: np.round(value, 2)})
+
+        return dict(sorted(flagged_images.items()))
+
+    def evaluate(self) -> dict:
+        """
+        Returns indices of outliers with and the issues identified for each
+
+        Returns
+        -------
+        Dict[int, Dict[str, float]]
+            Dictionary containing the indices of outliers and a dictionary issues and calculated values
+        """
+        self.stats.reset()
+        self.stats.update(self.images)
+        self.results = self.stats.compute()
+        return self._get_outliers()

dataeval/_internal/detectors/ood/ae.py

@@ -0,0 +1,77 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable
+
+import keras
+import numpy as np
+
+from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.models.tensorflow.autoencoder import AE
+from dataeval._internal.models.tensorflow.utils import predict_batch
+
+
+class OOD_AE(OODBase):
+    def __init__(self, model: AE) -> None:
+        """
+        Autoencoder based out-of-distribution detector.
+
+        Parameters
+        ----------
+        model : AE
+            An Autoencoder model.
+        """
+        super().__init__(model)
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable = keras.losses.MeanSquaredError(),
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Train the AE model with recommended loss function and optimizer.
+
+        Parameters
+        ----------
+        x_ref : np.ndarray
+            Training batch.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : Callable, default keras.losses.MeanSquaredError()
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        """
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X)
+
+        # reconstruct instances
+        X_recon = predict_batch(X, self.model, batch_size=batch_size)
+
+        # compute feature and instance level scores
+        fscore = np.power(X - X_recon, 2)
+        fscore_flat = fscore.reshape(fscore.shape[0], -1).copy()
+        n_score_features = int(np.ceil(fscore_flat.shape[1]))
+        sorted_fscore = np.sort(fscore_flat, axis=1)
+        sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
+        iscore = np.mean(sorted_fscore_perc, axis=1)
+
+        return OODScore(iscore, fscore)