dataeval 0.61.0__py3-none-any.whl

dataeval/_internal/detectors/ood/vaegmm.py

@@ -0,0 +1,79 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable
+
+import keras
+import numpy as np
+
+from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
+from dataeval._internal.models.tensorflow.gmm import gmm_energy
+from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM
+from dataeval._internal.models.tensorflow.utils import predict_batch
+
+
+class OOD_VAEGMM(OODGMMBase):
+    def __init__(self, model: VAEGMM, samples: int = 10) -> None:
+        """
+        VAE with Gaussian Mixture Model based outlier detector.
+
+        Parameters
+        ----------
+        model : VAEGMM
+            A VAEGMM model.
+        samples
+            Number of samples sampled to evaluate each instance.
+        """
+        super().__init__(model)
+        self.samples = samples
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable = LossGMM(elbo=Elbo(0.05)),
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Train the AE model with recommended loss function and optimizer.
+
+        Parameters
+        ----------
+        X : np.ndarray
+            Training batch.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : Callable, default LossGMM(elbo=Elbo(0.05))
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        """
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X)
+
+        # draw samples from latent space
+        X_samples = np.repeat(X, self.samples, axis=0)
+        _, z, _ = predict_batch(X_samples, self.model, batch_size=batch_size)
+
+        # compute average energy for samples
+        energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
+        energy_samples = energy.numpy().reshape((-1, self.samples))  # type: ignore
+        iscore = np.mean(energy_samples, axis=-1)
+        return OODScore(iscore)

dataeval/_internal/flags.py

@@ -0,0 +1,47 @@
+from enum import Flag, auto
+from typing import Union
+
+
+class auto_all:
+    def __get__(self, _, cls):
+        return ~cls(0)
+
+
+class ImageHash(Flag):
+    XXHASH = auto()
+    PCHASH = auto()
+    ALL = auto_all()
+
+
+class ImageProperty(Flag):
+    WIDTH = auto()
+    HEIGHT = auto()
+    SIZE = auto()
+    ASPECT_RATIO = auto()
+    CHANNELS = auto()
+    DEPTH = auto()
+    ALL = auto_all()
+
+
+class ImageVisuals(Flag):
+    BRIGHTNESS = auto()
+    BLURRINESS = auto()
+    MISSING = auto()
+    ZERO = auto()
+    ALL = auto_all()
+
+
+class ImageStatistics(Flag):
+    MEAN = auto()
+    STD = auto()
+    VAR = auto()
+    SKEW = auto()
+    KURTOSIS = auto()
+    ENTROPY = auto()
+    PERCENTILES = auto()
+    HISTOGRAM = auto()
+    ALL = auto_all()
+
+
+ImageStatsFlags = Union[ImageHash, ImageProperty, ImageVisuals, ImageStatistics]
+LinterFlags = Union[ImageProperty, ImageVisuals, ImageStatistics]

dataeval/_internal/metrics/base.py

@@ -0,0 +1,92 @@
+from abc import ABC, abstractmethod
+from typing import Callable, Dict, Generic, List, TypeVar
+
+TOutput = TypeVar("TOutput", bound=dict)
+TMethods = TypeVar("TMethods")
+TCallable = TypeVar("TCallable", bound=Callable)
+
+
+class MetricMixin(ABC, Generic[TOutput]):
+    @abstractmethod
+    def update(self, preds, targets): ...
+
+    @abstractmethod
+    def compute(self) -> TOutput: ...
+
+    @abstractmethod
+    def reset(self): ...
+
+
+class EvaluateMixin(ABC, Generic[TOutput]):
+    @abstractmethod
+    def evaluate(self) -> TOutput:
+        """Abstract method to calculate metric based off of constructor parameters"""
+
+
+class MethodsMixin(ABC, Generic[TMethods, TCallable]):
+    """
+    Use this mixin to define a mapping of functions to method names which
+    can be queried by the user and called internally with the appropriate
+    method name as the key.
+
+    Explicitly defining the Callable generic helps with type safety and
+    hinting for function signatures and recommended but optional.
+
+    e.g.:
+
+    def _mult(x: float, y: float) -> float:
+        return x * y
+
+    class MyMetric(MethodsMixin[Callable[float, float], float]):
+
+        def _methods(cls) -> Dict[str, Callable[float, float], float]:
+            return {
+                "ADD": lambda x, y: x + y,
+                "MULT":  _mult,
+                ...
+            }
+
+    Then during evaluate, you can call the method specified with the getter.
+
+    e.g.:
+
+        def evaluate(self):
+            return self._method(x, y)
+
+    The resulting class can be used like so.
+
+    m = MyMetric(1.0, 2.0, "ADD")
+    m.evaluate()       #  returns 3.0
+    m.method           #  returns "ADD"
+    MyMetric.methods() #  returns "['ADD', 'MULT']
+    m.method = "MULT"
+    m.evaluate()       #  returns 2.0
+    """
+
+    @classmethod
+    @abstractmethod
+    def _methods(cls) -> Dict[str, TCallable]:
+        """Abstract method returning available method functions for class"""
+
+    @property
+    def _method(self) -> TCallable:
+        return self._methods()[self.method]
+
+    @classmethod
+    def methods(cls) -> List[str]:
+        return list(cls._methods().keys())
+
+    @property
+    def method(self) -> str:
+        return self._method_key
+
+    @method.setter
+    def method(self, value: TMethods):
+        self._set_method(value)
+
+    def _set_method(self, value: TMethods):
+        """This setter is to fix pyright incorrect detection of
+        incorrectly overriding the 'method' property"""
+        if value not in self.methods():
+            raise KeyError(f"Specified method not available for class ({self.methods()}).")
+        self._method_key = value

dataeval/_internal/metrics/ber.py

@@ -0,0 +1,124 @@
+"""
+This module contains the implementation of the
+FR Test Statistic based estimate and the
+KNN based estimate for the Bayes Error Rate
+
+Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
+https://arxiv.org/abs/1811.06419
+"""
+
+from typing import Callable, Dict, Literal, Tuple
+
+import numpy as np
+from maite.protocols import ArrayLike
+from scipy.sparse import coo_matrix
+from scipy.stats import mode
+
+from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+
+from .utils import compute_neighbors, get_classes_counts, minimum_spanning_tree
+
+
+def _mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
+    M, N = get_classes_counts(y)
+
+    tree = coo_matrix(minimum_spanning_tree(X))
+    matches = np.sum([y[tree.row[i]] != y[tree.col[i]] for i in range(N - 1)])
+    deltas = matches / (2 * N)
+    upper = 2 * deltas
+    lower = ((M - 1) / (M)) * (1 - max(1 - 2 * ((M) / (M - 1)) * deltas, 0) ** 0.5)
+    return upper, lower
+
+
+def _knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
+    M, N = get_classes_counts(y)
+
+    # All features belong on second dimension
+    X = X.reshape((X.shape[0], -1))
+    nn_indices = compute_neighbors(X, X, k=k)
+    nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
+    modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
+    upper = float(np.count_nonzero(modal_class - y) / N)
+    lower = _knn_lowerbound(upper, M, k)
+    return upper, lower
+
+
+def _knn_lowerbound(value: float, classes: int, k: int) -> float:
+    "Several cases for computing the BER lower bound"
+    if value <= 1e-10:
+        return 0.0
+
+    if classes == 2 and k != 1:
+        if k > 5:
+            # Property 2 (Devroye, 1981) cited in Snoopy paper, not in snoopy repo
+            alpha = 0.3399
+            beta = 0.9749
+            a_k = alpha * np.sqrt(k) / (k - 3.25) * (1 + beta / (np.sqrt(k - 3)))
+            return value / (1 + a_k)
+        if k > 2:
+            return value / (1 + (1 / np.sqrt(k)))
+        # k == 2:
+        return value / 2
+
+    return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))
+
+
+_METHODS = Literal["MST", "KNN"]
+_FUNCTION = Callable[[np.ndarray, np.ndarray, int], Tuple[float, float]]
+
+
+class BER(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+    """
+    An estimator for Multi-class Bayes Error Rate using FR or KNN test statistic basis
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Array of images or image embeddings
+    labels : np.ndarray
+        Array of labels for each image or image embedding
+    method : Literal["MST", "KNN"], default "KNN"
+        Method to use when estimating the Bayes error rate
+    k : int, default 1
+        number of nearest neighbors for KNN estimator -- ignored by MST estimator
+
+
+    See Also
+    --------
+    `Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4) <https://arxiv.org/abs/1811.06419>`_
+
+    """
+
+    def __init__(self, data: ArrayLike, labels: ArrayLike, method: _METHODS = "KNN", k: int = 1) -> None:
+        self.data = data
+        self.labels = labels
+        self.k = k
+        self._set_method(method)
+
+    @classmethod
+    def _methods(
+        cls,
+    ) -> Dict[str, _FUNCTION]:
+        return {"MST": _mst, "KNN": _knn}
+
+    def evaluate(self) -> Dict[str, float]:
+        """
+        Calculates the Bayes Error Rate estimate using the provided method
+
+        Returns
+        -------
+        Dict[str, float]
+            ber : float
+                The estimated lower bounds of the Bayes Error Rate
+            ber_lower : float
+                The estimated upper bounds of the Bayes Error Rate
+
+        Raises
+        ------
+        ValueError
+            If unique classes M < 2
+        """
+        data = np.asarray(self.data)
+        labels = np.asarray(self.labels)
+        upper, lower = self._method(data, labels, self.k)
+        return {"ber": upper, "ber_lower": lower}

dataeval/_internal/metrics/coverage.py

@@ -0,0 +1,80 @@
+import math
+from typing import Literal, Tuple
+
+import numpy as np
+from scipy.spatial.distance import pdist, squareform
+
+
+class Coverage:
+    """
+    Class for evaluating coverage and identifying images/samples that are in undercovered regions.
+
+    This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
+
+    Parameters
+    ----------
+    embeddings : np.ndarray
+        n x p array of image embeddings from the dataset.
+    radius_type : Literal["adaptive", "naive"], default "adaptive"
+        The function used to determine radius.
+    k: int, default 20
+        Number of observations required in order to be covered.
+    percent: np.float64, default np.float(0.01)
+        Percent of observations to be considered uncovered. Only applies to adaptive radius.
+
+    Note
+    ----
+    Embeddings should be on the unit interval.
+    """
+
+    def __init__(
+        self,
+        embeddings: np.ndarray,
+        radius_type: Literal["adaptive", "naive"] = "adaptive",
+        k: int = 20,
+        percent: np.float64 = np.float64(0.01),
+    ):
+        self.embeddings = embeddings
+        self.radius_type = radius_type
+        self.k = k
+        self.percent = percent
+
+    def evaluate(self) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Perform a one-way chi-squared test between observation frequencies and expected frequencies that
+        tests the null hypothesis that the observed data has the expected frequencies.
+
+        Returns
+        -------
+        np.ndarray
+            Array of uncovered indices
+        np.ndarray
+            Array of critical value radii
+
+        Raises
+        ------
+        ValueError
+            If length of embeddings is less than or equal to k
+        ValueError
+            If radius_type is unknown
+        """
+
+        # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
+        n = len(self.embeddings)
+        if n <= self.k:
+            raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
+        mat = squareform(pdist(self.embeddings))
+        sorted_dists = np.sort(mat, axis=1)
+        crit = sorted_dists[:, self.k + 1]
+
+        d = np.shape(self.embeddings)[1]
+        if self.radius_type == "naive":
+            self.rho = (1 / math.sqrt(math.pi)) * ((2 * self.k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
+            pvals = np.where(crit > self.rho)[0]
+        elif self.radius_type == "adaptive":
+            # Use data adaptive cutoff
+            cutoff = int(n * self.percent)
+            pvals = np.argsort(crit)[::-1][:cutoff]
+        else:
+            raise ValueError("Invalid radius type.")
+        return pvals, crit

dataeval/_internal/metrics/divergence.py

@@ -0,0 +1,94 @@
+"""
+This module contains the implementation of HP Divergence
+using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
+"""
+
+from typing import Any, Callable, Dict, Literal
+
+import numpy as np
+
+from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+
+from .utils import compute_neighbors, minimum_spanning_tree
+
+
+def _mst(data: np.ndarray, labels: np.ndarray) -> int:
+    mst = minimum_spanning_tree(data).toarray()
+    edgelist = np.transpose(np.nonzero(mst))
+    errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
+    return errors
+
+
+def _fnn(data: np.ndarray, labels: np.ndarray) -> int:
+    nn_indices = compute_neighbors(data, data)
+    errors = np.sum(np.abs(labels[nn_indices] - labels))
+    return errors
+
+
+_METHODS = Literal["MST", "FNN"]
+_FUNCTION = Callable[[np.ndarray, np.ndarray], int]
+
+
+class Divergence(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+    """
+    Calculates the estimated divergence between two datasets
+
+    Parameters
+    ----------
+    data_a : np.ndarray
+        Array of images or image embeddings to compare
+    data_b : np.ndarray
+        Array of images or image embeddings to compare
+    method : Literal["MST, "FNN"], default "MST"
+        Method used to estimate dataset divergence
+
+    See Also
+    --------
+        For more information about this divergence, its formal definition,
+        and its associated estimators see https://arxiv.org/abs/1412.6534.
+
+    Warning
+    -------
+        MST is very slow in this implementation, this is unlike matlab where
+        they have comparable speeds
+        Overall, MST takes ~25x LONGER!!
+        Source of slowdown:
+        conversion to and from CSR format adds ~10% of the time diff between
+        1nn and scipy mst function the remaining 90%
+    """
+
+    def __init__(
+        self,
+        data_a: np.ndarray,
+        data_b: np.ndarray,
+        method: _METHODS = "MST",
+    ) -> None:
+        self.data_a = data_a
+        self.data_b = data_b
+        self._set_method(method)
+
+    @classmethod
+    def _methods(cls) -> Dict[str, _FUNCTION]:
+        return {"FNN": _fnn, "MST": _mst}
+
+    def evaluate(self) -> Dict[str, Any]:
+        """
+        Calculates the divergence and any errors between the datasets
+
+        Returns
+        -------
+        Dict[str, Any]
+            dp : float
+                divergence value between 0.0 and 1.0
+            errors : int
+                the number of differing edges
+        """
+        N = self.data_a.shape[0]
+        M = self.data_b.shape[0]
+
+        stacked_data = np.vstack((self.data_a, self.data_b))
+        labels = np.vstack([np.zeros([N, 1]), np.ones([M, 1])])
+
+        errors = self._method(stacked_data, labels)
+        dp = max(0.0, 1 - ((M + N) / (2 * M * N)) * errors)
+        return {"divergence": dp, "error": errors}

dataeval/_internal/metrics/hash.py

@@ -0,0 +1,79 @@
+import numpy as np
+import xxhash as xxh
+from PIL import Image
+from scipy.fftpack import dct
+
+from dataeval._internal.metrics.utils import normalize_image_shape, rescale
+
+HASH_SIZE = 8
+MAX_FACTOR = 4
+
+
+def pchash(image: np.ndarray) -> str:
+    """
+    Performs a perceptual hash on an image by resizing to a square NxN image
+    using the Lanczos algorithm where N is 32x32 or the largest multiple of
+    8 that is smaller than the input image dimensions.  The resampled image
+    is compressed using a discrete cosine transform and the lowest frequency
+    component is encoded as a bit array of greater or less than median value
+    and returned as a hex string.
+
+    Parameters
+    ----------
+    image : np.ndarray
+        An image as a numpy array in CxHxW format
+
+    Returns
+    -------
+    str
+        The hex string hash of the image using perceptual hashing
+    """
+    # Verify that the image is at least larger than an 8x8 image
+    min_dim = min(image.shape[-2:])
+    if min_dim < HASH_SIZE + 1:
+        raise ValueError(f"Image must be larger than {HASH_SIZE}x{HASH_SIZE} for fuzzy hashing.")
+
+    # Calculates the dimensions of the resized square image
+    resize_dim = HASH_SIZE * min((min_dim - 1) // HASH_SIZE, MAX_FACTOR)
+
+    # Normalizes the image to CxHxW and takes the mean over all the channels
+    normalized = np.mean(normalize_image_shape(image), axis=0).squeeze()
+
+    # Rescales the pixel values to an 8-bit 0-255 image
+    rescaled = rescale(normalized, 8).astype(np.uint8)
+
+    # Resizes the image using the Lanczos algorithm to a square image
+    im = np.array(Image.fromarray(rescaled).resize((resize_dim, resize_dim), Image.Resampling.LANCZOS))
+
+    # Performs discrete cosine transforms to compress the image information and takes the lowest frequency component
+    transform = dct(dct(im.T).T)[:HASH_SIZE, :HASH_SIZE]
+
+    # Encodes the transform as a bit array over the median value
+    diff = transform > np.median(transform)
+
+    # Pads the front of the bit array to a multiple of 8 with False
+    padded = np.full(int(np.ceil(diff.size / 8) * 8), False)
+    padded[-diff.size :] = diff.ravel()
+
+    # Converts the bit array to a hex string and strips leading 0s
+    hash_hex = np.packbits(padded).tobytes().hex().lstrip("0")
+    return hash_hex if hash_hex else "0"
+
+
+def xxhash(image: np.ndarray) -> str:
+    """
+    Performs a fast non-cryptographic hash using the xxhash algorithm
+    (xxhash.com) against the image as a flattened bytearray.  The hash
+    is returned as a hex string.
+
+    Parameters
+    ----------
+    image : np.ndarray
+        An image as a numpy array
+
+    Returns
+    -------
+    str
+        The hex string hash of the image using the xxHash algorithm
+    """
+    return xxh.xxh3_64_hexdigest(image.ravel().tobytes())