dataeval 0.61.0__py3-none-any.whl → 0.63.0__py3-none-any.whl

dataeval/_internal/detectors/linter.py

@@ -1,8 +1,9 @@
-from typing import Literal, Optional, Sequence, Union
+from typing import Iterable, Literal, Optional, Sequence, Union
 
 import numpy as np
 
 from dataeval._internal.flags import ImageProperty, ImageVisuals, LinterFlags
+from dataeval._internal.interop import ArrayLike
 from dataeval._internal.metrics.stats import ImageStats
 
 
@@ -30,25 +31,79 @@ def _get_outlier_mask(
 
 
 class Linter:
-    """
-    Calculates statistical outliers of a dataset using various statistical
-    tests applied to each image
+    r"""
+    Calculates statistical outliers of a dataset using various statistical tests applied to each image
+
+    Parameters
+    ----------
+    flags : [ImageProperty | ImageStatistics | ImageVisuals], default None
+        Metric(s) to calculate for each image - calculates all metrics if None
+    outlier_method : ["modzscore" | "zscore" | "iqr"], optional - default "modzscore"
+        Statistical method used to identify outliers
+    outlier_threshold : float, optional - default None
+        Threshold value for the given ``outlier_method``, above which data is considered an outlier.
+        Uses method specific default if `None`
+
+    Attributes
+    ----------
+    stats : ImageStats
+        Class to hold the value of each metric for each image
+
+    See Also
+    --------
+    Duplicates
+
+    Notes
+    ------
+    There are 3 different statistical methods:
+
+    - zscore
+    - modzscore
+    - iqr
+
+    | The z score method is based on the difference between the data point and the mean of the data.
+        The default threshold value for `zscore` is 3.
+    | Z score = :math:`|x_i - \mu| / \sigma`
+
+    | The modified z score method is based on the difference between the data point and the median of the data.
+        The default threshold value for `modzscore` is 3.5.
+    | Modified z score = :math:`0.6745 * |x_i - x̃| / MAD`, where :math:`MAD` is the median absolute deviation
+
+    | The interquartile range method is based on the difference between the data point and
+        the difference between the 75th and 25th qartile. The default threshold value for `iqr` is 1.5.
+    | Interquartile range = :math:`threshold * (Q_3 - Q_1)`
+
+    Examples
+    --------
+    Initialize the Linter class:
+
+    >>> lint = Linter()
+
+    Specifying specific metrics to analyze:
+
+    >>> lint = Linter(flags=[ImageProperty.SIZE, ImageVisuals.ALL])
+
+    Specifying an outlier method:
+
+    >>> lint = Linter(outlier_method="iqr")
+
+    Specifying an outlier method and threshold:
+
+    >>> lint = Linter(outlier_method="zscore", outlier_threshold=2.5)
     """
 
     def __init__(
         self,
-        images: np.ndarray,
         flags: Optional[Union[LinterFlags, Sequence[LinterFlags]]] = None,
+        outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
+        outlier_threshold: Optional[float] = None,
     ):
         flags = flags if flags is not None else (ImageProperty.ALL, ImageVisuals.ALL)
         self.stats = ImageStats(flags)
-        self.images = images
+        self.outlier_method: Literal["zscore", "modzscore", "iqr"] = outlier_method
+        self.outlier_threshold = outlier_threshold
 
-    def _get_outliers(
-        self,
-        outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
-        outlier_threshold: Optional[float] = None,
-    ) -> dict:
+    def _get_outliers(self) -> dict:
         flagged_images = {}
 
         for stat, values in self.results.items():
@@ -56,23 +111,37 @@ class Linter:
                 continue
 
             if values.ndim == 1 and np.std(values) != 0:
-                mask = _get_outlier_mask(values, outlier_method, outlier_threshold)
+                mask = _get_outlier_mask(values, self.outlier_method, self.outlier_threshold)
                 indices = np.flatnonzero(mask)
                 for i, value in zip(indices, values[mask]):
                     flagged_images.setdefault(i, {}).update({stat: np.round(value, 2)})
 
         return dict(sorted(flagged_images.items()))
 
-    def evaluate(self) -> dict:
+    def evaluate(self, images: Iterable[ArrayLike]) -> dict:
         """
-        Returns indices of outliers with and the issues identified for each
+        Returns indices of outliers with the issues identified for each
+
+        Parameters
+        ----------
+        images : Iterable[ArrayLike], shape - (N, C, H, W)
+            A dataset in an ArrayLike format.
+            Function expects the data to have 3 dimensions, CxHxW.
 
         Returns
         -------
         Dict[int, Dict[str, float]]
-            Dictionary containing the indices of outliers and a dictionary issues and calculated values
+            Dictionary containing the indices of outliers and a dictionary showing
+            the issues and calculated values for the given index.
+
+        Example
+        -------
+        Evaluate the dataset:
+
+        >>> lint.evaluate(images)
+        {18: {'brightness': 0.78}, 25: {'brightness': 0.98}}
         """
         self.stats.reset()
-        self.stats.update(self.images)
+        self.stats.update(images)
         self.results = self.stats.compute()
         return self._get_outliers()

dataeval/_internal/detectors/ood/ae.py

@@ -12,6 +12,7 @@ import keras
 import numpy as np
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AE
 from dataeval._internal.models.tensorflow.utils import predict_batch
 
@@ -30,7 +31,7 @@ class OOD_AE(OODBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float = 100.0,
         loss_fn: Callable = keras.losses.MeanSquaredError(),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
@@ -43,7 +44,7 @@ class OOD_AE(OODBase):
 
         Parameters
         ----------
-        x_ref : np.ndarray
+        x_ref : ArrayLike
             Training batch.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
@@ -58,10 +59,10 @@ class OOD_AE(OODBase):
         verbose : bool, default True
             Whether to print training progress.
         """
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+        super().fit(to_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
-        self._validate(X)
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X := to_numpy(X))
 
         # reconstruct instances
         X_recon = predict_batch(X, self.model, batch_size=batch_size)

dataeval/_internal/detectors/ood/aegmm.py

@@ -9,9 +9,9 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable
 
 import keras
-import numpy as np
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import LossGMM
@@ -32,7 +32,7 @@ class OOD_AEGMM(OODGMMBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float = 100.0,
         loss_fn: Callable = LossGMM(),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
@@ -45,7 +45,7 @@ class OOD_AEGMM(OODGMMBase):
 
         Parameters
         ----------
-        x_ref : np.ndarray
+        x_ref : ArrayLike
             Training batch.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
@@ -62,8 +62,8 @@ class OOD_AEGMM(OODGMMBase):
         """
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
-        self._validate(X)
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X := to_numpy(X))
         _, z, _ = predict_batch(X, self.model, batch_size=batch_size)
         energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
         return OODScore(energy.numpy())  # type: ignore

dataeval/_internal/detectors/ood/base.py

@@ -13,6 +13,7 @@ import keras
 import numpy as np
 import tensorflow as tf
 
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
 from dataeval._internal.models.tensorflow.trainer import trainer
 
@@ -66,13 +67,13 @@ class OODBase(ABC):
         self._validate(X)
 
     @abstractmethod
-    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
         """
         Compute instance and (optionally) feature level outlier scores.
 
         Parameters
         ----------
-        X : np.ndarray
+        X : ArrayLike
             Batch of instances.
         batch_size : int, default int(1e10)
             Batch size used when making predictions with the autoencoder.
@@ -87,7 +88,7 @@ class OODBase(ABC):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float,
         loss_fn: Callable,
         optimizer: keras.optimizers.Optimizer,
@@ -100,7 +101,7 @@ class OODBase(ABC):
 
         Parameters
         ----------
-        x_ref: : np.ndarray
+        x_ref: : ArrayLike
             Training batch.
         threshold_perc : float
             Percentage of reference data that is normal.
@@ -119,7 +120,7 @@ class OODBase(ABC):
         trainer(
             model=self.model,
             loss_fn=loss_fn,
-            x_train=x_ref,
+            x_train=to_numpy(x_ref),
             optimizer=optimizer,
             epochs=epochs,
             batch_size=batch_size,
@@ -132,7 +133,7 @@ class OODBase(ABC):
 
     def predict(
         self,
-        X: np.ndarray,
+        X: ArrayLike,
         batch_size: int = int(1e10),
         ood_type: Literal["feature", "instance"] = "instance",
     ) -> Dict[str, np.ndarray]:
@@ -141,18 +142,18 @@ class OODBase(ABC):
 
         Parameters
         ----------
-        X
+        X : ArrayLike
             Batch of instances.
-        ood_type
-            Predict out-of-distribution at the 'feature' or 'instance' level.
-        batch_size
+        batch_size : int, default int(1e10)
             Batch size used when making predictions with the autoencoder.
+        ood_type : Literal["feature", "instance"], default "instance"
+            Predict out-of-distribution at the 'feature' or 'instance' level.
 
         Returns
         -------
         Dictionary containing the outlier predictions and both feature and instance level outlier scores.
         """
-        self._validate_state(X)
+        self._validate_state(X := to_numpy(X))
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
         ood_pred = (score.get(ood_type) > self._threshold_score(ood_type)).astype(int)
@@ -171,7 +172,7 @@ class OODGMMBase(OODBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float,
         loss_fn: Callable[[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor], tf.Tensor],
         optimizer: keras.optimizers.Optimizer,
@@ -183,7 +184,7 @@ class OODGMMBase(OODBase):
         trainer(
             model=self.model,
             loss_fn=loss_fn,
-            x_train=x_ref,
+            x_train=to_numpy(x_ref),
             optimizer=optimizer,
             epochs=epochs,
             batch_size=batch_size,

dataeval/_internal/detectors/ood/llr.py

@@ -16,6 +16,7 @@ from keras.layers import Input
 from keras.models import Model
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
 from dataeval._internal.models.tensorflow.trainer import trainer
 from dataeval._internal.models.tensorflow.utils import predict_batch
@@ -125,7 +126,7 @@ class OOD_LLR(OODBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float = 100.0,
         loss_fn: Optional[Callable] = None,
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
@@ -141,7 +142,7 @@ class OOD_LLR(OODBase):
 
         Parameters
         ----------
-        x_ref : np.ndarray
+        x_ref : ArrayLike
             Training batch.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
@@ -163,6 +164,7 @@ class OOD_LLR(OODBase):
         mutate_batch_size: int, default int(1e10)
             Batch size used to generate the mutations for the background dataset.
         """
+        x_ref = to_numpy(x_ref)
         input_shape = x_ref.shape[1:]
         optimizer = optimizer() if isinstance(optimizer, type) else optimizer
         # Separate into two separate optimizers, one for semantic model and one for background model
@@ -275,10 +277,10 @@ class OOD_LLR(OODBase):
 
     def score(
         self,
-        X: np.ndarray,
+        X: ArrayLike,
         batch_size: int = int(1e10),
     ) -> OODScore:
-        self._validate(X)
+        self._validate(X := to_numpy(X))
         fscore = -self._llr(X, True, batch_size=batch_size)
         iscore = -self._llr(X, False, batch_size=batch_size)
         return OODScore(iscore, fscore)

dataeval/_internal/detectors/ood/vae.py

@@ -12,6 +12,7 @@ import keras
 import numpy as np
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAE
 from dataeval._internal.models.tensorflow.losses import Elbo
 from dataeval._internal.models.tensorflow.utils import predict_batch
@@ -34,7 +35,7 @@ class OOD_VAE(OODBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float = 100.0,
         loss_fn: Callable = Elbo(0.05),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
@@ -47,7 +48,7 @@ class OOD_VAE(OODBase):
 
         Parameters
         ----------
-        x_ref : np.ndarray
+        x_ref : ArrayLike
             Training batch.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
@@ -64,8 +65,8 @@ class OOD_VAE(OODBase):
         """
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
-        self._validate(X)
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X := to_numpy(X))
 
         # sample reconstructed instances
         X_samples = np.repeat(X, self.samples, axis=0)

dataeval/_internal/detectors/ood/vaegmm.py

@@ -12,6 +12,7 @@ import keras
 import numpy as np
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.interop import ArrayLike, to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM
@@ -35,7 +36,7 @@ class OOD_VAEGMM(OODGMMBase):
 
     def fit(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         threshold_perc: float = 100.0,
         loss_fn: Callable = LossGMM(elbo=Elbo(0.05)),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
@@ -48,7 +49,7 @@ class OOD_VAEGMM(OODGMMBase):
 
         Parameters
         ----------
-        X : np.ndarray
+        X : ArrayLike
             Training batch.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
@@ -65,8 +66,8 @@ class OOD_VAEGMM(OODGMMBase):
         """
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
-        self._validate(X)
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X := to_numpy(X))
 
         # draw samples from latent space
         X_samples = np.repeat(X, self.samples, axis=0)

dataeval/_internal/functional/ber.py

@@ -0,0 +1,63 @@
+from typing import Tuple
+
+import numpy as np
+from scipy.sparse import coo_matrix
+from scipy.stats import mode
+
+from dataeval._internal.functional.utils import compute_neighbors, get_classes_counts, minimum_spanning_tree
+
+
+def ber_mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using a minimum spanning tree
+
+    Parameters
+    ----------
+    X : np.ndarray (N, :)
+        Data points with arbitrary dimensionality
+    y : np.ndarray (N, 1)
+        Labels for each data point
+    """
+
+    M, N = get_classes_counts(y)
+
+    tree = coo_matrix(minimum_spanning_tree(X))
+    matches = np.sum([y[tree.row[i]] != y[tree.col[i]] for i in range(N - 1)])
+    deltas = matches / (2 * N)
+    upper = 2 * deltas
+    lower = ((M - 1) / (M)) * (1 - max(1 - 2 * ((M) / (M - 1)) * deltas, 0) ** 0.5)
+    return upper, lower
+
+
+def ber_knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using K-nearest neighbors"""
+
+    M, N = get_classes_counts(y)
+
+    # All features belong on second dimension
+    X = X.reshape((X.shape[0], -1))
+    nn_indices = compute_neighbors(X, X, k=k)
+    nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
+    modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
+    upper = float(np.count_nonzero(modal_class - y) / N)
+    lower = _knn_lowerbound(upper, M, k)
+    return upper, lower
+
+
+def _knn_lowerbound(value: float, classes: int, k: int) -> float:
+    """Several cases for computing the BER lower bound"""
+    if value <= 1e-10:
+        return 0.0
+
+    if classes == 2 and k != 1:
+        if k > 5:
+            # Property 2 (Devroye, 1981) cited in Snoopy paper, not in snoopy repo
+            alpha = 0.3399
+            beta = 0.9749
+            a_k = alpha * np.sqrt(k) / (k - 3.25) * (1 + beta / (np.sqrt(k - 3)))
+            return value / (1 + a_k)
+        if k > 2:
+            return value / (1 + (1 / np.sqrt(k)))
+        # k == 2:
+        return value / 2
+
+    return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))

dataeval/_internal/functional/coverage.py

@@ -0,0 +1,75 @@
+import math
+from typing import Literal, Tuple
+
+import numpy as np
+from scipy.spatial.distance import pdist, squareform
+
+
+def coverage(
+    embeddings: np.ndarray,
+    radius_type: Literal["adaptive", "naive"] = "adaptive",
+    k: int = 20,
+    percent: np.float64 = np.float64(0.01),
+) -> Tuple[np.ndarray, np.ndarray, float]:
+    """
+    Perform a one-way chi-squared test between observation frequencies and expected frequencies that
+    tests the null hypothesis that the observed data has the expected frequencies.
+
+    Parameters
+    ----------
+    embeddings : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+    radius_type : Literal["adaptive", "naive"], default "adaptive"
+        The function used to determine radius.
+    k: int, default 20
+        Number of observations required in order to be covered.
+        [1] suggests that a minimum of 20-50 samples is necessary.
+    percent: np.float64, default np.float(0.01)
+        Percent of observations to be considered uncovered. Only applies to adaptive radius.
+
+    Returns
+    -------
+    np.ndarray
+        Array of uncovered indices
+    np.ndarray
+        Array of critical value radii
+    float
+        Radius for coverage
+
+    Raises
+    ------
+    ValueError
+        If length of embeddings is less than or equal to k
+    ValueError
+        If radius_type is unknown
+
+    Note
+    ----
+    Embeddings should be on the unit interval.
+
+    Reference
+    ---------
+    This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
+    [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
+    """
+
+    # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
+    n = len(embeddings)
+    if n <= k:
+        raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
+    mat = squareform(pdist(embeddings))
+    sorted_dists = np.sort(mat, axis=1)
+    crit = sorted_dists[:, k + 1]
+
+    d = np.shape(embeddings)[1]
+    if radius_type == "naive":
+        rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
+        pvals = np.where(crit > rho)[0]
+    elif radius_type == "adaptive":
+        # Use data adaptive cutoff as rho
+        rho = int(n * percent)
+        pvals = np.argsort(crit)[::-1][:rho]
+    else:
+        raise ValueError("Invalid radius type.")
+    return pvals, crit, rho

dataeval/_internal/functional/divergence.py

@@ -0,0 +1,16 @@
+import numpy as np
+
+from .utils import compute_neighbors, minimum_spanning_tree
+
+
+def divergence_mst(data: np.ndarray, labels: np.ndarray) -> int:
+    mst = minimum_spanning_tree(data).toarray()
+    edgelist = np.transpose(np.nonzero(mst))
+    errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
+    return errors
+
+
+def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
+    nn_indices = compute_neighbors(data, data)
+    errors = np.sum(np.abs(labels[nn_indices] - labels))
+    return errors

dataeval/_internal/{metrics → functional}/hash.py

@@ -3,7 +3,7 @@ import xxhash as xxh
 from PIL import Image
 from scipy.fftpack import dct
 
-from dataeval._internal.metrics.utils import normalize_image_shape, rescale
+from dataeval._internal.functional.utils import normalize_image_shape, rescale
 
 HASH_SIZE = 8
 MAX_FACTOR = 4