dataeval 0.61.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/__init__.py

@@ -2,14 +2,14 @@ from importlib.util import find_spec
 
 from . import detectors, flags, metrics
 
-__version__ = "0.61.0"
+__version__ = "0.64.0"
 
 __all__ = ["detectors", "flags", "metrics"]
 
 if find_spec("torch") is not None:  # pragma: no cover
-    from . import models, workflows
+    from . import models, utils, workflows
 
-    __all__ += ["models", "workflows"]
+    __all__ += ["models", "utils", "workflows"]
 elif find_spec("tensorflow") is not None:  # pragma: no cover
     from . import models
 

dataeval/_internal/detectors/clusterer.py

@@ -1,9 +1,12 @@
 from typing import Dict, Iterable, List, NamedTuple, Tuple, Union, cast
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.cluster.hierarchy import linkage
 from scipy.spatial.distance import pdist, squareform
 
+from dataeval._internal.interop import to_numpy
+
 
 def extend_linkage(link_arr: np.ndarray) -> np.ndarray:
     """
@@ -102,20 +105,36 @@ class Clusterer:
 
     Parameters
     ----------
-    dataset : np.ndarray
-        An array of images or image embeddings to perform clustering
+    dataset : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensional space.
+
+    Warning
+    -------
+    The Clusterer class is heavily dependent on computational resources, and may fail due to insufficient memory.
+
+    Note
+    ----
+    The Clusterer works best when the length of the feature dimension, P, is less than 500.
+    If flattening a CxHxW image results in a dimension larger than 500, then it is recommended to reduce the dimensions.
+
+    Example
+    -------
+    Initialize the Clusterer class:
+
+    >>> cluster = Clusterer(dataset)
     """
 
-    def __init__(self, dataset: np.ndarray):
+    def __init__(self, dataset: ArrayLike):
         # Allows an update to dataset to reset the state rather than instantiate a new class
         self._on_init(dataset)
 
-    def _on_init(self, dataset: np.ndarray):
-        self._validate_data(dataset)
-        self._data: np.ndarray = dataset
-        self._num_samples = len(dataset)
+    def _on_init(self, dataset: ArrayLike):
+        self._data: np.ndarray = to_numpy(dataset)
+        self._validate_data(self._data)
+        self._num_samples = len(self._data)
 
-        self._darr: np.ndarray = pdist(dataset, metric="euclidean")
+        self._darr: np.ndarray = pdist(self._data, metric="euclidean")
         self._sqdmat: np.ndarray = squareform(self._darr)
         self._larr: np.ndarray = extend_linkage(linkage(self._darr))
         self._max_clusters: int = np.count_nonzero(self._larr[:, 3] == 2)
@@ -131,7 +150,7 @@ class Clusterer:
         return self._data
 
     @data.setter
-    def data(self, x: np.ndarray):
+    def data(self, x: ArrayLike):
         self._on_init(x)
 
     @property
@@ -450,20 +469,30 @@ class Clusterer:
 
         Returns
         -------
-
-        Dict[str, Union[List[int]], List[List[int]]]
-            Dictionary containing list of outliers, potential outliers, duplicates, and near duplicates in keys
-            "outliers", "potential_outliers", "duplicates", "near_duplicates" respectively
-        """
+        Dict[str, List[int]]
+            outliers :
+                List of indices that do not fall within a cluster
+            potential_outliers :
+                List of indices which are near the border between belonging in the cluster and being an outlier
+            duplicates :
+                List of groups of indices that are exact duplicates
+            potential_duplicates :
+                List of groups of indices which are not exact but closely related data points
+
+        Example
+        -------
+        >>> cluster.evaluate()
+        {'outliers': [18, 21, 34, 35, 45], 'potential_outliers': [13, 15, 42], 'duplicates': [[9, 24], [23, 48]], 'potential_duplicates': [[1, 11]]}
+        """  # noqa: E501
 
         outliers, potential_outliers = self.find_outliers(self.last_good_merge_levels)
-        duplicates, near_duplicates = self.find_duplicates(self.last_good_merge_levels)
+        duplicates, potential_duplicates = self.find_duplicates(self.last_good_merge_levels)
 
         ret = {
             "outliers": outliers,
             "potential_outliers": potential_outliers,
             "duplicates": duplicates,
-            "near_duplicates": near_duplicates,
+            "potential_duplicates": potential_duplicates,
         }
 
         return ret

dataeval/_internal/detectors/drift/base.py

@@ -8,10 +8,12 @@ Licensed under Apache Software License (Apache 2.0)
 
 from abc import ABC, abstractmethod
 from functools import wraps
-from random import random
 from typing import Callable, Dict, Literal, Optional, Tuple, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
+
+from dataeval._internal.interop import to_numpy
 
 
 def update_x_ref(fn):
@@ -94,7 +96,7 @@ class ReservoirSamplingUpdate(UpdateStrategy):
                 x_reservoir[n_ref, :] = item
                 n_ref += 1
             else:
-                r = int(random() * count)
+                r = np.random.randint(0, count)
                 if r < self.n:
                     x_reservoir[r, :] = item
         return x_reservoir
@@ -105,11 +107,11 @@ class BaseDrift:
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
         update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
     ) -> None:
         # Type checking
@@ -128,7 +130,7 @@ class BaseDrift:
         self.update_x_ref = update_x_ref
         self.preprocess_fn = preprocess_fn
         self.correction = correction
-        self.n = len(x_ref)
+        self.n = len(self._x_ref)  # type: ignore
 
         # Ref counter for preprocessed x
         self._x_refcount = 0
@@ -140,9 +142,10 @@ class BaseDrift:
             if self.preprocess_fn is not None:
                 self._x_ref = self.preprocess_fn(self._x_ref)
 
+        self._x_ref = to_numpy(self._x_ref)
         return self._x_ref
 
-    def _preprocess(self, x: np.ndarray) -> np.ndarray:
+    def _preprocess(self, x: ArrayLike) -> ArrayLike:
         """Data preprocessing before computing the drift scores."""
         if self.preprocess_fn is not None:
             x = self.preprocess_fn(x)
@@ -159,11 +162,11 @@ class BaseUnivariateDrift(BaseDrift):
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
         update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
         n_features: Optional[int] = None,
     ) -> None:
@@ -188,14 +191,14 @@ class BaseUnivariateDrift(BaseDrift):
                 self._n_features = self.x_ref.reshape(self.x_ref.shape[0], -1).shape[-1]
             else:
                 # infer number of features after applying preprocessing step
-                x = self.preprocess_fn(self.x_ref[0:1])
+                x = to_numpy(self.preprocess_fn(self._x_ref[0:1]))  # type: ignore
                 self._n_features = x.reshape(x.shape[0], -1).shape[-1]
 
         return self._n_features
 
     @preprocess_x
     @abstractmethod
-    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
         """Abstract method to calculate feature score after preprocessing"""
 
     def _apply_correction(self, p_vals: np.ndarray) -> Tuple[int, float]:
@@ -221,7 +224,7 @@ class BaseUnivariateDrift(BaseDrift):
     @update_x_ref
     def predict(
         self,
-        x: np.ndarray,
+        x: ArrayLike,
         drift_type: Literal["batch", "feature"] = "batch",
     ) -> Dict[str, Union[int, float, np.ndarray]]:
         """
@@ -230,7 +233,7 @@ class BaseUnivariateDrift(BaseDrift):
 
         Parameters
         ----------
-        x : np.ndarray
+        x : ArrayLike
             Batch of instances.
         drift_type : Literal["batch", "feature"], default "batch"
             Predict drift at the 'feature' or 'batch' level. For 'batch', the test

dataeval/_internal/detectors/drift/cvm.py

@@ -9,8 +9,11 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable, Literal, Optional, Tuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.stats import cramervonmises_2samp
 
+from dataeval._internal.interop import to_numpy
+
 from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
 
 
@@ -23,7 +26,7 @@ class DriftCVM(BaseUnivariateDrift):
 
     Parameters
     ----------
-    x_ref : np.ndarray
+    x_ref : ArrayLike
         Data used as reference distribution.
     p_val : float, default 0.05
         p-value used for significance of the statistical test for each feature.
@@ -40,7 +43,7 @@ class DriftCVM(BaseUnivariateDrift):
         :py:class:`dataeval.detectors.LastSeenUpdateStrategy`
         or via reservoir sampling with
         :py:class:`dataeval.detectors.ReservoirSamplingUpdateStrategy`.
-    preprocess_fn : Optional[Callable[[np.ndarray], np.ndarray]], default None
+    preprocess_fn : Optional[Callable[[ArrayLike], ArrayLike]], default None
         Function to preprocess the data before computing the data drift metrics.
         Typically a dimensionality reduction technique.
     correction : Literal["bonferroni", "fdr"], default "bonferroni"
@@ -54,11 +57,11 @@ class DriftCVM(BaseUnivariateDrift):
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
         update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
         n_features: Optional[int] = None,
     ) -> None:
@@ -73,25 +76,26 @@ class DriftCVM(BaseUnivariateDrift):
         )
 
     @preprocess_x
-    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
         """
         Performs the two-sample Cramér-von Mises test(s), computing the p-value and
         test statistic per feature.
 
         Parameters
         ----------
-        x
+        x : ArrayLike
             Batch of instances.
 
         Returns
         -------
         Feature level p-values and CVM statistics.
         """
-        x = x.reshape(x.shape[0], -1)
+        x_np = to_numpy(x)
+        x_np = x_np.reshape(x_np.shape[0], -1)
         x_ref = self.x_ref.reshape(self.x_ref.shape[0], -1)
         p_val = np.zeros(self.n_features, dtype=np.float32)
         dist = np.zeros_like(p_val)
         for f in range(self.n_features):
-            result = cramervonmises_2samp(x_ref[:, f], x[:, f], method="auto")
+            result = cramervonmises_2samp(x_ref[:, f], x_np[:, f], method="auto")
             p_val[f], dist[f] = result.pvalue, result.statistic
         return p_val, dist

dataeval/_internal/detectors/drift/ks.py

@@ -9,8 +9,11 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable, Literal, Optional, Tuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.stats import ks_2samp
 
+from dataeval._internal.interop import to_numpy
+
 from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
 
 
@@ -55,11 +58,11 @@ class DriftKS(BaseUnivariateDrift):
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
         update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
         correction: Literal["bonferroni", "fdr"] = "bonferroni",
         alternative: Literal["two-sided", "less", "greater"] = "two-sided",
         n_features: Optional[int] = None,
@@ -78,7 +81,7 @@ class DriftKS(BaseUnivariateDrift):
         self.alternative = alternative
 
     @preprocess_x
-    def score(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+    def score(self, x: ArrayLike) -> Tuple[np.ndarray, np.ndarray]:
         """
         Compute K-S scores and statistics per feature.
 
@@ -91,6 +94,7 @@ class DriftKS(BaseUnivariateDrift):
         -------
         Feature level p-values and K-S statistics.
         """
+        x = to_numpy(x)
         x = x.reshape(x.shape[0], -1)
         x_ref = self.x_ref.reshape(self.x_ref.shape[0], -1)
         p_val = np.zeros(self.n_features, dtype=np.float32)

dataeval/_internal/detectors/drift/mmd.py

@@ -8,8 +8,10 @@ Licensed under Apache Software License (Apache 2.0)
 
 from typing import Callable, Dict, Optional, Tuple, Union
 
-import numpy as np
 import torch
+from numpy.typing import ArrayLike
+
+from dataeval._internal.interop import to_numpy
 
 from .base import BaseDrift, UpdateStrategy, preprocess_x, update_x_ref
 from .torch import GaussianRBF, get_device, mmd2_from_kernel_matrix
@@ -21,7 +23,7 @@ class DriftMMD(BaseDrift):
 
     Parameters
     ----------
-    x_ref : np.ndarray
+    x_ref : ArrayLike
         Data used as reference distribution.
     p_val : float, default 0.05
         p-value used for the significance of the permutation test.
@@ -44,7 +46,7 @@ class DriftMMD(BaseDrift):
         Function to preprocess the data before computing the data drift metrics.
     kernel : Callable, default :py:class:`dataeval.detectors.GaussianRBF`
         Kernel used for the MMD computation, defaults to Gaussian RBF kernel.
-    sigma : Optional[np.ndarray], default None
+    sigma : Optional[ArrayLike], default None
         Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple
         bandwidth values as an array. The kernel evaluation is then averaged over
         those bandwidths.
@@ -59,13 +61,13 @@ class DriftMMD(BaseDrift):
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
         update_x_ref: Optional[UpdateStrategy] = None,
-        preprocess_fn: Optional[Callable[[np.ndarray], np.ndarray]] = None,
+        preprocess_fn: Optional[Callable[[ArrayLike], ArrayLike]] = None,
         kernel: Callable = GaussianRBF,
-        sigma: Optional[np.ndarray] = None,
+        sigma: Optional[ArrayLike] = None,
         configure_kernel_from_x_ref: bool = True,
         n_permutations: int = 100,
         device: Optional[str] = None,
@@ -73,7 +75,7 @@ class DriftMMD(BaseDrift):
         super().__init__(x_ref, p_val, x_ref_preprocessed, update_x_ref, preprocess_fn)
 
         self.infer_sigma = configure_kernel_from_x_ref
-        if configure_kernel_from_x_ref and isinstance(sigma, np.ndarray):
+        if configure_kernel_from_x_ref and sigma is not None:
             self.infer_sigma = False
 
         self.n_permutations = n_permutations  # nb of iterations through permutation test
@@ -82,7 +84,7 @@ class DriftMMD(BaseDrift):
         self.device = get_device(device)
 
         # initialize kernel
-        sigma_tensor = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, np.ndarray) else None
+        sigma_tensor = torch.from_numpy(to_numpy(sigma)).to(self.device) if sigma is not None else None
         self.kernel = kernel(sigma_tensor).to(self.device) if kernel == GaussianRBF else kernel
 
         # compute kernel matrix for the reference data
@@ -102,7 +104,7 @@ class DriftMMD(BaseDrift):
         return kernel_mat
 
     @preprocess_x
-    def score(self, x: np.ndarray) -> Tuple[float, float, float]:
+    def score(self, x: ArrayLike) -> Tuple[float, float, float]:
         """
         Compute the p-value resulting from a permutation test using the maximum mean
         discrepancy as a distance measure between the reference data and the data to
@@ -110,7 +112,7 @@ class DriftMMD(BaseDrift):
 
         Parameters
         ----------
-        x
+        x : ArrayLike
             Batch of instances.
 
         Returns
@@ -118,6 +120,7 @@ class DriftMMD(BaseDrift):
         p-value obtained from the permutation test, the MMD^2 between the reference and
         test set, and the MMD^2 threshold above which drift is flagged.
         """
+        x = to_numpy(x)
         x_ref = torch.from_numpy(self.x_ref).to(self.device)
         n = x.shape[0]
         kernel_mat = self._kernel_matrix(x_ref, torch.from_numpy(x).to(self.device))
@@ -137,7 +140,7 @@ class DriftMMD(BaseDrift):
     @update_x_ref
     def predict(
         self,
-        x: np.ndarray,
+        x: ArrayLike,
     ) -> Dict[str, Union[int, float]]:
         """
         Predict whether a batch of data has drifted from the reference data and then
@@ -145,7 +148,7 @@ class DriftMMD(BaseDrift):
 
         Parameters
         ----------
-        x
+        x : ArrayLike
             Batch of instances.
 
         Returns

dataeval/_internal/detectors/drift/uncertainty.py

@@ -10,6 +10,7 @@ from functools import partial
 from typing import Callable, Dict, Literal, Optional, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.special import softmax
 from scipy.stats import entropy
 
@@ -64,7 +65,7 @@ class DriftUncertainty:
 
     Parameters
     ----------
-    x_ref : np.ndarray
+    x_ref : ArrayLike
         Data used as reference distribution. Should be disjoint from the data the
         model was trained on for accurate p-values.
     model : Callable
@@ -100,7 +101,7 @@ class DriftUncertainty:
 
     def __init__(
         self,
-        x_ref: np.ndarray,
+        x_ref: ArrayLike,
         model: Callable,
         p_val: float = 0.05,
         x_ref_preprocessed: bool = False,
@@ -130,16 +131,16 @@ class DriftUncertainty:
             p_val=p_val,
             x_ref_preprocessed=x_ref_preprocessed,
             update_x_ref=update_x_ref,
-            preprocess_fn=preprocess_fn,
+            preprocess_fn=preprocess_fn,  # type: ignore
         )
 
-    def predict(self, x: np.ndarray) -> Dict[str, Union[int, float, np.ndarray]]:
+    def predict(self, x: ArrayLike) -> Dict[str, Union[int, float, np.ndarray]]:
         """
         Predict whether a batch of data has drifted from the reference data.
 
         Parameters
         ----------
-        x
+        x : ArrayLike
             Batch of instances.
 
         Returns

dataeval/_internal/detectors/duplicates.py

@@ -1,6 +1,6 @@
-from typing import Dict, List, Literal
+from typing import Dict, Iterable, List, Literal
 
-import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.flags import ImageHash
 from dataeval._internal.metrics.stats import ImageStats
@@ -10,14 +10,21 @@ class Duplicates:
     """
     Finds the duplicate images in a dataset using xxhash for exact duplicates
     and pchash for near duplicates
+
+    Attributes
+    ----------
+    stats : ImageStats(flags=ImageHash.ALL)
+        Base stats class with the flags for checking duplicates
+
+    Example
+    -------
+    Initialize the Duplicates class:
+
+    >>> dups = Duplicates()
     """
 
-    def __init__(
-        self,
-        images: np.ndarray,
-    ):
+    def __init__(self):
         self.stats = ImageStats(ImageHash.ALL)
-        self.images = images
 
     def _get_duplicates(self) -> dict:
         exact = {}
@@ -34,16 +41,33 @@ class Duplicates:
             "near": sorted(near),
         }
 
-    def evaluate(self) -> Dict[Literal["exact", "near"], List[int]]:
+    def evaluate(self, images: Iterable[ArrayLike]) -> Dict[Literal["exact", "near"], List[int]]:
         """
         Returns duplicate image indices for both exact matches and near matches
 
+        Parameters
+        ----------
+        images : Iterable[ArrayLike], shape - (N, C, H, W)
+            A set of images in an ArrayLike format
+
         Returns
         -------
-        Dict[Literal["exact", "near"], List[int]]
-            Dictionary of exact and near match indices
+        Dict[str, List[int]]
+            exact :
+                List of groups of indices that are exact matches
+            near :
+                List of groups of indices that are near matches
+
+        See Also
+        --------
+        ImageStats
+
+        Example
+        -------
+        >>> dups.evaluate(images)
+        {'exact': [[3, 20], [16, 37]], 'near': [[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]]}
         """
         self.stats.reset()
-        self.stats.update(self.images)
+        self.stats.update(images)
         self.results = self.stats.compute()
         return self._get_duplicates()