dataeval 0.74.1__py3-none-any.whl → 0.75.0__py3-none-any.whl

dataeval/__init__.py

@@ -1,17 +1,40 @@
-__version__ = "0.74.1"
+"""
+DataEval provides a simple interface to characterize image data and its impact on model performance
+across classification and object-detection tasks. It also provides capabilities to select and curate
+datasets to test and train performant, robust, unbiased and reliable AI models and monitor for data
+shifts that impact performance of deployed models.
+"""
 
-from importlib.util import find_spec
+from __future__ import annotations
 
-_IS_TORCH_AVAILABLE = find_spec("torch") is not None
-_IS_TORCHVISION_AVAILABLE = find_spec("torchvision") is not None
+__all__ = ["detectors", "log", "metrics", "utils", "workflows"]
+__version__ = "0.75.0"
 
-del find_spec
+import logging
 
-from dataeval import detectors, metrics  # noqa: E402
+from dataeval import detectors, metrics, utils, workflows
 
-__all__ = ["detectors", "metrics"]
+logging.getLogger(__name__).addHandler(logging.NullHandler())
 
-if _IS_TORCH_AVAILABLE:
-    from dataeval import utils, workflows
 
-    __all__ += ["utils", "workflows"]
+def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> None:
+    """
+    Helper for quickly adding a StreamHandler to the logger. Useful for debugging.
+
+    Parameters
+    ----------
+    level : int, default logging.DEBUG(10)
+        Set the logging level for the logger
+    handler : logging.Handler, optional
+        Sets the logging handler for the logger if provided, otherwise logger will be
+        provided with a StreamHandler
+    """
+    import logging
+
+    logger = logging.getLogger(__name__)
+    if handler is None:
+        handler = logging.StreamHandler() if handler is None else handler
+        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.debug(f"Added logging handler {handler} to logger: {__name__}")

dataeval/detectors/__init__.py

@@ -2,6 +2,6 @@
 Detectors can determine if a dataset or individual images in a dataset are indicative of a specific issue.
 """
 
-from dataeval.detectors import drift, linters, ood
-
 __all__ = ["drift", "linters", "ood"]
+
+from dataeval.detectors import drift, linters, ood

dataeval/detectors/drift/__init__.py

@@ -2,19 +2,21 @@
 :term:`Drift` detectors identify if the statistical properties of the data has changed.
 """
 
-from dataeval import _IS_TORCH_AVAILABLE
+__all__ = [
+    "DriftCVM",
+    "DriftKS",
+    "DriftMMD",
+    "DriftMMDOutput",
+    "DriftOutput",
+    "DriftUncertainty",
+    "preprocess_drift",
+    "updates",
+]
+
 from dataeval.detectors.drift import updates
 from dataeval.detectors.drift.base import DriftOutput
 from dataeval.detectors.drift.cvm import DriftCVM
 from dataeval.detectors.drift.ks import DriftKS
-
-__all__ = ["DriftCVM", "DriftKS", "DriftOutput", "updates"]
-
-if _IS_TORCH_AVAILABLE:
-    from dataeval.detectors.drift.mmd import DriftMMD, DriftMMDOutput
-    from dataeval.detectors.drift.torch import preprocess_drift
-    from dataeval.detectors.drift.uncertainty import DriftUncertainty
-
-    __all__ += ["DriftMMD", "DriftMMDOutput", "DriftUncertainty", "preprocess_drift"]
-
-del _IS_TORCH_AVAILABLE
+from dataeval.detectors.drift.mmd import DriftMMD, DriftMMDOutput
+from dataeval.detectors.drift.torch import preprocess_drift
+from dataeval.detectors.drift.uncertainty import DriftUncertainty

dataeval/detectors/drift/base.py

@@ -8,7 +8,7 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["DriftOutput"]
+__all__ = []
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass

dataeval/detectors/drift/cvm.py

@@ -8,7 +8,7 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["DriftCVM"]
+__all__ = []
 
 from typing import Callable, Literal
 

dataeval/detectors/drift/ks.py

@@ -8,7 +8,7 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["DriftKS"]
+__all__ = []
 
 from typing import Callable, Literal
 

dataeval/detectors/drift/mmd.py

@@ -8,7 +8,7 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["DriftMMD", "DriftMMDOutput"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Callable
@@ -17,9 +17,10 @@ import torch
 from numpy.typing import ArrayLike
 
 from dataeval.detectors.drift.base import BaseDrift, DriftBaseOutput, UpdateStrategy, preprocess_x, update_x_ref
-from dataeval.detectors.drift.torch import _GaussianRBF, _mmd2_from_kernel_matrix, get_device
+from dataeval.detectors.drift.torch import GaussianRBF, mmd2_from_kernel_matrix
 from dataeval.interop import as_numpy
 from dataeval.output import set_metadata
+from dataeval.utils.torch.internal import get_device
 
 
 @dataclass(frozen=True)
@@ -109,7 +110,7 @@ class DriftMMD(BaseDrift):
 
         # initialize kernel
         sigma_tensor = torch.from_numpy(as_numpy(sigma)).to(self.device) if sigma is not None else None
-        self._kernel = _GaussianRBF(sigma_tensor).to(self.device)
+        self._kernel = GaussianRBF(sigma_tensor).to(self.device)
 
         # compute kernel matrix for the reference data
         if self._infer_sigma or isinstance(sigma_tensor, torch.Tensor):
@@ -150,9 +151,9 @@ class DriftMMD(BaseDrift):
         n = x.shape[0]
         kernel_mat = self._kernel_matrix(x_ref, torch.from_numpy(x).to(self.device))
         kernel_mat = kernel_mat - torch.diag(kernel_mat.diag())  # zero diagonal
-        mmd2 = _mmd2_from_kernel_matrix(kernel_mat, n, permute=False, zero_diag=False)
+        mmd2 = mmd2_from_kernel_matrix(kernel_mat, n, permute=False, zero_diag=False)
         mmd2_permuted = torch.Tensor(
-            [_mmd2_from_kernel_matrix(kernel_mat, n, permute=True, zero_diag=False) for _ in range(self.n_permutations)]
+            [mmd2_from_kernel_matrix(kernel_mat, n, permute=True, zero_diag=False) for _ in range(self.n_permutations)]
         )
         mmd2, mmd2_permuted = mmd2.detach().cpu(), mmd2_permuted.detach().cpu()
         p_val = (mmd2 <= mmd2_permuted).float().mean()

dataeval/detectors/drift/torch.py

@@ -17,10 +17,10 @@ import torch
 import torch.nn as nn
 from numpy.typing import NDArray
 
-from dataeval.utils.torch.utils import get_device, predict_batch
+from dataeval.utils.torch.internal import get_device, predict_batch
 
 
-def _mmd2_from_kernel_matrix(
+def mmd2_from_kernel_matrix(
     kernel_mat: torch.Tensor, m: int, permute: bool = False, zero_diag: bool = True
 ) -> torch.Tensor:
     """
@@ -127,7 +127,7 @@ def _squared_pairwise_distance(
 
 def sigma_median(x: torch.Tensor, y: torch.Tensor, dist: torch.Tensor) -> torch.Tensor:
     """
-    Bandwidth estimation using the median heuristic :cite:t:`Gretton2012`.
+    Bandwidth estimation using the median heuristic `Gretton2012`
 
     Parameters
     ----------
@@ -151,7 +151,7 @@ def sigma_median(x: torch.Tensor, y: torch.Tensor, dist: torch.Tensor) -> torch.
     return sigma
 
 
-class _GaussianRBF(nn.Module):
+class GaussianRBF(nn.Module):
     """
     Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2).
 
@@ -179,18 +179,18 @@ class _GaussianRBF(nn.Module):
     ) -> None:
         super().__init__()
         init_sigma_fn = sigma_median if init_sigma_fn is None else init_sigma_fn
-        self.config = {
+        self.config: dict[str, Any] = {
             "sigma": sigma,
             "trainable": trainable,
             "init_sigma_fn": init_sigma_fn,
         }
         if sigma is None:
-            self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable)
-            self.init_required = True
+            self.log_sigma: nn.Parameter = nn.Parameter(torch.empty(1), requires_grad=trainable)
+            self.init_required: bool = True
         else:
             sigma = sigma.reshape(-1)  # [Ns,]
-            self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable)
-            self.init_required = False
+            self.log_sigma: nn.Parameter = nn.Parameter(sigma.log(), requires_grad=trainable)
+            self.init_required: bool = False
         self.init_sigma_fn = init_sigma_fn
         self.trainable = trainable
 
@@ -200,8 +200,8 @@ class _GaussianRBF(nn.Module):
 
     def forward(
         self,
-        x: np.ndarray | torch.Tensor,
-        y: np.ndarray | torch.Tensor,
+        x: np.ndarray[Any, Any] | torch.Tensor,
+        y: np.ndarray[Any, Any] | torch.Tensor,
         infer_sigma: bool = False,
     ) -> torch.Tensor:
         x, y = torch.as_tensor(x), torch.as_tensor(y)
@@ -213,7 +213,7 @@ class _GaussianRBF(nn.Module):
             sigma = self.init_sigma_fn(x, y, dist)
             with torch.no_grad():
                 self.log_sigma.copy_(sigma.log().clone())
-            self.init_required = False
+            self.init_required: bool = False
 
         gamma = 1.0 / (2.0 * self.sigma**2)  # [Ns,]
         # TODO: do matrix multiplication after all?

dataeval/detectors/drift/uncertainty.py

@@ -8,7 +8,7 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["DriftUncertainty"]
+__all__ = []
 
 from functools import partial
 from typing import Callable, Literal
@@ -20,7 +20,8 @@ from scipy.stats import entropy
 
 from dataeval.detectors.drift.base import DriftOutput, UpdateStrategy
 from dataeval.detectors.drift.ks import DriftKS
-from dataeval.detectors.drift.torch import get_device, preprocess_drift
+from dataeval.detectors.drift.torch import preprocess_drift
+from dataeval.utils.torch.internal import get_device
 
 
 def classifier_uncertainty(

dataeval/detectors/linters/__init__.py

@@ -2,10 +2,6 @@
 Linters help identify potential issues in training and test data and are an important aspect of data cleaning.
 """
 
-from dataeval.detectors.linters.clusterer import Clusterer, ClustererOutput
-from dataeval.detectors.linters.duplicates import Duplicates, DuplicatesOutput
-from dataeval.detectors.linters.outliers import Outliers, OutliersOutput
-
 __all__ = [
     "Clusterer",
     "ClustererOutput",
@@ -14,3 +10,7 @@ __all__ = [
     "Outliers",
     "OutliersOutput",
 ]
+
+from dataeval.detectors.linters.clusterer import Clusterer, ClustererOutput
+from dataeval.detectors.linters.duplicates import Duplicates, DuplicatesOutput
+from dataeval.detectors.linters.outliers import Outliers, OutliersOutput

dataeval/detectors/linters/clusterer.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["ClustererOutput", "Clusterer"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Any, Iterable, NamedTuple, cast
@@ -147,12 +147,6 @@ class Clusterer:
     ----
     The Clusterer works best when the length of the feature dimension, P, is less than 500.
     If flattening a CxHxW image results in a dimension larger than 500, then it is recommended to reduce the dimensions.
-
-    Example
-    -------
-    Initialize the Clusterer class:
-
-    >>> cluster = Clusterer(dataset)
     """
 
     def __init__(self, dataset: ArrayLike) -> None:
@@ -506,6 +500,7 @@ class Clusterer:
 
         Example
         -------
+        >>> cluster = Clusterer(clusterer_images)
         >>> cluster.evaluate()
         ClustererOutput(outliers=[18, 21, 34, 35, 45], potential_outliers=[13, 15, 42], duplicates=[[9, 24], [23, 48]], potential_duplicates=[[1, 11]])
         """  # noqa: E501

dataeval/detectors/linters/duplicates.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["DuplicatesOutput", "Duplicates"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Generic, Iterable, Sequence, TypeVar, overload
@@ -51,13 +51,6 @@ class Duplicates:
     ----------
     only_exact : bool, default False
         Only inspect the dataset for exact image matches
-
-    Example
-    -------
-    Initialize the Duplicates class:
-
-    >>> all_dupes = Duplicates()
-    >>> exact_dupes = Duplicates(only_exact=True)
     """
 
     def __init__(self, only_exact: bool = False) -> None:
@@ -73,7 +66,8 @@ class Duplicates:
         if not self.only_exact:
             near_dict: dict[int, list] = {}
             for i, value in enumerate(stats["pchash"]):
-                near_dict.setdefault(value, []).append(i)
+                if value:
+                    near_dict.setdefault(value, []).append(i)
             near = [sorted(v) for v in near_dict.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
         else:
             near = []
@@ -112,6 +106,7 @@ class Duplicates:
 
         Example
         -------
+        >>> exact_dupes = Duplicates(only_exact=True)
         >>> exact_dupes.from_stats([hashes1, hashes2])
         DuplicatesOutput(exact=[{0: [3, 20]}, {0: [16], 1: [12]}], near=[])
         """
@@ -159,7 +154,8 @@ class Duplicates:
 
         Example
         -------
-        >>> all_dupes.evaluate(images)
+        >>> all_dupes = Duplicates()
+        >>> all_dupes.evaluate(duplicate_images)
         DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
         """  # noqa: E501
         self.stats = hashstats(data)

dataeval/detectors/linters/outliers.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["OutliersOutput", "Outliers"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
@@ -188,6 +188,7 @@ class Outliers:
         -------
         Evaluate the dataset:
 
+        >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=3.5)
         >>> results = outliers.from_stats([stats1, stats2])
         >>> len(results)
         2
@@ -248,7 +249,8 @@ class Outliers:
         -------
         Evaluate the dataset:
 
-        >>> results = outliers.evaluate(images)
+        >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=3.5)
+        >>> results = outliers.evaluate(outlier_images)
         >>> list(results.issues)
         [10, 12]
         >>> results.issues[10]

dataeval/detectors/ood/__init__.py

@@ -2,14 +2,7 @@
 Out-of-distribution (OOD)` detectors identify data that is different from the data used to train a particular model.
 """
 
-from dataeval import _IS_TORCH_AVAILABLE
-from dataeval.detectors.ood.base import OODOutput, OODScoreOutput
+__all__ = ["OODOutput", "OODScoreOutput", "OOD_AE"]
 
-__all__ = ["OODOutput", "OODScoreOutput"]
-
-if _IS_TORCH_AVAILABLE:
-    from dataeval.detectors.ood.ae_torch import OOD_AE
-
-    __all__ += ["OOD_AE"]
-
-del _IS_TORCH_AVAILABLE
+from dataeval.detectors.ood.ae import OOD_AE
+from dataeval.detectors.ood.output import OODOutput, OODScoreOutput

dataeval/detectors/ood/{ae_torch.py → ae.py}

@@ -10,16 +10,18 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
+__all__ = []
+
 from typing import Callable
 
 import numpy as np
 import torch
 from numpy.typing import ArrayLike
 
-from dataeval.detectors.ood.base import OODScoreOutput
-from dataeval.detectors.ood.base_torch import OODBase
+from dataeval.detectors.ood.base import OODBase
+from dataeval.detectors.ood.output import OODScoreOutput
 from dataeval.interop import as_numpy
-from dataeval.utils.torch.utils import predict_batch
+from dataeval.utils.torch.internal import predict_batch
 
 
 class OOD_AE(OODBase):
@@ -28,7 +30,7 @@ class OOD_AE(OODBase):
 
     Parameters
     ----------
-    model : AriaAutoencoder
+    model : Autoencoder
         An Autoencoder model.
     """