dataeval 0.82.0__py3-none-any.whl → 0.83.0__py3-none-any.whl

dataeval/__init__.py

@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
 from __future__ import annotations
 
 __all__ = ["config", "detectors", "log", "metrics", "typing", "utils", "workflows"]
-__version__ = "0.82.0"
+__version__ = "0.83.0"
 
 import logging
 
@@ -34,7 +34,12 @@ def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> N
     logger = logging.getLogger(__name__)
     if handler is None:
         handler = logging.StreamHandler() if handler is None else handler
-        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+        handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s %(levelname)-8s %(name)s.%(filename)s:%(lineno)s - %(funcName)10s() | %(message)s"
+            )
+        )
     logger.addHandler(handler)
     logger.setLevel(level)
+    logging.DEBUG
     logger.debug(f"Added logging handler {handler} to logger: {__name__}")

dataeval/config.py

@@ -4,36 +4,71 @@ Global configuration settings for DataEval.
 
 from __future__ import annotations
 
-__all__ = ["get_device", "set_device", "get_max_processes", "set_max_processes"]
+__all__ = ["get_device", "set_device", "get_max_processes", "set_max_processes", "DeviceLike"]
 
+import sys
+from typing import Union
+
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
+import numpy as np
 import torch
-from torch import device
 
-_device: device | None = None
+### GLOBALS ###
+
+_device: torch.device | None = None
 _processes: int | None = None
+_seed: int | None = None
+
+### CONSTS ###
 
+EPSILON = 1e-10
 
-def set_device(device: str | device | int) -> None:
+### TYPES ###
+
+DeviceLike: TypeAlias = Union[int, str, tuple[str, int], torch.device]
+"""
+Type alias for types that are acceptable for specifying a torch.device.
+
+See Also
+--------
+`torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
+"""
+
+### FUNCS ###
+
+
+def _todevice(device: DeviceLike) -> torch.device:
+    return torch.device(*device) if isinstance(device, tuple) else torch.device(device)
+
+
+def set_device(device: DeviceLike) -> None:
     """
     Sets the default device to use when executing against a PyTorch backend.
 
     Parameters
     ----------
-    device : str or int or `torch.device`
-        The default device to use. See `torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
-        documentation for more information.
+    device : DeviceLike
+        The default device to use. See documentation for more information.
+
+    See Also
+    --------
+    `torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
     """
     global _device
-    _device = torch.device(device)
+    _device = _todevice(device)
 
 
-def get_device(override: str | device | int | None = None) -> torch.device:
+def get_device(override: DeviceLike | None = None) -> torch.device:
     """
     Returns the PyTorch device to use.
 
     Parameters
     ----------
-    override : str or int or `torch.device` or None, default None
+    override : DeviceLike or None, default None
         The user specified override if provided, otherwise returns the default device.
 
     Returns
@@ -44,7 +79,7 @@ def get_device(override: str | device | int | None = None) -> torch.device:
         global _device
         return torch.get_default_device() if _device is None else _device
     else:
-        return torch.device(override)
+        return _todevice(override)
 
 
 def set_max_processes(processes: int | None) -> None:
@@ -75,3 +110,35 @@ def get_max_processes() -> int | None:
     """
     global _processes
     return _processes
+
+
+def set_seed(seed: int | None, all_generators: bool = False) -> None:
+    """
+    Sets the seed for use by classes that allow for a random state or seed.
+
+    Parameters
+    ----------
+    seed : int or None
+        The seed to use.
+    all_generators : bool, default False
+        Whether to set the seed for all generators, including NumPy and PyTorch.
+    """
+    global _seed
+    _seed = seed
+
+    if all_generators:
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+
+
+def get_seed() -> int | None:
+    """
+    Returns the seed for random state or seed.
+
+    Returns
+    -------
+    int or None
+        The seed to use.
+    """
+    global _seed
+    return _seed

dataeval/detectors/drift/_mmd.py

@@ -14,7 +14,7 @@ from typing import Callable
 
 import torch
 
-from dataeval.config import get_device
+from dataeval.config import DeviceLike, get_device
 from dataeval.detectors.drift._base import BaseDrift, UpdateStrategy, preprocess_x, update_x_ref
 from dataeval.detectors.drift._torch import GaussianRBF, mmd2_from_kernel_matrix
 from dataeval.outputs import DriftMMDOutput
@@ -31,7 +31,7 @@ class DriftMMD(BaseDrift):
     ----------
     x_ref : ArrayLike
         Data used as reference distribution.
-    p_val : float | None, default 0.05
+    p_val : float or None, default 0.05
         :term:`P-value` used for significance of the statistical test for each feature.
         If the FDR correction method is used, this corresponds to the acceptable
         q-value.
@@ -39,14 +39,14 @@ class DriftMMD(BaseDrift):
         Whether the given reference data ``x_ref`` has been preprocessed yet.
         If ``True``, only the test data ``x`` will be preprocessed at prediction time.
         If ``False``, the reference data will also be preprocessed.
-    update_x_ref : UpdateStrategy | None, default None
+    update_x_ref : UpdateStrategy or None, default None
         Reference data can optionally be updated using an UpdateStrategy class. Update
         using the last n instances seen by the detector with LastSeenUpdateStrategy
         or via reservoir sampling with ReservoirSamplingUpdateStrategy.
-    preprocess_fn : Callable | None, default None
+    preprocess_fn : Callable or None, default None
         Function to preprocess the data before computing the data drift metrics.
         Typically a :term:`dimensionality reduction<Dimensionality Reduction>` technique.
-    sigma : ArrayLike | None, default None
+    sigma : ArrayLike or None, default None
         Optionally set the internal GaussianRBF kernel bandwidth. Can also pass multiple
         bandwidth values as an array. The kernel evaluation is then averaged over
         those bandwidths.
@@ -54,9 +54,9 @@ class DriftMMD(BaseDrift):
         Whether to already configure the kernel bandwidth from the reference data.
     n_permutations : int, default 100
         Number of permutations used in the permutation test.
-    device : str | None, default None
-        Device type used. The default None uses the GPU and falls back on CPU.
-        Can be specified by passing either 'cuda', 'gpu' or 'cpu'.
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
 
     Example
     -------
@@ -84,7 +84,7 @@ class DriftMMD(BaseDrift):
         sigma: ArrayLike | None = None,
         configure_kernel_from_x_ref: bool = True,
         n_permutations: int = 100,
-        device: str | torch.device | None = None,
+        device: DeviceLike | None = None,
     ) -> None:
         super().__init__(x_ref, p_val, x_ref_preprocessed, update_x_ref, preprocess_fn)
 

dataeval/detectors/drift/_torch.py

@@ -17,7 +17,7 @@ import torch
 import torch.nn as nn
 from numpy.typing import NDArray
 
-from dataeval.config import get_device
+from dataeval.config import DeviceLike, get_device
 from dataeval.utils.torch._internal import predict_batch
 
 
@@ -59,7 +59,7 @@ def mmd2_from_kernel_matrix(
 def preprocess_drift(
     x: NDArray[Any],
     model: nn.Module,
-    device: str | torch.device | None = None,
+    device: DeviceLike | None = None,
     preprocess_batch_fn: Callable | None = None,
     batch_size: int = int(1e10),
     dtype: type[np.generic] | torch.dtype = np.float32,
@@ -73,15 +73,15 @@ def preprocess_drift(
         Batch of instances.
     model : nn.Module
         Model used for preprocessing.
-    device : torch.device | None, default None
-        Device type used. The default None tries to use the GPU and falls back on CPU.
-        Can be specified by passing either torch.device('cuda') or torch.device('cpu').
-    preprocess_batch_fn : Callable | None, default None
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
+    preprocess_batch_fn : Callable or None, default None
         Optional batch preprocessing function. For example to convert a list of objects
         to a batch which can be processed by the PyTorch model.
     batch_size : int, default 1e10
         Batch size used during prediction.
-    dtype : np.dtype | torch.dtype, default np.float32
+    dtype : np.dtype or torch.dtype, default np.float32
         Model output type, either a :term:`NumPy` or torch dtype, e.g. np.float32 or torch.float32.
 
     Returns

dataeval/detectors/drift/_uncertainty.py

@@ -85,20 +85,20 @@ class DriftUncertainty:
         Whether the given reference data ``x_ref`` has been preprocessed yet.
         If ``True``, only the test data ``x`` will be preprocessed at prediction time.
         If ``False``, the reference data will also be preprocessed.
-    update_x_ref : UpdateStrategy | None, default None
+    update_x_ref : UpdateStrategy or None, default None
         Reference data can optionally be updated using an UpdateStrategy class. Update
         using the last n instances seen by the detector with LastSeenUpdateStrategy
         or via reservoir sampling with ReservoirSamplingUpdateStrategy.
-    preds_type : "probs" | "logits", default "probs"
+    preds_type : "probs" or "logits", default "probs"
         Type of prediction output by the model. Options are 'probs' (in [0,1]) or
         'logits' (in [-inf,inf]).
     batch_size : int, default 32
         Batch size used to evaluate model. Only relevant when backend has been
         specified for batch prediction.
-    preprocess_batch_fn : Callable | None, default None
+    preprocess_batch_fn : Callable or None, default None
         Optional batch preprocessing function. For example to convert a list of
         objects to a batch which can be processed by the model.
-    device : str | None, default None
+    device : DeviceLike or None, default None
         Device type used. The default None tries to use the GPU and falls back on
         CPU if needed. Can be specified by passing either 'cuda' or 'cpu'.
 

dataeval/detectors/linters/duplicates.py

@@ -88,13 +88,13 @@ class Duplicates:
         """
 
         if isinstance(hashes, HashStatsOutput):
-            return DuplicatesOutput(**self._get_duplicates(hashes.dict()))
+            return DuplicatesOutput(**self._get_duplicates(hashes.data()))
 
         if not isinstance(hashes, Sequence):
             raise TypeError("Invalid stats output type; only use output from hashstats.")
 
         combined, dataset_steps = combine_stats(hashes)
-        duplicates = self._get_duplicates(combined.dict())
+        duplicates = self._get_duplicates(combined.data())
 
         # split up results from combined dataset into individual dataset buckets
         for dup_type, dup_list in duplicates.items():
@@ -136,5 +136,5 @@ class Duplicates:
         """  # noqa: E501
         images = Images(data) if isinstance(data, Dataset) else data
         self.stats = hashstats(images)
-        duplicates = self._get_duplicates(self.stats.dict())
+        duplicates = self._get_duplicates(self.stats.data())
         return DuplicatesOutput(**duplicates)

dataeval/detectors/linters/outliers.py

@@ -169,7 +169,7 @@ class Outliers:
         {}
         """  # noqa: E501
         if isinstance(stats, (ImageStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)):
-            return OutliersOutput(self._get_outliers(stats.dict()))
+            return OutliersOutput(self._get_outliers(stats.data()))
 
         if not isinstance(stats, Sequence):
             raise TypeError(
@@ -189,7 +189,7 @@ class Outliers:
         output_list: list[dict[int, dict[str, float]]] = [{} for _ in stats]
         for _, indices in stats_map.items():
             substats, dataset_steps = combine_stats([stats[i] for i in indices])
-            outliers = self._get_outliers(substats.dict())
+            outliers = self._get_outliers(substats.data())
             for idx, issue in outliers.items():
                 k, v = get_dataset_step_from_idx(idx, dataset_steps)
                 output_list[indices[k]][v] = issue
@@ -225,5 +225,5 @@ class Outliers:
         """
         images = Images(data) if isinstance(data, Dataset) else data
         self.stats = imagestats(images)
-        outliers = self._get_outliers(self.stats.dict())
+        outliers = self._get_outliers(self.stats.data())
         return OutliersOutput(outliers)

dataeval/detectors/ood/ae.py

@@ -18,6 +18,7 @@ import numpy as np
 import torch
 from numpy.typing import NDArray
 
+from dataeval.config import DeviceLike
 from dataeval.detectors.ood.base import OODBase
 from dataeval.outputs import OODScoreOutput
 from dataeval.typing import ArrayLike
@@ -33,9 +34,9 @@ class OOD_AE(OODBase):
     model : torch.nn.Module
         An autoencoder model to use for encoding and reconstruction of images
         for detection of out-of-distribution samples.
-    device : str or torch.Device or None, default None
-        The device to use for the detector. None will default to the global
-        configuration selection if set, otherwise "cuda" then "cpu" by availability.
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
 
     Example
     -------
@@ -57,7 +58,7 @@ class OOD_AE(OODBase):
     array([ True,  True, False,  True,  True,  True,  True,  True])
     """
 
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         super().__init__(model, device)
 
     def fit(

dataeval/detectors/ood/base.py

@@ -14,7 +14,7 @@ from typing import Callable, cast
 
 import torch
 
-from dataeval.config import get_device
+from dataeval.config import DeviceLike, get_device
 from dataeval.detectors.ood.mixin import OODBaseMixin, OODFitMixin, OODGMMMixin
 from dataeval.typing import ArrayLike
 from dataeval.utils._array import to_numpy
@@ -23,7 +23,7 @@ from dataeval.utils.torch._internal import trainer
 
 
 class OODBase(OODBaseMixin[torch.nn.Module], OODFitMixin[Callable[..., torch.nn.Module], torch.optim.Optimizer]):
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         self.device: torch.device = get_device(device)
         super().__init__(model)
 

dataeval/detectors/ood/mixin.py

@@ -157,4 +157,4 @@ class OODBaseMixin(Generic[TModel], ABC):
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
         ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
-        return OODOutput(is_ood=ood_pred, **score.dict())
+        return OODOutput(is_ood=ood_pred, **score.data())

dataeval/detectors/ood/vae.py

@@ -17,6 +17,7 @@ from typing import Callable
 import numpy as np
 import torch
 
+from dataeval.config import DeviceLike
 from dataeval.detectors.ood.base import OODBase
 from dataeval.outputs import OODScoreOutput
 from dataeval.typing import ArrayLike
@@ -34,7 +35,7 @@ class OOD_VAE(OODBase):
         An Autoencoder model.
     """
 
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         super().__init__(model, device)
 
     def fit(

dataeval/metadata/__init__.py

@@ -1,6 +1,6 @@
 """Explanatory functions using metadata and additional features such as ood or drift"""
 
-__all__ = ["most_deviated_factors", "metadata_distance"]
+__all__ = ["find_ood_predictors", "metadata_distance", "find_most_deviated_factors"]
 
 from dataeval.metadata._distance import metadata_distance
-from dataeval.metadata._ood import most_deviated_factors
+from dataeval.metadata._ood import find_most_deviated_factors, find_ood_predictors

dataeval/metadata/_distance.py

@@ -10,7 +10,8 @@ from scipy.stats import iqr, ks_2samp
 from scipy.stats import wasserstein_distance as emd
 
 from dataeval.metadata._utils import _compare_keys, _validate_factors_and_data
-from dataeval.outputs._base import MappingOutput
+from dataeval.outputs import MetadataDistanceOutput, MetadataDistanceValues
+from dataeval.outputs._base import set_metadata
 from dataeval.typing import ArrayLike
 from dataeval.utils.data import Metadata
 
@@ -23,41 +24,6 @@ class KSType(NamedTuple):
     pvalue: float
 
 
-class MetadataKSResult(NamedTuple):
-    """
-    Attributes
-    ----------
-    statistic : float
-        the KS statistic
-    location : float
-        The value at which the KS statistic has its maximum, measured in IQR-normalized units relative
-        to the median of the reference distribution.
-    dist : float
-        The Earth Mover's Distance normalized by the interquartile range (IQR) of the reference
-    pvalue : float
-        The p-value from the KS two-sample test
-    """
-
-    statistic: float
-    location: float
-    dist: float
-    pvalue: float
-
-
-class KSOutput(MappingOutput[str, MetadataKSResult]):
-    """
-    Output class for results of ks_2samp featurewise comparisons of new metadata to reference metadata.
-
-
-    Attributes
-    ----------
-    key: str
-        Metadata feature names
-    value: :class:`MetadataKSResult`
-        Output per feature name containing the statistic, statistic location, distance, and pvalue.
-    """
-
-
 def _calculate_drift(x1: ArrayLike, x2: ArrayLike) -> float:
     """Calculates the shift magnitude between x1 and x2 scaled by x1"""
 
@@ -74,7 +40,8 @@ def _calculate_drift(x1: ArrayLike, x2: ArrayLike) -> float:
     return distance if xmin == xmax else distance / (xmax - xmin)
 
 
-def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
+@set_metadata
+def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> MetadataDistanceOutput:
     """
     Measures the feature-wise distance between two continuous metadata distributions and
     computes a p-value to evaluate its significance.
@@ -90,8 +57,8 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
 
     Returns
     -------
-    dict[str, KstestResult]
-        A dictionary with keys corresponding to metadata feature names, and values that are KstestResult objects, as
+    MetadataDistanceOutput
+        A mapping with keys corresponding to metadata feature names, and values that are KstestResult objects, as
         defined by scipy.stats.ks_2samp.
 
     See Also
@@ -110,7 +77,7 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
     >>> list(output)
     ['time', 'altitude']
     >>> output["time"]
-    MetadataKSResult(statistic=1.0, location=0.44354838709677413, dist=2.7, pvalue=0.0)
+    MetadataDistanceValues(statistic=1.0, location=0.44354838709677413, dist=2.7, pvalue=0.0)
     """
 
     _compare_keys(metadata1.continuous_factor_names, metadata2.continuous_factor_names)
@@ -134,7 +101,7 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
         )
 
     # Set default for statistic, location, and magnitude to zero and pvalue to one
-    results: dict[str, MetadataKSResult] = {}
+    results: dict[str, MetadataDistanceValues] = {}
 
     # Per factor
     for i, fname in enumerate(fnames):
@@ -147,7 +114,7 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
 
         # Default case
         if xmin == xmax:
-            results[fname] = MetadataKSResult(statistic=0.0, location=0.0, dist=0.0, pvalue=1.0)
+            results[fname] = MetadataDistanceValues(statistic=0.0, location=0.0, dist=0.0, pvalue=1.0)
             continue
 
         ks_result = cast(KSType, ks_2samp(fdata1, fdata2, method="asymp"))
@@ -157,11 +124,11 @@ def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> KSOutput:
 
         drift = _calculate_drift(fdata1, fdata2)
 
-        results[fname] = MetadataKSResult(
+        results[fname] = MetadataDistanceValues(
             statistic=ks_result.statistic,
             location=loc,
             dist=drift,
             pvalue=ks_result.pvalue,
         )
 
-    return KSOutput(results)
+    return MetadataDistanceOutput(results)