dataeval 0.74.2__py3-none-any.whl → 0.75.0__py3-none-any.whl

dataeval/detectors/ood/base.py

@@ -8,94 +8,30 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
-__all__ = ["OODOutput", "OODScoreOutput"]
+__all__ = []
 
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Callable, Generic, Literal, TypeVar
+from typing import Callable, cast
 
-import numpy as np
-from numpy.typing import ArrayLike, NDArray
+import torch
+from numpy.typing import ArrayLike
 
+from dataeval.detectors.ood.mixin import OODBaseMixin, OODFitMixin, OODGMMMixin
 from dataeval.interop import to_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.gmm import GaussianMixtureModelParams
-
-
-@dataclass(frozen=True)
-class OODOutput(Output):
-    """
-    Output class for predictions from :class:`OOD_AE`, :class:`OOD_AEGMM`, :class:`OOD_LLR`,
-    :class:`OOD_VAE`, and :class:`OOD_VAEGMM` out-of-distribution detectors
-
-    Attributes
-    ----------
-    is_ood : NDArray
-        Array of images that are detected as :term:Out-of-Distribution (OOD)`
-    instance_score : NDArray
-        Instance score of the evaluated dataset
-    feature_score : NDArray | None
-        Feature score, if available, of the evaluated dataset
-    """
-
-    is_ood: NDArray[np.bool_]
-    instance_score: NDArray[np.float32]
-    feature_score: NDArray[np.float32] | None
-
-
-@dataclass(frozen=True)
-class OODScoreOutput(Output):
-    """
-    Output class for instance and feature scores from :class:`OOD_AE`, :class:`OOD_AEGMM`,
-    :class:`OOD_LLR`, :class:`OOD_VAE`, and :class:`OOD_VAEGMM` out-of-distribution detectors
-
-    Parameters
-    ----------
-    instance_score : NDArray
-        Instance score of the evaluated dataset.
-    feature_score : NDArray | None, default None
-        Feature score, if available, of the evaluated dataset.
-    """
-
-    instance_score: NDArray[np.float32]
-    feature_score: NDArray[np.float32] | None = None
-
-    def get(self, ood_type: Literal["instance", "feature"]) -> NDArray[np.float32]:
-        """
-        Returns either the instance or feature score
-
-        Parameters
-        ----------
-        ood_type : "instance" | "feature"
-
-        Returns
-        -------
-        NDArray
-            Either the instance or feature score based on input selection
-        """
-        return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
-
-
-TGMMData = TypeVar("TGMMData")
+from dataeval.utils.torch.gmm import GaussianMixtureModelParams, gmm_params
+from dataeval.utils.torch.internal import get_device, trainer
 
 
-class OODGMMMixin(Generic[TGMMData]):
-    _gmm_params: GaussianMixtureModelParams[TGMMData]
+class OODBase(OODBaseMixin[torch.nn.Module], OODFitMixin[Callable[..., torch.nn.Module], torch.optim.Optimizer]):
+    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+        self.device: torch.device = get_device(device)
+        super().__init__(model)
 
-
-TModel = TypeVar("TModel", bound=Callable)
-TLossFn = TypeVar("TLossFn", bound=Callable)
-TOptimizer = TypeVar("TOptimizer")
-
-
-class OODFitMixin(Generic[TLossFn, TOptimizer], ABC):
-    @abstractmethod
     def fit(
         self,
         x_ref: ArrayLike,
         threshold_perc: float,
-        loss_fn: TLossFn | None,
-        optimizer: TOptimizer | None,
+        loss_fn: Callable[..., torch.nn.Module] | None,
+        optimizer: torch.optim.Optimizer | None,
         epochs: int,
         batch_size: int,
         verbose: bool,
@@ -109,9 +45,9 @@ class OODFitMixin(Generic[TLossFn, TOptimizer], ABC):
             Training data.
         threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
-        loss_fn : TLossFn
+        loss_fn : Callable | None, default None
             Loss function used for training.
-        optimizer : TOptimizer
+        optimizer : Optimizer, default keras.optimizers.Adam
             Optimizer used for training.
         epochs : int, default 20
             Number of training epochs.
@@ -121,87 +57,54 @@ class OODFitMixin(Generic[TLossFn, TOptimizer], ABC):
             Whether to print training progress.
         """
 
-
-class OODBaseMixin(Generic[TModel], ABC):
-    _ref_score: OODScoreOutput
-    _threshold_perc: float
-    _data_info: tuple[tuple, type] | None = None
-
-    def __init__(
+        # Train the model
+        trainer(
+            model=self.model,
+            x_train=to_numpy(x_ref),
+            y_train=None,
+            loss_fn=loss_fn,
+            optimizer=optimizer,
+            preprocess_fn=None,
+            epochs=epochs,
+            batch_size=batch_size,
+            device=self.device,
+            verbose=verbose,
+        )
+
+        # Infer the threshold values
+        self._ref_score = self.score(x_ref, batch_size)
+        self._threshold_perc = threshold_perc
+
+
+class OODBaseGMM(OODBase, OODGMMMixin[GaussianMixtureModelParams]):
+    def fit(
         self,
-        model: TModel,
+        x_ref: ArrayLike,
+        threshold_perc: float,
+        loss_fn: Callable[..., torch.nn.Module] | None,
+        optimizer: torch.optim.Optimizer | None,
+        epochs: int,
+        batch_size: int,
+        verbose: bool,
     ) -> None:
-        self.model = model
-
-    def _get_data_info(self, X: NDArray) -> tuple[tuple, type]:
-        if not isinstance(X, np.ndarray):
-            raise TypeError("Dataset should of type: `NDArray`.")
-        return X.shape[1:], X.dtype.type
-
-    def _validate(self, X: NDArray) -> None:
-        check_data_info = self._get_data_info(X)
-        if self._data_info is not None and check_data_info != self._data_info:
-            raise RuntimeError(f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
-                               Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}.")
-
-    def _validate_state(self, X: NDArray) -> None:
-        attrs = [k for c in self.__class__.mro()[:-1][::-1] if hasattr(c, "__annotations__") for k in c.__annotations__]
-        if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
-            raise RuntimeError("Metric needs to be `fit` before method call.")
-        self._validate(X)
-
-    @abstractmethod
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput: ...
-
-    @set_metadata
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        """
-        Compute the :term:`out of distribution<Out-of-distribution (OOD)>` scores for a given dataset.
-
-        Parameters
-        ----------
-        X : ArrayLike
-            Input data to score.
-        batch_size : int, default 1e10
-            Number of instances to process in each batch.
-            Use a smaller batch size if your dataset is large or if you encounter memory issues.
-
-        Returns
-        -------
-        OODScoreOutput
-            An object containing the instance-level and feature-level OOD scores.
-        """
-        return self._score(X, batch_size)
-
-    def _threshold_score(self, ood_type: Literal["feature", "instance"] = "instance") -> np.floating:
-        return np.percentile(self._ref_score.get(ood_type), self._threshold_perc)
-
-    @set_metadata
-    def predict(
-        self,
-        X: ArrayLike,
-        batch_size: int = int(1e10),
-        ood_type: Literal["feature", "instance"] = "instance",
-    ) -> OODOutput:
-        """
-        Predict whether instances are :term:`out of distribution<Out-of-distribution (OOD)>` or not.
-
-        Parameters
-        ----------
-        X : ArrayLike
-            Input data for out-of-distribution prediction.
-        batch_size : int, default 1e10
-            Number of instances to process in each batch.
-        ood_type : "feature" | "instance", default "instance"
-            Predict out-of-distribution at the 'feature' or 'instance' level.
-
-        Returns
-        -------
-        Dictionary containing the outlier predictions for the selected level,
-        and the OOD scores for the data including both 'instance' and 'feature' (if present) level scores.
-        """
-        self._validate_state(X := to_numpy(X))
-        # compute outlier scores
-        score = self.score(X, batch_size=batch_size)
-        ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
-        return OODOutput(is_ood=ood_pred, **score.dict())
+        # Train the model
+        trainer(
+            model=self.model,
+            x_train=to_numpy(x_ref),
+            y_train=None,
+            loss_fn=loss_fn,
+            optimizer=optimizer,
+            preprocess_fn=None,
+            epochs=epochs,
+            batch_size=batch_size,
+            device=self.device,
+            verbose=verbose,
+        )
+
+        # Calculate the GMM parameters
+        _, z, gamma = cast(tuple[torch.Tensor, torch.Tensor, torch.Tensor], self.model(x_ref))
+        self._gmm_params = gmm_params(z, gamma)
+
+        # Infer the threshold values
+        self._ref_score = self.score(x_ref, batch_size)
+        self._threshold_perc = threshold_perc

dataeval/detectors/ood/metadata_ks_compare.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+__all__ = []
+
 import numbers
 import warnings
 from typing import Any, Mapping, NamedTuple
@@ -40,51 +42,41 @@ class KSOutput(MappingOutput[str, MetadataKSResult]):
 def meta_distribution_compare(
     md0: Mapping[str, list[Any] | NDArray[Any]], md1: Mapping[str, list[Any] | NDArray[Any]]
 ) -> KSOutput:
-    """Measures the featurewise distance between two metadata distributions, and computes a p-value to evaluate its
-        significance.
-
-        Uses the Earth Mover's Distance and the Kolmogorov-Smirnov two-sample test, featurewise.
-
-        Parameters
-        ----------
-        md0 : Mapping[str, list[Any] | NDArray[Any]]
-            A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
-        md1 : Mapping[str, list[Any] | NDArray[Any]]
-            Another set of arrays of values, indexed by metadata feature names, with one value per data example per
-            feature.
-
-        Returns
-        -------
-        dict[str, KstestResult]
-            A dictionary with keys corresponding to metadata feature names, and values that are KstestResult objects, as
-            defined by scipy.stats.ks_2samp. These values also have two additional attributes: shift_magnitude and
-            statistic_location. The first is the Earth Mover's Distance normalized by the interquartile range (IQR) of
-            the reference, while the second is the value at which the KS statistic has its maximum, measured in
-            IQR-normalized units relative to the median of the reference distribution.
-
-        Examples
-        --------
-        Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
-        altitude.
-
-    >>> import numpy
+    """
+    Measures the featurewise distance between two metadata distributions, and computes a p-value to evaluate its
+    significance.
+
+    Uses the Earth Mover's Distance and the Kolmogorov-Smirnov two-sample test, featurewise.
+
+    Parameters
+    ----------
+    md0 : Mapping[str, list[Any] | NDArray[Any]]
+        A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
+    md1 : Mapping[str, list[Any] | NDArray[Any]]
+        Another set of arrays of values, indexed by metadata feature names, with one value per data example per
+        feature.
+
+    Returns
+    -------
+    dict[str, KstestResult]
+        A dictionary with keys corresponding to metadata feature names, and values that are KstestResult objects, as
+        defined by scipy.stats.ks_2samp. These values also have two additional attributes: shift_magnitude and
+        statistic_location. The first is the Earth Mover's Distance normalized by the interquartile range (IQR) of
+        the reference, while the second is the value at which the KS statistic has its maximum, measured in
+        IQR-normalized units relative to the median of the reference distribution.
+
+    Examples
+    --------
+    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
+    altitude.
+
     >>> md0 = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
     >>> md1 = {"time": [7.8, 9.10, 11.12], "altitude": [532, 9876, 211101]}
-    >>> md_out = meta_distribution_compare(md0, md1).mdc
+    >>> md_out = meta_distribution_compare(md0, md1)
     >>> for k, v in md_out.items():
-    >>>     print(k)
-    >>>     for kv in v:
-    >>>         print("\t", f"{kv}: {v[kv]:.3f}")
-    time
-            statistic: 1.000
-            statistic_location: 0.444
-            shift_magnitude: 2.700
-            pvalue: 0.000
-    altitude
-            statistic: 0.333
-            statistic_location: 0.478
-            shift_magnitude: 0.749
-            pvalue: 0.944
+    ...     print(f"{k}: { {kv: round(vv, 3) for kv, vv in v._asdict().items()} }")
+    time: {'statistic': 1.0, 'statistic_location': 0.444, 'shift_magnitude': 2.7, 'pvalue': 0.0}
+    altitude: {'statistic': 0.333, 'statistic_location': 0.478, 'shift_magnitude': 0.749, 'pvalue': 0.944}
     """
 
     if (metadata_keys := md0.keys()) != md1.keys():

dataeval/detectors/ood/metadata_least_likely.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+__all__ = []
+
 import numbers
 import warnings
 from typing import Any
@@ -41,11 +43,9 @@ def get_least_likely_features(
     Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
     altitude, as shown below.
 
-    >>> from dataeval._internal.metrics.metadata_least_likely import get_least_likely_features
-    >>> import numpy
     >>> metadata = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
     >>> new_metadata = {"time": [7.8, 11.12], "altitude": [532, -211101]}
-    >>> is_ood = numpy.array([True, True])
+    >>> is_ood = np.array([True, True])
     >>> get_least_likely_features(metadata, new_metadata, is_ood)
     [('time', 2.0), ('altitude', 33.245346)]
     """

dataeval/detectors/ood/metadata_ood_mi.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+__all__ = []
+
 import numbers
 import warnings
 from typing import Any
@@ -51,11 +53,10 @@ def get_metadata_ood_mi(
     --------
     Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and altitude.
 
-        >>> import numpy
-        >>> metadata = {"time": numpy.linspace(0, 10, 100), "altitude": numpy.linspace(0, 16, 100) ** 2}
-        >>> is_ood = metadata["altitude"] > 100
-        >>> print(get_metadata_ood_mi(metadata, is_ood, discrete_features=False))
-        {'time': 0.933074285817367, 'altitude': 0.9407686591507002}
+    >>> metadata = {"time": np.linspace(0, 10, 100), "altitude": np.linspace(0, 16, 100) ** 2}
+    >>> is_ood = metadata["altitude"] > 100
+    >>> get_metadata_ood_mi(metadata, is_ood, discrete_features=False, random_state=0)
+    {'time': 0.9359596758173668, 'altitude': 0.9407686591507002}
     """
     numerical_keys = [k for k, v in metadata.items() if all(isinstance(vi, numbers.Number) for vi in v)]
     if len(numerical_keys) < len(metadata):

dataeval/detectors/ood/mixin.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from dataeval.detectors.ood.output import OODOutput, OODScoreOutput
+
+__all__ = []
+
+from abc import ABC, abstractmethod
+from typing import Callable, Generic, Literal, TypeVar
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+from dataeval.interop import to_numpy
+from dataeval.output import set_metadata
+
+TGMMParams = TypeVar("TGMMParams")
+
+
+class OODGMMMixin(Generic[TGMMParams]):
+    _gmm_params: TGMMParams
+
+
+TModel = TypeVar("TModel", bound=Callable)
+TLossFn = TypeVar("TLossFn", bound=Callable)
+TOptimizer = TypeVar("TOptimizer")
+
+
+class OODFitMixin(Generic[TLossFn, TOptimizer], ABC):
+    @abstractmethod
+    def fit(
+        self,
+        x_ref: ArrayLike,
+        threshold_perc: float,
+        loss_fn: TLossFn | None,
+        optimizer: TOptimizer | None,
+        epochs: int,
+        batch_size: int,
+        verbose: bool,
+    ) -> None:
+        """
+        Train the model and infer the threshold value.
+
+        Parameters
+        ----------
+        x_ref : ArrayLike
+            Training data.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : TLossFn
+            Loss function used for training.
+        optimizer : TOptimizer
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        """
+
+
+class OODBaseMixin(Generic[TModel], ABC):
+    _ref_score: OODScoreOutput
+    _threshold_perc: float
+    _data_info: tuple[tuple, type] | None = None
+
+    def __init__(
+        self,
+        model: TModel,
+    ) -> None:
+        self.model = model
+
+    def _get_data_info(self, X: NDArray) -> tuple[tuple, type]:
+        if not isinstance(X, np.ndarray):
+            raise TypeError("Dataset should of type: `NDArray`.")
+        return X.shape[1:], X.dtype.type
+
+    def _validate(self, X: NDArray) -> None:
+        check_data_info = self._get_data_info(X)
+        if self._data_info is not None and check_data_info != self._data_info:
+            raise RuntimeError(
+                f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
+                               Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}."
+            )
+
+    def _validate_state(self, X: NDArray) -> None:
+        attrs = [k for c in self.__class__.mro()[:-1][::-1] if hasattr(c, "__annotations__") for k in c.__annotations__]
+        if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
+            raise RuntimeError("Metric needs to be `fit` before method call.")
+        self._validate(X)
+
+    @abstractmethod
+    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput: ...
+
+    @set_metadata
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
+        """
+        Compute the :term:`out of distribution<Out-of-distribution (OOD)>` scores for a given dataset.
+
+        Parameters
+        ----------
+        X : ArrayLike
+            Input data to score.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+            Use a smaller batch size if your dataset is large or if you encounter memory issues.
+
+        Returns
+        -------
+        OODScoreOutput
+            An object containing the instance-level and feature-level OOD scores.
+        """
+        return self._score(X, batch_size)
+
+    def _threshold_score(self, ood_type: Literal["feature", "instance"] = "instance") -> np.floating:
+        return np.percentile(self._ref_score.get(ood_type), self._threshold_perc)
+
+    @set_metadata
+    def predict(
+        self,
+        X: ArrayLike,
+        batch_size: int = int(1e10),
+        ood_type: Literal["feature", "instance"] = "instance",
+    ) -> OODOutput:
+        """
+        Predict whether instances are :term:`out of distribution<Out-of-distribution (OOD)>` or not.
+
+        Parameters
+        ----------
+        X : ArrayLike
+            Input data for out-of-distribution prediction.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+        ood_type : "feature" | "instance", default "instance"
+            Predict out-of-distribution at the 'feature' or 'instance' level.
+
+        Returns
+        -------
+        Dictionary containing the outlier predictions for the selected level,
+        and the OOD scores for the data including both 'instance' and 'feature' (if present) level scores.
+        """
+        self._validate_state(X := to_numpy(X))
+        # compute outlier scores
+        score = self.score(X, batch_size=batch_size)
+        ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
+        return OODOutput(is_ood=ood_pred, **score.dict())

dataeval/detectors/ood/output.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+__all__ = []
+
+from dataclasses import dataclass
+from typing import Literal
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval.output import Output
+
+
+@dataclass(frozen=True)
+class OODOutput(Output):
+    """
+    Output class for predictions from out-of-distribution detectors.
+
+    Attributes
+    ----------
+    is_ood : NDArray
+        Array of images that are detected as :term:Out-of-Distribution (OOD)`
+    instance_score : NDArray
+        Instance score of the evaluated dataset
+    feature_score : NDArray | None
+        Feature score, if available, of the evaluated dataset
+    """
+
+    is_ood: NDArray[np.bool_]
+    instance_score: NDArray[np.float32]
+    feature_score: NDArray[np.float32] | None
+
+
+@dataclass(frozen=True)
+class OODScoreOutput(Output):
+    """
+    Output class for instance and feature scores from out-of-distribution detectors.
+
+    Parameters
+    ----------
+    instance_score : NDArray
+        Instance score of the evaluated dataset.
+    feature_score : NDArray | None, default None
+        Feature score, if available, of the evaluated dataset.
+    """
+
+    instance_score: NDArray[np.float32]
+    feature_score: NDArray[np.float32] | None = None
+
+    def get(self, ood_type: Literal["instance", "feature"]) -> NDArray[np.float32]:
+        """
+        Returns either the instance or feature score
+
+        Parameters
+        ----------
+        ood_type : "instance" | "feature"
+
+        Returns
+        -------
+        NDArray
+            Either the instance or feature score based on input selection
+        """
+        return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score

dataeval/interop.py

@@ -1,18 +1,19 @@
-from __future__ import annotations
-
-from types import ModuleType
+"""Utility functions for interoperability with different array types."""
 
-from dataeval.logging import LogMessage
+from __future__ import annotations
 
-__all__ = ["as_numpy", "to_numpy", "to_numpy_iter"]
+__all__ = []
 
 import logging
 from importlib import import_module
+from types import ModuleType
 from typing import Any, Iterable, Iterator
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
+from dataeval.log import LogMessage
+
 _logger = logging.getLogger(__name__)
 
 _MODULE_CACHE = {}

dataeval/{logging.py → log.py}

@@ -1,3 +1,5 @@
+__all__ = []
+
 from typing import Callable
 
 

dataeval/metrics/__init__.py

@@ -3,6 +3,6 @@ Metrics are a way to measure the performance of your models or datasets that
 can then be analyzed in the context of a given problem.
 """
 
-from dataeval.metrics import bias, estimators, stats
-
 __all__ = ["bias", "estimators", "stats"]
+
+from dataeval.metrics import bias, estimators, stats