dataeval 0.65.0__py3-none-any.whl → 0.67.0__py3-none-any.whl

dataeval/_internal/detectors/ood/vaegmm.py

@@ -6,10 +6,13 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
+from __future__ import annotations
+
 from typing import Callable
 
 import keras
 import numpy as np
+import tensorflow as tf
 from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
@@ -21,17 +24,18 @@ from dataeval._internal.models.tensorflow.utils import predict_batch
 
 
 class OOD_VAEGMM(OODGMMBase):
-    def __init__(self, model: VAEGMM, samples: int = 10) -> None:
-        """
-        VAE with Gaussian Mixture Model based outlier detector.
+    """
+    VAE with Gaussian Mixture Model based outlier detector.
 
-        Parameters
-        ----------
-        model : VAEGMM
-            A VAEGMM model.
-        samples
-            Number of samples sampled to evaluate each instance.
-        """
+    Parameters
+    ----------
+    model : VAEGMM
+        A VAEGMM model.
+    samples
+        Number of samples sampled to evaluate each instance.
+    """
+
+    def __init__(self, model: VAEGMM, samples: int = 10) -> None:
         super().__init__(model)
         self.samples = samples
 
@@ -39,35 +43,37 @@ class OOD_VAEGMM(OODGMMBase):
         self,
         x_ref: ArrayLike,
         threshold_perc: float = 100.0,
-        loss_fn: Callable = LossGMM(elbo=Elbo(0.05)),
+        loss_fn: Callable[..., tf.Tensor] | None = None,
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
     ) -> None:
+        if loss_fn is None:
+            loss_fn = LossGMM(elbo=Elbo(0.05))
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
         """
-        Train the AE model with recommended loss function and optimizer.
+        Compute the out-of-distribution (OOD) score for a given dataset.
 
         Parameters
         ----------
         X : ArrayLike
-            Training batch.
-        threshold_perc : float, default 100.0
-            Percentage of reference data that is normal.
-        loss_fn : Callable, default LossGMM(elbo=Elbo(0.05))
-            Loss function used for training.
-        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
-            Optimizer used for training.
-        epochs : int, default 20
-            Number of training epochs.
-        batch_size : int, default 64
-            Batch size used for training.
-        verbose : bool, default True
-            Whether to print training progress.
-        """
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+            Input data to score.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+            Use a smaller batch size if your dataset is large or if you encounter memory issues.
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        Returns
+        -------
+        OODScore
+            An object containing the instance-level OOD score.
+
+        Note
+        ----
+        This model does not produce a feature level score like the OOD_AE or OOD_VAE models.
+        """
         self._validate(X := to_numpy(X))
 
         # draw samples from latent space

dataeval/_internal/detectors/{linter.py → outliers.py}

@@ -1,17 +1,18 @@
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Dict, Iterable, Literal, Optional
+from typing import Iterable, Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.flags import verify_supported
+from dataeval._internal.flags import ImageStat, to_distinct, verify_supported
+from dataeval._internal.metrics.stats import StatsOutput, imagestats
 from dataeval._internal.output import OutputMetadata, set_metadata
-from dataeval.flags import ImageStat
-from dataeval.metrics import imagestats
 
 
 @dataclass(frozen=True)
-class LinterOutput(OutputMetadata):
+class OutliersOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -20,11 +21,11 @@ class LinterOutput(OutputMetadata):
         the issues and calculated values for the given index.
     """
 
-    issues: Dict[int, Dict[str, float]]
+    issues: dict[int, dict[str, float]]
 
 
 def _get_outlier_mask(
-    values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
+    values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: float | None
 ) -> NDArray:
     if method == "zscore":
         threshold = threshold if threshold else 3.0
@@ -46,7 +47,7 @@ def _get_outlier_mask(
         raise ValueError("Outlier method must be 'zscore' 'modzscore' or 'iqr'.")
 
 
-class Linter:
+class Outliers:
     r"""
     Calculates statistical outliers of a dataset using various statistical tests applied to each image
 
@@ -92,28 +93,28 @@ class Linter:
 
     Examples
     --------
-    Initialize the Linter class:
+    Initialize the Outliers class:
 
-    >>> lint = Linter()
+    >>> outliers = Outliers()
 
     Specifying specific metrics to analyze:
 
-    >>> lint = Linter(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
+    >>> outliers = Outliers(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
 
     Specifying an outlier method:
 
-    >>> lint = Linter(outlier_method="iqr")
+    >>> outliers = Outliers(outlier_method="iqr")
 
     Specifying an outlier method and threshold:
 
-    >>> lint = Linter(outlier_method="zscore", outlier_threshold=2.5)
+    >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=2.5)
     """
 
     def __init__(
         self,
         flags: ImageStat = ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
-        outlier_threshold: Optional[float] = None,
+        outlier_threshold: float | None = None,
     ):
         verify_supported(flags, ImageStat.ALL_STATS)
         self.flags = flags
@@ -123,11 +124,9 @@ class Linter:
     def _get_outliers(self) -> dict:
         flagged_images = {}
         stats_dict = self.stats.dict()
+        supported = to_distinct(ImageStat.ALL_STATS)
         for stat, values in stats_dict.items():
-            if not isinstance(values, np.ndarray):
-                continue
-
-            if values.ndim == 1 and np.std(values) != 0:
+            if stat in supported.values() and values.ndim == 1 and np.std(values) != 0:
                 mask = _get_outlier_mask(values, self.outlier_method, self.outlier_threshold)
                 indices = np.flatnonzero(mask)
                 for i, value in zip(indices, values[mask]):
@@ -136,19 +135,18 @@ class Linter:
         return dict(sorted(flagged_images.items()))
 
     @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
-    def evaluate(self, images: Iterable[ArrayLike]) -> LinterOutput:
+    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput) -> OutliersOutput:
         """
         Returns indices of outliers with the issues identified for each
 
         Parameters
         ----------
-        images : Iterable[ArrayLike], shape - (N, C, H, W)
-            A dataset in an ArrayLike format.
-            Function expects the data to have 3 dimensions, CxHxW.
+        data : Iterable[ArrayLike], shape - (C, H, W) | StatsOutput
+            A dataset of images in an ArrayLike format or the output from an imagestats metric analysis
 
         Returns
         -------
-        LinterOutput
+        OutliersOutput
             Output class containing the indices of outliers and a dictionary showing
             the issues and calculated values for the given index.
 
@@ -156,8 +154,16 @@ class Linter:
         -------
         Evaluate the dataset:
 
-        >>> lint.evaluate(images)
-        LinterOutput(issues={18: {'brightness': 0.78}, 25: {'brightness': 0.98}})
+        >>> outliers.evaluate(images)
+        OutliersOutput(issues={18: {'brightness': 0.78}, 25: {'brightness': 0.98}})
         """
-        self.stats = imagestats(images, self.flags)
-        return LinterOutput(self._get_outliers())
+        if isinstance(data, StatsOutput):
+            flags = set(to_distinct(self.flags).values())
+            stats = set(data.dict())
+            missing = flags - stats
+            if missing:
+                raise ValueError(f"StatsOutput is missing {missing} from the required stats: {flags}.")
+            self.stats = data
+        else:
+            self.stats = imagestats(data, self.flags)
+        return OutliersOutput(self._get_outliers())

dataeval/_internal/flags.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 from enum import IntFlag, auto
 from functools import reduce
-from typing import Dict, Iterable, TypeVar, Union, cast
+from typing import Iterable, TypeVar, cast
 
 TFlag = TypeVar("TFlag", bound=IntFlag)
 
@@ -47,7 +49,7 @@ def is_distinct(flag: IntFlag) -> bool:
     return (flag & (flag - 1) == 0) and flag != 0
 
 
-def to_distinct(flag: TFlag) -> Dict[TFlag, str]:
+def to_distinct(flag: TFlag) -> dict[TFlag, str]:
     """
     Returns a distinct set of all flags set on the input flag and their names
 
@@ -61,7 +63,7 @@ def to_distinct(flag: TFlag) -> Dict[TFlag, str]:
         return {f: f.name.lower() for f in list(flag.__class__) if f & flag and is_distinct(f) and f.name}
 
 
-def verify_supported(flag: TFlag, flags: Union[TFlag, Iterable[TFlag]]):
+def verify_supported(flag: TFlag, flags: TFlag | Iterable[TFlag]):
     supported = flags if isinstance(flags, flag.__class__) else cast(TFlag, reduce(lambda a, b: a | b, flags))  # type: ignore
     unsupported = flag & ~supported
     if unsupported:

dataeval/_internal/interop.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from importlib import import_module
-from typing import Iterable, Optional
+from typing import Iterable
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -20,7 +22,7 @@ def try_import(module_name):
     return module
 
 
-def to_numpy(array: Optional[ArrayLike]) -> NDArray:
+def to_numpy(array: ArrayLike | None) -> NDArray:
     if array is None:
         return np.ndarray([])
 

dataeval/_internal/metrics/balance.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import warnings
 from dataclasses import dataclass
-from typing import Dict, List, Sequence
+from typing import Sequence
 
 import numpy as np
 from numpy.typing import NDArray
@@ -43,7 +45,7 @@ def validate_num_neighbors(num_neighbors: int) -> int:
 
 
 @set_metadata("dataeval.metrics")
-def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Mutual information (MI) between factors (class label, metadata, label/image properties)
 
@@ -71,6 +73,22 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
     we attempt to infer whether a variable is categorical by the fraction of unique
     values in the dataset.
 
+    Example
+    -------
+    Return balance (mutual information) of factors with class_labels
+
+    >>> balance(class_labels, metadata).mutual_information[0]
+    array([0.99999822, 0.13363788, 0.        , 0.02994455])
+
+    Return balance (mutual information) of metadata factors with class_labels
+    and each other
+
+    >>> balance(class_labels, metadata).mutual_information
+    array([[0.99999822, 0.13363788, 0.        , 0.02994455],
+           [0.13363788, 0.99999843, 0.01389763, 0.09725766],
+           [0.        , 0.01389763, 0.48549233, 0.15314612],
+           [0.02994455, 0.09725766, 0.15314612, 0.99999856]])
+
     See Also
     --------
     sklearn.feature_selection.mutual_info_classif
@@ -96,14 +114,15 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
                 tgt,
                 discrete_features=is_categorical,  # type: ignore
                 n_neighbors=num_neighbors,
+                random_state=0,
             )
         else:
-            # continuous variables
             mi[idx, :] = mutual_info_regression(
                 data,
                 tgt,
                 discrete_features=is_categorical,  # type: ignore
                 n_neighbors=num_neighbors,
+                random_state=0,
             )
 
     ent_all = entropy(data, names, is_categorical, normalized=False)
@@ -115,7 +134,7 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
 
 
 @set_metadata("dataeval.metrics")
-def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+def balance_classwise(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Compute mutual information (analogous to correlation) between metadata factors
     (class label, metadata, label/image properties) with individual class labels.
@@ -143,6 +162,15 @@ def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_nei
         (num_classes x num_factors) estimate of mutual information between
         num_factors metadata factors and individual class labels.
 
+    Example
+    -------
+    Return classwise balance (mutual information) of factors with individual class_labels
+
+    >>> balance_classwise(class_labels, metadata).mutual_information
+    array([[0.13363788, 0.54085156, 0.        ],
+           [0.13363788, 0.54085156, 0.        ]])
+
+
     See Also
     --------
     sklearn.feature_selection.mutual_info_classif
@@ -177,6 +205,7 @@ def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_nei
             tgt,
             discrete_features=cat_mask,  # type: ignore
             n_neighbors=num_neighbors,
+            random_state=0,
         )
 
     # let this recompute for all features including class label

dataeval/_internal/metrics/ber.py

@@ -7,8 +7,10 @@ Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
 """
 
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Literal, Tuple
+from typing import Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -35,7 +37,7 @@ class BEROutput(OutputMetadata):
     ber_lower: float
 
 
-def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
+def ber_mst(X: NDArray, y: NDArray) -> tuple[float, float]:
     """Calculates the Bayes Error Rate using a minimum spanning tree
 
     Parameters
@@ -60,7 +62,7 @@ def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
     return upper, lower
 
 
-def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
+def ber_knn(X: NDArray, y: NDArray, k: int) -> tuple[float, float]:
     """Calculates the Bayes Error Rate using K-nearest neighbors
 
     Parameters
@@ -135,7 +137,7 @@ def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN",
     Examples
     --------
     >>> import sklearn.datasets as dsets
-    >>> from dataeval.metrics import ber
+    >>> from dataeval.metrics.estimators import ber
 
     >>> images, labels = dsets.make_blobs(n_samples=50, centers=2, n_features=2, random_state=0)
 

dataeval/_internal/metrics/diversity.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Dict, List, Literal, Optional, Sequence
+from typing import Literal, Sequence
 
 import numpy as np
 from numpy.typing import NDArray
@@ -22,9 +24,9 @@ class DiversityOutput(OutputMetadata):
 
 def diversity_shannon(
     data: NDArray,
-    names: List[str],
-    is_categorical: List[bool],
-    subset_mask: Optional[NDArray[np.bool_]] = None,
+    names: list[str],
+    is_categorical: list[bool],
+    subset_mask: NDArray[np.bool_] | None = None,
 ) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
@@ -37,7 +39,14 @@ def diversity_shannon(
 
     Parameters
     ----------
-    subset_mask: Optional[NDArray[np.bool_]]
+    data: NDArray
+        Array containing numerical values for metadata factors
+    names: list[str]
+        Names of metadata factors -- keys of the metadata dictionary
+    is_categorical: list[bool]
+        List of flags to identify whether variables are categorical (True) or
+        continuous (False)
+    subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -58,38 +67,44 @@ def diversity_shannon(
     ent_unnormalized = entropy(data, names, is_categorical, normalized=False, subset_mask=subset_mask)
     # normalize by global counts rather than classwise counts
     num_bins = get_num_bins(data, names, is_categorical=is_categorical, subset_mask=subset_mask)
-    return ent_unnormalized / np.log(num_bins)
+    ent_norm = np.empty(ent_unnormalized.shape)
+    ent_norm[num_bins != 1] = ent_unnormalized[num_bins != 1] / np.log(num_bins[num_bins != 1])
+    ent_norm[num_bins == 1] = 0
+    return ent_norm
 
 
 def diversity_simpson(
     data: NDArray,
-    names: List[str],
-    is_categorical: List[bool],
-    subset_mask: Optional[NDArray[np.bool_]] = None,
+    names: list[str],
+    is_categorical: list[bool],
+    subset_mask: NDArray[np.bool_] | None = None,
 ) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
 
-    We define diversity as a normalized form of the inverse Simpson diversity
-    index.
+    We define diversity as the inverse Simpson diversity index linearly rescaled to the unit interval.
 
     diversity = 1 implies that samples are evenly distributed across a particular factor
-    diversity = 1/num_categories implies that all samples belong to one category/bin
+    diversity = 0 implies that all samples belong to one category/bin
 
     Parameters
     ----------
-    subset_mask: Optional[NDArray[np.bool_]]
+    data: NDArray
+        Array containing numerical values for metadata factors
+    names: list[str]
+        Names of metadata factors -- keys of the metadata dictionary
+    is_categorical: list[bool]
+        List of flags to identify whether variables are categorical (True) or
+        continuous (False)
+    subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
     -----
     For continuous variables, histogram bins are chosen automatically.  See
         numpy.histogram for details.
-    The expression is undefined for q=1, but it approaches the Shannon entropy
-        in the limit.
-    If there is only one category, the diversity index takes a value of 1 =
-        1/N = 1/1.  Entropy will take a value of 0.
+    If there is only one category, the diversity index takes a value of 0.
 
     Returns
     -------
@@ -111,8 +126,8 @@ def diversity_simpson(
         # relative frequencies
         p_i = cnts / cnts.sum()
         # inverse Simpson index normalized by (number of bins)
-        ev_index[col] = 1 / np.sum(p_i**2) / num_bins[col]
-
+        s_0 = 1 / np.sum(p_i**2) / num_bins[col]
+        ev_index[col] = (s_0 * num_bins[col] - 1) / (num_bins[col] - 1)
     return ev_index
 
 
@@ -121,7 +136,7 @@ DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
 
 @set_metadata("dataeval.metrics")
 def diversity(
-    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity for discrete/categorical variables and, through standard
@@ -136,9 +151,8 @@ def diversity(
         List of class labels for each image
     metadata: List[Dict]
         List of metadata factors for each image
-    metric: Literal["shannon", "simpson"], default "simpson"
-        string variable indicating which diversity index should be used.
-        Permissible values include "simpson" and "shannon"
+    method: Literal["shannon", "simpson"], default "simpson"
+        Indicates which diversity index should be computed
 
     Notes
     -----
@@ -149,6 +163,19 @@ def diversity(
     DiversityOutput
         Diversity index per column of self.data or each factor in self.names
 
+    Example
+    -------
+    Compute Simpson diversity index of metadata and class labels
+
+    >>> diversity(class_labels, metadata, method="simpson").diversity_index
+    array([0.18103448, 0.18103448, 0.88636364])
+
+    Compute Shannon diversity index of metadata and class labels
+
+    >>> diversity(class_labels, metadata, method="shannon").diversity_index
+    array([0.37955133, 0.37955133, 0.96748876])
+
+
     See Also
     --------
     numpy.histogram
@@ -161,7 +188,7 @@ def diversity(
 
 @set_metadata("dataeval.metrics")
 def diversity_classwise(
-    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity for discrete/categorical variables and, through standard
@@ -171,7 +198,7 @@ def diversity_classwise(
     index.
 
     diversity = 1 implies that samples are evenly distributed across a particular factor
-    diversity = 1/num_categories implies that all samples belong to one category/bin
+    diversity = 0 implies that all samples belong to one category/bin
 
     Parameters
     ----------
@@ -179,18 +206,34 @@ def diversity_classwise(
         List of class labels for each image
     metadata: List[Dict]
         List of metadata factors for each image
+    method: Literal["shannon", "simpson"], default "simpson"
+        Indicates which diversity index should be computed
 
     Notes
     -----
     - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
-    - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
-    - If there is only one category, the diversity index takes a value of 1 = 1/N = 1/1. Entropy will take a value of 0.
+    - If there is only one category, the diversity index takes a value of 0.
 
     Returns
     -------
     DiversityOutput
         Diversity index [n_class x n_factor]
 
+    Example
+    -------
+    Compute classwise Simpson diversity index of metadata and class labels
+
+    >>> diversity_classwise(class_labels, metadata, method="simpson").diversity_index
+    array([[0.17241379, 0.39473684],
+           [0.2       , 0.2       ]])
+
+    Compute classwise Shannon diversity index of metadata and class labels
+
+    >>> diversity_classwise(class_labels, metadata, method="shannon").diversity_index
+    array([[0.43156028, 0.83224889],
+           [0.57938016, 0.57938016]])
+
+
     See Also
     --------
     numpy.histogram