dataeval 0.69.4__py3-none-any.whl → 0.70.1__py3-none-any.whl

dataeval/_internal/detectors/outliers.py

@@ -1,39 +1,47 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Iterable, Literal, Sequence, cast
-from warnings import warn
+from typing import Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
-from dataeval._internal.flags import ImageStat, to_distinct, verify_supported
-from dataeval._internal.metrics.stats import StatsOutput, imagestats
+from dataeval._internal.metrics.stats.base import BOX_COUNT, SOURCE_INDEX
+from dataeval._internal.metrics.stats.datasetstats import DatasetStatsOutput, datasetstats
+from dataeval._internal.metrics.stats.dimensionstats import DimensionStatsOutput
+from dataeval._internal.metrics.stats.pixelstats import PixelStatsOutput
+from dataeval._internal.metrics.stats.visualstats import VisualStatsOutput
 from dataeval._internal.output import OutputMetadata, set_metadata
 
 IndexIssueMap = dict[int, dict[str, float]]
-DatasetIndexIssueMap = dict[int, IndexIssueMap]
-"""
-Mapping of image indices to a dictionary of issue types and calculated values
-"""
+OutlierStatsOutput = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, list[IndexIssueMap])
 
 
 @dataclass(frozen=True)
-class OutliersOutput(OutputMetadata):
+class OutliersOutput(Generic[TIndexIssueMap], OutputMetadata):
     """
+    Output class for :class:`Outliers` lint detector
+
     Attributes
     ----------
-    issues : dict[int, dict[str, float]] | dict[int, dict[int, dict[str, float]]]
+    issues : dict[int, dict[str, float]] | list[dict[int, dict[str, float]]]
         Indices of image outliers with their associated issue type and calculated values.
 
     - For a single dataset, a dictionary containing the indices of outliers and
       a dictionary showing the issues and calculated values for the given index.
-    - For multiple datasets, a map of dataset indices to the indices of outliers
-      and their associated issues and calculated values.
+    - For multiple stats outputs, a list of dictionaries containing the indices of
+      outliers and their associated issues and calculated values.
     """
 
-    issues: IndexIssueMap | DatasetIndexIssueMap
+    issues: TIndexIssueMap
+
+    def __len__(self):
+        if isinstance(self.issues, dict):
+            return len(self.issues)
+        else:
+            return sum(len(d) for d in self.issues)
 
 
 def _get_outlier_mask(
@@ -43,7 +51,7 @@ def _get_outlier_mask(
         threshold = threshold if threshold else 3.0
         std = np.std(values)
         abs_diff = np.abs(values - np.mean(values))
-        return (abs_diff / std) > threshold
+        return std != 0 and (abs_diff / std) > threshold
     elif method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
@@ -65,9 +73,6 @@ class Outliers:
 
     Parameters
     ----------
-    flags : ImageStat, default ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS
-        Metric(s) to calculate for each image - calculates all metrics if None
-        Only supports ImageStat.ALL_STATS
     outlier_method : ["modzscore" | "zscore" | "iqr"], optional - default "modzscore"
         Statistical method used to identify outliers
     outlier_threshold : float, optional - default None
@@ -76,15 +81,15 @@ class Outliers:
 
     Attributes
     ----------
-    stats : dict[str, Any]
-        Dictionary to hold the value of each metric for each image
+    stats : tuple[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+        Various stats output classes that hold the value of each metric for each image
 
     See Also
     --------
     Duplicates
 
-    Notes
-    ------
+    Note
+    ----
     There are 3 different statistical methods:
 
     - zscore
@@ -109,52 +114,61 @@ class Outliers:
 
     >>> outliers = Outliers()
 
-    Specifying specific metrics to analyze:
-
-    >>> outliers = Outliers(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
-
     Specifying an outlier method:
 
     >>> outliers = Outliers(outlier_method="iqr")
 
     Specifying an outlier method and threshold:
 
-    >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=2.75)
+    >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=3.5)
     """
 
     def __init__(
         self,
-        flags: ImageStat = ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS,
+        use_dimension: bool = True,
+        use_pixel: bool = True,
+        use_visual: bool = True,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: float | None = None,
     ):
-        verify_supported(flags, ImageStat.ALL_STATS)
-        self.flags = flags
+        self.stats: DatasetStatsOutput
+        self.use_dimension = use_dimension
+        self.use_pixel = use_pixel
+        self.use_visual = use_visual
         self.outlier_method: Literal["zscore", "modzscore", "iqr"] = outlier_method
         self.outlier_threshold = outlier_threshold
 
-    def _get_outliers(self) -> dict:
-        flagged_images = {}
-        stats_dict = self.stats.dict()
-        supported = to_distinct(ImageStat.ALL_STATS)
-        for stat, values in stats_dict.items():
-            if stat in supported.values() and values.ndim == 1 and np.std(values) != 0:
-                mask = _get_outlier_mask(values, self.outlier_method, self.outlier_threshold)
+    def _get_outliers(self, stats: dict) -> dict[int, dict[str, float]]:
+        flagged_images: dict[int, dict[str, float]] = {}
+        for stat, values in stats.items():
+            if stat in (SOURCE_INDEX, BOX_COUNT):
+                continue
+            if values.ndim == 1:
+                mask = _get_outlier_mask(values.astype(np.float64), self.outlier_method, self.outlier_threshold)
                 indices = np.flatnonzero(mask)
                 for i, value in zip(indices, values[mask]):
-                    flagged_images.setdefault(i, {}).update({stat: np.round(value, 2)})
+                    flagged_images.setdefault(i, {}).update({stat: value})
 
         return dict(sorted(flagged_images.items()))
 
-    @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
-    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> OutliersOutput:
+    @overload
+    def from_stats(self, stats: OutlierStatsOutput | DatasetStatsOutput) -> OutliersOutput[IndexIssueMap]: ...
+
+    @overload
+    def from_stats(self, stats: Sequence[OutlierStatsOutput]) -> OutliersOutput[list[IndexIssueMap]]: ...
+
+    @set_metadata("dataeval.detectors", ["outlier_method", "outlier_threshold"])
+    def from_stats(
+        self, stats: OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+    ) -> OutliersOutput:
         """
         Returns indices of outliers with the issues identified for each
 
         Parameters
         ----------
-        data : Iterable[ArrayLike], shape - (C, H, W) | StatsOutput | Sequence[StatsOutput]
-            A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
+        stats : OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+            The output(s) from a dimensionstats, pixelstats, or visualstats metric
+            analysis or an aggregate DatasetStatsOutput
 
         Returns
         -------
@@ -162,36 +176,96 @@ class Outliers:
             Output class containing the indices of outliers and a dictionary showing
             the issues and calculated values for the given index.
 
+        See Also
+        --------
+        dimensionstats
+        pixelstats
+        visualstats
+
         Example
         -------
         Evaluate the dataset:
 
-        >>> outliers.evaluate(images)
-        OutliersOutput(issues={10: {'blurriness': 1.26, 'contrast': 1.06, 'zeros': 0.05}, 12: {'blurriness': 1.51, 'contrast': 1.06, 'zeros': 0.05}})
+        >>> results = outliers.from_stats([stats1, stats2])
+        >>> len(results)
+        2
+        >>> results.issues[0]
+        {10: {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128}, 12: {'std': 0.00536, 'var': 2.87e-05, 'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128}}
+        >>> results.issues[1]
+        {}
         """  # noqa: E501
-        stats, dataset_steps = combine_stats(data)
-
-        if isinstance(stats, StatsOutput):
-            selected_flags = set(to_distinct(self.flags).values())
-            provided = set(stats.dict())
-            missing = selected_flags - provided
-            if missing:
-                warn(
-                    f"StatsOutput provided {provided} and is missing {missing} \
-                        from the selected stat flags: {selected_flags}."
+        if isinstance(stats, DatasetStatsOutput):
+            outliers = self._get_outliers({k: v for o in stats.outputs() for k, v in o.dict().items()})
+            return OutliersOutput(outliers)
+
+        if isinstance(stats, (DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)):
+            return OutliersOutput(self._get_outliers(stats.dict()))
+
+        if not isinstance(stats, Sequence):
+            raise TypeError(
+                "Invalid stats output type; only use output from dimensionstats, pixelstats or visualstats."
+            )
+
+        stats_map: dict[type, list[int]] = {}
+        for i, stats_output in enumerate(stats):
+            if not isinstance(
+                stats_output, (DatasetStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)
+            ):
+                raise TypeError(
+                    "Invalid stats output type; only use output from dimensionstats, pixelstats or visualstats."
                 )
-            self.stats = stats
-        else:
-            self.stats = imagestats(cast(Iterable[ArrayLike], data), self.flags)
-
-        outliers = self._get_outliers()
+            stats_map.setdefault(type(stats_output), []).append(i)
 
-        # split up results from combined dataset into individual dataset buckets
-        if dataset_steps:
-            out_dict = {}
+        output_list: list[dict[int, dict[str, float]]] = [{} for _ in stats]
+        for _, indices in stats_map.items():
+            substats, dataset_steps = combine_stats([stats[i] for i in indices])
+            outliers = self._get_outliers(substats.dict())
             for idx, issue in outliers.items():
                 k, v = get_dataset_step_from_idx(idx, dataset_steps)
-                out_dict.setdefault(k, {})[v] = issue
-            outliers = out_dict
+                output_list[indices[k]][v] = issue
+
+        return OutliersOutput(output_list)
+
+    @set_metadata(
+        "dataeval.detectors",
+        [
+            "use_dimension",
+            "use_pixel",
+            "use_visual",
+            "outlier_method",
+            "outlier_threshold",
+        ],
+    )
+    def evaluate(self, data: Iterable[ArrayLike]) -> OutliersOutput[IndexIssueMap]:
+        """
+        Returns indices of outliers with the issues identified for each
+
+        Parameters
+        ----------
+        data : Iterable[ArrayLike], shape - (C, H, W)
+            A dataset of images in an ArrayLike format
 
+        Returns
+        -------
+        OutliersOutput
+            Output class containing the indices of outliers and a dictionary showing
+            the issues and calculated values for the given index.
+
+        Example
+        -------
+        Evaluate the dataset:
+
+        >>> results = outliers.evaluate(images)
+        >>> list(results.issues)
+        [10, 12]
+        >>> results.issues[10]
+        {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128, 'contrast': 1.25, 'zeros': 0.05493}
+        """
+        self.stats = datasetstats(
+            images=data,
+            use_dimension=self.use_dimension,
+            use_pixel=self.use_pixel,
+            use_visual=self.use_visual,
+        )
+        outliers = self._get_outliers({k: v for o in self.stats.outputs() for k, v in o.dict().items()})
         return OutliersOutput(outliers)

dataeval/_internal/interop.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from importlib import import_module
-from typing import Iterable
+from typing import Any, Iterable, Iterator
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -22,24 +22,28 @@ def try_import(module_name):
     return module
 
 
-def to_numpy(array: ArrayLike | None) -> NDArray:
+def as_numpy(array: ArrayLike | None) -> NDArray[Any]:
+    return to_numpy(array, copy=False)
+
+
+def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
     if array is None:
         return np.ndarray([])
 
     if isinstance(array, np.ndarray):
-        return array
+        return array.copy() if copy else array
 
     tf = try_import("tensorflow")
     if tf and tf.is_tensor(array):
-        return array.numpy()  # type: ignore
+        return array.numpy().copy() if copy else array.numpy()  # type: ignore
 
     torch = try_import("torch")
     if torch and isinstance(array, torch.Tensor):
-        return array.detach().cpu().numpy()  # type: ignore
+        return array.detach().cpu().numpy().copy() if copy else array.detach().cpu().numpy()  # type: ignore
 
-    return np.asarray(array)
+    return np.array(array, copy=copy)
 
 
-def to_numpy_iter(iterable: Iterable[ArrayLike]):
+def to_numpy_iter(iterable: Iterable[ArrayLike]) -> Iterator[NDArray[Any]]:
     for array in iterable:
         yield to_numpy(array)

dataeval/_internal/metrics/balance.py

@@ -2,10 +2,10 @@ from __future__ import annotations
 
 import warnings
 from dataclasses import dataclass
-from typing import Sequence
+from typing import Mapping
 
 import numpy as np
-from numpy.typing import NDArray
+from numpy.typing import ArrayLike, NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
 from dataeval._internal.metrics.utils import entropy, preprocess_metadata
@@ -15,6 +15,8 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class BalanceOutput(OutputMetadata):
     """
+    Output class for :func:`balance` bias metric
+
     Attributes
     ----------
     balance : NDArray[np.float64]
@@ -51,16 +53,16 @@ def validate_num_neighbors(num_neighbors: int) -> int:
 
 
 @set_metadata("dataeval.metrics")
-def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
+def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neighbors: int = 5) -> BalanceOutput:
     """
     Mutual information (MI) between factors (class label, metadata, label/image properties)
 
     Parameters
     ----------
-    class_labels: Sequence[int]
+    class_labels: ArrayLike
         List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
+    metadata: Mapping[str, ArrayLike]
+        Dict of lists of metadata factors for each image
     num_neighbors: int, default 5
         Number of nearest neighbors to use for computing MI between discrete
         and continuous variables.
@@ -71,8 +73,8 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
         (num_factors+1) x (num_factors+1) estimate of mutual information
         between num_factors metadata factors and class label. Symmetry is enforced.
 
-    Notes
-    -----
+    Note
+    ----
     We use `mutual_info_classif` from sklearn since class label is categorical.
     `mutual_info_classif` outputs are consistent up to O(1e-4) and depend on a random
     seed. MI is computed differently for categorical and continuous variables, and
@@ -90,9 +92,9 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
     Return intra/interfactor balance (mutual information)
 
     >>> bal.factors
-    array([[0.99999843, 0.03510422, 0.09725766],
-           [0.03510422, 0.08433558, 0.15621459],
-           [0.09725766, 0.15621459, 0.99999856]])
+    array([[0.99999843, 0.04133555, 0.09725766],
+           [0.04133555, 0.08433558, 0.1301489 ],
+           [0.09725766, 0.1301489 , 0.99999856]])
 
     Return classwise balance (mutual information) of factors with individual class_labels
 

dataeval/_internal/metrics/ber.py

@@ -17,7 +17,7 @@ from numpy.typing import ArrayLike, NDArray
 from scipy.sparse import coo_matrix
 from scipy.stats import mode
 
-from dataeval._internal.interop import to_numpy
+from dataeval._internal.interop import as_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
 from dataeval._internal.output import OutputMetadata, set_metadata
 
@@ -25,6 +25,8 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class BEROutput(OutputMetadata):
     """
+    Output class for :func:`ber` estimator metric
+
     Attributes
     ----------
     ber : float
@@ -145,7 +147,7 @@ def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN",
     BEROutput(ber=0.04, ber_lower=0.020416847668728033)
     """
     ber_fn = get_method(BER_FN_MAP, method)
-    X = to_numpy(images)
-    y = to_numpy(labels)
+    X = as_numpy(images)
+    y = as_numpy(labels)
     upper, lower = ber_fn(X, y, k) if method == "KNN" else ber_fn(X, y)
     return BEROutput(upper, lower)

dataeval/_internal/metrics/coverage.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import math
 from dataclasses import dataclass
 from typing import Literal
@@ -14,6 +16,8 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class CoverageOutput(OutputMetadata):
     """
+    Output class for :func:`coverage` bias metric
+
     Attributes
     ----------
     indices : NDArray

dataeval/_internal/metrics/divergence.py

@@ -3,13 +3,15 @@ This module contains the implementation of HP Divergence
 using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 """
 
+from __future__ import annotations
+
 from dataclasses import dataclass
 from typing import Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.interop import to_numpy
+from dataeval._internal.interop import as_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
 from dataeval._internal.output import OutputMetadata, set_metadata
 
@@ -17,6 +19,8 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class DivergenceOutput(OutputMetadata):
     """
+    Output class for :func:`divergence` estimator metric
+
     Attributes
     ----------
     divergence : float
@@ -96,8 +100,8 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     DivergenceOutput
         The divergence value (0.0..1.0) and the number of differing edges between the datasets
 
-    Notes
-    -----
+    Note
+    ----
     The divergence value indicates how similar the 2 datasets are
     with 0 indicating approximately identical data distributions.
 
@@ -123,8 +127,8 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     DivergenceOutput(divergence=0.28, errors=36.0)
     """
     div_fn = get_method(DIVERGENCE_FN_MAP, method)
-    a = to_numpy(data_a)
-    b = to_numpy(data_b)
+    a = as_numpy(data_a)
+    b = as_numpy(data_b)
     N = a.shape[0]
     M = b.shape[0]
 

dataeval/_internal/metrics/diversity.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Literal, Sequence
+from typing import Literal, Mapping
 
 import numpy as np
-from numpy.typing import NDArray
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
 from dataeval._internal.output import OutputMetadata, set_metadata
@@ -13,6 +13,8 @@ from dataeval._internal.output import OutputMetadata, set_metadata
 @dataclass(frozen=True)
 class DiversityOutput(OutputMetadata):
     """
+    Output class for :func:`diversity` bias metric
+
     Attributes
     ----------
     diversity_index : NDArray[np.float64]
@@ -52,8 +54,8 @@ def diversity_shannon(
     subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
-    Notes
-    -----
+    Note
+    ----
     For continuous variables, histogram bins are chosen automatically.  See `numpy.histogram` for details.
 
     Returns
@@ -103,8 +105,8 @@ def diversity_simpson(
     subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
-    Notes
-    -----
+    Note
+    ----
     For continuous variables, histogram bins are chosen automatically.  See
         numpy.histogram for details.
     If there is only one category, the diversity index takes a value of 0.
@@ -142,7 +144,7 @@ DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
 
 @set_metadata("dataeval.metrics")
 def diversity(
-    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity and classwise diversity for discrete/categorical variables and, through standard
@@ -155,15 +157,15 @@ def diversity(
 
     Parameters
     ----------
-    class_labels: Sequence[int]
+    class_labels: ArrayLike
         List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
+    metadata: Mapping[str, ArrayLike]
+        Dict of list of metadata factors for each image
     method: Literal["shannon", "simpson"], default "simpson"
         Indicates which diversity index should be computed
 
-    Notes
-    -----
+    Note
+    ----
     - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
     - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
     - If there is only one category, the diversity index takes a value of 1 = 1/N = 1/1. Entropy will take a value of 0.