dataeval 0.69.3__py3-none-any.whl → 0.70.0__py3-none-any.whl

dataeval/_internal/detectors/outliers.py

@@ -1,39 +1,45 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Iterable, Literal, Sequence, cast
-from warnings import warn
+from typing import Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.detectors.merged_stats import combine_stats, get_dataset_step_from_idx
-from dataeval._internal.flags import ImageStat, to_distinct, verify_supported
-from dataeval._internal.metrics.stats import StatsOutput, imagestats
+from dataeval._internal.metrics.stats.base import BOX_COUNT, SOURCE_INDEX
+from dataeval._internal.metrics.stats.datasetstats import DatasetStatsOutput, datasetstats
+from dataeval._internal.metrics.stats.dimensionstats import DimensionStatsOutput
+from dataeval._internal.metrics.stats.pixelstats import PixelStatsOutput
+from dataeval._internal.metrics.stats.visualstats import VisualStatsOutput
 from dataeval._internal.output import OutputMetadata, set_metadata
 
 IndexIssueMap = dict[int, dict[str, float]]
-DatasetIndexIssueMap = dict[int, IndexIssueMap]
-"""
-Mapping of image indices to a dictionary of issue types and calculated values
-"""
+OutlierStatsOutput = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, list[IndexIssueMap])
 
 
 @dataclass(frozen=True)
-class OutliersOutput(OutputMetadata):
+class OutliersOutput(Generic[TIndexIssueMap], OutputMetadata):
     """
     Attributes
     ----------
-    issues : dict[int, dict[str, float]] | dict[int, dict[int, dict[str, float]]]
+    issues : dict[int, dict[str, float]] | list[dict[int, dict[str, float]]]
         Indices of image outliers with their associated issue type and calculated values.
 
     - For a single dataset, a dictionary containing the indices of outliers and
       a dictionary showing the issues and calculated values for the given index.
-    - For multiple datasets, a map of dataset indices to the indices of outliers
-      and their associated issues and calculated values.
+    - For multiple stats outputs, a list of dictionaries containing the indices of
+      outliers and their associated issues and calculated values.
     """
 
-    issues: IndexIssueMap | DatasetIndexIssueMap
+    issues: TIndexIssueMap
+
+    def __len__(self):
+        if isinstance(self.issues, dict):
+            return len(self.issues)
+        else:
+            return sum(len(d) for d in self.issues)
 
 
 def _get_outlier_mask(
@@ -43,7 +49,7 @@ def _get_outlier_mask(
         threshold = threshold if threshold else 3.0
         std = np.std(values)
         abs_diff = np.abs(values - np.mean(values))
-        return (abs_diff / std) > threshold
+        return std != 0 and (abs_diff / std) > threshold
     elif method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
@@ -65,9 +71,6 @@ class Outliers:
 
     Parameters
     ----------
-    flags : ImageStat, default ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS
-        Metric(s) to calculate for each image - calculates all metrics if None
-        Only supports ImageStat.ALL_STATS
     outlier_method : ["modzscore" | "zscore" | "iqr"], optional - default "modzscore"
         Statistical method used to identify outliers
     outlier_threshold : float, optional - default None
@@ -76,8 +79,8 @@ class Outliers:
 
     Attributes
     ----------
-    stats : dict[str, Any]
-        Dictionary to hold the value of each metric for each image
+    stats : tuple[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+        Various stats output classes that hold the value of each metric for each image
 
     See Also
     --------
@@ -109,52 +112,61 @@ class Outliers:
 
     >>> outliers = Outliers()
 
-    Specifying specific metrics to analyze:
-
-    >>> outliers = Outliers(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
-
     Specifying an outlier method:
 
     >>> outliers = Outliers(outlier_method="iqr")
 
     Specifying an outlier method and threshold:
 
-    >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=2.75)
+    >>> outliers = Outliers(outlier_method="zscore", outlier_threshold=3.5)
     """
 
     def __init__(
         self,
-        flags: ImageStat = ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS,
+        use_dimension: bool = True,
+        use_pixel: bool = True,
+        use_visual: bool = True,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: float | None = None,
     ):
-        verify_supported(flags, ImageStat.ALL_STATS)
-        self.flags = flags
+        self.stats: DatasetStatsOutput
+        self.use_dimension = use_dimension
+        self.use_pixel = use_pixel
+        self.use_visual = use_visual
         self.outlier_method: Literal["zscore", "modzscore", "iqr"] = outlier_method
         self.outlier_threshold = outlier_threshold
 
-    def _get_outliers(self) -> dict:
-        flagged_images = {}
-        stats_dict = self.stats.dict()
-        supported = to_distinct(ImageStat.ALL_STATS)
-        for stat, values in stats_dict.items():
-            if stat in supported.values() and values.ndim == 1 and np.std(values) != 0:
-                mask = _get_outlier_mask(values, self.outlier_method, self.outlier_threshold)
+    def _get_outliers(self, stats: dict) -> dict[int, dict[str, float]]:
+        flagged_images: dict[int, dict[str, float]] = {}
+        for stat, values in stats.items():
+            if stat in (SOURCE_INDEX, BOX_COUNT):
+                continue
+            if values.ndim == 1:
+                mask = _get_outlier_mask(values.astype(np.float64), self.outlier_method, self.outlier_threshold)
                 indices = np.flatnonzero(mask)
                 for i, value in zip(indices, values[mask]):
-                    flagged_images.setdefault(i, {}).update({stat: np.round(value, 2)})
+                    flagged_images.setdefault(i, {}).update({stat: value})
 
         return dict(sorted(flagged_images.items()))
 
-    @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
-    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput | Sequence[StatsOutput]) -> OutliersOutput:
+    @overload
+    def from_stats(self, stats: OutlierStatsOutput | DatasetStatsOutput) -> OutliersOutput[IndexIssueMap]: ...
+
+    @overload
+    def from_stats(self, stats: Sequence[OutlierStatsOutput]) -> OutliersOutput[list[IndexIssueMap]]: ...
+
+    @set_metadata("dataeval.detectors", ["outlier_method", "outlier_threshold"])
+    def from_stats(
+        self, stats: OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+    ) -> OutliersOutput:
         """
         Returns indices of outliers with the issues identified for each
 
         Parameters
         ----------
-        data : Iterable[ArrayLike], shape - (C, H, W) | StatsOutput | Sequence[StatsOutput]
-            A dataset of images in an ArrayLike format or the output(s) from an imagestats metric analysis
+        stats : OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+            The output(s) from a dimensionstats, pixelstats, or visualstats metric
+            analysis or an aggregate DatasetStatsOutput
 
         Returns
         -------
@@ -162,36 +174,96 @@ class Outliers:
             Output class containing the indices of outliers and a dictionary showing
             the issues and calculated values for the given index.
 
+        See Also
+        --------
+        dimensionstats
+        pixelstats
+        visualstats
+
         Example
         -------
         Evaluate the dataset:
 
-        >>> outliers.evaluate(images)
-        OutliersOutput(issues={10: {'blurriness': 1.26, 'contrast': 1.06, 'zeros': 0.05}, 12: {'blurriness': 1.51, 'contrast': 1.06, 'zeros': 0.05}})
+        >>> results = outliers.from_stats([stats1, stats2])
+        >>> len(results)
+        2
+        >>> results.issues[0]
+        {10: {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128}, 12: {'std': 0.00536, 'var': 2.87e-05, 'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128}}
+        >>> results.issues[1]
+        {}
         """  # noqa: E501
-        stats, dataset_steps = combine_stats(data)
-
-        if isinstance(stats, StatsOutput):
-            selected_flags = set(to_distinct(self.flags).values())
-            provided = set(stats.dict())
-            missing = selected_flags - provided
-            if missing:
-                warn(
-                    f"StatsOutput provided {provided} and is missing {missing} \
-                        from the selected stat flags: {selected_flags}."
+        if isinstance(stats, DatasetStatsOutput):
+            outliers = self._get_outliers({k: v for o in stats.outputs() for k, v in o.dict().items()})
+            return OutliersOutput(outliers)
+
+        if isinstance(stats, (DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)):
+            return OutliersOutput(self._get_outliers(stats.dict()))
+
+        if not isinstance(stats, Sequence):
+            raise TypeError(
+                "Invalid stats output type; only use output from dimensionstats, pixelstats or visualstats."
+            )
+
+        stats_map: dict[type, list[int]] = {}
+        for i, stats_output in enumerate(stats):
+            if not isinstance(
+                stats_output, (DatasetStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)
+            ):
+                raise TypeError(
+                    "Invalid stats output type; only use output from dimensionstats, pixelstats or visualstats."
                 )
-            self.stats = stats
-        else:
-            self.stats = imagestats(cast(Iterable[ArrayLike], data), self.flags)
-
-        outliers = self._get_outliers()
+            stats_map.setdefault(type(stats_output), []).append(i)
 
-        # split up results from combined dataset into individual dataset buckets
-        if dataset_steps:
-            out_dict = {}
+        output_list: list[dict[int, dict[str, float]]] = [{} for _ in stats]
+        for _, indices in stats_map.items():
+            substats, dataset_steps = combine_stats([stats[i] for i in indices])
+            outliers = self._get_outliers(substats.dict())
             for idx, issue in outliers.items():
                 k, v = get_dataset_step_from_idx(idx, dataset_steps)
-                out_dict.setdefault(k, {})[v] = issue
-            outliers = out_dict
+                output_list[indices[k]][v] = issue
+
+        return OutliersOutput(output_list)
+
+    @set_metadata(
+        "dataeval.detectors",
+        [
+            "use_dimension",
+            "use_pixel",
+            "use_visual",
+            "outlier_method",
+            "outlier_threshold",
+        ],
+    )
+    def evaluate(self, data: Iterable[ArrayLike]) -> OutliersOutput[IndexIssueMap]:
+        """
+        Returns indices of outliers with the issues identified for each
 
+        Parameters
+        ----------
+        data : Iterable[ArrayLike], shape - (C, H, W)
+            A dataset of images in an ArrayLike format
+
+        Returns
+        -------
+        OutliersOutput
+            Output class containing the indices of outliers and a dictionary showing
+            the issues and calculated values for the given index.
+
+        Example
+        -------
+        Evaluate the dataset:
+
+        >>> results = outliers.evaluate(images)
+        >>> list(results.issues)
+        [10, 12]
+        >>> results.issues[10]
+        {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128, 'contrast': 1.25, 'zeros': 0.05493}
+        """
+        self.stats = datasetstats(
+            images=data,
+            use_dimension=self.use_dimension,
+            use_pixel=self.use_pixel,
+            use_visual=self.use_visual,
+        )
+        outliers = self._get_outliers({k: v for o in self.stats.outputs() for k, v in o.dict().items()})
         return OutliersOutput(outliers)

dataeval/_internal/interop.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from importlib import import_module
-from typing import Iterable
+from typing import Any, Iterable, Iterator
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -22,24 +22,28 @@ def try_import(module_name):
     return module
 
 
-def to_numpy(array: ArrayLike | None) -> NDArray:
+def as_numpy(array: ArrayLike | None) -> NDArray[Any]:
+    return to_numpy(array, copy=False)
+
+
+def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
     if array is None:
         return np.ndarray([])
 
     if isinstance(array, np.ndarray):
-        return array
+        return array.copy() if copy else array
 
     tf = try_import("tensorflow")
     if tf and tf.is_tensor(array):
-        return array.numpy()  # type: ignore
+        return array.numpy().copy() if copy else array.numpy()  # type: ignore
 
     torch = try_import("torch")
     if torch and isinstance(array, torch.Tensor):
-        return array.detach().cpu().numpy()  # type: ignore
+        return array.detach().cpu().numpy().copy() if copy else array.detach().cpu().numpy()  # type: ignore
 
-    return np.asarray(array)
+    return np.array(array, copy=copy)
 
 
-def to_numpy_iter(iterable: Iterable[ArrayLike]):
+def to_numpy_iter(iterable: Iterable[ArrayLike]) -> Iterator[NDArray[Any]]:
     for array in iterable:
         yield to_numpy(array)

dataeval/_internal/metrics/balance.py

@@ -2,10 +2,10 @@ from __future__ import annotations
 
 import warnings
 from dataclasses import dataclass
-from typing import Sequence
+from typing import Mapping
 
 import numpy as np
-from numpy.typing import NDArray
+from numpy.typing import ArrayLike, NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
 from dataeval._internal.metrics.utils import entropy, preprocess_metadata
@@ -51,16 +51,16 @@ def validate_num_neighbors(num_neighbors: int) -> int:
 
 
 @set_metadata("dataeval.metrics")
-def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
+def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neighbors: int = 5) -> BalanceOutput:
     """
     Mutual information (MI) between factors (class label, metadata, label/image properties)
 
     Parameters
     ----------
-    class_labels: Sequence[int]
+    class_labels: ArrayLike
         List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
+    metadata: Mapping[str, ArrayLike]
+        Dict of lists of metadata factors for each image
     num_neighbors: int, default 5
         Number of nearest neighbors to use for computing MI between discrete
         and continuous variables.
@@ -90,9 +90,9 @@ def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: in
     Return intra/interfactor balance (mutual information)
 
     >>> bal.factors
-    array([[0.99999843, 0.03510422, 0.09725766],
-           [0.03510422, 0.08433558, 0.15621459],
-           [0.09725766, 0.15621459, 0.99999856]])
+    array([[0.99999843, 0.04133555, 0.09725766],
+           [0.04133555, 0.08433558, 0.1301489 ],
+           [0.09725766, 0.1301489 , 0.99999856]])
 
     Return classwise balance (mutual information) of factors with individual class_labels
 

dataeval/_internal/metrics/ber.py

@@ -17,7 +17,7 @@ from numpy.typing import ArrayLike, NDArray
 from scipy.sparse import coo_matrix
 from scipy.stats import mode
 
-from dataeval._internal.interop import to_numpy
+from dataeval._internal.interop import as_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
 from dataeval._internal.output import OutputMetadata, set_metadata
 
@@ -145,7 +145,7 @@ def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN",
     BEROutput(ber=0.04, ber_lower=0.020416847668728033)
     """
     ber_fn = get_method(BER_FN_MAP, method)
-    X = to_numpy(images)
-    y = to_numpy(labels)
+    X = as_numpy(images)
+    y = as_numpy(labels)
     upper, lower = ber_fn(X, y, k) if method == "KNN" else ber_fn(X, y)
     return BEROutput(upper, lower)

dataeval/_internal/metrics/divergence.py

@@ -9,7 +9,7 @@ from typing import Literal
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.interop import to_numpy
+from dataeval._internal.interop import as_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
 from dataeval._internal.output import OutputMetadata, set_metadata
 
@@ -123,8 +123,8 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     DivergenceOutput(divergence=0.28, errors=36.0)
     """
     div_fn = get_method(DIVERGENCE_FN_MAP, method)
-    a = to_numpy(data_a)
-    b = to_numpy(data_b)
+    a = as_numpy(data_a)
+    b = as_numpy(data_b)
     N = a.shape[0]
     M = b.shape[0]
 

dataeval/_internal/metrics/diversity.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Literal, Sequence
+from typing import Literal, Mapping
 
 import numpy as np
-from numpy.typing import NDArray
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
 from dataeval._internal.output import OutputMetadata, set_metadata
@@ -142,7 +142,7 @@ DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
 
 @set_metadata("dataeval.metrics")
 def diversity(
-    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity and classwise diversity for discrete/categorical variables and, through standard
@@ -155,10 +155,10 @@ def diversity(
 
     Parameters
     ----------
-    class_labels: Sequence[int]
+    class_labels: ArrayLike
         List of class labels for each image
-    metadata: List[Dict]
-        List of metadata factors for each image
+    metadata: Mapping[str, ArrayLike]
+        Dict of list of metadata factors for each image
     method: Literal["shannon", "simpson"], default "simpson"
         Indicates which diversity index should be computed
 

dataeval/_internal/metrics/parity.py

@@ -62,8 +62,8 @@ def digitize_factor_bins(continuous_values: NDArray, bins: int, factor_name: str
 
 
 def format_discretize_factors(
-    data_factors: dict[str, NDArray], continuous_factor_bincounts: dict[str, int]
-) -> tuple[dict[str, NDArray], NDArray]:
+    data_factors: Mapping[str, NDArray], continuous_factor_bincounts: Mapping[str, int]
+) -> dict[str, NDArray]:
     """
     Sets up the internal list of metadata factors.
 
@@ -80,10 +80,9 @@ def format_discretize_factors(
 
     Returns
     -------
-    Tuple[Dict[str, NDArray], NDArray]
+    Dict[str, NDArray]
         - Intrinsic per-image metadata information with the formatting that input data_factors uses.
           Each key is a metadata factor, whose value is the discrete per-image factor values.
-        - Per-image labels, whose ith element is the label for the ith element of the dataset.
     """
 
     invalid_keys = set(continuous_factor_bincounts.keys()) - set(data_factors.keys())
@@ -103,8 +102,6 @@ def format_discretize_factors(
     if lengths[1:] != lengths[:-1]:
         raise ValueError("The lengths of each entry in the dictionary are not equal." f" Found lengths {lengths}")
 
-    labels = data_factors["class"]
-
     metadata_factors = {
         name: val
         if name not in continuous_factor_bincounts
@@ -113,7 +110,7 @@ def format_discretize_factors(
         if name != "class"
     }
 
-    return metadata_factors, labels
+    return metadata_factors
 
 
 def normalize_expected_dist(expected_dist: NDArray, observed_dist: NDArray) -> NDArray:
@@ -187,7 +184,8 @@ def validate_dist(label_dist: NDArray, label_name: str):
         warnings.warn(
             f"Labels {np.where(label_dist<5)[0]} in {label_name}"
             " dataset have frequencies less than 5. This may lead"
-            " to invalid chi-squared evaluation."
+            " to invalid chi-squared evaluation.",
+            UserWarning,
         )
 
 
@@ -280,8 +278,9 @@ def label_parity(
 
 @set_metadata("dataeval.metrics")
 def parity(
+    class_labels: ArrayLike,
     data_factors: Mapping[str, ArrayLike],
-    continuous_factor_bincounts: dict[str, int] | None = None,
+    continuous_factor_bincounts: Mapping[str, int] | None = None,
 ) -> ParityOutput[NDArray[np.float64]]:
     """
     Calculate chi-square statistics to assess the relationship between multiple factors and class labels.
@@ -292,10 +291,12 @@ def parity(
 
     Parameters
     ----------
+    class_labels: ArrayLike
+        List of class labels for each image
     data_factors: Mapping[str, ArrayLike]
-        The dataset factors, which are per-image attributes including class label and metadata.
+        The dataset factors, which are per-image metadata attributes.
         Each key of dataset_factors is a factor, whose value is the per-image factor values.
-    continuous_factor_bincounts : Dict[str, int] | None, default None
+    continuous_factor_bincounts : Mapping[str, int] | None, default None
         A dictionary specifying the number of bins for discretizing the continuous factors.
         The keys should correspond to the names of continuous factors in `data_factors`,
         and the values should be the number of bins to use for discretization.
@@ -329,21 +330,27 @@ def parity(
     --------
     Randomly creating some "continuous" and categorical variables using ``np.random.default_rng``
 
+    >>> labels = np_random_gen.choice([0, 1, 2], (100))
     >>> data_factors = {
     ...     "age": np_random_gen.choice([25, 30, 35, 45], (100)),
     ...     "income": np_random_gen.choice([50000, 65000, 80000], (100)),
     ...     "gender": np_random_gen.choice(["M", "F"], (100)),
-    ...     "class": np_random_gen.choice([0, 1, 2], (100)),
     ... }
     >>> continuous_factor_bincounts = {"age": 4, "income": 3}
-    >>> parity(data_factors, continuous_factor_bincounts)
-    ParityOutput(score=array([2.82329785, 1.60625584, 1.38377236]), p_value=array([0.83067563, 0.80766733, 0.5006309 ]))
+    >>> parity(labels, data_factors, continuous_factor_bincounts)
+    ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]))
     """
+    if len(np.shape(class_labels)) > 1:
+        raise ValueError(
+            f"Got class labels with {len(np.shape(class_labels))}-dimensional",
+            f" shape {np.shape(class_labels)}, but expected a 1-dimensional array.",
+        )
 
     data_factors_np = {k: to_numpy(v) for k, v in data_factors.items()}
     continuous_factor_bincounts = continuous_factor_bincounts if continuous_factor_bincounts else {}
 
-    factors, labels = format_discretize_factors(data_factors_np, continuous_factor_bincounts)
+    labels = to_numpy(class_labels)
+    factors = format_discretize_factors(data_factors_np, continuous_factor_bincounts)
 
     chi_scores = np.zeros(len(factors))
     p_values = np.zeros(len(factors))
@@ -396,7 +403,8 @@ def parity(
         message = "\n".join(factor_msg)
 
         warnings.warn(
-            f"The following factors did not meet the recommended 5 occurrences for each value-label combination. \nRecommend rerunning parity after adjusting the following factor-value-label combinations: \n{message}",  # noqa: E501
+            f"The following factors did not meet the recommended 5 occurrences for each value-label combination. \n\
+            Recommend rerunning parity after adjusting the following factor-value-label combinations: \n{message}",
             UserWarning,
         )