dataeval 0.81.0__py3-none-any.whl → 0.82.1__py3-none-any.whl

dataeval/detectors/linters/outliers.py

@@ -2,142 +2,19 @@ from __future__ import annotations
 
 __all__ = []
 
-import contextlib
-from dataclasses import dataclass
-from typing import Any, Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
+from typing import Any, Literal, Sequence, overload
 
 import numpy as np
 from numpy.typing import NDArray
-from torch.utils.data import Dataset
-
-from dataeval._output import Output, set_metadata
-from dataeval.metrics.stats._base import BOX_COUNT, SOURCE_INDEX, combine_stats, get_dataset_step_from_idx
-from dataeval.metrics.stats._datasetstats import DatasetStatsOutput, datasetstats
-from dataeval.metrics.stats._dimensionstats import DimensionStatsOutput
-from dataeval.metrics.stats._labelstats import LabelStatsOutput
-from dataeval.metrics.stats._pixelstats import PixelStatsOutput
-from dataeval.metrics.stats._visualstats import VisualStatsOutput
-from dataeval.typing import ArrayLike
-
-with contextlib.suppress(ImportError):
-    import pandas as pd
-
-
-IndexIssueMap = dict[int, dict[str, float]]
-OutlierStatsOutput = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
-TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, list[IndexIssueMap])
-
-
-def _reorganize_by_class_and_metric(result, lstats):
-    """Flip result from grouping by image to grouping by class and metric"""
-    metrics = {}
-    class_wise = {label: {} for label in lstats.image_indices_per_label}
-
-    # Group metrics and calculate class-wise counts
-    for img, group in result.items():
-        for extreme in group:
-            metrics.setdefault(extreme, []).append(img)
-            for label, images in lstats.image_indices_per_label.items():
-                if img in images:
-                    class_wise[label][extreme] = class_wise[label].get(extreme, 0) + 1
-
-    return metrics, class_wise
-
-
-def _create_table(metrics, class_wise):
-    """Create table for displaying the results"""
-    max_class_length = max(len(str(label)) for label in class_wise) + 2
-    max_total = max(len(metrics[group]) for group in metrics) + 2
-
-    table_header = " | ".join(
-        [f"{'Class':>{max_class_length}}"]
-        + [f"{group:^{max(5, len(str(group))) + 2}}" for group in sorted(metrics.keys())]
-        + [f"{'Total':<{max_total}}"]
-    )
-    table_rows = []
-
-    for class_cat, results in class_wise.items():
-        table_value = [f"{class_cat:>{max_class_length}}"]
-        total = 0
-        for group in sorted(metrics.keys()):
-            count = results.get(group, 0)
-            table_value.append(f"{count:^{max(5, len(str(group))) + 2}}")
-            total += count
-        table_value.append(f"{total:^{max_total}}")
-        table_rows.append(" | ".join(table_value))
-
-    table = [table_header] + table_rows
-    return table
-
-
-def _create_pandas_dataframe(class_wise):
-    """Create data for pandas dataframe"""
-    data = []
-    for label, metrics_dict in class_wise.items():
-        row = {"Class": label}
-        total = sum(metrics_dict.values())
-        row.update(metrics_dict)  # Add metric counts
-        row["Total"] = total
-        data.append(row)
-    return data
-
-
-@dataclass(frozen=True)
-class OutliersOutput(Generic[TIndexIssueMap], Output):
-    """
-    Output class for :class:`.Outliers` lint detector.
-
-    Attributes
-    ----------
-    issues : dict[int, dict[str, float]] | list[dict[int, dict[str, float]]]
-        Indices of image Outliers with their associated issue type and calculated values.
-
-    - For a single dataset, a dictionary containing the indices of outliers and
-      a dictionary showing the issues and calculated values for the given index.
-    - For multiple stats outputs, a list of dictionaries containing the indices of
-      outliers and their associated issues and calculated values.
-    """
 
-    issues: TIndexIssueMap
-
-    def __len__(self) -> int:
-        if isinstance(self.issues, dict):
-            return len(self.issues)
-        else:
-            return sum(len(d) for d in self.issues)
-
-    def to_table(self, labelstats: LabelStatsOutput) -> str:
-        if isinstance(self.issues, dict):
-            metrics, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
-            listed_table = _create_table(metrics, classwise)
-            table = "\n".join(listed_table)
-        else:
-            outertable = []
-            for d in self.issues:
-                metrics, classwise = _reorganize_by_class_and_metric(d, labelstats)
-                listed_table = _create_table(metrics, classwise)
-                str_table = "\n".join(listed_table)
-                outertable.append(str_table)
-            table = "\n\n".join(outertable)
-        return table
-
-    def to_dataframe(self, labelstats: LabelStatsOutput) -> pd.DataFrame:
-        import pandas as pd
-
-        if isinstance(self.issues, dict):
-            _, classwise = _reorganize_by_class_and_metric(self.issues, labelstats)
-            data = _create_pandas_dataframe(classwise)
-            df = pd.DataFrame(data)
-        else:
-            df_list = []
-            for i, d in enumerate(self.issues):
-                _, classwise = _reorganize_by_class_and_metric(d, labelstats)
-                data = _create_pandas_dataframe(classwise)
-                single_df = pd.DataFrame(data)
-                single_df["Dataset"] = i
-                df_list.append(single_df)
-            df = pd.concat(df_list)
-        return df
+from dataeval.metrics.stats._base import combine_stats, get_dataset_step_from_idx
+from dataeval.metrics.stats._imagestats import imagestats
+from dataeval.outputs import DimensionStatsOutput, ImageStatsOutput, OutliersOutput, PixelStatsOutput, VisualStatsOutput
+from dataeval.outputs._base import set_metadata
+from dataeval.outputs._linters import IndexIssueMap, OutlierStatsOutput
+from dataeval.outputs._stats import BOX_COUNT, SOURCE_INDEX
+from dataeval.typing import Array, Dataset
+from dataeval.utils.data._images import Images
 
 
 def _get_outlier_mask(
@@ -227,7 +104,7 @@ class Outliers:
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: float | None = None,
     ):
-        self.stats: DatasetStatsOutput
+        self.stats: ImageStatsOutput
         self.use_dimension = use_dimension
         self.use_pixel = use_pixel
         self.use_visual = use_visual
@@ -248,23 +125,23 @@ class Outliers:
         return dict(sorted(flagged_images.items()))
 
     @overload
-    def from_stats(self, stats: OutlierStatsOutput | DatasetStatsOutput) -> OutliersOutput[IndexIssueMap]: ...
+    def from_stats(self, stats: OutlierStatsOutput | ImageStatsOutput) -> OutliersOutput[IndexIssueMap]: ...
 
     @overload
     def from_stats(self, stats: Sequence[OutlierStatsOutput]) -> OutliersOutput[list[IndexIssueMap]]: ...
 
     @set_metadata(state=["outlier_method", "outlier_threshold"])
     def from_stats(
-        self, stats: OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+        self, stats: OutlierStatsOutput | ImageStatsOutput | Sequence[OutlierStatsOutput]
     ) -> OutliersOutput[IndexIssueMap] | OutliersOutput[list[IndexIssueMap]]:
         """
         Returns indices of Outliers with the issues identified for each.
 
         Parameters
         ----------
-        stats : OutlierStatsOutput | DatasetStatsOutput | Sequence[OutlierStatsOutput]
+        stats : OutlierStatsOutput | ImageStatsOutput | Sequence[OutlierStatsOutput]
             The output(s) from a dimensionstats, pixelstats, or visualstats metric
-            analysis or an aggregate DatasetStatsOutput
+            analysis or an aggregate ImageStatsOutput
 
         Returns
         -------
@@ -291,12 +168,8 @@ class Outliers:
         >>> results.issues[1]
         {}
         """  # noqa: E501
-        if isinstance(stats, DatasetStatsOutput):
-            outliers = self._get_outliers({k: v for o in stats._outputs() for k, v in o.dict().items()})
-            return OutliersOutput(outliers)
-
-        if isinstance(stats, (DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)):
-            return OutliersOutput(self._get_outliers(stats.dict()))
+        if isinstance(stats, (ImageStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)):
+            return OutliersOutput(self._get_outliers(stats.data()))
 
         if not isinstance(stats, Sequence):
             raise TypeError(
@@ -306,7 +179,7 @@ class Outliers:
         stats_map: dict[type, list[int]] = {}
         for i, stats_output in enumerate(stats):
             if not isinstance(
-                stats_output, (DatasetStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)
+                stats_output, (ImageStatsOutput, DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput)
             ):
                 raise TypeError(
                     "Invalid stats output type; only use output from dimensionstats, pixelstats or visualstats."
@@ -316,29 +189,22 @@ class Outliers:
         output_list: list[dict[int, dict[str, float]]] = [{} for _ in stats]
         for _, indices in stats_map.items():
             substats, dataset_steps = combine_stats([stats[i] for i in indices])
-            outliers = self._get_outliers(substats.dict())
+            outliers = self._get_outliers(substats.data())
             for idx, issue in outliers.items():
                 k, v = get_dataset_step_from_idx(idx, dataset_steps)
                 output_list[indices[k]][v] = issue
 
         return OutliersOutput(output_list)
 
-    @overload
-    def evaluate(self, data: Iterable[ArrayLike]) -> OutliersOutput[IndexIssueMap]: ...
-    @overload
-    def evaluate(self, data: Dataset[tuple[ArrayLike, Any, dict[str, Any]]]) -> OutliersOutput[IndexIssueMap]: ...
-
     @set_metadata(state=["use_dimension", "use_pixel", "use_visual", "outlier_method", "outlier_threshold"])
-    def evaluate(
-        self, data: Iterable[ArrayLike] | Dataset[tuple[ArrayLike, Any, dict[str, Any]]]
-    ) -> OutliersOutput[IndexIssueMap]:
+    def evaluate(self, data: Dataset[Array] | Dataset[tuple[Array, Any, Any]]) -> OutliersOutput[IndexIssueMap]:
         """
         Returns indices of Outliers with the issues identified for each
 
         Parameters
         ----------
-        data : Iterable[ArrayLike], shape - (C, H, W)
-            A dataset of images in an ArrayLike format
+        data : Iterable[Array], shape - (C, H, W)
+            A dataset of images in an Array format
 
         Returns
         -------
@@ -355,9 +221,9 @@ class Outliers:
         >>> list(results.issues)
         [10, 12]
         >>> results.issues[10]
-        {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128, 'contrast': 1.25, 'zeros': 0.05493}
+        {'contrast': 1.25, 'zeros': 0.05493, 'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128}
         """
-        images = (d[0] for d in data) if isinstance(data, Dataset) else data
-        self.stats = datasetstats(images=images)
-        outliers = self._get_outliers(self.stats.dict())
+        images = Images(data) if isinstance(data, Dataset) else data
+        self.stats = imagestats(images)
+        outliers = self._get_outliers(self.stats.data())
         return OutliersOutput(outliers)

dataeval/detectors/ood/__init__.py

@@ -5,4 +5,4 @@ Out-of-distribution (OOD) detectors identify data that is different from the dat
 __all__ = ["OODOutput", "OODScoreOutput", "OOD_AE"]
 
 from dataeval.detectors.ood.ae import OOD_AE
-from dataeval.detectors.ood.output import OODOutput, OODScoreOutput
+from dataeval.outputs._ood import OODOutput, OODScoreOutput

dataeval/detectors/ood/ae.py

@@ -18,8 +18,9 @@ import numpy as np
 import torch
 from numpy.typing import NDArray
 
+from dataeval.config import DeviceLike
 from dataeval.detectors.ood.base import OODBase
-from dataeval.detectors.ood.output import OODScoreOutput
+from dataeval.outputs import OODScoreOutput
 from dataeval.typing import ArrayLike
 from dataeval.utils.torch._internal import predict_batch
 
@@ -33,9 +34,9 @@ class OOD_AE(OODBase):
     model : torch.nn.Module
         An autoencoder model to use for encoding and reconstruction of images
         for detection of out-of-distribution samples.
-    device : str or torch.Device or None, default None
-        The device to use for the detector. None will default to the global
-        configuration selection if set, otherwise "cuda" then "cpu" by availability.
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
 
     Example
     -------
@@ -57,7 +58,7 @@ class OOD_AE(OODBase):
     array([ True,  True, False,  True,  True,  True,  True,  True])
     """
 
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         super().__init__(model, device)
 
     def fit(

dataeval/detectors/ood/base.py

@@ -14,7 +14,7 @@ from typing import Callable, cast
 
 import torch
 
-from dataeval.config import get_device
+from dataeval.config import DeviceLike, get_device
 from dataeval.detectors.ood.mixin import OODBaseMixin, OODFitMixin, OODGMMMixin
 from dataeval.typing import ArrayLike
 from dataeval.utils._array import to_numpy
@@ -23,7 +23,7 @@ from dataeval.utils.torch._internal import trainer
 
 
 class OODBase(OODBaseMixin[torch.nn.Module], OODFitMixin[Callable[..., torch.nn.Module], torch.optim.Optimizer]):
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         self.device: torch.device = get_device(device)
         super().__init__(model)
 

dataeval/detectors/ood/metadata_ood_mi.py

@@ -10,6 +10,8 @@ import numpy as np
 from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif
 
+from dataeval.config import get_seed
+
 # NATS2BITS is the reciprocal of natural log of 2. If you have an information/entropy-type quantity measured in nats,
 #   which is what many library functions return, multiply it by NATS2BITS to get it in bits.
 NATS2BITS = 1.442695
@@ -19,7 +21,6 @@ def get_metadata_ood_mi(
     metadata: dict[str, list[Any] | NDArray[Any]],
     is_ood: NDArray[np.bool_],
     discrete_features: str | bool | NDArray[np.bool_] = False,
-    random_state: int | None = None,
 ) -> dict[str, float]:
     """Computes mutual information between a set of metadata features and an out-of-distribution flag.
 
@@ -39,9 +40,6 @@ def get_metadata_ood_mi(
         A boolean array, with one value per example, that indicates which examples are OOD.
     discrete_features : str | bool | NDArray[np.bool_]
         Either a boolean array or a single boolean value, indicate which features take on discrete values.
-    random_state : int, optional - default None
-        Determines random number generation for small noise added to continuous variables. Set to a value for
-        reproducible results.
 
     Returns
     -------
@@ -55,7 +53,7 @@ def get_metadata_ood_mi(
 
     >>> metadata = {"time": np.linspace(0, 10, 100), "altitude": np.linspace(0, 16, 100) ** 2}
     >>> is_ood = metadata["altitude"] > 100
-    >>> get_metadata_ood_mi(metadata, is_ood, discrete_features=False, random_state=0)
+    >>> get_metadata_ood_mi(metadata, is_ood, discrete_features=False)
     {'time': 0.9359596758173668, 'altitude': 0.9407686591507002}
     """
     numerical_keys = [k for k, v in metadata.items() if all(isinstance(vi, numbers.Number) for vi in v)]
@@ -84,7 +82,7 @@ def get_metadata_ood_mi(
             Xscl,
             is_ood,
             discrete_features=discrete_features,  # type: ignore
-            random_state=random_state,
+            random_state=get_seed(),
         )
         * NATS2BITS
     )

dataeval/detectors/ood/mixin.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from dataeval.detectors.ood.output import OODOutput, OODScoreOutput
-
 __all__ = []
 
 from abc import ABC, abstractmethod
@@ -10,7 +8,8 @@ from typing import Callable, Generic, Literal, TypeVar
 import numpy as np
 from numpy.typing import NDArray
 
-from dataeval._output import set_metadata
+from dataeval.outputs import OODOutput, OODScoreOutput
+from dataeval.outputs._base import set_metadata
 from dataeval.typing import ArrayLike
 from dataeval.utils._array import as_numpy, to_numpy
 
@@ -158,4 +157,4 @@ class OODBaseMixin(Generic[TModel], ABC):
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
         ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
-        return OODOutput(is_ood=ood_pred, **score.dict())
+        return OODOutput(is_ood=ood_pred, **score.data())

dataeval/detectors/ood/vae.py

@@ -17,8 +17,9 @@ from typing import Callable
 import numpy as np
 import torch
 
+from dataeval.config import DeviceLike
 from dataeval.detectors.ood.base import OODBase
-from dataeval.detectors.ood.output import OODScoreOutput
+from dataeval.outputs import OODScoreOutput
 from dataeval.typing import ArrayLike
 from dataeval.utils._array import as_numpy
 from dataeval.utils.torch._internal import predict_batch
@@ -34,7 +35,7 @@ class OOD_VAE(OODBase):
         An Autoencoder model.
     """
 
-    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+    def __init__(self, model: torch.nn.Module, device: DeviceLike | None = None) -> None:
         super().__init__(model, device)
 
     def fit(

dataeval/metadata/__init__.py

@@ -1,5 +1,6 @@
 """Explanatory functions using metadata and additional features such as ood or drift"""
 
-__all__ = ["most_deviated_factors"]
+__all__ = ["most_deviated_factors", "metadata_distance"]
 
+from dataeval.metadata._distance import metadata_distance
 from dataeval.metadata._ood import most_deviated_factors

dataeval/metadata/_distance.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+__all__ = []
+
+import warnings
+from typing import NamedTuple, cast
+
+import numpy as np
+from scipy.stats import iqr, ks_2samp
+from scipy.stats import wasserstein_distance as emd
+
+from dataeval.metadata._utils import _compare_keys, _validate_factors_and_data
+from dataeval.outputs import MetadataDistanceOutput, MetadataDistanceValues
+from dataeval.outputs._base import set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils.data import Metadata
+
+
+class KSType(NamedTuple):
+    """Used to typehint scipy's internal hidden ks_2samp output"""
+
+    statistic: float
+    statistic_location: float
+    pvalue: float
+
+
+def _calculate_drift(x1: ArrayLike, x2: ArrayLike) -> float:
+    """Calculates the shift magnitude between x1 and x2 scaled by x1"""
+
+    distance = emd(x1, x2)
+
+    X = iqr(x1)
+
+    # Preferred scaling of x1
+    if X:
+        return distance / X
+
+    # Return if single-valued, else scale
+    xmin, xmax = np.min(x1), np.max(x1)
+    return distance if xmin == xmax else distance / (xmax - xmin)
+
+
+@set_metadata
+def metadata_distance(metadata1: Metadata, metadata2: Metadata) -> MetadataDistanceOutput:
+    """
+    Measures the feature-wise distance between two continuous metadata distributions and
+    computes a p-value to evaluate its significance.
+
+    Uses the Earth Mover's Distance and the Kolmogorov-Smirnov two-sample test, featurewise.
+
+    Parameters
+    ----------
+    metadata1 : Metadata
+        Class containing continuous factor names and values to be used as reference
+    metadata2 : Metadata
+        Class containing continuous factor names and values to be compare with the reference
+
+    Returns
+    -------
+    MetadataDistanceOutput
+        A mapping with keys corresponding to metadata feature names, and values that are KstestResult objects, as
+        defined by scipy.stats.ks_2samp.
+
+    See Also
+    --------
+    Earth mover's distance
+
+    Kolmogorov-Smirnov two-sample test
+
+    Note
+    ----
+    This function only applies to the continuous data
+
+    Examples
+    --------
+    >>> output = metadata_distance(metadata1, metadata2)
+    >>> list(output)
+    ['time', 'altitude']
+    >>> output["time"]
+    MetadataDistanceValues(statistic=1.0, location=0.44354838709677413, dist=2.7, pvalue=0.0)
+    """
+
+    _compare_keys(metadata1.continuous_factor_names, metadata2.continuous_factor_names)
+    fnames = metadata1.continuous_factor_names
+
+    cont1 = np.atleast_2d(metadata1.continuous_data)  # (S, F)
+    cont2 = np.atleast_2d(metadata2.continuous_data)  # (S, F)
+
+    _validate_factors_and_data(fnames, cont1)
+    _validate_factors_and_data(fnames, cont2)
+
+    N = len(cont1)
+    M = len(cont2)
+
+    # This is a simplified version of sqrt(N*M / N+M) < 4
+    if (N - 16) * (M - 16) < 256:
+        warnings.warn(
+            f"Sample sizes of {N}, {M} will yield unreliable p-values from the KS test. "
+            f"Recommended 32 samples per factor or at least 16 if one set has many more.",
+            UserWarning,
+        )
+
+    # Set default for statistic, location, and magnitude to zero and pvalue to one
+    results: dict[str, MetadataDistanceValues] = {}
+
+    # Per factor
+    for i, fname in enumerate(fnames):
+        fdata1 = cont1[:, i]  # (S, 1)
+        fdata2 = cont2[:, i]  # (S, 1)
+
+        # Min and max over both distributions
+        xmin = min(np.min(fdata1), np.min(fdata2))
+        xmax = max(np.max(fdata1), np.max(fdata2))
+
+        # Default case
+        if xmin == xmax:
+            results[fname] = MetadataDistanceValues(statistic=0.0, location=0.0, dist=0.0, pvalue=1.0)
+            continue
+
+        ks_result = cast(KSType, ks_2samp(fdata1, fdata2, method="asymp"))
+
+        # Normalized location
+        loc = float((ks_result.statistic_location - xmin) / (xmax - xmin))
+
+        drift = _calculate_drift(fdata1, fdata2)
+
+        results[fname] = MetadataDistanceValues(
+            statistic=ks_result.statistic,
+            location=loc,
+            dist=drift,
+            pvalue=ks_result.pvalue,
+        )
+
+    return MetadataDistanceOutput(results)