dataeval 0.81.0__py3-none-any.whl → 0.82.1__py3-none-any.whl

dataeval/metadata/_ood.py

@@ -7,51 +7,12 @@ import warnings
 import numpy as np
 from numpy.typing import NDArray
 
-from dataeval.detectors.ood import OODOutput
+from dataeval.metadata._utils import _compare_keys, _validate_factors_and_data
+from dataeval.outputs import MostDeviatedFactorsOutput, OODOutput
+from dataeval.outputs._base import set_metadata
 from dataeval.utils.data import Metadata
 
 
-def _validate_keys(keys1: list[str], keys2: list[str]) -> None:
-    """
-    Raises error when two lists are not equivalent including ordering
-
-    Parameters
-    ----------
-    keys1 : list of strings
-        List of strings to compare
-    keys2 : list of strings
-        List of strings to compare
-
-    Raises
-    ------
-    ValueError
-        If lists do not have the same values, value counts, or ordering
-    """
-
-    if keys1 != keys2:
-        raise ValueError(f"Metadata keys must be identical, got {keys1} and {keys2}")
-
-
-def _validate_factors_and_data(factors: list[str], data: NDArray) -> None:
-    """
-    Raises error when the number of factors and number of rows do not match
-
-    Parameters
-    ----------
-    factors : list of strings
-        List of factor names of size N
-    data : NDArray
-        Array of values with shape (M, N)
-
-    Raises
-    ------
-    ValueError
-        If the length of factors does not equal the length of the transposed data
-    """
-    if len(factors) != len(data.T):
-        raise ValueError(f"Factors and data have mismatched lengths. Got {len(factors)} and {len(data.T)}")
-
-
 def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[str], list[NDArray], list[NDArray]]:
     """
     Combines the factor names and data arrays of metadata_1 and metadata_2 when the names
@@ -93,7 +54,7 @@ def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[
 
     # Validate and attach discrete data
     if metadata_1.discrete_factor_names:
-        _validate_keys(metadata_1.discrete_factor_names, metadata_2.discrete_factor_names)
+        _compare_keys(metadata_1.discrete_factor_names, metadata_2.discrete_factor_names)
         _validate_factors_and_data(metadata_1.discrete_factor_names, metadata_1.discrete_data)
 
         factor_names.extend(metadata_1.discrete_factor_names)
@@ -102,7 +63,7 @@ def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[
 
     # Validate and attach continuous data
     if metadata_1.continuous_factor_names:
-        _validate_keys(metadata_1.continuous_factor_names, metadata_2.continuous_factor_names)
+        _compare_keys(metadata_1.continuous_factor_names, metadata_2.continuous_factor_names)
         _validate_factors_and_data(metadata_1.continuous_factor_names, metadata_1.continuous_data)
 
         factor_names.extend(metadata_1.continuous_factor_names)
@@ -159,11 +120,12 @@ def _calc_median_deviations(reference: NDArray, test: NDArray) -> NDArray:
     return np.abs(np.where(test_dev >= 0, test_dev / pscale, test_dev / nscale))  # (S_t, F)
 
 
+@set_metadata
 def most_deviated_factors(
     metadata_1: Metadata,
     metadata_2: Metadata,
     ood: OODOutput,
-) -> list[tuple[str, float]]:
+) -> MostDeviatedFactorsOutput:
     """
     Determines greatest deviation in metadata features per out of distribution sample in metadata_2.
 
@@ -189,13 +151,30 @@ def most_deviated_factors(
        and have equivalent factor names and lengths
     2. The flag at index `i` in :attr:`.OODOutput.is_ood` must correspond
        directly to sample `i` of `metadata_2` being out-of-distribution from `metadata_1`
+
+    Examples
+    --------
+
+    >>> from dataeval.detectors.ood import OODOutput
+
+    All samples are out-of-distribution
+
+    >>> is_ood = OODOutput(np.array([True, True, True]), np.array([]), np.array([]))
+    >>> most_deviated_factors(metadata1, metadata2, is_ood)
+    MostDeviatedFactorsOutput([('time', 2.0), ('time', 2.592), ('time', 3.51)])
+
+    If there are no out-of-distribution samples, a list is returned
+
+    >>> is_ood = OODOutput(np.array([False, False, False]), np.array([]), np.array([]))
+    >>> most_deviated_factors(metadata1, metadata2, is_ood)
+    MostDeviatedFactorsOutput([])
     """
 
     ood_mask: NDArray[np.bool] = ood.is_ood
 
     # No metadata correlated with out of distribution data
     if not any(ood_mask):
-        return []
+        return MostDeviatedFactorsOutput([])
 
     # Combines reference and test factor names and data if exists and match exactly
     # shape -> (samples, factors)
@@ -204,6 +183,7 @@ def most_deviated_factors(
         metadata_2=metadata_2,
     )
 
+    # Stack discrete and continuous factors as separate factors. Must have equal sample counts
     metadata_ref = np.hstack(md_1) if md_1 else np.array([])
     metadata_tst = np.hstack(md_2) if md_2 else np.array([])
 
@@ -212,7 +192,7 @@ def most_deviated_factors(
             f"At least 3 reference metadata samples are needed, got {len(metadata_ref)}",
             UserWarning,
         )
-        return []
+        return MostDeviatedFactorsOutput([])
 
     if len(metadata_tst) != len(ood_mask):
         raise ValueError(
@@ -226,7 +206,7 @@ def most_deviated_factors(
     deviations = _calc_median_deviations(metadata_ref, metadata_tst)
 
     # Get most impactful factor deviation of each sample for ood samples only
-    deviation = np.max(deviations, axis=1)[ood_mask]
+    deviation = np.max(deviations, axis=1)[ood_mask].astype(np.float16)
 
     # Get indices of most impactful factors for ood samples only
     max_factors = np.argmax(deviations, axis=1)[ood_mask]
@@ -235,4 +215,5 @@ def most_deviated_factors(
     most_ood_factors = np.array(factor_names)[max_factors].tolist()
 
     # List of tuples matching the factor name with its deviation
-    return [(factor, dev.item()) for factor, dev in zip(most_ood_factors, deviation)]
+
+    return MostDeviatedFactorsOutput([(factor, dev) for factor, dev in zip(most_ood_factors, deviation)])

dataeval/metadata/_utils.py

@@ -0,0 +1,44 @@
+__all__ = []
+
+from numpy.typing import NDArray
+
+
+def _compare_keys(keys1: list[str], keys2: list[str]) -> None:
+    """
+    Raises error when two lists are not equivalent including ordering
+
+    Parameters
+    ----------
+    keys1 : list of strings
+        List of strings to compare
+    keys2 : list of strings
+        List of strings to compare
+
+    Raises
+    ------
+    ValueError
+        If lists do not have the same values, value counts, or ordering
+    """
+
+    if keys1 != keys2:
+        raise ValueError(f"Metadata keys must be identical, got {keys1} and {keys2}")
+
+
+def _validate_factors_and_data(factors: list[str], data: NDArray) -> None:
+    """
+    Raises error when the number of factors and number of rows do not match
+
+    Parameters
+    ----------
+    factors : list of strings
+        List of factor names of size N
+    data : NDArray
+        Array of values with shape (M, N)
+
+    Raises
+    ------
+    ValueError
+        If the length of factors does not equal the length of the transposed data
+    """
+    if len(factors) != len(data.T):
+        raise ValueError(f"Factors and data have mismatched lengths. Got {len(factors)} and {len(data.T)}")

dataeval/metrics/bias/__init__.py

@@ -16,7 +16,8 @@ __all__ = [
     "parity",
 ]
 
-from dataeval.metrics.bias._balance import BalanceOutput, balance
-from dataeval.metrics.bias._coverage import CoverageOutput, coverage
-from dataeval.metrics.bias._diversity import DiversityOutput, diversity
-from dataeval.metrics.bias._parity import LabelParityOutput, ParityOutput, label_parity, parity
+from dataeval.metrics.bias._balance import balance
+from dataeval.metrics.bias._coverage import coverage
+from dataeval.metrics.bias._diversity import diversity
+from dataeval.metrics.bias._parity import label_parity, parity
+from dataeval.outputs._bias import BalanceOutput, CoverageOutput, DiversityOutput, LabelParityOutput, ParityOutput

dataeval/metrics/bias/_balance.py

@@ -2,150 +2,18 @@ from __future__ import annotations
 
 __all__ = []
 
-import contextlib
 import warnings
-from dataclasses import dataclass
-from typing import Any, Literal, overload
 
 import numpy as np
 import scipy as sp
-from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
-from dataeval._output import Output, set_metadata
+from dataeval.config import get_seed
+from dataeval.outputs import BalanceOutput
+from dataeval.outputs._base import set_metadata
 from dataeval.utils._bin import get_counts
-from dataeval.utils._plot import heatmap
 from dataeval.utils.data import Metadata
 
-with contextlib.suppress(ImportError):
-    from matplotlib.figure import Figure
-
-
-@dataclass(frozen=True)
-class BalanceOutput(Output):
-    """
-    Output class for :func:`.balance` :term:`bias<Bias>` metric.
-
-    Attributes
-    ----------
-    balance : NDArray[np.float64]
-        Estimate of mutual information between metadata factors and class label
-    factors : NDArray[np.float64]
-        Estimate of inter/intra-factor mutual information
-    classwise : NDArray[np.float64]
-        Estimate of mutual information between metadata factors and individual class labels
-    factor_names : list[str]
-        Names of each metadata factor
-    class_names : list[str]
-        List of the class labels present in the dataset
-    """
-
-    balance: NDArray[np.float64]
-    factors: NDArray[np.float64]
-    classwise: NDArray[np.float64]
-    factor_names: list[str]
-    class_names: list[str]
-
-    @overload
-    def _by_factor_type(
-        self,
-        attr: Literal["factor_names"],
-        factor_type: Literal["discrete", "continuous", "both"],
-    ) -> list[str]: ...
-
-    @overload
-    def _by_factor_type(
-        self,
-        attr: Literal["balance", "factors", "classwise"],
-        factor_type: Literal["discrete", "continuous", "both"],
-    ) -> NDArray[np.float64]: ...
-
-    def _by_factor_type(
-        self,
-        attr: Literal["balance", "factors", "classwise", "factor_names"],
-        factor_type: Literal["discrete", "continuous", "both"],
-    ) -> NDArray[np.float64] | list[str]:
-        # if not filtering by factor_type then just return the requested attribute without mask
-        if factor_type == "both":
-            return getattr(self, attr)
-
-        # create the mask for the selected factor_type
-        mask_lambda = (
-            (lambda x: "-continuous" not in x) if factor_type == "discrete" else (lambda x: "-discrete" not in x)
-        )
-
-        # return the masked attribute
-        if attr == "factor_names":
-            return [x.replace(f"-{factor_type}", "") for x in self.factor_names if mask_lambda(x)]
-        else:
-            factor_type_mask = [mask_lambda(x) for x in self.factor_names]
-            if attr == "factors":
-                return self.factors[factor_type_mask[1:]][:, factor_type_mask[1:]]
-            elif attr == "balance":
-                return self.balance[factor_type_mask]
-            elif attr == "classwise":
-                return self.classwise[:, factor_type_mask]
-
-    def plot(
-        self,
-        row_labels: list[Any] | NDArray[Any] | None = None,
-        col_labels: list[Any] | NDArray[Any] | None = None,
-        plot_classwise: bool = False,
-        factor_type: Literal["discrete", "continuous", "both"] = "discrete",
-    ) -> Figure:
-        """
-        Plot a heatmap of balance information
-
-        Parameters
-        ----------
-        row_labels : ArrayLike or None, default None
-            List/Array containing the labels for rows in the histogram
-        col_labels : ArrayLike or None, default None
-            List/Array containing the labels for columns in the histogram
-        plot_classwise : bool, default False
-            Whether to plot per-class balance instead of global balance
-        factor_type : "discrete", "continuous", or "both", default "discrete"
-            Whether to plot discretized values, continuous values, or to include both
-        """
-        if plot_classwise:
-            if row_labels is None:
-                row_labels = self.class_names
-            if col_labels is None:
-                col_labels = self._by_factor_type("factor_names", factor_type)
-
-            fig = heatmap(
-                self._by_factor_type("classwise", factor_type),
-                row_labels,
-                col_labels,
-                xlabel="Factors",
-                ylabel="Class",
-                cbarlabel="Normalized Mutual Information",
-            )
-        else:
-            # Combine balance and factors results
-            data = np.concatenate(
-                [
-                    self._by_factor_type("balance", factor_type)[np.newaxis, 1:],
-                    self._by_factor_type("factors", factor_type),
-                ],
-                axis=0,
-            )
-            # Create a mask for the upper triangle of the symmetrical array, ignoring the diagonal
-            mask = np.triu(data + 1, k=0) < 1
-            # Finalize the data for the plot, last row is last factor x last factor so it gets dropped
-            heat_data = np.where(mask, np.nan, data)[:-1]
-            # Creating label array for heat map axes
-            heat_labels = self._by_factor_type("factor_names", factor_type)
-
-            if row_labels is None:
-                row_labels = heat_labels[:-1]
-            if col_labels is None:
-                col_labels = heat_labels[1:]
-
-            fig = heatmap(heat_data, row_labels, col_labels, cbarlabel="Normalized Mutual Information")
-
-        return fig
-
 
 def _validate_num_neighbors(num_neighbors: int) -> int:
     if not isinstance(num_neighbors, (int, float)):
@@ -200,25 +68,22 @@ def balance(
 
     >>> bal = balance(metadata)
     >>> bal.balance
-    array([0.9999982 , 0.2494567 , 0.02994455, 0.13363788, 0.        ,
-           0.        ])
+    array([1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ])
 
     Return intra/interfactor balance (mutual information)
 
     >>> bal.factors
-    array([[0.99999935, 0.31360499, 0.26925848, 0.85201924, 0.36653548],
-           [0.31360499, 0.99999856, 0.09725766, 0.15836905, 1.98031993],
-           [0.26925848, 0.09725766, 0.99999846, 0.03713108, 0.01544656],
-           [0.85201924, 0.15836905, 0.03713108, 0.47450653, 0.25509664],
-           [0.36653548, 1.98031993, 0.01544656, 0.25509664, 1.06260686]])
+    array([[1.   , 0.314, 0.269, 0.852, 0.367],
+           [0.314, 1.   , 0.097, 0.158, 1.98 ],
+           [0.269, 0.097, 1.   , 0.037, 0.015],
+           [0.852, 0.158, 0.037, 0.475, 0.255],
+           [0.367, 1.98 , 0.015, 0.255, 1.063]])
 
     Return classwise balance (mutual information) of factors with individual class_labels
 
     >>> bal.classwise
-    array([[0.9999982 , 0.2494567 , 0.02994455, 0.13363788, 0.        ,
-            0.        ],
-           [0.9999982 , 0.2494567 , 0.02994455, 0.13363788, 0.        ,
-            0.        ]])
+    array([[1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ],
+           [1.   , 0.249, 0.03 , 0.134, 0.   , 0.   ]])
 
 
     See Also
@@ -227,6 +92,9 @@ def balance(
     sklearn.feature_selection.mutual_info_regression
     sklearn.metrics.mutual_info_score
     """
+    if not metadata.discrete_factor_names and not metadata.continuous_factor_names:
+        raise ValueError("No factors found in provided metadata.")
+
     num_neighbors = _validate_num_neighbors(num_neighbors)
 
     num_factors = metadata.total_num_factors
@@ -246,7 +114,7 @@ def balance(
                 data[:, idx],
                 discrete_features=is_discrete,  # type: ignore
                 n_neighbors=num_neighbors,
-                random_state=0,
+                random_state=get_seed(),
             )
         else:
             mi[idx, :] = mutual_info_classif(
@@ -254,7 +122,7 @@ def balance(
                 data[:, idx],
                 discrete_features=is_discrete,  # type: ignore
                 n_neighbors=num_neighbors,
-                random_state=0,
+                random_state=get_seed(),
             )
 
     # Normalization via entropy
@@ -283,7 +151,7 @@ def balance(
             tgt_bin[:, idx],
             discrete_features=is_discrete,  # type: ignore
             n_neighbors=num_neighbors,
-            random_state=0,
+            random_state=get_seed(),
         )
 
     # Classwise normalization via entropy

dataeval/metrics/bias/_coverage.py

@@ -2,118 +2,16 @@ from __future__ import annotations
 
 __all__ = []
 
-import contextlib
 import math
-from dataclasses import dataclass
-from typing import Any, Literal
+from typing import Literal
 
 import numpy as np
-from numpy.typing import NDArray
 from scipy.spatial.distance import pdist, squareform
 
-from dataeval._output import Output, set_metadata
+from dataeval.outputs import CoverageOutput
+from dataeval.outputs._base import set_metadata
 from dataeval.typing import ArrayLike
-from dataeval.utils._array import ensure_embeddings, flatten, to_numpy
-
-with contextlib.suppress(ImportError):
-    from matplotlib.figure import Figure
-
-
-def _plot(images: NDArray[Any], num_images: int) -> Figure:
-    """
-    Creates a single plot of all of the provided images
-
-    Parameters
-    ----------
-    images : NDArray
-        Array containing only the desired images to plot
-
-    Returns
-    -------
-    matplotlib.figure.Figure
-        Plot of all provided images
-    """
-    import matplotlib.pyplot as plt
-
-    num_images = min(num_images, len(images))
-
-    if images.ndim == 4:
-        images = np.moveaxis(images, 1, -1)
-    elif images.ndim == 3:
-        images = np.repeat(images[:, :, :, np.newaxis], 3, axis=-1)
-    else:
-        raise ValueError(
-            f"Expected a (N,C,H,W) or a (N, H, W) set of images, but got a {images.ndim}-dimensional set of images."
-        )
-
-    rows = int(np.ceil(num_images / 3))
-    fig, axs = plt.subplots(rows, 3, figsize=(9, 3 * rows))
-
-    if rows == 1:
-        for j in range(3):
-            if j >= len(images):
-                continue
-            axs[j].imshow(images[j])
-            axs[j].axis("off")
-    else:
-        for i in range(rows):
-            for j in range(3):
-                i_j = i * 3 + j
-                if i_j >= len(images):
-                    continue
-                axs[i, j].imshow(images[i_j])
-                axs[i, j].axis("off")
-
-    fig.tight_layout()
-    return fig
-
-
-@dataclass(frozen=True)
-class CoverageOutput(Output):
-    """
-    Output class for :func:`.coverage` :term:`bias<Bias>` metric.
-
-    Attributes
-    ----------
-    uncovered_indices : NDArray[np.intp]
-        Array of uncovered indices
-    critical_value_radii : NDArray[np.float64]
-        Array of critical value radii
-    coverage_radius : float
-        Radius for :term:`coverage<Coverage>`
-    """
-
-    uncovered_indices: NDArray[np.intp]
-    critical_value_radii: NDArray[np.float64]
-    coverage_radius: float
-
-    def plot(self, images: ArrayLike, top_k: int = 6) -> Figure:
-        """
-        Plot the top k images together for visualization
-
-        Parameters
-        ----------
-        images : ArrayLike
-            Original images (not embeddings) in (N, C, H, W) or (N, H, W) format
-        top_k : int, default 6
-            Number of images to plot (plotting assumes groups of 3)
-
-        Returns
-        -------
-        matplotlib.figure.Figure
-        """
-
-        # Determine which images to plot
-        highest_uncovered_indices = self.uncovered_indices[:top_k]
-
-        # Grab the images
-        images = to_numpy(images)
-        selected_images = images[highest_uncovered_indices]
-
-        # Plot the images
-        fig = _plot(selected_images, top_k)
-
-        return fig
+from dataeval.utils._array import ensure_embeddings, flatten
 
 
 @set_metadata