dataeval 0.72.1__py3-none-any.whl → 0.73.0__py3-none-any.whl

dataeval/metrics/__init__.py

@@ -3,6 +3,6 @@ Metrics are a way to measure the performance of your models or datasets that
 can then be analyzed in the context of a given problem.
 """
 
-from . import bias, estimators, stats
+from dataeval.metrics import bias, estimators, stats
 
 __all__ = ["bias", "estimators", "stats"]

dataeval/metrics/bias/__init__.py

@@ -3,10 +3,10 @@ Bias metrics check for skewed or imbalanced datasets and incomplete feature
 representation which may impact model performance.
 """
 
-from dataeval._internal.metrics.balance import BalanceOutput, balance
-from dataeval._internal.metrics.coverage import CoverageOutput, coverage
-from dataeval._internal.metrics.diversity import DiversityOutput, diversity
-from dataeval._internal.metrics.parity import ParityOutput, label_parity, parity
+from dataeval.metrics.bias.balance import BalanceOutput, balance
+from dataeval.metrics.bias.coverage import CoverageOutput, coverage
+from dataeval.metrics.bias.diversity import DiversityOutput, diversity
+from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity
 
 __all__ = [
     "balance",

dataeval/{_internal/metrics → metrics/bias}/balance.py

@@ -1,35 +1,98 @@
 from __future__ import annotations
 
+__all__ = ["BalanceOutput", "balance"]
+
+import contextlib
 import warnings
 from dataclasses import dataclass
-from typing import Mapping
+from typing import Any, Mapping
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
-from dataeval._internal.metrics.utils import entropy, preprocess_metadata
-from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.metrics.bias.metadata import entropy, heatmap, preprocess_metadata
+from dataeval.output import OutputMetadata, set_metadata
+
+with contextlib.suppress(ImportError):
+    from matplotlib.figure import Figure
 
 
 @dataclass(frozen=True)
 class BalanceOutput(OutputMetadata):
     """
-    Output class for :func:`balance` :term:`Bias` metric
+    Output class for :func:`balance` bias metric
 
     Attributes
     ----------
     balance : NDArray[np.float64]
-        Estimate of :term:`mutual information<Mutual Information (MI)>` between metadata factors and class label
+        Estimate of mutual information between metadata factors and class label
     factors : NDArray[np.float64]
         Estimate of inter/intra-factor mutual information
     classwise : NDArray[np.float64]
         Estimate of mutual information between metadata factors and individual class labels
+    class_list: NDArray
+        Array of the class labels present in the dataset
+    metadata_names: list[str]
+        Names of each metadata factor
     """
 
     balance: NDArray[np.float64]
     factors: NDArray[np.float64]
     classwise: NDArray[np.float64]
+    class_list: NDArray[Any]
+    metadata_names: list[str]
+
+    def plot(
+        self,
+        row_labels: list[Any] | NDArray[Any] | None = None,
+        col_labels: list[Any] | NDArray[Any] | None = None,
+        plot_classwise: bool = False,
+    ) -> Figure:
+        """
+        Plot a heatmap of balance information
+
+        Parameters
+        ----------
+        row_labels : ArrayLike | None, default None
+            List/Array containing the labels for rows in the histogram
+        col_labels : ArrayLike | None, default None
+            List/Array containing the labels for columns in the histogram
+        plot_classwise : bool, default False
+            Whether to plot per-class balance instead of global balance
+        """
+        if plot_classwise:
+            if row_labels is None:
+                row_labels = self.class_list
+            if col_labels is None:
+                col_labels = np.concatenate((["class"], self.metadata_names))
+
+            fig = heatmap(
+                self.classwise,
+                row_labels,
+                col_labels,
+                xlabel="Factors",
+                ylabel="Class",
+                cbarlabel="Normalized Mutual Information",
+            )
+        else:
+            # Combine balance and factors results
+            data = np.concatenate([self.balance[np.newaxis, 1:], self.factors], axis=0)
+            # Create a mask for the upper triangle of the symmetrical array, ignoring the diagonal
+            mask = np.triu(data + 1, k=0) < 1
+            # Finalize the data for the plot, last row is last factor x last factor so it gets dropped
+            heat_data = np.where(mask, np.nan, data)[:-1]
+            # Creating label array for heat map axes
+            heat_labels = np.concatenate((["class"], self.metadata_names))
+
+            if row_labels is None:
+                row_labels = heat_labels[:-1]
+            if col_labels is None:
+                col_labels = heat_labels[1:]
+
+            fig = heatmap(heat_data, row_labels, col_labels, cbarlabel="Normalized Mutual Information")
+
+        return fig
 
 
 def validate_num_neighbors(num_neighbors: int) -> int:
@@ -55,7 +118,7 @@ def validate_num_neighbors(num_neighbors: int) -> int:
 @set_metadata("dataeval.metrics")
 def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neighbors: int = 5) -> BalanceOutput:
     """
-    :term:`Mutual information (MI)` between factors (class label, metadata, label/image properties)
+    Mutual information (MI) between factors (class label, metadata, label/image properties)
 
     Parameters
     ----------
@@ -70,7 +133,7 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
     Returns
     -------
     BalanceOutput
-        (num_factors+1) x (num_factors+1) estimate of :term:`mutual information<Mutual Information (MI)>`
+        (num_factors+1) x (num_factors+1) estimate of mutual information
         between num_factors metadata factors and class label. Symmetry is enforced.
 
     Note
@@ -83,7 +146,7 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
 
     Example
     -------
-    Return :term:`balance<Balance>` (:term:`mutual information<Mutual Information (MI)>`) of factors with class_labels
+    Return balance (mutual information) of factors with class_labels
 
     >>> bal = balance(class_labels, metadata)
     >>> bal.balance
@@ -109,7 +172,7 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
     sklearn.metrics.mutual_info_score
     """
     num_neighbors = validate_num_neighbors(num_neighbors)
-    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    data, names, is_categorical, unique_labels = preprocess_metadata(class_labels, metadata)
     num_factors = len(names)
     mi = np.empty((num_factors, num_factors))
     mi[:] = np.nan
@@ -143,8 +206,7 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
 
     # unique class labels
     class_idx = names.index("class_label")
-    class_data = data[:, class_idx].astype(int)
-    u_cls = np.unique(class_data)
+    u_cls = np.unique(data[:, class_idx])
     num_classes = len(u_cls)
 
     # assume class is a factor
@@ -154,7 +216,7 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
     # categorical variables, excluding class label
     cat_mask = np.concatenate((is_categorical[:class_idx], is_categorical[(class_idx + 1) :]), axis=0).astype(int)
 
-    tgt_bin = np.stack([class_data == cls for cls in u_cls]).T.astype(int)
+    tgt_bin = np.stack([data[:, class_idx] == cls for cls in u_cls]).T.astype(int)
     ent_tgt_bin = entropy(
         tgt_bin, names=[str(idx) for idx in range(num_classes)], is_categorical=[True for idx in range(num_classes)]
     )
@@ -174,4 +236,4 @@ def balance(class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], num_neig
     norm_factor = 0.5 * np.add.outer(ent_tgt_bin, ent_all) + 1e-6
     classwise = classwise_mi / norm_factor
 
-    return BalanceOutput(balance, factors, classwise)
+    return BalanceOutput(balance, factors, classwise, unique_labels, list(metadata.keys()))

dataeval/{_internal/metrics → metrics/bias}/coverage.py

@@ -1,16 +1,23 @@
 from __future__ import annotations
 
+__all__ = ["CoverageOutput", "coverage"]
+
+import contextlib
 import math
 from dataclasses import dataclass
-from typing import Literal
+from typing import Any, Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.spatial.distance import pdist, squareform
 
-from dataeval._internal.interop import to_numpy
-from dataeval._internal.metrics.utils import flatten
-from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.interop import to_numpy
+from dataeval.metrics.bias.metadata import coverage_plot
+from dataeval.output import OutputMetadata, set_metadata
+from dataeval.utils.shared import flatten
+
+with contextlib.suppress(ImportError):
+    from matplotlib.figure import Figure
 
 
 @dataclass(frozen=True)
@@ -32,13 +39,40 @@ class CoverageOutput(OutputMetadata):
     radii: NDArray[np.float64]
     critical_value: float
 
+    def plot(
+        self,
+        images: NDArray[Any],
+        top_k: int = 6,
+    ) -> Figure:
+        """
+        Plot the top k images together for visualization
+
+        Parameters
+        ----------
+        images : ArrayLike
+            Original images (not embeddings) in (N, C, H, W) or (N, H, W) format
+        top_k : int, default 6
+            Number of images to plot (plotting assumes groups of 3)
+        """
+        # Determine which images to plot
+        highest_uncovered_indices = self.indices[:top_k]
+
+        # Grab the images
+        images = to_numpy(images)
+        selected_images = images[highest_uncovered_indices]
+
+        # Plot the images
+        fig = coverage_plot(selected_images, top_k)
+
+        return fig
+
 
-@set_metadata("dataeval.metrics")
+@set_metadata()
 def coverage(
     embeddings: ArrayLike,
     radius_type: Literal["adaptive", "naive"] = "adaptive",
     k: int = 20,
-    percent: np.float64 = np.float64(0.01),
+    percent: float = 0.01,
 ) -> CoverageOutput:
     """
     Class for evaluating :term:`coverage<Coverage>` and identifying images/samples that are in undercovered regions.
@@ -53,7 +87,7 @@ def coverage(
     k: int, default 20
         Number of observations required in order to be covered.
         [1] suggests that a minimum of 20-50 samples is necessary.
-    percent: np.float64, default np.float(0.01)
+    percent: float, default 0.01
         Percent of observations to be considered uncovered. Only applies to adaptive radius.
 
     Returns

dataeval/{_internal/metrics → metrics/bias}/diversity.py

@@ -1,13 +1,27 @@
 from __future__ import annotations
 
+__all__ = ["DiversityOutput", "diversity"]
+
+import contextlib
 from dataclasses import dataclass
-from typing import Literal, Mapping
+from typing import Any, Literal, Mapping
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
-from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.metrics.bias.metadata import (
+    diversity_bar_plot,
+    entropy,
+    get_counts,
+    get_num_bins,
+    heatmap,
+    preprocess_metadata,
+)
+from dataeval.output import OutputMetadata, set_metadata
+from dataeval.utils.shared import get_method
+
+with contextlib.suppress(ImportError):
+    from matplotlib.figure import Figure
 
 
 @dataclass(frozen=True)
@@ -21,18 +35,66 @@ class DiversityOutput(OutputMetadata):
         :term:`Diversity` index for classes and factors
     classwise : NDArray[np.float64]
         Classwise diversity index [n_class x n_factor]
+    class_list: NDArray[np.int64]
+        Class labels for each value in the dataset
+    metadata_names: list[str]
+        Names of each metadata factor
     """
 
     diversity_index: NDArray[np.float64]
     classwise: NDArray[np.float64]
+    class_list: NDArray[Any]
+    metadata_names: list[str]
+    method: Literal["shannon", "simpson"]
+
+    def plot(
+        self,
+        row_labels: list[Any] | NDArray[Any] | None = None,
+        col_labels: list[Any] | NDArray[Any] | None = None,
+        plot_classwise: bool = False,
+    ) -> Figure:
+        """
+        Plot a heatmap of diversity information
+
+        Parameters
+        ----------
+        row_labels : ArrayLike | None, default None
+            List/Array containing the labels for rows in the histogram
+        col_labels : ArrayLike | None, default None
+            List/Array containing the labels for columns in the histogram
+        plot_classwise : bool, default False
+            Whether to plot per-class balance instead of global balance
+        """
+        if plot_classwise:
+            if row_labels is None:
+                row_labels = self.class_list
+            if col_labels is None:
+                col_labels = self.metadata_names
+
+            fig = heatmap(
+                self.classwise,
+                row_labels,
+                col_labels,
+                xlabel="Factors",
+                ylabel="Class",
+                cbarlabel=f"Normalized {self.method.title()} Index",
+            )
+
+        else:
+            # Creating label array for heat map axes
+            heat_labels = np.concatenate((["class"], self.metadata_names))
+
+            fig = diversity_bar_plot(heat_labels, self.diversity_index)
+
+        return fig
 
 
 def diversity_shannon(
-    data: NDArray,
+    data: NDArray[Any],
     names: list[str],
     is_categorical: list[bool],
     subset_mask: NDArray[np.bool_] | None = None,
-) -> NDArray:
+) -> NDArray[np.float64]:
     """
     Compute :term:`diversity<Diversity>` for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -79,11 +141,11 @@ def diversity_shannon(
 
 
 def diversity_simpson(
-    data: NDArray,
+    data: NDArray[Any],
     names: list[str],
     is_categorical: list[bool],
     subset_mask: NDArray[np.bool_] | None = None,
-) -> NDArray:
+) -> NDArray[np.float64]:
     """
     Compute :term:`diversity<Diversity>` for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -139,10 +201,7 @@ def diversity_simpson(
     return ev_index
 
 
-DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
-
-
-@set_metadata("dataeval.metrics")
+@set_metadata()
 def diversity(
     class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
@@ -202,20 +261,18 @@ def diversity(
     --------
     numpy.histogram
     """
-    diversity_fn = get_method(DIVERSITY_FN_MAP, method)
-    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    diversity_fn = get_method({"simpson": diversity_simpson, "shannon": diversity_shannon}, method)
+    data, names, is_categorical, unique_labels = preprocess_metadata(class_labels, metadata)
     diversity_index = diversity_fn(data, names, is_categorical, None).astype(np.float64)
 
     class_idx = names.index("class_label")
-    class_lbl = data[:, class_idx]
-
-    u_classes = np.unique(class_lbl)
+    u_classes = np.unique(data[:, class_idx])
     num_factors = len(names)
     diversity = np.empty((len(u_classes), num_factors))
     diversity[:] = np.nan
     for idx, cls in enumerate(u_classes):
-        subset_mask = class_lbl == cls
+        subset_mask = data[:, class_idx] == cls
         diversity[idx, :] = diversity_fn(data, names, is_categorical, subset_mask)
     div_no_class = np.concatenate((diversity[:, :class_idx], diversity[:, (class_idx + 1) :]), axis=1)
 
-    return DiversityOutput(diversity_index, div_no_class)
+    return DiversityOutput(diversity_index, div_no_class, unique_labels, list(metadata.keys()), method)