dataeval 0.76.0__py3-none-any.whl → 0.81.0__py3-none-any.whl

dataeval/metrics/__init__.py

@@ -5,4 +5,4 @@ can then be analyzed in the context of a given problem.
 
 __all__ = ["bias", "estimators", "stats"]
 
-from dataeval.metrics import bias, estimators, stats
+from . import bias, estimators, stats

dataeval/metrics/bias/__init__.py

@@ -7,6 +7,7 @@ __all__ = [
     "BalanceOutput",
     "CoverageOutput",
     "DiversityOutput",
+    "LabelParityOutput",
     "ParityOutput",
     "balance",
     "coverage",
@@ -15,7 +16,7 @@ __all__ = [
     "parity",
 ]
 
-from dataeval.metrics.bias.balance import BalanceOutput, balance
-from dataeval.metrics.bias.coverage import CoverageOutput, coverage
-from dataeval.metrics.bias.diversity import DiversityOutput, diversity
-from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity
+from dataeval.metrics.bias._balance import BalanceOutput, balance
+from dataeval.metrics.bias._coverage import CoverageOutput, coverage
+from dataeval.metrics.bias._diversity import DiversityOutput, diversity
+from dataeval.metrics.bias._parity import LabelParityOutput, ParityOutput, label_parity, parity

dataeval/metrics/bias/{balance.py → _balance.py}

@@ -5,16 +5,17 @@ __all__ = []
 import contextlib
 import warnings
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Literal, overload
 
 import numpy as np
 import scipy as sp
 from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
-from dataeval.output import Output, set_metadata
-from dataeval.utils.metadata import Metadata, get_counts
-from dataeval.utils.plot import heatmap
+from dataeval._output import Output, set_metadata
+from dataeval.utils._bin import get_counts
+from dataeval.utils._plot import heatmap
+from dataeval.utils.data import Metadata
 
 with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
@@ -23,8 +24,8 @@ with contextlib.suppress(ImportError):
 @dataclass(frozen=True)
 class BalanceOutput(Output):
     """
-    Output class for :func:`balance` :term:`bias<Bias>` metric.
-    
+    Output class for :func:`.balance` :term:`bias<Bias>` metric.
+
     Attributes
     ----------
     balance : NDArray[np.float64]
@@ -35,21 +36,62 @@ class BalanceOutput(Output):
         Estimate of mutual information between metadata factors and individual class labels
     factor_names : list[str]
         Names of each metadata factor
-    class_list : NDArray
-        Array of the class labels present in the dataset
+    class_names : list[str]
+        List of the class labels present in the dataset
     """
 
     balance: NDArray[np.float64]
     factors: NDArray[np.float64]
     classwise: NDArray[np.float64]
     factor_names: list[str]
-    class_list: NDArray[Any]
+    class_names: list[str]
+
+    @overload
+    def _by_factor_type(
+        self,
+        attr: Literal["factor_names"],
+        factor_type: Literal["discrete", "continuous", "both"],
+    ) -> list[str]: ...
+
+    @overload
+    def _by_factor_type(
+        self,
+        attr: Literal["balance", "factors", "classwise"],
+        factor_type: Literal["discrete", "continuous", "both"],
+    ) -> NDArray[np.float64]: ...
+
+    def _by_factor_type(
+        self,
+        attr: Literal["balance", "factors", "classwise", "factor_names"],
+        factor_type: Literal["discrete", "continuous", "both"],
+    ) -> NDArray[np.float64] | list[str]:
+        # if not filtering by factor_type then just return the requested attribute without mask
+        if factor_type == "both":
+            return getattr(self, attr)
+
+        # create the mask for the selected factor_type
+        mask_lambda = (
+            (lambda x: "-continuous" not in x) if factor_type == "discrete" else (lambda x: "-discrete" not in x)
+        )
+
+        # return the masked attribute
+        if attr == "factor_names":
+            return [x.replace(f"-{factor_type}", "") for x in self.factor_names if mask_lambda(x)]
+        else:
+            factor_type_mask = [mask_lambda(x) for x in self.factor_names]
+            if attr == "factors":
+                return self.factors[factor_type_mask[1:]][:, factor_type_mask[1:]]
+            elif attr == "balance":
+                return self.balance[factor_type_mask]
+            elif attr == "classwise":
+                return self.classwise[:, factor_type_mask]
 
     def plot(
         self,
         row_labels: list[Any] | NDArray[Any] | None = None,
         col_labels: list[Any] | NDArray[Any] | None = None,
         plot_classwise: bool = False,
+        factor_type: Literal["discrete", "continuous", "both"] = "discrete",
     ) -> Figure:
         """
         Plot a heatmap of balance information
@@ -62,15 +104,17 @@ class BalanceOutput(Output):
             List/Array containing the labels for columns in the histogram
         plot_classwise : bool, default False
             Whether to plot per-class balance instead of global balance
+        factor_type : "discrete", "continuous", or "both", default "discrete"
+            Whether to plot discretized values, continuous values, or to include both
         """
         if plot_classwise:
             if row_labels is None:
-                row_labels = self.class_list
+                row_labels = self.class_names
             if col_labels is None:
-                col_labels = self.factor_names
+                col_labels = self._by_factor_type("factor_names", factor_type)
 
             fig = heatmap(
-                self.classwise,
+                self._by_factor_type("classwise", factor_type),
                 row_labels,
                 col_labels,
                 xlabel="Factors",
@@ -79,13 +123,19 @@ class BalanceOutput(Output):
             )
         else:
             # Combine balance and factors results
-            data = np.concatenate([self.balance[np.newaxis, 1:], self.factors], axis=0)
+            data = np.concatenate(
+                [
+                    self._by_factor_type("balance", factor_type)[np.newaxis, 1:],
+                    self._by_factor_type("factors", factor_type),
+                ],
+                axis=0,
+            )
             # Create a mask for the upper triangle of the symmetrical array, ignoring the diagonal
             mask = np.triu(data + 1, k=0) < 1
             # Finalize the data for the plot, last row is last factor x last factor so it gets dropped
             heat_data = np.where(mask, np.nan, data)[:-1]
             # Creating label array for heat map axes
-            heat_labels = self.factor_names
+            heat_labels = self._by_factor_type("factor_names", factor_type)
 
             if row_labels is None:
                 row_labels = heat_labels[:-1]
@@ -128,7 +178,7 @@ def balance(
     Parameters
     ----------
     metadata : Metadata
-        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
+        Preprocessed metadata
     num_neighbors : int, default 5
         Number of points to consider as neighbors
 
@@ -184,7 +234,7 @@ def balance(
     mi = np.full((num_factors, num_factors), np.nan, dtype=np.float32)
     data = np.hstack((metadata.class_labels[:, np.newaxis], metadata.discrete_data))
     discretized_data = data
-    if metadata.continuous_data is not None:
+    if len(metadata.continuous_data):
         data = np.hstack((data, metadata.continuous_data))
         discrete_idx = [metadata.discrete_factor_names.index(name) for name in metadata.continuous_factor_names]
         discretized_data = np.hstack((discretized_data, metadata.discrete_data[:, discrete_idx]))
@@ -218,7 +268,7 @@ def balance(
     factors = nmi[1:, 1:]
 
     # assume class is a factor
-    num_classes = metadata.class_names.size
+    num_classes = len(metadata.class_names)
     classwise_mi = np.full((num_classes, num_factors), np.nan, dtype=np.float32)
 
     # classwise targets

dataeval/metrics/bias/{coverage.py → _coverage.py}

@@ -8,12 +8,12 @@ from dataclasses import dataclass
 from typing import Any, Literal
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 from scipy.spatial.distance import pdist, squareform
 
-from dataeval.interop import to_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.shared import flatten
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import ensure_embeddings, flatten, to_numpy
 
 with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
@@ -71,21 +71,21 @@ def _plot(images: NDArray[Any], num_images: int) -> Figure:
 @dataclass(frozen=True)
 class CoverageOutput(Output):
     """
-    Output class for :func:`coverage` :term:`bias<Bias>` metric.
+    Output class for :func:`.coverage` :term:`bias<Bias>` metric.
 
     Attributes
     ----------
-    indices : NDArray[np.intp]
+    uncovered_indices : NDArray[np.intp]
         Array of uncovered indices
-    radii : NDArray[np.float64]
+    critical_value_radii : NDArray[np.float64]
         Array of critical value radii
-    critical_value : float
+    coverage_radius : float
         Radius for :term:`coverage<Coverage>`
     """
 
-    indices: NDArray[np.intp]
-    radii: NDArray[np.float64]
-    critical_value: float
+    uncovered_indices: NDArray[np.intp]
+    critical_value_radii: NDArray[np.float64]
+    coverage_radius: float
 
     def plot(self, images: ArrayLike, top_k: int = 6) -> Figure:
         """
@@ -102,8 +102,9 @@ class CoverageOutput(Output):
         -------
         matplotlib.figure.Figure
         """
+
         # Determine which images to plot
-        highest_uncovered_indices = self.indices[:top_k]
+        highest_uncovered_indices = self.uncovered_indices[:top_k]
 
         # Grab the images
         images = to_numpy(images)
@@ -119,7 +120,7 @@ class CoverageOutput(Output):
 def coverage(
     embeddings: ArrayLike,
     radius_type: Literal["adaptive", "naive"] = "adaptive",
-    k: int = 20,
+    num_observations: int = 20,
     percent: float = 0.01,
 ) -> CoverageOutput:
     """
@@ -128,11 +129,11 @@ def coverage(
     Parameters
     ----------
     embeddings : ArrayLike, shape - (N, P)
-        A dataset in an ArrayLike format.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+        Dataset embeddings as unit interval [0, 1].
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensional space.
     radius_type : {"adaptive", "naive"}, default "adaptive"
         The function used to determine radius.
-    k : int, default 20
+    num_observations : int, default 20
         Number of observations required in order to be covered.
         [1] suggests that a minimum of 20-50 samples is necessary.
     percent : float, default 0.01
@@ -146,7 +147,9 @@ def coverage(
     Raises
     ------
     ValueError
-        If length of :term:`embeddings<Embeddings>` is less than or equal to k
+        If embeddings are not unit interval [0-1]
+    ValueError
+        If length of :term:`embeddings<Embeddings>` is less than or equal to num_observations
     ValueError
         If radius_type is unknown
 
@@ -157,10 +160,10 @@ def coverage(
     Example
     -------
     >>> results = coverage(embeddings)
-    >>> results.indices
+    >>> results.uncovered_indices
     array([447, 412,   8,  32,  63])
-    >>> results.critical_value
-    0.8459038956941765
+    >>> results.coverage_radius
+    0.17592147193757596
 
     Reference
     ---------
@@ -169,26 +172,29 @@ def coverage(
     [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
     """
 
-    # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
-    embeddings = to_numpy(embeddings)
-    n = len(embeddings)
-    if n <= k:
+    # Calculate distance matrix, look at the (num_observations + 1)th farthest neighbor for each image.
+    embeddings = ensure_embeddings(embeddings, dtype=np.float64, unit_interval=True)
+    len_embeddings = len(embeddings)
+    if len_embeddings <= num_observations:
         raise ValueError(
-            f"Number of observations n={n} is less than or equal to the specified number of neighbors k={k}."
+            f"Length of embeddings ({len_embeddings}) is less than or equal to the specified number of \
+                observations ({num_observations})."
         )
-    mat = squareform(pdist(flatten(embeddings))).astype(np.float64)
-    sorted_dists = np.sort(mat, axis=1)
-    crit = sorted_dists[:, k + 1]
+    embeddings_matrix = squareform(pdist(flatten(embeddings))).astype(np.float64)
+    sorted_dists = np.sort(embeddings_matrix, axis=1)
+    critical_value_radii = sorted_dists[:, num_observations + 1]
 
     d = embeddings.shape[1]
     if radius_type == "naive":
-        rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
-        pvals = np.where(crit > rho)[0]
+        coverage_radius = (1 / math.sqrt(math.pi)) * (
+            (2 * num_observations * math.gamma(d / 2 + 1)) / (len_embeddings)
+        ) ** (1 / d)
+        uncovered_indices = np.where(critical_value_radii > coverage_radius)[0]
     elif radius_type == "adaptive":
-        # Use data adaptive cutoff as rho
-        selection = int(max(n * percent, 1))
-        pvals = np.argsort(crit)[::-1][:selection]
-        rho = float(np.mean(np.sort(crit)[::-1][selection - 1 : selection + 1]))
+        # Use data adaptive cutoff as coverage_radius
+        selection = int(max(len_embeddings * percent, 1))
+        uncovered_indices = np.argsort(critical_value_radii)[::-1][:selection]
+        coverage_radius = float(np.mean(np.sort(critical_value_radii)[::-1][selection - 1 : selection + 1]))
     else:
         raise ValueError(f"{radius_type} is an invalid radius type. Expected 'adaptive' or 'naive'")
-    return CoverageOutput(pvals, crit, rho)
+    return CoverageOutput(uncovered_indices, critical_value_radii, coverage_radius)

dataeval/metrics/bias/{diversity.py → _diversity.py}

@@ -8,12 +8,14 @@ from typing import Any, Literal
 
 import numpy as np
 import scipy as sp
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 
-from dataeval.output import Output, set_metadata
-from dataeval.utils.metadata import Metadata, get_counts
-from dataeval.utils.plot import heatmap
-from dataeval.utils.shared import get_method
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._bin import get_counts
+from dataeval.utils._method import get_method
+from dataeval.utils._plot import heatmap
+from dataeval.utils.data import Metadata
 
 with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
@@ -37,7 +39,7 @@ def _plot(labels: NDArray[Any], bar_heights: NDArray[Any]) -> Figure:
     """
     import matplotlib.pyplot as plt
 
-    fig, ax = plt.subplots(figsize=(10, 10))
+    fig, ax = plt.subplots(figsize=(8, 8))
 
     ax.bar(labels, bar_heights)
     ax.set_xlabel("Factors")
@@ -51,7 +53,7 @@ def _plot(labels: NDArray[Any], bar_heights: NDArray[Any]) -> Figure:
 @dataclass(frozen=True)
 class DiversityOutput(Output):
     """
-    Output class for :func:`diversity` :term:`bias<Bias>` metric.
+    Output class for :func:`.diversity` :term:`bias<Bias>` metric.
 
     Attributes
     ----------
@@ -61,14 +63,14 @@ class DiversityOutput(Output):
         Classwise diversity index [n_class x n_factor]
     factor_names : list[str]
         Names of each metadata factor
-    class_list : NDArray[Any]
+    class_names : list[str]
         Class labels for each value in the dataset
     """
 
     diversity_index: NDArray[np.double]
     classwise: NDArray[np.double]
     factor_names: list[str]
-    class_list: NDArray[Any]
+    class_names: list[str]
 
     def plot(
         self,
@@ -90,7 +92,7 @@ class DiversityOutput(Output):
         """
         if plot_classwise:
             if row_labels is None:
-                row_labels = self.class_list
+                row_labels = self.class_names
             if col_labels is None:
                 col_labels = self.factor_names
 
@@ -191,6 +193,9 @@ def diversity_simpson(
     return ev_index
 
 
+_DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
+
+
 @set_metadata
 def diversity(
     metadata: Metadata,
@@ -210,7 +215,7 @@ def diversity(
     Parameters
     ----------
     metadata : Metadata
-        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
+        Preprocessed metadata
     method : "simpson" or "shannon", default "simpson"
         The methodology used for defining diversity
 
@@ -251,7 +256,7 @@ def diversity(
     --------
     scipy.stats.entropy
     """
-    diversity_fn = get_method({"simpson": diversity_simpson, "shannon": diversity_shannon}, method)
+    diversity_fn = get_method(_DIVERSITY_FN_MAP, method)
     discretized_data = np.hstack((metadata.class_labels[:, np.newaxis], metadata.discrete_data))
     cnts = get_counts(discretized_data)
     num_bins = np.bincount(np.nonzero(cnts)[1])