dataeval 0.74.2__py3-none-any.whl → 0.76.0__py3-none-any.whl

dataeval/metrics/bias/__init__.py

@@ -1,24 +1,21 @@
 """
-Bias metrics check for skewed or imbalanced datasets and incomplete feature
+Bias metrics check for skewed or imbalanced datasets and incomplete feature \
 representation which may impact model performance.
 """
 
-from dataeval.metrics.bias.balance import BalanceOutput, balance
-from dataeval.metrics.bias.coverage import CoverageOutput, coverage
-from dataeval.metrics.bias.diversity import DiversityOutput, diversity
-from dataeval.metrics.bias.metadata_preprocessing import MetadataOutput, metadata_preprocessing
-from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity
-
 __all__ = [
+    "BalanceOutput",
+    "CoverageOutput",
+    "DiversityOutput",
+    "ParityOutput",
     "balance",
     "coverage",
     "diversity",
     "label_parity",
     "parity",
-    "metadata_preprocessing",
-    "BalanceOutput",
-    "CoverageOutput",
-    "DiversityOutput",
-    "ParityOutput",
-    "MetadataOutput",
 ]
+
+from dataeval.metrics.bias.balance import BalanceOutput, balance
+from dataeval.metrics.bias.coverage import CoverageOutput, coverage
+from dataeval.metrics.bias.diversity import DiversityOutput, diversity
+from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity

dataeval/metrics/bias/balance.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["BalanceOutput", "balance"]
+__all__ = []
 
 import contextlib
 import warnings
@@ -12,9 +12,9 @@ import scipy as sp
 from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
-from dataeval.metrics.bias.metadata_preprocessing import MetadataOutput
-from dataeval.metrics.bias.metadata_utils import get_counts, heatmap
 from dataeval.output import Output, set_metadata
+from dataeval.utils.metadata import Metadata, get_counts
+from dataeval.utils.plot import heatmap
 
 with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
@@ -23,8 +23,8 @@ with contextlib.suppress(ImportError):
 @dataclass(frozen=True)
 class BalanceOutput(Output):
     """
-    Output class for :func:`balance` bias metric
-
+    Output class for :func:`balance` :term:`bias<Bias>` metric.
+    
     Attributes
     ----------
     balance : NDArray[np.float64]
@@ -119,22 +119,24 @@ def _validate_num_neighbors(num_neighbors: int) -> int:
 
 @set_metadata
 def balance(
-    metadata: MetadataOutput,
+    metadata: Metadata,
     num_neighbors: int = 5,
 ) -> BalanceOutput:
     """
-    Mutual information (MI) between factors (class label, metadata, label/image properties)
+    Mutual information (MI) between factors (class label, metadata, label/image properties).
 
     Parameters
     ----------
-    metadata : MetadataOutput
-        Output after running `metadata_preprocessing`
+    metadata : Metadata
+        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
+    num_neighbors : int, default 5
+        Number of points to consider as neighbors
 
     Returns
     -------
     BalanceOutput
-        (num_factors+1) x (num_factors+1) estimate of mutual information
-        between num_factors metadata factors and class label. Symmetry is enforced.
+        (num_factors+1) x (num_factors+1) estimate of mutual information \
+            between num_factors metadata factors and class label. Symmetry is enforced.
 
     Note
     ----

dataeval/metrics/bias/coverage.py

@@ -1,18 +1,17 @@
 from __future__ import annotations
 
-__all__ = ["CoverageOutput", "coverage"]
+__all__ = []
 
 import contextlib
 import math
 from dataclasses import dataclass
-from typing import Literal
+from typing import Any, Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.spatial.distance import pdist, squareform
 
 from dataeval.interop import to_numpy
-from dataeval.metrics.bias.metadata_utils import coverage_plot
 from dataeval.output import Output, set_metadata
 from dataeval.utils.shared import flatten
 
@@ -20,10 +19,59 @@ with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
 
 
+def _plot(images: NDArray[Any], num_images: int) -> Figure:
+    """
+    Creates a single plot of all of the provided images
+
+    Parameters
+    ----------
+    images : NDArray
+        Array containing only the desired images to plot
+
+    Returns
+    -------
+    matplotlib.figure.Figure
+        Plot of all provided images
+    """
+    import matplotlib.pyplot as plt
+
+    num_images = min(num_images, len(images))
+
+    if images.ndim == 4:
+        images = np.moveaxis(images, 1, -1)
+    elif images.ndim == 3:
+        images = np.repeat(images[:, :, :, np.newaxis], 3, axis=-1)
+    else:
+        raise ValueError(
+            f"Expected a (N,C,H,W) or a (N, H, W) set of images, but got a {images.ndim}-dimensional set of images."
+        )
+
+    rows = int(np.ceil(num_images / 3))
+    fig, axs = plt.subplots(rows, 3, figsize=(9, 3 * rows))
+
+    if rows == 1:
+        for j in range(3):
+            if j >= len(images):
+                continue
+            axs[j].imshow(images[j])
+            axs[j].axis("off")
+    else:
+        for i in range(rows):
+            for j in range(3):
+                i_j = i * 3 + j
+                if i_j >= len(images):
+                    continue
+                axs[i, j].imshow(images[i_j])
+                axs[i, j].axis("off")
+
+    fig.tight_layout()
+    return fig
+
+
 @dataclass(frozen=True)
 class CoverageOutput(Output):
     """
-    Output class for :func:`coverage` :term:`bias<Bias>` metric
+    Output class for :func:`coverage` :term:`bias<Bias>` metric.
 
     Attributes
     ----------
@@ -62,7 +110,7 @@ class CoverageOutput(Output):
         selected_images = images[highest_uncovered_indices]
 
         # Plot the images
-        fig = coverage_plot(selected_images, top_k)
+        fig = _plot(selected_images, top_k)
 
         return fig
 

dataeval/metrics/bias/diversity.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["DiversityOutput", "diversity"]
+__all__ = []
 
 import contextlib
 from dataclasses import dataclass
@@ -10,19 +10,48 @@ import numpy as np
 import scipy as sp
 from numpy.typing import ArrayLike, NDArray
 
-from dataeval.metrics.bias.metadata_preprocessing import MetadataOutput
-from dataeval.metrics.bias.metadata_utils import diversity_bar_plot, get_counts, heatmap
 from dataeval.output import Output, set_metadata
+from dataeval.utils.metadata import Metadata, get_counts
+from dataeval.utils.plot import heatmap
 from dataeval.utils.shared import get_method
 
 with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
 
 
+def _plot(labels: NDArray[Any], bar_heights: NDArray[Any]) -> Figure:
+    """
+    Plots a formatted bar plot
+
+    Parameters
+    ----------
+    labels : NDArray
+        Array containing the labels for each bar
+    bar_heights : NDArray
+        Array containing the values for each bar
+
+    Returns
+    -------
+    matplotlib.figure.Figure
+        Bar plot figure
+    """
+    import matplotlib.pyplot as plt
+
+    fig, ax = plt.subplots(figsize=(10, 10))
+
+    ax.bar(labels, bar_heights)
+    ax.set_xlabel("Factors")
+
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+
+    fig.tight_layout()
+    return fig
+
+
 @dataclass(frozen=True)
 class DiversityOutput(Output):
     """
-    Output class for :func:`diversity` :term:`bias<Bias>` metric
+    Output class for :func:`diversity` :term:`bias<Bias>` metric.
 
     Attributes
     ----------
@@ -77,8 +106,7 @@ class DiversityOutput(Output):
         else:
             # Creating label array for heat map axes
             heat_labels = np.concatenate((["class"], self.factor_names))
-
-            fig = diversity_bar_plot(heat_labels, self.diversity_index)
+            fig = _plot(heat_labels, self.diversity_index)
 
         return fig
 
@@ -165,27 +193,26 @@ def diversity_simpson(
 
 @set_metadata
 def diversity(
-    metadata: MetadataOutput,
+    metadata: Metadata,
     method: Literal["simpson", "shannon"] = "simpson",
 ) -> DiversityOutput:
     """
-    Compute :term:`diversity<Diversity>` and classwise diversity for discrete/categorical variables and,
-    through standard histogram binning, for continuous variables.
+    Compute :term:`diversity<Diversity>` and classwise diversity for \
+        discrete/categorical variables through standard histogram binning, \
+        for continuous variables.
 
-    We define diversity as a normalized form of the inverse Simpson diversity index.
+    The method specified defines diversity as the inverse Simpson diversity index linearly rescaled to
+    the unit interval, or the normalized form of the Shannon entropy.
 
     diversity = 1 implies that samples are evenly distributed across a particular factor
     diversity = 0 implies that all samples belong to one category/bin
 
     Parameters
     ----------
-    metadata : MetadataOutput
-        Output after running `metadata_preprocessing`
-
-    Note
-    ----
-    - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
-    - If there is only one category, the diversity index takes a value of 0.
+    metadata : Metadata
+        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
+    method : "simpson" or "shannon", default "simpson"
+        The methodology used for defining diversity
 
     Returns
     -------
@@ -193,27 +220,32 @@ def diversity(
         Diversity index per column of self.data or each factor in self.names and
         classwise diversity [n_class x n_factor]
 
+    Note
+    ----
+    - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
+    - If there is only one category, the diversity index takes a value of 0.
+
     Example
     -------
-    Compute Simpson diversity index of metadata and class labels
+    Compute the diversity index of metadata and class labels
 
     >>> div_simp = diversity(metadata, method="simpson")
     >>> div_simp.diversity_index
-    array([0.72413793, 0.88636364, 0.72413793])
+    array([0.6       , 0.80882353, 1.        , 0.8       ])
 
     >>> div_simp.classwise
-    array([[0.69230769, 0.68965517],
-           [0.5       , 0.8       ]])
+    array([[0.5       , 0.8       , 0.8       ],
+           [0.63043478, 0.97560976, 0.52830189]])
 
     Compute Shannon diversity index of metadata and class labels
 
     >>> div_shan = diversity(metadata, method="shannon")
     >>> div_shan.diversity_index
-    array([0.8812909 , 0.96748876, 0.8812909 ])
+    array([0.81127812, 0.9426312 , 1.        , 0.91829583])
 
     >>> div_shan.classwise
-    array([[0.91651644, 0.86312057],
-           [0.68260619, 0.91829583]])
+    array([[0.68260619, 0.91829583, 0.91829583],
+           [0.81443569, 0.99107606, 0.76420451]])
 
     See Also
     --------

dataeval/metrics/bias/parity.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["ParityOutput", "parity", "label_parity"]
+__all__ = []
 
 import warnings
 from dataclasses import dataclass
@@ -12,8 +12,8 @@ from scipy.stats import chisquare
 from scipy.stats.contingency import chi2_contingency, crosstab
 
 from dataeval.interop import as_numpy, to_numpy
-from dataeval.metrics.bias.metadata_preprocessing import MetadataOutput
 from dataeval.output import Output, set_metadata
+from dataeval.utils.metadata import Metadata
 
 TData = TypeVar("TData", np.float64, NDArray[np.float64])
 
@@ -21,7 +21,7 @@ TData = TypeVar("TData", np.float64, NDArray[np.float64])
 @dataclass(frozen=True)
 class ParityOutput(Generic[TData], Output):
     """
-    Output class for :func:`parity` and :func:`label_parity` :term:`bias<Bias>` metrics
+    Output class for :func:`parity` and :func:`label_parity` :term:`bias<Bias>` metrics.
 
     Attributes
     ----------
@@ -123,8 +123,8 @@ def label_parity(
     num_classes: int | None = None,
 ) -> ParityOutput[np.float64]:
     """
-    Calculate the chi-square statistic to assess the :term:`parity<Parity>` between expected and
-    observed label distributions.
+    Calculate the chi-square statistic to assess the :term:`parity<Parity>` \
+    between expected and observed label distributions.
 
     This function computes the frequency distribution of classes in both expected and observed labels, normalizes
     the expected distribution to match the total number of observed labels, and then calculates the chi-square
@@ -167,8 +167,9 @@ def label_parity(
     --------
     Randomly creating some label distributions using ``np.random.default_rng``
 
-    >>> expected_labels = np_random_gen.choice([0, 1, 2, 3, 4], (100))
-    >>> observed_labels = np_random_gen.choice([2, 3, 0, 4, 1], (100))
+    >>> rng = np.random.default_rng(175)
+    >>> expected_labels = rng.choice([0, 1, 2, 3, 4], (100))
+    >>> observed_labels = rng.choice([2, 3, 0, 4, 1], (100))
     >>> label_parity(expected_labels, observed_labels)
     ParityOutput(score=14.007374204742625, p_value=0.0072715574616218, metadata_names=None)
     """
@@ -205,10 +206,10 @@ def label_parity(
 
 
 @set_metadata
-def parity(metadata: MetadataOutput) -> ParityOutput[NDArray[np.float64]]:
+def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
     """
-    Calculate chi-square statistics to assess the linear relationship between multiple factors
-    and class labels.
+    Calculate chi-square statistics to assess the linear relationship \
+    between multiple factors and class labels.
 
     This function computes the chi-square statistic for each metadata factor to determine if there is
     a significant relationship between the factor values and class labels. The chi-square statistic is
@@ -216,8 +217,8 @@ def parity(metadata: MetadataOutput) -> ParityOutput[NDArray[np.float64]]:
 
     Parameters
     ----------
-    metadata : MetadataOutput
-        Output after running `metadata_preprocessing`
+    metadata : Metadata
+        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
 
     Returns
     -------
@@ -249,16 +250,18 @@ def parity(metadata: MetadataOutput) -> ParityOutput[NDArray[np.float64]]:
     --------
     Randomly creating some "continuous" and categorical variables using ``np.random.default_rng``
 
-    >>> labels = np_random_gen.choice([0, 1, 2], (100))
+    >>> from dataeval.utils.metadata import preprocess
+    >>> rng = np.random.default_rng(175)
+    >>> labels = rng.choice([0, 1, 2], (100))
     >>> metadata_dict = [
     ...     {
-    ...         "age": list(np_random_gen.choice([25, 30, 35, 45], (100))),
-    ...         "income": list(np_random_gen.choice([50000, 65000, 80000], (100))),
-    ...         "gender": list(np_random_gen.choice(["M", "F"], (100))),
+    ...         "age": list(rng.choice([25, 30, 35, 45], (100))),
+    ...         "income": list(rng.choice([50000, 65000, 80000], (100))),
+    ...         "gender": list(rng.choice(["M", "F"], (100))),
     ...     }
     ... ]
     >>> continuous_factor_bincounts = {"age": 4, "income": 3}
-    >>> metadata = metadata_preprocessing(metadata_dict, labels, continuous_factor_bincounts)
+    >>> metadata = preprocess(metadata_dict, labels, continuous_factor_bincounts)
     >>> parity(metadata)
     ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]), metadata_names=['age', 'income', 'gender'])
     """  # noqa: E501

dataeval/metrics/estimators/__init__.py

@@ -2,8 +2,8 @@
 Estimators calculate performance bounds and the statistical distance between datasets.
 """
 
+__all__ = ["ber", "divergence", "uap", "BEROutput", "DivergenceOutput", "UAPOutput"]
+
 from dataeval.metrics.estimators.ber import BEROutput, ber
 from dataeval.metrics.estimators.divergence import DivergenceOutput, divergence
 from dataeval.metrics.estimators.uap import UAPOutput, uap
-
-__all__ = ["ber", "divergence", "uap", "BEROutput", "DivergenceOutput", "UAPOutput"]

dataeval/metrics/estimators/ber.py

@@ -5,11 +5,12 @@ KNN based estimate for the :term:`Bayes error rate<Bayes Error Rate (BER)>`
 
 Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
+
 """
 
 from __future__ import annotations
 
-__all__ = ["BEROutput", "ber"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Literal
@@ -27,7 +28,7 @@ from dataeval.utils.shared import compute_neighbors, get_classes_counts, get_met
 @dataclass(frozen=True)
 class BEROutput(Output):
     """
-    Output class for :func:`ber` estimator metric
+    Output class for :func:`ber` estimator metric.
 
     Attributes
     ----------
@@ -38,11 +39,12 @@ class BEROutput(Output):
     """
 
     ber: float
+
     ber_lower: float
 
 
 def ber_mst(images: NDArray[np.float64], labels: NDArray[np.int_], k: int = 1) -> tuple[float, float]:
-    """Calculates the :term:`Bayes error rate<Bayes Error Rate (BER)>` using a minimum spanning tree
+    """Calculates the :term:`Bayes error rate<Bayes Error Rate (BER)>` using a minimum spanning tree.
 
     Parameters
     ----------
@@ -117,7 +119,8 @@ def knn_lowerbound(value: float, classes: int, k: int) -> float:
 @set_metadata
 def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
-    An estimator for Multi-class :term:`Bayes error rate<Bayes Error Rate (BER)>` using FR or KNN test statistic basis
+    An estimator for Multi-class :term:`Bayes error rate<Bayes Error Rate (BER)>` \
+    using FR or KNN test statistic basis.
 
     Parameters
     ----------

dataeval/metrics/estimators/divergence.py

@@ -5,7 +5,7 @@ using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 
 from __future__ import annotations
 
-__all__ = ["DivergenceOutput", "divergence"]
+__all__ = []
 
 from dataclasses import dataclass
 from typing import Literal
@@ -21,7 +21,7 @@ from dataeval.utils.shared import compute_neighbors, get_method, minimum_spannin
 @dataclass(frozen=True)
 class DivergenceOutput(Output):
     """
-    Output class for :func:`divergence` estimator metric
+    Output class for :func:`divergence` estimator metric.
 
     Attributes
     ----------
@@ -59,7 +59,7 @@ def divergence_mst(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
 
 def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
     """
-    Calculates the estimated label errors based on their nearest neighbors
+    Calculates the estimated label errors based on their nearest neighbors.
 
     Parameters
     ----------
@@ -81,7 +81,7 @@ def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
 @set_metadata
 def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
-    Calculates the :term`divergence` and any errors between the datasets
+    Calculates the :term:`divergence` and any errors between the datasets.
 
     Parameters
     ----------

dataeval/metrics/estimators/uap.py

@@ -6,7 +6,7 @@ average precision<Upper-Bound Average Precision (UAP)>` using empirical mean pre
 
 from __future__ import annotations
 
-__all__ = ["UAPOutput", "uap"]
+__all__ = []
 
 from dataclasses import dataclass
 
@@ -20,7 +20,7 @@ from dataeval.output import Output, set_metadata
 @dataclass(frozen=True)
 class UAPOutput(Output):
     """
-    Output class for :func:`uap` estimator metric
+    Output class for :func:`uap` estimator metric.
 
     Attributes
     ----------
@@ -34,8 +34,8 @@ class UAPOutput(Output):
 @set_metadata
 def uap(labels: ArrayLike, scores: ArrayLike) -> UAPOutput:
     """
-    FR Test Statistic based estimate of the empirical mean precision for
-    the upperbound average precision
+    FR Test Statistic based estimate of the empirical mean precision for the \
+    upperbound average precision.
 
     Parameters
     ----------

dataeval/metrics/stats/__init__.py

@@ -1,8 +1,26 @@
 """
-Statistics metrics calculate a variety of image properties and pixel statistics
+Statistics metrics calculate a variety of image properties and pixel statistics \
 and label statistics against the images and labels of a dataset.
 """
 
+__all__ = [
+    "ChannelStatsOutput",
+    "DatasetStatsOutput",
+    "DimensionStatsOutput",
+    "HashStatsOutput",
+    "LabelStatsOutput",
+    "PixelStatsOutput",
+    "VisualStatsOutput",
+    "boxratiostats",
+    "channelstats",
+    "datasetstats",
+    "dimensionstats",
+    "hashstats",
+    "labelstats",
+    "pixelstats",
+    "visualstats",
+]
+
 from dataeval.metrics.stats.boxratiostats import boxratiostats
 from dataeval.metrics.stats.datasetstats import (
     ChannelStatsOutput,
@@ -15,21 +33,3 @@ from dataeval.metrics.stats.hashstats import HashStatsOutput, hashstats
 from dataeval.metrics.stats.labelstats import LabelStatsOutput, labelstats
 from dataeval.metrics.stats.pixelstats import PixelStatsOutput, pixelstats
 from dataeval.metrics.stats.visualstats import VisualStatsOutput, visualstats
-
-__all__ = [
-    "boxratiostats",
-    "channelstats",
-    "datasetstats",
-    "dimensionstats",
-    "hashstats",
-    "labelstats",
-    "pixelstats",
-    "visualstats",
-    "ChannelStatsOutput",
-    "DatasetStatsOutput",
-    "DimensionStatsOutput",
-    "HashStatsOutput",
-    "LabelStatsOutput",
-    "PixelStatsOutput",
-    "VisualStatsOutput",
-]