dataeval 0.72.1__py3-none-any.whl → 0.73.0__py3-none-any.whl

dataeval/metrics/bias/metadata.py

@@ -0,0 +1,358 @@
+from __future__ import annotations
+
+__all__ = []
+
+import contextlib
+from typing import Any, Mapping
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+from scipy.stats import entropy as sp_entropy
+
+from dataeval.interop import to_numpy
+
+with contextlib.suppress(ImportError):
+    from matplotlib.figure import Figure
+
+CLASS_LABEL = "class_label"
+
+
+def get_counts(
+    data: NDArray[np.int_], names: list[str], is_categorical: list[bool], subset_mask: NDArray[np.bool_] | None = None
+) -> tuple[dict[str, NDArray[np.int_]], dict[str, NDArray[np.int_]]]:
+    """
+    Initialize dictionary of histogram counts --- treat categorical values
+    as histogram bins.
+
+    Parameters
+    ----------
+    subset_mask: NDArray[np.bool_] | None
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Returns
+    -------
+    counts: Dict
+        histogram counts per metadata factor in `factors`.  Each
+        factor will have a different number of bins.  Counts get reused
+        across metrics, so hist_counts are cached but only if computed
+        globally, i.e. without masked samples.
+    """
+
+    hist_counts, hist_bins = {}, {}
+    # np.where needed to satisfy linter
+    mask = np.where(subset_mask if subset_mask is not None else np.ones(data.shape[0], dtype=bool))
+
+    for cdx, fn in enumerate(names):
+        # linter doesn't like double indexing
+        col_data = data[mask, cdx].squeeze()
+        if is_categorical[cdx]:
+            # if discrete, use unique values as bins
+            bins, cnts = np.unique(col_data, return_counts=True)
+        else:
+            bins = hist_bins.get(fn, "auto")
+            cnts, bins = np.histogram(col_data, bins=bins, density=True)
+
+        hist_counts[fn] = cnts
+        hist_bins[fn] = bins
+
+    return hist_counts, hist_bins
+
+
+def entropy(
+    data: NDArray[Any],
+    names: list[str],
+    is_categorical: list[bool],
+    normalized: bool = False,
+    subset_mask: NDArray[np.bool_] | None = None,
+) -> NDArray[np.float64]:
+    """
+    Meant for use with :term:`bias<Bias>` metrics, :term:`balance<Balance>`, :term:`diversity<Diversity>`,
+    ClasswiseBalance, and Classwise Diversity.
+
+    Compute entropy for discrete/categorical variables and for continuous variables through standard
+    histogram binning.
+
+    Parameters
+    ----------
+    normalized: bool
+        Flag that determines whether or not to normalize entropy by log(num_bins)
+    subset_mask: NDArray[np.bool_] | None
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Note
+    ----
+    For continuous variables, histogram bins are chosen automatically.  See
+    numpy.histogram for details.
+
+    Returns
+    -------
+    ent: NDArray[np.float64]
+        Entropy estimate per column of X
+
+    See Also
+    --------
+    numpy.histogram
+    scipy.stats.entropy
+    """
+
+    num_factors = len(names)
+    hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
+
+    ev_index = np.empty(num_factors)
+    for col, cnts in enumerate(hist_counts.values()):
+        # entropy in nats, normalizes counts
+        ev_index[col] = sp_entropy(cnts)
+        if normalized:
+            if len(cnts) == 1:
+                # log(0)
+                ev_index[col] = 0
+            else:
+                ev_index[col] /= np.log(len(cnts))
+    return ev_index
+
+
+def get_num_bins(
+    data: NDArray[Any], names: list[str], is_categorical: list[bool], subset_mask: NDArray[np.bool_] | None = None
+) -> NDArray[np.float64]:
+    """
+    Number of bins or unique values for each metadata factor, used to
+    normalize entropy/:term:`diversity<Diversity>`.
+
+    Parameters
+    ----------
+    subset_mask: NDArray[np.bool_] | None
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Returns
+    -------
+    NDArray[np.float64]
+    """
+    # likely cached
+    hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
+    num_bins = np.empty(len(hist_counts))
+    for idx, cnts in enumerate(hist_counts.values()):
+        num_bins[idx] = len(cnts)
+
+    return num_bins
+
+
+def infer_categorical(arr: NDArray[Any], threshold: float = 0.2) -> NDArray[Any]:
+    """
+    Compute fraction of feature values that are unique --- intended to be used
+    for inferring whether variables are categorical.
+    """
+    if arr.ndim == 1:
+        arr = np.expand_dims(arr, axis=1)
+    num_samples = arr.shape[0]
+    pct_unique = np.empty(arr.shape[1])
+    for col in range(arr.shape[1]):  # type: ignore
+        uvals = np.unique(arr[:, col], axis=0)
+        pct_unique[col] = len(uvals) / num_samples
+    return pct_unique < threshold
+
+
+def preprocess_metadata(
+    class_labels: ArrayLike, metadata: Mapping[str, ArrayLike], cat_thresh: float = 0.2
+) -> tuple[NDArray[Any], list[str], list[bool], NDArray[np.str_]]:
+    # if class_labels is not numeric
+    class_array = to_numpy(class_labels)
+    if not np.issubdtype(class_array.dtype, np.number):
+        unique_classes, numerical_labels = np.unique(class_array, return_inverse=True)
+    else:
+        numerical_labels = np.asarray(class_array, dtype=int)
+        unique_classes = np.unique(class_array)
+
+    # convert class_labels and dict of lists to matrix of metadata values
+    preprocessed_metadata = {CLASS_LABEL: numerical_labels}
+
+    # map columns of dict that are not numeric (e.g. string) to numeric values
+    # that mutual information and diversity functions can accommodate.  Each
+    # unique string receives a unique integer value.
+    for k, v in metadata.items():
+        if k == CLASS_LABEL:
+            k = "label_class"
+        # if not numeric
+        v = to_numpy(v)
+        if not np.issubdtype(v.dtype, np.number):
+            _, mapped_vals = np.unique(v, return_inverse=True)
+            preprocessed_metadata[k] = mapped_vals
+        else:
+            preprocessed_metadata[k] = v
+
+    data = np.stack(list(preprocessed_metadata.values()), axis=-1)
+    names = list(preprocessed_metadata.keys())
+    is_categorical = [infer_categorical(preprocessed_metadata[var], cat_thresh)[0] for var in names]
+
+    return data, names, is_categorical, unique_classes
+
+
+def heatmap(
+    data: NDArray[Any],
+    row_labels: list[str] | NDArray[Any],
+    col_labels: list[str] | NDArray[Any],
+    xlabel: str = "",
+    ylabel: str = "",
+    cbarlabel: str = "",
+) -> Figure:
+    """
+    Plots a formatted heatmap
+
+    Parameters
+    ----------
+    data : NDArray
+        Array containing numerical values for factors to plot
+    row_labels : ArrayLike
+        List/Array containing the labels for rows in the histogram
+    col_labels : ArrayLike
+        List/Array containing the labels for columns in the histogram
+    xlabel : str, default ""
+        X-axis label
+    ylabel : str, default ""
+        Y-axis label
+    cbarlabel : str, default ""
+        Label for the colorbar
+    """
+    import matplotlib
+    import matplotlib.pyplot as plt
+
+    fig, ax = plt.subplots(figsize=(10, 10))
+
+    # Plot the heatmap
+    im = ax.imshow(data, vmin=0, vmax=1.0)
+
+    # Create colorbar
+    cbar = fig.colorbar(im, shrink=0.5)
+    cbar.set_ticks([0.0, 0.25, 0.5, 0.75, 1.0])
+    cbar.set_ticklabels(["0.0", "0.25", "0.5", "0.75", "1.0"])
+    cbar.set_label(cbarlabel, loc="center")
+
+    # Show all ticks and label them with the respective list entries.
+    ax.set_xticks(np.arange(data.shape[1]), labels=col_labels)
+    ax.set_yticks(np.arange(data.shape[0]), labels=row_labels)
+
+    ax.tick_params(top=False, bottom=True, labeltop=False, labelbottom=True)
+    # Rotate the tick labels and set their alignment.
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+
+    # Turn spines off and create white grid.
+    ax.spines[:].set_visible(False)
+
+    ax.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True)
+    ax.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True)
+    ax.grid(which="minor", color="w", linestyle="-", linewidth=3)
+    ax.tick_params(which="minor", bottom=False, left=False)
+
+    if xlabel:
+        ax.set_xlabel(xlabel)
+    if ylabel:
+        ax.set_ylabel(ylabel)
+
+    valfmt = matplotlib.ticker.FuncFormatter(format_text)  # type: ignore
+
+    # Normalize the threshold to the images color range.
+    threshold = im.norm(1.0) / 2.0
+
+    # Set default alignment to center, but allow it to be
+    # overwritten by textkw.
+    kw = {"horizontalalignment": "center", "verticalalignment": "center"}
+
+    # Loop over the data and create a `Text` for each "pixel".
+    # Change the text's color depending on the data.
+    textcolors = ("white", "black")
+    texts = []
+    for i in range(data.shape[0]):
+        for j in range(data.shape[1]):
+            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
+            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)  # type: ignore
+            texts.append(text)
+
+    fig.tight_layout()
+    return fig
+
+
+# Function to define how the text is displayed in the heatmap
+def format_text(*args: str) -> str:
+    """
+    Helper function to format text for heatmap()
+
+    Parameters
+    ----------
+    *args: Tuple (str, str)
+        Text to be formatted. Second element is ignored, but is a
+        mandatory pass-through argument as per matplotlib.ticket.FuncFormatter
+
+    Returns
+    -------
+    str
+        Formatted text
+    """
+    x = args[0]
+    return f"{x:.2f}".replace("0.00", "0").replace("0.", ".").replace("nan", "")
+
+
+def diversity_bar_plot(labels: NDArray[Any], bar_heights: NDArray[Any]) -> Figure:
+    """
+    Plots a formatted bar plot
+
+    Parameters
+    ----------
+    labels : NDArray
+        Array containing the labels for each bar
+    bar_heights : NDArray
+        Array containing the values for each bar
+    """
+    import matplotlib.pyplot as plt
+
+    fig, ax = plt.subplots(figsize=(10, 10))
+
+    ax.bar(labels, bar_heights)
+    ax.set_xlabel("Factors")
+
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+
+    fig.tight_layout()
+    return fig
+
+
+def coverage_plot(images: NDArray[Any], num_images: int) -> Figure:
+    """
+    Creates a single plot of all of the provided images
+
+    Parameters
+    ----------
+    images : NDArray
+        Array containing only the desired images to plot
+    """
+    import matplotlib.pyplot as plt
+
+    num_images = min(num_images, len(images))
+
+    if images.ndim == 4:
+        images = np.moveaxis(images, 1, -1)
+    elif images.ndim == 3:
+        images = np.repeat(images[:, :, :, np.newaxis], 3, axis=-1)
+    else:
+        raise ValueError(
+            f"Expected a (N,C,H,W) or a (N, H, W) set of images, but got a {images.ndim}-dimensional set of images."
+        )
+
+    rows = np.ceil(num_images / 3).astype(int)
+    fig, axs = plt.subplots(rows, 3, figsize=(9, 3 * rows))
+
+    if rows == 1:
+        for j in range(3):
+            if j >= len(images):
+                continue
+            axs[j].imshow(images[j])
+            axs[j].axis("off")
+    else:
+        for i in range(rows):
+            for j in range(3):
+                i_j = i * 3 + j
+                if i_j >= len(images):
+                    continue
+                axs[i, j].imshow(images[i_j])
+                axs[i, j].axis("off")
+
+    fig.tight_layout()
+    return fig

dataeval/{_internal/metrics → metrics/bias}/parity.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
 
+__all__ = ["ParityOutput", "parity", "label_parity"]
+
 import warnings
 from dataclasses import dataclass
-from typing import Generic, Mapping, TypeVar
+from typing import Any, Generic, Mapping, TypeVar
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.stats import chi2_contingency, chisquare
 
-from dataeval._internal.interop import to_numpy
-from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.interop import to_numpy
+from dataeval.metrics.bias.metadata import CLASS_LABEL, preprocess_metadata
+from dataeval.output import OutputMetadata, set_metadata
 
 TData = TypeVar("TData", np.float64, NDArray[np.float64])
 
@@ -25,13 +28,16 @@ class ParityOutput(Generic[TData], OutputMetadata):
         chi-squared score(s) of the test
     p_value : np.float64 | NDArray[np.float64]
         p-value(s) of the test
+    metadata_names: list[str] | None
+        Names of each metadata factor
     """
 
     score: TData
     p_value: TData
+    metadata_names: list[str] | None
 
 
-def digitize_factor_bins(continuous_values: NDArray, bins: int, factor_name: str) -> NDArray:
+def digitize_factor_bins(continuous_values: NDArray[Any], bins: int, factor_name: str) -> NDArray[np.intp]:
     """
     Digitizes a list of values into a given number of bins.
 
@@ -64,8 +70,8 @@ def digitize_factor_bins(continuous_values: NDArray, bins: int, factor_name: str
 
 
 def format_discretize_factors(
-    data_factors: Mapping[str, NDArray], continuous_factor_bincounts: Mapping[str, int]
-) -> dict[str, NDArray]:
+    data: NDArray[Any], names: list[str], is_categorical: list[bool], continuous_factor_bincounts: Mapping[str, int]
+) -> dict[str, NDArray[Any]]:
     """
     Sets up the internal list of metadata factors.
 
@@ -87,35 +93,37 @@ def format_discretize_factors(
           Each key is a metadata factor, whose value is the discrete per-image factor values.
     """
 
-    invalid_keys = set(continuous_factor_bincounts.keys()) - set(data_factors.keys())
+    invalid_keys = set(continuous_factor_bincounts.keys()) - set(names)
     if invalid_keys:
         raise KeyError(
             f"The continuous factor(s) {invalid_keys} do not exist in data_factors. Delete these "
             "keys from `continuous_factor_names` or add corresponding entries to `data_factors`."
         )
 
+    warn = []
     metadata_factors = {}
-
-    # make sure each factor has the same number of entries
-    lengths = []
-    for arr in data_factors.values():
-        lengths.append(arr.shape)
-
-    if lengths[1:] != lengths[:-1]:
-        raise ValueError("The lengths of each entry in the dictionary are not equal." f" Found lengths {lengths}")
-
-    metadata_factors = {
-        name: val
-        if name not in continuous_factor_bincounts
-        else digitize_factor_bins(val, continuous_factor_bincounts[name], name)
-        for name, val in data_factors.items()
-        if name != "class"
-    }
+    for i, name in enumerate(names):
+        if name == CLASS_LABEL:
+            continue
+        if name in continuous_factor_bincounts:
+            metadata_factors[name] = digitize_factor_bins(data[:, i], continuous_factor_bincounts[name], name)
+        elif not is_categorical[i]:
+            warn.append(name)
+            metadata_factors[name] = data[:, i]
+        else:
+            metadata_factors[name] = data[:, i]
+
+    if warn:
+        warnings.warn(
+            f"The following factors appear to be continuous but did not have the desired number of bins specified: \n\
+            {warn}",
+            UserWarning,
+        )
 
     return metadata_factors
 
 
-def normalize_expected_dist(expected_dist: NDArray, observed_dist: NDArray) -> NDArray:
+def normalize_expected_dist(expected_dist: NDArray[Any], observed_dist: NDArray[Any]) -> NDArray[Any]:
     """
     Normalize the expected label distribution to match the total number of labels in the observed distribution.
 
@@ -162,7 +170,7 @@ def normalize_expected_dist(expected_dist: NDArray, observed_dist: NDArray) -> N
     return expected_dist
 
 
-def validate_dist(label_dist: NDArray, label_name: str):
+def validate_dist(label_dist: NDArray[Any], label_name: str) -> None:
     """
     Verifies that the given label distribution has labels and checks if
     any labels have frequencies less than 5.
@@ -191,7 +199,7 @@ def validate_dist(label_dist: NDArray, label_name: str):
         )
 
 
-@set_metadata("dataeval.metrics")
+@set_metadata()
 def label_parity(
     expected_labels: ArrayLike,
     observed_labels: ArrayLike,
@@ -245,7 +253,7 @@ def label_parity(
     >>> expected_labels = np_random_gen.choice([0, 1, 2, 3, 4], (100))
     >>> observed_labels = np_random_gen.choice([2, 3, 0, 4, 1], (100))
     >>> label_parity(expected_labels, observed_labels)
-    ParityOutput(score=14.007374204742625, p_value=0.0072715574616218)
+    ParityOutput(score=14.007374204742625, p_value=0.0072715574616218, metadata_names=None)
     """
 
     # Calculate
@@ -276,13 +284,13 @@ def label_parity(
         )
 
     cs, p = chisquare(f_obs=observed_dist, f_exp=expected_dist)
-    return ParityOutput(cs, p)
+    return ParityOutput(cs, p, None)
 
 
-@set_metadata("dataeval.metrics")
+@set_metadata()
 def parity(
     class_labels: ArrayLike,
-    data_factors: Mapping[str, ArrayLike],
+    metadata: Mapping[str, ArrayLike],
     continuous_factor_bincounts: Mapping[str, int] | None = None,
 ) -> ParityOutput[NDArray[np.float64]]:
     """
@@ -297,12 +305,12 @@ def parity(
     ----------
     class_labels: ArrayLike
         List of class labels for each image
-    data_factors: Mapping[str, ArrayLike]
+    metadata: Mapping[str, ArrayLike]
         The dataset factors, which are per-image metadata attributes.
         Each key of dataset_factors is a factor, whose value is the per-image factor values.
     continuous_factor_bincounts : Mapping[str, int] | None, default None
         A dictionary specifying the number of bins for discretizing the continuous factors.
-        The keys should correspond to the names of continuous factors in `data_factors`,
+        The keys should correspond to the names of continuous factors in `metadata`,
         and the values should be the number of bins to use for discretization.
         If not provided, no discretization is applied.
 
@@ -335,42 +343,44 @@ def parity(
     Randomly creating some "continuous" and categorical variables using ``np.random.default_rng``
 
     >>> labels = np_random_gen.choice([0, 1, 2], (100))
-    >>> data_factors = {
+    >>> metadata = {
     ...     "age": np_random_gen.choice([25, 30, 35, 45], (100)),
     ...     "income": np_random_gen.choice([50000, 65000, 80000], (100)),
     ...     "gender": np_random_gen.choice(["M", "F"], (100)),
     ... }
     >>> continuous_factor_bincounts = {"age": 4, "income": 3}
-    >>> parity(labels, data_factors, continuous_factor_bincounts)
-    ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]))
-    """
+    >>> parity(labels, metadata, continuous_factor_bincounts)
+    ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]), metadata_names=['age', 'income', 'gender'])
+    """  # noqa: E501
     if len(np.shape(class_labels)) > 1:
         raise ValueError(
             f"Got class labels with {len(np.shape(class_labels))}-dimensional",
             f" shape {np.shape(class_labels)}, but expected a 1-dimensional array.",
         )
 
-    data_factors_np = {k: to_numpy(v) for k, v in data_factors.items()}
+    data, names, is_categorical, _ = preprocess_metadata(class_labels, metadata)
     continuous_factor_bincounts = continuous_factor_bincounts if continuous_factor_bincounts else {}
 
-    labels = to_numpy(class_labels)
-    factors = format_discretize_factors(data_factors_np, continuous_factor_bincounts)
+    factors = format_discretize_factors(data, names, is_categorical, continuous_factor_bincounts)
+
+    # unique class labels
+    class_idx = names.index(CLASS_LABEL)
+    u_cls = np.unique(data[:, class_idx])
 
     chi_scores = np.zeros(len(factors))
     p_values = np.zeros(len(factors))
-    n_cls = len(np.unique(labels))
     not_enough_data = {}
     for i, (current_factor_name, factor_values) in enumerate(factors.items()):
         unique_factor_values = np.unique(factor_values)
-        contingency_matrix = np.zeros((len(unique_factor_values), n_cls))
+        contingency_matrix = np.zeros((len(unique_factor_values), u_cls.size))
         # Builds a contingency matrix where entry at index (r,c) represents
         # the frequency of current_factor_name achieving value unique_factor_values[r]
         # at a data point with class c.
 
         # TODO: Vectorize this nested for loop
         for fi, factor_value in enumerate(unique_factor_values):
-            for label in range(n_cls):
-                with_both = np.bitwise_and((labels == label), factor_values == factor_value)
+            for label in u_cls:
+                with_both = np.bitwise_and((data[:, class_idx] == label), factor_values == factor_value)
                 contingency_matrix[fi, label] = np.sum(with_both)
                 if 0 < contingency_matrix[fi, label] < 5:
                     if current_factor_name not in not_enough_data:
@@ -412,4 +422,4 @@ def parity(
             UserWarning,
         )
 
-    return ParityOutput(chi_scores, p_values)
+    return ParityOutput(chi_scores, p_values, list(metadata.keys()))

dataeval/metrics/estimators/__init__.py

@@ -2,8 +2,8 @@
 Estimators calculate performance bounds and the statistical distance between datasets.
 """
 
-from dataeval._internal.metrics.ber import BEROutput, ber
-from dataeval._internal.metrics.divergence import DivergenceOutput, divergence
-from dataeval._internal.metrics.uap import UAPOutput, uap
+from dataeval.metrics.estimators.ber import BEROutput, ber
+from dataeval.metrics.estimators.divergence import DivergenceOutput, divergence
+from dataeval.metrics.estimators.uap import UAPOutput, uap
 
 __all__ = ["ber", "divergence", "uap", "BEROutput", "DivergenceOutput", "UAPOutput"]