dataeval 0.61.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/_internal/metrics/diversity.py

@@ -0,0 +1,206 @@
+from typing import Dict, List, Literal, NamedTuple, Optional, Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
+
+
+class DiversityOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    diversity_index : NDArray[np.float64]
+        Diversity index for classes and factors
+    """
+
+    diversity_index: NDArray[np.float64]
+
+
+def diversity_shannon(
+    data: np.ndarray,
+    names: List[str],
+    is_categorical: List[bool],
+    subset_mask: Optional[np.ndarray] = None,
+) -> np.ndarray:
+    """
+    Compute diversity for discrete/categorical variables and, through standard
+    histogram binning, for continuous variables.
+
+    We define diversity as a normalized form of the Shannon entropy.
+
+    diversity = 1 implies that samples are evenly distributed across a particular factor
+    diversity = 0 implies that all samples belong to one category/bin
+
+    Parameters
+    ----------
+    subset_mask: Optional[np.ndarray[bool]]
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Notes
+    -----
+    For continuous variables, histogram bins are chosen automatically.  See `numpy.histogram` for details.
+
+    Returns
+    -------
+    diversity_index: np.ndarray
+        Diversity index per column of X
+
+    See Also
+    --------
+    numpy.histogram
+    """
+
+    # entropy computed using global auto bins so that we can properly normalize
+    ent_unnormalized = entropy(data, names, is_categorical, normalized=False, subset_mask=subset_mask)
+    # normalize by global counts rather than classwise counts
+    num_bins = get_num_bins(data, names, is_categorical=is_categorical, subset_mask=subset_mask)
+    return ent_unnormalized / np.log(num_bins)
+
+
+def diversity_simpson(
+    data: np.ndarray,
+    names: List[str],
+    is_categorical: List[bool],
+    subset_mask: Optional[np.ndarray] = None,
+) -> np.ndarray:
+    """
+    Compute diversity for discrete/categorical variables and, through standard
+    histogram binning, for continuous variables.
+
+    We define diversity as a normalized form of the inverse Simpson diversity
+    index.
+
+    diversity = 1 implies that samples are evenly distributed across a particular factor
+    diversity = 1/num_categories implies that all samples belong to one category/bin
+
+    Parameters
+    ----------
+    subset_mask: Optional[np.ndarray[bool]]
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Notes
+    -----
+    For continuous variables, histogram bins are chosen automatically.  See
+        numpy.histogram for details.
+    The expression is undefined for q=1, but it approaches the Shannon entropy
+        in the limit.
+    If there is only one category, the diversity index takes a value of 1 =
+        1/N = 1/1.  Entropy will take a value of 0.
+
+    Returns
+    -------
+    np.ndarray
+        Diversity index per column of X
+
+    See Also
+    --------
+    numpy.histogram
+    """
+
+    hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
+    # normalize by global counts, not classwise counts
+    num_bins = get_num_bins(data, names, is_categorical)
+
+    ev_index = np.empty(len(names))
+    # loop over columns for convenience
+    for col, cnts in enumerate(hist_counts.values()):
+        # relative frequencies
+        p_i = cnts / cnts.sum()
+        # inverse Simpson index normalized by (number of bins)
+        ev_index[col] = 1 / np.sum(p_i**2) / num_bins[col]
+
+    return ev_index
+
+
+DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
+
+
+def diversity(
+    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+) -> DiversityOutput:
+    """
+    Compute diversity for discrete/categorical variables and, through standard
+    histogram binning, for continuous variables.
+
+    diversity = 1 implies that samples are evenly distributed across a particular factor
+    diversity = 0 implies that all samples belong to one category/bin
+
+    Parameters
+    ----------
+    class_labels: Sequence[int]
+        List of class labels for each image
+    metadata: List[Dict]
+        List of metadata factors for each image
+    metric: Literal["shannon", "simpson"], default "simpson"
+        string variable indicating which diversity index should be used.
+        Permissible values include "simpson" and "shannon"
+
+    Notes
+    -----
+    - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
+
+    Returns
+    -------
+    DiversityOutput
+        Diversity index per column of self.data or each factor in self.names
+
+    See Also
+    --------
+    numpy.histogram
+    """
+    diversity_fn = get_method(DIVERSITY_FN_MAP, method)
+    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    diversity_index = diversity_fn(data, names, is_categorical, None).astype(np.float64)
+    return DiversityOutput(diversity_index)
+
+
+def diversity_classwise(
+    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+) -> DiversityOutput:
+    """
+    Compute diversity for discrete/categorical variables and, through standard
+    histogram binning, for continuous variables.
+
+    We define diversity as a normalized form of the inverse Simpson diversity
+    index.
+
+    diversity = 1 implies that samples are evenly distributed across a particular factor
+    diversity = 1/num_categories implies that all samples belong to one category/bin
+
+    Parameters
+    ----------
+    class_labels: Sequence[int]
+        List of class labels for each image
+    metadata: List[Dict]
+        List of metadata factors for each image
+
+    Notes
+    -----
+    - For continuous variables, histogram bins are chosen automatically. See numpy.histogram for details.
+    - The expression is undefined for q=1, but it approaches the Shannon entropy in the limit.
+    - If there is only one category, the diversity index takes a value of 1 = 1/N = 1/1. Entropy will take a value of 0.
+
+    Returns
+    -------
+    DiversityOutput
+        Diversity index [n_class x n_factor]
+
+    See Also
+    --------
+    numpy.histogram
+    """
+    diversity_fn = get_method(DIVERSITY_FN_MAP, method)
+    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    class_idx = names.index("class_label")
+    class_lbl = data[:, class_idx]
+
+    u_classes = np.unique(class_lbl)
+    num_factors = len(names)
+    diversity = np.empty((len(u_classes), num_factors))
+    diversity[:] = np.nan
+    for idx, cls in enumerate(u_classes):
+        subset_mask = class_lbl == cls
+        diversity[idx, :] = diversity_fn(data, names, is_categorical, subset_mask)
+    div_no_class = np.concatenate((diversity[:, :class_idx], diversity[:, (class_idx + 1) :]), axis=1)
+    return DiversityOutput(div_no_class)

dataeval/_internal/metrics/parity.py

@@ -1,180 +1,309 @@
 import warnings
-from typing import Optional, Tuple
+from typing import Dict, Mapping, NamedTuple, Optional, Tuple
 
 import numpy as np
-import scipy
+from numpy.typing import ArrayLike, NDArray
+from scipy.stats import chi2_contingency, chisquare
 
+from dataeval._internal.interop import to_numpy
 
-class Parity:
+
+class ParityOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    score : np.float64
+        chi-squared value of the test
+    p_value : np.float64
+        p-value of the test
+    """
+
+    score: np.float64
+    p_value: np.float64
+
+
+class ParityMetadataOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    scores : NDArray[np.float64]
+        chi-squared values of the test
+    p_values : NDArray[np.float64]
+        p-values of the test
+    """
+
+    score: NDArray[np.float64]
+    p_value: NDArray[np.float64]
+
+
+def digitize_factor_bins(continuous_values: np.ndarray, bins: int, factor_name: str):
+    """
+    Digitizes a list of values into a given number of bins.
+
+    Parameters
+    ----------
+    continuous_values: np.ndarray
+        The values to be digitized.
+    bins: int
+        The number of bins for the discrete values that continuous_values will be digitized into.
+    factor_name: str
+        The name of the factor to be digitized.
+
+    Returns
+    -------
+    np.ndarray
+        The digitized values
+
+    """
+    if not np.all([np.issubdtype(type(n), np.number) for n in continuous_values]):
+        raise TypeError(
+            f"Encountered a non-numeric value for factor {factor_name}, but the factor"
+            " was specified to be continuous. Ensure all occurrences of this factor are numeric types,"
+            f" or do not specify {factor_name} as a continuous factor."
+        )
+
+    _, bin_edges = np.histogram(continuous_values, bins=bins)
+    bin_edges[-1] = np.inf
+    bin_edges[0] = -np.inf
+    return np.digitize(continuous_values, bin_edges)
+
+
+def format_discretize_factors(
+    data_factors: dict[str, np.ndarray], continuous_factor_bincounts: Dict[str, int]
+) -> Tuple[dict, np.ndarray]:
+    """
+    Sets up the internal list of metadata factors.
+
+    Parameters
+    ----------
+    data_factors: Dict[str, np.ndarray]
+        The dataset factors, which are per-image attributes including class label and metadata.
+        Each key of dataset_factors is a factor, whose value is the per-image factor values.
+    continuous_factor_bincounts : Dict[str, int]
+        The factors in data_factors that have continuous values and the array of bin counts to
+        discretize values into. All factors are treated as having discrete values unless they
+        are specified as keys in this dictionary. Each element of this array must occur as a key
+        in data_factors.
+
+    Returns
+    -------
+    Dict[str, np.ndarray]
+        Intrinsic per-image metadata information with the formatting that input data_factors uses.
+        Each key is a metadata factor, whose value is the discrete per-image factor values.
+    np.ndarray
+        Per-image labels, whose ith element is the label for the ith element of the dataset.
+    """
+    invalid_keys = set(continuous_factor_bincounts.keys()) - set(data_factors.keys())
+    if invalid_keys:
+        raise KeyError(
+            f"The continuous factor(s) {invalid_keys} do not exist in data_factors. Delete these "
+            "keys from `continuous_factor_names` or add corresponding entries to `data_factors`."
+        )
+
+    metadata_factors = {}
+
+    # make sure each factor has the same number of entries
+    lengths = []
+    for arr in data_factors.values():
+        lengths.append(arr.shape)
+
+    if lengths[1:] != lengths[:-1]:
+        raise ValueError("The lengths of each entry in the dictionary are not equal." f" Found lengths {lengths}")
+
+    labels = data_factors["class"]
+
+    metadata_factors = {
+        name: val
+        if name not in continuous_factor_bincounts
+        else digitize_factor_bins(val, continuous_factor_bincounts[name], name)
+        for name, val in data_factors.items()
+        if name != "class"
+    }
+
+    return metadata_factors, labels
+
+
+def normalize_expected_dist(expected_dist: np.ndarray, observed_dist: np.ndarray) -> np.ndarray:
+    exp_sum = np.sum(expected_dist)
+    obs_sum = np.sum(observed_dist)
+
+    if exp_sum == 0:
+        raise ValueError(
+            f"Expected label distribution {expected_dist} is all zeros. "
+            "Ensure that Parity.expected_dist is set to a list "
+            "with at least one nonzero element"
+        )
+
+    # Renormalize expected distribution to have the same total number of labels as the observed dataset
+    if exp_sum != obs_sum:
+        expected_dist = expected_dist * obs_sum / exp_sum
+
+    return expected_dist
+
+
+def validate_dist(label_dist: np.ndarray, label_name: str):
+    """
+    Verifies that the given label distribution has labels and checks if
+    any labels have frequencies less than 5.
+
+    Parameters
+    ----------
+    label_dist : np.ndarray
+        Array representing label distributions
+
+    Raises
+    ------
+    ValueError
+        If label_dist is empty
+    Warning
+        If any elements of label_dist are less than 5
+    """
+    if not len(label_dist):
+        raise ValueError(f"No labels found in the {label_name} dataset")
+    if np.any(label_dist < 5):
+        warnings.warn(
+            f"Labels {np.where(label_dist<5)[0]} in {label_name}"
+            " dataset have frequencies less than 5. This may lead"
+            " to invalid chi-squared evaluation."
+        )
+        warnings.warn(
+            f"Labels {np.where(label_dist<5)[0]} in {label_name}"
+            " dataset have frequencies less than 5. This may lead"
+            " to invalid chi-squared evaluation."
+        )
+
+
+def parity(
+    expected_labels: ArrayLike,
+    observed_labels: ArrayLike,
+    num_classes: Optional[int] = None,
+) -> ParityOutput:
     """
-    Class for evaluating statistics of observed and expected class labels, including:
+    Perform a one-way chi-squared test between observation frequencies and expected frequencies that
+    tests the null hypothesis that the observed data has the expected frequencies.
 
-    - Chi Squared test for statistical independence between expected and observed labels
+    This function acts as an interface to the scipy.stats.chisquare method, which is documented at
+    https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html
 
     Parameters
     ----------
-    expected_labels : np.ndarray
+    expected_labels : ArrayLike
         List of class labels in the expected dataset
-    observed_labels : np.ndarray
+    observed_labels : ArrayLike
         List of class labels in the observed dataset
     num_classes : Optional[int]
         The number of unique classes in the datasets. If this is not specified, it will
         be inferred from the set of unique labels in expected_labels and observed_labels
+
+    Returns
+    -------
+    ParityOutput[np.float64]
+        chi-squared score and p-value of the test
+
+    Raises
+    ------
+    ValueError
+        If x is empty
+    """
+    # Calculate
+    if not num_classes:
+        num_classes = 0
+
+    # Calculate the class frequencies associated with the datasets
+    observed_dist = np.bincount(to_numpy(observed_labels), minlength=num_classes)
+    expected_dist = np.bincount(to_numpy(expected_labels), minlength=num_classes)
+
+    # Validate
+    validate_dist(observed_dist, "observed")
+
+    # Normalize
+    expected_dist = normalize_expected_dist(expected_dist, observed_dist)
+
+    # Validate normalized expected distribution
+    validate_dist(expected_dist, f"expected for {np.sum(observed_dist)} observations")
+
+    if len(observed_dist) != len(expected_dist):
+        raise ValueError(
+            f"Found {len(observed_dist)} unique classes in observed label distribution, "
+            f"but found {len(expected_dist)} unique classes in expected label distribution. "
+            "This can happen when some class ids have zero instances in one dataset but "
+            "not in the other. When initializing Parity, try setting the num_classes "
+            "parameter to the known number of unique class ids, so that classes with "
+            "zero instances are still included in the distributions."
+        )
+
+    cs, p = chisquare(f_obs=observed_dist, f_exp=expected_dist)
+    return ParityOutput(cs, p)
+
+
+def parity_metadata(
+    data_factors: Mapping[str, ArrayLike],
+    continuous_factor_bincounts: Optional[Dict[str, int]] = None,
+) -> ParityMetadataOutput:
+    """
+    Evaluates the statistical independence of metadata factors from class labels.
+    This performs a chi-square test, which provides a score and a p-value for
+    statistical independence between each pair of a metadata factor and a class label.
+    A high score with a low p-value suggests that a metadata factor is strongly
+    correlated with a class label.
+
+    Parameters
+    ----------
+    data_factors: Mapping[str, ArrayLike]
+        The dataset factors, which are per-image attributes including class label and metadata.
+        Each key of dataset_factors is a factor, whose value is the per-image factor values.
+    continuous_factor_bincounts : Optional[Dict[str, int]], default None
+        The factors in data_factors that have continuous values and the array of bin counts to
+        discretize values into. All factors are treated as having discrete values unless they
+        are specified as keys in this dictionary. Each element of this array must occur as a key
+        in data_factors.
+
+    Returns
+    -------
+    ParityOutput[NDArray[np.float64]]
+        Arrays of length (num_factors) whose (i)th element corresponds to the
+        chi-square score and p-value for the relationship between factor i and
+        the class labels in the dataset.
     """
+    data_factors_np = {k: to_numpy(v) for k, v in data_factors.items()}
+    continuous_factor_bincounts = continuous_factor_bincounts if continuous_factor_bincounts else {}
+
+    factors, labels = format_discretize_factors(data_factors_np, continuous_factor_bincounts)
+
+    chi_scores = np.zeros(len(factors))
+    p_values = np.zeros(len(factors))
+    n_cls = len(np.unique(labels))
+    for i, (current_factor_name, factor_values) in enumerate(factors.items()):
+        unique_factor_values = np.unique(factor_values)
+        contingency_matrix = np.zeros((len(unique_factor_values), n_cls))
+        # Builds a contingency matrix where entry at index (r,c) represents
+        # the frequency of current_factor_name achieving value unique_factor_values[r]
+        # at a data point with class c.
+
+        # TODO: Vectorize this nested for loop
+        for fi, factor_value in enumerate(unique_factor_values):
+            for label in range(n_cls):
+                with_both = np.bitwise_and((labels == label), factor_values == factor_value)
+                contingency_matrix[fi, label] = np.sum(with_both)
+                if 0 < contingency_matrix[fi, label] < 5:
+                    warnings.warn(
+                        f"Factor {current_factor_name} value {factor_value} co-occurs "
+                        f"only {contingency_matrix[fi, label]} times with label {label}. "
+                        "This can cause inaccurate chi_square calculation. Recommend"
+                        "ensuring each label occurs either 0 times or at least 5 times. "
+                        "Alternatively, digitize any continuous-valued factors "
+                        "into fewer bins."
+                    )
+
+        # This deletes rows containing only zeros,
+        # because scipy.stats.chi2_contingency fails when there are rows containing only zeros.
+        rowsums = np.sum(contingency_matrix, axis=1)
+        rowmask = np.where(rowsums)
+        contingency_matrix = contingency_matrix[rowmask]
+
+        chi2, p, _, _ = chi2_contingency(contingency_matrix)
+
+        chi_scores[i] = chi2
+        p_values[i] = p
 
-    def __init__(self, expected_labels: np.ndarray, observed_labels: np.ndarray, num_classes: Optional[int] = None):
-        self.set_labels(expected_labels, observed_labels, num_classes)
-
-    def set_labels(self, expected_labels: np.ndarray, observed_labels: np.ndarray, num_classes: Optional[int] = None):
-        """
-        Calculates the label distributions for expected and observed labels
-        and performs validation on the results.
-
-        Parameters
-        ----------
-        expected_labels : np.ndarray
-            List of class labels in the expected dataset
-        observed_labels : np.ndarray
-            List of class labels in the observed dataset
-        num_classes : Optional[int]
-            The number of unique classes in the datasets. If this is not specified, it will
-            be inferred from the set of unique labels in expected_labels and observed_labels
-
-        Raises
-        ------
-        ValueError
-            If x is empty
-        """
-        self.num_classes = num_classes
-
-        # Calculate
-        observed_dist = self._calculate_label_dist(observed_labels)
-        expected_dist = self._calculate_label_dist(expected_labels)
-
-        # Validate
-        self._validate_dist(observed_dist, "observed")
-
-        # Normalize
-        expected_dist = self._normalize_expected_dist(expected_dist, observed_dist)
-
-        # Validate normalized expected distribution
-        self._validate_dist(expected_dist, f"expected for {np.sum(observed_dist)} observations")
-        self._validate_class_balance(expected_dist, observed_dist)
-
-        self._observed_dist = observed_dist
-        self._expected_dist = expected_dist
-
-    def _normalize_expected_dist(self, expected_dist: np.ndarray, observed_dist: np.ndarray) -> np.ndarray:
-        exp_sum = np.sum(expected_dist)
-        obs_sum = np.sum(observed_dist)
-
-        if exp_sum == 0:
-            raise ValueError(
-                f"Expected label distribution {expected_dist} is all zeros. "
-                "Ensure that Parity.expected_dist is set to a list "
-                "with at least one nonzero element"
-            )
-
-        # Renormalize expected distribution to have the same total number of labels as the observed dataset
-        if exp_sum != obs_sum:
-            expected_dist = expected_dist * obs_sum / exp_sum
-
-        return expected_dist
-
-    def _calculate_label_dist(self, labels: np.ndarray) -> np.ndarray:
-        """
-        Calculate the class frequencies associated with a dataset
-
-        Parameters
-        ----------
-        labels : np.ndarray
-            List of class labels in a dataset
-
-        Returns
-        -------
-        label_dist : np.ndarray
-            Array representing label distributions
-        """
-        label_dist = np.bincount(labels, minlength=(self.num_classes if self.num_classes else 0))
-        return label_dist
-
-    def _validate_class_balance(self, expected_dist: np.ndarray, observed_dist: np.ndarray):
-        """
-        Check if the numbers of unique classes in the datasets are unequal
-
-        Parameters
-        ----------
-        expected_dist : np.ndarray
-            Array representing expected label distributions
-        observed_dist : np.ndarray
-            Array representing observed label distributions
-
-        Raises
-        ------
-        ValueError
-            When exp_ld and obs_ld do not have the same number of classes
-        """
-        exp_n_cls = len(expected_dist)
-        obs_n_cls = len(observed_dist)
-        if exp_n_cls != obs_n_cls:
-            raise ValueError(
-                f"Found {obs_n_cls} unique classes in observed label distribution, "
-                f"but found {exp_n_cls} unique classes in expected label distribution,"
-                "This can happen when some class ids have zero instances in one dataset but "
-                "not in the other. When initializing Parity, "
-                "try setting the num_classes parameter to the known number of unique class ids, "
-                "so that classes with zero instances are still included in the distributions."
-            )
-
-    def _validate_dist(self, label_dist: np.ndarray, label_name: str):
-        """
-        Verifies that the given label distribution has labels and checks if
-        any labels have frequencies less than 5.
-
-        Parameters
-        ----------
-        label_dist : np.ndarray
-            Array representing label distributions
-
-        Raises
-        ------
-        ValueError
-            If label_dist is empty
-        Warning
-            If any elements of label_dist are less than 5
-        """
-        if not len(label_dist):
-            raise ValueError(f"No labels found in the {label_name} dataset")
-        if np.any(label_dist < 5):
-            warnings.warn(
-                f"Labels {np.where(label_dist<5)[0]} in {label_name}"
-                " dataset have frequencies less than 5. This may lead"
-                " to invalid chi-squared evaluation."
-            )
-            warnings.warn(
-                f"Labels {np.where(label_dist<5)[0]} in {label_name}"
-                " dataset have frequencies less than 5. This may lead"
-                " to invalid chi-squared evaluation."
-            )
-
-    def evaluate(self) -> Tuple[np.float64, np.float64]:
-        """
-        Perform a one-way chi-squared test between observation frequencies and expected frequencies that
-        tests the null hypothesis that the observed data has the expected frequencies.
-
-        This function acts as an interface to the scipy.stats.chisquare method, which is documented at
-        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html
-        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html
-
-        Returns
-        -------
-        np.float64
-            chi-squared value of the test
-        np.float64
-            p-value of the test
-        """
-        cs_result = scipy.stats.chisquare(f_obs=self._observed_dist, f_exp=self._expected_dist)
-
-        chisquared = cs_result.statistic
-        p_value = cs_result.pvalue
-        return chisquared, p_value
+    return ParityMetadataOutput(chi_scores, p_values)