dataeval 0.64.0__py3-none-any.whl → 0.66.0__py3-none-any.whl

dataeval/_internal/metrics/balance.py

@@ -1,14 +1,19 @@
+from __future__ import annotations
+
 import warnings
-from typing import Dict, List, NamedTuple, Sequence
+from dataclasses import dataclass
+from typing import Sequence
 
 import numpy as np
 from numpy.typing import NDArray
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
 from dataeval._internal.metrics.utils import entropy, preprocess_metadata
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class BalanceOutput(NamedTuple):
+@dataclass(frozen=True)
+class BalanceOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -39,7 +44,8 @@ def validate_num_neighbors(num_neighbors: int) -> int:
     return num_neighbors
 
 
-def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+@set_metadata("dataeval.metrics")
+def balance(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Mutual information (MI) between factors (class label, metadata, label/image properties)
 
@@ -67,6 +73,22 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
     we attempt to infer whether a variable is categorical by the fraction of unique
     values in the dataset.
 
+    Example
+    -------
+    Return balance (mutual information) of factors with class_labels
+
+    >>> balance(class_labels, metadata).mutual_information[0]
+    array([0.99999822, 0.13363788, 0.        , 0.02994455])
+
+    Return balance (mutual information) of metadata factors with class_labels
+    and each other
+
+    >>> balance(class_labels, metadata).mutual_information
+    array([[0.99999822, 0.13363788, 0.        , 0.02994455],
+           [0.13363788, 0.99999843, 0.01389763, 0.09725766],
+           [0.        , 0.01389763, 0.48549233, 0.15314612],
+           [0.02994455, 0.09725766, 0.15314612, 0.99999856]])
+
     See Also
     --------
     sklearn.feature_selection.mutual_info_classif
@@ -83,20 +105,24 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
         tgt = data[:, idx]
 
         if is_categorical[idx]:
+            if tgt.dtype == float:
+                # map to unique integers if categorical
+                _, tgt = np.unique(tgt, return_inverse=True)
             # categorical target
             mi[idx, :] = mutual_info_classif(
                 data,
                 tgt,
                 discrete_features=is_categorical,  # type: ignore
                 n_neighbors=num_neighbors,
+                random_state=0,
             )
         else:
-            # continuous variables
             mi[idx, :] = mutual_info_regression(
                 data,
                 tgt,
                 discrete_features=is_categorical,  # type: ignore
                 n_neighbors=num_neighbors,
+                random_state=0,
             )
 
     ent_all = entropy(data, names, is_categorical, normalized=False)
@@ -107,7 +133,8 @@ def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: in
     return BalanceOutput(nmi)
 
 
-def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+@set_metadata("dataeval.metrics")
+def balance_classwise(class_labels: Sequence[int], metadata: list[dict], num_neighbors: int = 5) -> BalanceOutput:
     """
     Compute mutual information (analogous to correlation) between metadata factors
     (class label, metadata, label/image properties) with individual class labels.
@@ -135,6 +162,15 @@ def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_nei
         (num_classes x num_factors) estimate of mutual information between
         num_factors metadata factors and individual class labels.
 
+    Example
+    -------
+    Return classwise balance (mutual information) of factors with individual class_labels
+
+    >>> balance_classwise(class_labels, metadata).mutual_information
+    array([[0.13363788, 0.54085156, 0.        ],
+           [0.13363788, 0.54085156, 0.        ]])
+
+
     See Also
     --------
     sklearn.feature_selection.mutual_info_classif
@@ -169,6 +205,7 @@ def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_nei
             tgt,
             discrete_features=cat_mask,  # type: ignore
             n_neighbors=num_neighbors,
+            random_state=0,
         )
 
     # let this recompute for all features including class label

dataeval/_internal/metrics/ber.py

@@ -7,7 +7,10 @@ Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
 """
 
-from typing import Literal, NamedTuple, Tuple
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
@@ -16,9 +19,11 @@ from scipy.stats import mode
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class BEROutput(NamedTuple):
+@dataclass(frozen=True)
+class BEROutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -32,7 +37,7 @@ class BEROutput(NamedTuple):
     ber_lower: float
 
 
-def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
+def ber_mst(X: NDArray, y: NDArray) -> tuple[float, float]:
     """Calculates the Bayes Error Rate using a minimum spanning tree
 
     Parameters
@@ -57,7 +62,7 @@ def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
     return upper, lower
 
 
-def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
+def ber_knn(X: NDArray, y: NDArray, k: int) -> tuple[float, float]:
     """Calculates the Bayes Error Rate using K-nearest neighbors
 
     Parameters
@@ -73,9 +78,6 @@ def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
         The upper and lower bounds of the bayes error rate
     """
     M, N = get_classes_counts(y)
-
-    # All features belong on second dimension
-    X = X.reshape((X.shape[0], -1))
     nn_indices = compute_neighbors(X, X, k=k)
     nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
     modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
@@ -107,6 +109,7 @@ def knn_lowerbound(value: float, classes: int, k: int) -> float:
 BER_FN_MAP = {"KNN": ber_knn, "MST": ber_mst}
 
 
+@set_metadata("dataeval.metrics")
 def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
     An estimator for Multi-class Bayes Error Rate using FR or KNN test statistic basis
@@ -134,7 +137,7 @@ def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN",
     Examples
     --------
     >>> import sklearn.datasets as dsets
-    >>> from dataeval.metrics import ber
+    >>> from dataeval.metrics.estimators import ber
 
     >>> images, labels = dsets.make_blobs(n_samples=50, centers=2, n_features=2, random_state=0)
 

dataeval/_internal/metrics/coverage.py

@@ -1,20 +1,24 @@
 import math
-from typing import Literal, NamedTuple
+from dataclasses import dataclass
+from typing import Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.spatial.distance import pdist, squareform
 
 from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import flatten
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class CoverageOutput(NamedTuple):
+@dataclass(frozen=True)
+class CoverageOutput(OutputMetadata):
     """
     Attributes
     ----------
-    indices : np.ndarray
+    indices : NDArray
         Array of uncovered indices
-    radii : np.ndarray
+    radii : NDArray
         Array of critical value radii
     critical_value : float
         Radius for coverage
@@ -25,6 +29,7 @@ class CoverageOutput(NamedTuple):
     critical_value: float
 
 
+@set_metadata("dataeval.metrics")
 def coverage(
     embeddings: ArrayLike,
     radius_type: Literal["adaptive", "naive"] = "adaptive",
@@ -87,12 +92,14 @@ def coverage(
     embeddings = to_numpy(embeddings)
     n = len(embeddings)
     if n <= k:
-        raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
-    mat = squareform(pdist(embeddings)).astype(np.float64)
+        raise ValueError(
+            f"Number of observations n={n} is less than or equal to the specified number of neighbors k={k}."
+        )
+    mat = squareform(pdist(flatten(embeddings))).astype(np.float64)
     sorted_dists = np.sort(mat, axis=1)
     crit = sorted_dists[:, k + 1]
 
-    d = np.shape(embeddings)[1]
+    d = embeddings.shape[1]
     if radius_type == "naive":
         rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
         pvals = np.where(crit > rho)[0]
@@ -101,5 +108,5 @@ def coverage(
         rho = int(n * percent)
         pvals = np.argsort(crit)[::-1][:rho]
     else:
-        raise ValueError("Invalid radius type.")
+        raise ValueError(f"{radius_type} is an invalid radius type. Expected 'adaptive' or 'naive'")
     return CoverageOutput(pvals, crit, rho)

dataeval/_internal/metrics/divergence.py

@@ -3,16 +3,19 @@ This module contains the implementation of HP Divergence
 using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 """
 
-from typing import Literal, NamedTuple
+from dataclasses import dataclass
+from typing import Literal
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class DivergenceOutput(NamedTuple):
+@dataclass(frozen=True)
+class DivergenceOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -26,14 +29,44 @@ class DivergenceOutput(NamedTuple):
     errors: int
 
 
-def divergence_mst(data: np.ndarray, labels: np.ndarray) -> int:
+def divergence_mst(data: NDArray, labels: NDArray) -> int:
+    """
+    Calculates the estimated label errors based on the minimum spanning tree
+
+    Parameters
+    ----------
+    data : NDArray, shape - (N, ... )
+        Input images to be grouped
+    labels : NDArray
+        Corresponding labels for each data point
+
+    Returns
+    -------
+    int
+        Number of label errors when creating the minimum spanning tree
+    """
     mst = minimum_spanning_tree(data).toarray()
     edgelist = np.transpose(np.nonzero(mst))
     errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
     return errors
 
 
-def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
+def divergence_fnn(data: NDArray, labels: NDArray) -> int:
+    """
+    Calculates the estimated label errors based on their nearest neighbors
+
+    Parameters
+    ----------
+    data : NDArray, shape - (N, ... )
+        Input images to be grouped
+    labels : NDArray
+        Corresponding labels for each data point
+
+    Returns
+    -------
+    int
+        Number of label errors when finding nearest neighbors
+    """
     nn_indices = compute_neighbors(data, data)
     errors = np.sum(np.abs(labels[nn_indices] - labels))
     return errors
@@ -42,6 +75,7 @@ def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
 DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}
 
 
+@set_metadata("dataeval.metrics")
 def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
     Calculates the divergence and any errors between the datasets
@@ -50,10 +84,10 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     ----------
     data_a : ArrayLike, shape - (N, P)
         A dataset in an ArrayLike format to compare.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
     data_b : ArrayLike, shape - (N, P)
         A dataset in an ArrayLike format to compare.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
     method : Literal["MST, "FNN"], default "FNN"
         Method used to estimate dataset divergence
 

dataeval/_internal/metrics/diversity.py

@@ -1,12 +1,17 @@
-from typing import Dict, List, Literal, NamedTuple, Optional, Sequence
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal, Sequence
 
 import numpy as np
 from numpy.typing import NDArray
 
 from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class DiversityOutput(NamedTuple):
+@dataclass(frozen=True)
+class DiversityOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -18,11 +23,11 @@ class DiversityOutput(NamedTuple):
 
 
 def diversity_shannon(
-    data: np.ndarray,
-    names: List[str],
-    is_categorical: List[bool],
-    subset_mask: Optional[np.ndarray] = None,
-) -> np.ndarray:
+    data: NDArray,
+    names: list[str],
+    is_categorical: list[bool],
+    subset_mask: NDArray[np.bool_] | None = None,
+) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -34,7 +39,7 @@ def diversity_shannon(
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -43,7 +48,7 @@ def diversity_shannon(
 
     Returns
     -------
-    diversity_index: np.ndarray
+    diversity_index: NDArray
         Diversity index per column of X
 
     See Also
@@ -55,15 +60,18 @@ def diversity_shannon(
     ent_unnormalized = entropy(data, names, is_categorical, normalized=False, subset_mask=subset_mask)
     # normalize by global counts rather than classwise counts
     num_bins = get_num_bins(data, names, is_categorical=is_categorical, subset_mask=subset_mask)
-    return ent_unnormalized / np.log(num_bins)
+    ent_norm = np.empty(ent_unnormalized.shape)
+    ent_norm[num_bins != 1] = ent_unnormalized[num_bins != 1] / np.log(num_bins[num_bins != 1])
+    ent_norm[num_bins == 1] = 0
+    return ent_norm
 
 
 def diversity_simpson(
-    data: np.ndarray,
-    names: List[str],
-    is_categorical: List[bool],
-    subset_mask: Optional[np.ndarray] = None,
-) -> np.ndarray:
+    data: NDArray,
+    names: list[str],
+    is_categorical: list[bool],
+    subset_mask: NDArray[np.bool_] | None = None,
+) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -76,7 +84,7 @@ def diversity_simpson(
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: NDArray[np.bool_] | None
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -90,7 +98,7 @@ def diversity_simpson(
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Diversity index per column of X
 
     See Also
@@ -116,8 +124,9 @@ def diversity_simpson(
 DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
 
 
+@set_metadata("dataeval.metrics")
 def diversity(
-    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity for discrete/categorical variables and, through standard
@@ -145,6 +154,19 @@ def diversity(
     DiversityOutput
         Diversity index per column of self.data or each factor in self.names
 
+    Example
+    -------
+    Compute Simpson diversity index of metadata and class labels
+
+    >>> diversity(class_labels, metadata, method="simpson").diversity_index
+    array([0.34482759, 0.34482759, 0.90909091])
+
+    Compute Shannon diversity index of metadata and class labels
+
+    >>> diversity(class_labels, metadata, method="shannon").diversity_index
+    array([0.37955133, 0.37955133, 0.96748876])
+
+
     See Also
     --------
     numpy.histogram
@@ -155,8 +177,9 @@ def diversity(
     return DiversityOutput(diversity_index)
 
 
+@set_metadata("dataeval.metrics")
 def diversity_classwise(
-    class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
+    class_labels: Sequence[int], metadata: list[dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
     """
     Compute diversity for discrete/categorical variables and, through standard
@@ -186,6 +209,21 @@ def diversity_classwise(
     DiversityOutput
         Diversity index [n_class x n_factor]
 
+    Example
+    -------
+    Compute classwise Simpson diversity index of metadata and class labels
+
+    >>> diversity_classwise(class_labels, metadata, method="simpson").diversity_index
+    array([[0.33793103, 0.51578947],
+           [0.36      , 0.36      ]])
+
+    Compute classwise Shannon diversity index of metadata and class labels
+
+    >>> diversity_classwise(class_labels, metadata, method="shannon").diversity_index
+    array([[0.43156028, 0.83224889],
+           [0.57938016, 0.57938016]])
+
+
     See Also
     --------
     numpy.histogram