dataeval 0.64.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

dataeval/_internal/metrics/coverage.py

@@ -1,20 +1,24 @@
 import math
-from typing import Literal, NamedTuple
+from dataclasses import dataclass
+from typing import Literal
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.spatial.distance import pdist, squareform
 
 from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import flatten
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class CoverageOutput(NamedTuple):
+@dataclass(frozen=True)
+class CoverageOutput(OutputMetadata):
     """
     Attributes
     ----------
-    indices : np.ndarray
+    indices : NDArray
         Array of uncovered indices
-    radii : np.ndarray
+    radii : NDArray
         Array of critical value radii
     critical_value : float
         Radius for coverage
@@ -25,6 +29,7 @@ class CoverageOutput(NamedTuple):
     critical_value: float
 
 
+@set_metadata("dataeval.metrics")
 def coverage(
     embeddings: ArrayLike,
     radius_type: Literal["adaptive", "naive"] = "adaptive",
@@ -87,12 +92,14 @@ def coverage(
     embeddings = to_numpy(embeddings)
     n = len(embeddings)
     if n <= k:
-        raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
-    mat = squareform(pdist(embeddings)).astype(np.float64)
+        raise ValueError(
+            f"Number of observations n={n} is less than or equal to the specified number of neighbors k={k}."
+        )
+    mat = squareform(pdist(flatten(embeddings))).astype(np.float64)
     sorted_dists = np.sort(mat, axis=1)
     crit = sorted_dists[:, k + 1]
 
-    d = np.shape(embeddings)[1]
+    d = embeddings.shape[1]
     if radius_type == "naive":
         rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
         pvals = np.where(crit > rho)[0]
@@ -101,5 +108,5 @@ def coverage(
         rho = int(n * percent)
         pvals = np.argsort(crit)[::-1][:rho]
     else:
-        raise ValueError("Invalid radius type.")
+        raise ValueError(f"{radius_type} is an invalid radius type. Expected 'adaptive' or 'naive'")
     return CoverageOutput(pvals, crit, rho)

dataeval/_internal/metrics/divergence.py

@@ -3,16 +3,19 @@ This module contains the implementation of HP Divergence
 using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 """
 
-from typing import Literal, NamedTuple
+from dataclasses import dataclass
+from typing import Literal
 
 import numpy as np
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class DivergenceOutput(NamedTuple):
+@dataclass(frozen=True)
+class DivergenceOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -26,14 +29,44 @@ class DivergenceOutput(NamedTuple):
     errors: int
 
 
-def divergence_mst(data: np.ndarray, labels: np.ndarray) -> int:
+def divergence_mst(data: NDArray, labels: NDArray) -> int:
+    """
+    Calculates the estimated label errors based on the minimum spanning tree
+
+    Parameters
+    ----------
+    data : NDArray, shape - (N, ... )
+        Input images to be grouped
+    labels : NDArray
+        Corresponding labels for each data point
+
+    Returns
+    -------
+    int
+        Number of label errors when creating the minimum spanning tree
+    """
     mst = minimum_spanning_tree(data).toarray()
     edgelist = np.transpose(np.nonzero(mst))
     errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
     return errors
 
 
-def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
+def divergence_fnn(data: NDArray, labels: NDArray) -> int:
+    """
+    Calculates the estimated label errors based on their nearest neighbors
+
+    Parameters
+    ----------
+    data : NDArray, shape - (N, ... )
+        Input images to be grouped
+    labels : NDArray
+        Corresponding labels for each data point
+
+    Returns
+    -------
+    int
+        Number of label errors when finding nearest neighbors
+    """
     nn_indices = compute_neighbors(data, data)
     errors = np.sum(np.abs(labels[nn_indices] - labels))
     return errors
@@ -42,6 +75,7 @@ def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
 DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}
 
 
+@set_metadata("dataeval.metrics")
 def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
     Calculates the divergence and any errors between the datasets
@@ -50,10 +84,10 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     ----------
     data_a : ArrayLike, shape - (N, P)
         A dataset in an ArrayLike format to compare.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
     data_b : ArrayLike, shape - (N, P)
         A dataset in an ArrayLike format to compare.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
     method : Literal["MST, "FNN"], default "FNN"
         Method used to estimate dataset divergence
 

dataeval/_internal/metrics/diversity.py

@@ -1,12 +1,15 @@
-from typing import Dict, List, Literal, NamedTuple, Optional, Sequence
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Optional, Sequence
 
 import numpy as np
 from numpy.typing import NDArray
 
 from dataeval._internal.metrics.utils import entropy, get_counts, get_method, get_num_bins, preprocess_metadata
+from dataeval._internal.output import OutputMetadata, set_metadata
 
 
-class DiversityOutput(NamedTuple):
+@dataclass(frozen=True)
+class DiversityOutput(OutputMetadata):
     """
     Attributes
     ----------
@@ -18,11 +21,11 @@ class DiversityOutput(NamedTuple):
 
 
 def diversity_shannon(
-    data: np.ndarray,
+    data: NDArray,
     names: List[str],
     is_categorical: List[bool],
-    subset_mask: Optional[np.ndarray] = None,
-) -> np.ndarray:
+    subset_mask: Optional[NDArray[np.bool_]] = None,
+) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -34,7 +37,7 @@ def diversity_shannon(
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: Optional[NDArray[np.bool_]]
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -43,7 +46,7 @@ def diversity_shannon(
 
     Returns
     -------
-    diversity_index: np.ndarray
+    diversity_index: NDArray
         Diversity index per column of X
 
     See Also
@@ -59,11 +62,11 @@ def diversity_shannon(
 
 
 def diversity_simpson(
-    data: np.ndarray,
+    data: NDArray,
     names: List[str],
     is_categorical: List[bool],
-    subset_mask: Optional[np.ndarray] = None,
-) -> np.ndarray:
+    subset_mask: Optional[NDArray[np.bool_]] = None,
+) -> NDArray:
     """
     Compute diversity for discrete/categorical variables and, through standard
     histogram binning, for continuous variables.
@@ -76,7 +79,7 @@ def diversity_simpson(
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: Optional[NDArray[np.bool_]]
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -90,7 +93,7 @@ def diversity_simpson(
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Diversity index per column of X
 
     See Also
@@ -116,6 +119,7 @@ def diversity_simpson(
 DIVERSITY_FN_MAP = {"simpson": diversity_simpson, "shannon": diversity_shannon}
 
 
+@set_metadata("dataeval.metrics")
 def diversity(
     class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:
@@ -155,6 +159,7 @@ def diversity(
     return DiversityOutput(diversity_index)
 
 
+@set_metadata("dataeval.metrics")
 def diversity_classwise(
     class_labels: Sequence[int], metadata: List[Dict], method: Literal["shannon", "simpson"] = "simpson"
 ) -> DiversityOutput:

dataeval/_internal/metrics/parity.py

@@ -1,48 +1,39 @@
 import warnings
-from typing import Dict, Mapping, NamedTuple, Optional, Tuple
+from dataclasses import dataclass
+from typing import Dict, Generic, Mapping, Optional, Tuple, TypeVar
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 from scipy.stats import chi2_contingency, chisquare
 
 from dataeval._internal.interop import to_numpy
+from dataeval._internal.output import OutputMetadata, set_metadata
 
-
-class ParityOutput(NamedTuple):
-    """
-    Attributes
-    ----------
-    score : np.float64
-        chi-squared value of the test
-    p_value : np.float64
-        p-value of the test
-    """
-
-    score: np.float64
-    p_value: np.float64
+TData = TypeVar("TData", np.float64, NDArray[np.float64])
 
 
-class ParityMetadataOutput(NamedTuple):
+@dataclass(frozen=True)
+class ParityOutput(Generic[TData], OutputMetadata):
     """
     Attributes
     ----------
-    scores : NDArray[np.float64]
-        chi-squared values of the test
-    p_values : NDArray[np.float64]
-        p-values of the test
+    score : np.float64 | NDArray[np.float64]
+        chi-squared score(s) of the test
+    p_value : np.float64 | NDArray[np.float64]
+        p-value(s) of the test
     """
 
-    score: NDArray[np.float64]
-    p_value: NDArray[np.float64]
+    score: TData
+    p_value: TData
 
 
-def digitize_factor_bins(continuous_values: np.ndarray, bins: int, factor_name: str):
+def digitize_factor_bins(continuous_values: NDArray, bins: int, factor_name: str) -> NDArray:
     """
     Digitizes a list of values into a given number of bins.
 
     Parameters
     ----------
-    continuous_values: np.ndarray
+    continuous_values: NDArray
         The values to be digitized.
     bins: int
         The number of bins for the discrete values that continuous_values will be digitized into.
@@ -51,7 +42,7 @@ def digitize_factor_bins(continuous_values: np.ndarray, bins: int, factor_name:
 
     Returns
     -------
-    np.ndarray
+    NDArray
         The digitized values
 
     """
@@ -69,14 +60,14 @@ def digitize_factor_bins(continuous_values: np.ndarray, bins: int, factor_name:
 
 
 def format_discretize_factors(
-    data_factors: dict[str, np.ndarray], continuous_factor_bincounts: Dict[str, int]
-) -> Tuple[dict, np.ndarray]:
+    data_factors: Dict[str, NDArray], continuous_factor_bincounts: Dict[str, int]
+) -> Tuple[Dict[str, NDArray], NDArray]:
     """
     Sets up the internal list of metadata factors.
 
     Parameters
     ----------
-    data_factors: Dict[str, np.ndarray]
+    data_factors: Dict[str, NDArray]
         The dataset factors, which are per-image attributes including class label and metadata.
         Each key of dataset_factors is a factor, whose value is the per-image factor values.
     continuous_factor_bincounts : Dict[str, int]
@@ -87,11 +78,10 @@ def format_discretize_factors(
 
     Returns
     -------
-    Dict[str, np.ndarray]
-        Intrinsic per-image metadata information with the formatting that input data_factors uses.
-        Each key is a metadata factor, whose value is the discrete per-image factor values.
-    np.ndarray
-        Per-image labels, whose ith element is the label for the ith element of the dataset.
+    Tuple[Dict[str, NDArray], NDArray]
+        - Intrinsic per-image metadata information with the formatting that input data_factors uses.
+          Each key is a metadata factor, whose value is the discrete per-image factor values.
+        - Per-image labels, whose ith element is the label for the ith element of the dataset.
     """
     invalid_keys = set(continuous_factor_bincounts.keys()) - set(data_factors.keys())
     if invalid_keys:
@@ -123,7 +113,7 @@ def format_discretize_factors(
     return metadata_factors, labels
 
 
-def normalize_expected_dist(expected_dist: np.ndarray, observed_dist: np.ndarray) -> np.ndarray:
+def normalize_expected_dist(expected_dist: NDArray, observed_dist: NDArray) -> NDArray:
     exp_sum = np.sum(expected_dist)
     obs_sum = np.sum(observed_dist)
 
@@ -141,14 +131,14 @@ def normalize_expected_dist(expected_dist: np.ndarray, observed_dist: np.ndarray
     return expected_dist
 
 
-def validate_dist(label_dist: np.ndarray, label_name: str):
+def validate_dist(label_dist: NDArray, label_name: str):
     """
     Verifies that the given label distribution has labels and checks if
     any labels have frequencies less than 5.
 
     Parameters
     ----------
-    label_dist : np.ndarray
+    label_dist : NDArray
         Array representing label distributions
 
     Raises
@@ -166,18 +156,14 @@ def validate_dist(label_dist: np.ndarray, label_name: str):
             " dataset have frequencies less than 5. This may lead"
             " to invalid chi-squared evaluation."
         )
-        warnings.warn(
-            f"Labels {np.where(label_dist<5)[0]} in {label_name}"
-            " dataset have frequencies less than 5. This may lead"
-            " to invalid chi-squared evaluation."
-        )
 
 
+@set_metadata("dataeval.metrics")
 def parity(
     expected_labels: ArrayLike,
     observed_labels: ArrayLike,
     num_classes: Optional[int] = None,
-) -> ParityOutput:
+) -> ParityOutput[np.float64]:
     """
     Perform a one-way chi-squared test between observation frequencies and expected frequencies that
     tests the null hypothesis that the observed data has the expected frequencies.
@@ -236,10 +222,11 @@ def parity(
     return ParityOutput(cs, p)
 
 
+@set_metadata("dataeval.metrics")
 def parity_metadata(
     data_factors: Mapping[str, ArrayLike],
     continuous_factor_bincounts: Optional[Dict[str, int]] = None,
-) -> ParityMetadataOutput:
+) -> ParityOutput[NDArray[np.float64]]:
     """
     Evaluates the statistical independence of metadata factors from class labels.
     This performs a chi-square test, which provides a score and a p-value for
@@ -306,4 +293,4 @@ def parity_metadata(
         chi_scores[i] = chi2
         p_values[i] = p
 
-    return ParityMetadataOutput(chi_scores, p_values)
+    return ParityOutput(chi_scores, p_values)