dataeval 0.63.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/_internal/metrics/ber.py

@@ -7,68 +7,142 @@ Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
 """
 
-from typing import Callable, Dict, Literal, Tuple
+from typing import Literal, NamedTuple, Tuple
 
 import numpy as np
+from numpy.typing import ArrayLike, NDArray
+from scipy.sparse import coo_matrix
+from scipy.stats import mode
 
-from dataeval._internal.functional.ber import ber_knn, ber_mst
-from dataeval._internal.interop import ArrayLike, to_numpy
-from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
 
-_METHODS = Literal["MST", "KNN"]
-_FUNCTION = Callable[[np.ndarray, np.ndarray, int], Tuple[float, float]]
+
+class BEROutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    ber : float
+        The upper bounds of the Bayes Error Rate
+    ber_lower : float
+        The lower bounds of the Bayes Error Rate
+    """
+
+    ber: float
+    ber_lower: float
+
+
+def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using a minimum spanning tree
+
+    Parameters
+    ----------
+    X : NDArray, shape - (N, ... )
+        n_samples containing n_features
+    y : NDArray, shape - (N, 1)
+        Labels corresponding to each sample
+
+    Returns
+    -------
+    Tuple[float, float]
+        The upper and lower bounds of the bayes error rate
+    """
+    M, N = get_classes_counts(y)
+
+    tree = coo_matrix(minimum_spanning_tree(X))
+    matches = np.sum([y[tree.row[i]] != y[tree.col[i]] for i in range(N - 1)])
+    deltas = matches / (2 * N)
+    upper = 2 * deltas
+    lower = ((M - 1) / (M)) * (1 - max(1 - 2 * ((M) / (M - 1)) * deltas, 0) ** 0.5)
+    return upper, lower
 
 
-class BER(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using K-nearest neighbors
+
+    Parameters
+    ----------
+    X : NDArray, shape - (N, ... )
+        n_samples containing n_features
+    y : NDArray, shape - (N, 1)
+        Labels corresponding to each sample
+
+    Returns
+    -------
+    Tuple[float, float]
+        The upper and lower bounds of the bayes error rate
+    """
+    M, N = get_classes_counts(y)
+
+    # All features belong on second dimension
+    X = X.reshape((X.shape[0], -1))
+    nn_indices = compute_neighbors(X, X, k=k)
+    nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
+    modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
+    upper = float(np.count_nonzero(modal_class - y) / N)
+    lower = knn_lowerbound(upper, M, k)
+    return upper, lower
+
+
+def knn_lowerbound(value: float, classes: int, k: int) -> float:
+    """Several cases for computing the BER lower bound"""
+    if value <= 1e-10:
+        return 0.0
+
+    if classes == 2 and k != 1:
+        if k > 5:
+            # Property 2 (Devroye, 1981) cited in Snoopy paper, not in snoopy repo
+            alpha = 0.3399
+            beta = 0.9749
+            a_k = alpha * np.sqrt(k) / (k - 3.25) * (1 + beta / (np.sqrt(k - 3)))
+            return value / (1 + a_k)
+        if k > 2:
+            return value / (1 + (1 / np.sqrt(k)))
+        # k == 2:
+        return value / 2
+
+    return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))
+
+
+BER_FN_MAP = {"KNN": ber_knn, "MST": ber_mst}
+
+
+def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
     An estimator for Multi-class Bayes Error Rate using FR or KNN test statistic basis
 
     Parameters
     ----------
-    method : Literal["MST", "KNN"], default "KNN"
-        Method to use when estimating the Bayes error rate
+    images : ArrayLike (N, ... )
+        Array of images or image embeddings
+    labels : ArrayLike (N, 1)
+        Array of labels for each image or image embedding
     k : int, default 1
-        number of nearest neighbors for KNN estimator -- ignored by MST estimator
+        Number of nearest neighbors for KNN estimator -- ignored by MST estimator
+    method : Literal["KNN", "MST"], default "KNN"
+        Method to use when estimating the Bayes error rate
 
+    Returns
+    -------
+    BEROutput
+        The upper and lower bounds of the Bayes Error Rate
 
-    See Also
+    References
+    ----------
+    [1] `Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4) <https://arxiv.org/abs/1811.06419>`_
+
+    Examples
     --------
-    `Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4) <https://arxiv.org/abs/1811.06419>`_
+    >>> import sklearn.datasets as dsets
+    >>> from dataeval.metrics import ber
 
-    """
+    >>> images, labels = dsets.make_blobs(n_samples=50, centers=2, n_features=2, random_state=0)
 
-    def __init__(self, method: _METHODS = "KNN", k: int = 1) -> None:
-        self.k: int = k
-        self._set_method(method)
-
-    @classmethod
-    def _methods(cls) -> Dict[str, _FUNCTION]:
-        return {"KNN": ber_knn, "MST": ber_mst}
-
-    def evaluate(self, images: ArrayLike, labels: ArrayLike) -> Dict[str, float]:
-        """
-        Calculates the Bayes Error Rate estimate using the provided method
-
-        Parameters
-        ----------
-        images : ArrayLike (N, : )
-            Array of images or image embeddings
-        labels : ArrayLike (N, 1)
-            Array of labels for each image or image embedding
-
-        Returns
-        -------
-        Dict[str, float]
-            ber : float
-                The estimated lower bounds of the Bayes Error Rate
-            ber_lower : float
-                The estimated upper bounds of the Bayes Error Rate
-
-        Raises
-        ------
-        ValueError
-            If unique classes M < 2
-        """
-
-        upper, lower = self._method(to_numpy(images), to_numpy(labels), self.k)
-        return {"ber": upper, "ber_lower": lower}
+    >>> ber(images, labels)
+    BEROutput(ber=0.04, ber_lower=0.020416847668728033)
+    """
+    ber_fn = get_method(BER_FN_MAP, method)
+    X = to_numpy(images)
+    y = to_numpy(labels)
+    upper, lower = ber_fn(X, y, k) if method == "KNN" else ber_fn(X, y)
+    return BEROutput(upper, lower)

dataeval/_internal/metrics/coverage.py

@@ -1,18 +1,44 @@
-from typing import Literal, Tuple
+import math
+from typing import Literal, NamedTuple
 
 import numpy as np
+from numpy.typing import ArrayLike, NDArray
+from scipy.spatial.distance import pdist, squareform
 
-from dataeval._internal.functional.coverage import coverage
-from dataeval._internal.interop import ArrayLike, to_numpy
-from dataeval._internal.metrics.base import EvaluateMixin
+from dataeval._internal.interop import to_numpy
 
 
-class Coverage(EvaluateMixin):
+class CoverageOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    indices : np.ndarray
+        Array of uncovered indices
+    radii : np.ndarray
+        Array of critical value radii
+    critical_value : float
+        Radius for coverage
+    """
+
+    indices: NDArray[np.intp]
+    radii: NDArray[np.float64]
+    critical_value: float
+
+
+def coverage(
+    embeddings: ArrayLike,
+    radius_type: Literal["adaptive", "naive"] = "adaptive",
+    k: int = 20,
+    percent: np.float64 = np.float64(0.01),
+) -> CoverageOutput:
     """
     Class for evaluating coverage and identifying images/samples that are in undercovered regions.
 
     Parameters
     ----------
+    embeddings : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
     radius_type : Literal["adaptive", "naive"], default "adaptive"
         The function used to determine radius.
     k: int, default 20
@@ -21,76 +47,59 @@ class Coverage(EvaluateMixin):
     percent: np.float64, default np.float(0.01)
         Percent of observations to be considered uncovered. Only applies to adaptive radius.
 
+    Returns
+    -------
+    CoverageOutput
+        Array of uncovered indices, critical value radii, and the radius for coverage
+
+    Raises
+    ------
+    ValueError
+        If length of embeddings is less than or equal to k
+    ValueError
+        If radius_type is unknown
+
+    Note
+    ----
+    Embeddings should be on the unit interval.
+
+    Example
+    -------
+    >>> coverage(embeddings)
+    CoverageOutput(indices=array([], dtype=int64), radii=array([0.59307666, 0.56956307, 0.56328616, 0.70660265, 0.57778087,
+           0.53738624, 0.58968217, 1.27721334, 0.84378694, 0.67767021,
+           0.69680335, 1.35532621, 0.59764166, 0.8691945 , 0.83627602,
+           0.84187303, 0.62212358, 1.09039732, 0.67956797, 0.60134383,
+           0.83713908, 0.91784263, 1.12901193, 0.73907618, 0.63943983,
+           0.61188447, 0.47872713, 0.57207771, 0.92885883, 0.54750511,
+           0.83015726, 1.20721778, 0.50421928, 0.98312246, 0.59764166,
+           0.61009202, 0.73864073, 1.0381061 , 0.77598609, 0.72984036,
+           0.67573006, 0.48056064, 1.00050879, 0.89532971, 0.58395529,
+           0.95954793, 0.60134383, 1.10096454, 0.51955314, 0.73038702]), critical_value=0)
+
     Reference
     ---------
     This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
     [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
-
-    Examples
-    --------
-    Initialize the Coverage class:
-
-    >>> cover = Coverage()
-
-    Adjusting parameters:
-
-    >>> cover = Coverage(k=5, percent=0.1)
-    """
-
-    def __init__(
-        self,
-        radius_type: Literal["adaptive", "naive"] = "adaptive",
-        k: int = 20,
-        percent: np.float64 = np.float64(0.01),
-    ):
-        self.radius_type: Literal["adaptive", "naive"] = radius_type
-        self.k: int = k
-        self.percent: np.float64 = percent
-
-    def evaluate(self, embeddings: ArrayLike) -> Tuple[np.ndarray, np.ndarray, float]:
-        """
-        Perform a one-way chi-squared test between observation frequencies and expected frequencies that
-        tests the null hypothesis that the observed data has the expected frequencies.
-
-        Parameters
-        ----------
-        embeddings : ArrayLike, shape - (N, P)
-            A dataset in an ArrayLike format.
-            Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
-
-        Returns
-        -------
-        np.ndarray
-            Array of uncovered indices
-        np.ndarray
-            Array of critical value radii
-        float
-            Radius for coverage
-
-        Raises
-        ------
-        ValueError
-            If length of embeddings is less than or equal to k
-        ValueError
-            If radius_type is unknown
-
-        Note
-        ----
-        Embeddings should be on the unit interval.
-
-        Example
-        -------
-        >>> cover.evaluate(embeddings)
-        (array([31,  7, 22, 37, 11]), array([0.35938604, 0.26462789, 0.20319609, 0.34140912, 0.31069921,
-               0.2308378 , 0.33300179, 0.69881025, 0.53587532, 0.35689803,
-               0.39333634, 0.67497874, 0.21788128, 0.43510162, 0.38601861,
-               0.34171868, 0.16941337, 0.66438044, 0.20319609, 0.19732733,
-               0.48660288, 0.5135814 , 0.69352653, 0.26946943, 0.31120605,
-               0.33067705, 0.30508271, 0.32802489, 0.51805702, 0.31120605,
-               0.40843265, 0.74996768, 0.31069921, 0.52263763, 0.26654013,
-               0.33113507, 0.40814838, 0.67723008, 0.48124375, 0.37243185,
-               0.29760001, 0.30907904, 0.59023236, 0.57778087, 0.21839853,
-               0.46067782, 0.31078966, 0.65199049, 0.26410603, 0.19542706]))
-        """
-
-        return coverage(to_numpy(embeddings), self.radius_type, self.k, self.percent)
+    """  # noqa: E501
+
+    # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
+    embeddings = to_numpy(embeddings)
+    n = len(embeddings)
+    if n <= k:
+        raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
+    mat = squareform(pdist(embeddings)).astype(np.float64)
+    sorted_dists = np.sort(mat, axis=1)
+    crit = sorted_dists[:, k + 1]
+
+    d = np.shape(embeddings)[1]
+    if radius_type == "naive":
+        rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
+        pvals = np.where(crit > rho)[0]
+    elif radius_type == "adaptive":
+        # Use data adaptive cutoff as rho
+        rho = int(n * percent)
+        pvals = np.argsort(crit)[::-1][:rho]
+    else:
+        raise ValueError("Invalid radius type.")
+    return CoverageOutput(pvals, crit, rho)

dataeval/_internal/metrics/divergence.py

@@ -3,27 +3,70 @@ This module contains the implementation of HP Divergence
 using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 """
 
-from typing import Any, Callable, Dict, Literal
+from typing import Literal, NamedTuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 
-from dataeval._internal.functional.divergence import divergence_fnn, divergence_mst
-from dataeval._internal.interop import ArrayLike, to_numpy
-from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
 
-_METHODS = Literal["MST", "FNN"]
-_FUNCTION = Callable[[np.ndarray, np.ndarray], int]
+
+class DivergenceOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    divergence : float
+        Divergence value calculated between 2 datasets ranging between 0.0 and 1.0
+    errors : int
+        The number of differing edges between the datasets
+    """
+
+    divergence: float
+    errors: int
+
+
+def divergence_mst(data: np.ndarray, labels: np.ndarray) -> int:
+    mst = minimum_spanning_tree(data).toarray()
+    edgelist = np.transpose(np.nonzero(mst))
+    errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
+    return errors
+
+
+def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
+    nn_indices = compute_neighbors(data, data)
+    errors = np.sum(np.abs(labels[nn_indices] - labels))
+    return errors
 
 
-class Divergence(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}
+
+
+def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
-    Calculates the estimated HP divergence between two datasets
+    Calculates the divergence and any errors between the datasets
 
     Parameters
     ----------
-    method : Literal["MST, "FNN"], default "MST"
+    data_a : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format to compare.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+    data_b : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format to compare.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+    method : Literal["MST, "FNN"], default "FNN"
         Method used to estimate dataset divergence
 
+    Returns
+    -------
+    DivergenceOutput
+        The divergence value (0.0..1.0) and the number of differing edges between the datasets
+
+    Notes
+    -----
+    The divergence value indicates how similar the 2 datasets are
+    with 0 indicating approximately identical data distributions.
+
     Warning
     -------
         MST is very slow in this implementation, this is unlike matlab where
@@ -40,63 +83,20 @@ class Divergence(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
 
     Examples
     --------
-    Initialize the Divergence class:
-
-    >>> divert = Divergence()
+    Evaluate the datasets:
 
-    Specify the method:
-
-    >>> divert = Divergence(method="FNN")
+    >>> divergence(datasetA, datasetB)
+    DivergenceOutput(divergence=0.28, errors=36.0)
     """
-
-    def __init__(self, method: _METHODS = "MST") -> None:
-        self._set_method(method)
-
-    @classmethod
-    def _methods(cls) -> Dict[str, _FUNCTION]:
-        return {"FNN": divergence_fnn, "MST": divergence_mst}
-
-    def evaluate(self, data_a: ArrayLike, data_b: ArrayLike) -> Dict[str, Any]:
-        """
-        Calculates the divergence and any errors between the datasets
-
-        Parameters
-        ----------
-        data_a : ArrayLike, shape - (N, P)
-            A dataset in an ArrayLike format to compare.
-            Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
-        data_b : ArrayLike, shape - (N, P)
-            A dataset in an ArrayLike format to compare.
-            Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
-
-        Returns
-        -------
-        Dict[str, Any]
-            divergence : float
-                divergence value between 0.0 and 1.0
-            error : int
-                the number of differing edges between the datasets
-
-        Notes
-        -----
-        The divergence value indicates how similar the 2 datasets are
-        with 0 indicating approximately identical data distributions.
-
-        Examples
-        --------
-        Evaluate the datasets:
-
-        >>> divert.evaluate(datasetA, datasetB)
-        {'divergence': 0.28, 'error': 36.0}
-        """
-        a = to_numpy(data_a)
-        b = to_numpy(data_b)
-        N = a.shape[0]
-        M = b.shape[0]
-
-        stacked_data = np.vstack((a, b))
-        labels = np.vstack([np.zeros([N, 1]), np.ones([M, 1])])
-
-        errors = self._method(stacked_data, labels)
-        dp = max(0.0, 1 - ((M + N) / (2 * M * N)) * errors)
-        return {"divergence": dp, "error": errors}
+    div_fn = get_method(DIVERGENCE_FN_MAP, method)
+    a = to_numpy(data_a)
+    b = to_numpy(data_b)
+    N = a.shape[0]
+    M = b.shape[0]
+
+    stacked_data = np.vstack((a, b))
+    labels = np.vstack([np.zeros([N, 1]), np.ones([M, 1])])
+
+    errors = div_fn(stacked_data, labels)
+    dp = max(0.0, 1 - ((M + N) / (2 * M * N)) * errors)
+    return DivergenceOutput(dp, errors)