dataeval 0.61.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/_internal/metrics/base.py

@@ -1,92 +1,10 @@
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, Generic, List, TypeVar
+from typing import Generic, TypeVar
 
 TOutput = TypeVar("TOutput", bound=dict)
-TMethods = TypeVar("TMethods")
-TCallable = TypeVar("TCallable", bound=Callable)
-
-
-class MetricMixin(ABC, Generic[TOutput]):
-    @abstractmethod
-    def update(self, preds, targets): ...
-
-    @abstractmethod
-    def compute(self) -> TOutput: ...
-
-    @abstractmethod
-    def reset(self): ...
 
 
 class EvaluateMixin(ABC, Generic[TOutput]):
     @abstractmethod
-    def evaluate(self) -> TOutput:
+    def evaluate(self, *args, **kwargs) -> TOutput:
         """Abstract method to calculate metric based off of constructor parameters"""
-
-
-class MethodsMixin(ABC, Generic[TMethods, TCallable]):
-    """
-    Use this mixin to define a mapping of functions to method names which
-    can be queried by the user and called internally with the appropriate
-    method name as the key.
-
-    Explicitly defining the Callable generic helps with type safety and
-    hinting for function signatures and recommended but optional.
-
-    e.g.:
-
-    def _mult(x: float, y: float) -> float:
-        return x * y
-
-    class MyMetric(MethodsMixin[Callable[float, float], float]):
-
-        def _methods(cls) -> Dict[str, Callable[float, float], float]:
-            return {
-                "ADD": lambda x, y: x + y,
-                "MULT":  _mult,
-                ...
-            }
-
-    Then during evaluate, you can call the method specified with the getter.
-
-    e.g.:
-
-        def evaluate(self):
-            return self._method(x, y)
-
-    The resulting class can be used like so.
-
-    m = MyMetric(1.0, 2.0, "ADD")
-    m.evaluate()       #  returns 3.0
-    m.method           #  returns "ADD"
-    MyMetric.methods() #  returns "['ADD', 'MULT']
-    m.method = "MULT"
-    m.evaluate()       #  returns 2.0
-    """
-
-    @classmethod
-    @abstractmethod
-    def _methods(cls) -> Dict[str, TCallable]:
-        """Abstract method returning available method functions for class"""
-
-    @property
-    def _method(self) -> TCallable:
-        return self._methods()[self.method]
-
-    @classmethod
-    def methods(cls) -> List[str]:
-        return list(cls._methods().keys())
-
-    @property
-    def method(self) -> str:
-        return self._method_key
-
-    @method.setter
-    def method(self, value: TMethods):
-        self._set_method(value)
-
-    def _set_method(self, value: TMethods):
-        """This setter is to fix pyright incorrect detection of
-        incorrectly overriding the 'method' property"""
-        if value not in self.methods():
-            raise KeyError(f"Specified method not available for class ({self.methods()}).")
-        self._method_key = value

dataeval/_internal/metrics/ber.py

@@ -7,19 +7,46 @@ Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4)
 https://arxiv.org/abs/1811.06419
 """
 
-from typing import Callable, Dict, Literal, Tuple
+from typing import Literal, NamedTuple, Tuple
 
 import numpy as np
-from maite.protocols import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 from scipy.sparse import coo_matrix
 from scipy.stats import mode
 
-from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
 
-from .utils import compute_neighbors, get_classes_counts, minimum_spanning_tree
+
+class BEROutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    ber : float
+        The upper bounds of the Bayes Error Rate
+    ber_lower : float
+        The lower bounds of the Bayes Error Rate
+    """
+
+    ber: float
+    ber_lower: float
 
 
-def _mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
+def ber_mst(X: NDArray, y: NDArray) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using a minimum spanning tree
+
+    Parameters
+    ----------
+    X : NDArray, shape - (N, ... )
+        n_samples containing n_features
+    y : NDArray, shape - (N, 1)
+        Labels corresponding to each sample
+
+    Returns
+    -------
+    Tuple[float, float]
+        The upper and lower bounds of the bayes error rate
+    """
     M, N = get_classes_counts(y)
 
     tree = coo_matrix(minimum_spanning_tree(X))
@@ -30,7 +57,21 @@ def _mst(X: np.ndarray, y: np.ndarray, _: int) -> Tuple[float, float]:
     return upper, lower
 
 
-def _knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
+def ber_knn(X: NDArray, y: NDArray, k: int) -> Tuple[float, float]:
+    """Calculates the Bayes Error Rate using K-nearest neighbors
+
+    Parameters
+    ----------
+    X : NDArray, shape - (N, ... )
+        n_samples containing n_features
+    y : NDArray, shape - (N, 1)
+        Labels corresponding to each sample
+
+    Returns
+    -------
+    Tuple[float, float]
+        The upper and lower bounds of the bayes error rate
+    """
     M, N = get_classes_counts(y)
 
     # All features belong on second dimension
@@ -39,12 +80,12 @@ def _knn(X: np.ndarray, y: np.ndarray, k: int) -> Tuple[float, float]:
     nn_indices = np.expand_dims(nn_indices, axis=1) if nn_indices.ndim == 1 else nn_indices
     modal_class = mode(y[nn_indices], axis=1, keepdims=True).mode.squeeze()
     upper = float(np.count_nonzero(modal_class - y) / N)
-    lower = _knn_lowerbound(upper, M, k)
+    lower = knn_lowerbound(upper, M, k)
     return upper, lower
 
 
-def _knn_lowerbound(value: float, classes: int, k: int) -> float:
-    "Several cases for computing the BER lower bound"
+def knn_lowerbound(value: float, classes: int, k: int) -> float:
+    """Several cases for computing the BER lower bound"""
     if value <= 1e-10:
         return 0.0
 
@@ -63,62 +104,45 @@ def _knn_lowerbound(value: float, classes: int, k: int) -> float:
     return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))
 
 
-_METHODS = Literal["MST", "KNN"]
-_FUNCTION = Callable[[np.ndarray, np.ndarray, int], Tuple[float, float]]
+BER_FN_MAP = {"KNN": ber_knn, "MST": ber_mst}
 
 
-class BER(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
     An estimator for Multi-class Bayes Error Rate using FR or KNN test statistic basis
 
     Parameters
     ----------
-    data : np.ndarray
+    images : ArrayLike (N, ... )
         Array of images or image embeddings
-    labels : np.ndarray
+    labels : ArrayLike (N, 1)
         Array of labels for each image or image embedding
-    method : Literal["MST", "KNN"], default "KNN"
-        Method to use when estimating the Bayes error rate
     k : int, default 1
-        number of nearest neighbors for KNN estimator -- ignored by MST estimator
+        Number of nearest neighbors for KNN estimator -- ignored by MST estimator
+    method : Literal["KNN", "MST"], default "KNN"
+        Method to use when estimating the Bayes error rate
 
+    Returns
+    -------
+    BEROutput
+        The upper and lower bounds of the Bayes Error Rate
 
-    See Also
+    References
+    ----------
+    [1] `Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4) <https://arxiv.org/abs/1811.06419>`_
+
+    Examples
     --------
-    `Learning to Bound the Multi-class Bayes Error (Th. 3 and Th. 4) <https://arxiv.org/abs/1811.06419>`_
+    >>> import sklearn.datasets as dsets
+    >>> from dataeval.metrics import ber
 
-    """
+    >>> images, labels = dsets.make_blobs(n_samples=50, centers=2, n_features=2, random_state=0)
 
-    def __init__(self, data: ArrayLike, labels: ArrayLike, method: _METHODS = "KNN", k: int = 1) -> None:
-        self.data = data
-        self.labels = labels
-        self.k = k
-        self._set_method(method)
-
-    @classmethod
-    def _methods(
-        cls,
-    ) -> Dict[str, _FUNCTION]:
-        return {"MST": _mst, "KNN": _knn}
-
-    def evaluate(self) -> Dict[str, float]:
-        """
-        Calculates the Bayes Error Rate estimate using the provided method
-
-        Returns
-        -------
-        Dict[str, float]
-            ber : float
-                The estimated lower bounds of the Bayes Error Rate
-            ber_lower : float
-                The estimated upper bounds of the Bayes Error Rate
-
-        Raises
-        ------
-        ValueError
-            If unique classes M < 2
-        """
-        data = np.asarray(self.data)
-        labels = np.asarray(self.labels)
-        upper, lower = self._method(data, labels, self.k)
-        return {"ber": upper, "ber_lower": lower}
+    >>> ber(images, labels)
+    BEROutput(ber=0.04, ber_lower=0.020416847668728033)
+    """
+    ber_fn = get_method(BER_FN_MAP, method)
+    X = to_numpy(images)
+    y = to_numpy(labels)
+    upper, lower = ber_fn(X, y, k) if method == "KNN" else ber_fn(X, y)
+    return BEROutput(upper, lower)

dataeval/_internal/metrics/coverage.py

@@ -1,80 +1,105 @@
 import math
-from typing import Literal, Tuple
+from typing import Literal, NamedTuple
 
 import numpy as np
+from numpy.typing import ArrayLike, NDArray
 from scipy.spatial.distance import pdist, squareform
 
+from dataeval._internal.interop import to_numpy
 
-class Coverage:
+
+class CoverageOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    indices : np.ndarray
+        Array of uncovered indices
+    radii : np.ndarray
+        Array of critical value radii
+    critical_value : float
+        Radius for coverage
     """
-    Class for evaluating coverage and identifying images/samples that are in undercovered regions.
 
-    This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
+    indices: NDArray[np.intp]
+    radii: NDArray[np.float64]
+    critical_value: float
+
+
+def coverage(
+    embeddings: ArrayLike,
+    radius_type: Literal["adaptive", "naive"] = "adaptive",
+    k: int = 20,
+    percent: np.float64 = np.float64(0.01),
+) -> CoverageOutput:
+    """
+    Class for evaluating coverage and identifying images/samples that are in undercovered regions.
 
     Parameters
     ----------
-    embeddings : np.ndarray
-        n x p array of image embeddings from the dataset.
+    embeddings : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
     radius_type : Literal["adaptive", "naive"], default "adaptive"
         The function used to determine radius.
     k: int, default 20
         Number of observations required in order to be covered.
+        [1] suggests that a minimum of 20-50 samples is necessary.
     percent: np.float64, default np.float(0.01)
         Percent of observations to be considered uncovered. Only applies to adaptive radius.
 
+    Returns
+    -------
+    CoverageOutput
+        Array of uncovered indices, critical value radii, and the radius for coverage
+
+    Raises
+    ------
+    ValueError
+        If length of embeddings is less than or equal to k
+    ValueError
+        If radius_type is unknown
+
     Note
     ----
     Embeddings should be on the unit interval.
-    """
 
-    def __init__(
-        self,
-        embeddings: np.ndarray,
-        radius_type: Literal["adaptive", "naive"] = "adaptive",
-        k: int = 20,
-        percent: np.float64 = np.float64(0.01),
-    ):
-        self.embeddings = embeddings
-        self.radius_type = radius_type
-        self.k = k
-        self.percent = percent
+    Example
+    -------
+    >>> coverage(embeddings)
+    CoverageOutput(indices=array([], dtype=int64), radii=array([0.59307666, 0.56956307, 0.56328616, 0.70660265, 0.57778087,
+           0.53738624, 0.58968217, 1.27721334, 0.84378694, 0.67767021,
+           0.69680335, 1.35532621, 0.59764166, 0.8691945 , 0.83627602,
+           0.84187303, 0.62212358, 1.09039732, 0.67956797, 0.60134383,
+           0.83713908, 0.91784263, 1.12901193, 0.73907618, 0.63943983,
+           0.61188447, 0.47872713, 0.57207771, 0.92885883, 0.54750511,
+           0.83015726, 1.20721778, 0.50421928, 0.98312246, 0.59764166,
+           0.61009202, 0.73864073, 1.0381061 , 0.77598609, 0.72984036,
+           0.67573006, 0.48056064, 1.00050879, 0.89532971, 0.58395529,
+           0.95954793, 0.60134383, 1.10096454, 0.51955314, 0.73038702]), critical_value=0)
 
-    def evaluate(self) -> Tuple[np.ndarray, np.ndarray]:
-        """
-        Perform a one-way chi-squared test between observation frequencies and expected frequencies that
-        tests the null hypothesis that the observed data has the expected frequencies.
-
-        Returns
-        -------
-        np.ndarray
-            Array of uncovered indices
-        np.ndarray
-            Array of critical value radii
-
-        Raises
-        ------
-        ValueError
-            If length of embeddings is less than or equal to k
-        ValueError
-            If radius_type is unknown
-        """
+    Reference
+    ---------
+    This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
+    [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
+    """  # noqa: E501
 
-        # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
-        n = len(self.embeddings)
-        if n <= self.k:
-            raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
-        mat = squareform(pdist(self.embeddings))
-        sorted_dists = np.sort(mat, axis=1)
-        crit = sorted_dists[:, self.k + 1]
+    # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
+    embeddings = to_numpy(embeddings)
+    n = len(embeddings)
+    if n <= k:
+        raise ValueError("Number of observations less than or equal to the specified number of neighbors.")
+    mat = squareform(pdist(embeddings)).astype(np.float64)
+    sorted_dists = np.sort(mat, axis=1)
+    crit = sorted_dists[:, k + 1]
 
-        d = np.shape(self.embeddings)[1]
-        if self.radius_type == "naive":
-            self.rho = (1 / math.sqrt(math.pi)) * ((2 * self.k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
-            pvals = np.where(crit > self.rho)[0]
-        elif self.radius_type == "adaptive":
-            # Use data adaptive cutoff
-            cutoff = int(n * self.percent)
-            pvals = np.argsort(crit)[::-1][:cutoff]
-        else:
-            raise ValueError("Invalid radius type.")
-        return pvals, crit
+    d = np.shape(embeddings)[1]
+    if radius_type == "naive":
+        rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
+        pvals = np.where(crit > rho)[0]
+    elif radius_type == "adaptive":
+        # Use data adaptive cutoff as rho
+        rho = int(n * percent)
+        pvals = np.argsort(crit)[::-1][:rho]
+    else:
+        raise ValueError("Invalid radius type.")
+    return CoverageOutput(pvals, crit, rho)

dataeval/_internal/metrics/divergence.py

@@ -3,49 +3,69 @@ This module contains the implementation of HP Divergence
 using the Fast Nearest Neighbor and Minimum Spanning Tree algorithms
 """
 
-from typing import Any, Callable, Dict, Literal
+from typing import Literal, NamedTuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 
-from dataeval._internal.metrics.base import EvaluateMixin, MethodsMixin
+from dataeval._internal.interop import to_numpy
+from dataeval._internal.metrics.utils import compute_neighbors, get_method, minimum_spanning_tree
 
-from .utils import compute_neighbors, minimum_spanning_tree
 
+class DivergenceOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    divergence : float
+        Divergence value calculated between 2 datasets ranging between 0.0 and 1.0
+    errors : int
+        The number of differing edges between the datasets
+    """
+
+    divergence: float
+    errors: int
 
-def _mst(data: np.ndarray, labels: np.ndarray) -> int:
+
+def divergence_mst(data: np.ndarray, labels: np.ndarray) -> int:
     mst = minimum_spanning_tree(data).toarray()
     edgelist = np.transpose(np.nonzero(mst))
     errors = np.sum(labels[edgelist[:, 0]] != labels[edgelist[:, 1]])
     return errors
 
 
-def _fnn(data: np.ndarray, labels: np.ndarray) -> int:
+def divergence_fnn(data: np.ndarray, labels: np.ndarray) -> int:
     nn_indices = compute_neighbors(data, data)
     errors = np.sum(np.abs(labels[nn_indices] - labels))
     return errors
 
 
-_METHODS = Literal["MST", "FNN"]
-_FUNCTION = Callable[[np.ndarray, np.ndarray], int]
+DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}
 
 
-class Divergence(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
+def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
-    Calculates the estimated divergence between two datasets
+    Calculates the divergence and any errors between the datasets
 
     Parameters
     ----------
-    data_a : np.ndarray
-        Array of images or image embeddings to compare
-    data_b : np.ndarray
-        Array of images or image embeddings to compare
-    method : Literal["MST, "FNN"], default "MST"
+    data_a : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format to compare.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+    data_b : ArrayLike, shape - (N, P)
+        A dataset in an ArrayLike format to compare.
+        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
+    method : Literal["MST, "FNN"], default "FNN"
         Method used to estimate dataset divergence
 
-    See Also
-    --------
-        For more information about this divergence, its formal definition,
-        and its associated estimators see https://arxiv.org/abs/1412.6534.
+    Returns
+    -------
+    DivergenceOutput
+        The divergence value (0.0..1.0) and the number of differing edges between the datasets
+
+    Notes
+    -----
+    The divergence value indicates how similar the 2 datasets are
+    with 0 indicating approximately identical data distributions.
 
     Warning
     -------
@@ -55,40 +75,28 @@ class Divergence(EvaluateMixin, MethodsMixin[_METHODS, _FUNCTION]):
         Source of slowdown:
         conversion to and from CSR format adds ~10% of the time diff between
         1nn and scipy mst function the remaining 90%
-    """
 
-    def __init__(
-        self,
-        data_a: np.ndarray,
-        data_b: np.ndarray,
-        method: _METHODS = "MST",
-    ) -> None:
-        self.data_a = data_a
-        self.data_b = data_b
-        self._set_method(method)
-
-    @classmethod
-    def _methods(cls) -> Dict[str, _FUNCTION]:
-        return {"FNN": _fnn, "MST": _mst}
-
-    def evaluate(self) -> Dict[str, Any]:
-        """
-        Calculates the divergence and any errors between the datasets
-
-        Returns
-        -------
-        Dict[str, Any]
-            dp : float
-                divergence value between 0.0 and 1.0
-            errors : int
-                the number of differing edges
-        """
-        N = self.data_a.shape[0]
-        M = self.data_b.shape[0]
-
-        stacked_data = np.vstack((self.data_a, self.data_b))
-        labels = np.vstack([np.zeros([N, 1]), np.ones([M, 1])])
-
-        errors = self._method(stacked_data, labels)
-        dp = max(0.0, 1 - ((M + N) / (2 * M * N)) * errors)
-        return {"divergence": dp, "error": errors}
+    References
+    ----------
+    For more information about this divergence, its formal definition,
+    and its associated estimators see https://arxiv.org/abs/1412.6534.
+
+    Examples
+    --------
+    Evaluate the datasets:
+
+    >>> divergence(datasetA, datasetB)
+    DivergenceOutput(divergence=0.28, errors=36.0)
+    """
+    div_fn = get_method(DIVERGENCE_FN_MAP, method)
+    a = to_numpy(data_a)
+    b = to_numpy(data_b)
+    N = a.shape[0]
+    M = b.shape[0]
+
+    stacked_data = np.vstack((a, b))
+    labels = np.vstack([np.zeros([N, 1]), np.ones([M, 1])])
+
+    errors = div_fn(stacked_data, labels)
+    dp = max(0.0, 1 - ((M + N) / (2 * M * N)) * errors)
+    return DivergenceOutput(dp, errors)