dataeval 0.76.0__py3-none-any.whl → 0.81.0__py3-none-any.whl

dataeval/metrics/bias/{parity.py → _parity.py}

@@ -2,40 +2,86 @@ from __future__ import annotations
 
 __all__ = []
 
+import contextlib
 import warnings
 from dataclasses import dataclass
 from typing import Any, Generic, TypeVar
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 from scipy.stats import chisquare
 from scipy.stats.contingency import chi2_contingency, crosstab
 
-from dataeval.interop import as_numpy, to_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.metadata import Metadata
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy
+from dataeval.utils.data import Metadata
+
+with contextlib.suppress(ImportError):
+    import pandas as pd
 
 TData = TypeVar("TData", np.float64, NDArray[np.float64])
 
 
 @dataclass(frozen=True)
-class ParityOutput(Generic[TData], Output):
+class BaseParityOutput(Generic[TData], Output):
+    score: TData
+    p_value: TData
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Exports the parity output results to a pandas DataFrame.
+
+        Returns
+        -------
+        pd.DataFrame
+        """
+        import pandas as pd
+
+        return pd.DataFrame(
+            index=self.factor_names,  # type: ignore - list[str] is documented as acceptable index type
+            data={
+                "score": self.score.round(2),
+                "p-value": self.p_value.round(2),
+            },
+        )
+
+
+@dataclass(frozen=True)
+class LabelParityOutput(BaseParityOutput[np.float64]):
+    """
+    Output class for :func:`.label_parity` :term:`bias<Bias>` metrics.
+
+    Attributes
+    ----------
+    score : np.float64
+        chi-squared score(s) of the test
+    p_value : np.float64
+        p-value(s) of the test
+    """
+
+
+@dataclass(frozen=True)
+class ParityOutput(BaseParityOutput[NDArray[np.float64]]):
     """
-    Output class for :func:`parity` and :func:`label_parity` :term:`bias<Bias>` metrics.
+    Output class for :func:`.parity` :term:`bias<Bias>` metrics.
 
     Attributes
     ----------
-    score : np.float64 | NDArray[np.float64]
+    score : NDArray[np.float64]
         chi-squared score(s) of the test
-    p_value : np.float64 | NDArray[np.float64]
+    p_value : NDArray[np.float64]
         p-value(s) of the test
-    metadata_names : list[str] | None
+    factor_names : list[str]
         Names of each metadata factor
+    insufficient_data: dict
+        Dictionary of metadata factors with less than 5 class occurrences per value
     """
 
-    score: TData
-    p_value: TData
-    metadata_names: list[str] | None
+    # score: NDArray[np.float64]
+    # p_value: NDArray[np.float64]
+    factor_names: list[str]
+    insufficient_data: dict[str, dict[int, dict[str, int]]]
 
 
 def normalize_expected_dist(expected_dist: NDArray[Any], observed_dist: NDArray[Any]) -> NDArray[Any]:
@@ -109,7 +155,7 @@ def validate_dist(label_dist: NDArray[Any], label_name: str) -> None:
         raise ValueError(f"No labels found in the {label_name} dataset")
     if np.any(label_dist < 5):
         warnings.warn(
-            f"Labels {np.where(label_dist<5)[0]} in {label_name}"
+            f"Labels {np.where(label_dist < 5)[0]} in {label_name}"
             " dataset have frequencies less than 5. This may lead"
             " to invalid chi-squared evaluation.",
             UserWarning,
@@ -121,7 +167,7 @@ def label_parity(
     expected_labels: ArrayLike,
     observed_labels: ArrayLike,
     num_classes: int | None = None,
-) -> ParityOutput[np.float64]:
+) -> LabelParityOutput:
     """
     Calculate the chi-square statistic to assess the :term:`parity<Parity>` \
     between expected and observed label distributions.
@@ -142,7 +188,7 @@ def label_parity(
 
     Returns
     -------
-    ParityOutput[np.float64]
+    LabelParityOutput
         chi-squared score and :term`P-Value` of the test
 
     Raises
@@ -171,7 +217,7 @@ def label_parity(
     >>> expected_labels = rng.choice([0, 1, 2, 3, 4], (100))
     >>> observed_labels = rng.choice([2, 3, 0, 4, 1], (100))
     >>> label_parity(expected_labels, observed_labels)
-    ParityOutput(score=14.007374204742625, p_value=0.0072715574616218, metadata_names=None)
+    LabelParityOutput(score=14.007374204742625, p_value=0.0072715574616218)
     """
 
     # Calculate
@@ -179,8 +225,8 @@ def label_parity(
         num_classes = 0
 
     # Calculate the class frequencies associated with the datasets
-    observed_dist = np.bincount(to_numpy(observed_labels), minlength=num_classes)
-    expected_dist = np.bincount(to_numpy(expected_labels), minlength=num_classes)
+    observed_dist = np.bincount(as_numpy(observed_labels), minlength=num_classes)
+    expected_dist = np.bincount(as_numpy(expected_labels), minlength=num_classes)
 
     # Validate
     validate_dist(observed_dist, "observed")
@@ -202,11 +248,11 @@ def label_parity(
         )
 
     cs, p = chisquare(f_obs=observed_dist, f_exp=expected_dist)
-    return ParityOutput(cs, p, None)
+    return LabelParityOutput(cs, p)
 
 
 @set_metadata
-def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
+def parity(metadata: Metadata) -> ParityOutput:
     """
     Calculate chi-square statistics to assess the linear relationship \
     between multiple factors and class labels.
@@ -218,7 +264,7 @@ def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
     Parameters
     ----------
     metadata : Metadata
-        Preprocessed metadata from :func:`dataeval.utils.metadata.preprocess`
+        Preprocessed metadata
 
     Returns
     -------
@@ -250,24 +296,21 @@ def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
     --------
     Randomly creating some "continuous" and categorical variables using ``np.random.default_rng``
 
-    >>> from dataeval.utils.metadata import preprocess
-    >>> rng = np.random.default_rng(175)
-    >>> labels = rng.choice([0, 1, 2], (100))
-    >>> metadata_dict = [
-    ...     {
-    ...         "age": list(rng.choice([25, 30, 35, 45], (100))),
-    ...         "income": list(rng.choice([50000, 65000, 80000], (100))),
-    ...         "gender": list(rng.choice(["M", "F"], (100))),
-    ...     }
-    ... ]
-    >>> continuous_factor_bincounts = {"age": 4, "income": 3}
-    >>> metadata = preprocess(metadata_dict, labels, continuous_factor_bincounts)
+    >>> metadata = generate_random_metadata(
+    ...     labels=["doctor", "artist", "teacher"],
+    ...     factors={
+    ...         "age": [25, 30, 35, 45],
+    ...         "income": [50000, 65000, 80000],
+    ...         "gender": ["M", "F"]},
+    ...     length=100,
+    ...     random_seed=175)
+    >>> metadata.continuous_factor_bins = {"age": 4, "income": 3}
     >>> parity(metadata)
-    ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]), metadata_names=['age', 'income', 'gender'])
+    ParityOutput(score=array([7.35731943, 5.46711299, 0.51506212]), p_value=array([0.28906231, 0.24263543, 0.77295762]), factor_names=['age', 'income', 'gender'], insufficient_data={'age': {3: {'artist': 4}, 4: {'artist': 4, 'teacher': 3}}, 'income': {1: {'artist': 3}}})
     """  # noqa: E501
     chi_scores = np.zeros(metadata.discrete_data.shape[1])
     p_values = np.zeros_like(chi_scores)
-    not_enough_data = {}
+    insufficient_data = {}
     for i, col_data in enumerate(metadata.discrete_data.T):
         # Builds a contingency matrix where entry at index (r,c) represents
         # the frequency of current_factor_name achieving value unique_factor_values[r]
@@ -281,14 +324,14 @@ def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
         current_factor_name = metadata.discrete_factor_names[i]
         for int_factor, int_class in zip(counts[0], counts[1]):
             if contingency_matrix[int_factor, int_class] > 0:
-                factor_category = unique_factor_values[int_factor]
-                if current_factor_name not in not_enough_data:
-                    not_enough_data[current_factor_name] = {}
-                if factor_category not in not_enough_data[current_factor_name]:
-                    not_enough_data[current_factor_name][factor_category] = []
-                not_enough_data[current_factor_name][factor_category].append(
-                    (metadata.class_names[int_class], int(contingency_matrix[int_factor, int_class]))
-                )
+                factor_category = unique_factor_values[int_factor].item()
+                if current_factor_name not in insufficient_data:
+                    insufficient_data[current_factor_name] = {}
+                if factor_category not in insufficient_data[current_factor_name]:
+                    insufficient_data[current_factor_name][factor_category] = {}
+                class_name = metadata.class_names[int_class]
+                class_count = contingency_matrix[int_factor, int_class].item()
+                insufficient_data[current_factor_name][factor_category][class_name] = class_count
 
         # This deletes rows containing only zeros,
         # because scipy.stats.chi2_contingency fails when there are rows containing only zeros.
@@ -301,24 +344,7 @@ def parity(metadata: Metadata) -> ParityOutput[NDArray[np.float64]]:
         chi_scores[i] = chi2
         p_values[i] = p
 
-    if not_enough_data:
-        factor_msg = []
-        for factor, fact_dict in not_enough_data.items():
-            stacked_msg = []
-            for key, value in fact_dict.items():
-                msg = []
-                for item in value:
-                    msg.append(f"label {item[0]}: {item[1]} occurrences")
-                flat_msg = "\n\t\t".join(msg)
-                stacked_msg.append(f"value {key} - {flat_msg}\n\t")
-            factor_msg.append(factor + " - " + "".join(stacked_msg))
-
-        message = "\n".join(factor_msg)
-
-        warnings.warn(
-            f"The following factors did not meet the recommended 5 occurrences for each value-label combination. \n\
-            Recommend rerunning parity after adjusting the following factor-value-label combinations: \n{message}",
-            UserWarning,
-        )
+    if insufficient_data:
+        warnings.warn("Some factors did not meet the recommended 5 occurrences for each value-label combination.")
 
-    return ParityOutput(chi_scores, p_values, metadata.discrete_factor_names)
+    return ParityOutput(chi_scores, p_values, metadata.discrete_factor_names, insufficient_data)

dataeval/metrics/estimators/__init__.py

@@ -2,8 +2,18 @@
 Estimators calculate performance bounds and the statistical distance between datasets.
 """
 
-__all__ = ["ber", "divergence", "uap", "BEROutput", "DivergenceOutput", "UAPOutput"]
+__all__ = [
+    "ber",
+    "clusterer",
+    "divergence",
+    "uap",
+    "BEROutput",
+    "ClustererOutput",
+    "DivergenceOutput",
+    "UAPOutput",
+]
 
-from dataeval.metrics.estimators.ber import BEROutput, ber
-from dataeval.metrics.estimators.divergence import DivergenceOutput, divergence
-from dataeval.metrics.estimators.uap import UAPOutput, uap
+from dataeval.metrics.estimators._ber import BEROutput, ber
+from dataeval.metrics.estimators._clusterer import ClustererOutput, clusterer
+from dataeval.metrics.estimators._divergence import DivergenceOutput, divergence
+from dataeval.metrics.estimators._uap import UAPOutput, uap

dataeval/metrics/estimators/{ber.py → _ber.py}

@@ -16,19 +16,21 @@ from dataclasses import dataclass
 from typing import Literal
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 from scipy.sparse import coo_matrix
 from scipy.stats import mode
 
-from dataeval.interop import as_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.shared import compute_neighbors, get_classes_counts, get_method, minimum_spanning_tree
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy, ensure_embeddings
+from dataeval.utils._method import get_method
+from dataeval.utils._mst import compute_neighbors, minimum_spanning_tree
 
 
 @dataclass(frozen=True)
 class BEROutput(Output):
     """
-    Output class for :func:`ber` estimator metric.
+    Output class for :func:`.ber` estimator metric.
 
     Attributes
     ----------
@@ -116,18 +118,21 @@ def knn_lowerbound(value: float, classes: int, k: int) -> float:
     return ((classes - 1) / classes) * (1 - np.sqrt(max(0, 1 - ((classes / (classes - 1)) * value))))
 
 
+_BER_FN_MAP = {"KNN": ber_knn, "MST": ber_mst}
+
+
 @set_metadata
-def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
+def ber(embeddings: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN", "MST"] = "KNN") -> BEROutput:
     """
     An estimator for Multi-class :term:`Bayes error rate<Bayes Error Rate (BER)>` \
     using FR or KNN test statistic basis.
 
     Parameters
     ----------
-    images : ArrayLike (N, ... )
-        Array of images or image :term:`embeddings<Embeddings>`
+    embeddings : ArrayLike (N, ... )
+        Array of image :term:`embeddings<Embeddings>`
     labels : ArrayLike (N, 1)
-        Array of labels for each image or image embedding
+        Array of labels for each image
     k : int, default 1
         Number of nearest neighbors for KNN estimator -- ignored by MST estimator
     method : Literal["KNN", "MST"], default "KNN"
@@ -152,8 +157,34 @@ def ber(images: ArrayLike, labels: ArrayLike, k: int = 1, method: Literal["KNN",
     >>> ber(images, labels)
     BEROutput(ber=0.04, ber_lower=0.020416847668728033)
     """
-    ber_fn = get_method({"KNN": ber_knn, "MST": ber_mst}, method)
-    X = as_numpy(images)
+    ber_fn = get_method(_BER_FN_MAP, method)
+    X = ensure_embeddings(embeddings, dtype=np.float64)
     y = as_numpy(labels)
     upper, lower = ber_fn(X, y, k)
     return BEROutput(upper, lower)
+
+
+def get_classes_counts(labels: NDArray[np.int_]) -> tuple[int, int]:
+    """
+    Returns the classes and counts of from an array of labels
+
+    Parameters
+    ----------
+    label : NDArray
+        Numpy labels array
+
+    Returns
+    -------
+        Classes and counts
+
+    Raises
+    ------
+    ValueError
+        If the number of unique classes is less than 2
+    """
+    classes, counts = np.unique(labels, return_counts=True)
+    M = len(classes)
+    if M < 2:
+        raise ValueError("Label vector contains less than 2 classes!")
+    N = int(np.sum(counts))
+    return M, N

dataeval/metrics/estimators/_clusterer.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+__all__ = []
+
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval._output import Output
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy
+
+
+@dataclass(frozen=True)
+class ClustererOutput(Output):
+    """
+    Output class for :func:`.clusterer`.
+
+    Attributes
+    ----------
+    clusters : NDArray[int]
+        Assigned clusters
+    mst : NDArray[int]
+        The minimum spanning tree of the data
+    linkage_tree : NDArray[float]
+        The linkage array of the data
+    condensed_tree : NDArray[float]
+        The condensed tree of the data
+    membership_strengths : NDArray[float]
+        The strength of the data point belonging to the assigned cluster
+    """
+
+    clusters: NDArray[np.int_]
+    mst: NDArray[np.double]
+    linkage_tree: NDArray[np.double]
+    condensed_tree: NDArray[np.double]
+    membership_strengths: NDArray[np.double]
+
+    def find_outliers(self) -> NDArray[np.int_]:
+        """
+        Retrieves Outliers based on when the sample was added to the cluster
+        and how far it was from the cluster when it was added
+
+        Returns
+        -------
+        NDArray[int]
+            A numpy array of the outlier indices
+        """
+        return np.nonzero(self.clusters == -1)[0]
+
+    def find_duplicates(self) -> tuple[list[list[int]], list[list[int]]]:
+        """
+        Finds duplicate and near duplicate data based on cluster average distance
+
+        Returns
+        -------
+        Tuple[List[List[int]], List[List[int]]]
+            The exact :term:`duplicates<Duplicates>` and near duplicates as lists of related indices
+        """
+        # Delay load numba compiled functions
+        from dataeval.utils._clusterer import compare_links_to_cluster_std, sorted_union_find
+
+        exact_indices, near_indices = compare_links_to_cluster_std(self.mst, self.clusters)
+        exact_dupes = sorted_union_find(exact_indices)
+        near_dupes = sorted_union_find(near_indices)
+
+        return [[int(ii) for ii in il] for il in exact_dupes], [[int(ii) for ii in il] for il in near_dupes]
+
+
+def clusterer(data: ArrayLike) -> ClustererOutput:
+    """
+    Uses hierarchical clustering on the flattened data and returns clustering
+    information.
+
+    Parameters
+    ----------
+    data : ArrayLike, shape - (N, ...)
+        A dataset in an ArrayLike format. Function expects the data to have 2
+        or more dimensions which will flatten to (N, P) where N number of
+        observations in a P-dimensional space.
+
+    Returns
+    -------
+    :class:`.ClustererOutput`
+
+    Note
+    ----
+    The clusterer works best when the length of the feature dimension, P, is
+    less than 500. If flattening a CxHxW image results in a dimension larger
+    than 500, then it is recommended to reduce the dimensions.
+
+    Example
+    -------
+    >>> clusterer(clusterer_images).clusters
+    array([ 2,  0,  0,  0,  0,  0,  4,  0,  3,  1,  1,  0,  2,  0,  0,  0,  0,
+            4,  2,  0,  0,  1,  2,  0,  1,  3,  0,  3,  3,  4,  0,  0,  3,  0,
+            3, -1,  0,  0,  2,  4,  3,  4,  0,  1,  0, -1,  3,  0,  0,  0])
+    """
+    # Delay load numba compiled functions
+    from dataeval.utils._clusterer import cluster
+
+    c = cluster(data)
+    return ClustererOutput(c.clusters, c.mst, c.linkage_tree, as_numpy(c.condensed_tree), c.membership_strengths)

dataeval/metrics/estimators/{divergence.py → _divergence.py}

@@ -11,17 +11,19 @@ from dataclasses import dataclass
 from typing import Literal
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 
-from dataeval.interop import as_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.shared import compute_neighbors, get_method, minimum_spanning_tree
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import ensure_embeddings
+from dataeval.utils._method import get_method
+from dataeval.utils._mst import compute_neighbors, minimum_spanning_tree
 
 
 @dataclass(frozen=True)
 class DivergenceOutput(Output):
     """
-    Output class for :func:`divergence` estimator metric.
+    Output class for :func:`.divergence` estimator metric.
 
     Attributes
     ----------
@@ -78,18 +80,21 @@ def divergence_fnn(data: NDArray[np.float64], labels: NDArray[np.int_]) -> int:
     return errors
 
 
+_DIVERGENCE_FN_MAP = {"FNN": divergence_fnn, "MST": divergence_mst}
+
+
 @set_metadata
-def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
+def divergence(emb_a: ArrayLike, emb_b: ArrayLike, method: Literal["FNN", "MST"] = "FNN") -> DivergenceOutput:
     """
     Calculates the :term:`divergence` and any errors between the datasets.
 
     Parameters
     ----------
-    data_a : ArrayLike, shape - (N, P)
-        A dataset in an ArrayLike format to compare.
+    emb_a : ArrayLike, shape - (N, P)
+        Image embeddings in an ArrayLike format to compare.
         Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
-    data_b : ArrayLike, shape - (N, P)
-        A dataset in an ArrayLike format to compare.
+    emb_b : ArrayLike, shape - (N, P)
+        Image embeddings in an ArrayLike format to compare.
         Function expects the data to have 2 dimensions, N number of observations in a P-dimensionial space.
     method : Literal["MST, "FNN"], default "FNN"
         Method used to estimate dataset :term:`divergence<Divergence>`
@@ -125,9 +130,9 @@ def divergence(data_a: ArrayLike, data_b: ArrayLike, method: Literal["FNN", "MST
     >>> divergence(datasetA, datasetB)
     DivergenceOutput(divergence=0.28, errors=36)
     """
-    div_fn = get_method({"FNN": divergence_fnn, "MST": divergence_mst}, method)
-    a = as_numpy(data_a)
-    b = as_numpy(data_b)
+    div_fn = get_method(_DIVERGENCE_FN_MAP, method)
+    a = ensure_embeddings(emb_a, dtype=np.float64)
+    b = ensure_embeddings(emb_b, dtype=np.float64)
     N = a.shape[0]
     M = b.shape[0]
 

dataeval/metrics/estimators/{uap.py → _uap.py}

@@ -10,17 +10,17 @@ __all__ = []
 
 from dataclasses import dataclass
 
-from numpy.typing import ArrayLike
 from sklearn.metrics import average_precision_score
 
-from dataeval.interop import as_numpy
-from dataeval.output import Output, set_metadata
+from dataeval._output import Output, set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy
 
 
 @dataclass(frozen=True)
 class UAPOutput(Output):
     """
-    Output class for :func:`uap` estimator metric.
+    Output class for :func:`.uap` estimator metric.
 
     Attributes
     ----------

dataeval/metrics/stats/__init__.py

@@ -21,15 +21,15 @@ __all__ = [
     "visualstats",
 ]
 
-from dataeval.metrics.stats.boxratiostats import boxratiostats
-from dataeval.metrics.stats.datasetstats import (
+from dataeval.metrics.stats._boxratiostats import boxratiostats
+from dataeval.metrics.stats._datasetstats import (
     ChannelStatsOutput,
     DatasetStatsOutput,
     channelstats,
     datasetstats,
 )
-from dataeval.metrics.stats.dimensionstats import DimensionStatsOutput, dimensionstats
-from dataeval.metrics.stats.hashstats import HashStatsOutput, hashstats
-from dataeval.metrics.stats.labelstats import LabelStatsOutput, labelstats
-from dataeval.metrics.stats.pixelstats import PixelStatsOutput, pixelstats
-from dataeval.metrics.stats.visualstats import VisualStatsOutput, visualstats
+from dataeval.metrics.stats._dimensionstats import DimensionStatsOutput, dimensionstats
+from dataeval.metrics.stats._hashstats import HashStatsOutput, hashstats
+from dataeval.metrics.stats._labelstats import LabelStatsOutput, labelstats
+from dataeval.metrics.stats._pixelstats import PixelStatsOutput, pixelstats
+from dataeval.metrics.stats._visualstats import VisualStatsOutput, visualstats