dataeval 0.82.0__py3-none-any.whl → 0.83.0__py3-none-any.whl

dataeval/utils/data/selections/_prioritize.py

@@ -0,0 +1,296 @@
+from __future__ import annotations
+
+__all__ = []
+
+import logging
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any, Literal, overload
+
+import numpy as np
+import torch
+from numpy.typing import NDArray
+from sklearn.cluster import KMeans
+from sklearn.metrics import pairwise_distances
+
+from dataeval.config import EPSILON, DeviceLike, get_seed
+from dataeval.utils.data import Embeddings, Select
+from dataeval.utils.data._selection import Selection, SelectionStage
+
+_logger = logging.getLogger(__name__)
+
+
+class _Clusters:
+    __slots__ = ["labels", "cluster_centers", "unique_labels"]
+
+    labels: NDArray[np.intp]
+    cluster_centers: NDArray[np.float64]
+    unique_labels: NDArray[np.intp]
+
+    def __init__(self, labels: NDArray[np.intp], cluster_centers: NDArray[np.float64]) -> None:
+        self.labels = labels
+        self.cluster_centers = cluster_centers
+        self.unique_labels = np.unique(labels)
+
+    def _dist2center(self, X: NDArray[np.float64]) -> NDArray[np.float64]:
+        dist = np.zeros(self.labels.shape)
+        for lab in self.unique_labels:
+            dist[self.labels == lab] = np.linalg.norm(X[self.labels == lab, :] - self.cluster_centers[lab, :], axis=1)
+        return dist
+
+    def _complexity(self, X: NDArray[np.float64]) -> NDArray[np.float64]:
+        num_clst_intra = int(np.maximum(np.minimum(int(self.unique_labels.shape[0] / 5), 20), 1))
+        d_intra = np.zeros(self.unique_labels.shape)
+        d_inter = np.zeros(self.unique_labels.shape)
+        for cdx, lab in enumerate(self.unique_labels):
+            d_intra[cdx] = np.mean(np.linalg.norm(X[self.labels == lab, :] - self.cluster_centers[cdx, :], axis=1))
+            d_inter[cdx] = np.mean(
+                np.linalg.norm(self.cluster_centers - self.cluster_centers[cdx, :], axis=1)[:num_clst_intra]
+            )
+        cj = d_intra * d_inter
+        tau = 0.1
+        exp = np.exp(cj / tau)
+        prob: NDArray[np.float64] = exp / np.sum(exp)
+        return prob
+
+    def _sort_by_weights(self, X: NDArray[np.float64]) -> NDArray[np.intp]:
+        pr = self._complexity(X)
+        d2c = self._dist2center(X)
+        inds_per_clst: list[NDArray[np.intp]] = []
+        for lab in zip(self.unique_labels):
+            inds = np.nonzero(self.labels == lab)[0]
+            # 'hardest' first
+            srt_inds = np.argsort(d2c[inds])[::-1]
+            inds_per_clst.append(inds[srt_inds])
+        glob_inds: list[NDArray[np.intp]] = []
+        while not bool(np.all([arr.size == 0 for arr in inds_per_clst])):
+            clst_ind = np.random.choice(self.unique_labels, 1, p=pr)[0]
+            if inds_per_clst[clst_ind].size > 0:
+                glob_inds.append(inds_per_clst[clst_ind][0])
+            else:
+                continue
+            inds_per_clst[clst_ind] = inds_per_clst[clst_ind][1:]
+        # sorted hardest first; reverse for consistency
+        return np.array(glob_inds[::-1])
+
+
+class _Sorter(ABC):
+    @abstractmethod
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]: ...
+
+
+class _KNNSorter(_Sorter):
+    def __init__(self, samples: int, k: int | None) -> None:
+        if k is None or k <= 0:
+            k = int(np.sqrt(samples))
+            _logger._log(logging.INFO, f"Setting k to default value of {k}", {"k": k, "samples": samples})
+        elif k >= samples:
+            raise ValueError(f"k={k} should be less than dataset size ({samples})")
+        elif k >= samples / 10 and k > np.sqrt(samples):
+            warnings.warn(
+                f"Variable k={k} is large with respect to dataset size but valid; "
+                + f"a nominal recommendation is k={int(np.sqrt(samples))}"
+            )
+        self._k = k
+
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        if reference is None:
+            dists = pairwise_distances(embeddings, embeddings)
+            np.fill_diagonal(dists, np.inf)
+        else:
+            dists = pairwise_distances(embeddings, reference)
+        inds = np.argsort(np.sort(dists, axis=1)[:, self._k])
+        return inds
+
+
+class _KMeansSorter(_Sorter):
+    def __init__(self, samples: int, c: int | None, n_init: int | Literal["auto", "warn"] = "auto") -> None:
+        if c is None or c <= 0:
+            c = int(np.sqrt(samples))
+            _logger._log(logging.INFO, f"Setting the value of num_clusters to a default value of {c}", {})
+        if c >= samples:
+            raise ValueError(f"c={c} should be less than dataset size ({samples})")
+        self._c = c
+        self._n_init = n_init
+
+    def _get_clusters(self, embeddings: NDArray[Any]) -> _Clusters:
+        clst = KMeans(n_clusters=self._c, init="k-means++", n_init=self._n_init, random_state=get_seed())  # type: ignore - n_init allows int but is typed as str
+        clst.fit(embeddings)
+        if clst.labels_ is None or clst.cluster_centers_ is None:
+            raise ValueError("Clustering failed to produce labels or cluster centers")
+        return _Clusters(clst.labels_, clst.cluster_centers_)
+
+
+class _KMeansDistanceSorter(_KMeansSorter):
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        clst = self._get_clusters(embeddings if reference is None else reference)
+        inds = np.argsort(clst._dist2center(embeddings))
+        return inds
+
+
+class _KMeansComplexitySorter(_KMeansSorter):
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        clst = self._get_clusters(embeddings if reference is None else reference)
+        inds = clst._sort_by_weights(embeddings)
+        return inds
+
+
+class Prioritize(Selection[Any]):
+    """
+    Prioritizes the dataset by sort order in the embedding space.
+
+    Parameters
+    ----------
+    model : torch.nn.Module
+        Model to use for encoding images
+    batch_size : int
+        Batch size to use when encoding images
+    device : DeviceLike or None
+        Device to use for encoding images
+    method : Literal["knn", "kmeans_distance", "kmeans_complexity"]
+        Method to use for prioritization
+    k : int | None, default None
+        Number of nearest neighbors to use for prioritization (knn only)
+    c : int | None, default None
+        Number of clusters to use for prioritization (kmeans only)
+    """
+
+    stage = SelectionStage.ORDER
+
+    @overload
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["knn"],
+        *,
+        k: int | None = None,
+    ) -> None: ...
+
+    @overload
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["kmeans_distance", "kmeans_complexity"],
+        *,
+        c: int | None = None,
+    ) -> None: ...
+
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["knn", "kmeans_distance", "kmeans_complexity"],
+        *,
+        k: int | None = None,
+        c: int | None = None,
+    ) -> None:
+        if method not in ("knn", "kmeans_distance", "kmeans_complexity"):
+            raise ValueError(f"Invalid prioritization method: {method}")
+        self._model = model
+        self._batch_size = batch_size
+        self._device = device
+        self._method = method
+        self._embeddings: Embeddings | None = None
+        self._reference: Embeddings | None = None
+        self._k = k
+        self._c = c
+
+    @overload
+    @classmethod
+    def using(
+        cls,
+        method: Literal["knn"],
+        *,
+        k: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize: ...
+
+    @overload
+    @classmethod
+    def using(
+        cls,
+        method: Literal["kmeans_distance", "kmeans_complexity"],
+        *,
+        c: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize: ...
+
+    @classmethod
+    def using(
+        cls,
+        method: Literal["knn", "kmeans_distance", "kmeans_complexity"],
+        *,
+        k: int | None = None,
+        c: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize:
+        """
+        Prioritizes the dataset by sort order in the embedding space using existing
+        embeddings and/or reference dataset embeddings.
+
+        Parameters
+        ----------
+        method : Literal["knn", "kmeans_distance", "kmeans_complexity"]
+            Method to use for prioritization
+        embeddings : Embeddings or None, default None
+            Embeddings to use for prioritization
+        reference : Embeddings or None, default None
+            Reference embeddings to prioritize relative to
+        k : int or None, default None
+            Number of nearest neighbors to use for prioritization (knn only)
+        c : int or None, default None
+            Number of clusters to use for prioritization (kmeans, cluster only)
+
+        Notes
+        -----
+        At least one of `embeddings` or `reference` must be provided.
+        """
+        emb_params: Embeddings | None = embeddings if embeddings is not None else reference
+        if emb_params is None:
+            raise ValueError("Must provide at least embeddings or reference embeddings.")
+        prioritize = Prioritize(emb_params._model, emb_params.batch_size, emb_params.device, method)
+        prioritize._k = k
+        prioritize._c = c
+        prioritize._embeddings = embeddings
+        prioritize._reference = reference
+        return prioritize
+
+    def _get_sorter(self, samples: int) -> _Sorter:
+        if self._method == "knn":
+            return _KNNSorter(samples, self._k)
+        elif self._method == "kmeans_distance":
+            return _KMeansDistanceSorter(samples, self._c)
+        else:  # self._method == "kmeans_complexity"
+            return _KMeansComplexitySorter(samples, self._c)
+
+    def _to_normalized_ndarray(self, embeddings: Embeddings, selection: list[int] | None = None) -> NDArray[Any]:
+        emb: NDArray[Any] = embeddings.to_tensor(selection).cpu().numpy()
+        emb /= max(np.max(np.linalg.norm(emb, axis=1)), EPSILON)
+        return emb
+
+    def __call__(self, dataset: Select[Any]) -> None:
+        # Initialize sorter
+        self._sorter = self._get_sorter(len(dataset._selection))
+        # Extract and normalize embeddings
+        embeddings = (
+            Embeddings(dataset, batch_size=self._batch_size, model=self._model, device=self._device)
+            if self._embeddings is None
+            else self._embeddings
+        )
+        if len(dataset._selection) != len(embeddings):
+            raise ValueError(
+                "Size of embeddings do not match the size of the selection: "
+                + f"embeddings={len(embeddings)}, selection={len(dataset._selection)}"
+            )
+        emb = self._to_normalized_ndarray(embeddings, dataset._selection)
+        ref = None if self._reference is None else self._to_normalized_ndarray(self._reference)
+        # Sort indices
+        dataset._selection = self._sorter._sort(emb, ref).tolist()

dataeval/utils/data/selections/_shuffle.py

@@ -2,10 +2,14 @@ from __future__ import annotations
 
 __all__ = []
 
-from typing import Any
+from typing import Any, Sequence
 
 import numpy as np
+from numpy.random import BitGenerator, Generator, SeedSequence
+from numpy.typing import NDArray
 
+from dataeval.typing import Array, ArrayLike
+from dataeval.utils._array import as_numpy
 from dataeval.utils.data._selection import Select, Selection, SelectionStage
 
 
@@ -15,14 +19,19 @@ class Shuffle(Selection[Any]):
 
     Parameters
     ----------
-    seed
+    seed : int, ArrayLike, SeedSequence, BitGenerator, Generator or None, default None
         Seed for the random number generator.
+
+    See Also
+    --------
+    `NumPy Random Generator <https://numpy.org/doc/stable/reference/random/generator.html>`_
     """
 
+    seed: int | NDArray[Any] | SeedSequence | BitGenerator | Generator | None
     stage = SelectionStage.ORDER
 
-    def __init__(self, seed: int):
-        self.seed = seed
+    def __init__(self, seed: int | ArrayLike | SeedSequence | BitGenerator | Generator | None = None):
+        self.seed = as_numpy(seed) if isinstance(seed, (Sequence, Array)) else seed
 
     def __call__(self, dataset: Select[Any]) -> None:
         rng = np.random.default_rng(self.seed)

dataeval/utils/torch/_gmm.py

@@ -16,6 +16,8 @@ from typing import TypeVar
 import numpy as np
 import torch
 
+from dataeval.config import EPSILON
+
 TGMMData = TypeVar("TGMMData")
 
 
@@ -74,8 +76,7 @@ def gmm_params(z: torch.Tensor, gamma: torch.Tensor) -> GaussianMixtureModelPara
 
     # cholesky decomposition of covariance and determinant derivation
     D = cov.shape[1]
-    eps = 1e-6
-    L = torch.linalg.cholesky(cov + torch.eye(D) * eps)  # K x D x D
+    L = torch.linalg.cholesky(cov + torch.eye(D) * EPSILON)  # K x D x D
     log_det_cov = 2.0 * torch.sum(torch.log(torch.diagonal(L, dim1=-2, dim2=-1)), 1)  # K
 
     return GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)

dataeval/utils/torch/_internal.py

@@ -11,13 +11,13 @@ from numpy.typing import NDArray
 from torch.utils.data import DataLoader, TensorDataset
 from tqdm import tqdm
 
-from dataeval.config import get_device
+from dataeval.config import DeviceLike, get_device
 
 
 def predict_batch(
     x: NDArray[Any] | torch.Tensor,
     model: Callable | torch.nn.Module | torch.nn.Sequential,
-    device: torch.device | None = None,
+    device: DeviceLike | None = None,
     batch_size: int = int(1e10),
     preprocess_fn: Callable[[torch.Tensor], torch.Tensor] | None = None,
     dtype: type[np.generic] | torch.dtype = np.float32,
@@ -31,9 +31,9 @@ def predict_batch(
         Batch of instances.
     model : Callable | nn.Module | nn.Sequential
         PyTorch model.
-    device : torch.device | None, default None
-        Device type used. The default None tries to use the GPU and falls back on CPU.
-        Can be specified by passing either torch.device('cuda') or torch.device('cpu').
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
     batch_size : int, default 1e10
         Batch size used during prediction.
     preprocess_fn : Callable | None, default None

dataeval/utils/torch/trainer.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from dataeval.config import DeviceLike, get_device
+
 __all__ = ["AETrainer"]
 
 from typing import Any
@@ -25,9 +27,9 @@ class AETrainer:
     ----------
     model : nn.Module
         The model to be trained.
-    device : str or torch.device, default "auto"
-        The hardware device to use for training.
-        If "auto", the device will be set to "cuda" if available, otherwise "cpu".
+    device : DeviceLike or None, default None
+        The hardware device to use if specified, otherwise uses the DataEval
+        default or torch default.
     batch_size : int, default 8
         The number of images to process in a batch.
     """
@@ -35,13 +37,11 @@ class AETrainer:
     def __init__(
         self,
         model: nn.Module,
-        device: str | torch.device = "auto",
+        device: DeviceLike | None = None,
         batch_size: int = 8,
     ):
-        if device == "auto":
-            device = "cuda" if torch.cuda.is_available() else "cpu"
-        self.device: torch.device = torch.device(device)
-        self.model: nn.Module = model.to(device)
+        self.device: torch.device = get_device(device)
+        self.model: nn.Module = model.to(self.device)
         self.batch_size = batch_size
 
     def train(self, dataset: Dataset[Any], epochs: int = 25) -> list[float]:

{dataeval-0.82.0.dist-info → dataeval-0.83.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.82.0
+Version: 0.83.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -50,9 +50,9 @@ and reference material, please visit our documentation on
 
 <!-- start tagline -->
 
-DataEval curates datasets to train and test performant, robust, unbiased and
-reliable AI models and monitors for data shifts that impact performance of
-deployed models.
+DataEval analyzes datasets and models to give users the ability to train and
+test performant, unbiased, and reliable AI models and monitor data for
+impactful shifts to deployed models.
 
 <!-- end tagline -->
 

dataeval-0.83.0.dist-info/RECORD

@@ -0,0 +1,105 @@
+dataeval/__init__.py,sha256=uL-JSd_dKVJpGx4H8f6aOiQVpli46zeTLFqjb4Pa69c,1636
+dataeval/_log.py,sha256=Mn5bRWO0cgtAYd5VGYSFiPgu57ta3zoktrtHAZ1m3dU,357
+dataeval/config.py,sha256=oQ0XQsgIF4_z4n1j0Di6B-JCRUFzzPgJgpQUm3ZlYhs,3539
+dataeval/detectors/__init__.py,sha256=3Sg-XWlwr75zEEH3hZKA4nWMtGvaRlnfzTWvZG_Ak6U,189
+dataeval/detectors/drift/__init__.py,sha256=6is_XBtG1d-vUbhHvqXGOdnAwxJ7NA5yRfURn7pCeIw,651
+dataeval/detectors/drift/_base.py,sha256=mJdKvyROgWvz-p1VlAIJqUI6BAj9ss8riUvR5An5wIw,13459
+dataeval/detectors/drift/_cvm.py,sha256=H2w-I0eMD7yP-CSmpdodeJ0-TYznJT7w_H7JuobESow,3859
+dataeval/detectors/drift/_ks.py,sha256=-5k3RBPA3kadX7oD14Wc52rAqQf1udwFeW7Qf3Sv4Tw,4058
+dataeval/detectors/drift/_mmd.py,sha256=NEXowx9UHIvmEKS8sqssw6PMLJMh0BZPhRNX1hYlkz4,7239
+dataeval/detectors/drift/_torch.py,sha256=VrFCyTaRrUslFPy_mYZ4UL70LZ8faH4eHwLurZ9qqNE,7628
+dataeval/detectors/drift/_uncertainty.py,sha256=O5h6_bJbeQEE660SLLP8k-EHqImmKegIcxzcnUKI7X4,5714
+dataeval/detectors/drift/updates.py,sha256=Btu2iaZW7fbO59G1w5v3ykFot0YPzy2U6VjF0d440VE,2195
+dataeval/detectors/linters/__init__.py,sha256=xn2zPwUcmsuf-Jd9uw6AVI11C9z1b1Y9fYtuFnXenZ0,404
+dataeval/detectors/linters/duplicates.py,sha256=tcxniL8rRZkDdQqfuS502UmfKxS3a7iRA22Dtt_vQIk,4935
+dataeval/detectors/linters/outliers.py,sha256=Hln2dPQZjF_uV2QYptA_o6ZF3ugyCImVT-XLDB2-q3A,9042
+dataeval/detectors/ood/__init__.py,sha256=juCYBDs7CQEAtMhnEpPqF6uTrOIH9kTBSuQ_GRw6a8o,283
+dataeval/detectors/ood/ae.py,sha256=YQfhB1ShQLjM1V4uCz9Oo2tCZpOfAZ_-SBCAl4Ac67Y,2921
+dataeval/detectors/ood/base.py,sha256=9b-Ljznf0lB1SXF4F_Aj3eJ4Y3ijGEDPMjucUsWOGJM,3051
+dataeval/detectors/ood/mixin.py,sha256=0_o-1HPvgf3-Lf1MSOIfjj5UB8LTLEBGYtJJfyCCzwc,5431
+dataeval/detectors/ood/vae.py,sha256=Fcq0-WbLhzYCgYOAJPBklHm7yuXmFJuEpBkhgwM5kiA,2291
+dataeval/metadata/__init__.py,sha256=XDDmJbOZBNM6pL0r6Nbu6oMRoyAh22IDkPYGndNlkZU,316
+dataeval/metadata/_distance.py,sha256=xsXMMg1pJkHcEZ-KIlqv9YOGYVID3ELjt3-fr1QVnOs,4082
+dataeval/metadata/_ood.py,sha256=HbS5MusWl62hjixUAd-xaaT0KXkYY1M-MlnUaAI_-8M,12751
+dataeval/metadata/_utils.py,sha256=r8qBJT83RblobD5W5zyTVi6vYi51Dwkqswizdbzss-M,1169
+dataeval/metrics/__init__.py,sha256=8VC8q3HuJN3o_WN51Ae2_wXznl3RMXIvA5GYVcy7vr8,225
+dataeval/metrics/bias/__init__.py,sha256=1yTLmgiu1kwT_7ZWcjOUbj8R0NJ0DjGoCuWdA0_T7kc,683
+dataeval/metrics/bias/_balance.py,sha256=UnUgbPk2ybFfS5qxv8e_uim7RxamWj0UQP71x3omGs0,6158
+dataeval/metrics/bias/_coverage.py,sha256=PeUoOiaghUEdn6Ov8z2-am7-fnBVIPcFbJK7Ty5JObA,3647
+dataeval/metrics/bias/_diversity.py,sha256=U_l4oYjH39rON2Io0BdCIwJxxob0cKTW8bZNufG0CWs,5820
+dataeval/metrics/bias/_parity.py,sha256=8JRZv4wLpxN9zTvMDlcpKgz-2nO-9eVjqccODcf2nbw,11535
+dataeval/metrics/estimators/__init__.py,sha256=Pnds8uIyAovt2fKqZjiHCIP_kVoBWlVllekYuK5UmmU,568
+dataeval/metrics/estimators/_ber.py,sha256=C30E5LiGGTAfo31zWFYDptDg0R7CTJGJ-a60YgzSkYY,5382
+dataeval/metrics/estimators/_clusterer.py,sha256=1HrpihGTJ63IkNSOy4Ibw633Gllkm1RxKmoKT5MOgt0,1434
+dataeval/metrics/estimators/_divergence.py,sha256=QDWl1lyAYoO9D3Ho7qOHSk6ud8Gi2MGuXEsYwO1HxvA,4043
+dataeval/metrics/estimators/_uap.py,sha256=BULEBbJ9BQ1IcTeZf0x7iI60QHAWCccBOM97FIu9VXA,1928
+dataeval/metrics/stats/__init__.py,sha256=6tA_9nbbM5ObJ6cds8Y1VBtTQiTOxrpGQSFLu_lWGGA,1098
+dataeval/metrics/stats/_base.py,sha256=rn0CrRCvVh3QLDEi_JlOFVUoQ-xtclnOoHt_o1E26J4,10656
+dataeval/metrics/stats/_boxratiostats.py,sha256=8Kd2FTZ5PLNYZfdAjU_R385gb0Z16JY0L9H_d5ZhgQs,6341
+dataeval/metrics/stats/_dimensionstats.py,sha256=h2wCLn4UuW7-GV6tM5E1SqSeGa_-4ie9oaEXpSC7EKI,2690
+dataeval/metrics/stats/_hashstats.py,sha256=yD6cXMvOo10-xtwUr7ftBRbCqMhReNfQJMInEWV_8Mk,4757
+dataeval/metrics/stats/_imagestats.py,sha256=hyjijPXAfUIJ1lwWiIyYK9VSLiq7Vg2-YhJ5Q8s1rkY,2979
+dataeval/metrics/stats/_labelstats.py,sha256=PtGyqj4RHw0cyLAWAR9FzZGqgA81AtxLGHZiuMAL2h0,4100
+dataeval/metrics/stats/_pixelstats.py,sha256=Q0-ldG-znDYBP_qTqm6S4qYm0ZV5FTTHf8MlyGHSYEc,3235
+dataeval/metrics/stats/_visualstats.py,sha256=ZxBDTerZ8ixibY2pGl7mwwcIz3DWl-k_Jb4YwBjHLNw,3686
+dataeval/outputs/__init__.py,sha256=uxTAr1Kn0QNwC7zn1U_5WBAgwZxupM3JGgD25DyO6yI,1655
+dataeval/outputs/_base.py,sha256=aZFbgybnZSQ3ws7QYRLTbDFqUfBFRVtIwX2LZfeGFUA,5703
+dataeval/outputs/_bias.py,sha256=O5RHbTUJDwkwJfz2-YoOfRb4eDl5Tg1UFVtvs025wfA,12173
+dataeval/outputs/_drift.py,sha256=gOiu2C-ERTWiRqlP0auMYxPBGdm9HecWPqWfg7I4tZg,2015
+dataeval/outputs/_estimators.py,sha256=a2oAIxxEDZ9WLGfMWH8KD-BVUS_SnULRPR-iI9hFPoQ,3047
+dataeval/outputs/_linters.py,sha256=YOdjrfm8ypdRrqYOaPM9nc6wVJI3-ita3Haj7LHDNaw,6416
+dataeval/outputs/_metadata.py,sha256=ffZgpX8KWURPHXpOWjbvJ2KRqWQkS2nWuIjKUzoHhMI,1710
+dataeval/outputs/_ood.py,sha256=suLKVXULGtXH0rq9eXHI1d3d2jhGmItJtz4QiQd47A4,1718
+dataeval/outputs/_stats.py,sha256=PhRdyWWZxewzenFx0MxK9y9ZLE2MnMA-a4-JeSJ_Bs8,13180
+dataeval/outputs/_utils.py,sha256=HHlGC7sk416m_3Bgn075Qdblz_aPup_UOafJpB0RuXY,893
+dataeval/outputs/_workflows.py,sha256=MkRD6ubI4NCBXb9v3kjXy64cUGs3G-JKkBdOpRD9XVE,10750
+dataeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dataeval/typing.py,sha256=YQ1KteeK1zf2mcWwngWwQP8EC3pI4WsvAzp_x179b4g,6568
+dataeval/utils/__init__.py,sha256=T8F8zJh4ZAeu0wDzfpld92I2zJg9mWBmkGCHrDPU7gk,264
+dataeval/utils/_array.py,sha256=fc04sYShIdsRS4qtG1UCnlGGk-yVRxlOHTNAmW7NpDY,4990
+dataeval/utils/_bin.py,sha256=nylthmsC3vzLHLhlUMACvZs--h7xvAh9Pt75InaQJW8,7322
+dataeval/utils/_clusterer.py,sha256=fw5x-2QN0TIbiodDKHZxRgxKHINedpPcOklzce0Rbjg,5436
+dataeval/utils/_fast_mst.py,sha256=4_7ykVihCL5jWtxcGnrecIsDQo65kUml9SZ1JxgBZYY,7172
+dataeval/utils/_image.py,sha256=capzF_X5H0jy0PmTP3Hf52GFgLqrnfU6gS4tiwck9jo,1939
+dataeval/utils/_method.py,sha256=9B9JQbgqWJBRhQJb7glajUtWaQzUTIUuvrZ9_bisxsM,394
+dataeval/utils/_mst.py,sha256=f0vXytTUjlOS6AyL7c6PkXmaHuuGUK-vMLpq-5xMgxk,2183
+dataeval/utils/_plot.py,sha256=mTRQNbJsA42QMiOwZbJaH8sNYgP996QFDEGVVE9HSgY,7076
+dataeval/utils/data/__init__.py,sha256=vldQ2ZXl8gnI3s4vAGqUUVi6dc_R58F3JMSpbCOyFRI,820
+dataeval/utils/data/_dataset.py,sha256=tjZUJnxj9IY71GKqdKltrwufkn0EC0S3a6ylrW5Bc2s,7756
+dataeval/utils/data/_embeddings.py,sha256=fKGFJXhb4ajnBE3jrKxIvBAhBQ6HpcYYkpO_sAk3jTE,3669
+dataeval/utils/data/_images.py,sha256=pv_vvpH8hWxPgLvjeVC2mZiyZivZFNLARNIOXam5ceY,1984
+dataeval/utils/data/_metadata.py,sha256=VqeePp7NtoFFWzmIhH4fn-cjrnATpgzgzs-d73cnBXM,14370
+dataeval/utils/data/_selection.py,sha256=nlslafwAfoZ5d5K_v9bIIvij-UP0NcalKqH4Nw7A-S4,4553
+dataeval/utils/data/_split.py,sha256=YdsqTRjKbdSfg8w0f4XgX7j0uOSdtfzvvyObAzyqgI0,18433
+dataeval/utils/data/_targets.py,sha256=ws5d9wRiDkIuOV7GSAKNxzgSm6AWTgb0BFroQK5nAmM,3057
+dataeval/utils/data/collate.py,sha256=Z5nmBnWV_IoJzMp_tj8RCKjMJA9sSCY_zZITqISGixc,3865
+dataeval/utils/data/datasets/__init__.py,sha256=jBrswiERrvBx4pJQJZIq_B5UE-Wy8a2_SBfM2crG8R8,511
+dataeval/utils/data/datasets/_base.py,sha256=CZ-hb-yWPLdnTQ3pURJMcityQ42ZNYj_Lbb1P5Junn4,8793
+dataeval/utils/data/datasets/_cifar10.py,sha256=I6HKksE2escos1aTdiZJObtiVXChBlez5BDa0eBfJ_Y,5449
+dataeval/utils/data/datasets/_fileio.py,sha256=SixIk5nIlIwJdX9zjNXS10vHA3hL8aaYbqHsDg1xSpY,6447
+dataeval/utils/data/datasets/_milco.py,sha256=ScBe7Ux-J9Kxs33jeKffhWKeSb8GCrWznTyEUt95Vt4,6369
+dataeval/utils/data/datasets/_mixin.py,sha256=FJgZP_cpJkgAHA3j3ai_j3Wt7aFSEjIMVmt9NpvVXzg,1757
+dataeval/utils/data/datasets/_mnist.py,sha256=iWWI9mq6TbZm7eTL9btzqjCNMhgXrLHQeMKENr7USsk,7988
+dataeval/utils/data/datasets/_ships.py,sha256=p3fScYLW2f1wUEPOroCX5nOFti0vMOSjeYltj6ox53U,4777
+dataeval/utils/data/datasets/_types.py,sha256=iSKyHXRlGuomXs0FHK6md8lXLQrQQ4fxgVOwr4o81bo,1089
+dataeval/utils/data/datasets/_voc.py,sha256=4poEer_G_mUBcz6eAro0Tc29CjdgjEAlms0Eu0tLBzE,14842
+dataeval/utils/data/selections/__init__.py,sha256=k86OpqGPkjT1MrOir5fOZ3AIq5UR81Az9ek7l1-GdIM,565
+dataeval/utils/data/selections/_classfilter.py,sha256=opSF8CGv4x1hUMe-GTQOu3UwJK80DzT0nJOV0l2uaW4,2404
+dataeval/utils/data/selections/_indices.py,sha256=QdLgXN7GABCvGPYe28PV1RAc_RSP_nZOyCvEpKRBdWg,636
+dataeval/utils/data/selections/_limit.py,sha256=ECvHRsp7OF4LZw2tE4sGqqJ085kjC-hd2c7QDMfvXr8,518
+dataeval/utils/data/selections/_prioritize.py,sha256=EAA4_uFVV7MmemhhufGmP7eunnbtyTc-TzgcnvRK5OE,11333
+dataeval/utils/data/selections/_reverse.py,sha256=6SWpELC9Wgx-kPqzhDrPNn4NKU6FqDJveLrxV4D2Ypk,374
+dataeval/utils/data/selections/_shuffle.py,sha256=kY3xJvVbBArdrJu_u6mXmxk1HdNmmDE4w7MmxbevUmU,1178
+dataeval/utils/metadata.py,sha256=X8Hu4LdCzAaE9uk1hI4BflmFve_VOQCqK9lXq0sk9ow,14196
+dataeval/utils/torch/__init__.py,sha256=dn5mjCrFp0b1aL_UEURhONU0Ag0cmXoTOBSGagpkTiA,325
+dataeval/utils/torch/_blocks.py,sha256=HVhBTMMD5NA4qheMUgyol1KWiKZDIuc8k5j4RcMKmhk,1466
+dataeval/utils/torch/_gmm.py,sha256=XM68GNEP97EjaB1U49-ZXRb81d0CEFnPS910alrcB3g,3740
+dataeval/utils/torch/_internal.py,sha256=23DCnF7C7N3tZgZUpT2nyH7mMb8Pi4GcnQyjK0BKHpg,5735
+dataeval/utils/torch/models.py,sha256=hmroEs6C6jQ5tAoZa71RFeIvXLxfXrTJSFH_jG2LGQU,9749
+dataeval/utils/torch/trainer.py,sha256=iUotX4OdirH8-ZtjdpU8gbJavkYW9YY9qpA2mAlFy1Y,5520
+dataeval/workflows/__init__.py,sha256=ou8y0KO-d6W5lgmcyLjKlf-J_ckP3vilW7wHkgiDlZ4,255
+dataeval/workflows/sufficiency.py,sha256=mjKmfRrAjShLUFIARv5o8yT5fnFvDsS5Qu6ujIPUgQg,8497
+dataeval-0.83.0.dist-info/LICENSE.txt,sha256=uAooygKWvX6NbU9Ran9oG2msttoG8aeTeHSTe5JeCnY,1061
+dataeval-0.83.0.dist-info/METADATA,sha256=lVRLNQcl2DYQDo7GHpFv_z133aD5hn-uOCkXXltGK5s,5320
+dataeval-0.83.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+dataeval-0.83.0.dist-info/RECORD,,

dataeval/detectors/ood/metadata_ood_mi.py

@@ -1,93 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import numbers
-import warnings
-from typing import Any
-
-import numpy as np
-from numpy.typing import NDArray
-from sklearn.feature_selection import mutual_info_classif
-
-# NATS2BITS is the reciprocal of natural log of 2. If you have an information/entropy-type quantity measured in nats,
-#   which is what many library functions return, multiply it by NATS2BITS to get it in bits.
-NATS2BITS = 1.442695
-
-
-def get_metadata_ood_mi(
-    metadata: dict[str, list[Any] | NDArray[Any]],
-    is_ood: NDArray[np.bool_],
-    discrete_features: str | bool | NDArray[np.bool_] = False,
-    random_state: int | None = None,
-) -> dict[str, float]:
-    """Computes mutual information between a set of metadata features and an out-of-distribution flag.
-
-    Given a metadata dictionary `metadata` (where each key maps to one scalar metadata feature per example), and a
-    corresponding boolean flag `is_ood` indicating whether each example falls out-of-distribution (OOD) relative to a
-    reference dataset, this function finds the strength of association between each metadata feature and `is_ood` by
-    computing their mutual information. Metadata features may be either discrete or continuous; set the
-    `discrete_features` keyword to a bool array set to True for each feature that is discrete, or pass one bool to apply
-    to all features. Returns a dict indicating the strength of association between each individual feature and the OOD
-    flag, measured in bits.
-
-    Parameters
-    ----------
-    metadata : dict[str, list[Any] | NDArray[Any]]
-        A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
-    is_ood : NDArray[np.bool_]
-        A boolean array, with one value per example, that indicates which examples are OOD.
-    discrete_features : str | bool | NDArray[np.bool_]
-        Either a boolean array or a single boolean value, indicate which features take on discrete values.
-    random_state : int, optional - default None
-        Determines random number generation for small noise added to continuous variables. Set to a value for
-        reproducible results.
-
-    Returns
-    -------
-    dict[str, float]
-        A dictionary with keys corresponding to metadata feature names, and values indicating the strength of
-        association between each named feature and the OOD flag, as mutual information measured in bits.
-
-    Examples
-    --------
-    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and altitude.
-
-    >>> metadata = {"time": np.linspace(0, 10, 100), "altitude": np.linspace(0, 16, 100) ** 2}
-    >>> is_ood = metadata["altitude"] > 100
-    >>> get_metadata_ood_mi(metadata, is_ood, discrete_features=False, random_state=0)
-    {'time': 0.9359596758173668, 'altitude': 0.9407686591507002}
-    """
-    numerical_keys = [k for k, v in metadata.items() if all(isinstance(vi, numbers.Number) for vi in v)]
-    if len(numerical_keys) < len(metadata):
-        warnings.warn(
-            f"Processing {numerical_keys}, others are non-numerical and will be skipped.",
-            UserWarning,
-        )
-
-    md_lengths = {len(np.atleast_1d(v)) for v in metadata.values()}
-    if len(md_lengths) > 1:
-        raise ValueError(f"Metadata features have differing sizes: {md_lengths}")
-
-    if len(is_ood) != (mdl := md_lengths.pop()):
-        raise ValueError(
-            f"OOD flag and metadata features need to be same size, but are different sizes: {len(is_ood)} and {mdl}."
-        )
-
-    X = np.array([metadata[k] for k in numerical_keys]).T
-
-    X0, dX = np.mean(X, axis=0), np.std(X, axis=0, ddof=1)
-    Xscl = (X - X0) / dX
-
-    mutual_info_values = (
-        mutual_info_classif(
-            Xscl,
-            is_ood,
-            discrete_features=discrete_features,  # type: ignore
-            random_state=random_state,
-        )
-        * NATS2BITS
-    )
-
-    mi_dict = {k: mutual_info_values[i] for i, k in enumerate(numerical_keys)}
-    return mi_dict