dataeval 0.82.1__py3-none-any.whl → 0.84.0__py3-none-any.whl

dataeval/utils/data/datasets/_voc.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 __all__ = []
 
 from pathlib import Path
-from typing import Any, Literal, Sequence, TypeVar
+from typing import TYPE_CHECKING, Any, Literal, Sequence, TypeVar
 
 import torch
 from defusedxml.ElementTree import parse
@@ -16,7 +16,10 @@ from dataeval.utils.data.datasets._base import (
     DataLocation,
 )
 from dataeval.utils.data.datasets._mixin import BaseDatasetNumpyMixin, BaseDatasetTorchMixin
-from dataeval.utils.data.datasets._types import ObjectDetectionTarget, SegmentationTarget, Transform
+from dataeval.utils.data.datasets._types import ObjectDetectionTarget, SegmentationTarget
+
+if TYPE_CHECKING:
+    from dataeval.typing import Transform
 
 _TArray = TypeVar("_TArray")
 _TTarget = TypeVar("_TTarget")
@@ -201,6 +204,8 @@ class BaseVOCDataset(BaseDataset[_TArray, _TTarget, list[str]]):
         boxes: list[list[float]] = []
         label_str = []
         root = parse(annotation).getroot()
+        if root is None:
+            raise ValueError(f"Unable to parse {annotation}")
         num_objects = len(root.findall("object"))
         additional_meta: dict[str, Any] = {
             "folder": [root.findtext("folder", default="") for _ in range(num_objects)],
@@ -253,21 +258,27 @@ class VOCDetection(
         If "base", then the combined dataset of "train" and "val" is returned.
     year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
         The dataset year.
-    transforms : Transform | Sequence[Transform] | None, default None
+    transforms : Transform, Sequence[Transform] or None, default None
         Transform(s) to apply to the data.
     verbose : bool, default False
         If True, outputs print statements.
 
     Attributes
     ----------
-    index2label : dict
+    path : pathlib.Path
+        Location of the folder containing the data.
+    image_set : "train", "val", "test" or "base"
+        The selected image set from the dataset.
+    index2label : dict[int, str]
         Dictionary which translates from class integers to the associated class strings.
-    label2index : dict
+    label2index : dict[str, int]
         Dictionary which translates from class strings to the associated class integers.
-    path : Path
-        Location of the folder containing the data.
-    metadata : dict
-        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    metadata : DatasetMetadata
+        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
+    transforms : Sequence[Transform]
+        The transforms to be applied to the data.
+    size : int
+        The size of the dataset.
     """
 
 
@@ -277,7 +288,7 @@ class VOCDetectionTorch(
     BaseDatasetTorchMixin,
 ):
     """
-    `Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset.
+    `Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Detection Dataset as PyTorch tensors.
 
     Parameters
     ----------
@@ -291,21 +302,27 @@ class VOCDetectionTorch(
         If "base", then the combined dataset of "train" and "val" is returned.
     year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
         The dataset year.
-    transforms : Transform | Sequence[Transform] | None, default None
+    transforms : Transform, Sequence[Transform] or None, default None
         Transform(s) to apply to the data.
     verbose : bool, default False
         If True, outputs print statements.
 
     Attributes
     ----------
-    index2label : dict
+    path : pathlib.Path
+        Location of the folder containing the data.
+    image_set : "train", "val", "test" or "base"
+        The selected image set from the dataset.
+    index2label : dict[int, str]
         Dictionary which translates from class integers to the associated class strings.
-    label2index : dict
+    label2index : dict[str, int]
         Dictionary which translates from class strings to the associated class integers.
-    path : Path
-        Location of the folder containing the data.
-    metadata : dict
-        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    metadata : DatasetMetadata
+        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
+    transforms : Sequence[Transform]
+        The transforms to be applied to the data.
+    size : int
+        The size of the dataset.
     """
 
 
@@ -329,21 +346,27 @@ class VOCSegmentation(
         If "base", then the combined dataset of "train" and "val" is returned.
     year : "2007", "2008", "2009", "2010", "2011" or "2012", default "2012"
         The dataset year.
-    transforms : Transform | Sequence[Transform] | None, default None
+    transforms : Transform, Sequence[Transform] or None, default None
         Transform(s) to apply to the data.
     verbose : bool, default False
         If True, outputs print statements.
 
     Attributes
     ----------
-    index2label : dict
+    path : pathlib.Path
+        Location of the folder containing the data.
+    image_set : "train", "val", "test" or "base"
+        The selected image set from the dataset.
+    index2label : dict[int, str]
         Dictionary which translates from class integers to the associated class strings.
-    label2index : dict
+    label2index : dict[str, int]
         Dictionary which translates from class strings to the associated class integers.
-    path : Path
-        Location of the folder containing the data.
-    metadata : dict
-        Dictionary containing Dataset metadata, such as `id` which returns the dataset class name.
+    metadata : DatasetMetadata
+        Typed dictionary containing dataset metadata, such as `id` which returns the dataset class name.
+    transforms : Sequence[Transform]
+        The transforms to be applied to the data.
+    size : int
+        The size of the dataset.
     """
 
     def _load_data(self) -> tuple[list[str], list[str], dict[str, list[Any]]]:

dataeval/utils/data/selections/__init__.py

@@ -4,6 +4,7 @@ __all__ = [
     "ClassFilter",
     "Indices",
     "Limit",
+    "Prioritize",
     "Reverse",
     "Shuffle",
 ]
@@ -11,5 +12,6 @@ __all__ = [
 from dataeval.utils.data.selections._classfilter import ClassFilter
 from dataeval.utils.data.selections._indices import Indices
 from dataeval.utils.data.selections._limit import Limit
+from dataeval.utils.data.selections._prioritize import Prioritize
 from dataeval.utils.data.selections._reverse import Reverse
 from dataeval.utils.data.selections._shuffle import Shuffle

dataeval/utils/data/selections/_classfilter.py

@@ -10,7 +10,7 @@ from dataeval.typing import Array, ImageClassificationDatum
 from dataeval.utils._array import as_numpy
 from dataeval.utils.data._selection import Select, Selection, SelectionStage
 
-TImageClassificationDatum = TypeVar("TImageClassificationDatum", bound=ImageClassificationDatum, covariant=True)
+TImageClassificationDatum = TypeVar("TImageClassificationDatum", bound=ImageClassificationDatum)
 
 
 class ClassFilter(Selection[TImageClassificationDatum]):

dataeval/utils/data/selections/_prioritize.py

@@ -0,0 +1,296 @@
+from __future__ import annotations
+
+__all__ = []
+
+import logging
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any, Literal, overload
+
+import numpy as np
+import torch
+from numpy.typing import NDArray
+from sklearn.cluster import KMeans
+from sklearn.metrics import pairwise_distances
+
+from dataeval.config import EPSILON, DeviceLike, get_seed
+from dataeval.utils.data import Embeddings, Select
+from dataeval.utils.data._selection import Selection, SelectionStage
+
+_logger = logging.getLogger(__name__)
+
+
+class _Clusters:
+    __slots__ = ["labels", "cluster_centers", "unique_labels"]
+
+    labels: NDArray[np.intp]
+    cluster_centers: NDArray[np.float64]
+    unique_labels: NDArray[np.intp]
+
+    def __init__(self, labels: NDArray[np.intp], cluster_centers: NDArray[np.float64]) -> None:
+        self.labels = labels
+        self.cluster_centers = cluster_centers
+        self.unique_labels = np.unique(labels)
+
+    def _dist2center(self, X: NDArray[np.float64]) -> NDArray[np.float64]:
+        dist = np.zeros(self.labels.shape)
+        for lab in self.unique_labels:
+            dist[self.labels == lab] = np.linalg.norm(X[self.labels == lab, :] - self.cluster_centers[lab, :], axis=1)
+        return dist
+
+    def _complexity(self, X: NDArray[np.float64]) -> NDArray[np.float64]:
+        num_clst_intra = int(np.maximum(np.minimum(int(self.unique_labels.shape[0] / 5), 20), 1))
+        d_intra = np.zeros(self.unique_labels.shape)
+        d_inter = np.zeros(self.unique_labels.shape)
+        for cdx, lab in enumerate(self.unique_labels):
+            d_intra[cdx] = np.mean(np.linalg.norm(X[self.labels == lab, :] - self.cluster_centers[cdx, :], axis=1))
+            d_inter[cdx] = np.mean(
+                np.linalg.norm(self.cluster_centers - self.cluster_centers[cdx, :], axis=1)[:num_clst_intra]
+            )
+        cj = d_intra * d_inter
+        tau = 0.1
+        exp = np.exp(cj / tau)
+        prob: NDArray[np.float64] = exp / np.sum(exp)
+        return prob
+
+    def _sort_by_weights(self, X: NDArray[np.float64]) -> NDArray[np.intp]:
+        pr = self._complexity(X)
+        d2c = self._dist2center(X)
+        inds_per_clst: list[NDArray[np.intp]] = []
+        for lab in zip(self.unique_labels):
+            inds = np.nonzero(self.labels == lab)[0]
+            # 'hardest' first
+            srt_inds = np.argsort(d2c[inds])[::-1]
+            inds_per_clst.append(inds[srt_inds])
+        glob_inds: list[NDArray[np.intp]] = []
+        while not bool(np.all([arr.size == 0 for arr in inds_per_clst])):
+            clst_ind = np.random.choice(self.unique_labels, 1, p=pr)[0]
+            if inds_per_clst[clst_ind].size > 0:
+                glob_inds.append(inds_per_clst[clst_ind][0])
+            else:
+                continue
+            inds_per_clst[clst_ind] = inds_per_clst[clst_ind][1:]
+        # sorted hardest first; reverse for consistency
+        return np.array(glob_inds[::-1])
+
+
+class _Sorter(ABC):
+    @abstractmethod
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]: ...
+
+
+class _KNNSorter(_Sorter):
+    def __init__(self, samples: int, k: int | None) -> None:
+        if k is None or k <= 0:
+            k = int(np.sqrt(samples))
+            _logger._log(logging.INFO, f"Setting k to default value of {k}", {"k": k, "samples": samples})
+        elif k >= samples:
+            raise ValueError(f"k={k} should be less than dataset size ({samples})")
+        elif k >= samples / 10 and k > np.sqrt(samples):
+            warnings.warn(
+                f"Variable k={k} is large with respect to dataset size but valid; "
+                + f"a nominal recommendation is k={int(np.sqrt(samples))}"
+            )
+        self._k = k
+
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        if reference is None:
+            dists = pairwise_distances(embeddings, embeddings)
+            np.fill_diagonal(dists, np.inf)
+        else:
+            dists = pairwise_distances(embeddings, reference)
+        inds = np.argsort(np.sort(dists, axis=1)[:, self._k])
+        return inds
+
+
+class _KMeansSorter(_Sorter):
+    def __init__(self, samples: int, c: int | None, n_init: int | Literal["auto", "warn"] = "auto") -> None:
+        if c is None or c <= 0:
+            c = int(np.sqrt(samples))
+            _logger._log(logging.INFO, f"Setting the value of num_clusters to a default value of {c}", {})
+        if c >= samples:
+            raise ValueError(f"c={c} should be less than dataset size ({samples})")
+        self._c = c
+        self._n_init = n_init
+
+    def _get_clusters(self, embeddings: NDArray[Any]) -> _Clusters:
+        clst = KMeans(n_clusters=self._c, init="k-means++", n_init=self._n_init, random_state=get_seed())  # type: ignore - n_init allows int but is typed as str
+        clst.fit(embeddings)
+        if clst.labels_ is None or clst.cluster_centers_ is None:
+            raise ValueError("Clustering failed to produce labels or cluster centers")
+        return _Clusters(clst.labels_, clst.cluster_centers_)
+
+
+class _KMeansDistanceSorter(_KMeansSorter):
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        clst = self._get_clusters(embeddings if reference is None else reference)
+        inds = np.argsort(clst._dist2center(embeddings))
+        return inds
+
+
+class _KMeansComplexitySorter(_KMeansSorter):
+    def _sort(self, embeddings: NDArray[Any], reference: NDArray[Any] | None = None) -> NDArray[np.intp]:
+        clst = self._get_clusters(embeddings if reference is None else reference)
+        inds = clst._sort_by_weights(embeddings)
+        return inds
+
+
+class Prioritize(Selection[Any]):
+    """
+    Prioritizes the dataset by sort order in the embedding space.
+
+    Parameters
+    ----------
+    model : torch.nn.Module
+        Model to use for encoding images
+    batch_size : int
+        Batch size to use when encoding images
+    device : DeviceLike or None
+        Device to use for encoding images
+    method : Literal["knn", "kmeans_distance", "kmeans_complexity"]
+        Method to use for prioritization
+    k : int | None, default None
+        Number of nearest neighbors to use for prioritization (knn only)
+    c : int | None, default None
+        Number of clusters to use for prioritization (kmeans only)
+    """
+
+    stage = SelectionStage.ORDER
+
+    @overload
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["knn"],
+        *,
+        k: int | None = None,
+    ) -> None: ...
+
+    @overload
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["kmeans_distance", "kmeans_complexity"],
+        *,
+        c: int | None = None,
+    ) -> None: ...
+
+    def __init__(
+        self,
+        model: torch.nn.Module,
+        batch_size: int,
+        device: DeviceLike | None,
+        method: Literal["knn", "kmeans_distance", "kmeans_complexity"],
+        *,
+        k: int | None = None,
+        c: int | None = None,
+    ) -> None:
+        if method not in ("knn", "kmeans_distance", "kmeans_complexity"):
+            raise ValueError(f"Invalid prioritization method: {method}")
+        self._model = model
+        self._batch_size = batch_size
+        self._device = device
+        self._method = method
+        self._embeddings: Embeddings | None = None
+        self._reference: Embeddings | None = None
+        self._k = k
+        self._c = c
+
+    @overload
+    @classmethod
+    def using(
+        cls,
+        method: Literal["knn"],
+        *,
+        k: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize: ...
+
+    @overload
+    @classmethod
+    def using(
+        cls,
+        method: Literal["kmeans_distance", "kmeans_complexity"],
+        *,
+        c: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize: ...
+
+    @classmethod
+    def using(
+        cls,
+        method: Literal["knn", "kmeans_distance", "kmeans_complexity"],
+        *,
+        k: int | None = None,
+        c: int | None = None,
+        embeddings: Embeddings | None = None,
+        reference: Embeddings | None = None,
+    ) -> Prioritize:
+        """
+        Prioritizes the dataset by sort order in the embedding space using existing
+        embeddings and/or reference dataset embeddings.
+
+        Parameters
+        ----------
+        method : Literal["knn", "kmeans_distance", "kmeans_complexity"]
+            Method to use for prioritization
+        embeddings : Embeddings or None, default None
+            Embeddings to use for prioritization
+        reference : Embeddings or None, default None
+            Reference embeddings to prioritize relative to
+        k : int or None, default None
+            Number of nearest neighbors to use for prioritization (knn only)
+        c : int or None, default None
+            Number of clusters to use for prioritization (kmeans, cluster only)
+
+        Notes
+        -----
+        At least one of `embeddings` or `reference` must be provided.
+        """
+        emb_params: Embeddings | None = embeddings if embeddings is not None else reference
+        if emb_params is None:
+            raise ValueError("Must provide at least embeddings or reference embeddings.")
+        prioritize = Prioritize(emb_params._model, emb_params.batch_size, emb_params.device, method)
+        prioritize._k = k
+        prioritize._c = c
+        prioritize._embeddings = embeddings
+        prioritize._reference = reference
+        return prioritize
+
+    def _get_sorter(self, samples: int) -> _Sorter:
+        if self._method == "knn":
+            return _KNNSorter(samples, self._k)
+        elif self._method == "kmeans_distance":
+            return _KMeansDistanceSorter(samples, self._c)
+        else:  # self._method == "kmeans_complexity"
+            return _KMeansComplexitySorter(samples, self._c)
+
+    def _to_normalized_ndarray(self, embeddings: Embeddings, selection: list[int] | None = None) -> NDArray[Any]:
+        emb: NDArray[Any] = embeddings.to_tensor(selection).cpu().numpy()
+        emb /= max(np.max(np.linalg.norm(emb, axis=1)), EPSILON)
+        return emb
+
+    def __call__(self, dataset: Select[Any]) -> None:
+        # Initialize sorter
+        self._sorter = self._get_sorter(len(dataset._selection))
+        # Extract and normalize embeddings
+        embeddings = (
+            Embeddings(dataset, batch_size=self._batch_size, model=self._model, device=self._device)
+            if self._embeddings is None
+            else self._embeddings
+        )
+        if len(dataset._selection) != len(embeddings):
+            raise ValueError(
+                "Size of embeddings do not match the size of the selection: "
+                + f"embeddings={len(embeddings)}, selection={len(dataset._selection)}"
+            )
+        emb = self._to_normalized_ndarray(embeddings, dataset._selection)
+        ref = None if self._reference is None else self._to_normalized_ndarray(self._reference)
+        # Sort indices
+        dataset._selection = self._sorter._sort(emb, ref).tolist()

dataeval/utils/data/selections/_shuffle.py

@@ -2,10 +2,14 @@ from __future__ import annotations
 
 __all__ = []
 
-from typing import Any
+from typing import Any, Sequence
 
 import numpy as np
+from numpy.random import BitGenerator, Generator, SeedSequence
+from numpy.typing import NDArray
 
+from dataeval.typing import Array, ArrayLike
+from dataeval.utils._array import as_numpy
 from dataeval.utils.data._selection import Select, Selection, SelectionStage
 
 
@@ -15,14 +19,19 @@ class Shuffle(Selection[Any]):
 
     Parameters
     ----------
-    seed
+    seed : int, ArrayLike, SeedSequence, BitGenerator, Generator or None, default None
         Seed for the random number generator.
+
+    See Also
+    --------
+    `NumPy Random Generator <https://numpy.org/doc/stable/reference/random/generator.html>`_
     """
 
+    seed: int | NDArray[Any] | SeedSequence | BitGenerator | Generator | None
     stage = SelectionStage.ORDER
 
-    def __init__(self, seed: int):
-        self.seed = seed
+    def __init__(self, seed: int | ArrayLike | SeedSequence | BitGenerator | Generator | None = None):
+        self.seed = as_numpy(seed) if isinstance(seed, (Sequence, Array)) else seed
 
     def __call__(self, dataset: Select[Any]) -> None:
         rng = np.random.default_rng(self.seed)

dataeval/utils/metadata.py

@@ -214,7 +214,7 @@ def flatten(
                 output[k] = cv
             else:
                 dropped_inner.setdefault(k, set()).add(DropReason.INCONSISTENT_KEY)
-        elif not isinstance(cv, list):
+        else:
             output[k] = cv if not size else [cv] * size
 
     if fully_qualified:

dataeval/utils/torch/_gmm.py

@@ -16,6 +16,8 @@ from typing import TypeVar
 import numpy as np
 import torch
 
+from dataeval.config import EPSILON
+
 TGMMData = TypeVar("TGMMData")
 
 
@@ -74,8 +76,7 @@ def gmm_params(z: torch.Tensor, gamma: torch.Tensor) -> GaussianMixtureModelPara
 
     # cholesky decomposition of covariance and determinant derivation
     D = cov.shape[1]
-    eps = 1e-6
-    L = torch.linalg.cholesky(cov + torch.eye(D) * eps)  # K x D x D
+    L = torch.linalg.cholesky(cov + torch.eye(D) * EPSILON)  # K x D x D
     log_det_cov = 2.0 * torch.sum(torch.log(torch.diagonal(L, dim1=-2, dim2=-1)), 1)  # K
 
     return GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)

{dataeval-0.82.1.dist-info → dataeval-0.84.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.82.1
+Version: 0.84.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -50,9 +50,9 @@ and reference material, please visit our documentation on
 
 <!-- start tagline -->
 
-DataEval curates datasets to train and test performant, robust, unbiased and
-reliable AI models and monitors for data shifts that impact performance of
-deployed models.
+DataEval analyzes datasets and models to give users the ability to train and
+test performant, unbiased, and reliable AI models and monitor data for
+impactful shifts to deployed models.
 
 <!-- end tagline -->