dataeval 0.61.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/_internal/metrics/stats.py

@@ -1,26 +1,29 @@
+from abc import abstractmethod
 from enum import Flag
 from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Sequence, TypeVar, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.stats import entropy, kurtosis, skew
 
 from dataeval._internal.flags import ImageHash, ImageProperty, ImageStatistics, ImageStatsFlags, ImageVisuals
-from dataeval._internal.metrics.base import MetricMixin
-from dataeval._internal.metrics.hash import pchash, xxhash
-from dataeval._internal.metrics.utils import edge_filter, get_bitdepth, normalize_image_shape, rescale
+from dataeval._internal.interop import to_numpy_iter
+from dataeval._internal.metrics.base import EvaluateMixin
+from dataeval._internal.metrics.utils import edge_filter, get_bitdepth, normalize_image_shape, pchash, rescale, xxhash
 
 QUARTILES = (0, 25, 50, 75, 100)
 
-TBatch = TypeVar("TBatch", bound=Sequence)
+TBatch = TypeVar("TBatch", bound=Sequence[ArrayLike])
 TFlag = TypeVar("TFlag", bound=Flag)
 
 
-class BaseStatsMetric(MetricMixin, Generic[TBatch, TFlag]):
+class BaseStatsMetric(EvaluateMixin, Generic[TBatch, TFlag]):
     def __init__(self, flags: TFlag):
         self.flags = flags
         self.results = []
 
-    def update(self, preds: TBatch, targets=None) -> None:
+    @abstractmethod
+    def update(self, images: TBatch) -> None:
         """
         Updates internal metric cache for later calculation
 
@@ -66,6 +69,16 @@ class BaseStatsMetric(MetricMixin, Generic[TBatch, TFlag]):
         )
         return [flag.name.lower() for flag in flags if flag.name is not None]
 
+    def evaluate(self, images: TBatch) -> Dict[str, Any]:
+        """Calculate metric results given a single batch of images"""
+        if self.results:
+            raise RuntimeError("Call reset before calling evaluate")
+
+        self.update(images)
+        results = self.compute()
+        self.reset()
+        return results
+
 
 class ImageHashMetric(BaseStatsMetric):
     """
@@ -80,12 +93,12 @@ class ImageHashMetric(BaseStatsMetric):
     def __init__(self, flags: ImageHash = ImageHash.ALL):
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for data in preds:
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
             results = self._map(
                 {
-                    ImageHash.XXHASH: lambda: xxhash(data),
-                    ImageHash.PCHASH: lambda: pchash(data),
+                    ImageHash.XXHASH: lambda: xxhash(image),
+                    ImageHash.PCHASH: lambda: pchash(image),
                 }
             )
             self.results.append(results)
@@ -104,16 +117,16 @@ class ImagePropertyMetric(BaseStatsMetric):
     def __init__(self, flags: ImageProperty = ImageProperty.ALL):
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for data in preds:
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
             results = self._map(
                 {
-                    ImageProperty.WIDTH: lambda: np.int32(data.shape[-1]),
-                    ImageProperty.HEIGHT: lambda: np.int32(data.shape[-2]),
-                    ImageProperty.SIZE: lambda: np.int32(data.shape[-1] * data.shape[-2]),
-                    ImageProperty.ASPECT_RATIO: lambda: data.shape[-1] / np.int32(data.shape[-2]),
-                    ImageProperty.CHANNELS: lambda: data.shape[-3],
-                    ImageProperty.DEPTH: lambda: get_bitdepth(data).depth,
+                    ImageProperty.WIDTH: lambda: np.int32(image.shape[-1]),
+                    ImageProperty.HEIGHT: lambda: np.int32(image.shape[-2]),
+                    ImageProperty.SIZE: lambda: np.int32(image.shape[-1] * image.shape[-2]),
+                    ImageProperty.ASPECT_RATIO: lambda: image.shape[-1] / np.int32(image.shape[-2]),
+                    ImageProperty.CHANNELS: lambda: image.shape[-3],
+                    ImageProperty.DEPTH: lambda: get_bitdepth(image).depth,
                 }
             )
             self.results.append(results)
@@ -132,14 +145,14 @@ class ImageVisualsMetric(BaseStatsMetric):
     def __init__(self, flags: ImageVisuals = ImageVisuals.ALL):
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for data in preds:
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
             results = self._map(
                 {
-                    ImageVisuals.BRIGHTNESS: lambda: np.mean(rescale(data)),
-                    ImageVisuals.BLURRINESS: lambda: np.std(edge_filter(np.mean(data, axis=0))),
-                    ImageVisuals.MISSING: lambda: np.sum(np.isnan(data)),
-                    ImageVisuals.ZERO: lambda: np.int32(np.count_nonzero(data == 0)),
+                    ImageVisuals.BRIGHTNESS: lambda: np.mean(rescale(image)),
+                    ImageVisuals.BLURRINESS: lambda: np.std(edge_filter(np.mean(image, axis=0))),
+                    ImageVisuals.MISSING: lambda: np.sum(np.isnan(image)),
+                    ImageVisuals.ZERO: lambda: np.int32(np.count_nonzero(image == 0)),
                 }
             )
             self.results.append(results)
@@ -158,9 +171,9 @@ class ImageStatisticsMetric(BaseStatsMetric):
     def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for data in preds:
-            scaled = rescale(data)
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
+            scaled = rescale(image)
             if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
                 hist = np.histogram(scaled, bins=256, range=(0, 1))[0]
 
@@ -192,10 +205,10 @@ class ChannelStatisticsMetric(BaseStatsMetric):
     def __init__(self, flags: ImageStatistics = ImageStatistics.ALL):
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for data in preds:
-            scaled = rescale(data)
-            flattened = scaled.reshape(data.shape[0], -1)
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
+            scaled = rescale(image)
+            flattened = scaled.reshape(image.shape[0], -1)
 
             if (ImageStatistics.HISTOGRAM | ImageStatistics.ENTROPY) & self.flags:
                 hist = np.apply_along_axis(lambda x: np.histogram(x, bins=256, range=(0, 1))[0], 1, flattened)
@@ -253,8 +266,8 @@ class ImageStats(BaseAggregateMetric):
         super().__init__(flags)
         self._length = 0
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for image in preds:
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
             self._length += 1
             img = normalize_image_shape(image)
             for metric in self._metrics_dict:
@@ -295,8 +308,8 @@ class ChannelStats(BaseAggregateMetric):
     def __init__(self, flags: Optional[ImageStatistics] = None) -> None:
         super().__init__(flags)
 
-    def update(self, preds: Iterable[np.ndarray], targets=None) -> None:
-        for image in preds:
+    def update(self, images: Iterable[ArrayLike]) -> None:
+        for image in to_numpy_iter(images):
             img = normalize_image_shape(image)
             for metric in self._metrics_dict:
                 metric.update([img])

dataeval/_internal/metrics/uap.py

@@ -4,42 +4,47 @@ FR Test Statistic based estimate for the upperbound
 average precision using empirical mean precision
 """
 
-from typing import Dict
+from typing import NamedTuple
 
-import numpy as np
+from numpy.typing import ArrayLike
 from sklearn.metrics import average_precision_score
 
-from dataeval._internal.metrics.base import EvaluateMixin
+from dataeval._internal.interop import to_numpy
 
 
-class UAP(EvaluateMixin):
+class UAPOutput(NamedTuple):
     """
-    FR Test Statistic based estimate of the empirical mean precision
+    Attributes
+    ----------
+    uap : float
+        The empirical mean precision estimate
+    """
+
+    uap: float
+
+
+def uap(labels: ArrayLike, scores: ArrayLike) -> UAPOutput:
+    """
+    FR Test Statistic based estimate of the empirical mean precision for
+    the upperbound average precision
 
     Parameters
     ----------
-    labels : np.ndarray
+    labels : ArrayLike
         A numpy array of n_samples of class labels with M unique classes.
-
-    scores : np.ndarray
+    scores : ArrayLike
         A 2D array of class probabilities per image
+
+    Returns
+    -------
+    Dict[str, float]
+        uap : The empirical mean precision estimate
+
+    Raises
+    ------
+    ValueError
+        If unique classes M < 2
     """
 
-    def __init__(self, labels: np.ndarray, scores: np.ndarray) -> None:
-        self.labels = labels
-        self.scores = scores
-
-    def evaluate(self) -> Dict[str, float]:
-        """
-        Returns
-        -------
-        Dict[str, float]
-            uap : The empirical mean precision estimate
-
-        Raises
-        ------
-        ValueError
-            If unique classes M < 2
-        """
-        uap = float(average_precision_score(self.labels, self.scores, average="weighted"))
-        return {"uap": uap}
+    precision = float(average_precision_score(to_numpy(labels), to_numpy(scores), average="weighted"))
+    return UAPOutput(precision)

dataeval/_internal/metrics/utils.py

@@ -1,15 +1,180 @@
-from typing import Any, Literal, NamedTuple, Tuple, Union
+from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Sequence, Tuple, Union
 
 import numpy as np
+import xxhash as xxh
+from PIL import Image
+from scipy.fftpack import dct
 from scipy.signal import convolve2d
 from scipy.sparse import csr_matrix
 from scipy.sparse.csgraph import minimum_spanning_tree as mst
 from scipy.spatial.distance import pdist, squareform
+from scipy.stats import entropy as sp_entropy
 from sklearn.neighbors import NearestNeighbors
 
 EPSILON = 1e-5
 EDGE_KERNEL = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype=np.int8)
 BIT_DEPTH = (1, 8, 12, 16, 32)
+HASH_SIZE = 8
+MAX_FACTOR = 4
+
+
+def get_method(method_map: Dict[str, Callable], method: str) -> Callable:
+    if method not in method_map:
+        raise ValueError(f"Specified method {method} is not a valid method: {method_map}.")
+    return method_map[method]
+
+
+def get_counts(
+    data: np.ndarray, names: List[str], is_categorical: List[bool], subset_mask: Optional[np.ndarray] = None
+) -> tuple[Dict, Dict]:
+    """
+    Initialize dictionary of histogram counts --- treat categorical values
+    as histogram bins.
+
+    Parameters
+    ----------
+    subset_mask: Optional[np.ndarray[bool]]
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Returns
+    -------
+    counts: Dict
+        histogram counts per metadata factor in `factors`.  Each
+        factor will have a different number of bins.  Counts get reused
+        across metrics, so hist_counts are cached but only if computed
+        globally, i.e. without masked samples.
+    """
+
+    hist_counts, hist_bins = {}, {}
+    # np.where needed to satisfy linter
+    mask = np.where(subset_mask if subset_mask is not None else np.ones(data.shape[0], dtype=bool))
+
+    for cdx, fn in enumerate(names):
+        # linter doesn't like double indexing
+        col_data = data[mask, cdx].squeeze()
+        if is_categorical[cdx]:
+            # if discrete, use unique values as bins
+            bins, cnts = np.unique(col_data, return_counts=True)
+        else:
+            bins = hist_bins.get(fn, "auto")
+            cnts, bins = np.histogram(col_data, bins=bins, density=True)
+
+        hist_counts[fn] = cnts
+        hist_bins[fn] = bins
+
+    return hist_counts, hist_bins
+
+
+def entropy(
+    data: np.ndarray,
+    names: List[str],
+    is_categorical: List[bool],
+    normalized: bool = False,
+    subset_mask: Optional[np.ndarray] = None,
+) -> np.ndarray:
+    """
+    Meant for use with Bias metrics, Balance, Diversity, ClasswiseBalance,
+    and Classwise Diversity.
+
+    Compute entropy for discrete/categorical variables and, through standard
+    histogram binning, for continuous variables.
+
+    Parameters
+    ----------
+    normalized: bool
+        Flag that determines whether or not to normalize entropy by log(num_bins)
+    subset_mask: Optional[np.ndarray[bool]]
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Notes
+    -----
+    For continuous variables, histogram bins are chosen automatically.  See
+    numpy.histogram for details.
+
+    Returns
+    -------
+    ent: np.ndarray[float]
+        Entropy estimate per column of X
+
+    See Also
+    --------
+    numpy.histogram
+    scipy.stats.entropy
+    """
+
+    num_factors = len(names)
+    hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
+
+    ev_index = np.empty(num_factors)
+    for col, cnts in enumerate(hist_counts.values()):
+        # entropy in nats, normalizes counts
+        ev_index[col] = sp_entropy(cnts)
+        if normalized:
+            if len(cnts) == 1:
+                # log(0)
+                ev_index[col] = 0
+            else:
+                ev_index[col] /= np.log(len(cnts))
+    return ev_index
+
+
+def get_num_bins(
+    data: np.ndarray, names: List[str], is_categorical: List[bool], subset_mask: Optional[np.ndarray] = None
+) -> np.ndarray:
+    """
+    Number of bins or unique values for each metadata factor, used to
+    normalize entropy/diversity.
+
+    Parameters
+    ----------
+    subset_mask: Optional[np.ndarray[bool]]
+        Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+    """
+    # likely cached
+    hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
+    num_bins = np.empty(len(hist_counts))
+    for idx, cnts in enumerate(hist_counts.values()):
+        num_bins[idx] = len(cnts)
+
+    return num_bins
+
+
+def infer_categorical(X: np.ndarray, threshold: float = 0.5) -> np.ndarray:
+    """
+    Compute fraction of feature values that are unique --- intended to be used
+    for inferring whether variables are categorical.
+    """
+    if X.ndim == 1:
+        X = np.expand_dims(X, axis=1)
+    num_samples = X.shape[0]
+    pct_unique = np.empty(X.shape[1])
+    for col in range(X.shape[1]):  # type: ignore
+        uvals = np.unique(X[:, col], axis=0)
+        pct_unique[col] = len(uvals) / num_samples
+    return pct_unique < threshold
+
+
+def preprocess_metadata(class_labels: Sequence[int], metadata: List[Dict]) -> Tuple[np.ndarray, List[str], List[bool]]:
+    # convert class_labels and list of metadata dicts to dict of ndarrays
+    metadata_dict: Dict[str, np.ndarray] = {
+        "class_label": np.asarray(class_labels, dtype=int),
+        **{k: np.array([d[k] for d in metadata]) for k in metadata[0]},
+    }
+
+    # map columns of dict that are not numeric (e.g. string) to numeric values
+    # that mutual information and diversity functions can accommodate.  Each
+    # unique string receives a unique integer value.
+    for k, v in metadata_dict.items():
+        # if not numeric
+        if not np.issubdtype(v.dtype, np.number):
+            _, mapped_vals = np.unique(v, return_inverse=True)
+            metadata_dict[k] = mapped_vals
+
+    data = np.stack(list(metadata_dict.values()), axis=-1)
+    names = list(metadata_dict.keys())
+    is_categorical = [infer_categorical(metadata_dict[var], 0.25)[0] for var in names]
+
+    return data, names, is_categorical
 
 
 def minimum_spanning_tree(X: np.ndarray) -> Any:
@@ -89,7 +254,7 @@ def compute_neighbors(
 
     See Also
     --------
-    :func:`sklearn.neighbors.NearestNeighbors`
+    sklearn.neighbors.NearestNeighbors
     """
 
     nbrs = NearestNeighbors(n_neighbors=k + 1, algorithm=algorithm).fit(B)
@@ -156,3 +321,73 @@ def edge_filter(image: np.ndarray, offset: float = 0.5) -> np.ndarray:
     edges = convolve2d(image, EDGE_KERNEL, mode="same", boundary="symm") + offset
     np.clip(edges, 0, 255, edges)
     return edges
+
+
+def pchash(image: np.ndarray) -> str:
+    """
+    Performs a perceptual hash on an image by resizing to a square NxN image
+    using the Lanczos algorithm where N is 32x32 or the largest multiple of
+    8 that is smaller than the input image dimensions.  The resampled image
+    is compressed using a discrete cosine transform and the lowest frequency
+    component is encoded as a bit array of greater or less than median value
+    and returned as a hex string.
+
+    Parameters
+    ----------
+    image : np.ndarray
+        An image as a numpy array in CxHxW format
+
+    Returns
+    -------
+    str
+        The hex string hash of the image using perceptual hashing
+    """
+    # Verify that the image is at least larger than an 8x8 image
+    min_dim = min(image.shape[-2:])
+    if min_dim < HASH_SIZE + 1:
+        raise ValueError(f"Image must be larger than {HASH_SIZE}x{HASH_SIZE} for fuzzy hashing.")
+
+    # Calculates the dimensions of the resized square image
+    resize_dim = HASH_SIZE * min((min_dim - 1) // HASH_SIZE, MAX_FACTOR)
+
+    # Normalizes the image to CxHxW and takes the mean over all the channels
+    normalized = np.mean(normalize_image_shape(image), axis=0).squeeze()
+
+    # Rescales the pixel values to an 8-bit 0-255 image
+    rescaled = rescale(normalized, 8).astype(np.uint8)
+
+    # Resizes the image using the Lanczos algorithm to a square image
+    im = np.array(Image.fromarray(rescaled).resize((resize_dim, resize_dim), Image.Resampling.LANCZOS))
+
+    # Performs discrete cosine transforms to compress the image information and takes the lowest frequency component
+    transform = dct(dct(im.T).T)[:HASH_SIZE, :HASH_SIZE]
+
+    # Encodes the transform as a bit array over the median value
+    diff = transform > np.median(transform)
+
+    # Pads the front of the bit array to a multiple of 8 with False
+    padded = np.full(int(np.ceil(diff.size / 8) * 8), False)
+    padded[-diff.size :] = diff.ravel()
+
+    # Converts the bit array to a hex string and strips leading 0s
+    hash_hex = np.packbits(padded).tobytes().hex().lstrip("0")
+    return hash_hex if hash_hex else "0"
+
+
+def xxhash(image: np.ndarray) -> str:
+    """
+    Performs a fast non-cryptographic hash using the xxhash algorithm
+    (xxhash.com) against the image as a flattened bytearray.  The hash
+    is returned as a hex string.
+
+    Parameters
+    ----------
+    image : np.ndarray
+        An image as a numpy array
+
+    Returns
+    -------
+    str
+        The hex string hash of the image using the xxHash algorithm
+    """
+    return xxh.xxh3_64_hexdigest(image.ravel().tobytes())

dataeval/_internal/utils.py

@@ -0,0 +1,64 @@
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from torch.utils.data import Dataset
+
+
+def read_dataset(dataset: Dataset) -> List[List[Any]]:
+    """
+    Extract information from a dataset at each index into a individual lists of each information position
+
+    Parameters
+    ----------
+    dataset : torch.utils.data.Dataset
+        Input dataset
+
+    Returns
+    -------
+    List[List[Any]]
+        All objects in individual lists based on return position from dataset
+
+    Warning
+    -------
+    No type checking is done between lists or data inside lists
+
+    See Also
+    --------
+    torch.utils.data.Dataset
+
+    Examples
+    --------
+    >>> import numpy as np
+
+    >>> data = np.ones((10, 3, 3))
+    >>> labels = np.ones((10,))
+    >>> class ICDataset:
+    ...     def __init__(self, data, labels):
+    ...         self.data = data
+    ...         self.labels = labels
+
+    ...     def __getitem__(self, idx):
+    ...         return self.data[idx], self.labels[idx]
+
+    >>> ds = ICDataset(data, labels)
+
+    >>> result = read_dataset(ds)
+    >>> assert len(result) == 2
+    True
+    >>> assert result[0].shape == (10, 3, 3)  # 10 3x3 images
+    True
+    >>> assert result[1].shape == (10,)  # 10 labels
+    True
+    """
+
+    ddict: Dict[int, List] = defaultdict(list)
+
+    for data in dataset:
+        # Convert to tuple if single return (e.g. images only)
+        if not isinstance(data, tuple):
+            data = (data,)
+
+        for i, d in enumerate(data):
+            ddict[i].append(d)
+
+    return list(ddict.values())

dataeval/metrics/__init__.py

@@ -1,7 +1,27 @@
-from dataeval._internal.metrics.ber import BER
-from dataeval._internal.metrics.divergence import Divergence
-from dataeval._internal.metrics.parity import Parity
+from typing import List
+
+__all__: List[str] = []
+
+from dataeval._internal.metrics.balance import balance, balance_classwise
+from dataeval._internal.metrics.ber import ber
+from dataeval._internal.metrics.coverage import coverage
+from dataeval._internal.metrics.divergence import divergence
+from dataeval._internal.metrics.diversity import diversity, diversity_classwise
+from dataeval._internal.metrics.parity import parity, parity_metadata
 from dataeval._internal.metrics.stats import ChannelStats, ImageStats
-from dataeval._internal.metrics.uap import UAP
+from dataeval._internal.metrics.uap import uap
 
-__all__ = ["BER", "Divergence", "Parity", "UAP", "ChannelStats", "ImageStats"]
+__all__ += [
+    "balance",
+    "balance_classwise",
+    "ber",
+    "coverage",
+    "divergence",
+    "diversity",
+    "diversity_classwise",
+    "parity",
+    "parity_metadata",
+    "ChannelStats",
+    "ImageStats",
+    "uap",
+]

dataeval/utils/__init__.py

@@ -0,0 +1,9 @@
+from importlib.util import find_spec
+from typing import List
+
+__all__: List[str] = []
+
+if find_spec("torch") is not None:  # pragma: no cover
+    from dataeval._internal.utils import read_dataset
+
+    __all__ += ["read_dataset"]

{dataeval-0.61.0.dist-info → dataeval-0.64.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataeval
-Version: 0.61.0
+Version: 0.64.0
 Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
 Home-page: https://dataeval.ai/
 License: MIT
@@ -23,7 +23,6 @@ Provides-Extra: all
 Provides-Extra: tensorflow
 Provides-Extra: torch
 Requires-Dist: hdbscan (>=0.8.36)
-Requires-Dist: maite
 Requires-Dist: matplotlib ; extra == "torch" or extra == "all"
 Requires-Dist: numpy (>1.24.3)
 Requires-Dist: nvidia-cudnn-cu11 (>=8.6.0.163) ; extra == "tensorflow" or extra == "torch" or extra == "all"