dataeval 0.83.0__py3-none-any.whl → 0.84.0__py3-none-any.whl

dataeval/__init__.py

@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
 from __future__ import annotations
 
 __all__ = ["config", "detectors", "log", "metrics", "typing", "utils", "workflows"]
-__version__ = "0.83.0"
+__version__ = "0.84.0"
 
 import logging
 

dataeval/config.py

@@ -45,13 +45,13 @@ def _todevice(device: DeviceLike) -> torch.device:
     return torch.device(*device) if isinstance(device, tuple) else torch.device(device)
 
 
-def set_device(device: DeviceLike) -> None:
+def set_device(device: DeviceLike | None) -> None:
     """
     Sets the default device to use when executing against a PyTorch backend.
 
     Parameters
     ----------
-    device : DeviceLike
+    device : DeviceLike or None
         The default device to use. See documentation for more information.
 
     See Also
@@ -59,7 +59,7 @@ def set_device(device: DeviceLike) -> None:
     `torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
     """
     global _device
-    _device = _todevice(device)
+    _device = None if device is None else _todevice(device)
 
 
 def get_device(override: DeviceLike | None = None) -> torch.device:

dataeval/metrics/bias/__init__.py

@@ -6,10 +6,12 @@ representation which may impact model performance.
 __all__ = [
     "BalanceOutput",
     "CoverageOutput",
+    "CompletenessOutput",
     "DiversityOutput",
     "LabelParityOutput",
     "ParityOutput",
     "balance",
+    "completeness",
     "coverage",
     "diversity",
     "label_parity",
@@ -17,7 +19,15 @@ __all__ = [
 ]
 
 from dataeval.metrics.bias._balance import balance
+from dataeval.metrics.bias._completeness import completeness
 from dataeval.metrics.bias._coverage import coverage
 from dataeval.metrics.bias._diversity import diversity
 from dataeval.metrics.bias._parity import label_parity, parity
-from dataeval.outputs._bias import BalanceOutput, CoverageOutput, DiversityOutput, LabelParityOutput, ParityOutput
+from dataeval.outputs._bias import (
+    BalanceOutput,
+    CompletenessOutput,
+    CoverageOutput,
+    DiversityOutput,
+    LabelParityOutput,
+    ParityOutput,
+)

dataeval/metrics/bias/_completeness.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import itertools
+
+__all__ = []
+
+
+import numpy as np
+
+from dataeval.config import EPSILON
+from dataeval.outputs import CompletenessOutput
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import ensure_embeddings
+
+
+def completeness(embeddings: ArrayLike, quantiles: int) -> CompletenessOutput:
+    """
+    Calculate the fraction of boxes in a grid defined by quantiles that
+    contain at least one data point.
+    Also returns the center coordinates of each empty box.
+
+    Parameters
+    ----------
+    embeddings : ArrayLike
+        Embedded dataset (or other low-dimensional data) (nxp)
+    quantiles : int
+        number of quantile values to use for partitioning each dimension
+        e.g., 1 would create a grid of 2^p boxes, 2, 3^p etc..
+
+    Returns
+    -------
+    CompletenessOutput
+        - fraction_filled: float - Fraction of boxes that contain at least one
+          data point
+        - empty_box_centers: List[np.ndarray] - List of coordinates for centers of empty
+          boxes
+
+    Raises
+    ------
+    ValueError
+        If embeddings are too high-dimensional (>10)
+    ValueError
+        If there are too many quantiles (>2)
+    ValueError
+        If embedding is invalid shape
+
+    Example
+    -------
+    >>> embs = np.array([[1, 0], [0, 1], [1, 1]])
+    >>> quantiles = 1
+    >>> result = completeness(embs, quantiles)
+    >>> result.fraction_filled
+    0.75
+
+    Reference
+    ---------
+    This implementation is based on https://arxiv.org/abs/2002.03147.
+
+    [1] Byun, Taejoon, and Sanjai Rayadurgam. “Manifold for Machine Learning Assurance.”
+    Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering
+    """
+    # Ensure proper data format
+    embeddings = ensure_embeddings(embeddings, dtype=np.float64, unit_interval=False)
+
+    # Get data dimensions
+    n, p = embeddings.shape
+    if quantiles > 2 or quantiles <= 0:
+        raise ValueError(
+            f"Number of quantiles ({quantiles}) is greater than 2 or is nonpositive. \
+            The metric scales exponentially in this value. Please 1 or 2 quantiles."
+        )
+    if p > 10:
+        raise ValueError(
+            f"Dimension of embeddings ({p}) is greater than 10. \
+            The metric scales exponentially in this value. Please reduce the embedding dimension."
+        )
+    if n == 0 or p == 0:
+        raise ValueError("Your provided embeddings do not contain any data!")
+    # n+2 edges partition the embedding dimension (e.g. [0,0.5,1] for quantiles = 1)
+    quantile_vec = np.linspace(0, 1, quantiles + 2)
+
+    # Calculate the bin edges for each dimension based on quantiles
+    bin_edges = []
+    for dim in range(p):
+        # Calculate the quantile values for this feature
+        edges = np.array(np.quantile(embeddings[:, dim], quantile_vec))
+        # Make sure the last bin contains all the remaining points
+        edges[-1] += EPSILON
+        bin_edges.append(edges)
+    # Convert each data point into its corresponding grid cell indices
+    grid_indices = []
+    for dim in range(p):
+        # For each dimension, find which bin each data point belongs to
+        # Digitize is 1 indexed so we subtract 1
+        indices = np.digitize(embeddings[:, dim], bin_edges[dim]) - 1
+        grid_indices.append(indices)
+
+    # Make the rows the data point and the column the grid index
+    grid_coords = np.array(grid_indices).T
+
+    # Use set to find unique tuple of grid coordinates
+    occupied_cells = set(map(tuple, grid_coords))
+
+    # For the fraction
+    num_occupied_cells = len(occupied_cells)
+
+    # Calculate total possible cells in the grid
+    num_bins_per_dim = [len(edges) - 1 for edges in bin_edges]
+    total_possible_cells = np.prod(num_bins_per_dim)
+
+    # Generate all possible grid cells
+    all_cells = set(itertools.product(*[range(bins) for bins in num_bins_per_dim]))
+
+    # Find the empty cells (cells with no data points)
+    empty_cells = all_cells - occupied_cells
+
+    # Calculate center points of empty boxes
+    empty_box_centers = []
+    for cell in empty_cells:
+        center_coords = []
+        for dim, idx in enumerate(cell):
+            # Calculate center of the bin as midpoint between edges
+            center = (bin_edges[dim][idx] + bin_edges[dim][idx + 1]) / 2
+            center_coords.append(center)
+        empty_box_centers.append(np.array(center_coords))
+
+    # Calculate the fraction
+    fraction = float(num_occupied_cells / total_possible_cells)
+    empty_box_centers = np.array(empty_box_centers)
+    return CompletenessOutput(fraction, empty_box_centers)

dataeval/metrics/stats/_base.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 __all__ = []
 
+import math
 import re
 import warnings
 from collections import ChainMap
@@ -9,7 +10,7 @@ from copy import deepcopy
 from dataclasses import dataclass
 from functools import partial
 from multiprocessing import Pool
-from typing import Any, Callable, Generic, Iterable, Sequence, TypeVar
+from typing import Any, Callable, Generic, Iterable, Sequence, TypeVar, cast
 
 import numpy as np
 import tqdm
@@ -23,20 +24,7 @@ from dataeval.utils._image import normalize_image_shape, rescale
 
 DTYPE_REGEX = re.compile(r"NDArray\[np\.(.*?)\]")
 
-
-def normalize_box_shape(bounding_box: NDArray[Any]) -> NDArray[Any]:
-    """
-    Normalizes the bounding box shape into (N,4).
-    """
-    ndim = bounding_box.ndim
-    if ndim == 1:
-        return np.expand_dims(bounding_box, axis=0)
-    elif ndim > 2:
-        raise ValueError("Bounding boxes must have 2 dimensions: (# of boxes in an image, [X,Y,W,H]) -> (N,4)")
-    else:
-        return bounding_box
-
-
+BoundingBox = tuple[float, float, float, float]
 TStatsOutput = TypeVar("TStatsOutput", bound=BaseStatsOutput, covariant=True)
 
 
@@ -46,11 +34,15 @@ class StatsProcessor(Generic[TStatsOutput]):
     image_function_map: dict[str, Callable[[StatsProcessor[TStatsOutput]], Any]] = {}
     channel_function_map: dict[str, Callable[[StatsProcessor[TStatsOutput]], Any]] = {}
 
-    def __init__(self, image: NDArray[Any], box: NDArray[Any] | None, per_channel: bool) -> None:
+    def __init__(self, image: NDArray[Any], box: BoundingBox | None, per_channel: bool) -> None:
         self.raw = image
         self.width: int = image.shape[-1]
         self.height: int = image.shape[-2]
-        self.box: NDArray[np.int64] = np.array([0, 0, self.width, self.height]) if box is None else box.astype(np.int64)
+        box = BoundingBox((0, 0, self.width, self.height)) if box is None else box
+        # Clip the bounding box to image
+        x0, y0 = (min(j, max(0, math.floor(box[i]))) for i, j in zip((0, 1), (self.width - 1, self.height - 1)))
+        x1, y1 = (min(j, max(1, math.ceil(box[i]))) for i, j in zip((2, 3), (self.width, self.height)))
+        self.box: NDArray[np.int64] = np.array([x0, y0, x1, y1], dtype=np.int64)
         self._per_channel = per_channel
         self._image = None
         self._shape = None
@@ -123,18 +115,16 @@ class StatsProcessorOutput:
 def process_stats(
     i: int,
     image: ArrayLike,
-    target: Any,
-    per_box: bool,
+    boxes: list[BoundingBox] | None,
     per_channel: bool,
     stats_processor_cls: Iterable[type[StatsProcessor[TStatsOutput]]],
 ) -> StatsProcessorOutput:
     image = to_numpy(image)
-    boxes = to_numpy(target.boxes) if isinstance(target, ObjectDetectionTarget) else None
     results_list: list[dict[str, Any]] = []
     source_indices: list[SourceIndex] = []
     box_counts: list[int] = []
     warnings_list: list[str] = []
-    for i_b, box in [(None, None)] if boxes is None else enumerate(normalize_box_shape(boxes)):
+    for i_b, box in [(None, None)] if boxes is None else enumerate(boxes):
         processor_list = [p(image, box, per_channel) for p in stats_processor_cls]
         if any(not p._is_valid_slice for p in processor_list) and i_b is not None and box is not None:
             warnings_list.append(f"Bounding box [{i}][{i_b}]: {box} is out of bounds of {image.shape}.")
@@ -148,12 +138,24 @@ def process_stats(
 
 
 def process_stats_unpack(
-    args: tuple[int, ArrayLike, Any],
-    per_box: bool,
+    args: tuple[int, Array, list[BoundingBox] | None],
     per_channel: bool,
     stats_processor_cls: Iterable[type[StatsProcessor[TStatsOutput]]],
 ) -> StatsProcessorOutput:
-    return process_stats(*args, per_box=per_box, per_channel=per_channel, stats_processor_cls=stats_processor_cls)
+    return process_stats(*args, per_channel=per_channel, stats_processor_cls=stats_processor_cls)
+
+
+def _enumerate(dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]], per_box: bool):
+    for i in range(len(dataset)):
+        d = dataset[i]
+        image = d[0] if isinstance(d, tuple) else d
+        if per_box and isinstance(d, tuple) and isinstance(d[1], ObjectDetectionTarget):
+            boxes = cast(Array, d[1].boxes)
+            target = [BoundingBox(float(box[i]) for i in range(4)) for box in boxes]
+        else:
+            target = None
+
+        yield i, image, target
 
 
 def run_stats(
@@ -202,17 +204,11 @@ def run_stats(
     warning_list = []
     stats_processor_cls = stats_processor_cls if isinstance(stats_processor_cls, Iterable) else [stats_processor_cls]
 
-    def _enumerate(dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]], per_box: bool):
-        for i in range(len(dataset)):
-            d = dataset[i]
-            yield i, d[0] if isinstance(d, tuple) else d, d[1] if isinstance(d, tuple) and per_box else None
-
     with Pool(processes=get_max_processes()) as p:
         for r in tqdm.tqdm(
             p.imap(
                 partial(
                     process_stats_unpack,
-                    per_box=per_box,
                     per_channel=per_channel,
                     stats_processor_cls=stats_processor_cls,
                 ),

dataeval/metrics/stats/_labelstats.py

@@ -5,54 +5,16 @@ __all__ = []
 from collections import Counter, defaultdict
 from typing import Any, Mapping, TypeVar
 
-import numpy as np
-
 from dataeval.outputs import LabelStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import AnnotatedDataset, ArrayLike
-from dataeval.utils._array import as_numpy
+from dataeval.typing import AnnotatedDataset
 from dataeval.utils.data._metadata import Metadata
 
 TValue = TypeVar("TValue")
 
 
-def _ensure_2d(labels: ArrayLike) -> ArrayLike:
-    if isinstance(labels, np.ndarray):
-        return labels[:, None]
-    else:
-        return [[lbl] for lbl in labels]  # type: ignore
-
-
-def _get_list_depth(lst):
-    if isinstance(lst, list) and lst:
-        return 1 + max(_get_list_depth(item) for item in lst)
-    return 0
-
-
-def _check_labels_dimension(labels: ArrayLike) -> ArrayLike:
-    # Check for nested lists beyond 2 levels
-
-    if isinstance(labels, np.ndarray):
-        if labels.ndim == 1:
-            return _ensure_2d(labels)
-        elif labels.ndim == 2:
-            return labels
-        else:
-            raise ValueError("The label array must not have more than 2 dimensions.")
-    elif isinstance(labels, list):
-        depth = _get_list_depth(labels)
-        if depth == 1:
-            return _ensure_2d(labels)
-        elif depth == 2:
-            return labels
-        else:
-            raise ValueError("The label list must not be empty or have more than 2 levels of nesting.")
-    else:
-        raise TypeError("Labels must be either a NumPy array or a list.")
-
-
 def _sort_to_list(d: Mapping[int, TValue]) -> list[TValue]:
-    return [v for _, v in sorted(d.items())]
+    return [t[1] for t in sorted(d.items())]
 
 
 @set_metadata
@@ -98,12 +60,9 @@ def labelstats(dataset: Metadata | AnnotatedDataset[Any]) -> LabelStatsOutput:
     label_per_image: list[int] = []
 
     index2label = dict(enumerate(dataset.class_names))
-    labels = [target.labels.tolist() for target in dataset.targets]
-
-    labels_2d = _check_labels_dimension(labels)
 
-    for i, group in enumerate(labels_2d):
-        group = as_numpy(group).tolist()
+    for i, target in enumerate(dataset.targets):
+        group = target.labels.tolist()
 
         # Count occurrences of each label in all sublists
         label_counts.update(group)

dataeval/outputs/__init__.py

@@ -4,7 +4,7 @@ as well as runtime metadata for reproducibility and logging.
 """
 
 from ._base import ExecutionMetadata
-from ._bias import BalanceOutput, CoverageOutput, DiversityOutput, LabelParityOutput, ParityOutput
+from ._bias import BalanceOutput, CompletenessOutput, CoverageOutput, DiversityOutput, LabelParityOutput, ParityOutput
 from ._drift import DriftMMDOutput, DriftOutput
 from ._estimators import BEROutput, ClustererOutput, DivergenceOutput, UAPOutput
 from ._linters import DuplicatesOutput, OutliersOutput
@@ -29,6 +29,7 @@ __all__ = [
     "ChannelStatsOutput",
     "ClustererOutput",
     "CoverageOutput",
+    "CompletenessOutput",
     "DimensionStatsOutput",
     "DivergenceOutput",
     "DiversityOutput",

dataeval/outputs/_bias.py

@@ -14,9 +14,10 @@ with contextlib.suppress(ImportError):
     from matplotlib.figure import Figure
 
 from dataeval.outputs._base import Output
-from dataeval.typing import ArrayLike
-from dataeval.utils._array import to_numpy
+from dataeval.typing import ArrayLike, Dataset
+from dataeval.utils._array import as_numpy, channels_first_to_last
 from dataeval.utils._plot import heatmap
+from dataeval.utils.data._images import Images
 
 TData = TypeVar("TData", np.float64, NDArray[np.float64])
 
@@ -107,13 +108,13 @@ class CoverageOutput(Output):
     critical_value_radii: NDArray[np.float64]
     coverage_radius: float
 
-    def plot(self, images: ArrayLike, top_k: int = 6) -> Figure:
+    def plot(self, images: Images[Any] | Dataset[Any], top_k: int = 6) -> Figure:
         """
         Plot the top k images together for visualization.
 
         Parameters
         ----------
-        images : ArrayLike
+        images : Images or Dataset
             Original images (not embeddings) in (N, C, H, W) or (N, H, W) format
         top_k : int, default 6
             Number of images to plot (plotting assumes groups of 3)
@@ -130,46 +131,54 @@ class CoverageOutput(Output):
         import matplotlib.pyplot as plt
 
         # Determine which images to plot
-        highest_uncovered_indices = self.uncovered_indices[:top_k]
+        selected_indices = self.uncovered_indices[:top_k]
 
-        # Grab the images
-        selected_images = to_numpy(images)[highest_uncovered_indices]
+        images = Images(images) if isinstance(images, Dataset) else images
 
         # Plot the images
-        num_images = min(top_k, len(images))
-
-        ndim = selected_images.ndim
-        if ndim == 4:
-            selected_images = np.moveaxis(selected_images, 1, -1)
-        elif ndim == 3:
-            selected_images = np.repeat(selected_images[:, :, :, np.newaxis], 3, axis=-1)
-        else:
-            raise ValueError(
-                f"Expected a (N,C,H,W) or a (N, H, W) set of images, but got a {ndim}-dimensional set of images."
-            )
+        num_images = min(top_k, len(selected_indices))
 
         rows = int(np.ceil(num_images / 3))
         fig, axs = plt.subplots(rows, 3, figsize=(9, 3 * rows))
 
         if rows == 1:
             for j in range(3):
-                if j >= len(selected_images):
+                if j >= len(selected_indices):
                     continue
-                axs[j].imshow(selected_images[j])
+                image = channels_first_to_last(as_numpy(images[selected_indices[j]]))
+                axs[j].imshow(image)
                 axs[j].axis("off")
         else:
             for i in range(rows):
                 for j in range(3):
                     i_j = i * 3 + j
-                    if i_j >= len(selected_images):
+                    if i_j >= len(selected_indices):
                         continue
-                    axs[i, j].imshow(selected_images[i_j])
+                    image = channels_first_to_last(as_numpy(images[selected_indices[i_j]]))
+                    axs[i, j].imshow(image)
                     axs[i, j].axis("off")
 
         fig.tight_layout()
         return fig
 
 
+@dataclass(frozen=True)
+class CompletenessOutput(Output):
+    """
+    Output from the completeness function.
+
+    Attributes
+    ----------
+    fraction_filled : float
+        Fraction of boxes that contain at least one data point
+    empty_box_centers : List[np.ndarray]
+        List of coordinates for centers of empty boxes
+    """
+
+    fraction_filled: float
+    empty_box_centers: NDArray[np.float64]
+
+
 @dataclass(frozen=True)
 class BalanceOutput(Output):
     """

dataeval/outputs/_stats.py

@@ -4,7 +4,7 @@ __all__ = []
 
 import contextlib
 from dataclasses import dataclass
-from typing import Any, Iterable, Optional, Union
+from typing import Any, Iterable, NamedTuple, Optional, Union
 
 import numpy as np
 from numpy.typing import NDArray
@@ -22,8 +22,7 @@ SOURCE_INDEX = "source_index"
 BOX_COUNT = "box_count"
 
 
-@dataclass(frozen=True)
-class SourceIndex:
+class SourceIndex(NamedTuple):
     """
     The indices of the source image, box and channel.
 

dataeval/typing.py

@@ -23,7 +23,7 @@ __all__ = [
 import sys
 from typing import Any, Generic, Iterator, Protocol, Sequence, TypedDict, TypeVar, Union, runtime_checkable
 
-from typing_extensions import NotRequired, Required
+from typing_extensions import NotRequired, ReadOnly, Required
 
 if sys.version_info >= (3, 10):
     from typing import TypeAlias
@@ -91,8 +91,8 @@ class DatasetMetadata(TypedDict, total=False):
         A lookup table converting label value to class name
     """
 
-    id: Required[str]
-    index2label: NotRequired[dict[int, str]]
+    id: Required[ReadOnly[str]]
+    index2label: NotRequired[ReadOnly[dict[int, str]]]
 
 
 @runtime_checkable

dataeval/utils/_array.py

@@ -13,7 +13,7 @@ import torch
 from numpy.typing import NDArray
 
 from dataeval._log import LogMessage
-from dataeval.typing import ArrayLike
+from dataeval.typing import Array, ArrayLike
 
 _logger = logging.getLogger(__name__)
 
@@ -167,3 +167,28 @@ def flatten(array: ArrayLike) -> NDArray[Any]:
     """
     nparr = as_numpy(array)
     return nparr.reshape((nparr.shape[0], -1))
+
+
+_TArray = TypeVar("_TArray", bound=Array)
+
+
+def channels_first_to_last(array: _TArray) -> _TArray:
+    """
+    Converts array from channels first to channels last format
+
+    Parameters
+    ----------
+    array : ArrayLike
+        Input array
+
+    Returns
+    -------
+    ArrayLike
+        Converted array
+    """
+    if isinstance(array, np.ndarray):
+        return np.transpose(array, (1, 2, 0))
+    elif isinstance(array, torch.Tensor):
+        return torch.permute(array, (1, 2, 0))
+    else:
+        raise TypeError(f"Unsupported array type {type(array)} for conversion.")

dataeval/utils/data/_dataset.py

@@ -47,6 +47,8 @@ def _validate_data(
             or not len(bboxes[0][0]) == 4
         ):
             raise TypeError("Boxes must be a sequence of sequences of (x0, y0, x1, y1) for object detection.")
+    else:
+        raise ValueError(f"Unknown datum type '{datum_type}'. Must be 'ic' or 'od'.")
 
 
 def _find_max(arr: ArrayLike) -> Any: