dataeval 0.86.0__py3-none-any.whl → 0.86.1__py3-none-any.whl

dataeval/utils/_clusterer.py

@@ -4,6 +4,7 @@ __all__ = []
 
 import warnings
 from dataclasses import dataclass
+from typing import Any
 
 import numba
 import numpy as np
@@ -30,7 +31,9 @@ from dataeval.utils._fast_mst import calculate_neighbor_distances, minimum_spann
 
 
 @numba.njit(parallel=True, locals={"i": numba.types.int32})
-def compare_links_to_cluster_std(mst, clusters):
+def compare_links_to_cluster_std(
+    mst: NDArray[np.float32], clusters: NDArray[np.intp]
+) -> tuple[NDArray[np.int32], NDArray[np.int32]]:
     cluster_ids = np.unique(clusters)
     cluster_grouping = np.full(mst.shape[0], -1, dtype=np.int16)
 
@@ -79,7 +82,7 @@ def cluster(data: ArrayLike) -> ClusterData:
     cluster_selection_epsilon = 0.0
     # cluster_selection_method = "eom"
 
-    x = flatten(to_numpy(data))
+    x: NDArray[Any] = flatten(to_numpy(data))
     samples, features = x.shape  # Due to flatten(), we know shape has a length of 2
     if samples < 2:
         raise ValueError(f"Data should have at least 2 samples; got {samples}")
@@ -125,9 +128,9 @@ def cluster(data: ArrayLike) -> ClusterData:
     return ClusterData(clusters, mst, linkage_tree, condensed_tree, membership_strengths, kneighbors, kdistances)
 
 
-def sorted_union_find(index_groups):
+def sorted_union_find(index_groups: NDArray[np.int32]) -> list[list[np.int32]]:
     """Merges and sorts groups of indices that share any common index"""
-    groups = [[np.int32(x) for x in range(0)] for y in range(0)]
+    groups: list[list[np.int32]] = [[np.int32(x) for x in range(0)] for y in range(0)]
     uniques, inverse = np.unique(index_groups, return_inverse=True)
     inverse = inverse.flatten()
     disjoint_set = ds_rank_create(uniques.size)

dataeval/utils/_fast_mst.py

@@ -6,9 +6,11 @@
 __all__ = []
 
 import warnings
+from typing import Any
 
 import numba
 import numpy as np
+from numpy.typing import NDArray
 from sklearn.neighbors import NearestNeighbors
 
 with warnings.catch_warnings():
@@ -17,24 +19,26 @@ with warnings.catch_warnings():
 
 
 @numba.njit()
-def _ds_union_by_rank(disjoint_set, point, nbr):
+def _ds_union_by_rank(disjoint_set: tuple[NDArray[np.int32], NDArray[np.int32]], point: int, nbr: int) -> int:
     y = ds_find(disjoint_set, point)
     x = ds_find(disjoint_set, nbr)
 
     if x == y:
         return 0
 
-    if disjoint_set.rank[x] < disjoint_set.rank[y]:
+    if disjoint_set[1][x] < disjoint_set[1][y]:
         x, y = y, x
 
-    disjoint_set.parent[y] = x
-    if disjoint_set.rank[x] == disjoint_set.rank[y]:
-        disjoint_set.rank[x] += 1
+    disjoint_set[0][y] = x
+    if disjoint_set[1][x] == disjoint_set[1][y]:
+        disjoint_set[1][x] += 1
     return 1
 
 
 @numba.njit(locals={"i": numba.types.uint32, "nbr": numba.types.uint32, "dist": numba.types.float32})
-def _init_tree(n_neighbors, n_distance):
+def _init_tree(
+    n_neighbors: NDArray[np.intp], n_distance: NDArray[np.float32]
+) -> tuple[NDArray[np.float32], int, tuple[NDArray[np.int32], NDArray[np.int32]], NDArray[np.uint32]]:
     # Initial graph to hold tree connections
     tree = np.zeros((n_neighbors.size - 1, 3), dtype=np.float32)
     disjoint_set = ds_rank_create(n_neighbors.size)
@@ -56,7 +60,13 @@ def _init_tree(n_neighbors, n_distance):
 
 
 @numba.njit(locals={"i": numba.types.uint32, "nbr": numba.types.uint32})
-def _update_tree_by_distance(tree, int_tree, disjoint_set, n_neighbors, n_distance):
+def _update_tree_by_distance(
+    tree: NDArray[np.float32],
+    int_tree: int,
+    disjoint_set: tuple[NDArray[np.int32], NDArray[np.int32]],
+    n_neighbors: NDArray[np.uint32],
+    n_distance: NDArray[np.float32],
+) -> tuple[NDArray[np.float32], int, tuple[NDArray[np.int32], NDArray[np.int32]], NDArray[np.uint32]]:
     cluster_points = np.empty(n_neighbors.size, dtype=np.uint32)
     sort_dist = np.argsort(n_distance)
     dist_sorted = n_distance[sort_dist]
@@ -80,9 +90,9 @@ def _update_tree_by_distance(tree, int_tree, disjoint_set, n_neighbors, n_distan
 
 
 @numba.njit(locals={"i": numba.types.uint32})
-def _cluster_edges(tracker, last_idx, cluster_distances):
+def _cluster_edges(tracker: NDArray[Any], last_idx: int, cluster_distances: NDArray[Any]) -> list[NDArray[np.intp]]:
     cluster_ids = np.unique(tracker)
-    edge_points = []
+    edge_points: list[NDArray[np.intp]] = []
     for idx in range(cluster_ids.size):
         cluster_points = np.nonzero(tracker == cluster_ids[idx])[0]
         cluster_size = cluster_points.size
@@ -102,14 +112,16 @@ def _cluster_edges(tracker, last_idx, cluster_distances):
     return edge_points
 
 
-def _compute_nn(dataA, dataB, k):
+def _compute_nn(dataA: NDArray[Any], dataB: NDArray[Any], k: int) -> tuple[NDArray[np.int32], NDArray[np.float32]]:
     distances, neighbors = NearestNeighbors(n_neighbors=k + 1, algorithm="brute").fit(dataA).kneighbors(dataB)
     neighbors = np.array(neighbors[:, 1 : k + 1], dtype=np.int32)
     distances = np.array(distances[:, 1 : k + 1], dtype=np.float32)
     return neighbors, distances
 
 
-def _calculate_cluster_neighbors(data, groups, point_array):
+def _calculate_cluster_neighbors(
+    data: NDArray[Any], groups: list[NDArray[np.intp]], point_array: NDArray[Any]
+) -> tuple[NDArray[np.uint32], NDArray[np.float32]]:
     """Rerun nearest neighbor based on clusters"""
     cluster_neighbors = np.zeros(point_array.size, dtype=np.uint32)
     cluster_nbr_distances = np.full(point_array.size, np.inf, dtype=np.float32)
@@ -126,7 +138,9 @@ def _calculate_cluster_neighbors(data, groups, point_array):
     return cluster_neighbors, cluster_nbr_distances
 
 
-def minimum_spanning_tree(data, neighbors, distances):
+def minimum_spanning_tree(
+    data: NDArray[Any], neighbors: NDArray[np.int32], distances: NDArray[np.float32]
+) -> NDArray[np.float32]:
     # Transpose arrays to get number of samples along a row
     k_neighbors = neighbors.T.astype(np.uint32).copy()
     k_distances = distances.T.astype(np.float32).copy()
@@ -168,7 +182,7 @@ def minimum_spanning_tree(data, neighbors, distances):
     return tree
 
 
-def calculate_neighbor_distances(data: np.ndarray, k: int = 10):
+def calculate_neighbor_distances(data: np.ndarray, k: int = 10) -> tuple[NDArray[np.int32], NDArray[np.float32]]:
     # Have the potential to add in other distance calculations - supported calculations:
     # https://github.com/lmcinnes/pynndescent/blob/master/pynndescent/pynndescent_.py#L524
     try:

dataeval/utils/_image.py

@@ -12,6 +12,9 @@ from scipy.signal import convolve2d
 EDGE_KERNEL = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype=np.int8)
 BIT_DEPTH = (1, 8, 12, 16, 32)
 
+Box = tuple[int, int, int, int]
+"""Bounding box as tuple of integers in x0, y0, x1, y1 format."""
+
 
 @dataclass
 class BitDepth:
@@ -25,12 +28,11 @@ def get_bitdepth(image: NDArray[Any]) -> BitDepth:
     Approximates the bit depth of the image using the
     min and max pixel values.
     """
-    pmin, pmax = np.min(image), np.max(image)
+    pmin, pmax = np.nanmin(image), np.nanmax(image)
     if pmin < 0:
         return BitDepth(0, pmin, pmax)
-    else:
-        depth = ([x for x in BIT_DEPTH if 2**x > pmax] or [max(BIT_DEPTH)])[0]
-        return BitDepth(depth, 0, 2**depth - 1)
+    depth = ([x for x in BIT_DEPTH if 2**x > pmax] or [max(BIT_DEPTH)])[0]
+    return BitDepth(depth, 0, 2**depth - 1)
 
 
 def rescale(image: NDArray[Any], depth: int = 1) -> NDArray[Any]:
@@ -40,9 +42,8 @@ def rescale(image: NDArray[Any], depth: int = 1) -> NDArray[Any]:
     bitdepth = get_bitdepth(image)
     if bitdepth.depth == depth:
         return image
-    else:
-        normalized = (image + bitdepth.pmin) / (bitdepth.pmax - bitdepth.pmin)
-        return normalized * (2**depth - 1)
+    normalized = (image + bitdepth.pmin) / (bitdepth.pmax - bitdepth.pmin)
+    return normalized * (2**depth - 1)
 
 
 def normalize_image_shape(image: NDArray[Any]) -> NDArray[Any]:
@@ -52,13 +53,12 @@ def normalize_image_shape(image: NDArray[Any]) -> NDArray[Any]:
     ndim = image.ndim
     if ndim == 2:
         return np.expand_dims(image, axis=0)
-    elif ndim == 3:
+    if ndim == 3:
         return image
-    elif ndim > 3:
+    if ndim > 3:
         # Slice all but the last 3 dimensions
         return image[(0,) * (ndim - 3)]
-    else:
-        raise ValueError("Images must have 2 or more dimensions.")
+    raise ValueError("Images must have 2 or more dimensions.")
 
 
 def edge_filter(image: NDArray[Any], offset: float = 0.5) -> NDArray[np.uint8]:
@@ -71,3 +71,57 @@ def edge_filter(image: NDArray[Any], offset: float = 0.5) -> NDArray[np.uint8]:
     edges = convolve2d(image, EDGE_KERNEL, mode="same", boundary="symm") + offset
     np.clip(edges, 0, 255, edges)
     return edges
+
+
+def clip_box(image: NDArray[Any], box: Box) -> Box:
+    """
+    Clip the box to inside the provided image dimensions.
+    """
+    x0, y0, x1, y1 = box
+    h, w = image.shape[-2:]
+
+    return max(0, x0), max(0, y0), min(w, x1), min(h, y1)
+
+
+def is_valid_box(box: Box) -> bool:
+    """
+    Check if the box dimensions provided are a valid image.
+    """
+    return box[2] > box[0] and box[3] > box[1]
+
+
+def clip_and_pad(image: NDArray[Any], box: Box) -> NDArray[Any]:
+    """
+    Extract a region from an image based on a bounding box, clipping to image boundaries
+    and padding out-of-bounds areas with np.nan.
+
+    Parameters:
+    -----------
+    image : NDArray[Any]
+        Input image array in format C, H, W (channels first)
+    box : Box
+        Bounding box coordinates as (x0, y0, x1, y1) where (x0, y0) is top-left and (x1, y1) is bottom-right
+
+    Returns:
+    --------
+    NDArray[Any]
+        The extracted region with out-of-bounds areas padded with np.nan
+    """
+
+    # Create output array filled with NaN with a minimum size of 1x1
+    bw, bh = max(1, box[2] - box[0]), max(1, box[3] - box[1])
+
+    output = np.full((image.shape[-3] if image.ndim > 2 else 1, bh, bw), np.nan)
+
+    # Calculate source box
+    sbox = clip_box(image, box)
+
+    # Calculate destination box
+    x0, y0 = sbox[0] - box[0], sbox[1] - box[1]
+    x1, y1 = x0 + (sbox[2] - sbox[0]), y0 + (sbox[3] - sbox[1])
+
+    # Copy the source if valid from the image to the output
+    if is_valid_box(sbox):
+        output[:, y0:y1, x0:x1] = image[:, sbox[1] : sbox[3], sbox[0] : sbox[2]]
+
+    return output

dataeval/utils/_mst.py

@@ -83,6 +83,4 @@ def compute_neighbors(
 
     nbrs = NearestNeighbors(n_neighbors=k + 1, algorithm=algorithm).fit(B)
     nns = nbrs.kneighbors(A)[1]
-    nns = nns[:, 1:].squeeze()
-
-    return nns
+    return nns[:, 1:].squeeze()

dataeval/utils/_plot.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 __all__ = []
 
 import contextlib
+import math
 from typing import Any
 
 import numpy as np
@@ -160,11 +161,9 @@ def histogram_plot(
     import matplotlib.pyplot as plt
 
     num_metrics = len(data_dict)
-    if num_metrics > 2:
-        rows = int(len(data_dict) / 3)
-        fig, axs = plt.subplots(rows, 3, figsize=(10, rows * 2.5))
-    else:
-        fig, axs = plt.subplots(1, num_metrics, figsize=(4 * num_metrics, 4))
+    rows = math.ceil(num_metrics / 3)
+    cols = min(num_metrics, 3)
+    fig, axs = plt.subplots(rows, 3, figsize=(cols * 3 + 1, rows * 3))
 
     for ax, metric in zip(
         axs.flat,
@@ -178,6 +177,10 @@ def histogram_plot(
         ax.set_ylabel(ylabel)
         ax.set_xlabel(xlabel)
 
+    for ax in axs.flat[num_metrics:]:
+        ax.axis("off")
+        ax.set_visible(False)
+
     fig.tight_layout()
     return fig
 
@@ -216,11 +219,9 @@ def channel_histogram_plot(
     label_kwargs = {"label": [f"Channel {i}" for i in range(max_channels)]}
 
     num_metrics = len(data_keys)
-    if num_metrics > 2:
-        rows = int(len(data_keys) / 3)
-        fig, axs = plt.subplots(rows, 3, figsize=(10, rows * 2.5))
-    else:
-        fig, axs = plt.subplots(1, num_metrics, figsize=(4 * num_metrics, 4))
+    rows = math.ceil(num_metrics / 3)
+    cols = min(num_metrics, 3)
+    fig, axs = plt.subplots(rows, 3, figsize=(cols * 3 + 1, rows * 3))
 
     for ax, metric in zip(
         axs.flat,
@@ -245,5 +246,9 @@ def channel_histogram_plot(
         ax.set_ylabel(ylabel)
         ax.set_xlabel(xlabel)
 
+    for ax in axs.flat[num_metrics:]:
+        ax.axis("off")
+        ax.set_visible(False)
+
     fig.tight_layout()
     return fig

dataeval/utils/data/_dataset.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 __all__ = []
 
-from typing import Any, Generic, Iterable, Literal, Sequence, TypeVar
+from typing import Any, Generic, Iterable, Literal, Sequence, SupportsFloat, SupportsInt, TypeVar, cast
 
 from dataeval.typing import (
     Array,
@@ -17,8 +17,8 @@ from dataeval.utils._array import as_numpy
 def _validate_data(
     datum_type: Literal["ic", "od"],
     images: Array | Sequence[Array],
-    labels: Sequence[int] | Sequence[Sequence[int]],
-    bboxes: Sequence[Sequence[Sequence[float]]] | None,
+    labels: Array | Sequence[int] | Sequence[Array] | Sequence[Sequence[int]],
+    bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]] | None,
     metadata: Sequence[dict[str, Any]] | None,
 ) -> None:
     # Validate inputs
@@ -34,16 +34,21 @@ def _validate_data(
         raise ValueError(f"Number of metadata ({len(metadata)}) does not match number of images ({dataset_len}).")
 
     if datum_type == "ic":
-        if not isinstance(labels, Sequence) or not isinstance(labels[0], int):
+        if not isinstance(labels, (Sequence, Array)) or not isinstance(labels[0], (int, SupportsInt)):
             raise TypeError("Labels must be a sequence of integers for image classification.")
     elif datum_type == "od":
-        if not isinstance(labels, Sequence) or not isinstance(labels[0], Sequence) or not isinstance(labels[0][0], int):
+        if (
+            not isinstance(labels, (Sequence, Array))
+            or not isinstance(labels[0], (Sequence, Array))
+            or not isinstance(cast(Sequence[Any], labels[0])[0], (int, SupportsInt))
+        ):
             raise TypeError("Labels must be a sequence of sequences of integers for object detection.")
         if (
             bboxes is None
             or not isinstance(bboxes, (Sequence, Array))
             or not isinstance(bboxes[0], (Sequence, Array))
             or not isinstance(bboxes[0][0], (Sequence, Array))
+            or not isinstance(bboxes[0][0][0], (float, SupportsFloat))
             or not len(bboxes[0][0]) == 4
         ):
             raise TypeError("Boxes must be a sequence of sequences of (x0, y0, x1, y1) for object detection.")
@@ -52,11 +57,10 @@ def _validate_data(
 
 
 def _find_max(arr: ArrayLike) -> Any:
-    if isinstance(arr, (Iterable, Sequence, Array)):
+    if not isinstance(arr, (bytes, str)) and isinstance(arr, (Iterable, Sequence, Array)):
         if isinstance(arr[0], (Iterable, Sequence, Array)):
             return max([_find_max(x) for x in arr])  # type: ignore
-        else:
-            return max(arr)
+        return max(arr)
     return arr
 
 
@@ -92,12 +96,14 @@ class CustomImageClassificationDataset(BaseAnnotatedDataset[Sequence[int]], Imag
     def __init__(
         self,
         images: Array | Sequence[Array],
-        labels: Sequence[int],
+        labels: Array | Sequence[int],
         metadata: Sequence[dict[str, Any]] | None,
         classes: Sequence[str] | None,
         name: str | None = None,
     ) -> None:
-        super().__init__("ic", images, labels, metadata, classes)
+        super().__init__(
+            "ic", images, as_numpy(labels).tolist() if isinstance(labels, Array) else labels, metadata, classes
+        )
         if name is not None:
             self.__name__ = name
             self.__class__.__name__ = name
@@ -135,18 +141,24 @@ class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]]
     def __init__(
         self,
         images: Array | Sequence[Array],
-        labels: Sequence[Sequence[int]],
-        bboxes: Sequence[Sequence[Sequence[float]]],
+        labels: Array | Sequence[Array] | Sequence[Sequence[int]],
+        bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]],
         metadata: Sequence[dict[str, Any]] | None,
         classes: Sequence[str] | None,
         name: str | None = None,
     ) -> None:
-        super().__init__("od", images, labels, metadata, classes)
+        super().__init__(
+            "od",
+            images,
+            [as_numpy(label).tolist() if isinstance(label, Array) else label for label in labels],
+            metadata,
+            classes,
+        )
         if name is not None:
             self.__name__ = name
             self.__class__.__name__ = name
             self.__class__.__qualname__ = name
-        self._bboxes = bboxes
+        self._bboxes = [[as_numpy(box).tolist() if isinstance(box, Array) else box for box in bbox] for bbox in bboxes]
 
     @property
     def metadata(self) -> DatasetMetadata:
@@ -162,7 +174,7 @@ class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]]
 
 def to_image_classification_dataset(
     images: Array | Sequence[Array],
-    labels: Sequence[int],
+    labels: Array | Sequence[int],
     metadata: Sequence[dict[str, Any]] | None,
     classes: Sequence[str] | None,
     name: str | None = None,
@@ -174,7 +186,7 @@ def to_image_classification_dataset(
     ----------
     images : Array | Sequence[Array]
         The images to use in the dataset.
-    labels : Sequence[int]
+    labels : Array | Sequence[int]
         The labels to use in the dataset.
     metadata : Sequence[dict[str, Any]] | None
         The metadata to use in the dataset.
@@ -191,8 +203,8 @@ def to_image_classification_dataset(
 
 def to_object_detection_dataset(
     images: Array | Sequence[Array],
-    labels: Sequence[Sequence[int]],
-    bboxes: Sequence[Sequence[Sequence[float]]],
+    labels: Array | Sequence[Array] | Sequence[Sequence[int]],
+    bboxes: Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]],
     metadata: Sequence[dict[str, Any]] | None,
     classes: Sequence[str] | None,
     name: str | None = None,
@@ -204,9 +216,9 @@ def to_object_detection_dataset(
     ----------
     images : Array | Sequence[Array]
         The images to use in the dataset.
-    labels : Sequence[Sequence[int]]
+    labels : Array | Sequence[Array] | Sequence[Sequence[int]]
         The labels to use in the dataset.
-    bboxes : Sequence[Sequence[Sequence[float]]]
+    bboxes : Array | Sequence[Array] | Sequence[Sequence[Array]] | Sequence[Sequence[Sequence[float]]]
         The bounding boxes (x0,y0,x1,y0) to use in the dataset.
     metadata : Sequence[dict[str, Any]] | None
         The metadata to use in the dataset.