dataeval 0.86.9__py3-none-any.whl → 0.88.0__py3-none-any.whl

dataeval/outputs/_linters.py

@@ -2,11 +2,11 @@ from __future__ import annotations
 
 __all__ = []
 
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
-from typing import Generic, Mapping, Sequence, TypeVar, Union
+from typing import Generic, TypeAlias, TypeVar
 
 import pandas as pd
-from typing_extensions import TypeAlias
 
 from dataeval.outputs._base import Output
 from dataeval.outputs._stats import DimensionStatsOutput, LabelStatsOutput, PixelStatsOutput, VisualStatsOutput
@@ -16,7 +16,7 @@ DatasetDuplicateGroupMap: TypeAlias = Mapping[int, DuplicateGroup]
 TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateGroupMap)
 
 IndexIssueMap: TypeAlias = Mapping[int, Mapping[str, float]]
-OutlierStatsOutput: TypeAlias = Union[DimensionStatsOutput, PixelStatsOutput, VisualStatsOutput]
+OutlierStatsOutput: TypeAlias = DimensionStatsOutput | PixelStatsOutput | VisualStatsOutput
 TIndexIssueMap = TypeVar("TIndexIssueMap", IndexIssueMap, Sequence[IndexIssueMap])
 
 

dataeval/outputs/_stats.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 __all__ = []
 
+from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Iterable, Mapping, NamedTuple, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias
 
 import numpy as np
 import polars as pl
 from numpy.typing import NDArray
-from typing_extensions import TypeAlias
 
 from dataeval.outputs._base import Output
 from dataeval.utils._plot import channel_histogram_plot, histogram_plot
@@ -16,7 +16,7 @@ from dataeval.utils._plot import channel_histogram_plot, histogram_plot
 if TYPE_CHECKING:
     from matplotlib.figure import Figure
 
-OptionalRange: TypeAlias = Optional[Union[int, Iterable[int]]]
+OptionalRange: TypeAlias = int | Iterable[int] | None
 
 SOURCE_INDEX = "source_index"
 OBJECT_COUNT = "object_count"

dataeval/outputs/_utils.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 __all__ = []
 
+from collections.abc import Sequence
 from dataclasses import dataclass
-from typing import Sequence
 
 import numpy as np
 from numpy.typing import NDArray

dataeval/outputs/_workflows.py

@@ -4,8 +4,9 @@ __all__ = []
 
 import contextlib
 import warnings
-from dataclasses import dataclass
-from typing import Any, Iterable, Mapping, Sequence, cast
+from collections.abc import Iterable, Mapping, MutableMapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any, cast
 
 import numpy as np
 from numpy.typing import NDArray
@@ -76,10 +77,8 @@ def plot_measure(
     ax.set_title(f"{name} Sufficiency")
     ax.set_ylabel(f"{name}")
     ax.set_xlabel("Steps")
-
     # Plot measure over each step
     ax.scatter(steps, measure, label=f"Model Results ({name})", s=15, c="black")
-
     # Plot extrapolation
     ax.plot(
         projection,
@@ -92,7 +91,7 @@ def plot_measure(
     return fig
 
 
-def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
+def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.int64]:
     """
     Inverse function for f_out()
 
@@ -106,13 +105,27 @@ def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
     Returns
     -------
     NDArray
-        Array of sample sizes
+        Sample size or -1 if unachievable for each data point
     """
-    n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
-    return np.asarray(n_i, dtype=np.uint64)
+    with np.errstate(invalid="ignore"):
+        n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
+    unachievable_targets = np.isnan(n_i) | np.any(n_i > np.iinfo(np.int64).max)
+    if any(unachievable_targets):
+        with np.printoptions(suppress=True):
+            warnings.warn(
+                "Number of samples could not be determined for target(s): "
+                f"""{
+                    np.array2string(
+                        1 - y_i[unachievable_targets], separator=", ", formatter={"float": lambda x: f"{x}"}
+                    )
+                }""",
+                UserWarning,
+            )
+        n_i[unachievable_targets] = -1
+    return np.asarray(n_i, dtype=np.int64)
 
 
-def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.uint64]:
+def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.int64]:
     """Inverse function for project_steps()
 
     Parameters
@@ -125,14 +138,13 @@ def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np
     Returns
     -------
     NDArray
-        Array of sample sizes, or 0 if overflow
+        Samples required or -1 if unachievable for each target value
     """
     steps = f_inv_out(1 - np.array(targets), params)
-    steps[np.isnan(steps)] = 0
     return np.ceil(steps)
 
 
-def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[Any]:
+def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[np.float64]:
     """
     Retrieves the inverse power curve coefficients for the line of best fit.
     Global minimization is done via basin hopping. More info on this algorithm
@@ -178,11 +190,11 @@ def calc_params(p_i: NDArray[Any], n_i: NDArray[Any], niter: int) -> NDArray[Any
 
 
 def get_curve_params(
-    measures: Mapping[str, NDArray[Any]], ranges: NDArray[Any], niter: int
-) -> Mapping[str, NDArray[Any]]:
+    averaged_measures: MutableMapping[str, NDArray[Any]], ranges: NDArray[Any], niter: int
+) -> Mapping[str, NDArray[np.float64]]:
     """Calculates and aggregates parameters for both single and multi-class metrics"""
     output = {}
-    for name, measure in measures.items():
+    for name, measure in averaged_measures.items():
         measure = cast(np.ndarray, measure)
         if measure.ndim > 1:
             result = []
@@ -203,19 +215,25 @@ class SufficiencyOutput(Output):
     ----------
     steps : NDArray
         Array of sample sizes
-    measures : Dict[str, NDArray]
-        Average of values observed for each sample size step for each measure
+    measures : dict[str, NDArray]
+        3D array [runs, substep, classes] of values for all runs observed for each sample size step for each measure
+    averaged_measures : dict[str, NDArray]
+        Average of values for all runs observed for each sample size step for each measure
     n_iter : int, default 1000
         Number of iterations to perform in the basin-hopping curve-fit process
     """
 
     steps: NDArray[np.uint32]
-    measures: Mapping[str, NDArray[np.float64]]
+    measures: Mapping[str, NDArray[Any]]
+    averaged_measures: MutableMapping[str, NDArray[Any]] = field(default_factory=lambda: {})
     n_iter: int = 1000
 
     def __post_init__(self) -> None:
+        if len(self.averaged_measures) == 0:
+            for metric, values in self.measures.items():
+                self.averaged_measures[metric] = np.asarray(np.mean(values, axis=0)).T
         c = len(self.steps)
-        for m, v in self.measures.items():
+        for m, v in self.averaged_measures.items():
             c_v = v.shape[1] if v.ndim > 1 else len(v)
             if c != c_v:
                 raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
@@ -226,7 +244,7 @@ class SufficiencyOutput(Output):
         if self._params is None:
             self._params = {}
         if self.n_iter not in self._params:
-            self._params[self.n_iter] = get_curve_params(self.measures, self.steps, self.n_iter)
+            self._params[self.n_iter] = get_curve_params(self.averaged_measures, self.steps, self.n_iter)
         return self._params[self.n_iter]
 
     @set_metadata
@@ -259,16 +277,16 @@ class SufficiencyOutput(Output):
             raise ValueError("'projection' must consist of numerical values")
 
         output = {}
-        for name, measures in self.measures.items():
-            if measures.ndim > 1:
+        for name, averaged_measures in self.averaged_measures.items():
+            if averaged_measures.ndim > 1:
                 result = []
-                for i in range(len(measures)):
+                for i in range(len(averaged_measures)):
                     projected = project_steps(self.params[name][i], projection)
                     result.append(projected)
                 output[name] = np.array(result)
             else:
                 output[name] = project_steps(self.params[name], projection)
-        proj = SufficiencyOutput(projection, output, self.n_iter)
+        proj = SufficiencyOutput(projection, measures=self.measures, averaged_measures=output, n_iter=self.n_iter)
         proj._params = self._params
         return proj
 
@@ -304,11 +322,11 @@ class SufficiencyOutput(Output):
         plots = []
 
         # Create a plot for each measure on one figure
-        for name, measures in self.measures.items():
-            if measures.ndim > 1:
-                if class_names is not None and len(measures) != len(class_names):
+        for name, averaged_measures in self.averaged_measures.items():
+            if averaged_measures.ndim > 1:
+                if class_names is not None and len(averaged_measures) != len(class_names):
                     raise IndexError("Class name count does not align with measures")
-                for i, measure in enumerate(measures):
+                for i, measure in enumerate(averaged_measures):
                     class_name = str(i) if class_names is None else class_names[i]
                     fig = plot_measure(
                         f"{name}_{class_name}",
@@ -320,7 +338,7 @@ class SufficiencyOutput(Output):
                     plots.append(fig)
 
             else:
-                fig = plot_measure(name, self.steps, measures, self.params[name], extrapolated)
+                fig = plot_measure(name, self.steps, averaged_measures, self.params[name], extrapolated)
                 plots.append(fig)
 
         return plots
@@ -350,10 +368,10 @@ class SufficiencyOutput(Output):
 
         for name, target in targets.items():
             tarray = as_numpy(target)
-            if name not in self.measures:
+            if name not in self.averaged_measures:
                 continue
 
-            measure = self.measures[name]
+            measure = self.averaged_measures[name]
             if measure.ndim > 1:
                 projection[name] = np.zeros((len(measure), len(tarray)))
                 for i in range(len(measure)):

dataeval/typing.py

@@ -3,11 +3,12 @@ Common type protocols used for interoperability with DataEval.
 """
 
 __all__ = [
+    "AnnotatedDataset",
     "Array",
     "ArrayLike",
     "Dataset",
-    "AnnotatedDataset",
     "DatasetMetadata",
+    "DeviceLike",
     "ImageClassificationDatum",
     "ImageClassificationDataset",
     "ObjectDetectionTarget",
@@ -20,18 +21,21 @@ __all__ = [
 ]
 
 
-import sys
-from typing import Any, Generic, Iterator, Mapping, Protocol, TypedDict, TypeVar, runtime_checkable
+from collections.abc import Iterator, Mapping
+from typing import (
+    Any,
+    Generic,
+    Protocol,
+    TypeAlias,
+    TypedDict,
+    TypeVar,
+    runtime_checkable,
+)
 
 import numpy.typing
+import torch
 from typing_extensions import NotRequired, ReadOnly, Required
 
-if sys.version_info >= (3, 10):
-    from typing import TypeAlias
-else:
-    from typing_extensions import TypeAlias
-
-
 ArrayLike: TypeAlias = numpy.typing.ArrayLike
 """
 Type alias for a `Union` representing objects that can be coerced into an array.
@@ -42,6 +46,16 @@ See Also
 """
 
 
+DeviceLike: TypeAlias = int | str | tuple[str, int] | torch.device
+"""
+Type alias for a `Union` representing types that specify a torch.device.
+
+See Also
+--------
+`torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
+"""
+
+
 @runtime_checkable
 class Array(Protocol):
     """

dataeval/utils/__init__.py

@@ -4,6 +4,6 @@ in setting up data and architectures that are guaranteed to work with applicable
 DataEval metrics.
 """
 
-__all__ = ["data", "datasets", "torch"]
+__all__ = ["data", "torch"]
 
-from . import data, datasets, torch
+from . import data, torch

dataeval/utils/_array.py

@@ -4,9 +4,10 @@ __all__ = []
 
 import logging
 import warnings
+from collections.abc import Iterable, Iterator
 from importlib import import_module
 from types import ModuleType
-from typing import Any, Iterable, Iterator, Literal, TypeVar, overload
+from typing import Any, Literal, TypeVar, overload
 
 import numpy as np
 import torch
@@ -79,7 +80,7 @@ def rescale_array(array: NDArray[_np_dtype]) -> NDArray[_np_dtype]: ...
 def rescale_array(array: torch.Tensor) -> torch.Tensor: ...
 def rescale_array(array: Array | NDArray[_np_dtype] | torch.Tensor) -> Array | NDArray[_np_dtype] | torch.Tensor:
     """Rescale an array to the range [0, 1]"""
-    if isinstance(array, (np.ndarray, torch.Tensor)):
+    if isinstance(array, np.ndarray | torch.Tensor):
         arr_min = array.min()
         arr_max = array.max()
         return (array - arr_min) / (arr_max - arr_min)

dataeval/utils/_bin.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 __all__ = []
 
 import warnings
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
 
 import numpy as np
 from numpy.typing import NDArray
@@ -94,7 +95,7 @@ def bin_data(data: NDArray[Any], bin_method: str) -> NDArray[np.int_]:
     return np.digitize(data, bin_edges)
 
 
-def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.number[Any]]) -> bool:
+def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.number[Any]] | None = None) -> bool:
     """
     Determines whether the data is continuous or discrete using the Wasserstein distance.
 
@@ -113,11 +114,12 @@ def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.numbe
     measured from a uniform distribution is greater or less than 0.054, respectively.
     """
     # Check if the metadata is image specific
-    _, data_indices_unsorted = np.unique(data, return_index=True)
-    if data_indices_unsorted.size == image_indices.size:
-        data_indices = np.sort(data_indices_unsorted)
-        if (data_indices == image_indices).all():
-            data = data[data_indices]
+    if image_indices is not None:
+        _, data_indices_unsorted = np.unique(data, return_index=True)
+        if data_indices_unsorted.size == image_indices.size:
+            data_indices = np.sort(data_indices_unsorted)
+            if (data_indices == image_indices).all():
+                data = data[data_indices]
 
     n_examples = len(data)
 

dataeval/utils/_method.py

@@ -1,8 +1,7 @@
 from __future__ import annotations
 
-from typing import Callable, TypeVar
-
-from typing_extensions import ParamSpec
+from collections.abc import Callable
+from typing import ParamSpec, TypeVar
 
 P = ParamSpec("P")
 R = TypeVar("R")

dataeval/utils/_multiprocessing.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+__all__ = []
+
+from collections.abc import Callable, Iterable, Iterator
+from multiprocessing import Pool
+from typing import Any, TypeVar
+
+_S = TypeVar("_S")
+_T = TypeVar("_T")
+
+
+class PoolWrapper:
+    """
+    Wraps `multiprocessing.Pool` to allow for easy switching between
+    multiprocessing and single-threaded execution.
+
+    This helps with debugging and profiling, as well as usage with Jupyter notebooks
+    in VS Code, which does not support subprocess debugging.
+    """
+
+    def __init__(self, processes: int | None) -> None:
+        self.pool = Pool(processes) if processes is None or processes > 1 else None
+
+    def imap(self, func: Callable[[_S], _T], iterable: Iterable[_S]) -> Iterator[_T]:
+        return map(func, iterable) if self.pool is None else self.pool.imap(func, iterable)
+
+    def __enter__(self, *args: Any, **kwargs: Any) -> PoolWrapper:
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        if self.pool is not None:
+            self.pool.close()
+            self.pool.join()

dataeval/utils/_plot.py

@@ -4,7 +4,8 @@ __all__ = []
 
 import contextlib
 import math
-from typing import Any, Mapping, Sequence
+from collections.abc import Mapping, Sequence
+from typing import Any
 
 import numpy as np
 

dataeval/utils/data/__init__.py

@@ -1,11 +1,12 @@
 """Provides access to common Computer Vision datasets."""
 
-from dataeval.utils.data import collate, metadata
-from dataeval.utils.data._dataset import to_image_classification_dataset, to_object_detection_dataset
+from dataeval.utils.data import collate
+from dataeval.utils.data._merge import flatten, merge
+from dataeval.utils.data._validate import validate_dataset
 
 __all__ = [
     "collate",
-    "metadata",
-    "to_image_classification_dataset",
-    "to_object_detection_dataset",
+    "flatten",
+    "merge",
+    "validate_dataset",
 ]

dataeval/utils/data/{metadata.py → _merge.py}

@@ -7,8 +7,9 @@ from __future__ import annotations
 __all__ = ["merge", "flatten"]
 
 import warnings
+from collections.abc import Iterable, Mapping, Sequence
 from enum import Enum
-from typing import Any, Iterable, Literal, Mapping, Sequence, overload
+from typing import Any, Literal, overload
 
 import numpy as np
 from numpy.typing import NDArray
@@ -132,7 +133,7 @@ def _flatten_dict_inner(
         if isinstance(v, dict):
             fd, size = _flatten_dict_inner(v, dropped, new_keys, size=size, nested=nested)
             items.update(fd)
-        elif isinstance(v, (list, tuple)):
+        elif isinstance(v, list | tuple):
             if nested:
                 dropped.setdefault(parent_keys + (k,), set()).add(DropReason.NESTED_LIST)
             elif size is not None and size != len(v):

dataeval/utils/data/_validate.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+__all__ = []
+
+from collections.abc import Sequence, Sized
+from typing import Any, Literal
+
+from dataeval.config import EPSILON
+from dataeval.typing import Array, ObjectDetectionTarget
+from dataeval.utils._array import as_numpy
+
+
+class ValidationMessages:
+    DATASET_SIZED = "Dataset must be sized."
+    DATASET_INDEXABLE = "Dataset must be indexable."
+    DATASET_NONEMPTY = "Dataset must be non-empty."
+    DATASET_METADATA = "Dataset must have a 'metadata' attribute."
+    DATASET_METADATA_TYPE = "Dataset metadata must be a dictionary."
+    DATASET_METADATA_FORMAT = "Dataset metadata must contain an 'id' key."
+    DATUM_TYPE = "Dataset datum must be a tuple."
+    DATUM_FORMAT = "Dataset datum must contain 3 elements: image, target, metadata."
+    DATUM_IMAGE_TYPE = "Images must be 3-dimensional arrays."
+    DATUM_IMAGE_FORMAT = "Images must be in CHW format."
+    DATUM_TARGET_IC_TYPE = "ImageClassificationDataset targets must be one-dimensional arrays."
+    DATUM_TARGET_IC_FORMAT = "ImageClassificationDataset targets must be one-hot encoded or pseudo-probabilities."
+    DATUM_TARGET_OD_TYPE = "ObjectDetectionDataset targets must be have 'boxes', 'labels' and 'scores'."
+    DATUM_TARGET_OD_LABELS_TYPE = "ObjectDetectionTarget labels must be one-dimensional (N,) arrays."
+    DATUM_TARGET_OD_BOXES_TYPE = "ObjectDetectionTarget boxes must be two-dimensional (N, 4) arrays in xxyy format."
+    DATUM_TARGET_OD_SCORES_TYPE = "ObjectDetectionTarget scores must be one (N,) or two-dimensional (N, M) arrays."
+    DATUM_TARGET_TYPE = "Target is not a valid ImageClassification or ObjectDetection target type."
+    DATUM_METADATA_TYPE = "Datum metadata must be a dictionary."
+    DATUM_METADATA_FORMAT = "Datum metadata must contain an 'id' key."
+
+
+def _validate_dataset_type(dataset: Any) -> list[str]:
+    issues = []
+    is_sized = isinstance(dataset, Sized)
+    is_indexable = hasattr(dataset, "__getitem__")
+    if not is_sized:
+        issues.append(ValidationMessages.DATASET_SIZED)
+    if not is_indexable:
+        issues.append(ValidationMessages.DATASET_INDEXABLE)
+    if is_sized and len(dataset) == 0:
+        issues.append(ValidationMessages.DATASET_NONEMPTY)
+    return issues
+
+
+def _validate_dataset_metadata(dataset: Any) -> list[str]:
+    issues = []
+    if not hasattr(dataset, "metadata"):
+        issues.append(ValidationMessages.DATASET_METADATA)
+    metadata = getattr(dataset, "metadata", None)
+    if not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATASET_METADATA_TYPE)
+    if not isinstance(metadata, dict) or "id" not in metadata:
+        issues.append(ValidationMessages.DATASET_METADATA_FORMAT)
+    return issues
+
+
+def _validate_datum_type(datum: Any) -> list[str]:
+    issues = []
+    if not isinstance(datum, tuple):
+        issues.append(ValidationMessages.DATUM_TYPE)
+    if datum is None or isinstance(datum, Sized) and len(datum) != 3:
+        issues.append(ValidationMessages.DATUM_FORMAT)
+    return issues
+
+
+def _validate_datum_image(image: Any) -> list[str]:
+    issues = []
+    if not isinstance(image, Array) or len(image.shape) != 3:
+        issues.append(ValidationMessages.DATUM_IMAGE_TYPE)
+    if (
+        not isinstance(image, Array)
+        or len(image.shape) == 3
+        and (image.shape[0] > image.shape[1] or image.shape[0] > image.shape[2])
+    ):
+        issues.append(ValidationMessages.DATUM_IMAGE_FORMAT)
+    return issues
+
+
+def _validate_datum_target_ic(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, Array) or len(target.shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_TYPE)
+    if target is None or sum(target) > 1 + EPSILON or sum(target) < 1 - EPSILON:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_FORMAT)
+    return issues
+
+
+def _validate_datum_target_od(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, ObjectDetectionTarget):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_TYPE)
+    od_target: ObjectDetectionTarget | None = target if isinstance(target, ObjectDetectionTarget) else None
+    if od_target is None or len(as_numpy(od_target.labels).shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_OD_LABELS_TYPE)
+    if (
+        od_target is None
+        or len(as_numpy(od_target.boxes).shape) != 2
+        or (len(as_numpy(od_target.boxes).shape) == 2 and as_numpy(od_target.boxes).shape[1] != 4)
+    ):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_BOXES_TYPE)
+    if od_target is None or len(as_numpy(od_target.scores).shape) not in (1, 2):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_SCORES_TYPE)
+    return issues
+
+
+def _detect_target_type(target: Any) -> Literal["ic", "od", "auto"]:
+    if isinstance(target, Array):
+        return "ic"
+    if isinstance(target, ObjectDetectionTarget):
+        return "od"
+    return "auto"
+
+
+def _validate_datum_target(target: Any, target_type: Literal["ic", "od", "auto"]) -> list[str]:
+    issues = []
+    target_type = _detect_target_type(target) if target_type == "auto" else target_type
+    if target_type == "ic":
+        issues.extend(_validate_datum_target_ic(target))
+    elif target_type == "od":
+        issues.extend(_validate_datum_target_od(target))
+    else:
+        issues.append(ValidationMessages.DATUM_TARGET_TYPE)
+    return issues
+
+
+def _validate_datum_metadata(metadata: Any) -> list[str]:
+    issues = []
+    if metadata is None or not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATUM_METADATA_TYPE)
+    if metadata is None or isinstance(metadata, dict) and "id" not in metadata:
+        issues.append(ValidationMessages.DATUM_METADATA_FORMAT)
+    return issues
+
+
+def validate_dataset(dataset: Any, dataset_type: Literal["ic", "od", "auto"] = "auto") -> None:
+    """
+    Validate a dataset for compliance with MAITE protocol.
+
+    Parameters
+    ----------
+    dataset: Any
+        Dataset to validate.
+    dataset_type: "ic", "od", or "auto", default "auto"
+        Dataset type, if known.
+
+    Raises
+    ------
+    ValueError
+        Raises exception if dataset is invalid with a list of validation issues.
+    """
+    issues = []
+    issues.extend(_validate_dataset_type(dataset))
+    datum = None if issues else dataset[0]  # type: ignore
+    issues.extend(_validate_dataset_metadata(dataset))
+    issues.extend(_validate_datum_type(datum))
+
+    is_seq = isinstance(datum, Sequence)
+    datum_len = len(datum) if is_seq else 0
+    image = datum[0] if is_seq and datum_len > 0 else None
+    target = datum[1] if is_seq and datum_len > 1 else None
+    metadata = datum[2] if is_seq and datum_len > 2 else None
+    issues.extend(_validate_datum_image(image))
+    issues.extend(_validate_datum_target(target, dataset_type))
+    issues.extend(_validate_datum_metadata(metadata))
+
+    if issues:
+        raise ValueError("Dataset validation issues found:\n - " + "\n - ".join(issues))

dataeval/utils/data/collate.py

@@ -6,7 +6,8 @@ from __future__ import annotations
 
 __all__ = ["list_collate_fn", "numpy_collate_fn", "torch_collate_fn"]
 
-from typing import Any, Iterable, Sequence, TypeVar
+from collections.abc import Iterable, Sequence
+from typing import Any, TypeVar
 
 import numpy as np
 import torch

dataeval/utils/torch/_internal.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 
 __all__ = []
 
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 import numpy as np
 import torch