dataeval 0.86.8__py3-none-any.whl → 0.87.0__py3-none-any.whl

dataeval/metrics/bias/_balance.py

@@ -73,9 +73,9 @@ def balance(
     Return intra/interfactor balance (mutual information)
 
     >>> bal.factors
-    array([[1.   , 0.017, 0.015],
-           [0.017, 0.445, 0.245],
-           [0.015, 0.245, 1.063]])
+    array([[1.   , 0.   , 0.015],
+           [0.   , 0.08 , 0.011],
+           [0.015, 0.011, 1.063]])
 
     Return classwise balance (mutual information) of factors with individual class_labels
 
@@ -95,32 +95,39 @@ def balance(
 
     num_neighbors = _validate_num_neighbors(num_neighbors)
 
-    data = metadata.discretized_data
     factor_types = {"class_label": "categorical"} | {k: v.factor_type for k, v in metadata.factor_info.items()}
     is_discrete = [factor_type != "continuous" for factor_type in factor_types.values()]
     num_factors = len(factor_types)
     class_labels = metadata.class_labels
 
     mi = np.full((num_factors, num_factors), np.nan, dtype=np.float32)
-    data = np.hstack((class_labels[:, np.newaxis], data))
+
+    # Use numeric data for MI
+    data = np.hstack((class_labels[:, np.newaxis], metadata.digitized_data))
+
+    # Present discrete features composed of distinct values as continuous for `mutual_info_classif`
+    for i, factor_type in enumerate(factor_types):
+        if len(data) == len(np.unique(data[:, i])):
+            is_discrete[i] = False
+            factor_types[factor_type] = "continuous"
+
+    mutual_info_fn_map = {
+        "categorical": mutual_info_classif,
+        "discrete": mutual_info_classif,
+        "continuous": mutual_info_regression,
+    }
 
     for idx, factor_type in enumerate(factor_types.values()):
-        if factor_type != "continuous":
-            mi[idx, :] = mutual_info_classif(
-                data,
-                data[:, idx],
-                discrete_features=is_discrete,  # type: ignore - sklearn function not typed
-                n_neighbors=num_neighbors,
-                random_state=get_seed(),
-            )
-        else:
-            mi[idx, :] = mutual_info_regression(
-                data,
-                data[:, idx],
-                discrete_features=is_discrete,  # type: ignore - sklearn function not typed
-                n_neighbors=num_neighbors,
-                random_state=get_seed(),
-            )
+        mi[idx, :] = mutual_info_fn_map[factor_type](
+            data,
+            data[:, idx],
+            discrete_features=is_discrete,
+            n_neighbors=num_neighbors,
+            random_state=get_seed(),
+        )
+
+    # Use binned data for classwise MI
+    data = np.hstack((class_labels[:, np.newaxis], metadata.binned_data))
 
     # Normalization via entropy
     bin_cnts = get_counts(data)

dataeval/metrics/bias/_diversity.py

@@ -162,12 +162,12 @@ def diversity(
         raise ValueError("No factors found in provided metadata.")
 
     diversity_fn = get_method(_DIVERSITY_FN_MAP, method)
-    discretized_data = metadata.discretized_data
+    binned_data = metadata.binned_data
     factor_names = metadata.factor_names
     class_lbl = metadata.class_labels
 
-    class_labels_with_discretized_data = np.hstack((class_lbl[:, np.newaxis], discretized_data))
-    cnts = get_counts(class_labels_with_discretized_data)
+    class_labels_with_binned_data = np.hstack((class_lbl[:, np.newaxis], binned_data))
+    cnts = get_counts(class_labels_with_binned_data)
     num_bins = np.bincount(np.nonzero(cnts)[1])
     diversity_index = diversity_fn(cnts, num_bins)
 
@@ -176,7 +176,7 @@ def diversity(
     classwise_div = np.full((len(u_classes), num_factors), np.nan)
     for idx, cls in enumerate(u_classes):
         subset_mask = class_lbl == cls
-        cls_cnts = get_counts(discretized_data[subset_mask], min_num_bins=cnts.shape[0])
+        cls_cnts = get_counts(binned_data[subset_mask], min_num_bins=cnts.shape[0])
         classwise_div[idx, :] = diversity_fn(cls_cnts, num_bins[1:])
 
     return DiversityOutput(diversity_index, classwise_div, factor_names, metadata.class_names)

dataeval/metrics/bias/_parity.py

@@ -245,10 +245,10 @@ def parity(metadata: Metadata) -> ParityOutput:
     if not metadata.factor_names:
         raise ValueError("No factors found in provided metadata.")
 
-    chi_scores = np.zeros(metadata.discretized_data.shape[1])
+    chi_scores = np.zeros(metadata.binned_data.shape[1])
     p_values = np.zeros_like(chi_scores)
     insufficient_data: defaultdict[str, defaultdict[int, dict[str, int]]] = defaultdict(lambda: defaultdict(dict))
-    for i, col_data in enumerate(metadata.discretized_data.T):
+    for i, col_data in enumerate(metadata.binned_data.T):
         # Builds a contingency matrix where entry at index (r,c) represents
         # the frequency of current_factor_name achieving value unique_factor_values[r]
         # at a data point with class c.

dataeval/metrics/stats/_hashstats.py

@@ -8,8 +8,9 @@ from typing import Any, Callable
 
 import numpy as np
 import xxhash as xxh
-from PIL import Image
+from numpy.typing import NDArray
 from scipy.fftpack import dct
+from scipy.ndimage import zoom
 
 from dataeval.metrics.stats._base import StatsProcessor, run_stats
 from dataeval.outputs import HashStatsOutput
@@ -18,10 +19,26 @@ from dataeval.typing import ArrayLike, Dataset
 from dataeval.utils._array import as_numpy
 from dataeval.utils._image import normalize_image_shape, rescale
 
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
 HASH_SIZE = 8
 MAX_FACTOR = 4
 
 
+def _resize(image: NDArray[np.uint8], resize_dim: int, use_pil: bool = True) -> NDArray[np.uint8]:
+    """Resizes a grayscale (HxW) 8-bit image using PIL or scipy.ndimage.zoom."""
+
+    # Use PIL if available, otherwise resize and resample with scipy.ndimage.zoom
+    if use_pil and Image is not None:
+        return np.array(Image.fromarray(image).resize((resize_dim, resize_dim), Image.Resampling.LANCZOS))
+
+    zoom_factors = (resize_dim / image.shape[0], resize_dim / image.shape[1])
+    return np.clip(zoom(image, zoom_factors, order=5, mode="reflect"), 0, 255, dtype=np.uint8)
+
+
 def pchash(image: ArrayLike) -> str:
     """
     Performs a perceptual hash on an image by resizing to a square NxN image
@@ -59,7 +76,7 @@ def pchash(image: ArrayLike) -> str:
     rescaled = rescale(normalized, 8).astype(np.uint8)
 
     # Resizes the image using the Lanczos algorithm to a square image
-    im = np.array(Image.fromarray(rescaled).resize((resize_dim, resize_dim), Image.Resampling.LANCZOS))
+    im = _resize(rescaled, resize_dim)
 
     # Performs discrete cosine transforms to compress the image information and takes the lowest frequency component
     transform = dct(dct(im.T).T)[:HASH_SIZE, :HASH_SIZE]

dataeval/outputs/_workflows.py

@@ -92,7 +92,7 @@ def plot_measure(
     return fig
 
 
-def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
+def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.int64]:
     """
     Inverse function for f_out()
 
@@ -106,13 +106,27 @@ def f_inv_out(y_i: NDArray[Any], x: NDArray[Any]) -> NDArray[np.uint64]:
     Returns
     -------
     NDArray
-        Array of sample sizes
+        Sample size or -1 if unachievable for each data point
     """
-    n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
-    return np.asarray(n_i, dtype=np.uint64)
+    with np.errstate(invalid="ignore"):
+        n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
+    unachievable_targets = np.isnan(n_i) | np.any(n_i > np.iinfo(np.int64).max)
+    if any(unachievable_targets):
+        with np.printoptions(suppress=True):
+            warnings.warn(
+                "Number of samples could not be determined for target(s): "
+                f"""{
+                    np.array2string(
+                        1 - y_i[unachievable_targets], separator=", ", formatter={"float": lambda x: f"{x}"}
+                    )
+                }""",
+                UserWarning,
+            )
+        n_i[unachievable_targets] = -1
+    return np.asarray(n_i, dtype=np.int64)
 
 
-def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.uint64]:
+def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np.int64]:
     """Inverse function for project_steps()
 
     Parameters
@@ -125,10 +139,9 @@ def inv_project_steps(params: NDArray[Any], targets: NDArray[Any]) -> NDArray[np
     Returns
     -------
     NDArray
-        Array of sample sizes, or 0 if overflow
+        Samples required or -1 if unachievable for each target value
     """
     steps = f_inv_out(1 - np.array(targets), params)
-    steps[np.isnan(steps)] = 0
     return np.ceil(steps)
 
 

dataeval/typing.py

@@ -3,11 +3,12 @@ Common type protocols used for interoperability with DataEval.
 """
 
 __all__ = [
+    "AnnotatedDataset",
     "Array",
     "ArrayLike",
     "Dataset",
-    "AnnotatedDataset",
     "DatasetMetadata",
+    "DeviceLike",
     "ImageClassificationDatum",
     "ImageClassificationDataset",
     "ObjectDetectionTarget",
@@ -21,9 +22,10 @@ __all__ = [
 
 
 import sys
-from typing import Any, Generic, Iterator, Mapping, Protocol, TypedDict, TypeVar, runtime_checkable
+from typing import Any, Generic, Iterator, Mapping, Protocol, TypedDict, TypeVar, Union, runtime_checkable
 
 import numpy.typing
+import torch
 from typing_extensions import NotRequired, ReadOnly, Required
 
 if sys.version_info >= (3, 10):
@@ -42,6 +44,16 @@ See Also
 """
 
 
+DeviceLike: TypeAlias = Union[int, str, tuple[str, int], torch.device]
+"""
+Type alias for a `Union` representing types that specify a torch.device.
+
+See Also
+--------
+`torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
+"""
+
+
 @runtime_checkable
 class Array(Protocol):
     """

dataeval/utils/__init__.py

@@ -4,6 +4,6 @@ in setting up data and architectures that are guaranteed to work with applicable
 DataEval metrics.
 """
 
-__all__ = ["data", "datasets", "torch"]
+__all__ = ["data", "torch"]
 
-from . import data, datasets, torch
+from . import data, torch

dataeval/utils/_bin.py

@@ -94,7 +94,7 @@ def bin_data(data: NDArray[Any], bin_method: str) -> NDArray[np.int_]:
     return np.digitize(data, bin_edges)
 
 
-def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.number[Any]]) -> bool:
+def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.number[Any]] | None = None) -> bool:
     """
     Determines whether the data is continuous or discrete using the Wasserstein distance.
 
@@ -113,11 +113,12 @@ def is_continuous(data: NDArray[np.number[Any]], image_indices: NDArray[np.numbe
     measured from a uniform distribution is greater or less than 0.054, respectively.
     """
     # Check if the metadata is image specific
-    _, data_indices_unsorted = np.unique(data, return_index=True)
-    if data_indices_unsorted.size == image_indices.size:
-        data_indices = np.sort(data_indices_unsorted)
-        if (data_indices == image_indices).all():
-            data = data[data_indices]
+    if image_indices is not None:
+        _, data_indices_unsorted = np.unique(data, return_index=True)
+        if data_indices_unsorted.size == image_indices.size:
+            data_indices = np.sort(data_indices_unsorted)
+            if (data_indices == image_indices).all():
+                data = data[data_indices]
 
     n_examples = len(data)
 

dataeval/utils/data/__init__.py

@@ -2,10 +2,12 @@
 
 from dataeval.utils.data import collate, metadata
 from dataeval.utils.data._dataset import to_image_classification_dataset, to_object_detection_dataset
+from dataeval.utils.data._validate import validate_dataset
 
 __all__ = [
     "collate",
     "metadata",
     "to_image_classification_dataset",
     "to_object_detection_dataset",
+    "validate_dataset",
 ]

dataeval/utils/data/_dataset.py

@@ -14,6 +14,10 @@ from dataeval.typing import (
 from dataeval.utils._array import as_numpy
 
 
+def _ensure_id(index: int, metadata: dict[str, Any]) -> dict[str, Any]:
+    return {"id": index, **metadata} if "id" not in metadata else metadata
+
+
 def _validate_data(
     datum_type: Literal["ic", "od"],
     images: Array | Sequence[Array],
@@ -128,16 +132,19 @@ class CustomImageClassificationDataset(BaseAnnotatedDataset[Sequence[int]], Imag
         return (
             self._images[idx],
             as_numpy(one_hot),
-            self._metadata[idx] if self._metadata is not None else {},
+            _ensure_id(idx, self._metadata[idx] if self._metadata is not None else {}),
         )
 
 
 class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]], ObjectDetectionDataset):
     class ObjectDetectionTarget:
-        def __init__(self, labels: Sequence[int], bboxes: Sequence[Sequence[float]]) -> None:
+        def __init__(self, labels: Sequence[int], bboxes: Sequence[Sequence[float]], class_count: int) -> None:
             self._labels = labels
             self._bboxes = bboxes
-            self._scores = [1.0] * len(labels)
+            one_hot = [[0.0] * class_count] * len(labels)
+            for i, label in enumerate(labels):
+                one_hot[i][label] = 1.0
+            self._scores = one_hot
 
         @property
         def labels(self) -> Sequence[int]:
@@ -148,7 +155,7 @@ class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]]
             return self._bboxes
 
         @property
-        def scores(self) -> Sequence[float]:
+        def scores(self) -> Sequence[Sequence[float]]:
             return self._scores
 
     def __init__(
@@ -180,8 +187,8 @@ class CustomObjectDetectionDataset(BaseAnnotatedDataset[Sequence[Sequence[int]]]
     def __getitem__(self, idx: int, /) -> tuple[Array, ObjectDetectionTarget, dict[str, Any]]:
         return (
             self._images[idx],
-            self.ObjectDetectionTarget(self._labels[idx], self._bboxes[idx]),
-            self._metadata[idx] if self._metadata is not None else {},
+            self.ObjectDetectionTarget(self._labels[idx], self._bboxes[idx], len(self._classes)),
+            _ensure_id(idx, self._metadata[idx] if self._metadata is not None else {}),
         )
 
 

dataeval/utils/data/_validate.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+__all__ = []
+
+from typing import Any, Literal, Sequence, Sized
+
+from dataeval.config import EPSILON
+from dataeval.typing import Array, ObjectDetectionTarget
+from dataeval.utils._array import as_numpy
+
+
+class ValidationMessages:
+    DATASET_SIZED = "Dataset must be sized."
+    DATASET_INDEXABLE = "Dataset must be indexable."
+    DATASET_NONEMPTY = "Dataset must be non-empty."
+    DATASET_METADATA = "Dataset must have a 'metadata' attribute."
+    DATASET_METADATA_TYPE = "Dataset metadata must be a dictionary."
+    DATASET_METADATA_FORMAT = "Dataset metadata must contain an 'id' key."
+    DATUM_TYPE = "Dataset datum must be a tuple."
+    DATUM_FORMAT = "Dataset datum must contain 3 elements: image, target, metadata."
+    DATUM_IMAGE_TYPE = "Images must be 3-dimensional arrays."
+    DATUM_IMAGE_FORMAT = "Images must be in CHW format."
+    DATUM_TARGET_IC_TYPE = "ImageClassificationDataset targets must be one-dimensional arrays."
+    DATUM_TARGET_IC_FORMAT = "ImageClassificationDataset targets must be one-hot encoded or pseudo-probabilities."
+    DATUM_TARGET_OD_TYPE = "ObjectDetectionDataset targets must be have 'boxes', 'labels' and 'scores'."
+    DATUM_TARGET_OD_LABELS_TYPE = "ObjectDetectionTarget labels must be one-dimensional (N,) arrays."
+    DATUM_TARGET_OD_BOXES_TYPE = "ObjectDetectionTarget boxes must be two-dimensional (N, 4) arrays in xxyy format."
+    DATUM_TARGET_OD_SCORES_TYPE = "ObjectDetectionTarget scores must be one (N,) or two-dimensional (N, M) arrays."
+    DATUM_TARGET_TYPE = "Target is not a valid ImageClassification or ObjectDetection target type."
+    DATUM_METADATA_TYPE = "Datum metadata must be a dictionary."
+    DATUM_METADATA_FORMAT = "Datum metadata must contain an 'id' key."
+
+
+def _validate_dataset_type(dataset: Any) -> list[str]:
+    issues = []
+    is_sized = isinstance(dataset, Sized)
+    is_indexable = hasattr(dataset, "__getitem__")
+    if not is_sized:
+        issues.append(ValidationMessages.DATASET_SIZED)
+    if not is_indexable:
+        issues.append(ValidationMessages.DATASET_INDEXABLE)
+    if is_sized and len(dataset) == 0:
+        issues.append(ValidationMessages.DATASET_NONEMPTY)
+    return issues
+
+
+def _validate_dataset_metadata(dataset: Any) -> list[str]:
+    issues = []
+    if not hasattr(dataset, "metadata"):
+        issues.append(ValidationMessages.DATASET_METADATA)
+    metadata = getattr(dataset, "metadata", None)
+    if not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATASET_METADATA_TYPE)
+    if not isinstance(metadata, dict) or "id" not in metadata:
+        issues.append(ValidationMessages.DATASET_METADATA_FORMAT)
+    return issues
+
+
+def _validate_datum_type(datum: Any) -> list[str]:
+    issues = []
+    if not isinstance(datum, tuple):
+        issues.append(ValidationMessages.DATUM_TYPE)
+    if datum is None or isinstance(datum, Sized) and len(datum) != 3:
+        issues.append(ValidationMessages.DATUM_FORMAT)
+    return issues
+
+
+def _validate_datum_image(image: Any) -> list[str]:
+    issues = []
+    if not isinstance(image, Array) or len(image.shape) != 3:
+        issues.append(ValidationMessages.DATUM_IMAGE_TYPE)
+    if (
+        not isinstance(image, Array)
+        or len(image.shape) == 3
+        and (image.shape[0] > image.shape[1] or image.shape[0] > image.shape[2])
+    ):
+        issues.append(ValidationMessages.DATUM_IMAGE_FORMAT)
+    return issues
+
+
+def _validate_datum_target_ic(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, Array) or len(target.shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_TYPE)
+    if target is None or sum(target) > 1 + EPSILON or sum(target) < 1 - EPSILON:
+        issues.append(ValidationMessages.DATUM_TARGET_IC_FORMAT)
+    return issues
+
+
+def _validate_datum_target_od(target: Any) -> list[str]:
+    issues = []
+    if not isinstance(target, ObjectDetectionTarget):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_TYPE)
+    od_target: ObjectDetectionTarget | None = target if isinstance(target, ObjectDetectionTarget) else None
+    if od_target is None or len(as_numpy(od_target.labels).shape) != 1:
+        issues.append(ValidationMessages.DATUM_TARGET_OD_LABELS_TYPE)
+    if (
+        od_target is None
+        or len(as_numpy(od_target.boxes).shape) != 2
+        or (len(as_numpy(od_target.boxes).shape) == 2 and as_numpy(od_target.boxes).shape[1] != 4)
+    ):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_BOXES_TYPE)
+    if od_target is None or len(as_numpy(od_target.scores).shape) not in (1, 2):
+        issues.append(ValidationMessages.DATUM_TARGET_OD_SCORES_TYPE)
+    return issues
+
+
+def _detect_target_type(target: Any) -> Literal["ic", "od", "auto"]:
+    if isinstance(target, Array):
+        return "ic"
+    if isinstance(target, ObjectDetectionTarget):
+        return "od"
+    return "auto"
+
+
+def _validate_datum_target(target: Any, target_type: Literal["ic", "od", "auto"]) -> list[str]:
+    issues = []
+    target_type = _detect_target_type(target) if target_type == "auto" else target_type
+    if target_type == "ic":
+        issues.extend(_validate_datum_target_ic(target))
+    elif target_type == "od":
+        issues.extend(_validate_datum_target_od(target))
+    else:
+        issues.append(ValidationMessages.DATUM_TARGET_TYPE)
+    return issues
+
+
+def _validate_datum_metadata(metadata: Any) -> list[str]:
+    issues = []
+    if metadata is None or not isinstance(metadata, dict):
+        issues.append(ValidationMessages.DATUM_METADATA_TYPE)
+    if metadata is None or isinstance(metadata, dict) and "id" not in metadata:
+        issues.append(ValidationMessages.DATUM_METADATA_FORMAT)
+    return issues
+
+
+def validate_dataset(dataset: Any, dataset_type: Literal["ic", "od", "auto"] = "auto") -> None:
+    """
+    Validate a dataset for compliance with MAITE protocol.
+
+    Parameters
+    ----------
+    dataset: Any
+        Dataset to validate.
+    dataset_type: "ic", "od", or "auto", default "auto"
+        Dataset type, if known.
+
+    Raises
+    ------
+    ValueError
+        Raises exception if dataset is invalid with a list of validation issues.
+    """
+    issues = []
+    issues.extend(_validate_dataset_type(dataset))
+    datum = None if issues else dataset[0]  # type: ignore
+    issues.extend(_validate_dataset_metadata(dataset))
+    issues.extend(_validate_datum_type(datum))
+
+    is_seq = isinstance(datum, Sequence)
+    datum_len = len(datum) if is_seq else 0
+    image = datum[0] if is_seq and datum_len > 0 else None
+    target = datum[1] if is_seq and datum_len > 1 else None
+    metadata = datum[2] if is_seq and datum_len > 2 else None
+    issues.extend(_validate_datum_image(image))
+    issues.extend(_validate_datum_target(target, dataset_type))
+    issues.extend(_validate_datum_metadata(metadata))
+
+    if issues:
+        raise ValueError("Dataset validation issues found:\n - " + "\n - ".join(issues))

dataeval/workflows/sufficiency.py

@@ -57,22 +57,29 @@ class Sufficiency(Generic[T]):
     test_ds : torch.Dataset
         Data that will be used for every run's evaluation
     train_fn : Callable[[nn.Module, Dataset, Sequence[int]], None]
-        Function which takes a model (torch.nn.Module), a dataset
-        (torch.utils.data.Dataset), indices to train on and executes model
+        Function which takes a model, a dataset, and indices to train on and then executes model
         training against the data.
     eval_fn : Callable[[nn.Module, Dataset], Mapping[str, float | ArrayLike]]
-        Function which takes a model (torch.nn.Module), a dataset
-        (torch.utils.data.Dataset) and returns a dictionary of metric
-        values (Mapping[str, float]) which is used to assess model performance
+        Function which takes a model, a dataset and returns a dictionary of metric
+        values which is used to assess model performance
         given the model and data.
     runs : int, default 1
-        Number of models to run over all subsets
+        Number of models to train over the entire dataset.
     substeps : int, default 5
-        Total number of dataset partitions that each model will train on
+        The number of steps that each model will be trained and evaluated on.
     train_kwargs : Mapping | None, default None
         Additional arguments required for custom training function
     eval_kwargs : Mapping | None, default None
         Additional arguments required for custom evaluation function
+
+    Warning
+    -------
+    Since each run is trained sequentially, increasing the parameter `runs` can significantly increase runtime.
+
+    Note
+    ----
+    Substeps is overridden by the parameter `eval_at` in :meth:`.Sufficiency.evaluate`
+
     """
 
     def __init__(
@@ -159,13 +166,22 @@ class Sufficiency(Generic[T]):
     @set_metadata(state=["runs", "substeps"])
     def evaluate(self, eval_at: int | Iterable[int] | None = None) -> SufficiencyOutput:
         """
-        Creates data indices, trains models, and returns plotting data
+        Train and evaluate a model over multiple substeps
+
+        This function trains a model up to each step calculated from substeps. The model is then evaluated
+        at that step and trained from 0 to the next step. This repeats for all substeps. Once a model has been
+        trained and evaluated at all substeps, if runs is greater than one, the model weights are reset and
+        the process is repeated.
+
+        During each evaluation, the metrics returned as a dictionary by the given evaluation function are stored
+        and then averaged over when all runs are complete.
 
         Parameters
         ----------
         eval_at : int | Iterable[int] | None, default None
-            Specify this to collect accuracies over a specific set of dataset lengths, rather
-            than letting :term:`sufficiency<Sufficiency>` internally create the lengths to evaluate at.
+            Specify this to collect metrics over a specific set of dataset lengths.
+            If `None`, evaluates at each step is calculated by
+            `np.geomspace` over the length of the dataset for self.substeps
 
         Returns
         -------
@@ -179,6 +195,8 @@ class Sufficiency(Generic[T]):
 
         Examples
         --------
+        Default runs and substeps
+
         >>> suff = Sufficiency(
         ...     model=model,
         ...     train_ds=train_ds,
@@ -190,6 +208,31 @@ class Sufficiency(Generic[T]):
         ... )
         >>> suff.evaluate()
         SufficiencyOutput(steps=array([  1,   3,  10,  31, 100], dtype=uint32), measures={'test': array([1., 1., 1., 1., 1.])}, n_iter=1000)
+
+        Evaluate at a single value
+
+        >>> suff = Sufficiency(
+        ...     model=model,
+        ...     train_ds=train_ds,
+        ...     test_ds=test_ds,
+        ...     train_fn=train_fn,
+        ...     eval_fn=eval_fn,
+        ... )
+        >>> suff.evaluate(eval_at=50)
+        SufficiencyOutput(steps=array([50]), measures={'test': array([1.])}, n_iter=1000)
+
+        Evaluating at linear steps from 0-100 inclusive
+
+        >>> suff = Sufficiency(
+        ...     model=model,
+        ...     train_ds=train_ds,
+        ...     test_ds=test_ds,
+        ...     train_fn=train_fn,
+        ...     eval_fn=eval_fn,
+        ... )
+        >>> suff.evaluate(eval_at=np.arange(0, 101, 20))
+        SufficiencyOutput(steps=array([  0,  20,  40,  60,  80, 100]), measures={'test': array([1., 1., 1., 1., 1., 1.])}, n_iter=1000)
+
         """  # noqa: E501
         if eval_at is not None:
             ranges = np.asarray(list(eval_at) if isinstance(eval_at, Iterable) else [eval_at])