dataeval 0.82.1__py3-none-any.whl → 0.83.0__py3-none-any.whl

dataeval/__init__.py

@@ -8,7 +8,7 @@ shifts that impact performance of deployed models.
 from __future__ import annotations
 
 __all__ = ["config", "detectors", "log", "metrics", "typing", "utils", "workflows"]
-__version__ = "0.82.1"
+__version__ = "0.83.0"
 
 import logging
 
@@ -34,7 +34,12 @@ def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> N
     logger = logging.getLogger(__name__)
     if handler is None:
         handler = logging.StreamHandler() if handler is None else handler
-        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+        handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s %(levelname)-8s %(name)s.%(filename)s:%(lineno)s - %(funcName)10s() | %(message)s"
+            )
+        )
     logger.addHandler(handler)
     logger.setLevel(level)
+    logging.DEBUG
     logger.debug(f"Added logging handler {handler} to logger: {__name__}")

dataeval/config.py

@@ -17,10 +17,18 @@ else:
 import numpy as np
 import torch
 
+### GLOBALS ###
+
 _device: torch.device | None = None
 _processes: int | None = None
 _seed: int | None = None
 
+### CONSTS ###
+
+EPSILON = 1e-10
+
+### TYPES ###
+
 DeviceLike: TypeAlias = Union[int, str, tuple[str, int], torch.device]
 """
 Type alias for types that are acceptable for specifying a torch.device.
@@ -30,6 +38,8 @@ See Also
 `torch.device <https://pytorch.org/docs/stable/tensor_attributes.html#torch.device>`_
 """
 
+### FUNCS ###
+
 
 def _todevice(device: DeviceLike) -> torch.device:
     return torch.device(*device) if isinstance(device, tuple) else torch.device(device)

dataeval/metadata/__init__.py

@@ -1,6 +1,6 @@
 """Explanatory functions using metadata and additional features such as ood or drift"""
 
-__all__ = ["most_deviated_factors", "metadata_distance"]
+__all__ = ["find_ood_predictors", "metadata_distance", "find_most_deviated_factors"]
 
 from dataeval.metadata._distance import metadata_distance
-from dataeval.metadata._ood import most_deviated_factors
+from dataeval.metadata._ood import find_most_deviated_factors, find_ood_predictors

dataeval/metadata/_ood.py

@@ -6,14 +6,44 @@ import warnings
 
 import numpy as np
 from numpy.typing import NDArray
+from sklearn.feature_selection import mutual_info_classif
 
+from dataeval.config import get_seed
 from dataeval.metadata._utils import _compare_keys, _validate_factors_and_data
-from dataeval.outputs import MostDeviatedFactorsOutput, OODOutput
+from dataeval.outputs import MostDeviatedFactorsOutput, OODOutput, OODPredictorOutput
 from dataeval.outputs._base import set_metadata
 from dataeval.utils.data import Metadata
 
 
-def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[str], list[NDArray], list[NDArray]]:
+def _combine_discrete_continuous(metadata: Metadata) -> tuple[list[str], NDArray[np.float64]]:
+    """Combines the discrete and continuous data of a :class:`Metadata` object
+
+    Returns
+    -------
+    Tuple[list[str], NDArray]
+        The combined list of factors names and the combined discrete and continuous data
+
+    Note
+    ----
+    Discrete and continuous data must have the same number of samples
+    """
+    names = []
+    data = []
+
+    if metadata.discrete_factor_names and metadata.discrete_data.size != 0:
+        names.extend(metadata.discrete_factor_names)
+        data.append(metadata.discrete_data)
+
+    if metadata.continuous_factor_names and metadata.continuous_data.size != 0:
+        names.extend(metadata.continuous_factor_names)
+        data.append(metadata.continuous_data)
+
+    return names, np.hstack(data, dtype=np.float64) if data else np.array([], dtype=np.float64)
+
+
+def _combine_metadata(
+    metadata_1: Metadata, metadata_2: Metadata
+) -> tuple[list[str], list[NDArray[np.float64 | np.int64]], list[NDArray[np.int64 | np.float64]]]:
     """
     Combines the factor names and data arrays of metadata_1 and metadata_2 when the names
     match exactly and data has the same number of columns (factors).
@@ -42,8 +72,8 @@ def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[
         If the length of keys do not match the length of the data
     """
     factor_names: list[str] = []
-    m1_data: list[NDArray] = []
-    m2_data: list[NDArray] = []
+    m1_data: list[NDArray[np.int64 | np.float64]] = []
+    m2_data: list[NDArray[np.int64 | np.float64]] = []
 
     # Both metadata must have the same number of factors (cols), but not necessarily samples (row)
     if metadata_1.total_num_factors != metadata_2.total_num_factors:
@@ -121,36 +151,37 @@ def _calc_median_deviations(reference: NDArray, test: NDArray) -> NDArray:
 
 
 @set_metadata
-def most_deviated_factors(
-    metadata_1: Metadata,
-    metadata_2: Metadata,
+def find_most_deviated_factors(
+    metadata_ref: Metadata,
+    metadata_tst: Metadata,
     ood: OODOutput,
 ) -> MostDeviatedFactorsOutput:
     """
-    Determines greatest deviation in metadata features per out of distribution sample in metadata_2.
+    Determine greatest deviation in metadata features per out of distribution sample in test metadata.
 
     Parameters
     ----------
-    metadata_1 : Metadata
+    metadata_ref : Metadata
         A reference set of Metadata containing factor names and samples
         with discrete and/or continuous values per factor
-    metadata_2 : Metadata
+    metadata_tst : Metadata
         The set of Metadata that is tested against the reference metadata.
         This set must have the same number of features but does not require the same number of samples.
     ood : OODOutput
-        A class output by the DataEval's OOD functions that contains which examples are OOD.
+        A class output by DataEval's OOD functions that contains which examples are OOD.
 
     Returns
     -------
-    list[tuple[str, float]]
-        An array of the factor name and deviation of the highest metadata deviation for each OOD example in metadata_2.
+    MostDeviatedFactorsOutput
+        An output class containing the factor name and deviation of the highest metadata deviations for each
+        OOD example in the test metadata.
 
     Notes
     -----
     1. Both :class:`.Metadata` inputs must have discrete and continuous data in the shape (samples, factors)
        and have equivalent factor names and lengths
     2. The flag at index `i` in :attr:`.OODOutput.is_ood` must correspond
-       directly to sample `i` of `metadata_2` being out-of-distribution from `metadata_1`
+       directly to sample `i` of `metadata_tst` being out-of-distribution from `metadata_ref`
 
     Examples
     --------
@@ -160,13 +191,13 @@ def most_deviated_factors(
     All samples are out-of-distribution
 
     >>> is_ood = OODOutput(np.array([True, True, True]), np.array([]), np.array([]))
-    >>> most_deviated_factors(metadata1, metadata2, is_ood)
+    >>> find_most_deviated_factors(metadata1, metadata2, is_ood)
     MostDeviatedFactorsOutput([('time', 2.0), ('time', 2.592), ('time', 3.51)])
 
-    If there are no out-of-distribution samples, a list is returned
+    No samples are out-of-distribution
 
     >>> is_ood = OODOutput(np.array([False, False, False]), np.array([]), np.array([]))
-    >>> most_deviated_factors(metadata1, metadata2, is_ood)
+    >>> find_most_deviated_factors(metadata1, metadata2, is_ood)
     MostDeviatedFactorsOutput([])
     """
 
@@ -179,31 +210,30 @@ def most_deviated_factors(
     # Combines reference and test factor names and data if exists and match exactly
     # shape -> (samples, factors)
     factor_names, md_1, md_2 = _combine_metadata(
-        metadata_1=metadata_1,
-        metadata_2=metadata_2,
+        metadata_1=metadata_ref,
+        metadata_2=metadata_tst,
     )
 
     # Stack discrete and continuous factors as separate factors. Must have equal sample counts
-    metadata_ref = np.hstack(md_1) if md_1 else np.array([])
-    metadata_tst = np.hstack(md_2) if md_2 else np.array([])
+    ref_data = np.hstack(md_1) if md_1 else np.array([])  # (S, Fd + Fc)
+    tst_data = np.hstack(md_2) if md_2 else np.array([])  # (S, Fd + Fc)
 
-    if len(metadata_ref) < 3:
+    if len(ref_data) < 3:
         warnings.warn(
-            f"At least 3 reference metadata samples are needed, got {len(metadata_ref)}",
+            f"At least 3 reference metadata samples are needed, got {len(ref_data)}",
             UserWarning,
         )
         return MostDeviatedFactorsOutput([])
 
-    if len(metadata_tst) != len(ood_mask):
+    if len(tst_data) != len(ood_mask):
         raise ValueError(
-            f"ood and test metadata must have the same length, "
-            f"got {len(ood_mask)} and {len(metadata_tst)} respectively."
+            f"ood and test metadata must have the same length, got {len(ood_mask)} and {len(tst_data)} respectively."
         )
 
     # Calculates deviations of all samples in m2_data
     # from the median values of the corresponding index in m1_data
     # Guaranteed for inputs to not be empty
-    deviations = _calc_median_deviations(metadata_ref, metadata_tst)
+    deviations = _calc_median_deviations(ref_data, tst_data)
 
     # Get most impactful factor deviation of each sample for ood samples only
     deviation = np.max(deviations, axis=1)[ood_mask].astype(np.float16)
@@ -217,3 +247,90 @@ def most_deviated_factors(
     # List of tuples matching the factor name with its deviation
 
     return MostDeviatedFactorsOutput([(factor, dev) for factor, dev in zip(most_ood_factors, deviation)])
+
+
+_NATS2BITS = 1.442695
+"""
+_NATS2BITS is the reciprocal of natural log of 2. If you have an information/entropy-type quantity measured in nats,
+which is what many library functions return, multiply it by _NATS2BITS to get it in bits.
+"""
+
+
+def find_ood_predictors(
+    metadata: Metadata,
+    ood: OODOutput,
+) -> OODPredictorOutput:
+    """Computes mutual information between a set of metadata features and per sample out-of-distribution flags.
+
+    Given a set of metadata features per sample and a corresponding OODOutput that indicates whether a sample was
+    determined to be out of distribution, this function calculates the mutual information between each factor and being
+    out of distribution. In other words, it finds which metadata factors most likely correlate to an
+    out of distribution sample.
+
+    Note
+    ----
+    A high mutual information between a factor and ood samples is an indication of correlation, but not causation.
+    Additional analysis should be done to determine how to handle factors with a high mutual information.
+
+
+    Parameters
+    ----------
+    metadata : Metadata
+        A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
+    ood : OODOutput
+        A class output by DataEval's OOD functions that contains which examples are OOD.
+
+    Returns
+    -------
+    OODPredictorOutput
+        A dictionary with keys corresponding to metadata feature names, and values indicating the strength of
+        association between each named feature and the OOD flag, as mutual information measured in bits.
+
+    Examples
+    --------
+    >>> from dataeval.outputs import OODOutput
+
+    All samples are out-of-distribution
+
+    >>> is_ood = OODOutput(np.array([True, True, True]), np.array([]), np.array([]))
+    >>> find_ood_predictors(metadata1, is_ood)
+    OODPredictorOutput({'time': 8.008566032557951e-17, 'altitude': 8.008566032557951e-17})
+
+    No out-of-distribution samples
+
+    >> is_ood = OODOutput(np.array([False, False, False]), np.array([]), np.array([]))
+    >> find_ood_predictors(metadata1, is_ood)
+    OODPredictorOutput({})
+    """
+
+    ood_mask: NDArray[np.bool] = ood.is_ood
+
+    discrete_features_count = len(metadata.discrete_factor_names)
+    factors, data = _combine_discrete_continuous(metadata)  # (F, ), (S, F) => F = Fd + Fc
+
+    # No metadata correlated with out of distribution data, return 0.0 for all factors
+    if not any(ood_mask):
+        return OODPredictorOutput(dict.fromkeys(factors, 0.0))
+
+    if len(data) != len(ood_mask):
+        raise ValueError(
+            f"ood and metadata must have the same length, got {len(ood_mask)} and {len(data)} respectively."
+        )
+
+    # Calculate mean, std of each factor over all samples
+    scaled_data = (data - np.mean(data, axis=0)) / np.std(data, axis=0, ddof=1)  # (S, F)
+
+    discrete_features = np.zeros_like(factors, dtype=np.bool)
+    discrete_features[:discrete_features_count] = True
+
+    mutual_info_values = (
+        mutual_info_classif(
+            X=scaled_data,
+            y=ood_mask,
+            discrete_features=discrete_features,  # type: ignore -> sklearn issue - NDArray[bool] not of accepted type Union[ArrayLike, 'auto']
+            random_state=get_seed(),
+        )
+        * _NATS2BITS
+    )
+
+    return OODPredictorOutput({k: mutual_info_values[i] for i, k in enumerate(factors)})

dataeval/metrics/bias/_balance.py

@@ -8,7 +8,7 @@ import numpy as np
 import scipy as sp
 from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 
-from dataeval.config import get_seed
+from dataeval.config import EPSILON, get_seed
 from dataeval.outputs import BalanceOutput
 from dataeval.outputs._base import set_metadata
 from dataeval.utils._bin import get_counts
@@ -128,7 +128,7 @@ def balance(
     # Normalization via entropy
     bin_cnts = get_counts(discretized_data)
     ent_factor = sp.stats.entropy(bin_cnts, axis=0)
-    norm_factor = 0.5 * np.add.outer(ent_factor, ent_factor) + 1e-6
+    norm_factor = 0.5 * np.add.outer(ent_factor, ent_factor) + EPSILON
 
     # in principle MI should be symmetric, but it is not in practice.
     nmi = 0.5 * (mi + mi.T) / norm_factor
@@ -157,7 +157,7 @@ def balance(
     # Classwise normalization via entropy
     classwise_bin_cnts = get_counts(tgt_bin)
     ent_tgt_bin = sp.stats.entropy(classwise_bin_cnts, axis=0)
-    norm_factor = 0.5 * np.add.outer(ent_tgt_bin, ent_factor) + 1e-6
+    norm_factor = 0.5 * np.add.outer(ent_tgt_bin, ent_factor) + EPSILON
     classwise = classwise_mi / norm_factor
 
     # Grabbing factor names for plotting function

dataeval/metrics/estimators/_ber.py

@@ -19,6 +19,7 @@ from numpy.typing import NDArray
 from scipy.sparse import coo_matrix
 from scipy.stats import mode
 
+from dataeval.config import EPSILON
 from dataeval.outputs import BEROutput
 from dataeval.outputs._base import set_metadata
 from dataeval.typing import ArrayLike
@@ -82,7 +83,7 @@ def ber_knn(images: NDArray[np.float64], labels: NDArray[np.int_], k: int) -> tu
 
 def knn_lowerbound(value: float, classes: int, k: int) -> float:
     """Several cases for computing the BER lower bound"""
-    if value <= 1e-10:
+    if value <= EPSILON:
         return 0.0
 
     if classes == 2 and k != 1:

dataeval/metrics/stats/_base.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from dataclasses import dataclass
 from functools import partial
 from multiprocessing import Pool
-from typing import Any, Callable, Generic, Iterable, Sequence, TypeVar, cast
+from typing import Any, Callable, Generic, Iterable, Sequence, TypeVar
 
 import numpy as np
 import tqdm
@@ -17,7 +17,7 @@ from numpy.typing import NDArray
 
 from dataeval.config import get_max_processes
 from dataeval.outputs._stats import BaseStatsOutput, SourceIndex
-from dataeval.typing import ArrayLike, Dataset, ObjectDetectionTarget
+from dataeval.typing import Array, ArrayLike, Dataset, ObjectDetectionTarget
 from dataeval.utils._array import to_numpy
 from dataeval.utils._image import normalize_image_shape, rescale
 
@@ -122,22 +122,19 @@ class StatsProcessorOutput:
 
 def process_stats(
     i: int,
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    image: ArrayLike,
+    target: Any,
     per_box: bool,
     per_channel: bool,
     stats_processor_cls: Iterable[type[StatsProcessor[TStatsOutput]]],
 ) -> StatsProcessorOutput:
-    data = dataset[i]
-    image, target = (to_numpy(cast(ArrayLike, data[0])), data[1]) if isinstance(data, tuple) else (to_numpy(data), None)
-    target = None if not isinstance(target, ObjectDetectionTarget) else target
-    boxes = to_numpy(target.boxes) if target is not None else None
+    image = to_numpy(image)
+    boxes = to_numpy(target.boxes) if isinstance(target, ObjectDetectionTarget) else None
     results_list: list[dict[str, Any]] = []
     source_indices: list[SourceIndex] = []
     box_counts: list[int] = []
     warnings_list: list[str] = []
-    nboxes = [None] if boxes is None or not per_box else normalize_box_shape(boxes)
-    for i_b, box in enumerate(nboxes):
-        i_b = None if box is None else i_b
+    for i_b, box in [(None, None)] if boxes is None else enumerate(normalize_box_shape(boxes)):
         processor_list = [p(image, box, per_channel) for p in stats_processor_cls]
         if any(not p._is_valid_slice for p in processor_list) and i_b is not None and box is not None:
             warnings_list.append(f"Bounding box [{i}][{i_b}]: {box} is out of bounds of {image.shape}.")
@@ -151,17 +148,16 @@ def process_stats(
 
 
 def process_stats_unpack(
-    i: int,
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    args: tuple[int, ArrayLike, Any],
     per_box: bool,
     per_channel: bool,
     stats_processor_cls: Iterable[type[StatsProcessor[TStatsOutput]]],
 ) -> StatsProcessorOutput:
-    return process_stats(i, dataset, per_box=per_box, per_channel=per_channel, stats_processor_cls=stats_processor_cls)
+    return process_stats(*args, per_box=per_box, per_channel=per_channel, stats_processor_cls=stats_processor_cls)
 
 
 def run_stats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     per_box: bool,
     per_channel: bool,
     stats_processor_cls: Iterable[type[StatsProcessor[TStatsOutput]]],
@@ -175,7 +171,7 @@ def run_stats(
 
     Parameters
     ----------
-    data : Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]]
+    data : Dataset[Array] | Dataset[tuple[Array, Any, Any]]
         A dataset of images and targets to compute statistics on.
     per_box : bool
         A flag which determines if the statistics should be evaluated on a per-box basis or not.
@@ -206,18 +202,21 @@ def run_stats(
     warning_list = []
     stats_processor_cls = stats_processor_cls if isinstance(stats_processor_cls, Iterable) else [stats_processor_cls]
 
-    # TODO: Introduce global controls for CPU job parallelism and GPU configurations
+    def _enumerate(dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]], per_box: bool):
+        for i in range(len(dataset)):
+            d = dataset[i]
+            yield i, d[0] if isinstance(d, tuple) else d, d[1] if isinstance(d, tuple) and per_box else None
+
     with Pool(processes=get_max_processes()) as p:
         for r in tqdm.tqdm(
             p.imap(
                 partial(
                     process_stats_unpack,
-                    dataset=dataset,
                     per_box=per_box,
                     per_channel=per_channel,
                     stats_processor_cls=stats_processor_cls,
                 ),
-                range(len(dataset)),
+                _enumerate(dataset, per_box),
             ),
             total=len(dataset),
         ):

dataeval/metrics/stats/_dimensionstats.py

@@ -9,7 +9,7 @@ import numpy as np
 from dataeval.metrics.stats._base import StatsProcessor, run_stats
 from dataeval.outputs import DimensionStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import ArrayLike, Dataset
+from dataeval.typing import Array, Dataset
 from dataeval.utils._image import get_bitdepth
 
 
@@ -34,7 +34,7 @@ class DimensionStatsProcessor(StatsProcessor[DimensionStatsOutput]):
 
 @set_metadata
 def dimensionstats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
 ) -> DimensionStatsOutput:

dataeval/metrics/stats/_hashstats.py

@@ -14,7 +14,7 @@ from scipy.fftpack import dct
 from dataeval.metrics.stats._base import StatsProcessor, run_stats
 from dataeval.outputs import HashStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import ArrayLike, Dataset
+from dataeval.typing import Array, ArrayLike, Dataset
 from dataeval.utils._array import as_numpy
 from dataeval.utils._image import normalize_image_shape, rescale
 
@@ -105,7 +105,7 @@ class HashStatsProcessor(StatsProcessor[HashStatsOutput]):
 
 @set_metadata
 def hashstats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
 ) -> HashStatsOutput:

dataeval/metrics/stats/_imagestats.py

@@ -10,12 +10,12 @@ from dataeval.metrics.stats._pixelstats import PixelStatsProcessor
 from dataeval.metrics.stats._visualstats import VisualStatsProcessor
 from dataeval.outputs import ChannelStatsOutput, ImageStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import ArrayLike, Dataset
+from dataeval.typing import Array, Dataset
 
 
 @overload
 def imagestats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
     per_channel: Literal[True],
@@ -24,7 +24,7 @@ def imagestats(
 
 @overload
 def imagestats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
     per_channel: Literal[False] = False,
@@ -33,7 +33,7 @@ def imagestats(
 
 @set_metadata
 def imagestats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
     per_channel: bool = False,

dataeval/metrics/stats/_pixelstats.py

@@ -10,7 +10,7 @@ from scipy.stats import entropy, kurtosis, skew
 from dataeval.metrics.stats._base import StatsProcessor, run_stats
 from dataeval.outputs import PixelStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import ArrayLike, Dataset
+from dataeval.typing import Array, Dataset
 
 
 class PixelStatsProcessor(StatsProcessor[PixelStatsOutput]):
@@ -37,7 +37,7 @@ class PixelStatsProcessor(StatsProcessor[PixelStatsOutput]):
 
 @set_metadata
 def pixelstats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
     per_channel: bool = False,

dataeval/metrics/stats/_visualstats.py

@@ -9,7 +9,7 @@ import numpy as np
 from dataeval.metrics.stats._base import StatsProcessor, run_stats
 from dataeval.outputs import VisualStatsOutput
 from dataeval.outputs._base import set_metadata
-from dataeval.typing import ArrayLike, Dataset
+from dataeval.typing import Array, Dataset
 from dataeval.utils._image import edge_filter
 
 QUARTILES = (0, 25, 50, 75, 100)
@@ -44,7 +44,7 @@ class VisualStatsProcessor(StatsProcessor[VisualStatsOutput]):
 
 @set_metadata
 def visualstats(
-    dataset: Dataset[ArrayLike] | Dataset[tuple[ArrayLike, Any, Any]],
+    dataset: Dataset[Array] | Dataset[tuple[Array, Any, Any]],
     *,
     per_box: bool = False,
     per_channel: bool = False,

dataeval/outputs/__init__.py

@@ -8,7 +8,7 @@ from ._bias import BalanceOutput, CoverageOutput, DiversityOutput, LabelParityOu
 from ._drift import DriftMMDOutput, DriftOutput
 from ._estimators import BEROutput, ClustererOutput, DivergenceOutput, UAPOutput
 from ._linters import DuplicatesOutput, OutliersOutput
-from ._metadata import MetadataDistanceOutput, MetadataDistanceValues, MostDeviatedFactorsOutput
+from ._metadata import MetadataDistanceOutput, MetadataDistanceValues, MostDeviatedFactorsOutput, OODPredictorOutput
 from ._ood import OODOutput, OODScoreOutput
 from ._stats import (
     ChannelStatsOutput,
@@ -44,6 +44,7 @@ __all__ = [
     "MetadataDistanceValues",
     "MostDeviatedFactorsOutput",
     "OODOutput",
+    "OODPredictorOutput",
     "OODScoreOutput",
     "OutliersOutput",
     "ParityOutput",

dataeval/outputs/_metadata.py

@@ -52,3 +52,10 @@ class MetadataDistanceOutput(MappingOutput[str, MetadataDistanceValues]):
     value : :class:`.MetadataDistanceValues`
         Output per feature name containing the statistic, statistic location, distance, and pvalue.
     """
+
+
+class OODPredictorOutput(MappingOutput[str, float]):
+    """
+    Output class for results of :func:`find_ood_predictors` for the
+    mutual information between factors and being out of distribution
+    """