dataeval 0.76.1__py3-none-any.whl → 0.82.0__py3-none-any.whl

dataeval/detectors/ood/metadata_ks_compare.py

@@ -1,129 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import numbers
-import warnings
-from typing import Any, Mapping, NamedTuple
-
-import numpy as np
-from numpy.typing import NDArray
-from scipy.stats import iqr, ks_2samp
-from scipy.stats import wasserstein_distance as emd
-
-from dataeval.output import MappingOutput, set_metadata
-
-
-class MetadataKSResult(NamedTuple):
-    statistic: float
-    statistic_location: float
-    shift_magnitude: float
-    pvalue: float
-
-
-class KSOutput(MappingOutput[str, MetadataKSResult]):
-    """
-    Output dictionary class for results of ks_2samp featurewise comparisons of new metadata to reference metadata.
-
-    Attributes
-    ----------
-    key: str
-        Metadata feature names
-    value: NamedTuple[float, float, float, float]
-        Each value contains four floats, which are:
-        - statistic: the KS statistic itself
-        - statistic_location: its location within the range of the reference metadata
-        - shift_magnitude: the shift of new metadata relative to reference
-        - pvalue: the p-value from the KS two-sample test
-    """
-
-
-@set_metadata
-def meta_distribution_compare(
-    md0: Mapping[str, list[Any] | NDArray[Any]], md1: Mapping[str, list[Any] | NDArray[Any]]
-) -> KSOutput:
-    """
-    Measures the featurewise distance between two metadata distributions, and computes a p-value to evaluate its
-    significance.
-
-    Uses the Earth Mover's Distance and the Kolmogorov-Smirnov two-sample test, featurewise.
-
-    Parameters
-    ----------
-    md0 : Mapping[str, list[Any] | NDArray[Any]]
-        A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
-    md1 : Mapping[str, list[Any] | NDArray[Any]]
-        Another set of arrays of values, indexed by metadata feature names, with one value per data example per
-        feature.
-
-    Returns
-    -------
-    dict[str, KstestResult]
-        A dictionary with keys corresponding to metadata feature names, and values that are KstestResult objects, as
-        defined by scipy.stats.ks_2samp. These values also have two additional attributes: shift_magnitude and
-        statistic_location. The first is the Earth Mover's Distance normalized by the interquartile range (IQR) of
-        the reference, while the second is the value at which the KS statistic has its maximum, measured in
-        IQR-normalized units relative to the median of the reference distribution.
-
-    Examples
-    --------
-    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
-    altitude.
-
-    >>> md0 = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
-    >>> md1 = {"time": [7.8, 9.10, 11.12], "altitude": [532, 9876, 211101]}
-    >>> md_out = meta_distribution_compare(md0, md1)
-    >>> for k, v in md_out.items():
-    ...     print(f"{k}: { {kv: round(vv, 3) for kv, vv in v._asdict().items()} }")
-    time: {'statistic': 1.0, 'statistic_location': 0.444, 'shift_magnitude': 2.7, 'pvalue': 0.0}
-    altitude: {'statistic': 0.333, 'statistic_location': 0.478, 'shift_magnitude': 0.749, 'pvalue': 0.944}
-    """
-
-    if (metadata_keys := md0.keys()) != md1.keys():
-        raise ValueError(f"Both sets of metadata keys must be identical: {list(md0)}, {list(md1)}")
-
-    mdc = {}  # output dict
-    for k in metadata_keys:
-        mdc.update({k: {}})
-
-        x0, x1 = list(md0[k]), list(md1[k])
-
-        allx = x0 + x1  # "+" sign concatenates lists.
-
-        if not all(isinstance(allxi, numbers.Number) for allxi in allx):  # NB: np.nan *is* a number in this context.
-            continue  # non-numeric features will return an empty dict for feature k
-
-        # from Numerical Recipes in C, 3rd ed. p. 737. If too few points, warn and keep going.
-        if np.sqrt(((N := len(x0)) * (M := len(x1))) / (N + M)) < 4:
-            warnings.warn(
-                f"Sample sizes of {N}, {M} for feature {k} will yield unreliable p-values from the KS test.",
-                UserWarning,
-            )
-
-        xmin, xmax = min(allx), max(allx)
-        if xmin == xmax:  # only one value in this feature, so fill in the obvious results for feature k
-            mdc[k] = MetadataKSResult(
-                **{"statistic": 0.0, "statistic_location": 0.0, "shift_magnitude": 0.0, "pvalue": 1.0}
-            )
-            continue
-
-        ks_result = ks_2samp(x0, x1, method="asymp")
-        dev = ks_result.statistic_location - xmin  #  pyright: ignore  (KSresult type)
-        loc = dev / (xmax - xmin) if xmax > xmin else dev
-
-        dX = iqr(x0)  # preferred value of dX, which is the scale of the the md0 values for feature k
-        dX = (max(x0) - min(x0)) / 2.0 if dX == 0 else dX  # reasonable alternative value of dX, when iqr is zero.
-        dX = 1.0 if dX == 0 else dX  # if dX is *still* zero, just avoid division by zero this way
-
-        drift = emd(x0, x1) / dX
-
-        mdc[k] = MetadataKSResult(
-            **{
-                "statistic": ks_result.statistic,  #  pyright: ignore
-                "statistic_location": loc,
-                "shift_magnitude": drift,
-                "pvalue": ks_result.pvalue,  #  pyright: ignore
-            }
-        )
-
-    return KSOutput(mdc)

dataeval/detectors/ood/metadata_least_likely.py

@@ -1,119 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import numbers
-import warnings
-from typing import Any
-
-import numpy as np
-from numpy.typing import NDArray
-
-
-def get_least_likely_features(
-    metadata: dict[str, list[Any] | NDArray[Any]],
-    new_metadata: dict[str, list[Any] | NDArray[Any]],
-    is_ood: NDArray[np.bool_],
-) -> list[tuple[str, float]]:
-    """Computes which metadata feature is most out-of-distribution (OOD) relative to a reference metadata set.
-
-    Given a reference metadata dictionary `metadata` (where each key maps to one scalar metadata feature), a second
-    metadata dictionary, and a corresponding boolean flag `is_ood` indicating whether each new example falls
-    out-of-distribution (OOD) relative to the reference, this function finds which metadata feature is the most OOD,
-    for each OOD example.
-
-    Parameters
-    ----------
-    metadata: dict[str, list[Any] | NDArray[Any]]
-        A reference set of arrays of values, indexed by metadata feature names, with one value per data example per
-        feature.
-    new_metadata: dict[str, list[Any] | NDArray[Any]]
-        A second metadata set, to be tested against the reference metadata. It is ok if the two meta data objects
-        hold different numbers of examples.
-    is_ood: NDArray[np.bool_]
-        A boolean array, with one value per new_metadata example, that indicates which examples are OOD.
-
-    Returns
-    -------
-    list[tuple[str, float]]
-        An array of names of the features of each OOD new_metadata example that were the most OOD.
-
-    Examples
-    --------
-    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
-    altitude, as shown below.
-
-    >>> metadata = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
-    >>> new_metadata = {"time": [7.8, 11.12], "altitude": [532, -211101]}
-    >>> is_ood = np.array([True, True])
-    >>> get_least_likely_features(metadata, new_metadata, is_ood)
-    [('time', 2.0), ('altitude', 33.245346)]
-    """
-    # Raise errors for bad inputs...
-
-    if metadata.keys() != new_metadata.keys():
-        raise ValueError(f"Reference and test metadata keys must be identical: {list(metadata)}, {list(new_metadata)}")
-
-    md_lengths = {len(np.atleast_1d(v)) for v in metadata.values()}
-    new_md_lengths = {len(np.atleast_1d(v)) for v in new_metadata.values()}
-    if len(md_lengths) > 1 or len(new_md_lengths) > 1:
-        raise ValueError(f"All features must have same length, got lengths {md_lengths}, {new_md_lengths}")
-
-    n_reference, n_new = md_lengths.pop(), new_md_lengths.pop()  # possibly different numbers of metadata examples
-
-    if n_new != len(is_ood):
-        raise ValueError(f"is_ood flag must have same length as new metadata {n_new} but has length {len(is_ood)}.")
-
-    if n_reference < 3:  # too hard to define "in-distribution" with this few reference samples.
-        warnings.warn(
-            "We need at least 3 reference metadata examples to determine which "
-            f"features are least likely, but only got {n_reference}",
-            UserWarning,
-        )
-        return []
-
-    if not any(is_ood):
-        return []
-
-    # ...inputs are good, look for most deviant standardized features.
-
-    # largest standardized absolute deviation from the median observed so far for each example
-    deviation = np.zeros_like(is_ood, dtype=np.float32)
-
-    # name of feature that corresponds to `deviation` for each example
-    kmax = np.empty(len(is_ood), dtype=object)
-
-    for k, v in metadata.items():
-        # exclude cases where random happens to be out on tails, not interesting.
-        if k == "random":
-            continue
-
-        # Skip non-numerical features
-        if not all(isinstance(vi, numbers.Number) for vi in v):  # NB: np.nan *is* a number in this context.
-            continue
-
-        # Get standardization parameters from metadata
-        loc = np.median(v)  # ok, because we checked all were numeric
-        dev = np.asarray(v) - loc  # need to make array from v since it could be a list here.
-        posdev, negdev = dev[dev > 0], dev[dev < 0]
-        pos_scale = np.median(posdev) if posdev.any() else 1.0
-        neg_scale = np.abs(np.median(negdev)) if negdev.any() else 1.0
-
-        x, x0, dxp, dxn = np.atleast_1d(new_metadata[k]), loc, pos_scale, neg_scale  # just abbreviations
-        dxp = dxp if dxp > 0 else 1.0  # avoids dividing by zero below
-        dxn = dxn if dxn > 0 else 1.0
-
-        # xdev must be floating-point to avoid getting zero in an integer division.
-        xdev = (x - x0).astype(np.float64)
-        pos = xdev >= 0
-
-        X = np.zeros_like(xdev)
-        X[pos], X[~pos] = xdev[pos] / dxp, xdev[~pos] / dxn  # keeping track of possible asymmetry of x, but...
-        # ...below here, only need to think about absolute deviation.
-
-        abig = np.abs(X) > deviation
-        kmax[abig] = k
-        deviation[abig] = np.abs(X[abig])
-
-    unlikely_features = list(zip(kmax[is_ood], deviation[is_ood]))  # feature names, along with how far out they are.
-    return unlikely_features

dataeval/interop.py

@@ -1,69 +0,0 @@
-"""Utility functions for interoperability with different array types."""
-
-from __future__ import annotations
-
-__all__ = []
-
-import logging
-from importlib import import_module
-from types import ModuleType
-from typing import Any, Iterable, Iterator
-
-import numpy as np
-from numpy.typing import ArrayLike, NDArray
-
-from dataeval.log import LogMessage
-
-_logger = logging.getLogger(__name__)
-
-_MODULE_CACHE = {}
-
-
-def _try_import(module_name) -> ModuleType | None:
-    if module_name in _MODULE_CACHE:
-        return _MODULE_CACHE[module_name]
-
-    try:
-        module = import_module(module_name)
-    except ImportError:  # pragma: no cover - covered by test_mindeps.py
-        _logger.log(logging.INFO, f"Unable to import {module_name}.")
-        module = None
-
-    _MODULE_CACHE[module_name] = module
-    return module
-
-
-def as_numpy(array: ArrayLike | None) -> NDArray[Any]:
-    """Converts an ArrayLike to Numpy array without copying (if possible)"""
-    return to_numpy(array, copy=False)
-
-
-def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
-    """Converts an ArrayLike to new Numpy array"""
-    if array is None:
-        return np.ndarray([])
-
-    if isinstance(array, np.ndarray):
-        return array.copy() if copy else array
-
-    if array.__class__.__module__.startswith("tensorflow"):  # pragma: no cover - removed tf from deps
-        tf = _try_import("tensorflow")
-        if tf and tf.is_tensor(array):
-            _logger.log(logging.INFO, "Converting Tensorflow array to NumPy array.")
-            return array.numpy().copy() if copy else array.numpy()  # type: ignore
-
-    if array.__class__.__module__.startswith("torch"):
-        torch = _try_import("torch")
-        if torch and isinstance(array, torch.Tensor):
-            _logger.log(logging.INFO, "Converting PyTorch array to NumPy array.")
-            numpy = array.detach().cpu().numpy().copy() if copy else array.detach().cpu().numpy()  # type: ignore
-            _logger.log(logging.DEBUG, LogMessage(lambda: f"{str(array)} -> {str(numpy)}"))
-            return numpy
-
-    return np.array(array) if copy else np.asarray(array)
-
-
-def to_numpy_iter(iterable: Iterable[ArrayLike]) -> Iterator[NDArray[Any]]:
-    """Yields an iterator of numpy arrays from an ArrayLike"""
-    for array in iterable:
-        yield to_numpy(array)

dataeval/metrics/bias/coverage.py

@@ -1,194 +0,0 @@
-from __future__ import annotations
-
-__all__ = []
-
-import contextlib
-import math
-from dataclasses import dataclass
-from typing import Any, Literal
-
-import numpy as np
-from numpy.typing import ArrayLike, NDArray
-from scipy.spatial.distance import pdist, squareform
-
-from dataeval.interop import to_numpy
-from dataeval.output import Output, set_metadata
-from dataeval.utils.shared import flatten
-
-with contextlib.suppress(ImportError):
-    from matplotlib.figure import Figure
-
-
-def _plot(images: NDArray[Any], num_images: int) -> Figure:
-    """
-    Creates a single plot of all of the provided images
-
-    Parameters
-    ----------
-    images : NDArray
-        Array containing only the desired images to plot
-
-    Returns
-    -------
-    matplotlib.figure.Figure
-        Plot of all provided images
-    """
-    import matplotlib.pyplot as plt
-
-    num_images = min(num_images, len(images))
-
-    if images.ndim == 4:
-        images = np.moveaxis(images, 1, -1)
-    elif images.ndim == 3:
-        images = np.repeat(images[:, :, :, np.newaxis], 3, axis=-1)
-    else:
-        raise ValueError(
-            f"Expected a (N,C,H,W) or a (N, H, W) set of images, but got a {images.ndim}-dimensional set of images."
-        )
-
-    rows = int(np.ceil(num_images / 3))
-    fig, axs = plt.subplots(rows, 3, figsize=(9, 3 * rows))
-
-    if rows == 1:
-        for j in range(3):
-            if j >= len(images):
-                continue
-            axs[j].imshow(images[j])
-            axs[j].axis("off")
-    else:
-        for i in range(rows):
-            for j in range(3):
-                i_j = i * 3 + j
-                if i_j >= len(images):
-                    continue
-                axs[i, j].imshow(images[i_j])
-                axs[i, j].axis("off")
-
-    fig.tight_layout()
-    return fig
-
-
-@dataclass(frozen=True)
-class CoverageOutput(Output):
-    """
-    Output class for :func:`coverage` :term:`bias<Bias>` metric.
-
-    Attributes
-    ----------
-    indices : NDArray[np.intp]
-        Array of uncovered indices
-    radii : NDArray[np.float64]
-        Array of critical value radii
-    critical_value : float
-        Radius for :term:`coverage<Coverage>`
-    """
-
-    indices: NDArray[np.intp]
-    radii: NDArray[np.float64]
-    critical_value: float
-
-    def plot(self, images: ArrayLike, top_k: int = 6) -> Figure:
-        """
-        Plot the top k images together for visualization
-
-        Parameters
-        ----------
-        images : ArrayLike
-            Original images (not embeddings) in (N, C, H, W) or (N, H, W) format
-        top_k : int, default 6
-            Number of images to plot (plotting assumes groups of 3)
-
-        Returns
-        -------
-        matplotlib.figure.Figure
-        """
-        # Determine which images to plot
-        highest_uncovered_indices = self.indices[:top_k]
-
-        # Grab the images
-        images = to_numpy(images)
-        selected_images = images[highest_uncovered_indices]
-
-        # Plot the images
-        fig = _plot(selected_images, top_k)
-
-        return fig
-
-
-@set_metadata
-def coverage(
-    embeddings: ArrayLike,
-    radius_type: Literal["adaptive", "naive"] = "adaptive",
-    k: int = 20,
-    percent: float = 0.01,
-) -> CoverageOutput:
-    """
-    Class for evaluating :term:`coverage<Coverage>` and identifying images/samples that are in undercovered regions.
-
-    Parameters
-    ----------
-    embeddings : ArrayLike, shape - (N, P)
-        A dataset in an ArrayLike format.
-        Function expects the data to have 2 dimensions, N number of observations in a P-dimesionial space.
-    radius_type : {"adaptive", "naive"}, default "adaptive"
-        The function used to determine radius.
-    k : int, default 20
-        Number of observations required in order to be covered.
-        [1] suggests that a minimum of 20-50 samples is necessary.
-    percent : float, default 0.01
-        Percent of observations to be considered uncovered. Only applies to adaptive radius.
-
-    Returns
-    -------
-    CoverageOutput
-        Array of uncovered indices, critical value radii, and the radius for coverage
-
-    Raises
-    ------
-    ValueError
-        If length of :term:`embeddings<Embeddings>` is less than or equal to k
-    ValueError
-        If radius_type is unknown
-
-    Note
-    ----
-    Embeddings should be on the unit interval [0-1].
-
-    Example
-    -------
-    >>> results = coverage(embeddings)
-    >>> results.indices
-    array([447, 412,   8,  32,  63])
-    >>> results.critical_value
-    0.8459038956941765
-
-    Reference
-    ---------
-    This implementation is based on https://dl.acm.org/doi/abs/10.1145/3448016.3457315.
-
-    [1] Seymour Sudman. 1976. Applied sampling. Academic Press New York (1976).
-    """
-
-    # Calculate distance matrix, look at the (k+1)th farthest neighbor for each image.
-    embeddings = to_numpy(embeddings)
-    n = len(embeddings)
-    if n <= k:
-        raise ValueError(
-            f"Number of observations n={n} is less than or equal to the specified number of neighbors k={k}."
-        )
-    mat = squareform(pdist(flatten(embeddings))).astype(np.float64)
-    sorted_dists = np.sort(mat, axis=1)
-    crit = sorted_dists[:, k + 1]
-
-    d = embeddings.shape[1]
-    if radius_type == "naive":
-        rho = (1 / math.sqrt(math.pi)) * ((2 * k * math.gamma(d / 2 + 1)) / (n)) ** (1 / d)
-        pvals = np.where(crit > rho)[0]
-    elif radius_type == "adaptive":
-        # Use data adaptive cutoff as rho
-        selection = int(max(n * percent, 1))
-        pvals = np.argsort(crit)[::-1][:selection]
-        rho = float(np.mean(np.sort(crit)[::-1][selection - 1 : selection + 1]))
-    else:
-        raise ValueError(f"{radius_type} is an invalid radius type. Expected 'adaptive' or 'naive'")
-    return CoverageOutput(pvals, crit, rho)