dataeval 0.76.1__py3-none-any.whl → 0.81.0__py3-none-any.whl

dataeval/detectors/linters/duplicates.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 __all__ = []
 
 from dataclasses import dataclass
-from typing import Generic, Iterable, Sequence, TypeVar, overload
+from typing import Any, Generic, Iterable, Sequence, TypeVar, overload
 
-from numpy.typing import ArrayLike
+from torch.utils.data import Dataset
 
-from dataeval.detectors.linters.merged_stats import combine_stats, get_dataset_step_from_idx
-from dataeval.metrics.stats.hashstats import HashStatsOutput, hashstats
-from dataeval.output import Output, set_metadata
+from dataeval._output import Output, set_metadata
+from dataeval.metrics.stats._base import combine_stats, get_dataset_step_from_idx
+from dataeval.metrics.stats._hashstats import HashStatsOutput, hashstats
+from dataeval.typing import ArrayLike
 
 DuplicateGroup = list[int]
 DatasetDuplicateGroupMap = dict[int, DuplicateGroup]
@@ -19,7 +20,7 @@ TIndexCollection = TypeVar("TIndexCollection", DuplicateGroup, DatasetDuplicateG
 @dataclass(frozen=True)
 class DuplicatesOutput(Generic[TIndexCollection], Output):
     """
-    Output class for :class:`Duplicates` lint detector.
+    Output class for :class:`.Duplicates` lint detector.
 
     Attributes
     ----------
@@ -133,8 +134,15 @@ class Duplicates:
 
         return DuplicatesOutput(**duplicates)
 
+    @overload
+    def evaluate(self, data: Iterable[ArrayLike]) -> DuplicatesOutput[DuplicateGroup]: ...
+    @overload
+    def evaluate(self, data: Dataset[tuple[ArrayLike, Any, dict[str, Any]]]) -> DuplicatesOutput[DuplicateGroup]: ...
+
     @set_metadata(state=["only_exact"])
-    def evaluate(self, data: Iterable[ArrayLike]) -> DuplicatesOutput[DuplicateGroup]:
+    def evaluate(
+        self, data: Iterable[ArrayLike] | Dataset[tuple[ArrayLike, Any, dict[str, Any]]]
+    ) -> DuplicatesOutput[DuplicateGroup]:
         """
         Returns duplicate image indices for both exact matches and near matches
 
@@ -158,6 +166,7 @@ class Duplicates:
         >>> all_dupes.evaluate(duplicate_images)
         DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
         """  # noqa: E501
-        self.stats = hashstats(data)
+        images = (d[0] for d in data) if isinstance(data, Dataset) else data
+        self.stats = hashstats(images)
         duplicates = self._get_duplicates(self.stats.dict())
         return DuplicatesOutput(**duplicates)

dataeval/detectors/linters/outliers.py

@@ -4,19 +4,20 @@ __all__ = []
 
 import contextlib
 from dataclasses import dataclass
-from typing import Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
+from typing import Any, Generic, Iterable, Literal, Sequence, TypeVar, Union, overload
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
-
-from dataeval.detectors.linters.merged_stats import combine_stats, get_dataset_step_from_idx
-from dataeval.metrics.stats.base import BOX_COUNT, SOURCE_INDEX
-from dataeval.metrics.stats.datasetstats import DatasetStatsOutput, datasetstats
-from dataeval.metrics.stats.dimensionstats import DimensionStatsOutput
-from dataeval.metrics.stats.labelstats import LabelStatsOutput
-from dataeval.metrics.stats.pixelstats import PixelStatsOutput
-from dataeval.metrics.stats.visualstats import VisualStatsOutput
-from dataeval.output import Output, set_metadata
+from numpy.typing import NDArray
+from torch.utils.data import Dataset
+
+from dataeval._output import Output, set_metadata
+from dataeval.metrics.stats._base import BOX_COUNT, SOURCE_INDEX, combine_stats, get_dataset_step_from_idx
+from dataeval.metrics.stats._datasetstats import DatasetStatsOutput, datasetstats
+from dataeval.metrics.stats._dimensionstats import DimensionStatsOutput
+from dataeval.metrics.stats._labelstats import LabelStatsOutput
+from dataeval.metrics.stats._pixelstats import PixelStatsOutput
+from dataeval.metrics.stats._visualstats import VisualStatsOutput
+from dataeval.typing import ArrayLike
 
 with contextlib.suppress(ImportError):
     import pandas as pd
@@ -84,7 +85,7 @@ def _create_pandas_dataframe(class_wise):
 @dataclass(frozen=True)
 class OutliersOutput(Generic[TIndexIssueMap], Output):
     """
-    Output class for :class:`Outliers` lint detector.
+    Output class for :class:`.Outliers` lint detector.
 
     Attributes
     ----------
@@ -322,8 +323,15 @@ class Outliers:
 
         return OutliersOutput(output_list)
 
+    @overload
+    def evaluate(self, data: Iterable[ArrayLike]) -> OutliersOutput[IndexIssueMap]: ...
+    @overload
+    def evaluate(self, data: Dataset[tuple[ArrayLike, Any, dict[str, Any]]]) -> OutliersOutput[IndexIssueMap]: ...
+
     @set_metadata(state=["use_dimension", "use_pixel", "use_visual", "outlier_method", "outlier_threshold"])
-    def evaluate(self, data: Iterable[ArrayLike]) -> OutliersOutput[IndexIssueMap]:
+    def evaluate(
+        self, data: Iterable[ArrayLike] | Dataset[tuple[ArrayLike, Any, dict[str, Any]]]
+    ) -> OutliersOutput[IndexIssueMap]:
         """
         Returns indices of Outliers with the issues identified for each
 
@@ -349,6 +357,7 @@ class Outliers:
         >>> results.issues[10]
         {'skew': -3.906, 'kurtosis': 13.266, 'entropy': 0.2128, 'contrast': 1.25, 'zeros': 0.05493}
         """
-        self.stats = datasetstats(images=data)
+        images = (d[0] for d in data) if isinstance(data, Dataset) else data
+        self.stats = datasetstats(images=images)
         outliers = self._get_outliers(self.stats.dict())
         return OutliersOutput(outliers)

dataeval/detectors/ood/ae.py

@@ -16,12 +16,12 @@ from typing import Callable
 
 import numpy as np
 import torch
-from numpy.typing import ArrayLike
+from numpy.typing import NDArray
 
 from dataeval.detectors.ood.base import OODBase
 from dataeval.detectors.ood.output import OODScoreOutput
-from dataeval.interop import as_numpy
-from dataeval.utils.torch.internal import predict_batch
+from dataeval.typing import ArrayLike
+from dataeval.utils.torch._internal import predict_batch
 
 
 class OOD_AE(OODBase):
@@ -30,8 +30,31 @@ class OOD_AE(OODBase):
 
     Parameters
     ----------
-    model : Autoencoder
-        An Autoencoder model.
+    model : torch.nn.Module
+        An autoencoder model to use for encoding and reconstruction of images
+        for detection of out-of-distribution samples.
+    device : str or torch.Device or None, default None
+        The device to use for the detector. None will default to the global
+        configuration selection if set, otherwise "cuda" then "cpu" by availability.
+
+    Example
+    -------
+    Perform out-of-distribution detection on test data.
+
+    >>> from dataeval.utils.torch.models import AE
+
+    >>> input_shape = train_images[0].shape
+    >>> ood = OOD_AE(AE(input_shape))
+
+    Train the autoencoder using the training data.
+
+    >>> ood.fit(train_images, threshold_perc=99, epochs=20)
+
+    Test for out-of-distribution samples on the test data.
+
+    >>> output = ood.predict(test_images)
+    >>> output.is_ood
+    array([ True,  True, False,  True,  True,  True,  True,  True])
     """
 
     def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
@@ -55,9 +78,7 @@ class OOD_AE(OODBase):
 
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := as_numpy(X))
-
+    def _score(self, X: NDArray[np.float32], batch_size: int = int(1e10)) -> OODScoreOutput:
         # reconstruct instances
         X_recon = predict_batch(X, self.model, batch_size=batch_size)
 

dataeval/detectors/ood/base.py

@@ -13,12 +13,13 @@ __all__ = []
 from typing import Callable, cast
 
 import torch
-from numpy.typing import ArrayLike
 
+from dataeval.config import get_device
 from dataeval.detectors.ood.mixin import OODBaseMixin, OODFitMixin, OODGMMMixin
-from dataeval.interop import to_numpy
-from dataeval.utils.torch.gmm import GaussianMixtureModelParams, gmm_params
-from dataeval.utils.torch.internal import get_device, trainer
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import to_numpy
+from dataeval.utils.torch._gmm import GaussianMixtureModelParams, gmm_params
+from dataeval.utils.torch._internal import trainer
 
 
 class OODBase(OODBaseMixin[torch.nn.Module], OODFitMixin[Callable[..., torch.nn.Module], torch.optim.Optimizer]):

dataeval/detectors/ood/metadata_ks_compare.py

@@ -11,7 +11,7 @@ from numpy.typing import NDArray
 from scipy.stats import iqr, ks_2samp
 from scipy.stats import wasserstein_distance as emd
 
-from dataeval.output import MappingOutput, set_metadata
+from dataeval._output import MappingOutput, set_metadata
 
 
 class MetadataKSResult(NamedTuple):

dataeval/detectors/ood/mixin.py

@@ -8,10 +8,11 @@ from abc import ABC, abstractmethod
 from typing import Callable, Generic, Literal, TypeVar
 
 import numpy as np
-from numpy.typing import ArrayLike, NDArray
+from numpy.typing import NDArray
 
-from dataeval.interop import to_numpy
-from dataeval.output import set_metadata
+from dataeval._output import set_metadata
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy, to_numpy
 
 TGMMParams = TypeVar("TGMMParams")
 
@@ -73,6 +74,9 @@ class OODBaseMixin(Generic[TModel], ABC):
     def _get_data_info(self, X: NDArray) -> tuple[tuple, type]:
         if not isinstance(X, np.ndarray):
             raise TypeError("Dataset should of type: `NDArray`.")
+        if np.min(X) < 0 or np.max(X) > 1:
+            raise ValueError("Embeddings must be on the unit interval [0-1].")
+
         return X.shape[1:], X.dtype.type
 
     def _validate(self, X: NDArray) -> None:
@@ -90,7 +94,7 @@ class OODBaseMixin(Generic[TModel], ABC):
         self._validate(X)
 
     @abstractmethod
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput: ...
+    def _score(self, X: NDArray[np.float32], batch_size: int = int(1e10)) -> OODScoreOutput: ...
 
     @set_metadata
     def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
@@ -105,11 +109,17 @@ class OODBaseMixin(Generic[TModel], ABC):
             Number of instances to process in each batch.
             Use a smaller batch size if your dataset is large or if you encounter memory issues.
 
+        Raises
+        ------
+        ValueError
+            X input data must be unit interval [0-1].
+
         Returns
         -------
         OODScoreOutput
             An object containing the instance-level and feature-level OOD scores.
         """
+        self._validate(X := as_numpy(X).astype(np.float32))
         return self._score(X, batch_size)
 
     def _threshold_score(self, ood_type: Literal["feature", "instance"] = "instance") -> np.floating:
@@ -134,12 +144,17 @@ class OODBaseMixin(Generic[TModel], ABC):
         ood_type : "feature" | "instance", default "instance"
             Predict out-of-distribution at the 'feature' or 'instance' level.
 
+        Raises
+        ------
+        ValueError
+            X input data must be unit interval [0-1].
+
         Returns
         -------
         Dictionary containing the outlier predictions for the selected level,
         and the OOD scores for the data including both 'instance' and 'feature' (if present) level scores.
         """
-        self._validate_state(X := to_numpy(X))
+        self._validate_state(X := to_numpy(X).astype(np.float32))
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
         ood_pred = score.get(ood_type) > self._threshold_score(ood_type)

dataeval/detectors/ood/output.py

@@ -8,7 +8,7 @@ from typing import Literal
 import numpy as np
 from numpy.typing import NDArray
 
-from dataeval.output import Output
+from dataeval._output import Output
 
 
 @dataclass(frozen=True)

dataeval/detectors/ood/vae.py

@@ -0,0 +1,73 @@
+"""
+Adapted for Pytorch from
+
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from __future__ import annotations
+
+__all__ = []
+
+from typing import Callable
+
+import numpy as np
+import torch
+
+from dataeval.detectors.ood.base import OODBase
+from dataeval.detectors.ood.output import OODScoreOutput
+from dataeval.typing import ArrayLike
+from dataeval.utils._array import as_numpy
+from dataeval.utils.torch._internal import predict_batch
+
+
+class OOD_VAE(OODBase):
+    """
+    Autoencoder based out-of-distribution detector.
+
+    Parameters
+    ----------
+    model : Autoencoder
+        An Autoencoder model.
+    """
+
+    def __init__(self, model: torch.nn.Module, device: str | torch.device | None = None) -> None:
+        super().__init__(model, device)
+
+    def fit(
+        self,
+        x_ref: ArrayLike,
+        threshold_perc: float,
+        loss_fn: Callable[..., torch.nn.Module] | None = None,
+        optimizer: torch.optim.Optimizer | None = None,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = False,
+    ) -> None:
+        if loss_fn is None:
+            loss_fn = torch.nn.MSELoss()
+
+        if optimizer is None:
+            optimizer = torch.optim.Adam(self.model.parameters(), lr=1e-4)
+
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
+        self._validate(X := as_numpy(X))
+
+        # reconstruct instances
+        X_recon = predict_batch(X, self.model, batch_size=batch_size)[0]  # don't need mu or logvar from model
+
+        # compute feature and instance level scores
+        fscore = np.power(X.reshape((len(X), -1)) - X_recon, 2)
+        # fscore_flat = fscore.reshape(fscore.shape[0], -1).copy()
+        # n_score_features = int(np.ceil(fscore_flat.shape[1]))
+        # sorted_fscore = np.sort(fscore_flat, axis=1)
+        # sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
+        # iscore = np.mean(sorted_fscore_perc, axis=1)
+        iscore = np.sum(fscore, axis=1)
+
+        return OODScoreOutput(iscore, fscore)

dataeval/metadata/__init__.py

@@ -0,0 +1,5 @@
+"""Explanatory functions using metadata and additional features such as ood or drift"""
+
+__all__ = ["most_deviated_factors"]
+
+from dataeval.metadata._ood import most_deviated_factors

dataeval/metadata/_ood.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+__all__ = []
+
+import warnings
+
+import numpy as np
+from numpy.typing import NDArray
+
+from dataeval.detectors.ood import OODOutput
+from dataeval.utils.data import Metadata
+
+
+def _validate_keys(keys1: list[str], keys2: list[str]) -> None:
+    """
+    Raises error when two lists are not equivalent including ordering
+
+    Parameters
+    ----------
+    keys1 : list of strings
+        List of strings to compare
+    keys2 : list of strings
+        List of strings to compare
+
+    Raises
+    ------
+    ValueError
+        If lists do not have the same values, value counts, or ordering
+    """
+
+    if keys1 != keys2:
+        raise ValueError(f"Metadata keys must be identical, got {keys1} and {keys2}")
+
+
+def _validate_factors_and_data(factors: list[str], data: NDArray) -> None:
+    """
+    Raises error when the number of factors and number of rows do not match
+
+    Parameters
+    ----------
+    factors : list of strings
+        List of factor names of size N
+    data : NDArray
+        Array of values with shape (M, N)
+
+    Raises
+    ------
+    ValueError
+        If the length of factors does not equal the length of the transposed data
+    """
+    if len(factors) != len(data.T):
+        raise ValueError(f"Factors and data have mismatched lengths. Got {len(factors)} and {len(data.T)}")
+
+
+def _combine_metadata(metadata_1: Metadata, metadata_2: Metadata) -> tuple[list[str], list[NDArray], list[NDArray]]:
+    """
+    Combines the factor names and data arrays of metadata_1 and metadata_2 when the names
+    match exactly and data has the same number of columns (factors).
+
+    Parameters
+    ----------
+    metadata_1 : Metadata
+        The set of factor names used as reference to determine the correct factor names and length of data
+    metadata_2 : Metadata
+        The compared set of factor names and data that must match metadata_1
+
+    Returns
+    -------
+    list[str]
+        The combined discrete and continuous factor names in that order.
+    list[NDArray]
+        Combined discrete and continuous data of metadata_1
+    list[NDArray]
+        Combined discrete and continuous data of metadata_2
+
+    Raises
+    ------
+    ValueError
+        If keys do not match in metadata_1 and metadata_2
+    ValueError
+        If the length of keys do not match the length of the data
+    """
+    factor_names: list[str] = []
+    m1_data: list[NDArray] = []
+    m2_data: list[NDArray] = []
+
+    # Both metadata must have the same number of factors (cols), but not necessarily samples (row)
+    if metadata_1.total_num_factors != metadata_2.total_num_factors:
+        raise ValueError(
+            f"Number of factors differs between metadata_1 ({metadata_1.total_num_factors}) "
+            f"and metadata_2 ({metadata_2.total_num_factors})"
+        )
+
+    # Validate and attach discrete data
+    if metadata_1.discrete_factor_names:
+        _validate_keys(metadata_1.discrete_factor_names, metadata_2.discrete_factor_names)
+        _validate_factors_and_data(metadata_1.discrete_factor_names, metadata_1.discrete_data)
+
+        factor_names.extend(metadata_1.discrete_factor_names)
+        m1_data.append(metadata_1.discrete_data)
+        m2_data.append(metadata_2.discrete_data)
+
+    # Validate and attach continuous data
+    if metadata_1.continuous_factor_names:
+        _validate_keys(metadata_1.continuous_factor_names, metadata_2.continuous_factor_names)
+        _validate_factors_and_data(metadata_1.continuous_factor_names, metadata_1.continuous_data)
+
+        factor_names.extend(metadata_1.continuous_factor_names)
+        m1_data.append(metadata_1.continuous_data)
+        m2_data.append(metadata_2.continuous_data)
+
+    # Turns list of discrete and continuous into one array
+    return factor_names, m1_data, m2_data
+
+
+def _calc_median_deviations(reference: NDArray, test: NDArray) -> NDArray:
+    """
+    Calculates deviations of the test data from the median of the reference data
+
+    Parameters
+    ----------
+    reference : NDArray
+        Reference values of shape (samples, factors)
+    test : NDArray
+        Incoming values where each sample's factors will be compared to the median of
+        the reference set corresponding factors
+
+    Returns
+    -------
+    NDArray
+        Scaled positive and negative deviations of the test data from the reference.
+
+    Note
+    ----
+    All return values are in the range [0, pos_inf]
+    """
+
+    # Take median over samples (rows)
+    ref_median = np.median(reference, axis=0)  # (F, )
+
+    # Shift reference and test distributions by reference
+    ref_dev = reference - ref_median  # (S, F) - F
+    test_dev = test - ref_median  # (S_t, F) - F
+
+    # Separate positive and negative distributions
+    # Fills with nans to keep shape in both 1-D and N-D matrices
+    pdev = np.where(ref_dev > 0, ref_dev, np.nan)  # (S, F)
+    ndev = np.where(ref_dev < 0, ref_dev, np.nan)  # (S, F)
+
+    # Calculate middle of positive and negative distributions per feature
+    pscale = np.nanmedian(pdev, axis=0)  # (F, )
+    nscale = np.abs(np.nanmedian(ndev, axis=0))  # (F, )
+
+    # Replace 0's for division. Negatives should not happen
+    pscale = np.where(pscale > 0, pscale, 1.0)  # (F, )
+    nscale = np.where(nscale > 0, nscale, 1.0)  # (F, )
+
+    # Scales positive values by positive scale and negative values by negative
+    return np.abs(np.where(test_dev >= 0, test_dev / pscale, test_dev / nscale))  # (S_t, F)
+
+
+def most_deviated_factors(
+    metadata_1: Metadata,
+    metadata_2: Metadata,
+    ood: OODOutput,
+) -> list[tuple[str, float]]:
+    """
+    Determines greatest deviation in metadata features per out of distribution sample in metadata_2.
+
+    Parameters
+    ----------
+    metadata_1 : Metadata
+        A reference set of Metadata containing factor names and samples
+        with discrete and/or continuous values per factor
+    metadata_2 : Metadata
+        The set of Metadata that is tested against the reference metadata.
+        This set must have the same number of features but does not require the same number of samples.
+    ood : OODOutput
+        A class output by the DataEval's OOD functions that contains which examples are OOD.
+
+    Returns
+    -------
+    list[tuple[str, float]]
+        An array of the factor name and deviation of the highest metadata deviation for each OOD example in metadata_2.
+
+    Notes
+    -----
+    1. Both :class:`.Metadata` inputs must have discrete and continuous data in the shape (samples, factors)
+       and have equivalent factor names and lengths
+    2. The flag at index `i` in :attr:`.OODOutput.is_ood` must correspond
+       directly to sample `i` of `metadata_2` being out-of-distribution from `metadata_1`
+    """
+
+    ood_mask: NDArray[np.bool] = ood.is_ood
+
+    # No metadata correlated with out of distribution data
+    if not any(ood_mask):
+        return []
+
+    # Combines reference and test factor names and data if exists and match exactly
+    # shape -> (samples, factors)
+    factor_names, md_1, md_2 = _combine_metadata(
+        metadata_1=metadata_1,
+        metadata_2=metadata_2,
+    )
+
+    metadata_ref = np.hstack(md_1) if md_1 else np.array([])
+    metadata_tst = np.hstack(md_2) if md_2 else np.array([])
+
+    if len(metadata_ref) < 3:
+        warnings.warn(
+            f"At least 3 reference metadata samples are needed, got {len(metadata_ref)}",
+            UserWarning,
+        )
+        return []
+
+    if len(metadata_tst) != len(ood_mask):
+        raise ValueError(
+            f"ood and test metadata must have the same length, "
+            f"got {len(ood_mask)} and {len(metadata_tst)} respectively."
+        )
+
+    # Calculates deviations of all samples in m2_data
+    # from the median values of the corresponding index in m1_data
+    # Guaranteed for inputs to not be empty
+    deviations = _calc_median_deviations(metadata_ref, metadata_tst)
+
+    # Get most impactful factor deviation of each sample for ood samples only
+    deviation = np.max(deviations, axis=1)[ood_mask]
+
+    # Get indices of most impactful factors for ood samples only
+    max_factors = np.argmax(deviations, axis=1)[ood_mask]
+
+    # Get names of most impactful factors TODO: Find better way than np.dtype(<U4)
+    most_ood_factors = np.array(factor_names)[max_factors].tolist()
+
+    # List of tuples matching the factor name with its deviation
+    return [(factor, dev.item()) for factor, dev in zip(most_ood_factors, deviation)]

dataeval/metrics/__init__.py

@@ -5,4 +5,4 @@ can then be analyzed in the context of a given problem.
 
 __all__ = ["bias", "estimators", "stats"]
 
-from dataeval.metrics import bias, estimators, stats
+from . import bias, estimators, stats

dataeval/metrics/bias/__init__.py

@@ -7,6 +7,7 @@ __all__ = [
     "BalanceOutput",
     "CoverageOutput",
     "DiversityOutput",
+    "LabelParityOutput",
     "ParityOutput",
     "balance",
     "coverage",
@@ -15,7 +16,7 @@ __all__ = [
     "parity",
 ]
 
-from dataeval.metrics.bias.balance import BalanceOutput, balance
-from dataeval.metrics.bias.coverage import CoverageOutput, coverage
-from dataeval.metrics.bias.diversity import DiversityOutput, diversity
-from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity
+from dataeval.metrics.bias._balance import BalanceOutput, balance
+from dataeval.metrics.bias._coverage import CoverageOutput, coverage
+from dataeval.metrics.bias._diversity import DiversityOutput, diversity
+from dataeval.metrics.bias._parity import LabelParityOutput, ParityOutput, label_parity, parity