dataeval 0.72.1__py3-none-any.whl → 0.72.2__py3-none-any.whl

dataeval/{_internal/detectors → detectors}/ood/llr.py

@@ -8,6 +8,8 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
+__all__ = ["OOD_LLR"]
+
 from functools import partial
 from typing import Callable
 
@@ -18,15 +20,14 @@ from numpy.typing import ArrayLike, NDArray
 from tf_keras.layers import Input
 from tf_keras.models import Model
 
-from dataeval._internal.detectors.ood.base import OODBase, OODScoreOutput
-from dataeval._internal.interop import to_numpy
-from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
-from dataeval._internal.models.tensorflow.trainer import trainer
-from dataeval._internal.models.tensorflow.utils import predict_batch
-from dataeval._internal.output import set_metadata
+from dataeval.detectors.ood.base import OODBase, OODScoreOutput
+from dataeval.interop import to_numpy
+from dataeval.utils.tensorflow._internal.pixelcnn import PixelCNN
+from dataeval.utils.tensorflow._internal.trainer import trainer
+from dataeval.utils.tensorflow._internal.utils import predict_batch
 
 
-def build_model(
+def _build_model(
     dist: PixelCNN, input_shape: tuple | None = None, filepath: str | None = None
 ) -> tuple[keras.Model, PixelCNN]:
     """
@@ -54,11 +55,11 @@ def build_model(
     return model, dist
 
 
-def mutate_categorical(
+def _mutate_categorical(
     X: NDArray,
     rate: float,
     seed: int = 0,
-    feature_range: tuple = (0, 255),
+    feature_range: tuple[int, int] = (0, 255),
 ) -> tf.Tensor:
     """
     Randomly change integer feature values to values within a set range
@@ -113,17 +114,17 @@ class OOD_LLR(OODBase):
         log_prob: Callable | None = None,
         sequential: bool = False,
     ) -> None:
-        self.dist_s = model
-        self.dist_b = (
+        self.dist_s: PixelCNN = model
+        self.dist_b: PixelCNN = (
             model.copy()
             if hasattr(model, "copy")
             else keras.models.clone_model(model)
             if model_background is None
             else model_background
         )
-        self.has_log_prob = hasattr(model, "log_prob")
-        self.sequential = sequential
-        self.log_prob = log_prob
+        self.has_log_prob: bool = hasattr(model, "log_prob")
+        self.sequential: bool = sequential
+        self.log_prob: Callable | None = log_prob
 
         self._ref_score: OODScoreOutput
         self._threshold_perc: float
@@ -138,8 +139,12 @@ class OOD_LLR(OODBase):
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
-        mutate_fn: Callable = mutate_categorical,
-        mutate_fn_kwargs: dict = {"rate": 0.2, "seed": 0, "feature_range": (0, 255)},
+        mutate_fn: Callable = _mutate_categorical,
+        mutate_fn_kwargs: dict[str, float | int | tuple[int, int]] = {
+            "rate": 0.2,
+            "seed": 0,
+            "feature_range": (0, 255),
+        },
         mutate_batch_size: int = int(1e10),
     ) -> None:
         """
@@ -200,11 +205,11 @@ class OOD_LLR(OODBase):
 
         if use_build:
             # build and train semantic model
-            self.model_s = build_model(self.dist_s, input_shape)[0]
+            self.model_s: keras.Model = _build_model(self.dist_s, input_shape)[0]
             self.model_s.compile(optimizer=optimizer_s)
             self.model_s.fit(X, **kwargs)
             # build and train background model
-            self.model_b = build_model(self.dist_b, input_shape)[0]
+            self.model_b: keras.Model = _build_model(self.dist_b, input_shape)[0]
             self.model_b.compile(optimizer=optimizer_b)
             self.model_b.fit(X_back, **kwargs)
         else:
@@ -280,8 +285,7 @@ class OOD_LLR(OODBase):
         logp_b = logp_fn(self.dist_b, X, return_per_feature=return_per_feature, batch_size=batch_size)
         return logp_s - logp_b
 
-    @set_metadata("dataeval.detectors")
-    def score(
+    def _score(
         self,
         X: ArrayLike,
         batch_size: int = int(1e10),

dataeval/detectors/ood/metadata_ks_compare.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import numbers
+import warnings
+from typing import Any, Mapping
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.stats import iqr, ks_2samp
+from scipy.stats import wasserstein_distance as emd
+
+
+def meta_distribution_compare(
+    md0: Mapping[str, list[Any] | NDArray[Any]], md1: Mapping[str, list[Any] | NDArray[Any]]
+) -> dict[str, dict[str, float]]:
+    """Measures the featurewise distance between two metadata distributions, and computes a p-value to evaluate its
+        significance.
+
+        Uses the Earth Mover's Distance and the Kolmogorov-Smirnov two-sample test, featurewise.
+
+        Parameters
+        ----------
+        md0 : Mapping[str, list[Any] | NDArray[Any]]
+            A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
+        md1 : Mapping[str, list[Any] | NDArray[Any]]
+            Another set of arrays of values, indexed by metadata feature names, with one value per data example per
+            feature.
+
+        Returns
+        -------
+        dict[str, KstestResult]
+            A dictionary with keys corresponding to metadata feature names, and values that are KstestResult objects, as
+            defined by scipy.stats.ks_2samp. These values also have two additional attributes: shift_magnitude and
+            statistic_location. The first is the Earth Mover's Distance normalized by the interquartile range (IQR) of
+            the reference, while the second is the value at which the KS statistic has its maximum, measured in
+            IQR-normalized units relative to the median of the reference distribution.
+
+        Examples
+        --------
+        Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
+        altitude.
+
+    >>> import numpy
+    >>> md0 = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
+    >>> md1 = {"time": [7.8, 9.10, 11.12], "altitude": [532, 9876, 211101]}
+    >>> md_out = meta_distribution_compare(md0, md1)
+    >>> for k, v in md_out.items():
+    >>>     print(k)
+    >>>     for kv in v:
+    >>>         print("\t", f"{kv}: {v[kv]:.3f}")
+    time
+             statistic_location: 0.444
+             shift_magnitude: 2.700
+             pvalue: 0.000
+    altitude
+             statistic_location: 0.478
+             shift_magnitude: 0.749
+             pvalue: 0.944
+    """
+
+    if (metadata_keys := md0.keys()) != md1.keys():
+        raise ValueError(f"Both sets of metadata keys must be identical: {list(md0)}, {list(md1)}")
+
+    mdc_dict = {}  # output dict
+    for k in metadata_keys:
+        mdc_dict.update({k: {}})
+
+        x0, x1 = list(md0[k]), list(md1[k])
+
+        allx = x0 + x1  # "+" sign concatenates lists.
+
+        if not all(isinstance(allxi, numbers.Number) for allxi in allx):  # NB: np.nan *is* a number in this context.
+            continue  # non-numeric features will return an empty dict for feature k
+
+        # from Numerical Recipes in C, 3rd ed. p. 737. If too few points, warn and keep going.
+        if np.sqrt(((N := len(x0)) * (M := len(x1))) / (N + M)) < 4:
+            warnings.warn(
+                f"Sample sizes of {N}, {M} for feature {k} will yield unreliable p-values from the KS test.",
+                UserWarning,
+            )
+
+        xmin, xmax = min(allx), max(allx)
+        if xmin == xmax:  # only one value in this feature, so fill in the obvious results for feature k
+            mdc_dict[k].update({"statistic_location": 0.0, "shift_magnitude": 0.0, "pvalue": 1.0})
+            continue
+
+        ks_result = ks_2samp(x0, x1, method="asymp")
+        dev = ks_result.statistic_location - xmin  #  pyright: ignore  (KSresult type)
+        loc = dev / (xmax - xmin) if xmax > xmin else dev
+
+        dX = iqr(x0)  # preferred value of dX, which is the scale of the the md0 values for feature k
+        dX = (max(x0) - min(x0)) / 2.0 if dX == 0 else dX  # reasonable alternative value of dX, when iqr is zero.
+        dX = 1.0 if dX == 0 else dX  # if dX is *still* zero, just avoid division by zero this way
+
+        drift = emd(x0, x1) / dX
+
+        mdc_dict[k].update({"statistic_location": loc, "shift_magnitude": drift, "pvalue": ks_result.pvalue})  #  pyright: ignore
+
+    return mdc_dict

dataeval/detectors/ood/metadata_least_likely.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import numbers
+import warnings
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+def get_least_likely_features(
+    metadata: dict[str, list[Any] | NDArray[Any]],
+    new_metadata: dict[str, list[Any] | NDArray[Any]],
+    is_ood: NDArray[np.bool_],
+) -> list[tuple[str, float]]:
+    """Computes which metadata feature is most out-of-distribution (OOD) relative to a reference metadata set.
+
+    Given a reference metadata dictionary `metadata` (where each key maps to one scalar metadata feature), a second
+    metadata dictionary, and a corresponding boolean flag `is_ood` indicating whether each new example falls
+    out-of-distribution (OOD) relative to the reference, this function finds which metadata feature is the most OOD,
+    for each OOD example.
+
+    Parameters
+    ----------
+    metadata: dict[str, list[Any] | NDArray[Any]]
+        A reference set of arrays of values, indexed by metadata feature names, with one value per data example per
+        feature.
+    new_metadata: dict[str, list[Any] | NDArray[Any]]
+        A second metadata set, to be tested against the reference metadata. It is ok if the two meta data objects
+        hold different numbers of examples.
+    is_ood: NDArray[np.bool_]
+        A boolean array, with one value per new_metadata example, that indicates which examples are OOD.
+
+    Returns
+    -------
+    list[tuple[str, float]]
+        An array of names of the features of each OOD new_metadata example that were the most OOD.
+
+    Examples
+    --------
+    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and
+    altitude, as shown below.
+
+    >>> from dataeval._internal.metrics.metadata_least_likely import get_least_likely_features
+    >>> import numpy
+    >>> metadata = {"time": [1.2, 3.4, 5.6], "altitude": [235, 6789, 101112]}
+    >>> new_metadata = {"time": [7.8, 11.12], "altitude": [532, -211101]}
+    >>> is_ood = numpy.array([True, True])
+    >>> get_least_likely_features(metadata, new_metadata, is_ood)
+    [('time', 2.0), ('altitude', 33.245346)]
+    """
+    # Raise errors for bad inputs...
+
+    if metadata.keys() != new_metadata.keys():
+        raise ValueError(f"Reference and test metadata keys must be identical: {list(metadata)}, {list(new_metadata)}")
+
+    md_lengths = {len(np.atleast_1d(v)) for v in metadata.values()}
+    new_md_lengths = {len(np.atleast_1d(v)) for v in new_metadata.values()}
+    if len(md_lengths) > 1 or len(new_md_lengths) > 1:
+        raise ValueError(f"All features must have same length, got lengths {md_lengths}, {new_md_lengths}")
+
+    n_reference, n_new = md_lengths.pop(), new_md_lengths.pop()  # possibly different numbers of metadata examples
+
+    if n_new != len(is_ood):
+        raise ValueError(f"is_ood flag must have same length as new metadata {n_new} but has length {len(is_ood)}.")
+
+    if n_reference < 3:  # too hard to define "in-distribution" with this few reference samples.
+        warnings.warn(
+            "We need at least 3 reference metadata examples to determine which "
+            f"features are least likely, but only got {n_reference}",
+            UserWarning,
+        )
+        return []
+
+    if not any(is_ood):
+        return []
+
+    # ...inputs are good, look for most deviant standardized features.
+
+    # largest standardized absolute deviation from the median observed so far for each example
+    deviation = np.zeros_like(is_ood, dtype=np.float32)
+
+    # name of feature that corresponds to `deviation` for each example
+    kmax = np.empty(len(is_ood), dtype=object)
+
+    for k, v in metadata.items():
+        # exclude cases where random happens to be out on tails, not interesting.
+        if k == "random":
+            continue
+
+        # Skip non-numerical features
+        if not all(isinstance(vi, numbers.Number) for vi in v):  # NB: np.nan *is* a number in this context.
+            continue
+
+        # Get standardization parameters from metadata
+        loc = np.median(v)  # ok, because we checked all were numeric
+        dev = np.asarray(v) - loc  # need to make array from v since it could be a list here.
+        posdev, negdev = dev[dev > 0], dev[dev < 0]
+        pos_scale = np.median(posdev) if posdev.any() else 1.0
+        neg_scale = np.abs(np.median(negdev)) if negdev.any() else 1.0
+
+        x, x0, dxp, dxn = np.atleast_1d(new_metadata[k]), loc, pos_scale, neg_scale  # just abbreviations
+        dxp = dxp if dxp > 0 else 1.0  # avoids dividing by zero below
+        dxn = dxn if dxn > 0 else 1.0
+
+        # xdev must be floating-point to avoid getting zero in an integer division.
+        xdev = (x - x0).astype(np.float64)
+        pos = xdev >= 0
+
+        X = np.zeros_like(xdev)
+        X[pos], X[~pos] = xdev[pos] / dxp, xdev[~pos] / dxn  # keeping track of possible asymmetry of x, but...
+        # ...below here, only need to think about absolute deviation.
+
+        abig = np.abs(X) > deviation
+        kmax[abig] = k
+        deviation[abig] = np.abs(X[abig])
+
+    unlikely_features = list(zip(kmax[is_ood], deviation[is_ood]))  # feature names, along with how far out they are.
+    return unlikely_features

dataeval/detectors/ood/metadata_ood_mi.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import numbers
+import warnings
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+from sklearn.feature_selection import mutual_info_classif
+
+# NATS2BITS is the reciprocal of natural log of 2. If you have an information/entropy-type quantity measured in nats,
+#   which is what many library functions return, multiply it by NATS2BITS to get it in bits.
+NATS2BITS = 1.442695
+
+
+def get_metadata_ood_mi(
+    metadata: dict[str, list[Any] | NDArray[Any]],
+    is_ood: NDArray[np.bool_],
+    discrete_features: str | bool | NDArray[np.bool_] = False,
+    random_state: int | None = None,
+) -> dict[str, float]:
+    """Computes mutual information between a set of metadata features and an out-of-distribution flag.
+
+    Given a metadata dictionary `metadata` (where each key maps to one scalar metadata feature per example), and a
+    corresponding boolean flag `is_ood` indicating whether each example falls out-of-distribution (OOD) relative to a
+    reference dataset, this function finds the strength of association between each metadata feature and `is_ood` by
+    computing their mutual information. Metadata features may be either discrete or continuous; set the
+    `discrete_features` keyword to a bool array set to True for each feature that is discrete, or pass one bool to apply
+    to all features. Returns a dict indicating the strength of association between each individual feature and the OOD
+    flag, measured in bits.
+
+    Parameters
+    ----------
+    metadata : dict[str, list[Any] | NDArray[Any]]
+        A set of arrays of values, indexed by metadata feature names, with one value per data example per feature.
+    is_ood : NDArray[np.bool_]
+        A boolean array, with one value per example, that indicates which examples are OOD.
+    discrete_features : str | bool | NDArray[np.bool_]
+        Either a boolean array or a single boolean value, indicate which features take on discrete values.
+    random_state : int, optional - default None
+        Determines random number generation for small noise added to continuous variables. Set to a value for
+        reproducible results.
+
+    Returns
+    -------
+    dict[str, float]
+        A dictionary with keys corresponding to metadata feature names, and values indicating the strength of
+        association between each named feature and the OOD flag, as mutual information measured in bits.
+
+    Examples
+    --------
+    Imagine we have 3 data examples, and that the corresponding metadata contains 2 features called time and altitude.
+
+        >>> import numpy
+        >>> metadata = {"time": numpy.linspace(0, 10, 100), "altitude": numpy.linspace(0, 16, 100) ** 2}
+        >>> is_ood = metadata["altitude"] > 100
+        >>> print(get_metadata_ood_mi(metadata, is_ood, discrete_features=False))
+        {'time': 0.933074285817367, 'altitude': 0.9407686591507002}
+    """
+    numerical_keys = [k for k, v in metadata.items() if all(isinstance(vi, numbers.Number) for vi in v)]
+    if len(numerical_keys) < len(metadata):
+        warnings.warn(
+            f"Processing {numerical_keys}, others are non-numerical and will be skipped.",
+            UserWarning,
+        )
+
+    md_lengths = {len(np.atleast_1d(v)) for v in metadata.values()}
+    if len(md_lengths) > 1:
+        raise ValueError(f"Metadata features have differing sizes: {md_lengths}")
+
+    if len(is_ood) != (mdl := md_lengths.pop()):
+        raise ValueError(
+            f"OOD flag and metadata features need to be same size, but are different sizes: {len(is_ood)} and {mdl}."
+        )
+
+    X = np.array([metadata[k] for k in numerical_keys]).T
+
+    X0, dX = np.mean(X, axis=0), np.std(X, axis=0, ddof=1)
+    Xscl = (X - X0) / dX
+
+    mutual_info_values = (
+        mutual_info_classif(
+            Xscl,
+            is_ood,
+            discrete_features=discrete_features,  # type: ignore
+            random_state=random_state,
+        )
+        * NATS2BITS
+    )
+
+    mi_dict = {k: mutual_info_values[i] for i, k in enumerate(numerical_keys)}
+    return mi_dict

dataeval/{_internal/detectors → detectors}/ood/vae.py

@@ -8,6 +8,8 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
+__all__ = ["OOD_VAE"]
+
 from typing import Callable
 
 import numpy as np
@@ -15,12 +17,11 @@ import tensorflow as tf
 import tf_keras as keras
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODBase, OODScoreOutput
-from dataeval._internal.interop import to_numpy
-from dataeval._internal.models.tensorflow.autoencoder import VAE
-from dataeval._internal.models.tensorflow.losses import Elbo
-from dataeval._internal.models.tensorflow.utils import predict_batch
-from dataeval._internal.output import set_metadata
+from dataeval.detectors.ood.base import OODBase, OODScoreOutput
+from dataeval.interop import to_numpy
+from dataeval.utils.tensorflow._internal.autoencoder import VAE
+from dataeval.utils.tensorflow._internal.loss import Elbo
+from dataeval.utils.tensorflow._internal.utils import predict_batch
 
 
 class OOD_VAE(OODBase):
@@ -38,7 +39,7 @@ class OOD_VAE(OODBase):
     --------
     Instantiate an OOD detector metric with a generic dataset - batch of images with shape (3,25,25)
 
-    >>> metric = OOD_VAE(create_model(VAE, dataset[0].shape))
+    >>> metric = OOD_VAE(create_model("VAE", dataset[0].shape))
 
     Adjusting fit parameters,
     including setting the fit threshold at 85% for a training set with about 15% out-of-distribution
@@ -58,18 +59,15 @@ class OOD_VAE(OODBase):
         self,
         x_ref: ArrayLike,
         threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = None,
+        loss_fn: Callable[..., tf.Tensor] = Elbo(0.05),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
     ) -> None:
-        if loss_fn is None:
-            loss_fn = Elbo(0.05)
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    @set_metadata("dataeval.detectors")
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
+    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         self._validate(X := to_numpy(X))
 
         # sample reconstructed instances

dataeval/{_internal/detectors → detectors}/ood/vaegmm.py

@@ -8,6 +8,8 @@ Licensed under Apache Software License (Apache 2.0)
 
 from __future__ import annotations
 
+__all__ = ["OOD_VAEGMM"]
+
 from typing import Callable
 
 import numpy as np
@@ -15,13 +17,12 @@ import tensorflow as tf
 import tf_keras as keras
 from numpy.typing import ArrayLike
 
-from dataeval._internal.detectors.ood.base import OODGMMBase, OODScoreOutput
-from dataeval._internal.interop import to_numpy
-from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
-from dataeval._internal.models.tensorflow.gmm import gmm_energy
-from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM
-from dataeval._internal.models.tensorflow.utils import predict_batch
-from dataeval._internal.output import set_metadata
+from dataeval.detectors.ood.base import OODGMMBase, OODScoreOutput
+from dataeval.interop import to_numpy
+from dataeval.utils.tensorflow._internal.autoencoder import VAEGMM
+from dataeval.utils.tensorflow._internal.gmm import gmm_energy
+from dataeval.utils.tensorflow._internal.loss import Elbo, LossGMM
+from dataeval.utils.tensorflow._internal.utils import predict_batch
 
 
 class OOD_VAEGMM(OODGMMBase):
@@ -44,38 +45,15 @@ class OOD_VAEGMM(OODGMMBase):
         self,
         x_ref: ArrayLike,
         threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = None,
+        loss_fn: Callable[..., tf.Tensor] = LossGMM(elbo=Elbo(0.05)),
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
     ) -> None:
-        if loss_fn is None:
-            loss_fn = LossGMM(elbo=Elbo(0.05))
         super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
-    @set_metadata("dataeval.detectors")
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        """
-        Compute the out of distributuion<Out-of-distribution (OOD)>` score for a given dataset.
-
-        Parameters
-        ----------
-        X : ArrayLike
-            Input data to score.
-        batch_size : int, default 1e10
-            Number of instances to process in each batch.
-            Use a smaller batch size if your dataset is large or if you encounter memory issues.
-
-        Returns
-        -------
-        OODScoreOutput
-            An object containing the instance-level OOD score.
-
-        Note
-        ----
-        This model does not produce a feature level score like the OOD_AE or OOD_VAE models.
-        """
+    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
         self._validate(X := to_numpy(X))
 
         # draw samples from latent space

dataeval/{_internal/interop.py → interop.py}

@@ -1,43 +1,47 @@
 from __future__ import annotations
 
+__all__ = ["as_numpy", "to_numpy", "to_numpy_iter"]
+
 from importlib import import_module
 from typing import Any, Iterable, Iterator
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
-module_cache = {}
+_MODULE_CACHE = {}
 
 
-def try_import(module_name):
-    if module_name in module_cache:
-        return module_cache[module_name]
+def _try_import(module_name):
+    if module_name in _MODULE_CACHE:
+        return _MODULE_CACHE[module_name]
 
     try:
         module = import_module(module_name)
     except ImportError:  # pragma: no cover - covered by test_mindeps.py
         module = None
 
-    module_cache[module_name] = module
+    _MODULE_CACHE[module_name] = module
     return module
 
 
 def as_numpy(array: ArrayLike | None) -> NDArray[Any]:
+    """Converts an ArrayLike to Numpy array without copying (if possible)"""
     return to_numpy(array, copy=False)
 
 
 def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
+    """Converts an ArrayLike to new Numpy array"""
     if array is None:
         return np.ndarray([])
 
     if isinstance(array, np.ndarray):
         return array.copy() if copy else array
 
-    tf = try_import("tensorflow")
+    tf = _try_import("tensorflow")
     if tf and tf.is_tensor(array):
         return array.numpy().copy() if copy else array.numpy()  # type: ignore
 
-    torch = try_import("torch")
+    torch = _try_import("torch")
     if torch and isinstance(array, torch.Tensor):
         return array.detach().cpu().numpy().copy() if copy else array.detach().cpu().numpy()  # type: ignore
 
@@ -45,5 +49,6 @@ def to_numpy(array: ArrayLike | None, copy: bool = True) -> NDArray[Any]:
 
 
 def to_numpy_iter(iterable: Iterable[ArrayLike]) -> Iterator[NDArray[Any]]:
+    """Yields an iterator of numpy arrays from an ArrayLike"""
     for array in iterable:
         yield to_numpy(array)

dataeval/metrics/__init__.py

@@ -3,6 +3,6 @@ Metrics are a way to measure the performance of your models or datasets that
 can then be analyzed in the context of a given problem.
 """
 
-from . import bias, estimators, stats
+from dataeval.metrics import bias, estimators, stats
 
 __all__ = ["bias", "estimators", "stats"]

dataeval/metrics/bias/__init__.py

@@ -3,10 +3,10 @@ Bias metrics check for skewed or imbalanced datasets and incomplete feature
 representation which may impact model performance.
 """
 
-from dataeval._internal.metrics.balance import BalanceOutput, balance
-from dataeval._internal.metrics.coverage import CoverageOutput, coverage
-from dataeval._internal.metrics.diversity import DiversityOutput, diversity
-from dataeval._internal.metrics.parity import ParityOutput, label_parity, parity
+from dataeval.metrics.bias.balance import BalanceOutput, balance
+from dataeval.metrics.bias.coverage import CoverageOutput, coverage
+from dataeval.metrics.bias.diversity import DiversityOutput, diversity
+from dataeval.metrics.bias.parity import ParityOutput, label_parity, parity
 
 __all__ = [
     "balance",