dataeval 0.64.0__py3-none-any.whl → 0.66.0__py3-none-any.whl

dataeval/_internal/workflows/sufficiency.py

@@ -1,62 +1,88 @@
+from __future__ import annotations
+
 import warnings
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union, cast
+from dataclasses import dataclass
+from typing import Any, Callable, Sequence, cast
 
 import matplotlib.pyplot as plt
 import numpy as np
 import torch
 import torch.nn as nn
 from matplotlib.figure import Figure
+from numpy.typing import NDArray
 from scipy.optimize import basinhopping
 from torch.utils.data import Dataset
 
-from dataeval._internal.metrics.base import EvaluateMixin
+from dataeval._internal.output import OutputMetadata, set_metadata
+
 
-STEPS_KEY = "_STEPS_"
-PARAMS_KEY = "_CURVE_PARAMS_"
+@dataclass(frozen=True)
+class SufficiencyOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    steps : NDArray
+        Array of sample sizes
+    params : Dict[str, NDArray]
+        Inverse power curve coefficients for the line of best fit for each measure
+    measures : Dict[str, NDArray]
+        Average of values observed for each sample size step for each measure
+    """
 
-SufficiencyOutput = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+    steps: NDArray[np.uint32]
+    params: dict[str, NDArray[np.float64]]
+    measures: dict[str, NDArray[np.float64]]
 
+    def __post_init__(self):
+        c = len(self.steps)
+        if set(self.params) != set(self.measures):
+            raise ValueError("params and measures have a key mismatch")
+        for m, v in self.measures.items():
+            c_v = v.shape[1] if v.ndim > 1 else len(v)
+            if c != c_v:
+                raise ValueError(f"{m} does not contain the expected number ({c}) of data points.")
 
-def f_out(n_i: np.ndarray, x: np.ndarray) -> np.ndarray:
+
+def f_out(n_i: NDArray, x: NDArray) -> NDArray:
     """
     Calculates the line of best fit based on its free parameters
 
     Parameters
     ----------
-    n_i : np.ndarray
+    n_i : NDArray
         Array of sample sizes
-    x : np.ndarray
+    x : NDArray
         Array of inverse power curve coefficients
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Data points for the line of best fit
     """
     return x[0] * n_i ** (-x[1]) + x[2]
 
 
-def f_inv_out(y_i: np.ndarray, x: np.ndarray) -> np.ndarray:
+def f_inv_out(y_i: NDArray, x: NDArray) -> NDArray[np.uint64]:
     """
     Inverse function for f_out()
 
     Parameters
     ----------
-    y_i : np.ndarray
+    y_i : NDArray
         Data points for the line of best fit
-    x : np.ndarray
+    x : NDArray
         Array of inverse power curve coefficients
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Array of sample sizes
     """
     n_i = ((y_i - x[2]) / x[0]) ** (-1 / x[1])
-    return n_i
+    return np.asarray(n_i, dtype=np.uint64)
 
 
-def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
+def calc_params(p_i: NDArray, n_i: NDArray, niter: int) -> NDArray:
     """
     Retrieves the inverse power curve coefficients for the line of best fit.
     Global minimization is done via basin hopping. More info on this algorithm
@@ -64,9 +90,9 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
 
     Parameters
     ----------
-    p_i : np.ndarray
+    p_i : NDArray
         Array of corresponding losses
-    n_i : np.ndarray
+    n_i : NDArray
         Array of sample sizes
     niter : int
         Number of iterations to perform in the basin-hopping
@@ -74,7 +100,7 @@ def calc_params(p_i: np.ndarray, n_i: np.ndarray, niter: int) -> np.ndarray:
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Array of parameters to recreate line of best fit
     """
 
@@ -128,60 +154,46 @@ def validate_dataset_len(dataset: Dataset) -> int:
     return length
 
 
-def validate_output(data: SufficiencyOutput):
-    """Ensure the sufficiency data used is not malformed"""
-    if not all(key in data for key in [STEPS_KEY, PARAMS_KEY]):
-        raise KeyError(f"{STEPS_KEY} and {PARAMS_KEY} are required keys for Sufficiency output.")
-    c = len(data[STEPS_KEY])
-    for m, v in data.items():
-        if m in [STEPS_KEY, PARAMS_KEY]:
-            continue
-        v = cast(np.ndarray, v)
-        c_v = v.shape[1] if v.ndim > 1 else len(v)
-        if c != c_v:
-            raise ValueError("f{m} does not contain the expected number ({c}) of data points.")
-
-
-def project_steps(params: np.ndarray, projection: np.ndarray) -> np.ndarray:
+def project_steps(params: NDArray, projection: NDArray) -> NDArray:
     """Projects the measures for each value of X
 
     Parameters
     ----------
-    params : np.ndarray
+    params : NDArray
         Inverse power curve coefficients used to calculate projection
-    projection : np.ndarray
+    projection : NDArray
         Steps to extrapolate
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Extrapolated measure values at each projection step
 
     """
     return 1 - f_out(projection, params)
 
 
-def inv_project_steps(params: np.ndarray, targets: np.ndarray) -> np.ndarray:
+def inv_project_steps(params: NDArray, targets: NDArray) -> NDArray[np.uint64]:
     """Inverse function for project_steps()
 
     Parameters
     ----------
-    params : np.ndarray
+    params : NDArray
         Inverse power curve coefficients used to calculate projection
-    targets : np.ndarray
+    targets : NDArray
         Desired measure values
 
     Returns
     -------
-    np.ndarray
+    NDArray
         Array of sample sizes, or 0 if overflow
     """
     steps = f_inv_out(1 - np.array(targets), params)
     steps[np.isnan(steps)] = 0
-    return np.ceil(steps).astype(np.int64)
+    return np.ceil(steps)
 
 
-def get_curve_params(measures: Dict[str, np.ndarray], ranges: np.ndarray, niter: int) -> Dict[str, np.ndarray]:
+def get_curve_params(measures: dict[str, NDArray], ranges: NDArray, niter: int) -> dict[str, NDArray]:
     """Calculates and aggregates parameters for both single and multi-class metrics"""
     output = {}
     for name, measure in measures.items():
@@ -198,10 +210,10 @@ def get_curve_params(measures: Dict[str, np.ndarray], ranges: np.ndarray, niter:
 
 def plot_measure(
     name: str,
-    steps: np.ndarray,
-    measure: np.ndarray,
-    params: np.ndarray,
-    projection: np.ndarray,
+    steps: NDArray,
+    measure: NDArray,
+    params: NDArray,
+    projection: NDArray,
 ) -> Figure:
     fig = plt.figure()
     fig = cast(Figure, fig)
@@ -228,7 +240,7 @@ def plot_measure(
     return fig
 
 
-class Sufficiency(EvaluateMixin):
+class Sufficiency:
     """
     Project dataset sufficiency using given a model and evaluation criteria
 
@@ -236,15 +248,15 @@ class Sufficiency(EvaluateMixin):
     ----------
     model : nn.Module
         Model that will be trained for each subset of data
-    train_ds : Dataset
+    train_ds : torch.Dataset
         Full training data that will be split for each run
-    test_ds : Dataset
+    test_ds : torch.Dataset
         Data that will be used for every run's evaluation
     train_fn : Callable[[nn.Module, Dataset, Sequence[int]], None]
         Function which takes a model (torch.nn.Module), a dataset
         (torch.utils.data.Dataset), indices to train on and executes model
         training against the data.
-    eval_fn : Callable[[nn.Module, Dataset], Dict[str, float]]
+    eval_fn : Callable[[nn.Module, Dataset], Dict[str, float | NDArray]]
         Function which takes a model (torch.nn.Module), a dataset
         (torch.utils.data.Dataset) and returns a dictionary of metric
         values (Dict[str, float]) which is used to assess model performance
@@ -253,9 +265,9 @@ class Sufficiency(EvaluateMixin):
         Number of models to run over all subsets
     substeps : int, default 5
         Total number of dataset partitions that each model will train on
-    train_kwargs : Dict[str, Any] | None, default None
+    train_kwargs : Dict | None, default None
         Additional arguments required for custom training function
-    eval_kwargs : Dict[str, Any] | None, default None
+    eval_kwargs : Dict | None, default None
         Additional arguments required for custom evaluation function
     """
 
@@ -265,11 +277,11 @@ class Sufficiency(EvaluateMixin):
         train_ds: Dataset,
         test_ds: Dataset,
         train_fn: Callable[[nn.Module, Dataset, Sequence[int]], None],
-        eval_fn: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]],
+        eval_fn: Callable[[nn.Module, Dataset], dict[str, float] | dict[str, NDArray]],
         runs: int = 1,
         substeps: int = 5,
-        train_kwargs: Optional[Dict[str, Any]] = None,
-        eval_kwargs: Optional[Dict[str, Any]] = None,
+        train_kwargs: dict[str, Any] | None = None,
+        eval_kwargs: dict[str, Any] | None = None,
     ):
         self.model = model
         self.train_ds = train_ds
@@ -312,41 +324,42 @@ class Sufficiency(EvaluateMixin):
     @property
     def eval_fn(
         self,
-    ) -> Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]]:
+    ) -> Callable[[nn.Module, Dataset], dict[str, float] | dict[str, NDArray]]:
         return self._eval_fn
 
     @eval_fn.setter
     def eval_fn(
         self,
-        value: Callable[[nn.Module, Dataset], Union[Dict[str, float], Dict[str, np.ndarray]]],
+        value: Callable[[nn.Module, Dataset], dict[str, float] | dict[str, NDArray]],
     ):
         if not callable(value):
             raise TypeError("Must provide a callable for eval_fn.")
         self._eval_fn = value
 
     @property
-    def train_kwargs(self) -> Dict[str, Any]:
+    def train_kwargs(self) -> dict[str, Any]:
         return self._train_kwargs
 
     @train_kwargs.setter
-    def train_kwargs(self, value: Optional[Dict[str, Any]]):
+    def train_kwargs(self, value: dict[str, Any] | None):
         self._train_kwargs = {} if value is None else value
 
     @property
-    def eval_kwargs(self) -> Dict[str, Any]:
+    def eval_kwargs(self) -> dict[str, Any]:
         return self._eval_kwargs
 
     @eval_kwargs.setter
-    def eval_kwargs(self, value: Optional[Dict[str, Any]]):
+    def eval_kwargs(self, value: dict[str, Any] | None):
         self._eval_kwargs = {} if value is None else value
 
-    def evaluate(self, eval_at: Optional[np.ndarray] = None, niter: int = 1000) -> SufficiencyOutput:
+    @set_metadata("dataeval.workflows", ["runs", "substeps"])
+    def evaluate(self, eval_at: NDArray | None = None, niter: int = 1000) -> SufficiencyOutput:
         """
         Creates data indices, trains models, and returns plotting data
 
         Parameters
         ----------
-        eval_at : Optional[np.ndarray]
+        eval_at : NDArray | None, default None
             Specify this to collect accuracies over a specific set of dataset lengths, rather
             than letting Sufficiency internally create the lengths to evaluate at.
         niter : int, default 1000
@@ -354,9 +367,17 @@ class Sufficiency(EvaluateMixin):
 
         Returns
         -------
-        Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
-            Dictionary containing the average of each measure per substep
-        """
+        SufficiencyOutput
+            Dataclass containing the average of each measure per substep
+
+        Examples
+        --------
+        >>> suff = Sufficiency(
+        ...     model=model, train_ds=train_ds, test_ds=test_ds, train_fn=train_fn, eval_fn=eval_fn, runs=3, substeps=5
+        ... )
+        >>> suff.evaluate()
+        SufficiencyOutput(steps=array([  1,   3,  10,  31, 100], dtype=uint32), params={'test': array([ 0., 42.,  0.])}, measures={'test': array([1., 1., 1., 1., 1.])})
+        """  # noqa: E501
         if eval_at is not None:
             ranges = eval_at
         else:
@@ -365,7 +386,7 @@ class Sufficiency(EvaluateMixin):
                 self._length,
                 self.substeps,
             )  # Start, Stop, Num steps
-            ranges = np.geomspace(*geomshape).astype(np.int64)
+            ranges = np.geomspace(*geomshape, dtype=np.uint32)
         substeps = len(ranges)
         measures = {}
 
@@ -381,7 +402,7 @@ class Sufficiency(EvaluateMixin):
                 self.train_fn(
                     model,
                     self.train_ds,
-                    indices[:substep].tolist(),
+                    indices[: int(substep)].tolist(),
                     **self.train_kwargs,
                 )
 
@@ -390,9 +411,6 @@ class Sufficiency(EvaluateMixin):
 
                 # Keep track of each measures values
                 for name, value in measure.items():
-                    if name in [STEPS_KEY, PARAMS_KEY]:
-                        raise KeyError(f"Cannot use reserved name '{name}' as a metric name.")
-
                     # Sum result into current substep iteration to be averaged later
                     value = np.array(value).ravel()
                     if name not in measures:
@@ -402,66 +420,61 @@ class Sufficiency(EvaluateMixin):
         # The mean for each measure must be calculated before being returned
         measures = {k: (v / self.runs).T for k, v in measures.items()}
         params_output = get_curve_params(measures, ranges, niter)
-        output = {STEPS_KEY: ranges, PARAMS_KEY: params_output}
-        output.update(measures)
-        return output
+        return SufficiencyOutput(ranges, params_output, measures)
 
     @classmethod
     def project(
         cls,
         data: SufficiencyOutput,
-        projection: Union[int, Sequence[int], np.ndarray],
-    ) -> Dict[str, np.ndarray]:
+        projection: int | Sequence[int] | NDArray[np.uint],
+    ) -> SufficiencyOutput:
         """Projects the measures for each value of X
 
         Parameters
         ----------
-        data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+        data : SufficiencyOutput
             Dataclass containing the average of each measure per substep
-        steps : Union[int, np.ndarray]
+        projection : int | Sequence[int] | NDArray[np.uint]
             Step or steps to project
-        niter : int, default 200
-            Number of iterations to perform in the basin-hopping
-            numerical process to curve-fit data
+
+        Returns
+        -------
+        SufficiencyOutput
+            Dataclass containing the projected measures per projection
 
         Raises
         ------
-        KeyError
-            If STEPS_KEY or measure is not a valid key
         ValueError
             If the length of data points in the measures do not match
             If the steps are not int, Sequence[int] or an ndarray
         """
-        validate_output(data)
         projection = [projection] if isinstance(projection, int) else projection
         projection = np.array(projection) if isinstance(projection, Sequence) else projection
         if not isinstance(projection, np.ndarray):
             raise ValueError("'steps' must be an int, Sequence[int] or ndarray")
 
         output = {}
-        output[STEPS_KEY] = projection
-        for name, measures in data.items():
-            if name in [STEPS_KEY, PARAMS_KEY]:
-                continue
-            measures = cast(np.ndarray, measures)
+        for name, measures in data.measures.items():
             if measures.ndim > 1:
                 result = []
                 for i in range(len(measures)):
-                    projected = project_steps(data[PARAMS_KEY][name][i], projection)
+                    projected = project_steps(data.params[name][i], projection)
                     result.append(projected)
-                output[name] = np.array(result).T
+                output[name] = np.array(result)
             else:
-                output[name] = project_steps(data[PARAMS_KEY][name], projection)
-        return output
+                output[name] = project_steps(data.params[name], projection)
+        return SufficiencyOutput(projection, data.params, output)
 
     @classmethod
-    def plot(cls, data: SufficiencyOutput, class_names: Optional[Sequence[str]] = None) -> List[Figure]:
+    def plot(cls, data: SufficiencyOutput, class_names: Sequence[str] | None = None) -> list[Figure]:
         """Plotting function for data sufficiency tasks
 
         Parameters
         ----------
-        data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+        data : SufficiencyOutput
             Dataclass containing the average of each measure per substep
+        class_names : Sequence[str] | None, default None
+            List of class names
 
         Returns
         -------
@@ -470,29 +483,19 @@ class Sufficiency(EvaluateMixin):
 
         Raises
         ------
-        KeyError
-            If STEPS_KEY or measure is not a valid key
         ValueError
             If the length of data points in the measures do not match
         """
-        validate_output(data)
-
-        # X, y data
-        steps = cast(np.ndarray, data[STEPS_KEY])
-
         # Extrapolation parameters
-        last_X = steps[-1]
-        geomshape = (0.01 * last_X, last_X * 4, len(steps))
+        last_X = data.steps[-1]
+        geomshape = (0.01 * last_X, last_X * 4, len(data.steps))
         extrapolated = np.geomspace(*geomshape).astype(np.int64)
 
         # Stores all plots
         plots = []
 
         # Create a plot for each measure on one figure
-        for name, measures in data.items():
-            if name in [STEPS_KEY, PARAMS_KEY]:
-                continue
-            measures = cast(np.ndarray, measures)
+        for name, measures in data.measures.items():
             if measures.ndim > 1:
                 if class_names is not None and len(measures) != len(class_names):
                     raise IndexError("Class name count does not align with measures")
@@ -500,56 +503,54 @@ class Sufficiency(EvaluateMixin):
                     class_name = str(i) if class_names is None else class_names[i]
                     fig = plot_measure(
                         f"{name}_{class_name}",
-                        steps,
+                        data.steps,
                         measure,
-                        data[PARAMS_KEY][name][i],
+                        data.params[name][i],
                         extrapolated,
                     )
                     plots.append(fig)
 
             else:
-                fig = plot_measure(name, steps, measures, data[PARAMS_KEY][name], extrapolated)
+                fig = plot_measure(name, data.steps, measures, data.params[name], extrapolated)
                 plots.append(fig)
 
         return plots
 
     @classmethod
-    def inv_project(cls, targets: Dict[str, np.ndarray], data: SufficiencyOutput) -> Dict[str, np.ndarray]:
+    def inv_project(cls, targets: dict[str, NDArray], data: SufficiencyOutput) -> dict[str, NDArray]:
         """
         Calculate training samples needed to achieve target model metric values.
 
         Parameters
         ----------
-        targets : Dict[str, np.ndarray]
+        targets : Dict[str, NDArray]
             Dictionary of target metric scores (from 0.0 to 1.0) that we want
             to achieve, where the key is the name of the metric.
 
-        data : Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+        data : SufficiencyOutput
             Dataclass containing the average of each measure per substep
 
         Returns
         -------
-        Dict[str, np.ndarray]
+        Dict[str, NDArray]
             List of the number of training samples needed to achieve each
             corresponding entry in targets
         """
 
-        validate_output(data)
-
         projection = {}
 
         for name, target in targets.items():
-            if name not in data:
+            if name not in data.measures:
                 continue
 
-            measure = cast(np.ndarray, data[name])
+            measure = data.measures[name]
             if measure.ndim > 1:
                 projection[name] = np.zeros((len(measure), len(target)))
                 for i in range(len(measure)):
                     projection[name][i] = inv_project_steps(
-                        data[PARAMS_KEY][name][i], target[i] if target.ndim == measure.ndim else target
+                        data.params[name][i], target[i] if target.ndim == measure.ndim else target
                     )
             else:
-                projection[name] = inv_project_steps(data[PARAMS_KEY][name], target)
+                projection[name] = inv_project_steps(data.params[name], target)
 
         return projection

dataeval/detectors/__init__.py

@@ -1,29 +1,10 @@
-from importlib.util import find_spec
+from dataeval import _IS_TENSORFLOW_AVAILABLE
 
-from dataeval._internal.detectors.clusterer import Clusterer
-from dataeval._internal.detectors.drift.base import LastSeenUpdate, ReservoirSamplingUpdate
-from dataeval._internal.detectors.drift.cvm import DriftCVM
-from dataeval._internal.detectors.drift.ks import DriftKS
-from dataeval._internal.detectors.duplicates import Duplicates
-from dataeval._internal.detectors.linter import Linter
+from . import drift, linters
 
-__all__ = ["Clusterer", "Duplicates", "Linter", "DriftCVM", "DriftKS", "LastSeenUpdate", "ReservoirSamplingUpdate"]
+__all__ = ["drift", "linters"]
 
-if find_spec("torch") is not None:  # pragma: no cover
-    from dataeval._internal.detectors.drift.mmd import DriftMMD
-    from dataeval._internal.detectors.drift.torch import GaussianRBF, preprocess_drift
-    from dataeval._internal.detectors.drift.uncertainty import DriftUncertainty
+if _IS_TENSORFLOW_AVAILABLE:  # pragma: no cover
+    from . import ood
 
-    __all__ += ["DriftMMD", "GaussianRBF", "DriftUncertainty", "preprocess_drift"]
-
-if find_spec("tensorflow") is not None and find_spec("tensorflow_probability") is not None:  # pragma: no cover
-    from dataeval._internal.detectors.ood.ae import OOD_AE
-    from dataeval._internal.detectors.ood.aegmm import OOD_AEGMM
-    from dataeval._internal.detectors.ood.base import OODScore
-    from dataeval._internal.detectors.ood.llr import OOD_LLR
-    from dataeval._internal.detectors.ood.vae import OOD_VAE
-    from dataeval._internal.detectors.ood.vaegmm import OOD_VAEGMM
-
-    __all__ += ["OOD_AE", "OOD_AEGMM", "OOD_LLR", "OODScore", "OOD_VAE", "OOD_VAEGMM"]
-
-del find_spec
+    __all__ += ["ood"]

dataeval/detectors/drift/__init__.py

@@ -0,0 +1,16 @@
+from dataeval import _IS_TORCH_AVAILABLE
+from dataeval._internal.detectors.drift.cvm import DriftCVM
+from dataeval._internal.detectors.drift.ks import DriftKS
+
+from . import updates
+
+__all__ = ["DriftCVM", "DriftKS", "updates"]
+
+if _IS_TORCH_AVAILABLE:  # pragma: no cover
+    from dataeval._internal.detectors.drift.mmd import DriftMMD
+    from dataeval._internal.detectors.drift.torch import preprocess_drift
+    from dataeval._internal.detectors.drift.uncertainty import DriftUncertainty
+
+    from . import kernels
+
+    __all__ += ["DriftMMD", "DriftUncertainty", "kernels", "preprocess_drift"]

dataeval/detectors/drift/kernels/__init__.py

@@ -0,0 +1,6 @@
+from dataeval import _IS_TORCH_AVAILABLE
+
+if _IS_TORCH_AVAILABLE:  # pragma: no cover
+    from dataeval._internal.detectors.drift.torch import GaussianRBF
+
+    __all__ = ["GaussianRBF"]

dataeval/detectors/drift/updates/__init__.py

@@ -0,0 +1,3 @@
+from dataeval._internal.detectors.drift.base import LastSeenUpdate, ReservoirSamplingUpdate
+
+__all__ = ["LastSeenUpdate", "ReservoirSamplingUpdate"]

dataeval/detectors/linters/__init__.py

@@ -0,0 +1,5 @@
+from dataeval._internal.detectors.clusterer import Clusterer
+from dataeval._internal.detectors.duplicates import Duplicates
+from dataeval._internal.detectors.outliers import Outliers
+
+__all__ = ["Clusterer", "Duplicates", "Outliers"]

dataeval/detectors/ood/__init__.py

@@ -0,0 +1,11 @@
+from dataeval import _IS_TENSORFLOW_AVAILABLE
+
+if _IS_TENSORFLOW_AVAILABLE:  # pragma: no cover
+    from dataeval._internal.detectors.ood.ae import OOD_AE
+    from dataeval._internal.detectors.ood.aegmm import OOD_AEGMM
+    from dataeval._internal.detectors.ood.base import OODOutput, OODScore
+    from dataeval._internal.detectors.ood.llr import OOD_LLR
+    from dataeval._internal.detectors.ood.vae import OOD_VAE
+    from dataeval._internal.detectors.ood.vaegmm import OOD_VAEGMM
+
+    __all__ = ["OODOutput", "OODScore", "OOD_AE", "OOD_AEGMM", "OOD_LLR", "OOD_VAE", "OOD_VAEGMM"]

dataeval/flags/__init__.py

@@ -1,3 +1,3 @@
-from dataeval._internal.flags import ImageHash, ImageProperty, ImageStatistics, ImageVisuals
+from dataeval._internal.flags import ImageStat
 
-__all__ = ["ImageHash", "ImageProperty", "ImageStatistics", "ImageVisuals"]
+__all__ = ["ImageStat"]

dataeval/metrics/__init__.py

@@ -1,27 +1,3 @@
-from typing import List
+from . import bias, estimators, stats
 
-__all__: List[str] = []
-
-from dataeval._internal.metrics.balance import balance, balance_classwise
-from dataeval._internal.metrics.ber import ber
-from dataeval._internal.metrics.coverage import coverage
-from dataeval._internal.metrics.divergence import divergence
-from dataeval._internal.metrics.diversity import diversity, diversity_classwise
-from dataeval._internal.metrics.parity import parity, parity_metadata
-from dataeval._internal.metrics.stats import ChannelStats, ImageStats
-from dataeval._internal.metrics.uap import uap
-
-__all__ += [
-    "balance",
-    "balance_classwise",
-    "ber",
-    "coverage",
-    "divergence",
-    "diversity",
-    "diversity_classwise",
-    "parity",
-    "parity_metadata",
-    "ChannelStats",
-    "ImageStats",
-    "uap",
-]
+__all__ = ["bias", "estimators", "stats"]