dataeval 0.63.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

dataeval/__init__.py

@@ -2,14 +2,14 @@ from importlib.util import find_spec
 
 from . import detectors, flags, metrics
 
-__version__ = "0.63.0"
+__version__ = "0.64.0"
 
 __all__ = ["detectors", "flags", "metrics"]
 
 if find_spec("torch") is not None:  # pragma: no cover
-    from . import models, workflows
+    from . import models, utils, workflows
 
-    __all__ += ["models", "workflows"]
+    __all__ += ["models", "utils", "workflows"]
 elif find_spec("tensorflow") is not None:  # pragma: no cover
     from . import models
 

dataeval/_internal/detectors/clusterer.py

@@ -1,10 +1,11 @@
 from typing import Dict, Iterable, List, NamedTuple, Tuple, Union, cast
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.cluster.hierarchy import linkage
 from scipy.spatial.distance import pdist, squareform
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 
 
 def extend_linkage(link_arr: np.ndarray) -> np.ndarray:

dataeval/_internal/detectors/drift/base.py

@@ -11,8 +11,9 @@ from functools import wraps
 from typing import Callable, Dict, Literal, Optional, Tuple, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 
 
 def update_x_ref(fn):

dataeval/_internal/detectors/drift/cvm.py

@@ -9,9 +9,10 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable, Literal, Optional, Tuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.stats import cramervonmises_2samp
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 
 from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
 

dataeval/_internal/detectors/drift/ks.py

@@ -9,9 +9,10 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable, Literal, Optional, Tuple
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.stats import ks_2samp
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 
 from .base import BaseUnivariateDrift, UpdateStrategy, preprocess_x
 

dataeval/_internal/detectors/drift/mmd.py

@@ -9,8 +9,9 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable, Dict, Optional, Tuple, Union
 
 import torch
+from numpy.typing import ArrayLike
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 
 from .base import BaseDrift, UpdateStrategy, preprocess_x, update_x_ref
 from .torch import GaussianRBF, get_device, mmd2_from_kernel_matrix
@@ -74,7 +75,7 @@ class DriftMMD(BaseDrift):
         super().__init__(x_ref, p_val, x_ref_preprocessed, update_x_ref, preprocess_fn)
 
         self.infer_sigma = configure_kernel_from_x_ref
-        if configure_kernel_from_x_ref and isinstance(sigma, ArrayLike):
+        if configure_kernel_from_x_ref and sigma is not None:
             self.infer_sigma = False
 
         self.n_permutations = n_permutations  # nb of iterations through permutation test
@@ -83,7 +84,7 @@ class DriftMMD(BaseDrift):
         self.device = get_device(device)
 
         # initialize kernel
-        sigma_tensor = torch.from_numpy(to_numpy(sigma)).to(self.device) if isinstance(sigma, ArrayLike) else None
+        sigma_tensor = torch.from_numpy(to_numpy(sigma)).to(self.device) if sigma is not None else None
         self.kernel = kernel(sigma_tensor).to(self.device) if kernel == GaussianRBF else kernel
 
         # compute kernel matrix for the reference data

dataeval/_internal/detectors/drift/uncertainty.py

@@ -10,11 +10,10 @@ from functools import partial
 from typing import Callable, Dict, Literal, Optional, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
 from scipy.special import softmax
 from scipy.stats import entropy
 
-from dataeval._internal.interop import ArrayLike
-
 from .base import UpdateStrategy
 from .ks import DriftKS
 from .torch import get_device, preprocess_drift

dataeval/_internal/detectors/duplicates.py

@@ -1,7 +1,8 @@
 from typing import Dict, Iterable, List, Literal
 
+from numpy.typing import ArrayLike
+
 from dataeval._internal.flags import ImageHash
-from dataeval._internal.interop import ArrayLike
 from dataeval._internal.metrics.stats import ImageStats
 
 

dataeval/_internal/detectors/linter.py

@@ -1,9 +1,9 @@
 from typing import Iterable, Literal, Optional, Sequence, Union
 
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.flags import ImageProperty, ImageVisuals, LinterFlags
-from dataeval._internal.interop import ArrayLike
 from dataeval._internal.metrics.stats import ImageStats
 
 

dataeval/_internal/detectors/ood/ae.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AE
 from dataeval._internal.models.tensorflow.utils import predict_batch
 

dataeval/_internal/detectors/ood/aegmm.py

@@ -9,9 +9,10 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable
 
 import keras
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import LossGMM

dataeval/_internal/detectors/ood/base.py

@@ -12,8 +12,9 @@ from typing import Callable, Dict, List, Literal, NamedTuple, Optional, Tuple, c
 import keras
 import numpy as np
 import tensorflow as tf
+from numpy.typing import ArrayLike
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
 from dataeval._internal.models.tensorflow.trainer import trainer
 

dataeval/_internal/detectors/ood/llr.py

@@ -14,9 +14,10 @@ import numpy as np
 import tensorflow as tf
 from keras.layers import Input
 from keras.models import Model
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
 from dataeval._internal.models.tensorflow.trainer import trainer
 from dataeval._internal.models.tensorflow.utils import predict_batch
@@ -180,7 +181,7 @@ class OOD_LLR(OODBase):
 
         # create background data
         mutate_fn = partial(mutate_fn, **mutate_fn_kwargs)
-        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)
+        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)  # type: ignore
 
         # prepare sequential data
         if self.sequential and not self.has_log_prob:

dataeval/_internal/detectors/ood/vae.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAE
 from dataeval._internal.models.tensorflow.losses import Elbo
 from dataeval._internal.models.tensorflow.utils import predict_batch

dataeval/_internal/detectors/ood/vaegmm.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM

dataeval/_internal/interop.py

@@ -1,7 +1,8 @@
 from importlib import import_module
-from typing import Any, Iterable, Optional, runtime_checkable
+from typing import Iterable, Optional
 
 import numpy as np
+from numpy.typing import ArrayLike
 
 module_cache = {}
 
@@ -19,16 +20,6 @@ def try_import(module_name):
     return module
 
 
-try:
-    from maite.protocols import ArrayLike  # type: ignore
-except ImportError:  # pragma: no cover - covered by test_mindeps.py
-    from typing import Protocol
-
-    @runtime_checkable
-    class ArrayLike(Protocol):
-        def __array__(self) -> Any: ...
-
-
 def to_numpy(array: Optional[ArrayLike]) -> np.ndarray:
     if array is None:
         return np.ndarray([])

dataeval/_internal/metrics/balance.py

@@ -0,0 +1,180 @@
+import warnings
+from typing import Dict, List, NamedTuple, Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
+
+from dataeval._internal.metrics.utils import entropy, preprocess_metadata
+
+
+class BalanceOutput(NamedTuple):
+    """
+    Attributes
+    ----------
+    mutual_information : NDArray[np.float64]
+        Estimate of mutual information between metadata factors and class label
+    """
+
+    mutual_information: NDArray[np.float64]
+
+
+def validate_num_neighbors(num_neighbors: int) -> int:
+    if not isinstance(num_neighbors, (int, float)):
+        raise TypeError(
+            f"Variable {num_neighbors} is not real-valued numeric type."
+            "num_neighbors should be an int, greater than 0 and less than"
+            "the number of samples in the dataset"
+        )
+    if num_neighbors < 1:
+        raise ValueError(
+            f"Invalid value for {num_neighbors}."
+            "Choose a value greater than 0 and less than number of samples"
+            "in the dataset."
+        )
+    if isinstance(num_neighbors, float):
+        num_neighbors = int(num_neighbors)
+        warnings.warn(f"Variable {num_neighbors} is currently type float and will be truncated to type int.")
+
+    return num_neighbors
+
+
+def balance(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+    """
+    Mutual information (MI) between factors (class label, metadata, label/image properties)
+
+    Parameters
+    ----------
+    class_labels: Sequence[int]
+        List of class labels for each image
+    metadata: List[Dict]
+        List of metadata factors for each image
+    num_neighbors: int, default 5
+        Number of nearest neighbors to use for computing MI between discrete
+        and continuous variables.
+
+    Returns
+    -------
+    BalanceOutput
+        (num_factors+1) x (num_factors+1) estimate of mutual information
+        between num_factors metadata factors and class label. Symmetry is enforced.
+
+    Notes
+    -----
+    We use `mutual_info_classif` from sklearn since class label is categorical.
+    `mutual_info_classif` outputs are consistent up to O(1e-4) and depend on a random
+    seed. MI is computed differently for categorical and continuous variables, and
+    we attempt to infer whether a variable is categorical by the fraction of unique
+    values in the dataset.
+
+    See Also
+    --------
+    sklearn.feature_selection.mutual_info_classif
+    sklearn.feature_selection.mutual_info_regression
+    sklearn.metrics.mutual_info_score
+    """
+    num_neighbors = validate_num_neighbors(num_neighbors)
+    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    num_factors = len(names)
+    mi = np.empty((num_factors, num_factors))
+    mi[:] = np.nan
+
+    for idx in range(num_factors):
+        tgt = data[:, idx]
+
+        if is_categorical[idx]:
+            # categorical target
+            mi[idx, :] = mutual_info_classif(
+                data,
+                tgt,
+                discrete_features=is_categorical,  # type: ignore
+                n_neighbors=num_neighbors,
+            )
+        else:
+            # continuous variables
+            mi[idx, :] = mutual_info_regression(
+                data,
+                tgt,
+                discrete_features=is_categorical,  # type: ignore
+                n_neighbors=num_neighbors,
+            )
+
+    ent_all = entropy(data, names, is_categorical, normalized=False)
+    norm_factor = 0.5 * np.add.outer(ent_all, ent_all) + 1e-6
+    # in principle MI should be symmetric, but it is not in practice.
+    nmi = 0.5 * (mi + mi.T) / norm_factor
+
+    return BalanceOutput(nmi)
+
+
+def balance_classwise(class_labels: Sequence[int], metadata: List[Dict], num_neighbors: int = 5) -> BalanceOutput:
+    """
+    Compute mutual information (analogous to correlation) between metadata factors
+    (class label, metadata, label/image properties) with individual class labels.
+
+    Parameters
+    ----------
+    class_labels: Sequence[int]
+        List of class labels for each image
+    metadata: List[Dict]
+        List of metadata factors for each image
+    num_neighbors: int, default 5
+        Number of nearest neighbors to use for computing MI between discrete
+        and continuous variables.
+
+    Notes
+    -----
+    We use `mutual_info_classif` from sklearn since class label is categorical.
+    `mutual_info_classif` outputs are consistent up to O(1e-4) and depend on a random
+    seed. MI is computed differently for categorical and continuous variables, so we
+    have to specify with is_categorical.
+
+    Returns
+    -------
+    BalanceOutput
+        (num_classes x num_factors) estimate of mutual information between
+        num_factors metadata factors and individual class labels.
+
+    See Also
+    --------
+    sklearn.feature_selection.mutual_info_classif
+    sklearn.feature_selection.mutual_info_regression
+    sklearn.metrics.mutual_info_score
+    compute_mutual_information
+    """
+    num_neighbors = validate_num_neighbors(num_neighbors)
+    data, names, is_categorical = preprocess_metadata(class_labels, metadata)
+    num_factors = len(names)
+    # unique class labels
+    class_idx = names.index("class_label")
+    class_data = data[:, class_idx]
+    u_cls = np.unique(class_data)
+    num_classes = len(u_cls)
+
+    data_no_class = np.concatenate((data[:, :class_idx], data[:, (class_idx + 1) :]), axis=1)
+
+    # assume class is a factor
+    mi = np.empty((num_classes, num_factors - 1))
+    mi[:] = np.nan
+
+    # categorical variables, excluding class label
+    cat_mask = np.concatenate((is_categorical[:class_idx], is_categorical[(class_idx + 1) :]), axis=0).astype(int)
+
+    # classification MI for discrete/categorical features
+    for idx, cls in enumerate(u_cls):
+        tgt = class_data == cls
+        # units: nat
+        mi[idx, :] = mutual_info_classif(
+            data_no_class,
+            tgt,
+            discrete_features=cat_mask,  # type: ignore
+            n_neighbors=num_neighbors,
+        )
+
+    # let this recompute for all features including class label
+    ent_all = entropy(data, names, is_categorical)
+    ent_tgt = ent_all[class_idx]
+    ent_all = np.concatenate((ent_all[:class_idx], ent_all[(class_idx + 1) :]), axis=0)
+    norm_factor = 0.5 * np.add.outer(ent_tgt, ent_all) + 1e-6
+    nmi = mi / norm_factor
+    return BalanceOutput(nmi)

dataeval/_internal/metrics/base.py

@@ -1,92 +1,10 @@
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, Generic, List, TypeVar
+from typing import Generic, TypeVar
 
 TOutput = TypeVar("TOutput", bound=dict)
-TMethods = TypeVar("TMethods")
-TCallable = TypeVar("TCallable", bound=Callable)
-
-
-class MetricMixin(ABC, Generic[TOutput]):
-    @abstractmethod
-    def update(self, *args, **kwargs): ...
-
-    @abstractmethod
-    def compute(self) -> TOutput: ...
-
-    @abstractmethod
-    def reset(self): ...
 
 
 class EvaluateMixin(ABC, Generic[TOutput]):
     @abstractmethod
     def evaluate(self, *args, **kwargs) -> TOutput:
         """Abstract method to calculate metric based off of constructor parameters"""
-
-
-class MethodsMixin(ABC, Generic[TMethods, TCallable]):
-    """
-    Use this mixin to define a mapping of functions to method names which
-    can be queried by the user and called internally with the appropriate
-    method name as the key.
-
-    Explicitly defining the Callable generic helps with type safety and
-    hinting for function signatures and recommended but optional.
-
-    e.g.:
-
-    def _mult(x: float, y: float) -> float:
-        return x * y
-
-    class MyMetric(MethodsMixin[Callable[float, float], float]):
-
-        def _methods(cls) -> Dict[str, Callable[float, float], float]:
-            return {
-                "ADD": lambda x, y: x + y,
-                "MULT":  _mult,
-                ...
-            }
-
-    Then during evaluate, you can call the method specified with the getter.
-
-    e.g.:
-
-        def evaluate(self):
-            return self._method(x, y)
-
-    The resulting class can be used like so.
-
-    m = MyMetric(1.0, 2.0, "ADD")
-    m.evaluate()       #  returns 3.0
-    m.method           #  returns "ADD"
-    MyMetric.methods() #  returns "['ADD', 'MULT']
-    m.method = "MULT"
-    m.evaluate()       #  returns 2.0
-    """
-
-    @classmethod
-    @abstractmethod
-    def _methods(cls) -> Dict[str, TCallable]:
-        """Abstract method returning available method functions for class"""
-
-    @property
-    def _method(self) -> TCallable:
-        return self._methods()[self.method]
-
-    @classmethod
-    def methods(cls) -> List[str]:
-        return list(cls._methods().keys())
-
-    @property
-    def method(self) -> str:
-        return self._method_key
-
-    @method.setter
-    def method(self, value: TMethods):
-        self._set_method(value)
-
-    def _set_method(self, value: TMethods):
-        """This setter is to fix pyright incorrect detection of
-        incorrectly overriding the 'method' property"""
-        if value not in self.methods():
-            raise KeyError(f"Specified method not available for class ({self.methods()}).")
-        self._method_key = value