dataeval 0.63.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

dataeval/_internal/detectors/duplicates.py

@@ -1,8 +1,27 @@
-from typing import Dict, Iterable, List, Literal
+from dataclasses import dataclass
+from typing import Dict, Iterable, List
 
-from dataeval._internal.flags import ImageHash
-from dataeval._internal.interop import ArrayLike
-from dataeval._internal.metrics.stats import ImageStats
+from numpy.typing import ArrayLike
+
+from dataeval._internal.metrics.stats import StatsOutput
+from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.flags import ImageStat
+from dataeval.metrics import imagestats
+
+
+@dataclass(frozen=True)
+class DuplicatesOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    exact : List[List[int]]
+        Indices of images that are exact matches
+    near: List[List[int]]
+        Indices of images that are near matches
+    """
+
+    exact: List[List[int]]
+    near: List[List[int]]
 
 
 class Duplicates:
@@ -12,8 +31,8 @@ class Duplicates:
 
     Attributes
     ----------
-    stats : ImageStats(flags=ImageHash.ALL)
-        Base stats class with the flags for checking duplicates
+    stats : StatsOutput
+        Output class of stats
 
     Example
     -------
@@ -22,25 +41,36 @@ class Duplicates:
     >>> dups = Duplicates()
     """
 
-    def __init__(self):
-        self.stats = ImageStats(ImageHash.ALL)
+    def __init__(self, find_exact: bool = True, find_near: bool = True):
+        self.stats: StatsOutput
+        self.find_exact = find_exact
+        self.find_near = find_near
 
-    def _get_duplicates(self) -> dict:
-        exact = {}
-        near = {}
-        for i, value in enumerate(self.results["xxhash"]):
-            exact.setdefault(value, []).append(i)
-        for i, value in enumerate(self.results["pchash"]):
-            near.setdefault(value, []).append(i)
-        exact = [v for v in exact.values() if len(v) > 1]
-        near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+    def _get_duplicates(self) -> Dict[str, List[List[int]]]:
+        stats_dict = self.stats.dict()
+        if "xxhash" in stats_dict:
+            exact = {}
+            for i, value in enumerate(stats_dict["xxhash"]):
+                exact.setdefault(value, []).append(i)
+            exact = [v for v in exact.values() if len(v) > 1]
+        else:
+            exact = []
+
+        if "pchash" in stats_dict:
+            near = {}
+            for i, value in enumerate(stats_dict["pchash"]):
+                near.setdefault(value, []).append(i)
+            near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+        else:
+            near = []
 
         return {
             "exact": sorted(exact),
             "near": sorted(near),
         }
 
-    def evaluate(self, images: Iterable[ArrayLike]) -> Dict[Literal["exact", "near"], List[int]]:
+    @set_metadata("dataeval.detectors", ["find_exact", "find_near"])
+    def evaluate(self, images: Iterable[ArrayLike]) -> DuplicatesOutput:
         """
         Returns duplicate image indices for both exact matches and near matches
 
@@ -51,22 +81,19 @@ class Duplicates:
 
         Returns
         -------
-        Dict[str, List[int]]
-            exact :
-                List of groups of indices that are exact matches
-            near :
-                List of groups of indices that are near matches
+        DuplicatesOutput
+            List of groups of indices that are exact and near matches
 
         See Also
         --------
-        ImageStats
+        imagestats
 
         Example
         -------
         >>> dups.evaluate(images)
-        {'exact': [[3, 20], [16, 37]], 'near': [[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]]}
-        """
-        self.stats.reset()
-        self.stats.update(images)
-        self.results = self.stats.compute()
-        return self._get_duplicates()
+        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
+        """  # noqa: E501
+        flag_exact = ImageStat.XXHASH if self.find_exact else ImageStat(0)
+        flag_near = ImageStat.PCHASH if self.find_near else ImageStat(0)
+        self.stats = imagestats(images, flag_exact | flag_near)
+        return DuplicatesOutput(**self._get_duplicates())

dataeval/_internal/detectors/linter.py

@@ -1,15 +1,31 @@
-from typing import Iterable, Literal, Optional, Sequence, Union
+from dataclasses import dataclass
+from typing import Dict, Iterable, Literal, Optional
 
 import numpy as np
+from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.flags import ImageProperty, ImageVisuals, LinterFlags
-from dataeval._internal.interop import ArrayLike
-from dataeval._internal.metrics.stats import ImageStats
+from dataeval._internal.flags import verify_supported
+from dataeval._internal.output import OutputMetadata, set_metadata
+from dataeval.flags import ImageStat
+from dataeval.metrics import imagestats
+
+
+@dataclass(frozen=True)
+class LinterOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    issues : Dict[int, Dict[str, float]]
+        Dictionary containing the indices of outliers and a dictionary showing
+        the issues and calculated values for the given index.
+    """
+
+    issues: Dict[int, Dict[str, float]]
 
 
 def _get_outlier_mask(
-    values: np.ndarray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
-) -> np.ndarray:
+    values: NDArray, method: Literal["zscore", "modzscore", "iqr"], threshold: Optional[float]
+) -> NDArray:
     if method == "zscore":
         threshold = threshold if threshold else 3.0
         std = np.std(values)
@@ -18,7 +34,7 @@ def _get_outlier_mask(
     elif method == "modzscore":
         threshold = threshold if threshold else 3.5
         abs_diff = np.abs(values - np.median(values))
-        med_abs_diff = np.median(abs_diff)
+        med_abs_diff = np.median(abs_diff) if np.median(abs_diff) != 0 else np.mean(abs_diff)
         mod_z_score = 0.6745 * abs_diff / med_abs_diff
         return mod_z_score > threshold
     elif method == "iqr":
@@ -36,8 +52,9 @@ class Linter:
 
     Parameters
     ----------
-    flags : [ImageProperty | ImageStatistics | ImageVisuals], default None
+    flags : ImageStat, default ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS
         Metric(s) to calculate for each image - calculates all metrics if None
+        Only supports ImageStat.ALL_STATS
     outlier_method : ["modzscore" | "zscore" | "iqr"], optional - default "modzscore"
         Statistical method used to identify outliers
     outlier_threshold : float, optional - default None
@@ -46,8 +63,8 @@ class Linter:
 
     Attributes
     ----------
-    stats : ImageStats
-        Class to hold the value of each metric for each image
+    stats : Dict[str, Any]
+        Dictionary to hold the value of each metric for each image
 
     See Also
     --------
@@ -81,7 +98,7 @@ class Linter:
 
     Specifying specific metrics to analyze:
 
-    >>> lint = Linter(flags=[ImageProperty.SIZE, ImageVisuals.ALL])
+    >>> lint = Linter(flags=ImageStat.SIZE | ImageStat.ALL_VISUALS)
 
     Specifying an outlier method:
 
@@ -94,19 +111,19 @@ class Linter:
 
     def __init__(
         self,
-        flags: Optional[Union[LinterFlags, Sequence[LinterFlags]]] = None,
+        flags: ImageStat = ImageStat.ALL_PROPERTIES | ImageStat.ALL_VISUALS,
         outlier_method: Literal["zscore", "modzscore", "iqr"] = "modzscore",
         outlier_threshold: Optional[float] = None,
     ):
-        flags = flags if flags is not None else (ImageProperty.ALL, ImageVisuals.ALL)
-        self.stats = ImageStats(flags)
+        verify_supported(flags, ImageStat.ALL_STATS)
+        self.flags = flags
         self.outlier_method: Literal["zscore", "modzscore", "iqr"] = outlier_method
         self.outlier_threshold = outlier_threshold
 
     def _get_outliers(self) -> dict:
         flagged_images = {}
-
-        for stat, values in self.results.items():
+        stats_dict = self.stats.dict()
+        for stat, values in stats_dict.items():
             if not isinstance(values, np.ndarray):
                 continue
 
@@ -118,7 +135,8 @@ class Linter:
 
         return dict(sorted(flagged_images.items()))
 
-    def evaluate(self, images: Iterable[ArrayLike]) -> dict:
+    @set_metadata("dataeval.detectors", ["flags", "outlier_method", "outlier_threshold"])
+    def evaluate(self, images: Iterable[ArrayLike]) -> LinterOutput:
         """
         Returns indices of outliers with the issues identified for each
 
@@ -130,8 +148,8 @@ class Linter:
 
         Returns
         -------
-        Dict[int, Dict[str, float]]
-            Dictionary containing the indices of outliers and a dictionary showing
+        LinterOutput
+            Output class containing the indices of outliers and a dictionary showing
             the issues and calculated values for the given index.
 
         Example
@@ -139,9 +157,7 @@ class Linter:
         Evaluate the dataset:
 
         >>> lint.evaluate(images)
-        {18: {'brightness': 0.78}, 25: {'brightness': 0.98}}
+        LinterOutput(issues={18: {'brightness': 0.78}, 25: {'brightness': 0.98}})
         """
-        self.stats.reset()
-        self.stats.update(images)
-        self.results = self.stats.compute()
-        return self._get_outliers()
+        self.stats = imagestats(images, self.flags)
+        return LinterOutput(self._get_outliers())

dataeval/_internal/detectors/ood/ae.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AE
 from dataeval._internal.models.tensorflow.utils import predict_batch
 

dataeval/_internal/detectors/ood/aegmm.py

@@ -9,9 +9,10 @@ Licensed under Apache Software License (Apache 2.0)
 from typing import Callable
 
 import keras
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import AEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import LossGMM

dataeval/_internal/detectors/ood/base.py

@@ -7,15 +7,36 @@ Licensed under Apache Software License (Apache 2.0)
 """
 
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, List, Literal, NamedTuple, Optional, Tuple, cast
+from dataclasses import dataclass
+from typing import Callable, List, Literal, NamedTuple, Optional, Tuple, cast
 
 import keras
 import numpy as np
 import tensorflow as tf
+from numpy.typing import ArrayLike, NDArray
 
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
 from dataeval._internal.models.tensorflow.trainer import trainer
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+
+@dataclass(frozen=True)
+class OODOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    is_ood : NDArray[np.bool_]
+        Array of images that are detected as out of distribution
+    instance_score : NDArray[np.float32]
+        Instance score of the evaluated dataset
+    feature_score : Optional[NDArray[np.float32]]
+        Feature score, if available, of the evaluated dataset
+    """
+
+    is_ood: NDArray[np.bool_]
+    instance_score: NDArray[np.float32]
+    feature_score: Optional[NDArray[np.float32]]
 
 
 class OODScore(NamedTuple):
@@ -24,16 +45,16 @@ class OODScore(NamedTuple):
 
     Parameters
     ----------
-    instance_score : np.ndarray
+    instance_score : NDArray[np.float32]
         Instance score of the evaluated dataset.
-    feature_score : Optional[np.ndarray], default None
+    feature_score : Optional[NDArray[np.float32]], default None
         Feature score, if available, of the evaluated dataset.
     """
 
-    instance_score: np.ndarray
-    feature_score: Optional[np.ndarray] = None
+    instance_score: NDArray[np.float32]
+    feature_score: Optional[NDArray[np.float32]] = None
 
-    def get(self, ood_type: Literal["instance", "feature"]) -> np.ndarray:
+    def get(self, ood_type: Literal["instance", "feature"]) -> NDArray:
         return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
 
 
@@ -48,18 +69,18 @@ class OODBase(ABC):
         if not isinstance(model, keras.Model):
             raise TypeError("Model should be of type 'keras.Model'.")
 
-    def _get_data_info(self, X: np.ndarray) -> Tuple[tuple, type]:
+    def _get_data_info(self, X: NDArray) -> Tuple[tuple, type]:
         if not isinstance(X, np.ndarray):
-            raise TypeError("Dataset should of type: `np.ndarray`.")
+            raise TypeError("Dataset should of type: `NDArray`.")
         return X.shape[1:], X.dtype.type
 
-    def _validate(self, X: np.ndarray) -> None:
+    def _validate(self, X: NDArray) -> None:
         check_data_info = self._get_data_info(X)
         if self._data_info is not None and check_data_info != self._data_info:
             raise RuntimeError(f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
                                Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}.")
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: Optional[List[str]] = None) -> None:
         attrs = ["_data_info", "_threshold_perc", "_ref_score"]
         attrs = attrs if additional_attrs is None else attrs + additional_attrs
         if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
@@ -131,12 +152,13 @@ class OODBase(ABC):
         self._ref_score = self.score(x_ref, batch_size)
         self._threshold_perc = threshold_perc
 
+    @set_metadata("dataeval.detectors")
     def predict(
         self,
         X: ArrayLike,
         batch_size: int = int(1e10),
         ood_type: Literal["feature", "instance"] = "instance",
-    ) -> Dict[str, np.ndarray]:
+    ) -> OODOutput:
         """
         Predict whether instances are out-of-distribution or not.
 
@@ -156,8 +178,8 @@ class OODBase(ABC):
         self._validate_state(X := to_numpy(X))
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
-        ood_pred = (score.get(ood_type) > self._threshold_score(ood_type)).astype(int)
-        return {**{"is_ood": ood_pred}, **score._asdict()}
+        ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
+        return OODOutput(is_ood=ood_pred, **score._asdict())
 
 
 class OODGMMBase(OODBase):
@@ -165,7 +187,7 @@ class OODGMMBase(OODBase):
         super().__init__(model)
         self.gmm_params: GaussianMixtureModelParams
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: Optional[List[str]] = None) -> None:
         if additional_attrs is None:
             additional_attrs = ["gmm_params"]
         super()._validate_state(X, additional_attrs)

dataeval/_internal/detectors/ood/llr.py

@@ -14,9 +14,10 @@ import numpy as np
 import tensorflow as tf
 from keras.layers import Input
 from keras.models import Model
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
 from dataeval._internal.models.tensorflow.trainer import trainer
 from dataeval._internal.models.tensorflow.utils import predict_batch
@@ -51,7 +52,7 @@ def build_model(
 
 
 def mutate_categorical(
-    X: np.ndarray,
+    X: NDArray,
     rate: float,
     seed: int = 0,
     feature_range: tuple = (0, 255),
@@ -180,7 +181,7 @@ class OOD_LLR(OODBase):
 
         # create background data
         mutate_fn = partial(mutate_fn, **mutate_fn_kwargs)
-        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)
+        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)  # type: ignore
 
         # prepare sequential data
         if self.sequential and not self.has_log_prob:
@@ -220,10 +221,10 @@ class OOD_LLR(OODBase):
     def _logp(
         self,
         dist,
-        X: np.ndarray,
+        X: NDArray,
         return_per_feature: bool = False,
         batch_size: int = int(1e10),
-    ) -> np.ndarray:
+    ) -> NDArray:
         """
         Compute log probability of a batch of instances under the generative model.
         """
@@ -234,10 +235,10 @@ class OOD_LLR(OODBase):
     def _logp_alt(
         self,
         model: keras.Model,
-        X: np.ndarray,
+        X: NDArray,
         return_per_feature: bool = False,
         batch_size: int = int(1e10),
-    ) -> np.ndarray:
+    ) -> NDArray:
         """
         Compute log probability of a batch of instances with the user defined log_prob function.
         """
@@ -253,7 +254,7 @@ class OOD_LLR(OODBase):
             axis = tuple(np.arange(len(logp.shape))[1:])
             return np.mean(logp, axis=axis)
 
-    def _llr(self, X: np.ndarray, return_per_feature: bool, batch_size: int = int(1e10)) -> np.ndarray:
+    def _llr(self, X: NDArray, return_per_feature: bool, batch_size: int = int(1e10)) -> NDArray:
         """
         Compute likelihood ratios.
 

dataeval/_internal/detectors/ood/vae.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAE
 from dataeval._internal.models.tensorflow.losses import Elbo
 from dataeval._internal.models.tensorflow.utils import predict_batch

dataeval/_internal/detectors/ood/vaegmm.py

@@ -10,9 +10,10 @@ from typing import Callable
 
 import keras
 import numpy as np
+from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
-from dataeval._internal.interop import ArrayLike, to_numpy
+from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.autoencoder import VAEGMM
 from dataeval._internal.models.tensorflow.gmm import gmm_energy
 from dataeval._internal.models.tensorflow.losses import Elbo, LossGMM

dataeval/_internal/flags.py

@@ -1,37 +1,31 @@
-from enum import Flag, auto
-from typing import Union
+from enum import IntFlag, auto
+from functools import reduce
+from typing import Dict, Iterable, TypeVar, Union, cast
 
+TFlag = TypeVar("TFlag", bound=IntFlag)
 
-class auto_all:
-    def __get__(self, _, cls):
-        return ~cls(0)
 
+class ImageStat(IntFlag):
+    """
+    Flags for calculating image and channel statistics
+    """
 
-class ImageHash(Flag):
+    # HASHES
     XXHASH = auto()
     PCHASH = auto()
-    ALL = auto_all()
-
-
-class ImageProperty(Flag):
+    # PROPERTIES
     WIDTH = auto()
     HEIGHT = auto()
     SIZE = auto()
     ASPECT_RATIO = auto()
     CHANNELS = auto()
     DEPTH = auto()
-    ALL = auto_all()
-
-
-class ImageVisuals(Flag):
+    # VISUALS
     BRIGHTNESS = auto()
     BLURRINESS = auto()
     MISSING = auto()
     ZERO = auto()
-    ALL = auto_all()
-
-
-class ImageStatistics(Flag):
+    # PIXEL STATS
     MEAN = auto()
     STD = auto()
     VAR = auto()
@@ -40,8 +34,35 @@ class ImageStatistics(Flag):
     ENTROPY = auto()
     PERCENTILES = auto()
     HISTOGRAM = auto()
-    ALL = auto_all()
+    # JOINT FLAGS
+    ALL_HASHES = XXHASH | PCHASH
+    ALL_PROPERTIES = WIDTH | HEIGHT | SIZE | ASPECT_RATIO | CHANNELS | DEPTH
+    ALL_VISUALS = BRIGHTNESS | BLURRINESS | MISSING | ZERO
+    ALL_PIXELSTATS = MEAN | STD | VAR | SKEW | KURTOSIS | ENTROPY | PERCENTILES | HISTOGRAM
+    ALL_STATS = ALL_PROPERTIES | ALL_VISUALS | ALL_PIXELSTATS
+    ALL = ALL_HASHES | ALL_STATS
+
+
+def is_distinct(flag: IntFlag) -> bool:
+    return (flag & (flag - 1) == 0) and flag != 0
+
+
+def to_distinct(flag: TFlag) -> Dict[TFlag, str]:
+    """
+    Returns a distinct set of all flags set on the input flag and their names
+
+    NOTE: this is supported natively in Python 3.11, but for earlier versions we need
+    to use a combination of list comprehension and bit fiddling to determine distinct
+    flag values from joint aliases.
+    """
+    if isinstance(flag, Iterable):  # >= py311
+        return {f: f.name.lower() for f in flag if f.name}
+    else:  # < py311
+        return {f: f.name.lower() for f in list(flag.__class__) if f & flag and is_distinct(f) and f.name}
 
 
-ImageStatsFlags = Union[ImageHash, ImageProperty, ImageVisuals, ImageStatistics]
-LinterFlags = Union[ImageProperty, ImageVisuals, ImageStatistics]
+def verify_supported(flag: TFlag, flags: Union[TFlag, Iterable[TFlag]]):
+    supported = flags if isinstance(flags, flag.__class__) else cast(TFlag, reduce(lambda a, b: a | b, flags))  # type: ignore
+    unsupported = flag & ~supported
+    if unsupported:
+        raise ValueError(f"Unsupported flags {unsupported} called.  Only {supported} flags are supported.")

dataeval/_internal/interop.py

@@ -1,7 +1,8 @@
 from importlib import import_module
-from typing import Any, Iterable, Optional, runtime_checkable
+from typing import Iterable, Optional
 
 import numpy as np
+from numpy.typing import ArrayLike, NDArray
 
 module_cache = {}
 
@@ -19,17 +20,7 @@ def try_import(module_name):
     return module
 
 
-try:
-    from maite.protocols import ArrayLike  # type: ignore
-except ImportError:  # pragma: no cover - covered by test_mindeps.py
-    from typing import Protocol
-
-    @runtime_checkable
-    class ArrayLike(Protocol):
-        def __array__(self) -> Any: ...
-
-
-def to_numpy(array: Optional[ArrayLike]) -> np.ndarray:
+def to_numpy(array: Optional[ArrayLike]) -> NDArray:
     if array is None:
         return np.ndarray([])