dataeval 0.64.0__py3-none-any.whl → 0.66.0__py3-none-any.whl

dataeval/_internal/detectors/duplicates.py

@@ -1,9 +1,28 @@
-from typing import Dict, Iterable, List, Literal
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable
 
 from numpy.typing import ArrayLike
 
-from dataeval._internal.flags import ImageHash
-from dataeval._internal.metrics.stats import ImageStats
+from dataeval._internal.flags import ImageStat
+from dataeval._internal.metrics.stats import StatsOutput, imagestats
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+
+@dataclass(frozen=True)
+class DuplicatesOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    exact : List[List[int]]
+        Indices of images that are exact matches
+    near: List[List[int]]
+        Indices of images that are near matches
+    """
+
+    exact: list[list[int]]
+    near: list[list[int]]
 
 
 class Duplicates:
@@ -13,8 +32,13 @@ class Duplicates:
 
     Attributes
     ----------
-    stats : ImageStats(flags=ImageHash.ALL)
-        Base stats class with the flags for checking duplicates
+    stats : StatsOutput
+        Output class of stats
+
+    Parameters
+    ----------
+    only_exact : bool, default False
+        Only inspect the dataset for exact image matches
 
     Example
     -------
@@ -23,51 +47,63 @@ class Duplicates:
     >>> dups = Duplicates()
     """
 
-    def __init__(self):
-        self.stats = ImageStats(ImageHash.ALL)
+    def __init__(self, only_exact: bool = False):
+        self.stats: StatsOutput
+        self.only_exact = only_exact
+
+    def _get_duplicates(self) -> dict[str, list[list[int]]]:
+        stats_dict = self.stats.dict()
+        if "xxhash" in stats_dict:
+            exact = {}
+            for i, value in enumerate(stats_dict["xxhash"]):
+                exact.setdefault(value, []).append(i)
+            exact = [v for v in exact.values() if len(v) > 1]
+        else:
+            exact = []
 
-    def _get_duplicates(self) -> dict:
-        exact = {}
-        near = {}
-        for i, value in enumerate(self.results["xxhash"]):
-            exact.setdefault(value, []).append(i)
-        for i, value in enumerate(self.results["pchash"]):
-            near.setdefault(value, []).append(i)
-        exact = [v for v in exact.values() if len(v) > 1]
-        near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+        if "pchash" in stats_dict and not self.only_exact:
+            near = {}
+            for i, value in enumerate(stats_dict["pchash"]):
+                near.setdefault(value, []).append(i)
+            near = [v for v in near.values() if len(v) > 1 and not any(set(v).issubset(x) for x in exact)]
+        else:
+            near = []
 
         return {
             "exact": sorted(exact),
             "near": sorted(near),
         }
 
-    def evaluate(self, images: Iterable[ArrayLike]) -> Dict[Literal["exact", "near"], List[int]]:
+    @set_metadata("dataeval.detectors", ["only_exact"])
+    def evaluate(self, data: Iterable[ArrayLike] | StatsOutput) -> DuplicatesOutput:
         """
         Returns duplicate image indices for both exact matches and near matches
 
         Parameters
         ----------
-        images : Iterable[ArrayLike], shape - (N, C, H, W)
-            A set of images in an ArrayLike format
+        data : Iterable[ArrayLike], shape - (N, C, H, W) | StatsOutput
+            A dataset of images in an ArrayLike format or the output from an imagestats metric analysis
 
         Returns
         -------
-        Dict[str, List[int]]
-            exact :
-                List of groups of indices that are exact matches
-            near :
-                List of groups of indices that are near matches
+        DuplicatesOutput
+            List of groups of indices that are exact and near matches
 
         See Also
         --------
-        ImageStats
+        imagestats
 
         Example
         -------
         >>> dups.evaluate(images)
-        {'exact': [[3, 20], [16, 37]], 'near': [[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]]}
-        """
-        self.stats.reset()
-        self.stats.update(images)
-        self.results = self.stats.compute()
-        return self._get_duplicates()
+        DuplicatesOutput(exact=[[3, 20], [16, 37]], near=[[3, 20, 22], [12, 18], [13, 36], [14, 31], [17, 27], [19, 38, 47]])
+        """  # noqa: E501
+        if isinstance(data, StatsOutput):
+            if not data.xxhash:
+                raise ValueError("StatsOutput must include xxhash information of the images.")
+            if not self.only_exact and not data.pchash:
+                raise ValueError("StatsOutput must include pchash information of the images for near matches.")
+            self.stats = data
+        else:
+            self.stats = imagestats(data, ImageStat.XXHASH | (ImageStat(0) if self.only_exact else ImageStat.PCHASH))
+        return DuplicatesOutput(**self._get_duplicates())

dataeval/_internal/detectors/ood/ae.py

@@ -6,10 +6,13 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
+from __future__ import annotations
+
 from typing import Callable
 
 import keras
 import numpy as np
+import tensorflow as tf
 from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODBase, OODScore
@@ -19,47 +22,30 @@ from dataeval._internal.models.tensorflow.utils import predict_batch
 
 
 class OOD_AE(OODBase):
-    def __init__(self, model: AE) -> None:
-        """
-        Autoencoder based out-of-distribution detector.
+    """
+    Autoencoder based out-of-distribution detector.
+
+    Parameters
+    ----------
+    model : AE
+        An Autoencoder model.
+    """
 
-        Parameters
-        ----------
-        model : AE
-            An Autoencoder model.
-        """
+    def __init__(self, model: AE) -> None:
         super().__init__(model)
 
     def fit(
         self,
         x_ref: ArrayLike,
         threshold_perc: float = 100.0,
-        loss_fn: Callable = keras.losses.MeanSquaredError(),
+        loss_fn: Callable[..., tf.Tensor] | None = None,
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
     ) -> None:
-        """
-        Train the AE model with recommended loss function and optimizer.
-
-        Parameters
-        ----------
-        x_ref : ArrayLike
-            Training batch.
-        threshold_perc : float, default 100.0
-            Percentage of reference data that is normal.
-        loss_fn : Callable, default keras.losses.MeanSquaredError()
-            Loss function used for training.
-        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
-            Optimizer used for training.
-        epochs : int, default 20
-            Number of training epochs.
-        batch_size : int, default 64
-            Batch size used for training.
-        verbose : bool, default True
-            Whether to print training progress.
-        """
+        if loss_fn is None:
+            loss_fn = keras.losses.MeanSquaredError()
         super().fit(to_numpy(x_ref), threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
 
     def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:

dataeval/_internal/detectors/ood/aegmm.py

@@ -6,9 +6,12 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
+from __future__ import annotations
+
 from typing import Callable
 
 import keras
+import tensorflow as tf
 from numpy.typing import ArrayLike
 
 from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
@@ -20,50 +23,53 @@ from dataeval._internal.models.tensorflow.utils import predict_batch
 
 
 class OOD_AEGMM(OODGMMBase):
-    def __init__(self, model: AEGMM) -> None:
-        """
-        AE with Gaussian Mixture Model based outlier detector.
+    """
+    AE with Gaussian Mixture Model based outlier detector.
 
-        Parameters
-        ----------
-        model : AEGMM
-            An AEGMM model.
-        """
+    Parameters
+    ----------
+    model : AEGMM
+        An AEGMM model.
+    """
+
+    def __init__(self, model: AEGMM) -> None:
         super().__init__(model)
 
     def fit(
         self,
         x_ref: ArrayLike,
         threshold_perc: float = 100.0,
-        loss_fn: Callable = LossGMM(),
+        loss_fn: Callable[..., tf.Tensor] | None = None,
         optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
         epochs: int = 20,
         batch_size: int = 64,
         verbose: bool = True,
     ) -> None:
+        if loss_fn is None:
+            loss_fn = LossGMM()
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
         """
-        Train the AEGMM model with recommended loss function and optimizer.
+        Compute the out-of-distribution (OOD) score for a given dataset.
 
         Parameters
         ----------
-        x_ref : ArrayLike
-            Training batch.
-        threshold_perc : float, default 100.0
-            Percentage of reference data that is normal.
-        loss_fn : Callable, default LossGMM()
-            Loss function used for training.
-        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
-            Optimizer used for training.
-        epochs : int, default 20
-            Number of training epochs.
-        batch_size : int, default 64
-            Batch size used for training.
-        verbose : bool, default True
-            Whether to print training progress.
-        """
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+        X : ArrayLike
+            Input data to score.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+            Use a smaller batch size if your dataset is large or if you encounter memory issues.
 
-    def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
+        Returns
+        -------
+        OODScore
+            An object containing the instance-level OOD score.
+
+        Note
+        ----
+        This model does not produce a feature level score like the OOD_AE or OOD_VAE models.
+        """
         self._validate(X := to_numpy(X))
         _, z, _ = predict_batch(X, self.model, batch_size=batch_size)
         energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)

dataeval/_internal/detectors/ood/base.py

@@ -6,17 +6,39 @@ Original code Copyright (c) 2023 Seldon Technologies Ltd
 Licensed under Apache Software License (Apache 2.0)
 """
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, List, Literal, NamedTuple, Optional, Tuple, cast
+from dataclasses import dataclass
+from typing import Callable, Literal, NamedTuple, cast
 
 import keras
 import numpy as np
 import tensorflow as tf
-from numpy.typing import ArrayLike
+from numpy.typing import ArrayLike, NDArray
 
 from dataeval._internal.interop import to_numpy
 from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
 from dataeval._internal.models.tensorflow.trainer import trainer
+from dataeval._internal.output import OutputMetadata, set_metadata
+
+
+@dataclass(frozen=True)
+class OODOutput(OutputMetadata):
+    """
+    Attributes
+    ----------
+    is_ood : NDArray
+        Array of images that are detected as out of distribution
+    instance_score : NDArray
+        Instance score of the evaluated dataset
+    feature_score : NDArray | None
+        Feature score, if available, of the evaluated dataset
+    """
+
+    is_ood: NDArray[np.bool_]
+    instance_score: NDArray[np.float32]
+    feature_score: NDArray[np.float32] | None
 
 
 class OODScore(NamedTuple):
@@ -25,16 +47,28 @@ class OODScore(NamedTuple):
 
     Parameters
     ----------
-    instance_score : np.ndarray
+    instance_score : NDArray
         Instance score of the evaluated dataset.
-    feature_score : Optional[np.ndarray], default None
+    feature_score : NDArray | None, default None
         Feature score, if available, of the evaluated dataset.
     """
 
-    instance_score: np.ndarray
-    feature_score: Optional[np.ndarray] = None
+    instance_score: NDArray[np.float32]
+    feature_score: NDArray[np.float32] | None = None
+
+    def get(self, ood_type: Literal["instance", "feature"]) -> NDArray:
+        """
+        Returns either the instance or feature score
+
+        Parameters
+        ----------
+        ood_type : "instance" | "feature"
 
-    def get(self, ood_type: Literal["instance", "feature"]) -> np.ndarray:
+        Returns
+        -------
+        NDArray
+            Either the instance or feature score based on input selection
+        """
         return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
 
 
@@ -44,23 +78,23 @@ class OODBase(ABC):
 
         self._ref_score: OODScore
         self._threshold_perc: float
-        self._data_info: Optional[Tuple[tuple, type]] = None
+        self._data_info: tuple[tuple, type] | None = None
 
         if not isinstance(model, keras.Model):
             raise TypeError("Model should be of type 'keras.Model'.")
 
-    def _get_data_info(self, X: np.ndarray) -> Tuple[tuple, type]:
+    def _get_data_info(self, X: NDArray) -> tuple[tuple, type]:
         if not isinstance(X, np.ndarray):
-            raise TypeError("Dataset should of type: `np.ndarray`.")
+            raise TypeError("Dataset should of type: `NDArray`.")
         return X.shape[1:], X.dtype.type
 
-    def _validate(self, X: np.ndarray) -> None:
+    def _validate(self, X: NDArray) -> None:
         check_data_info = self._get_data_info(X)
         if self._data_info is not None and check_data_info != self._data_info:
             raise RuntimeError(f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
                                Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}.")
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: list[str] | None = None) -> None:
         attrs = ["_data_info", "_threshold_perc", "_ref_score"]
         attrs = attrs if additional_attrs is None else attrs + additional_attrs
         if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
@@ -70,18 +104,20 @@ class OODBase(ABC):
     @abstractmethod
     def score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScore:
         """
-        Compute instance and (optionally) feature level outlier scores.
+        Compute the out-of-distribution (OOD) scores for a given dataset.
 
         Parameters
         ----------
         X : ArrayLike
-            Batch of instances.
-        batch_size : int, default int(1e10)
-            Batch size used when making predictions with the autoencoder.
+            Input data to score.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+            Use a smaller batch size if your dataset is large or if you encounter memory issues.
 
         Returns
         -------
-        Instance and feature level outlier scores.
+        OODScore
+            An object containing the instance-level and feature-level OOD scores.
         """
 
     def _threshold_score(self, ood_type: Literal["feature", "instance"] = "instance") -> np.floating:
@@ -90,33 +126,34 @@ class OODBase(ABC):
     def fit(
         self,
         x_ref: ArrayLike,
-        threshold_perc: float,
-        loss_fn: Callable,
-        optimizer: keras.optimizers.Optimizer,
-        epochs: int,
-        batch_size: int,
-        verbose: bool,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable[..., tf.Tensor] | None = None,
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
     ) -> None:
         """
         Train the model and infer the threshold value.
 
         Parameters
         ----------
-        x_ref: : ArrayLike
-            Training batch.
-        threshold_perc : float
+        x_ref : ArrayLike
+            Training data.
+        threshold_perc : float, default 100.0
             Percentage of reference data that is normal.
-        loss_fn : Callable
+        loss_fn : Callable | None, default None
             Loss function used for training.
-        optimizer : keras.optimizers.Optimizer
+        optimizer : Optimizer, default keras.optimizers.Adam
             Optimizer used for training.
-        epochs : int
+        epochs : int, default 20
             Number of training epochs.
-        batch_size : int
+        batch_size : int, default 64
             Batch size used for training.
-        verbose : bool
+        verbose : bool, default True
             Whether to print training progress.
         """
+
         # Train the model
         trainer(
             model=self.model,
@@ -132,33 +169,35 @@ class OODBase(ABC):
         self._ref_score = self.score(x_ref, batch_size)
         self._threshold_perc = threshold_perc
 
+    @set_metadata("dataeval.detectors")
     def predict(
         self,
         X: ArrayLike,
         batch_size: int = int(1e10),
         ood_type: Literal["feature", "instance"] = "instance",
-    ) -> Dict[str, np.ndarray]:
+    ) -> OODOutput:
         """
         Predict whether instances are out-of-distribution or not.
 
         Parameters
         ----------
         X : ArrayLike
-            Batch of instances.
-        batch_size : int, default int(1e10)
-            Batch size used when making predictions with the autoencoder.
-        ood_type : Literal["feature", "instance"], default "instance"
+            Input data for out-of-distribution prediction.
+        batch_size : int, default 1e10
+            Number of instances to process in each batch.
+        ood_type : "feature" | "instance", default "instance"
             Predict out-of-distribution at the 'feature' or 'instance' level.
 
         Returns
         -------
-        Dictionary containing the outlier predictions and both feature and instance level outlier scores.
+        Dictionary containing the outlier predictions for the selected level,
+        and the OOD scores for the data including both 'instance' and 'feature' (if present) level scores.
         """
         self._validate_state(X := to_numpy(X))
         # compute outlier scores
         score = self.score(X, batch_size=batch_size)
-        ood_pred = (score.get(ood_type) > self._threshold_score(ood_type)).astype(int)
-        return {**{"is_ood": ood_pred}, **score._asdict()}
+        ood_pred = score.get(ood_type) > self._threshold_score(ood_type)
+        return OODOutput(is_ood=ood_pred, **score._asdict())
 
 
 class OODGMMBase(OODBase):
@@ -166,7 +205,7 @@ class OODGMMBase(OODBase):
         super().__init__(model)
         self.gmm_params: GaussianMixtureModelParams
 
-    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+    def _validate_state(self, X: NDArray, additional_attrs: list[str] | None = None) -> None:
         if additional_attrs is None:
             additional_attrs = ["gmm_params"]
         super()._validate_state(X, additional_attrs)
@@ -174,12 +213,12 @@ class OODGMMBase(OODBase):
     def fit(
         self,
         x_ref: ArrayLike,
-        threshold_perc: float,
-        loss_fn: Callable[[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor], tf.Tensor],
-        optimizer: keras.optimizers.Optimizer,
-        epochs: int,
-        batch_size: int,
-        verbose: bool,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable[..., tf.Tensor] | None = None,
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
     ) -> None:
         # Train the model
         trainer(
@@ -193,7 +232,7 @@ class OODGMMBase(OODBase):
         )
 
         # Calculate the GMM parameters
-        _, z, gamma = cast(Tuple[tf.Tensor, tf.Tensor, tf.Tensor], self.model(x_ref))
+        _, z, gamma = cast(tuple[tf.Tensor, tf.Tensor, tf.Tensor], self.model(x_ref))
         self.gmm_params = gmm_params(z, gamma)
 
         # Infer the threshold values