dataeval 0.61.0__py3-none-any.whl

dataeval/_internal/detectors/ood/aegmm.py

@@ -0,0 +1,69 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable
+
+import keras
+import numpy as np
+
+from dataeval._internal.detectors.ood.base import OODGMMBase, OODScore
+from dataeval._internal.models.tensorflow.autoencoder import AEGMM
+from dataeval._internal.models.tensorflow.gmm import gmm_energy
+from dataeval._internal.models.tensorflow.losses import LossGMM
+from dataeval._internal.models.tensorflow.utils import predict_batch
+
+
+class OOD_AEGMM(OODGMMBase):
+    def __init__(self, model: AEGMM) -> None:
+        """
+        AE with Gaussian Mixture Model based outlier detector.
+
+        Parameters
+        ----------
+        model : AEGMM
+            An AEGMM model.
+        """
+        super().__init__(model)
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable = LossGMM(),
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Train the AEGMM model with recommended loss function and optimizer.
+
+        Parameters
+        ----------
+        x_ref : np.ndarray
+            Training batch.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : Callable, default LossGMM()
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        """
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X)
+        _, z, _ = predict_batch(X, self.model, batch_size=batch_size)
+        energy, _ = gmm_energy(z, self.gmm_params, return_mean=False)
+        return OODScore(energy.numpy())  # type: ignore

dataeval/_internal/detectors/ood/base.py

@@ -0,0 +1,199 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from abc import ABC, abstractmethod
+from typing import Callable, Dict, List, Literal, NamedTuple, Optional, Tuple, cast
+
+import keras
+import numpy as np
+import tensorflow as tf
+
+from dataeval._internal.models.tensorflow.gmm import GaussianMixtureModelParams, gmm_params
+from dataeval._internal.models.tensorflow.trainer import trainer
+
+
+class OODScore(NamedTuple):
+    """
+    NamedTuple containing the instance and (optionally) feature score.
+
+    Parameters
+    ----------
+    instance_score : np.ndarray
+        Instance score of the evaluated dataset.
+    feature_score : Optional[np.ndarray], default None
+        Feature score, if available, of the evaluated dataset.
+    """
+
+    instance_score: np.ndarray
+    feature_score: Optional[np.ndarray] = None
+
+    def get(self, ood_type: Literal["instance", "feature"]) -> np.ndarray:
+        return self.instance_score if ood_type == "instance" or self.feature_score is None else self.feature_score
+
+
+class OODBase(ABC):
+    def __init__(self, model: keras.Model) -> None:
+        self.model = model
+
+        self._ref_score: OODScore
+        self._threshold_perc: float
+        self._data_info: Optional[Tuple[tuple, type]] = None
+
+        if not isinstance(model, keras.Model):
+            raise TypeError("Model should be of type 'keras.Model'.")
+
+    def _get_data_info(self, X: np.ndarray) -> Tuple[tuple, type]:
+        if not isinstance(X, np.ndarray):
+            raise TypeError("Dataset should of type: `np.ndarray`.")
+        return X.shape[1:], X.dtype.type
+
+    def _validate(self, X: np.ndarray) -> None:
+        check_data_info = self._get_data_info(X)
+        if self._data_info is not None and check_data_info != self._data_info:
+            raise RuntimeError(f"Expect data of type: {self._data_info[1]} and shape: {self._data_info[0]}. \
+                               Provided data is type: {check_data_info[1]} and shape: {check_data_info[0]}.")
+
+    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+        attrs = ["_data_info", "_threshold_perc", "_ref_score"]
+        attrs = attrs if additional_attrs is None else attrs + additional_attrs
+        if not all(hasattr(self, attr) for attr in attrs) or any(getattr(self, attr) for attr in attrs) is None:
+            raise RuntimeError("Metric needs to be `fit` before method call.")
+        self._validate(X)
+
+    @abstractmethod
+    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+        """
+        Compute instance and (optionally) feature level outlier scores.
+
+        Parameters
+        ----------
+        X : np.ndarray
+            Batch of instances.
+        batch_size : int, default int(1e10)
+            Batch size used when making predictions with the autoencoder.
+
+        Returns
+        -------
+        Instance and feature level outlier scores.
+        """
+
+    def _threshold_score(self, ood_type: Literal["feature", "instance"] = "instance") -> np.floating:
+        return np.percentile(self._ref_score.get(ood_type), self._threshold_perc)
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float,
+        loss_fn: Callable,
+        optimizer: keras.optimizers.Optimizer,
+        epochs: int,
+        batch_size: int,
+        verbose: bool,
+    ) -> None:
+        """
+        Train the model and infer the threshold value.
+
+        Parameters
+        ----------
+        x_ref: : np.ndarray
+            Training batch.
+        threshold_perc : float
+            Percentage of reference data that is normal.
+        loss_fn : Callable
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer
+            Optimizer used for training.
+        epochs : int
+            Number of training epochs.
+        batch_size : int
+            Batch size used for training.
+        verbose : bool
+            Whether to print training progress.
+        """
+        # Train the model
+        trainer(
+            model=self.model,
+            loss_fn=loss_fn,
+            x_train=x_ref,
+            optimizer=optimizer,
+            epochs=epochs,
+            batch_size=batch_size,
+            verbose=verbose,
+        )
+
+        # Infer the threshold values
+        self._ref_score = self.score(x_ref, batch_size)
+        self._threshold_perc = threshold_perc
+
+    def predict(
+        self,
+        X: np.ndarray,
+        batch_size: int = int(1e10),
+        ood_type: Literal["feature", "instance"] = "instance",
+    ) -> Dict[str, np.ndarray]:
+        """
+        Predict whether instances are out-of-distribution or not.
+
+        Parameters
+        ----------
+        X
+            Batch of instances.
+        ood_type
+            Predict out-of-distribution at the 'feature' or 'instance' level.
+        batch_size
+            Batch size used when making predictions with the autoencoder.
+
+        Returns
+        -------
+        Dictionary containing the outlier predictions and both feature and instance level outlier scores.
+        """
+        self._validate_state(X)
+        # compute outlier scores
+        score = self.score(X, batch_size=batch_size)
+        ood_pred = (score.get(ood_type) > self._threshold_score(ood_type)).astype(int)
+        return {**{"is_ood": ood_pred}, **score._asdict()}
+
+
+class OODGMMBase(OODBase):
+    def __init__(self, model: keras.Model) -> None:
+        super().__init__(model)
+        self.gmm_params: GaussianMixtureModelParams
+
+    def _validate_state(self, X: np.ndarray, additional_attrs: Optional[List[str]] = None) -> None:
+        if additional_attrs is None:
+            additional_attrs = ["gmm_params"]
+        super()._validate_state(X, additional_attrs)
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float,
+        loss_fn: Callable[[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor], tf.Tensor],
+        optimizer: keras.optimizers.Optimizer,
+        epochs: int,
+        batch_size: int,
+        verbose: bool,
+    ) -> None:
+        # Train the model
+        trainer(
+            model=self.model,
+            loss_fn=loss_fn,
+            x_train=x_ref,
+            optimizer=optimizer,
+            epochs=epochs,
+            batch_size=batch_size,
+            verbose=verbose,
+        )
+
+        # Calculate the GMM parameters
+        _, z, gamma = cast(Tuple[tf.Tensor, tf.Tensor, tf.Tensor], self.model(x_ref))
+        self.gmm_params = gmm_params(z, gamma)
+
+        # Infer the threshold values
+        self._ref_score = self.score(x_ref, batch_size)
+        self._threshold_perc = threshold_perc

dataeval/_internal/detectors/ood/llr.py

@@ -0,0 +1,284 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from functools import partial
+from typing import Callable, Optional, Tuple
+
+import keras
+import numpy as np
+import tensorflow as tf
+from keras.layers import Input
+from keras.models import Model
+
+from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
+from dataeval._internal.models.tensorflow.trainer import trainer
+from dataeval._internal.models.tensorflow.utils import predict_batch
+
+
+def build_model(
+    dist: PixelCNN, input_shape: Optional[tuple] = None, filepath: Optional[str] = None
+) -> Tuple[keras.Model, PixelCNN]:
+    """
+    Create keras.Model from TF distribution.
+
+    Parameters
+    ----------
+    dist
+        TensorFlow distribution.
+    input_shape
+        Input shape of the model.
+    filepath
+        File to load model weights from.
+
+    Returns
+    -------
+    TensorFlow model.
+    """
+    x_in = Input(shape=input_shape)
+    log_prob = dist.log_prob(x_in)
+    model = Model(inputs=x_in, outputs=log_prob)
+    model.add_loss(-tf.reduce_mean(log_prob))
+    if isinstance(filepath, str):
+        model.load_weights(filepath)
+    return model, dist
+
+
+def mutate_categorical(
+    X: np.ndarray,
+    rate: float,
+    seed: int = 0,
+    feature_range: tuple = (0, 255),
+) -> tf.Tensor:
+    """
+    Randomly change integer feature values to values within a set range
+    with a specified permutation rate.
+
+    Parameters
+    ----------
+    X
+        Batch of data to be perturbed.
+    rate
+        Permutation rate (between 0 and 1).
+    seed
+        Random seed.
+    feature_range
+        Min and max range for perturbed features.
+
+    Returns
+    -------
+    Array with perturbed data.
+    """
+    frange = (feature_range[0] + 1, feature_range[1] + 1)
+    shape = X.shape
+    n_samples = np.prod(shape)
+    mask = tf.random.categorical(tf.math.log([[1.0 - rate, rate]]), n_samples, seed=seed, dtype=tf.int32)
+    mask = tf.reshape(mask, shape)
+    possible_mutations = tf.random.uniform(shape, minval=frange[0], maxval=frange[1], dtype=tf.int32, seed=seed + 1)
+    X = tf.math.floormod(tf.cast(X, tf.int32) + mask * possible_mutations, frange[1])  # type: ignore py38
+    return tf.cast(X, tf.float32)  # type: ignore
+
+
+class OOD_LLR(OODBase):
+    def __init__(
+        self,
+        model: PixelCNN,
+        model_background: Optional[PixelCNN] = None,
+        log_prob: Optional[Callable] = None,
+        sequential: bool = False,
+    ) -> None:
+        """
+        Likelihood Ratios based outlier detector.
+
+        Parameters
+        ----------
+        model : PixelCNN
+            Generative distribution model.
+        model_background : Optional[PixelCNN], default None
+            Optional model for the background. Only needed if it is different from `model`.
+        log_prob : Optional[Callable], default None
+            Function used to evaluate log probabilities under the model
+            if the model does not have a `log_prob` function.
+        sequential : bool, default False
+            Whether the data is sequential. Used to create targets during training.
+        """
+        self.dist_s = model
+        self.dist_b = (
+            model.copy()
+            if hasattr(model, "copy")
+            else keras.models.clone_model(model)
+            if model_background is None
+            else model_background
+        )
+        self.has_log_prob = hasattr(model, "log_prob")
+        self.sequential = sequential
+        self.log_prob = log_prob
+
+        self._ref_score: OODScore
+        self._threshold_perc: float
+        self._data_info: Optional[Tuple[tuple, type]] = None
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float = 100.0,
+        loss_fn: Optional[Callable] = None,
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
+        mutate_fn: Callable = mutate_categorical,
+        mutate_fn_kwargs: dict = {"rate": 0.2, "seed": 0, "feature_range": (0, 255)},
+        mutate_batch_size: int = int(1e10),
+    ) -> None:
+        """
+        Train semantic and background generative models.
+
+        Parameters
+        ----------
+        x_ref : np.ndarray
+            Training batch.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : Optional[Callable], default None
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        mutate_fn : Callable, default mutate_categorical
+            Mutation function used to generate the background dataset.
+        mutate_fn_kwargs : dict, default {"rate": 0.2, "seed": 0, "feature_range": (0, 255)}
+            Kwargs for the mutation function used to generate the background dataset.
+            Default values set for an image dataset.
+        mutate_batch_size: int, default int(1e10)
+            Batch size used to generate the mutations for the background dataset.
+        """
+        input_shape = x_ref.shape[1:]
+        optimizer = optimizer() if isinstance(optimizer, type) else optimizer
+        # Separate into two separate optimizers, one for semantic model and one for background model
+        optimizer_s = optimizer
+        optimizer_b = optimizer.__class__.from_config(optimizer.get_config())
+
+        # training arguments
+        kwargs = {
+            "epochs": epochs,
+            "batch_size": batch_size,
+            "verbose": verbose,
+        }
+
+        # create background data
+        mutate_fn = partial(mutate_fn, **mutate_fn_kwargs)
+        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)
+
+        # prepare sequential data
+        if self.sequential and not self.has_log_prob:
+            y, y_back = x_ref[:, 1:], X_back[:, 1:]  # type: ignore
+            X, X_back = x_ref[:, :-1], X_back[:, :-1]  # type: ignore
+        else:
+            X = x_ref
+            y, y_back = None, None
+
+        # check if model needs to be built
+        use_build = self.has_log_prob and not isinstance(self.dist_s, keras.Model)
+
+        if use_build:
+            # build and train semantic model
+            self.model_s = build_model(self.dist_s, input_shape)[0]
+            self.model_s.compile(optimizer=optimizer_s)
+            self.model_s.fit(X, **kwargs)
+            # build and train background model
+            self.model_b = build_model(self.dist_b, input_shape)[0]
+            self.model_b.compile(optimizer=optimizer_b)
+            self.model_b.fit(X_back, **kwargs)
+        else:
+            # train semantic model
+            args = [self.dist_s, X]
+            kwargs.update({"y_train": y, "loss_fn": loss_fn, "optimizer": optimizer_s})
+            trainer(*args, **kwargs)
+
+            # train background model
+            args = [self.dist_b, X_back]
+            kwargs.update({"y_train": y_back, "loss_fn": loss_fn, "optimizer": optimizer_b})
+            trainer(*args, **kwargs)
+
+        self._datainfo = self._get_data_info(x_ref)
+        self._ref_score = self.score(x_ref, batch_size=batch_size)
+        self._threshold_perc = threshold_perc
+
+    def _logp(
+        self,
+        dist,
+        X: np.ndarray,
+        return_per_feature: bool = False,
+        batch_size: int = int(1e10),
+    ) -> np.ndarray:
+        """
+        Compute log probability of a batch of instances under the generative model.
+        """
+        logp_fn = partial(dist.log_prob, return_per_feature=return_per_feature)
+        # TODO: TBD: can this be any of the other types from predict_batch? i.e. tf.Tensor or tuple
+        return predict_batch(X, logp_fn, batch_size=batch_size)  # type: ignore[return-value]
+
+    def _logp_alt(
+        self,
+        model: keras.Model,
+        X: np.ndarray,
+        return_per_feature: bool = False,
+        batch_size: int = int(1e10),
+    ) -> np.ndarray:
+        """
+        Compute log probability of a batch of instances with the user defined log_prob function.
+        """
+        if self.sequential:
+            y, X = X[:, 1:], X[:, :-1]
+        else:
+            y = X.copy()
+        y_preds = predict_batch(X, model, batch_size=batch_size)
+        logp = self.log_prob(y, y_preds).numpy()  # type: ignore
+        if return_per_feature:
+            return logp
+        else:
+            axis = tuple(np.arange(len(logp.shape))[1:])
+            return np.mean(logp, axis=axis)
+
+    def _llr(self, X: np.ndarray, return_per_feature: bool, batch_size: int = int(1e10)) -> np.ndarray:
+        """
+        Compute likelihood ratios.
+
+        Parameters
+        ----------
+        X
+            Batch of instances.
+        return_per_feature
+            Return likelihood ratio per feature.
+        batch_size
+            Batch size for the generative model evaluations.
+
+        Returns
+        -------
+        Likelihood ratios.
+        """
+        logp_fn = self._logp if not isinstance(self.log_prob, Callable) else self._logp_alt  # type: ignore
+        logp_s = logp_fn(self.dist_s, X, return_per_feature=return_per_feature, batch_size=batch_size)
+        logp_b = logp_fn(self.dist_b, X, return_per_feature=return_per_feature, batch_size=batch_size)
+        return logp_s - logp_b
+
+    def score(
+        self,
+        X: np.ndarray,
+        batch_size: int = int(1e10),
+    ) -> OODScore:
+        self._validate(X)
+        fscore = -self._llr(X, True, batch_size=batch_size)
+        iscore = -self._llr(X, False, batch_size=batch_size)
+        return OODScore(iscore, fscore)

dataeval/_internal/detectors/ood/vae.py

@@ -0,0 +1,86 @@
+"""
+Source code derived from Alibi-Detect 0.11.4
+https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
+
+Original code Copyright (c) 2023 Seldon Technologies Ltd
+Licensed under Apache Software License (Apache 2.0)
+"""
+
+from typing import Callable
+
+import keras
+import numpy as np
+
+from dataeval._internal.detectors.ood.base import OODBase, OODScore
+from dataeval._internal.models.tensorflow.autoencoder import VAE
+from dataeval._internal.models.tensorflow.losses import Elbo
+from dataeval._internal.models.tensorflow.utils import predict_batch
+
+
+class OOD_VAE(OODBase):
+    def __init__(self, model: VAE, samples: int = 10) -> None:
+        """
+        VAE based outlier detector.
+
+        Parameters
+        ----------
+        model : VAE
+            A VAE model.
+        samples : int, default 10
+            Number of samples sampled to evaluate each instance.
+        """
+        super().__init__(model)
+        self.samples = samples
+
+    def fit(
+        self,
+        x_ref: np.ndarray,
+        threshold_perc: float = 100.0,
+        loss_fn: Callable = Elbo(0.05),
+        optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
+        epochs: int = 20,
+        batch_size: int = 64,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Train the VAE model.
+
+        Parameters
+        ----------
+        x_ref : np.ndarray
+            Training batch.
+        threshold_perc : float, default 100.0
+            Percentage of reference data that is normal.
+        loss_fn : Callable, default Elbo(0.05)
+            Loss function used for training.
+        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
+            Optimizer used for training.
+        epochs : int, default 20
+            Number of training epochs.
+        batch_size : int, default 64
+            Batch size used for training.
+        verbose : bool, default True
+            Whether to print training progress.
+        """
+        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
+
+    def score(self, X: np.ndarray, batch_size: int = int(1e10)) -> OODScore:
+        self._validate(X)
+
+        # sample reconstructed instances
+        X_samples = np.repeat(X, self.samples, axis=0)
+        X_recon = predict_batch(X_samples, model=self.model, batch_size=batch_size)
+
+        # compute feature scores
+        fscore = np.power(X_samples - X_recon, 2)
+        fscore = fscore.reshape((-1, self.samples) + X_samples.shape[1:])
+        fscore = np.mean(fscore, axis=1)
+
+        # compute instance scores
+        fscore_flat = fscore.reshape(fscore.shape[0], -1).copy()
+        n_score_features = int(np.ceil(fscore_flat.shape[1]))
+        sorted_fscore = np.sort(fscore_flat, axis=1)
+        sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
+        iscore = np.mean(sorted_fscore_perc, axis=1)
+
+        return OODScore(iscore, fscore)