dataeval 0.74.0__py3-none-any.whl → 0.74.1__py3-none-any.whl

dataeval/detectors/ood/llr.py

@@ -1,302 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_LLR"]
-
-from functools import partial
-from typing import TYPE_CHECKING, Callable
-
-import numpy as np
-from numpy.typing import ArrayLike, NDArray
-
-from dataeval.detectors.ood.base import OODBaseMixin, OODScoreOutput
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.trainer import trainer
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-def _build_model(
-    dist: tf_models.PixelCNN, input_shape: tuple | None = None, filepath: str | None = None
-) -> tuple[keras.Model, tf_models.PixelCNN]:
-    """
-    Create keras.Model from TF distribution.
-
-    Parameters
-    ----------
-    dist
-        :term:`TensorFlow` distribution.
-    input_shape
-        Input shape of the model.
-    filepath
-        File to load model weights from.
-
-    Returns
-    -------
-    TensorFlow model.
-    """
-    x_in = keras.layers.Input(shape=input_shape)
-    log_prob = dist.log_prob(x_in)
-    model = keras.models.Model(inputs=x_in, outputs=log_prob)
-    model.add_loss(-tf.reduce_mean(log_prob))
-    if isinstance(filepath, str):
-        model.load_weights(filepath)
-    return model, dist
-
-
-def _mutate_categorical(
-    X: NDArray,
-    rate: float,
-    seed: int = 0,
-    feature_range: tuple[int, int] = (0, 255),
-) -> tf.Tensor:
-    """
-    Randomly change integer feature values to values within a set range
-    with a specified permutation rate.
-
-    Parameters
-    ----------
-    X
-        Batch of data to be perturbed.
-    rate
-        Permutation rate (between 0 and 1).
-    seed
-        Random seed.
-    feature_range
-        Min and max range for perturbed features.
-
-    Returns
-    -------
-    Array with perturbed data.
-    """
-    frange = (feature_range[0] + 1, feature_range[1] + 1)
-    shape = X.shape
-    n_samples = np.prod(shape)
-    mask = tf.random.categorical(tf.math.log([[1.0 - rate, rate]]), n_samples, seed=seed, dtype=tf.int32)
-    mask = tf.reshape(mask, shape)
-    possible_mutations = tf.random.uniform(shape, minval=frange[0], maxval=frange[1], dtype=tf.int32, seed=seed + 1)
-    X = tf.math.floormod(tf.cast(X, tf.int32) + mask * possible_mutations, frange[1])  # type: ignore py38
-    return tf.cast(X, tf.float32)  # type: ignore
-
-
-class OOD_LLR(OODBaseMixin[tf_models.PixelCNN]):
-    """
-    Likelihood Ratios based outlier detector.
-
-    Parameters
-    ----------
-    model : PixelCNN
-        Generative distribution model.
-    model_background : Optional[PixelCNN], default None
-        Optional model for the background. Only needed if it is different from `model`.
-    log_prob : Optional[Callable], default None
-        Function used to evaluate log probabilities under the model
-        if the model does not have a `log_prob` function.
-    sequential : bool, default False
-        Whether the data is sequential. Used to create targets during training.
-    """
-
-    def __init__(
-        self,
-        model: tf_models.PixelCNN,
-        model_background: tf_models.PixelCNN | None = None,
-        log_prob: Callable | None = None,
-        sequential: bool = False,
-    ) -> None:
-        self.dist_s: tf_models.PixelCNN = model
-        self.dist_b: tf_models.PixelCNN = (
-            model.copy()
-            if hasattr(model, "copy")
-            else keras.models.clone_model(model)
-            if model_background is None
-            else model_background
-        )
-        self.has_log_prob: bool = hasattr(model, "log_prob")
-        self.sequential: bool = sequential
-        self.log_prob: Callable | None = log_prob
-
-        self._ref_score: OODScoreOutput
-        self._threshold_perc: float
-        self._data_info: tuple[tuple, type] | None = None
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable | None = None,
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-        mutate_fn: Callable = _mutate_categorical,
-        mutate_fn_kwargs: dict[str, float | int | tuple[int, int]] = {
-            "rate": 0.2,
-            "seed": 0,
-            "feature_range": (0, 255),
-        },
-        mutate_batch_size: int = int(1e10),
-    ) -> None:
-        """
-        Train semantic and background generative models.
-
-        Parameters
-        ----------
-        x_ref : ArrayLike
-            Training data.
-        threshold_perc : float, default 100.0
-            Percentage of reference data that is normal.
-        loss_fn : Callable | None, default None
-            Loss function used for training.
-        optimizer : keras.optimizers.Optimizer, default keras.optimizers.Adam
-            Optimizer used for training.
-        epochs : int, default 20
-            Number of training epochs.
-        batch_size : int, default 64
-            Batch size used for training.
-        verbose : bool, default True
-            Whether to print training progress.
-        mutate_fn : Callable, default mutate_categorical
-            Mutation function used to generate the background dataset.
-        mutate_fn_kwargs : dict, default {"rate": 0.2, "seed": 0, "feature_range": (0, 255)}
-            Kwargs for the mutation function used to generate the background dataset.
-            Default values set for an image dataset.
-        mutate_batch_size: int, default int(1e10)
-            Batch size used to generate the mutations for the background dataset.
-        """
-        x_ref = to_numpy(x_ref)
-        input_shape = x_ref.shape[1:]
-        optimizer = keras.optimizers.Adam() if optimizer is None else optimizer
-        # Separate into two separate optimizers, one for semantic model and one for background model
-        optimizer_s = optimizer
-        optimizer_b = optimizer.__class__.from_config(optimizer.get_config())
-
-        # training arguments
-        kwargs = {
-            "epochs": epochs,
-            "batch_size": batch_size,
-            "verbose": verbose,
-        }
-
-        # create background data
-        mutate_fn = partial(mutate_fn, **mutate_fn_kwargs)
-        X_back = predict_batch(x_ref, mutate_fn, batch_size=mutate_batch_size, dtype=x_ref.dtype)  # type: ignore
-
-        # prepare sequential data
-        if self.sequential and not self.has_log_prob:
-            y, y_back = x_ref[:, 1:], X_back[:, 1:]  # type: ignore
-            X, X_back = x_ref[:, :-1], X_back[:, :-1]  # type: ignore
-        else:
-            X = x_ref
-            y, y_back = None, None
-
-        # check if model needs to be built
-        use_build = self.has_log_prob and not isinstance(self.dist_s, keras.Model)
-
-        if use_build:
-            # build and train semantic model
-            self.model_s: keras.Model = _build_model(self.dist_s, input_shape)[0]
-            self.model_s.compile(optimizer=optimizer_s)
-            self.model_s.fit(X, **kwargs)
-            # build and train background model
-            self.model_b: keras.Model = _build_model(self.dist_b, input_shape)[0]
-            self.model_b.compile(optimizer=optimizer_b)
-            self.model_b.fit(X_back, **kwargs)
-        else:
-            # train semantic model
-            args = [self.dist_s, X]
-            kwargs.update({"y_train": y, "loss_fn": loss_fn, "optimizer": optimizer_s})
-            trainer(*args, **kwargs)
-
-            # train background model
-            args = [self.dist_b, X_back]
-            kwargs.update({"y_train": y_back, "loss_fn": loss_fn, "optimizer": optimizer_b})
-            trainer(*args, **kwargs)
-
-        self._datainfo = self._get_data_info(x_ref)
-        self._ref_score = self.score(x_ref, batch_size=batch_size)
-        self._threshold_perc = threshold_perc
-
-    def _logp(
-        self,
-        dist,
-        X: NDArray,
-        return_per_feature: bool = False,
-        batch_size: int = int(1e10),
-    ) -> NDArray:
-        """
-        Compute log probability of a batch of instances under the :term:`generative model<Generative Model>`.
-        """
-        logp_fn = partial(dist.log_prob, return_per_feature=return_per_feature)
-        # TODO: TBD: can this be any of the other types from predict_batch? i.e. tf.Tensor or tuple
-        return predict_batch(X, logp_fn, batch_size=batch_size)  # type: ignore[return-value]
-
-    def _logp_alt(
-        self,
-        model: keras.Model,
-        X: NDArray,
-        return_per_feature: bool = False,
-        batch_size: int = int(1e10),
-    ) -> NDArray:
-        """
-        Compute log probability of a batch of instances with the user defined log_prob function.
-        """
-        if self.sequential:
-            y, X = X[:, 1:], X[:, :-1]
-        else:
-            y = X.copy()
-        y_preds = predict_batch(X, model, batch_size=batch_size)
-        logp = self.log_prob(y, y_preds).numpy()  # type: ignore
-        if return_per_feature:
-            return logp
-        else:
-            axis = tuple(np.arange(len(logp.shape))[1:])
-            return np.mean(logp, axis=axis)
-
-    def _llr(self, X: NDArray, return_per_feature: bool, batch_size: int = int(1e10)) -> NDArray:
-        """
-        Compute likelihood ratios.
-
-        Parameters
-        ----------
-        X
-            Batch of instances.
-        return_per_feature
-            Return likelihood ratio per feature.
-        batch_size
-            Batch size for the :term:`generative model<Generative Model>` evaluations.
-
-        Returns
-        -------
-        Likelihood ratios.
-        """
-        logp_fn = self._logp if not isinstance(self.log_prob, Callable) else self._logp_alt  # type: ignore
-        logp_s = logp_fn(self.dist_s, X, return_per_feature=return_per_feature, batch_size=batch_size)
-        logp_b = logp_fn(self.dist_b, X, return_per_feature=return_per_feature, batch_size=batch_size)
-        return logp_s - logp_b
-
-    def _score(
-        self,
-        X: ArrayLike,
-        batch_size: int = int(1e10),
-    ) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-        fscore = -self._llr(X, True, batch_size=batch_size)
-        iscore = -self._llr(X, False, batch_size=batch_size)
-        return OODScoreOutput(iscore, fscore)

dataeval/detectors/ood/vae.py

@@ -1,98 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_VAE"]
-
-from typing import TYPE_CHECKING, Callable
-
-import numpy as np
-from numpy.typing import ArrayLike
-
-from dataeval.detectors.ood.base import OODScoreOutput
-from dataeval.detectors.ood.base_tf import OODBase
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.loss import Elbo
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-class OOD_VAE(OODBase):
-    """
-    VAE based outlier detector.
-
-    Parameters
-    ----------
-    model : VAE
-        A VAE model.
-    samples : int, default 10
-        Number of samples sampled to evaluate each instance.
-
-    Examples
-    --------
-    Instantiate an OOD detector metric with a generic dataset - batch of images with shape (3,25,25)
-
-    >>> metric = OOD_VAE(create_model("VAE", dataset[0].shape))
-
-    Adjusting fit parameters,
-    including setting the fit threshold at 85% for a training set with about 15% out-of-distribution
-
-    >>> metric.fit(dataset, threshold_perc=85, batch_size=128, verbose=False)
-
-    Detect :term:`out of distribution<Out-of-Distribution (OOD)>` samples at the 'feature' level
-
-    >>> result = metric.predict(dataset, ood_type="feature")
-    """
-
-    def __init__(self, model: tf_models.VAE, samples: int = 10) -> None:
-        super().__init__(model)
-        self.samples = samples
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = Elbo(0.05),
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-    ) -> None:
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
-
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-
-        # sample reconstructed instances
-        X_samples = np.repeat(X, self.samples, axis=0)
-        X_recon = predict_batch(X_samples, model=self.model, batch_size=batch_size)
-
-        # compute feature scores
-        fscore = np.power(X_samples - X_recon, 2)
-        fscore = fscore.reshape((-1, self.samples) + X_samples.shape[1:])
-        fscore = np.mean(fscore, axis=1)
-
-        # compute instance scores
-        fscore_flat = fscore.reshape(fscore.shape[0], -1).copy()
-        n_score_features = int(np.ceil(fscore_flat.shape[1]))
-        sorted_fscore = np.sort(fscore_flat, axis=1)
-        sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
-        iscore = np.mean(sorted_fscore_perc, axis=1)
-
-        return OODScoreOutput(iscore, fscore)

dataeval/detectors/ood/vaegmm.py

@@ -1,76 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_VAEGMM"]
-
-from typing import TYPE_CHECKING, Callable
-
-import numpy as np
-from numpy.typing import ArrayLike
-
-from dataeval.detectors.ood.base import OODScoreOutput
-from dataeval.detectors.ood.base_tf import OODBaseGMM
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.gmm import gmm_energy
-from dataeval.utils.tensorflow._internal.loss import Elbo, LossGMM
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-class OOD_VAEGMM(OODBaseGMM):
-    """
-    VAE with Gaussian Mixture Model based outlier detector.
-
-    Parameters
-    ----------
-    model : VAEGMM
-        A VAEGMM model.
-    samples
-        Number of samples sampled to evaluate each instance.
-    """
-
-    def __init__(self, model: tf_models.VAEGMM, samples: int = 10) -> None:
-        super().__init__(model)
-        self.samples = samples
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = LossGMM(elbo=Elbo(0.05)),
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-    ) -> None:
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
-
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-
-        # draw samples from latent space
-        X_samples = np.repeat(X, self.samples, axis=0)
-        _, z, _ = predict_batch(X_samples, self.model, batch_size=batch_size)
-
-        # compute average energy for samples
-        energy, _ = gmm_energy(z, self._gmm_params, return_mean=False)
-        energy_samples = energy.numpy().reshape((-1, self.samples))  # type: ignore
-        iscore = np.mean(energy_samples, axis=-1)
-        return OODScoreOutput(iscore)

dataeval/utils/lazy.py

@@ -1,26 +0,0 @@
-from __future__ import annotations
-
-from functools import cached_property
-from importlib import import_module
-from typing import Any
-
-
-class LazyModule:
-    def __init__(self, name: str) -> None:
-        self._name = name
-
-    def __getattr__(self, key: str) -> Any:
-        return getattr(self._module, key)
-
-    @cached_property
-    def _module(self):
-        return import_module(self._name)
-
-
-LAZY_MODULES: dict[str, LazyModule] = {}
-
-
-def lazyload(name: str) -> LazyModule:
-    if name not in LAZY_MODULES:
-        LAZY_MODULES[name] = LazyModule(name)
-    return LAZY_MODULES[name]

dataeval/utils/tensorflow/__init__.py

@@ -1,19 +0,0 @@
-"""
-TensorFlow models are used in :term:`out of distribution<Out-of-distribution (OOD)>` detectors in the
-:mod:`dataeval.detectors.ood` module.
-
-DataEval provides basic default models through the utility :func:`dataeval.utils.tensorflow.create_model`.
-"""
-
-from dataeval import _IS_TENSORFLOW_AVAILABLE
-
-__all__ = []
-
-
-if _IS_TENSORFLOW_AVAILABLE:
-    import dataeval.utils.tensorflow.loss as loss
-    from dataeval.utils.tensorflow._internal.utils import create_model
-
-    __all__ = ["create_model", "loss"]
-
-del _IS_TENSORFLOW_AVAILABLE

dataeval/utils/tensorflow/_internal/gmm.py

@@ -1,103 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import numpy as np
-
-from dataeval.utils.gmm import GaussianMixtureModelParams
-from dataeval.utils.lazy import lazyload
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-else:
-    tf = lazyload("tensorflow")
-
-
-def gmm_params(z: tf.Tensor, gamma: tf.Tensor) -> GaussianMixtureModelParams[tf.Tensor]:
-    """
-    Compute parameters of Gaussian Mixture Model.
-
-    Parameters
-    ----------
-    z : tf.Tensor
-        Observations.
-    gamma : tf.Tensor
-        Mixture probabilities to derive mixture distribution weights from.
-
-    Returns
-    -------
-    GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
-        The parameters used to calculate energy.
-    """
-    # compute gmm parameters phi, mu and cov
-    N = gamma.shape[0]  # nb of samples in batch
-    sum_gamma = tf.reduce_sum(gamma, 0)  # K
-    phi = sum_gamma / N  # K
-    mu = tf.reduce_sum(tf.expand_dims(gamma, -1) * tf.expand_dims(z, 1), 0) / tf.expand_dims(
-        sum_gamma, -1
-    )  # K x D (D = latent_dim)
-    z_mu = tf.expand_dims(z, 1) - tf.expand_dims(mu, 0)  # N x K x D
-    z_mu_outer = tf.expand_dims(z_mu, -1) * tf.expand_dims(z_mu, -2)  # N x K x D x D
-    cov = tf.reduce_sum(tf.expand_dims(tf.expand_dims(gamma, -1), -1) * z_mu_outer, 0) / tf.expand_dims(
-        tf.expand_dims(sum_gamma, -1), -1
-    )  # K x D x D
-
-    # cholesky decomposition of covariance and determinant derivation
-    D = tf.shape(cov)[1]  # type: ignore
-    eps = 1e-6
-    L = tf.linalg.cholesky(cov + tf.eye(D) * eps)  # K x D x D
-    log_det_cov = 2.0 * tf.reduce_sum(tf.math.log(tf.linalg.diag_part(L)), 1)  # K
-
-    return GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
-
-
-def gmm_energy(
-    z: tf.Tensor,
-    params: GaussianMixtureModelParams[tf.Tensor],
-    return_mean: bool = True,
-) -> tuple[tf.Tensor, tf.Tensor]:
-    """
-    Compute sample energy from Gaussian Mixture Model.
-
-    Parameters
-    ----------
-    params : GaussianMixtureModelParams
-        The gaussian mixture model parameters.
-    return_mean : bool, default True
-        Take mean across all sample energies in a batch.
-
-    Returns
-    -------
-    sample_energy
-        The sample energy of the GMM.
-    cov_diag
-        The inverse sum of the diagonal components of the covariance matrix.
-    """
-    D = tf.shape(params.cov)[1]  # type: ignore
-    z_mu = tf.expand_dims(z, 1) - tf.expand_dims(params.mu, 0)  # N x K x D
-    z_mu_T = tf.transpose(z_mu, perm=[1, 2, 0])  # K x D x N
-    v = tf.linalg.triangular_solve(params.L, z_mu_T, lower=True)  # K x D x D
-
-    # rewrite sample energy in logsumexp format for numerical stability
-    logits = tf.math.log(tf.expand_dims(params.phi, -1)) - 0.5 * (
-        tf.reduce_sum(tf.square(v), 1)
-        + tf.cast(D, tf.float32) * tf.math.log(2.0 * np.pi)  # type: ignore py38
-        + tf.expand_dims(params.log_det_cov, -1)
-    )  # K x N
-    sample_energy = -tf.reduce_logsumexp(logits, axis=0)  # N
-
-    if return_mean:
-        sample_energy = tf.reduce_mean(sample_energy)
-
-    # inverse sum of variances
-    cov_diag = tf.reduce_sum(tf.divide(1, tf.linalg.diag_part(params.cov)))
-
-    return sample_energy, cov_diag