dataeval 0.74.0__py3-none-any.whl → 0.74.2__py3-none-any.whl

dataeval/detectors/ood/vae.py

@@ -1,98 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_VAE"]
-
-from typing import TYPE_CHECKING, Callable
-
-import numpy as np
-from numpy.typing import ArrayLike
-
-from dataeval.detectors.ood.base import OODScoreOutput
-from dataeval.detectors.ood.base_tf import OODBase
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.loss import Elbo
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-class OOD_VAE(OODBase):
-    """
-    VAE based outlier detector.
-
-    Parameters
-    ----------
-    model : VAE
-        A VAE model.
-    samples : int, default 10
-        Number of samples sampled to evaluate each instance.
-
-    Examples
-    --------
-    Instantiate an OOD detector metric with a generic dataset - batch of images with shape (3,25,25)
-
-    >>> metric = OOD_VAE(create_model("VAE", dataset[0].shape))
-
-    Adjusting fit parameters,
-    including setting the fit threshold at 85% for a training set with about 15% out-of-distribution
-
-    >>> metric.fit(dataset, threshold_perc=85, batch_size=128, verbose=False)
-
-    Detect :term:`out of distribution<Out-of-Distribution (OOD)>` samples at the 'feature' level
-
-    >>> result = metric.predict(dataset, ood_type="feature")
-    """
-
-    def __init__(self, model: tf_models.VAE, samples: int = 10) -> None:
-        super().__init__(model)
-        self.samples = samples
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = Elbo(0.05),
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-    ) -> None:
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
-
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-
-        # sample reconstructed instances
-        X_samples = np.repeat(X, self.samples, axis=0)
-        X_recon = predict_batch(X_samples, model=self.model, batch_size=batch_size)
-
-        # compute feature scores
-        fscore = np.power(X_samples - X_recon, 2)
-        fscore = fscore.reshape((-1, self.samples) + X_samples.shape[1:])
-        fscore = np.mean(fscore, axis=1)
-
-        # compute instance scores
-        fscore_flat = fscore.reshape(fscore.shape[0], -1).copy()
-        n_score_features = int(np.ceil(fscore_flat.shape[1]))
-        sorted_fscore = np.sort(fscore_flat, axis=1)
-        sorted_fscore_perc = sorted_fscore[:, -n_score_features:]
-        iscore = np.mean(sorted_fscore_perc, axis=1)
-
-        return OODScoreOutput(iscore, fscore)

dataeval/detectors/ood/vaegmm.py

@@ -1,76 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-__all__ = ["OOD_VAEGMM"]
-
-from typing import TYPE_CHECKING, Callable
-
-import numpy as np
-from numpy.typing import ArrayLike
-
-from dataeval.detectors.ood.base import OODScoreOutput
-from dataeval.detectors.ood.base_tf import OODBaseGMM
-from dataeval.interop import to_numpy
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.gmm import gmm_energy
-from dataeval.utils.tensorflow._internal.loss import Elbo, LossGMM
-from dataeval.utils.tensorflow._internal.utils import predict_batch
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tf_keras as keras
-
-    import dataeval.utils.tensorflow._internal.models as tf_models
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    tf_models = lazyload("dataeval.utils.tensorflow._internal.models")
-
-
-class OOD_VAEGMM(OODBaseGMM):
-    """
-    VAE with Gaussian Mixture Model based outlier detector.
-
-    Parameters
-    ----------
-    model : VAEGMM
-        A VAEGMM model.
-    samples
-        Number of samples sampled to evaluate each instance.
-    """
-
-    def __init__(self, model: tf_models.VAEGMM, samples: int = 10) -> None:
-        super().__init__(model)
-        self.samples = samples
-
-    def fit(
-        self,
-        x_ref: ArrayLike,
-        threshold_perc: float = 100.0,
-        loss_fn: Callable[..., tf.Tensor] | None = LossGMM(elbo=Elbo(0.05)),
-        optimizer: keras.optimizers.Optimizer | None = None,
-        epochs: int = 20,
-        batch_size: int = 64,
-        verbose: bool = True,
-    ) -> None:
-        super().fit(x_ref, threshold_perc, loss_fn, optimizer, epochs, batch_size, verbose)
-
-    def _score(self, X: ArrayLike, batch_size: int = int(1e10)) -> OODScoreOutput:
-        self._validate(X := to_numpy(X))
-
-        # draw samples from latent space
-        X_samples = np.repeat(X, self.samples, axis=0)
-        _, z, _ = predict_batch(X_samples, self.model, batch_size=batch_size)
-
-        # compute average energy for samples
-        energy, _ = gmm_energy(z, self._gmm_params, return_mean=False)
-        energy_samples = energy.numpy().reshape((-1, self.samples))  # type: ignore
-        iscore = np.mean(energy_samples, axis=-1)
-        return OODScoreOutput(iscore)

dataeval/utils/lazy.py

@@ -1,26 +0,0 @@
-from __future__ import annotations
-
-from functools import cached_property
-from importlib import import_module
-from typing import Any
-
-
-class LazyModule:
-    def __init__(self, name: str) -> None:
-        self._name = name
-
-    def __getattr__(self, key: str) -> Any:
-        return getattr(self._module, key)
-
-    @cached_property
-    def _module(self):
-        return import_module(self._name)
-
-
-LAZY_MODULES: dict[str, LazyModule] = {}
-
-
-def lazyload(name: str) -> LazyModule:
-    if name not in LAZY_MODULES:
-        LAZY_MODULES[name] = LazyModule(name)
-    return LAZY_MODULES[name]

dataeval/utils/tensorflow/__init__.py

@@ -1,19 +0,0 @@
-"""
-TensorFlow models are used in :term:`out of distribution<Out-of-distribution (OOD)>` detectors in the
-:mod:`dataeval.detectors.ood` module.
-
-DataEval provides basic default models through the utility :func:`dataeval.utils.tensorflow.create_model`.
-"""
-
-from dataeval import _IS_TENSORFLOW_AVAILABLE
-
-__all__ = []
-
-
-if _IS_TENSORFLOW_AVAILABLE:
-    import dataeval.utils.tensorflow.loss as loss
-    from dataeval.utils.tensorflow._internal.utils import create_model
-
-    __all__ = ["create_model", "loss"]
-
-del _IS_TENSORFLOW_AVAILABLE

dataeval/utils/tensorflow/_internal/gmm.py

@@ -1,103 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import numpy as np
-
-from dataeval.utils.gmm import GaussianMixtureModelParams
-from dataeval.utils.lazy import lazyload
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-else:
-    tf = lazyload("tensorflow")
-
-
-def gmm_params(z: tf.Tensor, gamma: tf.Tensor) -> GaussianMixtureModelParams[tf.Tensor]:
-    """
-    Compute parameters of Gaussian Mixture Model.
-
-    Parameters
-    ----------
-    z : tf.Tensor
-        Observations.
-    gamma : tf.Tensor
-        Mixture probabilities to derive mixture distribution weights from.
-
-    Returns
-    -------
-    GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
-        The parameters used to calculate energy.
-    """
-    # compute gmm parameters phi, mu and cov
-    N = gamma.shape[0]  # nb of samples in batch
-    sum_gamma = tf.reduce_sum(gamma, 0)  # K
-    phi = sum_gamma / N  # K
-    mu = tf.reduce_sum(tf.expand_dims(gamma, -1) * tf.expand_dims(z, 1), 0) / tf.expand_dims(
-        sum_gamma, -1
-    )  # K x D (D = latent_dim)
-    z_mu = tf.expand_dims(z, 1) - tf.expand_dims(mu, 0)  # N x K x D
-    z_mu_outer = tf.expand_dims(z_mu, -1) * tf.expand_dims(z_mu, -2)  # N x K x D x D
-    cov = tf.reduce_sum(tf.expand_dims(tf.expand_dims(gamma, -1), -1) * z_mu_outer, 0) / tf.expand_dims(
-        tf.expand_dims(sum_gamma, -1), -1
-    )  # K x D x D
-
-    # cholesky decomposition of covariance and determinant derivation
-    D = tf.shape(cov)[1]  # type: ignore
-    eps = 1e-6
-    L = tf.linalg.cholesky(cov + tf.eye(D) * eps)  # K x D x D
-    log_det_cov = 2.0 * tf.reduce_sum(tf.math.log(tf.linalg.diag_part(L)), 1)  # K
-
-    return GaussianMixtureModelParams(phi, mu, cov, L, log_det_cov)
-
-
-def gmm_energy(
-    z: tf.Tensor,
-    params: GaussianMixtureModelParams[tf.Tensor],
-    return_mean: bool = True,
-) -> tuple[tf.Tensor, tf.Tensor]:
-    """
-    Compute sample energy from Gaussian Mixture Model.
-
-    Parameters
-    ----------
-    params : GaussianMixtureModelParams
-        The gaussian mixture model parameters.
-    return_mean : bool, default True
-        Take mean across all sample energies in a batch.
-
-    Returns
-    -------
-    sample_energy
-        The sample energy of the GMM.
-    cov_diag
-        The inverse sum of the diagonal components of the covariance matrix.
-    """
-    D = tf.shape(params.cov)[1]  # type: ignore
-    z_mu = tf.expand_dims(z, 1) - tf.expand_dims(params.mu, 0)  # N x K x D
-    z_mu_T = tf.transpose(z_mu, perm=[1, 2, 0])  # K x D x N
-    v = tf.linalg.triangular_solve(params.L, z_mu_T, lower=True)  # K x D x D
-
-    # rewrite sample energy in logsumexp format for numerical stability
-    logits = tf.math.log(tf.expand_dims(params.phi, -1)) - 0.5 * (
-        tf.reduce_sum(tf.square(v), 1)
-        + tf.cast(D, tf.float32) * tf.math.log(2.0 * np.pi)  # type: ignore py38
-        + tf.expand_dims(params.log_det_cov, -1)
-    )  # K x N
-    sample_energy = -tf.reduce_logsumexp(logits, axis=0)  # N
-
-    if return_mean:
-        sample_energy = tf.reduce_mean(sample_energy)
-
-    # inverse sum of variances
-    cov_diag = tf.reduce_sum(tf.divide(1, tf.linalg.diag_part(params.cov)))
-
-    return sample_energy, cov_diag

dataeval/utils/tensorflow/_internal/loss.py

@@ -1,121 +0,0 @@
-"""
-Source code derived from Alibi-Detect 0.11.4
-https://github.com/SeldonIO/alibi-detect/tree/v0.11.4
-
-Original code Copyright (c) 2023 Seldon Technologies Ltd
-Licensed under Apache Software License (Apache 2.0)
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Literal, cast
-
-import numpy as np
-from numpy.typing import NDArray
-
-from dataeval.utils.lazy import lazyload
-from dataeval.utils.tensorflow._internal.gmm import gmm_energy, gmm_params
-
-if TYPE_CHECKING:
-    import tensorflow as tf
-    import tensorflow_probability.python.distributions.mvn_diag as mvn_diag
-    import tensorflow_probability.python.distributions.mvn_tril as mvn_tril
-    import tensorflow_probability.python.stats as tfp_stats
-    import tf_keras as keras
-else:
-    tf = lazyload("tensorflow")
-    keras = lazyload("tf_keras")
-    mvn_diag = lazyload("tensorflow_probability.python.distributions.mvn_diag")
-    mvn_tril = lazyload("tensorflow_probability.python.distributions.mvn_tril")
-    tfp_stats = lazyload("tensorflow_probability.python.stats")
-
-
-class Elbo:
-    """
-    Compute ELBO loss.
-
-    The covariance matrix can be specified by passing the full covariance matrix, the matrix
-    diagonal, or a scale identity multiplier. Only one of these should be specified. If none are specified, the
-    identity matrix is used.
-
-    Parameters
-    ----------
-    cov_type : Union[Literal["cov_full", "cov_diag"], float], default 1.0
-        Full covariance matrix, diagonal :term:`variance<Variance>` matrix, or scale identity multiplier.
-    x : ArrayLike, optional - default None
-        Dataset used to calculate the covariance matrix.  Required for full and diagonal covariance matrix types.
-    """
-
-    def __init__(
-        self,
-        cov_type: Literal["cov_full", "cov_diag"] | float = 1.0,
-        x: tf.Tensor | NDArray[np.float32] | None = None,
-    ):
-        if isinstance(cov_type, float):
-            self._cov = ("sim", cov_type)
-        elif cov_type in ["cov_full", "cov_diag"]:
-            x_np: NDArray[np.float32] = x.numpy().astype(np.float32) if tf.is_tensor(x) else x  # type: ignore
-            cov = tfp_stats.covariance(x_np.reshape(x_np.shape[0], -1))  # type: ignore py38
-            if cov_type == "cov_diag":  # infer standard deviation from covariance matrix
-                cov = tf.math.sqrt(tf.linalg.diag_part(cov))
-            self._cov = (cov_type, cov)
-        else:
-            raise ValueError("Only cov_full, cov_diag or sim value should be specified.")
-
-    def __call__(self, y_true: tf.Tensor, y_pred: tf.Tensor) -> tf.Tensor:
-        y_pred_flat = cast(tf.Tensor, keras.layers.Flatten()(y_pred))
-
-        if self._cov[0] == "cov_full":
-            y_mn = mvn_tril.MultivariateNormalTriL(y_pred_flat, scale_tril=tf.linalg.cholesky(self._cov[1]))
-        else:  # cov_diag and sim
-            cov_diag = self._cov[1] if self._cov[0] == "cov_diag" else self._cov[1] * tf.ones(y_pred_flat.shape[-1])
-            y_mn = mvn_diag.MultivariateNormalDiag(y_pred_flat, scale_diag=cov_diag)
-
-        loss = -tf.reduce_mean(y_mn.log_prob(keras.layers.Flatten()(y_true)))
-        return loss
-
-
-class LossGMM:
-    """
-    Loss function used for AE and VAE with GMM.
-
-    Parameters
-    ----------
-    w_recon : float, default 1e-7
-        Weight on elbo loss term.
-    w_energy : float, default 0.1
-        Weight on sample energy loss term.
-    w_cov_diag : float, default 0.005
-        Weight on covariance regularizing loss term.
-    elbo : Elbo, optional - default None
-        ELBO loss function used to calculate w_recon.
-    """
-
-    def __init__(
-        self,
-        w_recon: float = 1e-7,
-        w_energy: float = 0.1,
-        w_cov_diag: float = 0.005,
-        elbo: Elbo | None = None,
-    ):
-        self.w_recon = w_recon
-        self.w_energy = w_energy
-        self.w_cov_diag = w_cov_diag
-        self.elbo = elbo
-
-    def __call__(
-        self,
-        x_true: tf.Tensor,
-        x_pred: tf.Tensor,
-        z: tf.Tensor,
-        gamma: tf.Tensor,
-    ) -> tf.Tensor:
-        w_recon = (
-            tf.reduce_mean(tf.subtract(x_true, x_pred) ** 2)
-            if self.elbo is None
-            else tf.multiply(self.w_recon, self.elbo(x_true, x_pred))
-        )
-        sample_energy, cov_diag = gmm_energy(z, gmm_params(z, gamma))
-        w_energy = tf.multiply(self.w_energy, sample_energy)
-        w_cov_diag = tf.multiply(self.w_cov_diag, cov_diag)
-        return w_recon + w_energy + w_cov_diag