oodeel 0.4.0__py3-none-any.whl

oodeel/methods/gram.py

@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+import numpy as np
+from sklearn.model_selection import train_test_split
+
+from ..aggregator import BaseAggregator
+from ..types import DatasetType
+from ..types import TensorType
+from .base import FeatureBasedDetector
+
+EPSILON = 1e-6  # Numerical stability constant
+
+
+class Gram(FeatureBasedDetector):
+    r"""
+    "Detecting Out-of-Distribution Examples with Gram Matrices"
+    [link](https://proceedings.mlr.press/v119/sastry20a.html)
+
+    **Important Disclaimer**: Taking the statistics of min/max deviation, as in the
+    paper, raises some problems. The method may yield a score of zero for some tasks
+    because the sample extreme values become more extreme with larger sample sizes.
+    To mitigate this, we replace the min/max with the q / (1-q) quantile threshold,
+    where q is a parameter that controls the discriminative ability of the method.
+
+    This approach improved baseline performance in our experiments.
+
+    Args:
+        orders (Union[List[int], int]): Power orders to consider for the correlation
+            matrix. If an int is provided, it is converted to a list.
+        quantile (float): Quantile to consider for the correlations to build the
+            deviation threshold.
+        aggregator (Optional[BaseAggregator]): Aggregator to combine multi-layer scores.
+            If multiple layers are used and no aggregator is provided,
+            StdNormalizedAggregator is used by default. Defaults to None.
+    """
+
+    def __init__(
+        self,
+        orders: Union[List[int], int] = list(range(1, 6)),
+        quantile: float = 0.01,
+        aggregator: Optional[BaseAggregator] = None,
+        **kwargs,
+    ):
+        super().__init__(aggregator=aggregator, **kwargs)
+        if isinstance(orders, int):
+            orders = [orders]
+        self.orders: List[int] = orders
+        self.quantile = quantile
+
+        self.postproc_fns = None  # Will be set during fit
+        # Mapping class -> list (per-layer) of thresholds [lower, upper]
+        self.min_maxs: Dict[int, List[TensorType]] = {}
+
+    # === Public API (override of _fit_to_dataset) ===
+    def _fit_to_dataset(
+        self,
+        fit_dataset: DatasetType,
+        verbose: bool = False,
+        **kwargs,
+    ) -> None:
+        """Fit thresholds on Gram statistics from a dataset.
+
+        This method sets :attr:`postproc_fns` to compute Gram matrices for all
+        selected feature layers and then delegates the actual fitting to the
+        generic implementation in :class:`OODBaseDetector`.
+
+        Args:
+            fit_dataset: Dataset containing in-distribution samples.
+            verbose: Whether to display a progress bar during feature extraction.
+            **kwargs: Additional keyword arguments forwarded to :func:`_fit_layer`.
+
+        Returns:
+            None
+        """
+        n_layers = len(self.feature_extractor.feature_layers_id)
+        if self.postproc_fns is None:
+            self.postproc_fns = [self._stat] * n_layers
+
+        super()._fit_to_dataset(fit_dataset, verbose=verbose, **kwargs)
+
+    # === Per-layer logic ===
+    def _fit_layer(
+        self,
+        layer_id: int,
+        layer_stats: np.ndarray,
+        info: dict,
+        val_split: float = None,
+        **kwargs,
+    ) -> None:
+        """Fit thresholds for one layer and store validation data.
+
+        Args:
+            layer_id: Index of the processed layer.
+            layer_stats: Gram statistics for this layer.
+            info: Dictionary containing the logits of the training data.
+            val_split: Ratio of samples used for aggregator fitting.
+        """
+
+        preds_all = np.argmax(info["logits"], axis=1)
+
+        # initialize min_maxs if not already done.
+        if not self.min_maxs:
+            n_layers = len(self.feature_extractor.feature_layers_id)
+            self._classes = np.sort(np.unique(preds_all)).tolist()
+            self.min_maxs = {cls: [None] * n_layers for cls in self._classes}
+
+        # split the dataset into training and validation sets (as in original paper).
+        idx_all = np.arange(preds_all.shape[0])
+        train_idx, val_idx = (
+            train_test_split(idx_all, test_size=val_split, random_state=42)
+            if val_split is not None
+            else (idx_all, idx_all)
+        )
+
+        train_stats = layer_stats[train_idx]
+        val_stats = layer_stats[val_idx]
+        train_preds = preds_all[train_idx]
+        val_preds = preds_all[val_idx]
+
+        # compute min/max thresholds for each class
+        for cls in self._classes:
+            cls_mask = train_preds == cls
+            stats_cls_np = train_stats[cls_mask]
+            stats_cls_t = self.op.from_numpy(stats_cls_np)
+            lower = self.op.quantile(stats_cls_t, self.quantile, dim=0)
+            upper = self.op.quantile(stats_cls_t, 1 - self.quantile, dim=0)
+            self.min_maxs[cls][layer_id] = self.op.cat(
+                [self.op.unsqueeze(lower, -1), self.op.unsqueeze(upper, -1)],
+                dim=-1,
+            )
+
+        if getattr(self, "aggregator", None) is not None:
+            # Store validation data for the aggregator.
+            if not hasattr(self, "_val_stats"):
+                self._val_stats = []
+                self._val_preds = []
+            while len(self._val_stats) <= layer_id:
+                self._val_stats.append(None)
+                self._val_preds.append(None)
+            self._val_stats[layer_id] = val_stats
+            self._val_preds[layer_id] = val_preds
+
+    def _score_layer(
+        self,
+        layer_id: int,
+        layer_stats: TensorType,
+        info: dict,
+        fit: bool = False,
+        **kwargs,
+    ) -> np.ndarray:
+        """Score inputs for a single layer.
+
+        Args:
+            layer_id (int): Layer index.
+            layer_stats (TensorType): Gram stats.
+            info (dict): Dictionary containing auxiliary data, such as logits.
+            fit (bool): Whether scoring is performed during fitting. If `True`
+                the validation subset stored by :func:`_fit_layer` is used.
+
+        Returns:
+            np.ndarray: Deviation-based OOD scores.
+        """
+        if fit and hasattr(self, "_val_stats"):
+            layer_stats = self.op.from_numpy(self._val_stats[layer_id])
+            preds = self._val_preds[layer_id]
+        else:
+            preds = np.argmax(self.op.convert_to_numpy(info["logits"]), axis=1)
+
+        thr_batch = self.op.stack([self.min_maxs[int(lbl)][layer_id] for lbl in preds])
+        dev = self._deviation(layer_stats, thr_batch)
+        return self.op.convert_to_numpy(dev)
+
+    # === Internal utilities ===
+    def _stat(self, feature_map: TensorType) -> TensorType:
+        """Compute Gram statistics for a single layer.
+
+        Args:
+            feature_map (TensorType): Feature map of shape `[B, ...]`.
+
+        Returns:
+            TensorType: Statistics of shape `[B, n_orders, C]`.
+        """
+        fm_shape = feature_map.shape
+        stats = []
+        for p in self.orders:
+            # Raise the feature map to the specified order.
+            fm_p = feature_map**p
+            if len(fm_shape) == 2:
+                # Dense layers: compute outer product.
+                fm_p = self.op.einsum("bi,bj->bij", fm_p, fm_p)
+            else:
+                # Convolutional feature maps: flatten spatial dimensions.
+                if self.backend == "tensorflow":
+                    fm_p = self.op.reshape(
+                        self.op.einsum("i...j->ij...", fm_p),
+                        (fm_shape[0], fm_shape[-1], -1),
+                    )
+                else:
+                    fm_p = self.op.reshape(fm_p, (fm_shape[0], fm_shape[1], -1))
+                fm_p = self.op.matmul(fm_p, self.op.permute(fm_p, (0, 2, 1)))
+            # Normalize and recover the original power.
+            fm_p = self.op.sign(fm_p) * (self.op.abs(fm_p) ** (1 / p))
+            # Use only the lower triangular part.
+            fm_p = self.op.tril(fm_p)
+            # Aggregate row-wise.
+            fm_p = self.op.sum(fm_p, dim=2)
+            stats.append(fm_p)
+        return self.op.stack(stats, dim=1)
+
+    def _deviation(self, stats: TensorType, thresholds: TensorType) -> TensorType:
+        """Compute deviation of `stats` outside `thresholds`.
+
+        Args:
+            stats (TensorType): Gram stats, shape `[B, *, C]`.
+            thresholds (TensorType): Lower & upper bounds, shape `[B, *, C, 2]`.
+
+        Returns:
+            TensorType: Deviation values, shape `[B]`.
+        """
+        below = self.op.where(stats < thresholds[..., 0], 1.0, 0.0)
+        above = self.op.where(stats > thresholds[..., 1], 1.0, 0.0)
+        dev_low = (
+            (thresholds[..., 0] - stats) / (self.op.abs(thresholds[..., 0]) + EPSILON)
+        ) * below
+        dev_high = (
+            (stats - thresholds[..., 1]) / (self.op.abs(thresholds[..., 1]) + EPSILON)
+        ) * above
+        return self.op.sum(dev_low + dev_high, dim=(1, 2))
+
+    # === Properties ===
+    @property
+    def requires_to_fit_dataset(self) -> bool:
+        """
+        Indicates whether this OOD detector requires in-distribution data for fitting.
+
+        Returns:
+            bool: True, since fitting requires computing class-conditional statistics.
+        """
+        return True
+
+    @property
+    def requires_internal_features(self) -> bool:
+        """
+        Indicates whether this OOD detector utilizes internal model features.
+
+        Returns:
+            bool: True, as it operates on intermediate feature representations.
+        """
+        return True

oodeel/methods/mahalanobis.py

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+import numpy as np
+
+from ..aggregator import BaseAggregator
+from ..types import TensorType
+from .base import FeatureBasedDetector
+
+
+class Mahalanobis(FeatureBasedDetector):
+    """
+    "A Simple Unified Framework for Detecting Out-of-Distribution Samples and
+    Adversarial Attacks"
+    https://arxiv.org/abs/1807.03888
+
+    This detector computes the Mahalanobis distance between the feature representations
+    of input samples and class-conditional Gaussian distributions estimated from
+    in-distribution data. It supports multiple feature layers by computing statistics
+    (class means and a covariance matrix) for each layer. During inference, scores
+    computed for each layer are aggregated using a provided aggregator.
+
+    Args:
+        eps (float): Magnitude for gradient-based input perturbation. Defaults to
+            0.0014.
+        temperature (float, optional): Temperature parameter. Defaults to 1000.
+        aggregator (Optional[BaseAggregator]): Aggregator to combine scores from
+            multiple feature layers. If `None` and more than one layer is
+            used, a `StdNormalizedAggregator` is instantiated automatically.
+    """
+
+    def __init__(
+        self,
+        eps: float = 0.0014,
+        temperature: float = 1000,
+        aggregator: Optional[BaseAggregator] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            eps=eps, temperature=temperature, aggregator=aggregator, **kwargs
+        )
+        self._layer_stats: List[Tuple[Dict, np.ndarray]] = []
+        self._classes: Optional[np.ndarray] = None
+
+    # === Per-layer logic ===
+    def _fit_layer(
+        self,
+        layer_id: int,
+        layer_features: np.ndarray,
+        info: dict,
+        **kwargs,
+    ) -> None:
+        """Compute class statistics for one feature layer.
+
+        Args:
+            layer_id: Index of the processed layer.
+            layer_features: In-distribution features for this layer.
+            info: Dictionary containing the training labels.
+        """
+        labels = info["labels"]
+
+        if isinstance(layer_features, np.ndarray):
+            layer_features = self.op.from_numpy(layer_features)
+
+        mus, pinv_cov = self._compute_layer_stats(layer_features, labels)
+
+        self._layer_stats.append((mus, pinv_cov))
+
+    def _score_layer(
+        self,
+        layer_id: int,
+        layer_features: TensorType,
+        info: dict,
+        fit: bool = False,
+        **kwargs,
+    ) -> np.ndarray:
+        """Compute Mahalanobis confidence for one feature layer.
+
+        Args:
+            layer_id: Index of the processed layer.
+            layer_features: Feature tensor for the current batch.
+            info: Unused dictionary of auxiliary data.
+            fit: Whether scoring is performed during fitting. Unused here.
+
+        Returns:
+            np.ndarray: Negative Mahalanobis confidence scores.
+        """
+        mus, pinv_cov = self._layer_stats[layer_id]
+        feats = self.op.flatten(layer_features)
+        g_scores = self._gaussian_log_probs(feats, mus, pinv_cov)
+        max_score = self.op.max(g_scores, dim=1)
+        return -self.op.convert_to_numpy(max_score)
+
+    # === Internal utilities ===
+    def _compute_layer_stats(
+        self, layer_features: TensorType, labels: np.ndarray
+    ) -> Tuple[Dict[int, TensorType], np.ndarray]:
+        """
+        Compute class-conditional statistics for a given feature layer.
+
+        For each class present in the labels, this method computes the mean feature
+        vector. It also computes a weighted average covariance matrix (across all
+        classes) and its pseudo-inverse, which will later be used for computing
+        Mahalanobis distances.
+
+        Args:
+            layer_features (TensorType): Feature tensor for a specific layer extracted
+                from in-distribution data.
+            labels (np.ndarray): Corresponding labels for the in-distribution data.
+
+        Returns:
+            Tuple[Dict, TensorType]:
+                - A dictionary mapping each class label to its mean feature vector.
+                - The pseudo-inverse of the weighted average covariance matrix.
+        """
+        classes = np.sort(np.unique(labels))
+        labels = self.op.from_numpy(labels)  # convert to tensor
+
+        feats = self.op.flatten(layer_features)
+        n_total = feats.shape[0]
+
+        mus: Dict[int, TensorType] = {}
+        mean_cov: TensorType = None
+
+        for cls in classes:
+            idx = self.op.equal(labels, cls)
+            feats_cls = self.op.flatten(layer_features[idx])
+            mu = self.op.mean(feats_cls, dim=0)
+            mus[cls] = mu
+
+            zero_f = feats_cls - mu
+            cov_cls = self.op.matmul(self.op.t(zero_f), zero_f) / zero_f.shape[0]
+            weight = feats_cls.shape[0] / n_total
+            mean_cov = (
+                cov_cls * weight if mean_cov is None else mean_cov + cov_cls * weight
+            )
+
+        pinv_cov = self.op.pinv(mean_cov)  # type: ignore[arg-type]
+        if self._classes is None:
+            self._classes = classes
+        return mus, pinv_cov
+
+    def _gaussian_log_probs(
+        self, out_features: TensorType, mus: Dict[int, TensorType], pinv_cov: TensorType
+    ) -> TensorType:
+        """Compute unnormalised Gaussian log-probabilities for all classes.
+
+        Args:
+            out_features (TensorType): Features of shape [B, D].
+            mus (Dict[int, TensorType]): Class mean vectors.
+            pinv_cov (TensorType): Pseudo-inverse covariance matrix.
+
+        Returns:
+            TensorType: Log-probabilities with shape [B, n_classes].
+        """
+        scores = []
+        for cls in self._classes:  # type: ignore[assignment]
+            mu = mus[cls]
+            zero_f = out_features - mu
+            log_prob = -0.5 * self.op.diag(
+                self.op.matmul(self.op.matmul(zero_f, pinv_cov), self.op.t(zero_f))
+            )
+            scores.append(self.op.reshape(log_prob, (-1, 1)))
+        return self.op.cat(scores, dim=1)
+
+    # ===
+    @property
+    def requires_to_fit_dataset(self) -> bool:
+        """
+        Indicates whether this OOD detector requires in-distribution data for fitting.
+
+        Returns:
+            bool: True, since fitting requires computing class-conditional statistics.
+        """
+        return True
+
+    @property
+    def requires_internal_features(self) -> bool:
+        """
+        Indicates whether this OOD detector utilizes internal model features.
+
+        Returns:
+            bool: True, as it operates on intermediate feature representations.
+        """
+        return True

oodeel/methods/mls.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import numpy as np
+
+from ..types import TensorType
+from ..types import Tuple
+from .base import OODBaseDetector
+
+
+class MLS(OODBaseDetector):
+    """
+    Maximum Logit Scores method for OOD detection.
+    "Open-Set Recognition: a Good Closed-Set Classifier is All You Need?"
+    https://arxiv.org/abs/2110.06207,
+    and Maximum Softmax Score
+    "A Baseline for Detecting Misclassified and Out-of-Distribution Examples
+    in Neural Networks"
+    http://arxiv.org/abs/1610.02136
+
+    Args:
+        output_activation (str): activation function for the last layer. If "linear",
+            the method is MLS and if "softmax", the method is MSS.
+            Defaults to "linear".
+        use_react (bool): if true, apply ReAct method by clipping penultimate
+            activations under a threshold value.
+        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
+            the react clipping threshold defined as the q-th quantile penultimate layer
+            activations. Defaults to 0.8.
+    """
+
+    def __init__(
+        self,
+        output_activation: str = "linear",
+        use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
+        react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
+        **kwargs,
+    ):
+        super().__init__(
+            use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
+            react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
+        )
+        self.output_activation = output_activation
+
+    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
+        """
+        Computes an OOD score for input samples "inputs" based on
+        the distance to nearest neighbors in the feature space of self.model
+
+        Args:
+            inputs: input samples to score
+
+        Returns:
+            Tuple[np.ndarray]: scores, logits
+        """
+        # optional: apply input perturbation
+        if self.eps > 0:
+            inputs = self._input_perturbation(inputs, self.eps, self.temperature)
+
+        _, logits = self.feature_extractor.predict_tensor(inputs)
+        if self.output_activation == "softmax":
+            logits = self.op.softmax(logits)
+        logits = self.op.convert_to_numpy(logits)
+        scores = -np.max(logits, axis=1)
+        return scores
+
+    @property
+    def requires_to_fit_dataset(self) -> bool:
+        """
+        Whether an OOD detector needs a `fit_dataset` argument in the fit function.
+
+        Returns:
+            bool: True if `fit_dataset` is required else False.
+        """
+        return False
+
+    @property
+    def requires_internal_features(self) -> bool:
+        """
+        Whether an OOD detector acts on internal model features.
+
+        Returns:
+            bool: True if the detector perform computations on an intermediate layer
+            else False.
+        """
+        return False