oodeel 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

oodeel/methods/base.py

@@ -20,11 +20,13 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import inspect
 from abc import ABC
 from abc import abstractmethod
 from typing import get_args
 
 import numpy as np
+from tqdm import tqdm
 
 from ..extractor.feature_extractor import FeatureExtractor
 from ..types import Callable
@@ -38,25 +40,65 @@ from ..utils import import_backend_specific_stuff
 
 
 class OODBaseDetector(ABC):
-    """Base Class for methods that assign a score to unseen samples.
-
-    Args:
-        use_react (bool): if true, apply ReAct method by clipping penultimate
-            activations under a threshold value.
-        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
-            the react clipping threshold defined as the q-th quantile penultimate layer
-            activations. Defaults to 0.8.
+    """OODBaseDetector is an abstract base class for Out-of-Distribution (OOD)
+    detection.
+
+    Attributes:
+        feature_extractor (FeatureExtractor): The feature extractor instance.
+        use_react (bool): Flag to indicate if ReAct method is used.
+        use_scale (bool): Flag to indicate if scaling method is used.
+        use_ash (bool): Flag to indicate if ASH method is used.
+        react_quantile (float): Quantile value for ReAct threshold.
+        scale_percentile (float): Percentile value for scaling.
+        ash_percentile (float): Percentile value for ASH.
+        react_threshold (float): Threshold value for ReAct.
+        postproc_fns (List[Callable]): List of post-processing functions.
+
+    Methods:
+        __init__: Initializes the OODBaseDetector with specified parameters.
+        _score_tensor: Abstract method to compute OOD score for input samples.
+        _sanitize_posproc_fns: Sanitizes post-processing functions used at each layer
+        output.
+        fit: Prepares the detector for scoring by constructing the feature extractor
+        and calibrating on ID data.
+        _load_feature_extractor: Loads the feature extractor based on the model and
+        specified layers.
+        _fit_to_dataset: Abstract method to fit the OOD detector to a dataset.
+        score: Computes an OOD score for input samples.
+        compute_react_threshold: Computes the ReAct threshold using the fit dataset.
+        __call__: Convenience wrapper for the score method.
+        requires_to_fit_dataset: Property indicating if the detector needs a fit
+        dataset.
+        requires_internal_features: Property indicating if the detector acts on
+        internal model features.
     """
 
     def __init__(
         self,
-        use_react: bool = False,
-        react_quantile: float = 0.8,
+        use_react: Optional[bool] = False,
+        use_scale: Optional[bool] = False,
+        use_ash: Optional[bool] = False,
+        react_quantile: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
+        postproc_fns: Optional[List[Callable]] = None,
     ):
         self.feature_extractor: FeatureExtractor = None
         self.use_react = use_react
+        self.use_scale = use_scale
+        self.use_ash = use_ash
         self.react_quantile = react_quantile
+        self.scale_percentile = scale_percentile
+        self.ash_percentile = ash_percentile
         self.react_threshold = None
+        self.postproc_fns = self._sanitize_posproc_fns(postproc_fns)
+
+        if use_scale and use_react:
+            raise ValueError("Cannot use both ReAct and scale at the same time")
+        if use_scale and use_ash:
+            raise ValueError("Cannot use both ASH and scale at the same time")
+        if use_ash and use_react:
+            raise ValueError("Cannot use both ReAct and ASH at the same time")
 
     @abstractmethod
     def _score_tensor(self, inputs: TensorType) -> np.ndarray:
@@ -66,18 +108,44 @@ class OODBaseDetector(ABC):
 
         Args:
             inputs (TensorType): tensor to score
-
         Returns:
             Tuple[TensorType]: OOD scores, predicted logits
         """
         raise NotImplementedError()
 
+    def _sanitize_posproc_fns(
+        self,
+        postproc_fns: Union[List[Callable], None],
+    ) -> List[Callable]:
+        """Sanitize postproc fns used at each layer output of the feature extractor.
+
+        Args:
+            postproc_fns (Optional[List[Callable]], optional): List of postproc
+                functions, one per output layer. Defaults to None.
+
+        Returns:
+            List[Callable]: Sanitized postproc_fns list
+        """
+        if postproc_fns is not None:
+            assert len(postproc_fns) == len(
+                self.output_layers_id
+            ), "len of postproc_fns and output_layers_id must match"
+
+            def identity(x):
+                return x
+
+            postproc_fns = [identity if fn is None else fn for fn in postproc_fns]
+
+        return postproc_fns
+
     def fit(
         self,
         model: Callable,
         fit_dataset: Optional[Union[ItemType, DatasetType]] = None,
         feature_layers_id: List[Union[int, str]] = [],
         input_layer_id: Optional[Union[int, str]] = None,
+        verbose: bool = False,
+        **kwargs,
     ) -> None:
         """Prepare the detector for scoring:
         * Constructs the feature extractor based on the model
@@ -95,6 +163,7 @@ class OODBaseDetector(ABC):
                 layer of the feature extractor.
                 If int, the rank of the layer in the layer list
                 If str, the name of the layer. Defaults to None.
+            verbose (bool): if True, display a progress bar. Defaults to False.
         """
         (
             self.backend,
@@ -117,7 +186,7 @@ class OODBaseDetector(ABC):
                     " provided to compute react activation threshold"
                 )
             else:
-                self.compute_react_threshold(model, fit_dataset)
+                self.compute_react_threshold(model, fit_dataset, verbose=verbose)
 
         if (feature_layers_id == []) and (self.requires_internal_features):
             raise ValueError(
@@ -133,7 +202,9 @@ class OODBaseDetector(ABC):
         )
 
         if fit_dataset is not None:
-            self._fit_to_dataset(fit_dataset)
+            if "verbose" in inspect.signature(self._fit_to_dataset).parameters.keys():
+                kwargs.update({"verbose": verbose})
+            self._fit_to_dataset(fit_dataset, **kwargs)
 
     def _load_feature_extractor(
         self,
@@ -158,11 +229,18 @@ class OODBaseDetector(ABC):
         Returns:
             FeatureExtractor: a feature extractor instance
         """
+        if not self.use_ash:
+            self.ash_percentile = None
+        if not self.use_scale:
+            self.scale_percentile = None
+
         feature_extractor = self.FeatureExtractorClass(
             model,
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=self.react_threshold,
+            scale_percentile=self.scale_percentile,
+            ash_percentile=self.ash_percentile,
         )
         return feature_extractor
 
@@ -180,12 +258,14 @@ class OODBaseDetector(ABC):
     def score(
         self,
         dataset: Union[ItemType, DatasetType],
+        verbose: bool = False,
     ) -> np.ndarray:
         """
         Computes an OOD score for input samples "inputs".
 
         Args:
             dataset (Union[ItemType, DatasetType]): dataset or tensors to score
+            verbose (bool): if True, display a progress bar. Defaults to False.
 
         Returns:
             tuple: scores or list of scores (depending on the input) and a dictionary
@@ -209,7 +289,7 @@ class OODBaseDetector(ABC):
             scores = np.array([])
             logits = None
 
-            for item in dataset:
+            for item in tqdm(dataset, desc="Scoring", disable=not verbose):
                 tensor = self.data_handler.get_input_from_dataset_item(item)
                 score_batch = self._score_tensor(tensor)
                 logits_batch = self.op.convert_to_numpy(
@@ -240,10 +320,16 @@ class OODBaseDetector(ABC):
         info = dict(labels=labels, logits=logits)
         return scores, info
 
-    def compute_react_threshold(self, model: Callable, fit_dataset: DatasetType):
+    def compute_react_threshold(
+        self, model: Callable, fit_dataset: DatasetType, verbose: bool = False
+    ):
         penult_feat_extractor = self._load_feature_extractor(model, [-2])
-        unclipped_features, _ = penult_feat_extractor.predict(fit_dataset)
-        self.react_threshold = self.op.quantile(unclipped_features, self.react_quantile)
+        unclipped_features, _ = penult_feat_extractor.predict(
+            fit_dataset, verbose=verbose
+        )
+        self.react_threshold = self.op.quantile(
+            unclipped_features[0], self.react_quantile
+        )
 
     def __call__(self, inputs: Union[ItemType, DatasetType]) -> np.ndarray:
         """

oodeel/methods/dknn.py

@@ -38,30 +38,43 @@ class DKNN(OODBaseDetector):
     Args:
         nearest: number of nearest neighbors to consider.
             Defaults to 1.
+        use_gpu (bool): Flag to enable GPU acceleration for FAISS. Defaults to False.
     """
 
-    def __init__(
-        self,
-        nearest: int = 1,
-    ):
+    def __init__(self, nearest: int = 50, use_gpu: bool = False):
         super().__init__()
-
         self.index = None
         self.nearest = nearest
+        self.use_gpu = use_gpu
+
+        if self.use_gpu:
+            try:
+                self.res = faiss.StandardGpuResources()
+            except AttributeError as e:
+                raise ImportError(
+                    "faiss-gpu is not installed, but use_gpu was set to True."
+                    + "Please install faiss-gpu or set use_gpu to False."
+                ) from e
 
     def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
         """
         Constructs the index from ID data "fit_dataset", which will be used for
-        nearest neighbor search.
+        nearest neighbor search. Can operate on CPU or GPU based on the `use_gpu` flag.
 
         Args:
             fit_dataset: input dataset (ID) to construct the index with.
         """
         fit_projected, _ = self.feature_extractor.predict(fit_dataset)
-        fit_projected = self.op.convert_to_numpy(fit_projected)
+        fit_projected = self.op.convert_to_numpy(fit_projected[0])
         fit_projected = fit_projected.reshape(fit_projected.shape[0], -1)
         norm_fit_projected = self._l2_normalization(fit_projected)
-        self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+
+        if self.use_gpu:
+            cpu_index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+            self.index = faiss.index_cpu_to_gpu(self.res, 0, cpu_index)
+        else:
+            self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+
         self.index.add(norm_fit_projected)
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
@@ -77,7 +90,7 @@ class DKNN(OODBaseDetector):
         """
 
         input_projected, _ = self.feature_extractor.predict_tensor(inputs)
-        input_projected = self.op.convert_to_numpy(input_projected)
+        input_projected = self.op.convert_to_numpy(input_projected[0])
         input_projected = input_projected.reshape(input_projected.shape[0], -1)
         norm_input_projected = self._l2_normalization(input_projected)
         scores, _ = self.index.search(norm_input_projected, self.nearest)

oodeel/methods/energy.py

@@ -59,11 +59,19 @@ class Energy(OODBaseDetector):
     def __init__(
         self,
         use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
         react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
     ):
         super().__init__(
             use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
             react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:

oodeel/methods/entropy.py

@@ -52,11 +52,19 @@ class Entropy(OODBaseDetector):
     def __init__(
         self,
         use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
         react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
     ):
         super().__init__(
             use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
             react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:

oodeel/methods/gen.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import numpy as np
+
+from ..types import DatasetType
+from ..types import TensorType
+from ..types import Tuple
+from .base import OODBaseDetector
+
+
+class GEN(OODBaseDetector):
+    """
+    Generalized Entropy method for OOD detection.
+    "GEN: Pushing the Limits of Softmax-Based Out-of-Distribution Detection"
+    https://openaccess.thecvf.com/content/CVPR2023/html/Liu_GEN_Pushing_the_Limits_of_Softmax-Based_Out-of-Distribution_Detection_CVPR_2023_paper.html,
+
+    Args:
+        gamma (float): parameter for the generalized entropy. Must be between 0 and 1.
+            Defaults to 0.1.
+        k (int): number of softmax values to keep for the entropy computation. Only the
+            top-k softmax probabilities will be used. Defaults to 100.
+        use_react (bool): if true, apply ReAct method by clipping penultimate
+            activations under a threshold value.
+        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
+            the react clipping threshold defined as the q-th quantile penultimate layer
+            activations. Defaults to 0.8.
+    """
+
+    def __init__(
+        self,
+        gamma: float = 0.1,
+        k: int = 100,
+        use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
+        react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
+    ):
+        super().__init__(
+            use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
+            react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
+        )
+        self.gamma = gamma
+        self.k = k
+
+    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
+        """
+        Computes an OOD score for input samples "inputs" based on
+        the distance to nearest neighbors in the feature space of self.model
+
+        Args:
+            inputs: input samples to score
+
+        Returns:
+            Tuple[np.ndarray]: scores, logits
+        """
+
+        _, logits = self.feature_extractor.predict_tensor(inputs)
+        probs = self.op.softmax(logits)
+        probs = self.op.convert_to_numpy(probs)
+        probs = np.sort(probs)[:, -self.k :]  # Keep the k largest probabilities
+        scores = np.sum(probs**self.gamma * (1 - probs) ** (self.gamma), axis=-1)
+        return scores
+
+    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
+        """
+        Fits the OOD detector to fit_dataset.
+
+        Args:
+            fit_dataset: dataset to fit the OOD detector on
+        """
+        pass
+
+    @property
+    def requires_to_fit_dataset(self) -> bool:
+        """
+        Whether an OOD detector needs a `fit_dataset` argument in the fit function.
+
+        Returns:
+            bool: True if `fit_dataset` is required else False.
+        """
+        return False
+
+    @property
+    def requires_internal_features(self) -> bool:
+        """
+        Whether an OOD detector acts on internal model features.
+
+        Returns:
+            bool: True if the detector perform computations on an intermediate layer
+            else False.
+        """
+        return False