oodeel 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

oodeel/extractor/keras_feature_extractor.py

@@ -24,6 +24,8 @@ from typing import get_args
 from typing import Optional
 
 import tensorflow as tf
+import tensorflow_probability as tfp
+from tqdm import tqdm
 
 from ..datasets.tf_data_handler import TFDataHandler
 from ..types import Callable
@@ -54,6 +56,11 @@ class KerasFeatureExtractor(FeatureExtractor):
             Defaults to None.
         react_threshold: if not None, penultimate layer activations are clipped under
             this threshold value (useful for ReAct). Defaults to None.
+        scale_percentile: if not None, the features are scaled
+            following the method of Xu et al., ICLR 2024.
+            Defaults to None.
+        ash_percentile: if not None, the features are scaled following
+            the method of Djurisic et al., ICLR 2023.
     """
 
     def __init__(
@@ -62,6 +69,8 @@ class KerasFeatureExtractor(FeatureExtractor):
         feature_layers_id: List[Union[int, str]] = [-1],
         input_layer_id: Optional[Union[int, str]] = None,
         react_threshold: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
     ):
         if input_layer_id is None:
             input_layer_id = 0
@@ -70,6 +79,8 @@ class KerasFeatureExtractor(FeatureExtractor):
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=react_threshold,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
         self.backend = "tensorflow"
@@ -143,6 +154,43 @@ class KerasFeatureExtractor(FeatureExtractor):
             )
             # apply ultimate layer on clipped activations
             output_tensors.append(last_layer(x))
+
+        # === If SCALE method, scale activations from penultimate layer ===
+        # === If ASH method, scale and prune activations from penultimate layer ===
+        elif (self.scale_percentile is not None) or (self.ash_percentile is not None):
+            penultimate_layer = self.find_layer(self.model, -2)
+            penult_extractor = tf.keras.models.Model(
+                new_input, penultimate_layer.output
+            )
+            last_layer = self.find_layer(self.model, -1)
+
+            # apply scaling on penultimate activations
+            penultimate = penult_extractor(new_input)
+            if self.scale_percentile is not None:
+                output_percentile = tfp.stats.percentile(
+                    penultimate, 100 * self.scale_percentile, axis=1
+                )
+            else:
+                output_percentile = tfp.stats.percentile(
+                    penultimate, 100 * self.ash_percentile, axis=1
+                )
+
+            mask = penultimate > tf.reshape(output_percentile, (-1, 1))
+            filtered_penultimate = tf.where(
+                mask, penultimate, tf.zeros_like(penultimate)
+            )
+            s = tf.math.exp(
+                tf.reduce_sum(penultimate, axis=1)
+                / tf.reduce_sum(filtered_penultimate, axis=1)
+            )
+
+            if self.scale_percentile is not None:
+                x = penultimate * tf.expand_dims(s, 1)
+            else:
+                x = filtered_penultimate * tf.expand_dims(s, 1)
+            # apply ultimate layer on scaled activations
+            output_tensors.append(last_layer(x))
+
         else:
             output_tensors.append(self.find_layer(self.model, -1).output)
 
@@ -190,6 +238,7 @@ class KerasFeatureExtractor(FeatureExtractor):
         self,
         dataset: Union[ItemType, tf.data.Dataset],
         postproc_fns: Optional[List[Callable]] = None,
+        verbose: bool = False,
         **kwargs,
     ) -> Tuple[List[tf.Tensor], dict]:
         """Get the projection of the dataset in the feature space of self.model
@@ -198,6 +247,7 @@ class KerasFeatureExtractor(FeatureExtractor):
             dataset (Union[ItemType, tf.data.Dataset]): input dataset
             postproc_fns (Optional[Callable]): postprocessing function to apply to each
                 feature immediately after forward. Default to None.
+            verbose (bool): if True, display a progress bar. Defaults to False.
             kwargs (dict): additional arguments not considered for prediction
 
         Returns:
@@ -218,7 +268,7 @@ class KerasFeatureExtractor(FeatureExtractor):
             features = [None for i in range(len(self.feature_layers_id))]
             logits = None
             contains_labels = TFDataHandler.get_item_length(dataset) > 1
-            for elem in dataset:
+            for elem in tqdm(dataset, desc="Predicting", disable=not verbose):
                 tensor = TFDataHandler.get_input_from_dataset_item(elem)
                 features_batch, logits_batch = self.predict_tensor(tensor, postproc_fns)
 

oodeel/extractor/torch_feature_extractor.py

@@ -20,13 +20,13 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from collections import OrderedDict
 from typing import get_args
 from typing import Optional
 
 import torch
 from torch import nn
 from torch.utils.data import DataLoader
+from tqdm import tqdm
 
 from ..datasets.torch_data_handler import TorchDataHandler
 from ..types import Callable
@@ -57,6 +57,12 @@ class TorchFeatureExtractor(FeatureExtractor):
             Defaults to None.
         react_threshold: if not None, penultimate layer activations are clipped under
             this threshold value (useful for ReAct). Defaults to None.
+        scale_percentile: if not None, the features are scaled
+            following the method of Xu et al., ICLR 2024.
+            Defaults to None.
+        ash_percentile: if not None, the features are scaled following
+            the method of Djurisic et al., ICLR 2023.
+
     """
 
     def __init__(
@@ -65,6 +71,8 @@ class TorchFeatureExtractor(FeatureExtractor):
         feature_layers_id: List[Union[int, str]] = [],
         input_layer_id: Optional[Union[int, str]] = None,
         react_threshold: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
     ):
         model = model.eval()
         super().__init__(
@@ -72,6 +80,8 @@ class TorchFeatureExtractor(FeatureExtractor):
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=react_threshold,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
         self._device = next(model.parameters()).device
         self._features = {layer: torch.empty(0) for layer in self._hook_layers_id}
@@ -140,18 +150,43 @@ class TorchFeatureExtractor(FeatureExtractor):
 
     def prepare_extractor(self) -> None:
         """Prepare the feature extractor by adding hooks to self.model"""
-        # remove forward hooks attached to the model
-        self._clean_forward_hooks()
+        # prepare self.model for ood hooks (add _ood_handles attribute or
+        # remove ood forward hooks attached to the model)
+        self._prepare_ood_handles()
 
         # === If react method, clip activations from penultimate layer ===
         if self.react_threshold is not None:
-            pen_layer = self.find_layer(self.model, -2)
-            pen_layer.register_forward_hook(self._get_clip_hook(self.react_threshold))
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_clip_hook(self.react_threshold)
+                )
+            )
+
+        # === If SCALE method, scale activations from penultimate layer ===
+        if self.scale_percentile is not None:
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_scale_hook(self.scale_percentile)
+                )
+            )
+
+        # === If ASH method, scale and prune activations from penultimate layer ===
+        if self.ash_percentile is not None:
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_ash_hook(self.ash_percentile)
+                )
+            )
 
         # Register a hook to store feature values for each considered layer + last layer
         for layer_id in self._hook_layers_id:
             layer = self.find_layer(self.model, layer_id)
-            layer.register_forward_hook(self._get_features_hook(layer_id))
+            self.model._ood_handles.append(
+                layer.register_forward_hook(self._get_features_hook(layer_id))
+            )
 
         # Crop model if input layer is provided
         if not (self.input_layer_id) is None:
@@ -226,6 +261,7 @@ class TorchFeatureExtractor(FeatureExtractor):
         dataset: Union[DataLoader, ItemType],
         postproc_fns: Optional[List[Callable]] = None,
         detach: bool = True,
+        verbose: bool = False,
         **kwargs,
     ) -> Tuple[List[torch.Tensor], dict]:
         """Get the projection of the dataset in the feature space of self.model
@@ -236,6 +272,7 @@ class TorchFeatureExtractor(FeatureExtractor):
                 each feature immediately after forward. Default to None.
             detach (bool): if True, return features detached from the computational
                 graph. Defaults to True.
+            verbose (bool): if True, display a progress bar. Defaults to False.
             kwargs (dict): additional arguments not considered for prediction
 
         Returns:
@@ -257,7 +294,7 @@ class TorchFeatureExtractor(FeatureExtractor):
             logits = None
             batch = next(iter(dataset))
             contains_labels = isinstance(batch, (list, tuple)) and len(batch) > 1
-            for elem in dataset:
+            for elem in tqdm(dataset, desc="Predicting", disable=not verbose):
                 tensor = TorchDataHandler.get_input_from_dataset_item(elem)
                 features_batch, logits_batch = self.predict_tensor(
                     tensor, postproc_fns, detach=detach
@@ -308,24 +345,69 @@ class TorchFeatureExtractor(FeatureExtractor):
             Callable: hook function
         """
 
-        def hook(_, __, output):
-            output = torch.clip(output, max=threshold)
-            return output
+        def hook(_, input):
+            input = input[0]
+            input = torch.clip(input, max=threshold)
+            return input
+
+        return hook
+
+    def _get_scale_hook(self, percentile: float) -> Callable:
+        """
+        Hook that scales activation features.
+
+        Args:
+            threshold (float): threshold value
+
+        Returns:
+            Callable: hook function
+        """
+
+        def hook(_, input):
+            input = input[0]
+            output_percentile = torch.quantile(input, percentile, dim=1)
+            mask = input > output_percentile[:, None]
+            output_masked = input * mask
+            s = torch.exp(torch.sum(input, dim=1) / torch.sum(output_masked, dim=1))
+            s = torch.unsqueeze(s, 1)
+            input = input * s
+            return input
 
         return hook
 
-    def _clean_forward_hooks(self) -> None:
+    def _get_ash_hook(self, percentile: float) -> Callable:
         """
-        Remove all the forward hook attached to the model's layers. This function should
-        be called at the __init__, and prevent from accumulating the hooks when
-        defining a new TorchFeatureExtractor for the same model.
+        Hook that scales and prunes activation features under a threshold value
+
+        Args:
+            threshold (float): threshold value
+
+        Returns:
+            Callable: hook function
         """
 
-        def __clean_hooks(m: nn.Module):
-            for _, child in m._modules.items():
-                if child is not None:
-                    if hasattr(child, "_forward_hooks"):
-                        child._forward_hooks = OrderedDict()
-                    __clean_hooks(child)
+        def hook(_, input):
+            input = input[0]
+            output_percentile = torch.quantile(input, percentile, dim=1)
+            mask = input > output_percentile[:, None]
+            output_masked = input * mask
+            s = torch.exp(torch.sum(input, dim=1) / torch.sum(output_masked, dim=1))
+            s = torch.unsqueeze(s, 1)
+            input = output_masked * s
+            return input
+
+        return hook
 
-        return __clean_hooks(self.model)
+    def _prepare_ood_handles(self) -> None:
+        """
+        Prepare the model by either setting a new attribute to self.model
+        as a list which will contain all the ood specific hooks, or by cleaning
+        the existing ood specific hooks if the attribute already exists.
+        """
+
+        if not hasattr(self.model, "_ood_handles"):
+            setattr(self.model, "_ood_handles", [])
+        else:
+            for handle in self.model._ood_handles:
+                handle.remove()
+            self.model._ood_handles = []

oodeel/methods/__init__.py

@@ -23,10 +23,25 @@
 from .dknn import DKNN
 from .energy import Energy
 from .entropy import Entropy
+from .gen import GEN
 from .gram import Gram
 from .mahalanobis import Mahalanobis
 from .mls import MLS
 from .odin import ODIN
+from .rmds import RMDS
+from .she import SHE
 from .vim import VIM
 
-__all__ = ["MLS", "DKNN", "ODIN", "Energy", "VIM", "Mahalanobis", "Entropy", "Gram"]
+__all__ = [
+    "DKNN",
+    "Energy",
+    "Entropy",
+    "GEN",
+    "SHE",
+    "Gram",
+    "Mahalanobis",
+    "MLS",
+    "ODIN",
+    "RMDS",
+    "VIM",
+]

oodeel/methods/base.py

@@ -20,11 +20,13 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import inspect
 from abc import ABC
 from abc import abstractmethod
 from typing import get_args
 
 import numpy as np
+from tqdm import tqdm
 
 from ..extractor.feature_extractor import FeatureExtractor
 from ..types import Callable
@@ -38,28 +40,66 @@ from ..utils import import_backend_specific_stuff
 
 
 class OODBaseDetector(ABC):
-    """Base Class for methods that assign a score to unseen samples.
-
-    Args:
-        use_react (bool): if true, apply ReAct method by clipping penultimate
-            activations under a threshold value.
-        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
-            the react clipping threshold defined as the q-th quantile penultimate layer
-            activations. Defaults to 0.8.
+    """OODBaseDetector is an abstract base class for Out-of-Distribution (OOD)
+    detection.
+
+    Attributes:
+        feature_extractor (FeatureExtractor): The feature extractor instance.
+        use_react (bool): Flag to indicate if ReAct method is used.
+        use_scale (bool): Flag to indicate if scaling method is used.
+        use_ash (bool): Flag to indicate if ASH method is used.
+        react_quantile (float): Quantile value for ReAct threshold.
+        scale_percentile (float): Percentile value for scaling.
+        ash_percentile (float): Percentile value for ASH.
+        react_threshold (float): Threshold value for ReAct.
+        postproc_fns (List[Callable]): List of post-processing functions.
+
+    Methods:
+        __init__: Initializes the OODBaseDetector with specified parameters.
+        _score_tensor: Abstract method to compute OOD score for input samples.
+        _sanitize_posproc_fns: Sanitizes post-processing functions used at each layer
+        output.
+        fit: Prepares the detector for scoring by constructing the feature extractor
+        and calibrating on ID data.
+        _load_feature_extractor: Loads the feature extractor based on the model and
+        specified layers.
+        _fit_to_dataset: Abstract method to fit the OOD detector to a dataset.
+        score: Computes an OOD score for input samples.
+        compute_react_threshold: Computes the ReAct threshold using the fit dataset.
+        __call__: Convenience wrapper for the score method.
+        requires_to_fit_dataset: Property indicating if the detector needs a fit
+        dataset.
+        requires_internal_features: Property indicating if the detector acts on
+        internal model features.
     """
 
     def __init__(
         self,
-        use_react: bool = False,
-        react_quantile: float = 0.8,
-        postproc_fns: List[Callable] = None,
+        use_react: Optional[bool] = False,
+        use_scale: Optional[bool] = False,
+        use_ash: Optional[bool] = False,
+        react_quantile: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
+        postproc_fns: Optional[List[Callable]] = None,
     ):
         self.feature_extractor: FeatureExtractor = None
         self.use_react = use_react
+        self.use_scale = use_scale
+        self.use_ash = use_ash
         self.react_quantile = react_quantile
+        self.scale_percentile = scale_percentile
+        self.ash_percentile = ash_percentile
         self.react_threshold = None
         self.postproc_fns = self._sanitize_posproc_fns(postproc_fns)
 
+        if use_scale and use_react:
+            raise ValueError("Cannot use both ReAct and scale at the same time")
+        if use_scale and use_ash:
+            raise ValueError("Cannot use both ASH and scale at the same time")
+        if use_ash and use_react:
+            raise ValueError("Cannot use both ReAct and ASH at the same time")
+
     @abstractmethod
     def _score_tensor(self, inputs: TensorType) -> np.ndarray:
         """Computes an OOD score for input samples "inputs".
@@ -104,6 +144,7 @@ class OODBaseDetector(ABC):
         fit_dataset: Optional[Union[ItemType, DatasetType]] = None,
         feature_layers_id: List[Union[int, str]] = [],
         input_layer_id: Optional[Union[int, str]] = None,
+        verbose: bool = False,
         **kwargs,
     ) -> None:
         """Prepare the detector for scoring:
@@ -122,6 +163,7 @@ class OODBaseDetector(ABC):
                 layer of the feature extractor.
                 If int, the rank of the layer in the layer list
                 If str, the name of the layer. Defaults to None.
+            verbose (bool): if True, display a progress bar. Defaults to False.
         """
         (
             self.backend,
@@ -144,7 +186,7 @@ class OODBaseDetector(ABC):
                     " provided to compute react activation threshold"
                 )
             else:
-                self.compute_react_threshold(model, fit_dataset)
+                self.compute_react_threshold(model, fit_dataset, verbose=verbose)
 
         if (feature_layers_id == []) and (self.requires_internal_features):
             raise ValueError(
@@ -160,6 +202,8 @@ class OODBaseDetector(ABC):
         )
 
         if fit_dataset is not None:
+            if "verbose" in inspect.signature(self._fit_to_dataset).parameters.keys():
+                kwargs.update({"verbose": verbose})
             self._fit_to_dataset(fit_dataset, **kwargs)
 
     def _load_feature_extractor(
@@ -185,11 +229,18 @@ class OODBaseDetector(ABC):
         Returns:
             FeatureExtractor: a feature extractor instance
         """
+        if not self.use_ash:
+            self.ash_percentile = None
+        if not self.use_scale:
+            self.scale_percentile = None
+
         feature_extractor = self.FeatureExtractorClass(
             model,
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=self.react_threshold,
+            scale_percentile=self.scale_percentile,
+            ash_percentile=self.ash_percentile,
         )
         return feature_extractor
 
@@ -207,12 +258,14 @@ class OODBaseDetector(ABC):
     def score(
         self,
         dataset: Union[ItemType, DatasetType],
+        verbose: bool = False,
     ) -> np.ndarray:
         """
         Computes an OOD score for input samples "inputs".
 
         Args:
             dataset (Union[ItemType, DatasetType]): dataset or tensors to score
+            verbose (bool): if True, display a progress bar. Defaults to False.
 
         Returns:
             tuple: scores or list of scores (depending on the input) and a dictionary
@@ -236,7 +289,7 @@ class OODBaseDetector(ABC):
             scores = np.array([])
             logits = None
 
-            for item in dataset:
+            for item in tqdm(dataset, desc="Scoring", disable=not verbose):
                 tensor = self.data_handler.get_input_from_dataset_item(item)
                 score_batch = self._score_tensor(tensor)
                 logits_batch = self.op.convert_to_numpy(
@@ -267,9 +320,13 @@ class OODBaseDetector(ABC):
         info = dict(labels=labels, logits=logits)
         return scores, info
 
-    def compute_react_threshold(self, model: Callable, fit_dataset: DatasetType):
+    def compute_react_threshold(
+        self, model: Callable, fit_dataset: DatasetType, verbose: bool = False
+    ):
         penult_feat_extractor = self._load_feature_extractor(model, [-2])
-        unclipped_features, _ = penult_feat_extractor.predict(fit_dataset)
+        unclipped_features, _ = penult_feat_extractor.predict(
+            fit_dataset, verbose=verbose
+        )
         self.react_threshold = self.op.quantile(
             unclipped_features[0], self.react_quantile
         )

oodeel/methods/dknn.py

@@ -38,21 +38,28 @@ class DKNN(OODBaseDetector):
     Args:
         nearest: number of nearest neighbors to consider.
             Defaults to 1.
+        use_gpu (bool): Flag to enable GPU acceleration for FAISS. Defaults to False.
     """
 
-    def __init__(
-        self,
-        nearest: int = 1,
-    ):
+    def __init__(self, nearest: int = 50, use_gpu: bool = False):
         super().__init__()
-
         self.index = None
         self.nearest = nearest
+        self.use_gpu = use_gpu
+
+        if self.use_gpu:
+            try:
+                self.res = faiss.StandardGpuResources()
+            except AttributeError as e:
+                raise ImportError(
+                    "faiss-gpu is not installed, but use_gpu was set to True."
+                    + "Please install faiss-gpu or set use_gpu to False."
+                ) from e
 
     def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
         """
         Constructs the index from ID data "fit_dataset", which will be used for
-        nearest neighbor search.
+        nearest neighbor search. Can operate on CPU or GPU based on the `use_gpu` flag.
 
         Args:
             fit_dataset: input dataset (ID) to construct the index with.
@@ -61,7 +68,13 @@ class DKNN(OODBaseDetector):
         fit_projected = self.op.convert_to_numpy(fit_projected[0])
         fit_projected = fit_projected.reshape(fit_projected.shape[0], -1)
         norm_fit_projected = self._l2_normalization(fit_projected)
-        self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+
+        if self.use_gpu:
+            cpu_index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+            self.index = faiss.index_cpu_to_gpu(self.res, 0, cpu_index)
+        else:
+            self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
+
         self.index.add(norm_fit_projected)
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:

oodeel/methods/energy.py

@@ -59,11 +59,19 @@ class Energy(OODBaseDetector):
     def __init__(
         self,
         use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
         react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
     ):
         super().__init__(
             use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
             react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:

oodeel/methods/entropy.py

@@ -52,11 +52,19 @@ class Entropy(OODBaseDetector):
     def __init__(
         self,
         use_react: bool = False,
+        use_scale: bool = False,
+        use_ash: bool = False,
         react_quantile: float = 0.8,
+        scale_percentile: float = 0.85,
+        ash_percentile: float = 0.90,
     ):
         super().__init__(
             use_react=use_react,
+            use_scale=use_scale,
+            use_ash=use_ash,
             react_quantile=react_quantile,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
     def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]: