oodeel 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

oodeel/extractor/feature_extractor.py

@@ -25,6 +25,7 @@ from abc import abstractmethod
 
 from ..types import Callable
 from ..types import DatasetType
+from ..types import ItemType
 from ..types import List
 from ..types import Optional
 from ..types import TensorType
@@ -51,6 +52,11 @@ class FeatureExtractor(ABC):
             Defaults to None.
         react_threshold: if not None, penultimate layer activations are clipped under
             this threshold value (useful for ReAct). Defaults to None.
+        scale_percentile: if not None, the features are scaled
+            following the method of Xu et al., ICLR 2024.
+            Defaults to None.
+        ash_percentile: if not None, the features are scaled following
+            the method of Djurisic et al., ICLR 2023.
     """
 
     def __init__(
@@ -59,6 +65,8 @@ class FeatureExtractor(ABC):
         feature_layers_id: List[Union[int, str]] = [-1],
         input_layer_id: Union[int, str] = [0],
         react_threshold: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
     ):
         if not isinstance(feature_layers_id, list):
             feature_layers_id = [feature_layers_id]
@@ -66,6 +74,8 @@ class FeatureExtractor(ABC):
         self.feature_layers_id = feature_layers_id
         self.input_layer_id = input_layer_id
         self.react_threshold = react_threshold
+        self.scale_percentile = scale_percentile
+        self.ash_percentile = ash_percentile
         self.model = model
         self.extractor = self.prepare_extractor()
 
@@ -91,28 +101,39 @@ class FeatureExtractor(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    def predict_tensor(self, tensor: TensorType) -> Tuple[List[TensorType], TensorType]:
-        """
-        Projects input samples "inputs" into the feature space
+    def predict_tensor(
+        self,
+        tensor: TensorType,
+        postproc_fns: Optional[List[Callable]] = None,
+    ) -> Tuple[List[TensorType], TensorType]:
+        """Get the projection of tensor in the feature space of self.model
 
         Args:
-            tensor (TensorType): input tensor
+            tensor (TensorType): input tensor (or dataset elem)
+            postproc_fns (Optional[Callable]): postprocessing function to apply to each
+                feature immediately after forward. Default to None.
 
         Returns:
-            List[TensorType], TensorType: features, logits
+            Tuple[List[TensorType], TensorType]: features, logits
         """
         raise NotImplementedError()
 
     @abstractmethod
     def predict(
         self,
-        dataset: Union[DatasetType, TensorType],
+        dataset: Union[ItemType, DatasetType],
+        postproc_fns: Optional[List[Callable]] = None,
+        verbose: bool = False,
+        **kwargs,
     ) -> Tuple[List[TensorType], dict]:
-        """
-        Projects input samples "inputs" into the feature space for a batched dataset
+        """Get the projection of the dataset in the feature space of self.model
 
         Args:
-            dataset (Union[DatasetType, TensorType]): iterable of tensor batches
+            dataset (Union[ItemType, DatasetType]): input dataset
+            postproc_fns (Optional[Callable]): postprocessing function to apply to each
+                feature immediately after forward. Default to None.
+            verbose (bool): if True, display a progress bar. Defaults to False.
+            kwargs (dict): additional arguments not considered for prediction
 
         Returns:
             List[TensorType], dict: features and extra information (logits, labels) as a

oodeel/extractor/keras_feature_extractor.py

@@ -24,6 +24,8 @@ from typing import get_args
 from typing import Optional
 
 import tensorflow as tf
+import tensorflow_probability as tfp
+from tqdm import tqdm
 
 from ..datasets.tf_data_handler import TFDataHandler
 from ..types import Callable
@@ -54,6 +56,11 @@ class KerasFeatureExtractor(FeatureExtractor):
             Defaults to None.
         react_threshold: if not None, penultimate layer activations are clipped under
             this threshold value (useful for ReAct). Defaults to None.
+        scale_percentile: if not None, the features are scaled
+            following the method of Xu et al., ICLR 2024.
+            Defaults to None.
+        ash_percentile: if not None, the features are scaled following
+            the method of Djurisic et al., ICLR 2023.
     """
 
     def __init__(
@@ -62,6 +69,8 @@ class KerasFeatureExtractor(FeatureExtractor):
         feature_layers_id: List[Union[int, str]] = [-1],
         input_layer_id: Optional[Union[int, str]] = None,
         react_threshold: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
     ):
         if input_layer_id is None:
             input_layer_id = 0
@@ -70,6 +79,8 @@ class KerasFeatureExtractor(FeatureExtractor):
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=react_threshold,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
 
         self.backend = "tensorflow"
@@ -143,6 +154,43 @@ class KerasFeatureExtractor(FeatureExtractor):
             )
             # apply ultimate layer on clipped activations
             output_tensors.append(last_layer(x))
+
+        # === If SCALE method, scale activations from penultimate layer ===
+        # === If ASH method, scale and prune activations from penultimate layer ===
+        elif (self.scale_percentile is not None) or (self.ash_percentile is not None):
+            penultimate_layer = self.find_layer(self.model, -2)
+            penult_extractor = tf.keras.models.Model(
+                new_input, penultimate_layer.output
+            )
+            last_layer = self.find_layer(self.model, -1)
+
+            # apply scaling on penultimate activations
+            penultimate = penult_extractor(new_input)
+            if self.scale_percentile is not None:
+                output_percentile = tfp.stats.percentile(
+                    penultimate, 100 * self.scale_percentile, axis=1
+                )
+            else:
+                output_percentile = tfp.stats.percentile(
+                    penultimate, 100 * self.ash_percentile, axis=1
+                )
+
+            mask = penultimate > tf.reshape(output_percentile, (-1, 1))
+            filtered_penultimate = tf.where(
+                mask, penultimate, tf.zeros_like(penultimate)
+            )
+            s = tf.math.exp(
+                tf.reduce_sum(penultimate, axis=1)
+                / tf.reduce_sum(filtered_penultimate, axis=1)
+            )
+
+            if self.scale_percentile is not None:
+                x = penultimate * tf.expand_dims(s, 1)
+            else:
+                x = filtered_penultimate * tf.expand_dims(s, 1)
+            # apply ultimate layer on scaled activations
+            output_tensors.append(last_layer(x))
+
         else:
             output_tensors.append(self.find_layer(self.model, -1).output)
 
@@ -150,11 +198,17 @@ class KerasFeatureExtractor(FeatureExtractor):
         return extractor
 
     @sanitize_input
-    def predict_tensor(self, tensor: TensorType) -> Tuple[List[tf.Tensor], tf.Tensor]:
+    def predict_tensor(
+        self,
+        tensor: TensorType,
+        postproc_fns: Optional[List[Callable]] = None,
+    ) -> Tuple[List[tf.Tensor], tf.Tensor]:
         """Get the projection of tensor in the feature space of self.model
 
         Args:
             tensor (TensorType): input tensor (or dataset elem)
+            postproc_fns (Optional[List[Callable]]): postprocessing function to apply to
+                each feature immediately after forward. Default to None.
 
         Returns:
             Tuple[List[tf.Tensor], tf.Tensor]: features, logits
@@ -166,8 +220,12 @@ class KerasFeatureExtractor(FeatureExtractor):
 
         # split features and logits
         logits = features.pop()
-        if len(features) == 1:
-            features = features[0]
+
+        if postproc_fns is not None:
+            features = [
+                postproc_fn(feature)
+                for feature, postproc_fn in zip(features, postproc_fns)
+            ]
 
         self._last_logits = logits
         return features, logits
@@ -179,12 +237,17 @@ class KerasFeatureExtractor(FeatureExtractor):
     def predict(
         self,
         dataset: Union[ItemType, tf.data.Dataset],
+        postproc_fns: Optional[List[Callable]] = None,
+        verbose: bool = False,
         **kwargs,
     ) -> Tuple[List[tf.Tensor], dict]:
         """Get the projection of the dataset in the feature space of self.model
 
         Args:
             dataset (Union[ItemType, tf.data.Dataset]): input dataset
+            postproc_fns (Optional[Callable]): postprocessing function to apply to each
+                feature immediately after forward. Default to None.
+            verbose (bool): if True, display a progress bar. Defaults to False.
             kwargs (dict): additional arguments not considered for prediction
 
         Returns:
@@ -195,7 +258,7 @@ class KerasFeatureExtractor(FeatureExtractor):
 
         if isinstance(dataset, get_args(ItemType)):
             tensor = TFDataHandler.get_input_from_dataset_item(dataset)
-            features, logits = self.predict_tensor(tensor)
+            features, logits = self.predict_tensor(tensor, postproc_fns)
 
             # Get labels if dataset is a tuple/list
             if isinstance(dataset, (list, tuple)):
@@ -205,12 +268,10 @@ class KerasFeatureExtractor(FeatureExtractor):
             features = [None for i in range(len(self.feature_layers_id))]
             logits = None
             contains_labels = TFDataHandler.get_item_length(dataset) > 1
-            for elem in dataset:
+            for elem in tqdm(dataset, desc="Predicting", disable=not verbose):
                 tensor = TFDataHandler.get_input_from_dataset_item(elem)
-                features_batch, logits_batch = self.predict_tensor(tensor)
-                # concatenate features
-                if len(features) == 1:
-                    features_batch = [features_batch]
+                features_batch, logits_batch = self.predict_tensor(tensor, postproc_fns)
+
                 for i, f in enumerate(features_batch):
                     features[i] = (
                         f
@@ -234,10 +295,6 @@ class KerasFeatureExtractor(FeatureExtractor):
 
         # store extra information in a dict
         info = dict(labels=labels, logits=logits)
-
-        if len(features) == 1:
-            features = features[0]
-
         return features, info
 
     def get_weights(self, layer_id: Union[int, str]) -> List[tf.Tensor]:

oodeel/extractor/torch_feature_extractor.py

@@ -20,13 +20,13 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from collections import OrderedDict
 from typing import get_args
 from typing import Optional
 
 import torch
 from torch import nn
 from torch.utils.data import DataLoader
+from tqdm import tqdm
 
 from ..datasets.torch_data_handler import TorchDataHandler
 from ..types import Callable
@@ -57,6 +57,12 @@ class TorchFeatureExtractor(FeatureExtractor):
             Defaults to None.
         react_threshold: if not None, penultimate layer activations are clipped under
             this threshold value (useful for ReAct). Defaults to None.
+        scale_percentile: if not None, the features are scaled
+            following the method of Xu et al., ICLR 2024.
+            Defaults to None.
+        ash_percentile: if not None, the features are scaled following
+            the method of Djurisic et al., ICLR 2023.
+
     """
 
     def __init__(
@@ -65,6 +71,8 @@ class TorchFeatureExtractor(FeatureExtractor):
         feature_layers_id: List[Union[int, str]] = [],
         input_layer_id: Optional[Union[int, str]] = None,
         react_threshold: Optional[float] = None,
+        scale_percentile: Optional[float] = None,
+        ash_percentile: Optional[float] = None,
     ):
         model = model.eval()
         super().__init__(
@@ -72,6 +80,8 @@ class TorchFeatureExtractor(FeatureExtractor):
             feature_layers_id=feature_layers_id,
             input_layer_id=input_layer_id,
             react_threshold=react_threshold,
+            scale_percentile=scale_percentile,
+            ash_percentile=ash_percentile,
         )
         self._device = next(model.parameters()).device
         self._features = {layer: torch.empty(0) for layer in self._hook_layers_id}
@@ -140,18 +150,43 @@ class TorchFeatureExtractor(FeatureExtractor):
 
     def prepare_extractor(self) -> None:
         """Prepare the feature extractor by adding hooks to self.model"""
-        # remove forward hooks attached to the model
-        self._clean_forward_hooks()
+        # prepare self.model for ood hooks (add _ood_handles attribute or
+        # remove ood forward hooks attached to the model)
+        self._prepare_ood_handles()
 
         # === If react method, clip activations from penultimate layer ===
         if self.react_threshold is not None:
-            pen_layer = self.find_layer(self.model, -2)
-            pen_layer.register_forward_hook(self._get_clip_hook(self.react_threshold))
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_clip_hook(self.react_threshold)
+                )
+            )
+
+        # === If SCALE method, scale activations from penultimate layer ===
+        if self.scale_percentile is not None:
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_scale_hook(self.scale_percentile)
+                )
+            )
+
+        # === If ASH method, scale and prune activations from penultimate layer ===
+        if self.ash_percentile is not None:
+            pen_layer = self.find_layer(self.model, -1)
+            self.model._ood_handles.append(
+                pen_layer.register_forward_pre_hook(
+                    self._get_ash_hook(self.ash_percentile)
+                )
+            )
 
         # Register a hook to store feature values for each considered layer + last layer
         for layer_id in self._hook_layers_id:
             layer = self.find_layer(self.model, layer_id)
-            layer.register_forward_hook(self._get_features_hook(layer_id))
+            self.model._ood_handles.append(
+                layer.register_forward_hook(self._get_features_hook(layer_id))
+            )
 
         # Crop model if input layer is provided
         if not (self.input_layer_id) is None:
@@ -181,12 +216,17 @@ class TorchFeatureExtractor(FeatureExtractor):
 
     @sanitize_input
     def predict_tensor(
-        self, x: TensorType, detach: bool = True
+        self,
+        x: TensorType,
+        postproc_fns: Optional[List[Callable]] = None,
+        detach: bool = True,
     ) -> Tuple[List[torch.Tensor], torch.Tensor]:
         """Get the projection of tensor in the feature space of self.model
 
         Args:
             x (TensorType): input tensor (or dataset elem)
+            postproc_fns (Optional[List[Callable]]): postprocessing function to apply to
+                each feature immediately after forward. Default to None.
             detach (bool): if True, return features detached from the computational
                 graph. Defaults to True.
 
@@ -206,8 +246,12 @@ class TorchFeatureExtractor(FeatureExtractor):
 
         # split features and logits
         logits = features.pop()
-        if len(features) == 1:
-            features = features[0]
+
+        if postproc_fns is not None:
+            features = [
+                postproc_fn(feature)
+                for feature, postproc_fn in zip(features, postproc_fns)
+            ]
 
         self._last_logits = logits
         return features, logits
@@ -215,15 +259,20 @@ class TorchFeatureExtractor(FeatureExtractor):
     def predict(
         self,
         dataset: Union[DataLoader, ItemType],
+        postproc_fns: Optional[List[Callable]] = None,
         detach: bool = True,
+        verbose: bool = False,
         **kwargs,
     ) -> Tuple[List[torch.Tensor], dict]:
         """Get the projection of the dataset in the feature space of self.model
 
         Args:
             dataset (Union[DataLoader, ItemType]): input dataset
+            postproc_fns (Optional[List[Callable]]): postprocessing function to apply to
+                each feature immediately after forward. Default to None.
             detach (bool): if True, return features detached from the computational
                 graph. Defaults to True.
+            verbose (bool): if True, display a progress bar. Defaults to False.
             kwargs (dict): additional arguments not considered for prediction
 
         Returns:
@@ -234,7 +283,7 @@ class TorchFeatureExtractor(FeatureExtractor):
 
         if isinstance(dataset, get_args(ItemType)):
             tensor = TorchDataHandler.get_input_from_dataset_item(dataset)
-            features, logits = self.predict_tensor(tensor, detach=detach)
+            features, logits = self.predict_tensor(tensor, postproc_fns, detach=detach)
 
             # Get labels if dataset is a tuple/list
             if isinstance(dataset, (list, tuple)) and len(dataset) > 1:
@@ -245,14 +294,11 @@ class TorchFeatureExtractor(FeatureExtractor):
             logits = None
             batch = next(iter(dataset))
             contains_labels = isinstance(batch, (list, tuple)) and len(batch) > 1
-            for elem in dataset:
+            for elem in tqdm(dataset, desc="Predicting", disable=not verbose):
                 tensor = TorchDataHandler.get_input_from_dataset_item(elem)
                 features_batch, logits_batch = self.predict_tensor(
-                    tensor, detach=detach
+                    tensor, postproc_fns, detach=detach
                 )
-                # concatenate features
-                if len(features) == 1:
-                    features_batch = [features_batch]
                 for i, f in enumerate(features_batch):
                     features[i] = (
                         f if features[i] is None else torch.cat([features[i], f], dim=0)
@@ -274,10 +320,6 @@ class TorchFeatureExtractor(FeatureExtractor):
 
         # store extra information in a dict
         info = dict(labels=labels, logits=logits)
-
-        if len(features) == 1:
-            features = features[0]
-
         return features, info
 
     def get_weights(self, layer_id: Union[str, int]) -> List[torch.Tensor]:
@@ -303,24 +345,69 @@ class TorchFeatureExtractor(FeatureExtractor):
             Callable: hook function
         """
 
-        def hook(_, __, output):
-            output = torch.clip(output, max=threshold)
-            return output
+        def hook(_, input):
+            input = input[0]
+            input = torch.clip(input, max=threshold)
+            return input
 
         return hook
 
-    def _clean_forward_hooks(self) -> None:
+    def _get_scale_hook(self, percentile: float) -> Callable:
         """
-        Remove all the forward hook attached to the model's layers. This function should
-        be called at the __init__, and prevent from accumulating the hooks when
-        defining a new TorchFeatureExtractor for the same model.
+        Hook that scales activation features.
+
+        Args:
+            threshold (float): threshold value
+
+        Returns:
+            Callable: hook function
         """
 
-        def __clean_hooks(m: nn.Module):
-            for _, child in m._modules.items():
-                if child is not None:
-                    if hasattr(child, "_forward_hooks"):
-                        child._forward_hooks = OrderedDict()
-                    __clean_hooks(child)
+        def hook(_, input):
+            input = input[0]
+            output_percentile = torch.quantile(input, percentile, dim=1)
+            mask = input > output_percentile[:, None]
+            output_masked = input * mask
+            s = torch.exp(torch.sum(input, dim=1) / torch.sum(output_masked, dim=1))
+            s = torch.unsqueeze(s, 1)
+            input = input * s
+            return input
 
-        return __clean_hooks(self.model)
+        return hook
+
+    def _get_ash_hook(self, percentile: float) -> Callable:
+        """
+        Hook that scales and prunes activation features under a threshold value
+
+        Args:
+            threshold (float): threshold value
+
+        Returns:
+            Callable: hook function
+        """
+
+        def hook(_, input):
+            input = input[0]
+            output_percentile = torch.quantile(input, percentile, dim=1)
+            mask = input > output_percentile[:, None]
+            output_masked = input * mask
+            s = torch.exp(torch.sum(input, dim=1) / torch.sum(output_masked, dim=1))
+            s = torch.unsqueeze(s, 1)
+            input = output_masked * s
+            return input
+
+        return hook
+
+    def _prepare_ood_handles(self) -> None:
+        """
+        Prepare the model by either setting a new attribute to self.model
+        as a list which will contain all the ood specific hooks, or by cleaning
+        the existing ood specific hooks if the attribute already exists.
+        """
+
+        if not hasattr(self.model, "_ood_handles"):
+            setattr(self.model, "_ood_handles", [])
+        else:
+            for handle in self.model._ood_handles:
+                handle.remove()
+            self.model._ood_handles = []

oodeel/methods/__init__.py

@@ -23,9 +23,25 @@
 from .dknn import DKNN
 from .energy import Energy
 from .entropy import Entropy
+from .gen import GEN
+from .gram import Gram
 from .mahalanobis import Mahalanobis
 from .mls import MLS
 from .odin import ODIN
+from .rmds import RMDS
+from .she import SHE
 from .vim import VIM
 
-__all__ = ["MLS", "DKNN", "ODIN", "Energy", "VIM", "Mahalanobis", "Entropy"]
+__all__ = [
+    "DKNN",
+    "Energy",
+    "Entropy",
+    "GEN",
+    "SHE",
+    "Gram",
+    "Mahalanobis",
+    "MLS",
+    "ODIN",
+    "RMDS",
+    "VIM",
+]