oodeel 0.4.0__py3-none-any.whl

tests/tests_tensorflow/tf_methods_utils.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import os
+import random
+
+import numpy as np
+import tensorflow as tf
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import train_test_split
+
+from oodeel.datasets import OODDataset
+from oodeel.eval.metrics import bench_metrics
+
+model_path = os.path.expanduser("~/") + ".oodeel/saved_models"
+data_path = os.path.expanduser("~/") + ".oodeel/datasets"
+os.makedirs(model_path, exist_ok=True)
+os.makedirs(data_path, exist_ok=True)
+
+
+def load_blobs_data(batch_size=128, num_samples=10000, train_ratio=0.8):
+    # === data hparams ===
+    num_classes = 3
+    in_labels = [0, 1]
+    out_labels = [2, 3]
+    centers = np.array([[-4, -4], [4, 4], [-4, 4], [4, -4]])
+
+    # === generate data ===
+    X, y = make_blobs(num_samples, num_classes, centers=centers, random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, train_size=train_ratio, random_state=0
+    )
+
+    # === id / ood split ===
+    blobs_train = OODDataset((X_train, y_train), backend="tensorflow")
+    blobs_test = OODDataset((X_test, y_test), backend="tensorflow")
+    oods_fit, _ = blobs_train.split_by_class(in_labels, out_labels)
+    oods_in, oods_out = blobs_test.split_by_class(in_labels, out_labels)
+
+    # === prepare data (shuffle, batch) => torch dataloaders ===
+    ds_fit = oods_fit.prepare(batch_size=batch_size, shuffle=True)
+    ds_in = oods_in.prepare(batch_size=batch_size)
+    ds_out = oods_out.prepare(batch_size=batch_size)
+    return ds_fit, ds_in, ds_out
+
+
+def load_blob_mlp():
+    model_path_blobs = tf.keras.utils.get_file(
+        "blobs_mlp.h5",
+        origin="https://github.com/deel-ai/oodeel/blob/assets/test_models/"
+        + "blobs_mlp.h5?raw=True",
+        cache_dir=model_path,
+        cache_subdir="",
+    )
+    model = tf.keras.models.load_model(model_path_blobs)
+    return model
+
+
+def eval_detector_on_blobs(
+    detector,
+    auroc_thr=0.6,
+    fpr95_thr=0.3,
+    batch_size=128,
+    check_react_clipping=False,
+):
+    # seed
+    tf.random.set_seed(0)
+    np.random.seed(0)
+    random.seed(1)
+
+    # load data
+    ds_fit, ds_in, ds_out = load_blobs_data(batch_size)
+
+    # get classifier
+    model = load_blob_mlp()
+
+    # fit ood detector
+    if (
+        detector.requires_to_fit_dataset or detector.use_react
+    ) and detector.requires_internal_features:
+        if hasattr(detector, "aggregator") and detector.aggregator is not None:
+            detector.fit(model, feature_layers_id=[-2, -1], fit_dataset=ds_fit)
+        else:
+            detector.fit(model, feature_layers_id=[-2], fit_dataset=ds_fit)
+    elif detector.requires_to_fit_dataset or detector.use_react:
+        detector.fit(model, fit_dataset=ds_fit)
+    else:
+        detector.fit(model)
+
+    # ood scores
+    scores_in, info_in = detector.score(ds_in)
+    scores_out, info_out = detector.score(ds_out)
+    assert scores_in.shape == (1028,)
+    assert info_in["labels"].shape == (1028,)
+    assert info_in["logits"].shape == (1028, 2)
+    assert scores_out.shape == (972,)
+    assert info_out["labels"].shape == (972,)
+    assert info_out["logits"].shape == (972, 2)
+
+    # ood metrics: auroc, fpr95tpr
+    metrics = bench_metrics(
+        (scores_in, scores_out),
+        metrics=["auroc", "fpr95tpr"],
+    )
+    auroc, fpr95tpr = metrics["auroc"], metrics["fpr95tpr"]
+    assert auroc >= auroc_thr, f"got a score of {auroc}, below {auroc_thr}!"
+    assert fpr95tpr <= fpr95_thr, f"got a score of {fpr95tpr}, above {fpr95_thr}!"
+
+    # react specific test
+    # /!\ do it at the end of the test because it may affect the detector's behaviour
+    if check_react_clipping:
+        assert detector.react_threshold is not None
+        penult_feat_extractor = detector.FeatureExtractorClass(
+            model=detector.feature_extractor.extractor, feature_layers_id=[-2]
+        )
+        penult_features, _ = penult_feat_extractor.predict(ds_fit)
+        assert tf.reduce_max(penult_features) <= detector.react_threshold, (
+            "Maximum value of penultimate features"
+            + f" ({tf.reduce_max(penult_features)}) should be less than or equal to"
+            + f" the react threshold value ({detector.react_threshold})"
+        )

tests/tests_tensorflow/tools_tf.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import numpy as np
+import tensorflow as tf
+
+
+def almost_equal(arr1, arr2, epsilon=1e-6):
+    """Ensure two array are almost equal at an epsilon"""
+    return np.mean(np.abs(arr1 - arr2)) < epsilon
+
+
+def generate_model(input_shape=(32, 32, 3), output_shape=10):
+    model = tf.keras.models.Sequential()
+    model.add(tf.keras.layers.Input(shape=input_shape))
+    model.add(tf.keras.layers.Conv2D(4, kernel_size=(2, 2), activation="relu"))
+    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
+    model.add(tf.keras.layers.Dropout(0.25))
+    model.add(tf.keras.layers.Flatten())
+    model.add(tf.keras.layers.Dense(output_shape))
+    model.compile(loss="categorical_crossentropy", optimizer="sgd")
+
+    return model
+
+
+def generate_regression_model(features_shape, output_shape=1):
+    model = tf.keras.models.Sequential()
+    model.add(tf.keras.layers.Input(shape=features_shape))
+    model.add(tf.keras.layers.Flatten())
+    model.add(tf.keras.layers.Dense(4, activation="relu"))
+    model.add(tf.keras.layers.Dense(4, activation="relu"))
+    model.add(tf.keras.layers.Dense(output_shape))
+    model.compile(loss="mean_absolute_error", optimizer="sgd", metrics=["accuracy"])
+
+    return model
+
+
+def simplest_mlp(num_features, num_classes):
+    return tf.keras.models.Sequential(
+        [
+            tf.keras.layers.Input(shape=(num_features,)),
+            tf.keras.layers.Dense(64, activation="relu"),
+            tf.keras.layers.Dense(num_classes, activation="softmax"),
+        ]
+    )
+
+
+def generate_data(x_shape=(32, 32, 3), num_labels=10, samples=100, one_hot=True):
+    x = np.random.rand(samples, *x_shape).astype(np.float32)
+    x /= np.max(x)
+    if one_hot:
+        y = np.eye(num_labels)[np.random.randint(0, num_labels, samples)]
+    else:
+        y = np.random.randint(0, num_labels, samples)
+
+    return x, y
+
+
+def generate_data_tf(
+    x_shape=(32, 32, 3), num_labels=10, samples=100, one_hot=True, as_supervised=True
+):
+    x, y = generate_data(x_shape, num_labels, samples, one_hot)
+    if as_supervised:
+        dataset = tf.data.Dataset.from_tensor_slices((x, y))
+    else:
+        dataset = tf.data.Dataset.from_tensor_slices({"input": x, "label": y})
+    return dataset

tests/tests_torch/__init__.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from .tools_torch import ComplexNet
+from .tools_torch import generate_data
+from .tools_torch import generate_data_torch
+from .tools_torch import named_sequential_model
+from .tools_torch import Net
+from .tools_torch import sequential_model
+from .torch_methods_utils import eval_detector_on_blobs
+
+__all__ = [
+    "ComplexNet",
+    "generate_data_torch",
+    "generate_data",
+    "named_sequential_model",
+    "Net",
+    "sequential_model",
+]

tests/tests_torch/tools_torch.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from collections import OrderedDict
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import TensorDataset
+
+
+def almost_equal(arr1, arr2, epsilon=1e-6):
+    """Ensure two array are almost equal at an epsilon"""
+    return np.mean(np.abs(arr1 - arr2)) < epsilon
+
+
+class Net(nn.Module):
+    def __init__(self, num_classes=10):
+        super().__init__()
+        self.conv1 = nn.Conv2d(3, 6, 5)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv2 = nn.Conv2d(6, 16, 5)
+        self.fc1 = nn.Linear(16 * 5 * 5, 120)
+        self.fc2 = nn.Linear(120, 84)
+        self.fc3 = nn.Linear(84, num_classes)
+
+    def forward(self, x):
+        x = self.pool(F.relu(self.conv1(x)))
+        x = self.pool(F.relu(self.conv2(x)))
+        x = torch.flatten(x, 1)  # flatten all dimensions except batch
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = self.fc3(x)
+        return x
+
+
+class ComplexNet(nn.Module):
+    def __init__(self, num_classes=10):
+        super().__init__()
+
+        self.feature_extractor = nn.Sequential(
+            OrderedDict(
+                [
+                    ("conv1", nn.Conv2d(3, 6, 5)),
+                    ("relu1", nn.ReLU()),
+                    ("pool1", nn.MaxPool2d(2, 2)),
+                    ("conv2", nn.Conv2d(6, 16, 5)),
+                    ("relu2", nn.ReLU()),
+                    ("pool2", nn.MaxPool2d(2, 2)),
+                    ("flatten", nn.Flatten()),
+                ]
+            )
+        )
+
+        self.fcs = nn.Sequential(
+            OrderedDict(
+                [
+                    ("fc1", nn.Linear(16 * 5 * 5, 120)),
+                    ("fc2", nn.Linear(120, 84)),
+                    ("fc3", nn.Linear(84, num_classes)),
+                ]
+            )
+        )
+
+    def forward(self, x):
+        x = self.feature_extractor(x)
+        x = self.fcs(x)
+        return x
+
+
+def sequential_model(num_classes=10):
+    return nn.Sequential(
+        nn.Conv2d(3, 6, 5),
+        nn.ReLU(),
+        nn.MaxPool2d(2, 2),
+        nn.Conv2d(6, 16, 5),
+        nn.ReLU(),
+        nn.MaxPool2d(2, 2),
+        nn.Flatten(),
+        nn.Linear(16 * 5 * 5, 120),
+        nn.Linear(120, 84),
+        nn.Linear(84, num_classes),
+    )
+
+
+def named_sequential_model(num_classes=10):
+    return nn.Sequential(
+        OrderedDict(
+            [
+                ("conv1", nn.Conv2d(3, 6, 5)),
+                ("relu1", nn.ReLU()),
+                ("pool1", nn.MaxPool2d(2, 2)),
+                ("conv2", nn.Conv2d(6, 16, 5)),
+                ("relu2", nn.ReLU()),
+                ("pool2", nn.MaxPool2d(2, 2)),
+                ("flatten", nn.Flatten()),
+                ("fc1", nn.Linear(16 * 5 * 5, 120)),
+                ("fc2", nn.Linear(120, 84)),
+                ("fc3", nn.Linear(84, num_classes)),
+            ]
+        )
+    )
+
+
+def simplest_mlp(num_features, num_classes=10):
+    return nn.Sequential(
+        nn.Linear(num_features, 64),
+        nn.ReLU(),
+        nn.Linear(64, num_classes),
+    )
+
+
+def generate_data(x_shape=(3, 32, 32), num_labels=10, samples=100, one_hot=True):
+    x = np.random.rand(samples, *x_shape).astype(np.float32)
+    x /= np.max(x)
+    if one_hot:
+        y = np.eye(num_labels)[np.random.randint(0, num_labels, samples)]
+    else:
+        y = np.random.randint(0, num_labels, samples)
+
+    return x, y
+
+
+def generate_data_torch(
+    x_shape=(3, 32, 32), num_labels=10, samples=100, one_hot=True, with_labels=True
+):
+    x, y = generate_data(x_shape, num_labels, samples, one_hot)
+    if with_labels:
+        return TensorDataset(torch.Tensor(x), torch.Tensor(y))
+    else:
+        return TensorDataset(torch.Tensor(x))

tests/tests_torch/torch_methods_utils.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import os
+
+import numpy as np
+import torch
+from sklearn.datasets import make_blobs
+from sklearn.model_selection import train_test_split
+
+from oodeel.datasets import load_data_handler
+from oodeel.eval.metrics import bench_metrics
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model_path = os.path.expanduser("~/") + ".oodeel/saved_models"
+data_path = os.path.expanduser("~/") + ".oodeel/datasets"
+os.makedirs(model_path, exist_ok=True)
+os.makedirs(data_path, exist_ok=True)
+
+
+def load_blobs_data(batch_size=128, num_samples=10000, train_ratio=0.8):
+    # === data hparams ===
+    num_classes = 3
+    in_labels = [0, 1]
+    out_labels = [2, 3]
+    centers = np.array([[-4, -4], [4, 4], [-4, 4], [4, -4]])
+
+    # === generate data ===
+    X, y = make_blobs(num_samples, num_classes, centers=centers, random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, train_size=train_ratio, random_state=0
+    )
+
+    # === id / ood split ===
+    handler = load_data_handler("torch")
+    blobs_train = handler.load_dataset((X_train, y_train))
+    blobs_test = handler.load_dataset((X_test, y_test))
+    oods_fit, _ = handler.split_by_class(blobs_train, in_labels, out_labels)
+    oods_in, oods_out = handler.split_by_class(blobs_test, in_labels, out_labels)
+
+    # === prepare data (shuffle, batch) => torch dataloaders ===
+    ds_fit = handler.prepare(oods_fit, batch_size=batch_size, shuffle=True)
+    ds_in = handler.prepare(oods_in, batch_size=batch_size)
+    ds_out = handler.prepare(oods_out, batch_size=batch_size)
+    return ds_fit, ds_in, ds_out
+
+
+def load_blob_mlp():
+    # load model
+    model = torch.hub.load_state_dict_from_url(
+        "https://github.com/deel-ai/oodeel/blob/assets/"
+        + "test_models/blobs_mlp.pt?raw=True",
+        map_location=device,
+    )
+    model.eval()
+    return model
+
+
+def eval_detector_on_blobs(
+    detector,
+    auroc_thr=0.6,
+    fpr95_thr=0.3,
+    batch_size=128,
+    check_react_clipping=False,
+):
+    # seed
+    torch.manual_seed(1)
+
+    # load data
+    ds_fit, ds_in, ds_out = load_blobs_data(batch_size)
+
+    # get classifier
+    model = load_blob_mlp()
+
+    # fit ood detector
+    if detector.requires_to_fit_dataset or detector.use_react:
+        if hasattr(detector, "aggregator") and detector.aggregator is not None:
+            detector.fit(model, feature_layers_id=[-2, -1], fit_dataset=ds_fit)
+        else:
+            detector.fit(model, feature_layers_id=[-2], fit_dataset=ds_fit)
+    else:
+        detector.fit(model)
+
+    # ood scores
+    scores_in, info_in = detector.score(ds_in)
+    scores_out, info_out = detector.score(ds_out)
+    assert scores_in.shape == (1028,)
+    assert info_in["labels"].shape == (1028,)
+    assert info_in["logits"].shape == (1028, 2)
+    assert scores_out.shape == (972,)
+    assert info_out["labels"].shape == (972,)
+    assert info_out["logits"].shape == (972, 2)
+
+    # ood metrics: auroc, fpr95tpr
+    metrics = bench_metrics(
+        (scores_in, scores_out),
+        metrics=["auroc", "fpr95tpr"],
+    )
+    auroc, fpr95tpr = metrics["auroc"], metrics["fpr95tpr"]
+    assert auroc >= auroc_thr, f"got a score of {auroc}, below {auroc_thr}!"
+    assert fpr95tpr <= fpr95_thr, f"got a score of {fpr95tpr}, above {fpr95_thr}!"
+
+    # react specific test
+    # /!\ do it at the end of the test because it may affect the detector's behaviour
+
+    if check_react_clipping:
+        assert detector.react_threshold is not None
+
+        penult_feat_extractor = detector._load_feature_extractor(
+            model=model, feature_layers_id=[-1]
+        )
+        # penult_feat_extractor._prepare_ood_handles()
+
+        def hook(_, input):
+            penult_feat_extractor._features["test"] = input[0]
+
+        penult_feat_extractor.model[-1].register_forward_pre_hook(hook)
+        for x, y in ds_fit:
+            _ = penult_feat_extractor.predict_tensor(x)
+            assert (
+                torch.max(penult_feat_extractor._features["test"])
+                <= detector.react_threshold
+            ), (
+                "Maximum value of penultimate features"
+                + f" ({torch.max(penult_feat_extractor._features)})"
+                + " should be less than or equal to the react threshold value"
+                + f" ({detector.react_threshold})"
+            )