monai-weekly 1.5.dev2505__py3-none-any.whl → 1.5.dev2507__py3-none-any.whl

tests/handlers/test_handler_checkpoint_saver.py

@@ -0,0 +1,233 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+
+import torch
+import torch.optim as optim
+from ignite.engine import Engine
+from parameterized import parameterized
+
+from monai.handlers import CheckpointLoader, CheckpointSaver
+
+TEST_CASE_1 = [
+    True,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    False,
+    False,
+    True,
+    0,
+    None,
+    ["test_checkpoint_final_iteration=40.pt"],
+]
+
+TEST_CASE_2 = [
+    False,
+    None,
+    True,
+    "val_loss",
+    2,
+    None,
+    False,
+    True,
+    False,
+    False,
+    0,
+    None,
+    ["test_checkpoint_key_metric=32.pt", "test_checkpoint_key_metric=40.pt"],
+]
+
+TEST_CASE_3 = [
+    False,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    True,
+    False,
+    True,
+    2,
+    2,
+    ["test_checkpoint_epoch=2.pt", "test_checkpoint_epoch=4.pt"],
+]
+
+TEST_CASE_4 = [
+    False,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    False,
+    False,
+    False,
+    10,
+    2,
+    ["test_checkpoint_iteration=30.pt", "test_checkpoint_iteration=40.pt"],
+]
+
+TEST_CASE_5 = [
+    True,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    False,
+    False,
+    True,
+    0,
+    None,
+    ["test_checkpoint_final_iteration=40.pt"],
+    True,
+]
+
+TEST_CASE_6 = [True, "final_model.pt", False, None, 1, None, False, False, False, True, 0, None, ["final_model.pt"]]
+
+TEST_CASE_7 = [False, None, True, "val_loss", 1, "model.pt", False, False, False, True, 0, None, ["model.pt"]]
+
+TEST_CASE_8 = [False, None, True, "val_loss", 1, "model.pt", False, True, False, True, 0, None, ["model.pt"]]
+
+
+class TestHandlerCheckpointSaver(unittest.TestCase):
+
+    @parameterized.expand(
+        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
+    )
+    def test_file(
+        self,
+        save_final,
+        final_filename,
+        save_key_metric,
+        key_metric_name,
+        key_metric_n_saved,
+        key_metric_filename,
+        key_metric_save_state,
+        key_metric_greater_or_equal,
+        key_metric_negative_sign,
+        epoch_level,
+        save_interval,
+        n_saved,
+        filenames,
+        multi_devices=False,
+    ):
+        data = [0] * 8
+
+        # set up engine
+        def _train_func(engine, batch):
+            engine.state.metrics["val_loss"] = engine.state.iteration
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        net = torch.nn.PReLU()
+        if multi_devices:
+            net = torch.nn.DataParallel(net)
+        optimizer = optim.SGD(net.parameters(), lr=0.02)
+        with tempfile.TemporaryDirectory() as tempdir:
+            handler = CheckpointSaver(
+                tempdir,
+                {"net": net, "opt": optimizer},
+                "CheckpointSaver",
+                "test",
+                save_final,
+                final_filename,
+                save_key_metric,
+                key_metric_name,
+                key_metric_n_saved,
+                key_metric_filename,
+                key_metric_save_state,
+                key_metric_greater_or_equal,
+                key_metric_negative_sign,
+                epoch_level,
+                save_interval,
+                n_saved,
+            )
+            handler.attach(engine)
+            engine.run(data, max_epochs=2)
+            engine.run(data, max_epochs=5)
+            for filename in filenames:
+                self.assertTrue(os.path.exists(os.path.join(tempdir, filename)))
+
+    def test_exception(self):
+        net = torch.nn.PReLU()
+
+        # set up engine
+        def _train_func(engine, batch):
+            raise RuntimeError("test exception.")
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        with tempfile.TemporaryDirectory() as tempdir:
+            stats_handler = CheckpointSaver(tempdir, {"net": net}, save_final=True)
+            stats_handler.attach(engine)
+
+            with self.assertRaises(RuntimeError):
+                engine.run(range(3), max_epochs=2)
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "net_final_iteration=1.pt")))
+
+    def test_load_state_dict(self):
+        net = torch.nn.PReLU()
+
+        # set up engine
+        def _train_func(engine, batch):
+            engine.state.metrics["val_loss"] = engine.state.iteration
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        with tempfile.TemporaryDirectory() as tempdir:
+            engine = Engine(_train_func)
+            CheckpointSaver(
+                save_dir=tempdir,
+                save_dict={"net": net},
+                save_key_metric=True,
+                key_metric_name="val_loss",
+                key_metric_n_saved=2,
+                key_metric_save_state=True,
+                key_metric_negative_sign=True,
+            ).attach(engine)
+            engine.run(range(3), max_epochs=3)
+
+            saver = CheckpointSaver(
+                save_dir=tempdir,
+                save_dict={"net": net},
+                save_key_metric=True,
+                key_metric_name="val_loss",
+                key_metric_n_saved=2,
+                key_metric_negative_sign=True,
+            )
+            engine = Engine(_train_func)
+            CheckpointLoader(os.path.join(tempdir, "net_key_metric=-6.pt"), {"checkpointer": saver}).attach(engine)
+            engine.run(range(1), max_epochs=1)
+
+            resumed = saver._key_metric_checkpoint._saved
+            for i in range(2):
+                self.assertEqual(resumed[1 - i].priority, -3 * (i + 1))
+                self.assertEqual(resumed[1 - i].filename, f"net_key_metric=-{3 * (i + 1)}.pt")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_classification_saver.py

@@ -0,0 +1,64 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import csv
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+from ignite.engine import Engine
+
+from monai.data import decollate_batch
+from monai.data.csv_saver import CSVSaver
+from monai.handlers import ClassificationSaver
+
+
+class TestHandlerClassificationSaver(unittest.TestCase):
+
+    def test_saved_content(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                engine.state.batch = decollate_batch(batch)
+                return [torch.zeros(1) for _ in range(8)]
+
+            engine = Engine(_train_func)
+
+            # set up testing handler
+            saver = CSVSaver(output_dir=tempdir, filename="predictions2.csv", delimiter="\t")
+            ClassificationSaver(output_dir=tempdir, filename="predictions1.csv", delimiter="\t").attach(engine)
+            ClassificationSaver(saver=saver).attach(engine)
+
+            data = [{"filename_or_obj": ["testfile" + str(i) for i in range(8)]}]
+            engine.run(data, max_epochs=1)
+
+            def _test_file(filename):
+                filepath = os.path.join(tempdir, filename)
+                self.assertTrue(os.path.exists(filepath))
+                with open(filepath) as f:
+                    reader = csv.reader(f, delimiter="\t")
+                    i = 0
+                    for row in reader:
+                        self.assertEqual(row[0], "testfile" + str(i))
+                        self.assertEqual(np.array(row[1:]).astype(np.float32), 0.0)
+                        i += 1
+                    self.assertEqual(i, 8)
+
+            _test_file("predictions1.csv")
+            _test_file("predictions2.csv")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_classification_saver_dist.py

@@ -0,0 +1,77 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import csv
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+import torch.distributed as dist
+from ignite.engine import Engine
+
+from monai.data import decollate_batch
+from monai.handlers import ClassificationSaver
+from tests.test_utils import DistCall, DistTestCase
+
+
+class DistributedHandlerClassificationSaver(DistTestCase):
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_saved_content(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            rank = dist.get_rank()
+
+            # set up engine
+            def _train_func(engine, batch):
+                engine.state.batch = decollate_batch(batch)
+                return [torch.zeros(1) for _ in range(8 + rank * 2)]
+
+            engine = Engine(_train_func)
+
+            # set up testing handler
+            saver = ClassificationSaver(output_dir=tempdir, filename="predictions.csv", save_rank=1)
+            saver.attach(engine)
+
+            # rank 0 has 8 images, rank 1 has 10 images
+            data = [
+                {
+                    "filename_or_obj": ["testfile" + str(i) for i in range(8 * rank, (8 + rank) * (rank + 1))],
+                    "data_shape": torch.ones((8 + rank * 2, 1, 1)),
+                }
+            ]
+            # rank 1 has more iterations
+            if rank == 1:
+                data.append(
+                    {
+                        "filename_or_obj": ["testfile" + str(i) for i in range(18, 28)],
+                        "data_shape": torch.ones((10, 1, 1)),
+                    }
+                )
+
+            engine.run(data, max_epochs=1)
+            filepath = os.path.join(tempdir, "predictions.csv")
+            if rank == 1:
+                self.assertTrue(os.path.exists(filepath))
+                with open(filepath) as f:
+                    reader = csv.reader(f)
+                    i = 0
+                    for row in reader:
+                        self.assertEqual(row[0], "testfile" + str(i))
+                        self.assertEqual(np.array(row[1:]).astype(np.float32), 0.0)
+                        i += 1
+                    self.assertEqual(i, 28)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_clearml_image.py

@@ -0,0 +1,65 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from os import environ
+
+from monai.handlers import ClearMLImageHandler
+from monai.utils import optional_import
+
+Task, has_clearml = optional_import("clearml", name="Task")
+get_active_config_file, has_get_active_config_file = optional_import(
+    "clearml.backend_config.defs", name="get_active_config_file"
+)
+_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
+
+
+@unittest.skipUnless(has_clearml, "Requires 'clearml' installation")
+@unittest.skipUnless(has_tb, "Requires SummaryWriter installation")
+@unittest.skipIf(not has_get_active_config_file, "ClearML 'get_active_config_file' not found")
+class TestHandlerClearMLImageHandler(unittest.TestCase):
+
+    def test_task_init(self):
+        handle, path = tempfile.mkstemp()
+        with open(handle, "w") as new_config:
+            if get_active_config_file():
+                with open(get_active_config_file()) as old_config:
+                    new_config.write(old_config.read())
+            new_config.write(
+                "\nsdk.development.vcs_repo_detect_async: false\nsdk.development.report_use_subprocess: false\n"
+            )
+        environ["CLEARML_CONFIG_FILE"] = path
+        try:
+            Task.force_store_standalone_script(True)
+            Task.set_offline(offline_mode=True)
+            ClearMLImageHandler(
+                project_name="MONAI",
+                task_name="monai_experiment",
+                output_uri=True,
+                tags=None,
+                reuse_last_task_id=True,
+                continue_last_task=False,
+                auto_connect_frameworks=True,
+                auto_connect_arg_parser=False,
+            )
+        except Exception as exc:
+            self.fail(exc)
+        self.assertEqual(Task.current_task().name, "monai_experiment")
+        self.assertEqual(Task.current_task()._project_name[1], "MONAI")
+        # Close ClearML Task
+        Task.current_task().close()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_clearml_stats.py

@@ -0,0 +1,65 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from os import environ
+
+from monai.handlers import ClearMLStatsHandler
+from monai.utils import optional_import
+
+Task, has_clearml = optional_import("clearml", name="Task")
+get_active_config_file, has_get_active_config_file = optional_import(
+    "clearml.backend_config.defs", name="get_active_config_file"
+)
+_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
+
+
+@unittest.skipUnless(has_clearml, "Requires 'clearml' installation")
+@unittest.skipUnless(has_tb, "Requires SummaryWriter installation")
+@unittest.skipIf(not has_get_active_config_file, "ClearML 'get_active_config_file' not found")
+class TestHandlerClearMLStatsHandler(unittest.TestCase):
+
+    def test_task_init(self):
+        handle, path = tempfile.mkstemp()
+        with open(handle, "w") as new_config:
+            if get_active_config_file():
+                with open(get_active_config_file()) as old_config:
+                    new_config.write(old_config.read())
+            new_config.write(
+                "\nsdk.development.vcs_repo_detect_async: false\nsdk.development.report_use_subprocess: false\n"
+            )
+        environ["CLEARML_CONFIG_FILE"] = path
+        try:
+            Task.force_store_standalone_script(True)
+            Task.set_offline(offline_mode=True)
+            ClearMLStatsHandler(
+                project_name="MONAI",
+                task_name="monai_experiment",
+                output_uri=True,
+                tags=None,
+                reuse_last_task_id=True,
+                continue_last_task=False,
+                auto_connect_frameworks=True,
+                auto_connect_arg_parser=False,
+            )
+        except Exception as exc:
+            self.fail(exc)
+        self.assertEqual(Task.current_task().name, "monai_experiment")
+        self.assertEqual(Task.current_task()._project_name[1], "MONAI")
+        # Close ClearML Task
+        Task.current_task().close()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_confusion_matrix.py

@@ -0,0 +1,104 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+from typing import Any
+
+import torch
+from ignite.engine import Engine
+from parameterized import parameterized
+
+from monai.handlers import ConfusionMatrix
+from tests.test_utils import assert_allclose
+
+TEST_CASE_1 = [{"include_background": True, "save_details": False, "metric_name": "f1"}, 0.75]
+TEST_CASE_2 = [{"include_background": False, "save_details": False, "metric_name": "ppv"}, 1.0]
+TEST_CASE_3 = [{"save_details": False, "metric_name": "f1", "reduction": "mean_batch"}, torch.tensor([0.6667, 0.8000])]
+TEST_CASE_SEG_1 = [{"include_background": True, "metric_name": "tpr"}, 0.7]
+
+data_1: dict[Any, Any] = {
+    "y_pred": torch.tensor(
+        [
+            [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]],
+            [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]],
+        ]
+    ),
+    "y": torch.tensor(
+        [
+            [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]],
+            [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]],
+        ]
+    ),
+}
+
+data_2: dict[Any, Any] = {
+    "y_pred": torch.tensor([[[[0.0, 1.0], [1.0, 0.0]], [[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [0.0, 0.0]]]]),
+    "y": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]]]]),
+}
+
+
+class TestHandlerConfusionMatrix(unittest.TestCase):
+    # TODO test multi node averaged confusion matrix
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_compute(self, input_params, expected_avg):
+        metric = ConfusionMatrix(**input_params)
+        # test input a list of channel-first tensor
+        y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])]
+        y = [torch.Tensor([[0], [1]]), torch.Tensor([[0], [1]])]
+        metric.update([y_pred, y])
+
+        y_pred = torch.Tensor([[[0], [1]], [[1], [0]]])
+        y = torch.Tensor([[[0], [1]], [[1], [0]]])
+        metric.update([y_pred, y])
+
+        avg_metric = metric.compute()
+        assert_allclose(avg_metric, expected_avg, atol=1e-4, rtol=1e-4, type_test=False)
+
+    @parameterized.expand([TEST_CASE_SEG_1])
+    def test_compute_seg(self, input_params, expected_avg):
+        metric = ConfusionMatrix(**input_params)
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        metric.attach(engine, "confusion_matrix")
+
+        y_pred = data_1["y_pred"]
+        y = data_1["y"]
+        metric.update([y_pred, y])
+
+        y_pred = data_2["y_pred"]
+        y = data_2["y"]
+        metric.update([y_pred, y])
+
+        avg_metric = metric.compute()
+        self.assertAlmostEqual(avg_metric, expected_avg, places=4)
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_shape_mismatch(self, input_params, _expected):
+        metric = ConfusionMatrix(**input_params)
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = torch.Tensor([[0, 1], [1, 0]])
+            y = torch.ones((2, 3))
+            metric.update([y_pred, y])
+
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = torch.Tensor([[0, 1], [1, 0]])
+            y = torch.ones((3, 2))
+            metric.update([y_pred, y])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_confusion_matrix_dist.py

@@ -0,0 +1,70 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import numpy as np
+import torch
+import torch.distributed as dist
+from ignite.engine import Engine
+
+from monai.handlers import ConfusionMatrix
+from tests.test_utils import DistCall, DistTestCase
+
+
+class DistributedConfusionMatrix(DistTestCase):
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_compute(self):
+        self._compute()
+
+    def _compute(self):
+        device = f"cuda:{dist.get_rank()}" if torch.cuda.is_available() else "cpu"
+        metric = ConfusionMatrix(include_background=True, metric_name="tpr")
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        metric.attach(engine, "confusion_matrix")
+        if dist.get_rank() == 0:
+            y_pred = torch.tensor(
+                [
+                    [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]],
+                    [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]],
+                ],
+                device=device,
+            )
+            y = torch.tensor(
+                [
+                    [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]],
+                    [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]],
+                ],
+                device=device,
+            )
+            metric.update([y_pred, y])
+
+        if dist.get_rank() == 1:
+            y_pred = torch.tensor(
+                [[[[0.0, 1.0], [1.0, 0.0]], [[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [0.0, 0.0]]]], device=device
+            )
+            y = torch.tensor(
+                [[[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]]]], device=device
+            )
+            metric.update([y_pred, y])
+
+        avg_metric = metric.compute()
+        np.testing.assert_allclose(avg_metric, 0.7, rtol=1e-04, atol=1e-04)
+
+
+if __name__ == "__main__":
+    unittest.main()