monai-weekly 1.5.dev2506__py3-none-any.whl → 1.5.dev2508__py3-none-any.whl

tests/handlers/test_handler_stats.py

@@ -0,0 +1,281 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import tempfile
+import unittest
+from io import StringIO
+
+import torch
+from ignite.engine import Engine, Events
+from parameterized import parameterized
+
+from monai.handlers import StatsHandler
+
+
+def get_event_filter(e):
+
+    def event_filter(_, event):
+        if event in e:
+            return True
+        return False
+
+    return event_filter
+
+
+class TestHandlerStats(unittest.TestCase):
+
+    @parameterized.expand([[True], [get_event_filter([1, 2])]])
+    def test_metrics_print(self, epoch_log):
+        log_stream = StringIO()
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
+        key_to_handler = "test_logging"
+        key_to_print = "testing_metric"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return [torch.tensor(0.0)]
+
+        engine = Engine(_train_func)
+
+        # set up dummy metric
+        @engine.on(Events.EPOCH_COMPLETED)
+        def _update_metric(engine):
+            current_metric = engine.state.metrics.get(key_to_print, 0.1)
+            engine.state.metrics[key_to_print] = current_metric + 0.1
+
+        # set up testing handler
+        logger = logging.getLogger(key_to_handler)
+        logger.setLevel(logging.INFO)
+        logger.addHandler(log_handler)
+        stats_handler = StatsHandler(iteration_log=False, epoch_log=epoch_log, name=key_to_handler)
+        stats_handler.attach(engine)
+
+        max_epochs = 4
+        engine.run(range(3), max_epochs=max_epochs)
+
+        # check logging output
+        output_str = log_stream.getvalue()
+        log_handler.close()
+        has_key_word = re.compile(f".*{key_to_print}.*")
+        content_count = 0
+        for line in output_str.split("\n"):
+            if has_key_word.match(line):
+                content_count += 1
+        if epoch_log is True:
+            self.assertEqual(content_count, max_epochs)
+        else:
+            self.assertEqual(content_count, 2)  # 2 = len([1, 2]) from event_filter
+
+    @parameterized.expand([[True], [get_event_filter([1, 3])]])
+    def test_loss_print(self, iteration_log):
+        log_stream = StringIO()
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
+        key_to_handler = "test_logging"
+        key_to_print = "myLoss"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return [torch.tensor(0.0)]
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        logger = logging.getLogger(key_to_handler)
+        logger.setLevel(logging.INFO)
+        logger.addHandler(log_handler)
+        stats_handler = StatsHandler(
+            iteration_log=iteration_log, epoch_log=False, name=key_to_handler, tag_name=key_to_print
+        )
+        stats_handler.attach(engine)
+
+        num_iters = 3
+        max_epochs = 2
+        engine.run(range(num_iters), max_epochs=max_epochs)
+
+        # check logging output
+        output_str = log_stream.getvalue()
+        log_handler.close()
+        has_key_word = re.compile(f".*{key_to_print}.*")
+        content_count = 0
+        for line in output_str.split("\n"):
+            if has_key_word.match(line):
+                content_count += 1
+        if iteration_log is True:
+            self.assertEqual(content_count, num_iters * max_epochs)
+        else:
+            self.assertEqual(content_count, 2)  # 2 = len([1, 3]) from event_filter
+
+    def test_loss_dict(self):
+        log_stream = StringIO()
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
+        key_to_handler = "test_logging"
+        key_to_print = "myLoss1"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return [torch.tensor(0.0)]
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        logger = logging.getLogger(key_to_handler)
+        logger.setLevel(logging.INFO)
+        logger.addHandler(log_handler)
+        stats_handler = StatsHandler(name=key_to_handler, output_transform=lambda x: {key_to_print: x[0]})
+        stats_handler.attach(engine)
+
+        engine.run(range(3), max_epochs=2)
+
+        # check logging output
+        output_str = log_stream.getvalue()
+        log_handler.close()
+        has_key_word = re.compile(f".*{key_to_print}.*")
+        content_count = 0
+        for line in output_str.split("\n"):
+            if has_key_word.match(line):
+                content_count += 1
+        self.assertGreater(content_count, 0)
+
+    def test_loss_file(self):
+        key_to_handler = "test_logging"
+        key_to_print = "myLoss"
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            filename = os.path.join(tempdir, "test_loss_stats.log")
+            handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
+
+            # set up engine
+            def _train_func(engine, batch):
+                return [torch.tensor(0.0)]
+
+            engine = Engine(_train_func)
+
+            # set up testing handler
+            logger = logging.getLogger(key_to_handler)
+            logger.setLevel(logging.INFO)
+            logger.addHandler(handler)
+            stats_handler = StatsHandler(name=key_to_handler, tag_name=key_to_print)
+            stats_handler.attach(engine)
+
+            engine.run(range(3), max_epochs=2)
+            handler.close()
+            stats_handler.logger.removeHandler(handler)
+            with open(filename) as f:
+                output_str = f.read()
+                has_key_word = re.compile(f".*{key_to_print}.*")
+                content_count = 0
+                for line in output_str.split("\n"):
+                    if has_key_word.match(line):
+                        content_count += 1
+                self.assertGreater(content_count, 0)
+
+    def test_exception(self):
+        # set up engine
+        def _train_func(engine, batch):
+            raise RuntimeError("test exception.")
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        stats_handler = StatsHandler()
+        stats_handler.attach(engine)
+
+        with self.assertRaises(RuntimeError):
+            engine.run(range(3), max_epochs=2)
+
+    def test_attributes_print(self):
+        log_stream = StringIO()
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
+        key_to_handler = "test_logging"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return [torch.tensor(0.0)]
+
+        engine = Engine(_train_func)
+
+        # set up dummy metric
+        @engine.on(Events.EPOCH_COMPLETED)
+        def _update_metric(engine):
+            if not hasattr(engine.state, "test1"):
+                engine.state.test1 = 0.1
+                engine.state.test2 = 0.2
+            else:
+                engine.state.test1 += 0.1
+                engine.state.test2 += 0.2
+
+        # set up testing handler
+        logger = logging.getLogger(key_to_handler)
+        logger.setLevel(logging.INFO)
+        logger.addHandler(log_handler)
+        stats_handler = StatsHandler(name=key_to_handler, state_attributes=["test1", "test2", "test3"])
+        stats_handler.attach(engine)
+
+        engine.run(range(3), max_epochs=2)
+
+        # check logging output
+        output_str = log_stream.getvalue()
+        log_handler.close()
+        has_key_word = re.compile(".*State values.*")
+        content_count = 0
+        for line in output_str.split("\n"):
+            if has_key_word.match(line):
+                content_count += 1
+        self.assertGreater(content_count, 0)
+
+    def test_default_logger(self):
+        log_stream = StringIO()
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
+        key_to_print = "myLoss"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return [torch.tensor(0.0)]
+
+        engine = Engine(_train_func)
+        engine.logger.addHandler(log_handler)
+
+        # set up testing handler
+        stats_handler = StatsHandler(name=None, tag_name=key_to_print)
+        engine.logger.setLevel(logging.WARNING)
+        with self.assertWarns(Warning):  # engine logging level warn
+            stats_handler.attach(engine)
+        # leverage `engine.logger` to print info
+        engine.logger.setLevel(logging.INFO)
+        level = logging.root.getEffectiveLevel()
+        logging.basicConfig(level=logging.INFO)
+        engine.run(range(3), max_epochs=2)
+        logging.basicConfig(level=level)
+
+        # check logging output
+        output_str = log_stream.getvalue()
+        log_handler.close()
+        has_key_word = re.compile(f".*{key_to_print}.*")
+        content_count = 0
+        for line in output_str.split("\n"):
+            if has_key_word.match(line):
+                content_count += 1
+        self.assertGreater(content_count, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_surface_distance.py

@@ -0,0 +1,113 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import numpy as np
+import torch
+from ignite.engine import Engine
+
+from monai.handlers import SurfaceDistance
+from tests.test_utils import assert_allclose
+
+
+def create_spherical_seg_3d(
+    radius: float = 20.0, centre: tuple[int, int, int] = (49, 49, 49), im_shape: tuple[int, int, int] = (99, 99, 99)
+) -> np.ndarray:
+    """
+    Return a 3D image with a sphere inside. Voxel values will be
+    1 inside the sphere, and 0 elsewhere.
+
+    Args:
+        radius: radius of sphere (in terms of number of voxels, can be partial)
+        centre: location of sphere centre.
+        im_shape: shape of image to create
+
+    See also:
+        :py:meth:`~create_test_image_3d`
+    """
+    # Create image
+    image = np.zeros(im_shape, dtype=np.int32)
+    spy, spx, spz = np.ogrid[
+        -centre[0] : im_shape[0] - centre[0], -centre[1] : im_shape[1] - centre[1], -centre[2] : im_shape[2] - centre[2]
+    ]
+    circle = (spx * spx + spy * spy + spz * spz) <= radius * radius
+
+    image[circle] = 1
+    image[~circle] = 0
+    return image
+
+
+sampler_sphere = torch.Tensor(create_spherical_seg_3d(radius=20, centre=(20, 20, 20))).unsqueeze(0).unsqueeze(0)
+# test input a list of channel-first tensor
+sampler_sphere_gt = [torch.Tensor(create_spherical_seg_3d(radius=20, centre=(10, 20, 20))).unsqueeze(0)]
+sampler_sphere_zeros = torch.zeros_like(sampler_sphere)
+
+TEST_SAMPLE_1 = [sampler_sphere, sampler_sphere_gt]
+TEST_SAMPLE_2 = [sampler_sphere_gt, sampler_sphere_gt]
+TEST_SAMPLE_3 = [sampler_sphere_zeros, sampler_sphere_gt]
+TEST_SAMPLE_4 = [sampler_sphere_zeros, sampler_sphere_zeros]
+
+
+class TestHandlerSurfaceDistance(unittest.TestCase):
+    # TODO test multi node Surface Distance
+
+    def test_compute(self):
+        sur_metric = SurfaceDistance(include_background=True)
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        sur_metric.attach(engine, "surface_distance")
+
+        y_pred, y = TEST_SAMPLE_1
+        sur_metric.update([y_pred, y])
+        self.assertAlmostEqual(sur_metric.compute(), 4.17133, places=4)
+        y_pred, y = TEST_SAMPLE_2
+        sur_metric.update([y_pred, y])
+        self.assertAlmostEqual(sur_metric.compute(), 2.08566, places=4)
+        y_pred, y = TEST_SAMPLE_3
+        sur_metric.update([y_pred, y])
+        self.assertAlmostEqual(sur_metric.compute(), float("inf"))
+        y_pred, y = TEST_SAMPLE_4
+        sur_metric.update([y_pred, y])
+        self.assertAlmostEqual(sur_metric.compute(), float("inf"))
+
+    def test_shape_mismatch(self):
+        sur_metric = SurfaceDistance(include_background=True)
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = TEST_SAMPLE_1[0]
+            y = torch.ones((1, 1, 10, 10, 10))
+            sur_metric.update([y_pred, y])
+
+    def test_reduction(self):
+        sur_metric = SurfaceDistance(include_background=True, reduction="mean_channel")
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        sur_metric.attach(engine, "surface_distance")
+
+        y_pred, y = TEST_SAMPLE_1
+        sur_metric.update([y_pred, y])
+        y_pred, y = TEST_SAMPLE_2
+        sur_metric.update([y_pred, y])
+        assert_allclose(
+            sur_metric.compute().float(), torch.tensor([4.1713, 0.0000]), atol=1e-4, rtol=1e-4, type_test=False
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_tb_image.py

@@ -0,0 +1,61 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import glob
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+from ignite.engine import Engine, Events
+from parameterized import parameterized
+
+from monai.data import decollate_batch
+from monai.handlers import TensorBoardImageHandler
+from monai.utils import optional_import
+from tests.test_utils import SkipIfBeforePyTorchVersion
+
+_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
+
+TEST_CASES = [[[20, 20]], [[2, 20, 20]], [[3, 20, 20]], [[20, 20, 20]], [[2, 20, 20, 20]], [[2, 2, 20, 20, 20]]]
+
+
+@unittest.skipUnless(has_tb, "Requires SummaryWriter installation")
+@SkipIfBeforePyTorchVersion((1, 13))  # issue 6683
+class TestHandlerTBImage(unittest.TestCase):
+    @parameterized.expand(TEST_CASES)
+    def test_tb_image_shape(self, shape):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                engine.state.batch = decollate_batch(list(batch))
+                return [torch.zeros((1, 10, 10))]
+
+            engine = Engine(_train_func)
+
+            # set up testing handler
+            stats_handler = TensorBoardImageHandler(log_dir=tempdir)
+            engine.add_event_handler(Events.ITERATION_COMPLETED, stats_handler)
+
+            data = zip(
+                torch.as_tensor(np.random.normal(size=(10, 4, *shape))),
+                torch.as_tensor(np.random.normal(size=(10, 4, *shape))),
+            )
+            engine.run(data, epoch_length=10, max_epochs=1)
+            stats_handler.close()
+
+            self.assertTrue(len(glob.glob(tempdir)) > 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_tb_stats.py

@@ -0,0 +1,166 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import glob
+import tempfile
+import unittest
+from unittest.mock import MagicMock
+
+from ignite.engine import Engine, Events
+from parameterized import parameterized
+
+from monai.handlers import TensorBoardStatsHandler
+from monai.utils import optional_import
+
+SummaryWriter, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
+
+
+def get_event_filter(e):
+
+    def event_filter(_, event):
+        if event in e:
+            return True
+        return False
+
+    return event_filter
+
+
+@unittest.skipUnless(has_tb, "Requires SummaryWriter installation")
+class TestHandlerTBStats(unittest.TestCase):
+
+    def test_metrics_print(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                return [batch + 1.0]
+
+            engine = Engine(_train_func)
+
+            # set up dummy metric
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _update_metric(engine):
+                current_metric = engine.state.metrics.get("acc", 0.1)
+                engine.state.metrics["acc"] = current_metric + 0.1
+
+            # set up testing handler
+            stats_handler = TensorBoardStatsHandler(log_dir=tempdir, iteration_log=False, epoch_log=True)
+            stats_handler.attach(engine)
+            engine.run(range(3), max_epochs=2)
+            stats_handler.close()
+            # check logging output
+            self.assertTrue(len(glob.glob(tempdir)) > 0)
+
+    @parameterized.expand([[True], [get_event_filter([1, 2])]])
+    def test_metrics_print_mock(self, epoch_log):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                return [batch + 1.0]
+
+            engine = Engine(_train_func)
+
+            # set up dummy metric
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _update_metric(engine):
+                current_metric = engine.state.metrics.get("acc", 0.1)
+                engine.state.metrics["acc"] = current_metric + 0.1
+
+            # set up testing handler
+            stats_handler = TensorBoardStatsHandler(log_dir=tempdir, iteration_log=False, epoch_log=epoch_log)
+            stats_handler._default_epoch_writer = MagicMock()
+            stats_handler.attach(engine)
+
+            max_epochs = 4
+            engine.run(range(3), max_epochs=max_epochs)
+            stats_handler.close()
+            # check logging output
+            if epoch_log is True:
+                self.assertEqual(stats_handler._default_epoch_writer.call_count, max_epochs)
+            else:
+                self.assertEqual(stats_handler._default_epoch_writer.call_count, 2)  # 2 = len([1, 2]) from event_filter
+
+    def test_metrics_writer(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                return [batch + 1.0]
+
+            engine = Engine(_train_func)
+
+            # set up dummy metric
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _update_metric(engine):
+                current_metric = engine.state.metrics.get("acc", 0.1)
+                engine.state.metrics["acc"] = current_metric + 0.1
+                engine.state.test = current_metric
+
+            # set up testing handler
+            writer = SummaryWriter(log_dir=tempdir)
+            stats_handler = TensorBoardStatsHandler(
+                summary_writer=writer,
+                iteration_log=True,
+                epoch_log=False,
+                output_transform=lambda x: {"loss": x[0] * 2.0},
+                global_epoch_transform=lambda x: x * 3.0,
+                state_attributes=["test"],
+            )
+            stats_handler.attach(engine)
+            engine.run(range(3), max_epochs=2)
+            writer.close()
+            # check logging output
+            self.assertTrue(len(glob.glob(tempdir)) > 0)
+
+    @parameterized.expand([[True], [get_event_filter([1, 3])]])
+    def test_metrics_writer_mock(self, iteration_log):
+        with tempfile.TemporaryDirectory() as tempdir:
+            # set up engine
+            def _train_func(engine, batch):
+                return [batch + 1.0]
+
+            engine = Engine(_train_func)
+
+            # set up dummy metric
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _update_metric(engine):
+                current_metric = engine.state.metrics.get("acc", 0.1)
+                engine.state.metrics["acc"] = current_metric + 0.1
+                engine.state.test = current_metric
+
+            # set up testing handler
+            writer = SummaryWriter(log_dir=tempdir)
+            stats_handler = TensorBoardStatsHandler(
+                summary_writer=writer,
+                iteration_log=iteration_log,
+                epoch_log=False,
+                output_transform=lambda x: {"loss": x[0] * 2.0},
+                global_epoch_transform=lambda x: x * 3.0,
+                state_attributes=["test"],
+            )
+            stats_handler._default_iteration_writer = MagicMock()
+            stats_handler.attach(engine)
+
+            num_iters = 3
+            max_epochs = 2
+            engine.run(range(num_iters), max_epochs=max_epochs)
+            writer.close()
+
+            if iteration_log is True:
+                self.assertEqual(stats_handler._default_iteration_writer.call_count, num_iters * max_epochs)
+            else:
+                self.assertEqual(
+                    stats_handler._default_iteration_writer.call_count, 2
+                )  # 2 = len([1, 3]) from event_filter
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/handlers/test_handler_validation.py

@@ -0,0 +1,59 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import torch
+from ignite.engine import Engine
+
+from monai.data import Dataset
+from monai.engines import Evaluator
+from monai.handlers import ValidationHandler
+
+
+class TestEvaluator(Evaluator):
+    __test__ = False  # indicate to pytest that this class is not intended for collection
+
+    def _iteration(self, engine, batchdata):
+        engine.state.output = "called"
+        return engine.state.output
+
+
+class TestHandlerValidation(unittest.TestCase):
+
+    def test_content(self):
+        data = [0] * 8
+
+        # set up engine
+        def _train_func(engine, batch):
+            pass
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        val_data_loader = torch.utils.data.DataLoader(Dataset(data))
+        evaluator = TestEvaluator(torch.device("cpu:0"), val_data_loader)
+        ValidationHandler(interval=2, validator=evaluator, exec_at_start=True).attach(engine)
+        # test execution at start
+        engine.run(data, max_epochs=1)
+        self.assertEqual(evaluator.state.max_epochs, 1)
+        self.assertEqual(evaluator.state.epoch_length, 8)
+        self.assertEqual(evaluator.state.output, "called")
+
+        engine.run(data, max_epochs=5)
+        self.assertEqual(evaluator.state.max_epochs, 4)
+        self.assertEqual(evaluator.state.epoch_length, 8)
+
+
+if __name__ == "__main__":
+    unittest.main()