monai-weekly 1.5.dev2506__py3-none-any.whl → 1.5.dev2507__py3-none-any.whl

tests/optimizers/test_lr_finder.py

@@ -0,0 +1,108 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import pickle
+import random
+import sys
+import unittest
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import torch
+from torch.utils.data import DataLoader
+
+from monai.apps import MedNISTDataset
+from monai.networks.nets import DenseNet
+from monai.optimizers import LearningRateFinder
+from monai.transforms import Compose, EnsureChannelFirstd, LoadImaged, ScaleIntensityd, ToTensord
+from monai.utils import optional_import, set_determinism
+from monai.utils.misc import MONAIEnvVars
+from tests.test_utils import skip_if_downloading_fails
+
+if TYPE_CHECKING:
+    import matplotlib.pyplot as plt
+
+    has_matplotlib = True
+    has_pil = True
+else:
+    plt, has_matplotlib = optional_import("matplotlib.pyplot")
+    _, has_pil = optional_import("PIL.Image")
+
+RAND_SEED = 42
+random.seed(RAND_SEED)
+set_determinism(seed=RAND_SEED)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+@unittest.skipUnless(sys.platform == "linux", "requires linux")
+@unittest.skipUnless(has_pil, "requires PIL")
+class TestLRFinder(unittest.TestCase):
+    def setUp(self):
+        self.root_dir = MONAIEnvVars.data_dir()
+        if not self.root_dir:
+            self.root_dir = Path(__file__).parents[1] / "testing_data"
+
+        self.transforms = Compose(
+            [
+                LoadImaged(keys="image"),
+                EnsureChannelFirstd(keys="image", channel_dim="no_channel"),
+                ScaleIntensityd(keys="image"),
+                ToTensord(keys="image"),
+            ]
+        )
+
+    def test_lr_finder(self):
+        # 0.001 gives 54 examples
+        with skip_if_downloading_fails():
+            train_ds = MedNISTDataset(
+                root_dir=self.root_dir,
+                transform=self.transforms,
+                section="validation",
+                val_frac=0.001,
+                download=True,
+                num_workers=2,
+            )
+        train_loader = DataLoader(train_ds, batch_size=300, shuffle=True, num_workers=2)
+        num_classes = train_ds.get_num_classes()
+
+        model = DenseNet(
+            spatial_dims=2, in_channels=1, out_channels=num_classes, init_features=2, growth_rate=2, block_config=(2,)
+        )
+        loss_function = torch.nn.CrossEntropyLoss()
+        learning_rate = 1e-5
+        optimizer = torch.optim.Adam(model.parameters(), learning_rate)
+
+        lr_finder = LearningRateFinder(
+            model=model,
+            optimizer=optimizer,
+            criterion=loss_function,
+            device=device,
+            pickle_module=pickle,
+            pickle_protocol=4,
+        )
+        lr_finder.range_test(train_loader, val_loader=train_loader, end_lr=10.0, num_iter=5)
+        print(lr_finder.get_steepest_gradient(0, 0)[0])
+
+        if has_matplotlib:
+            ax = plt.subplot()
+            plt.show(block=False)
+            lr_finder.plot(0, 0, ax=ax)  # to inspect the loss-learning rate graph
+            plt.pause(3)
+            plt.close()
+
+        lr_finder.reset()  # to reset the model and optimizer to their initial state
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/optimizers/test_lr_scheduler.py

@@ -0,0 +1,71 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import torch
+from parameterized import parameterized
+
+from monai.optimizers.lr_scheduler import WarmupCosineSchedule
+
+
+class SchedulerTestNet(torch.nn.Module):
+
+    def __init__(self):
+        super().__init__()
+        self.conv1 = torch.nn.Conv2d(1, 1, 1)
+        self.conv2 = torch.nn.Conv2d(1, 1, 1)
+
+    def forward(self, x):
+        return self.conv2(torch.nn.functional.relu(self.conv1(x)))
+
+
+TEST_CASE_LRSCHEDULER = [
+    [{"warmup_steps": 2, "t_total": 10}, [0.000, 0.500, 1.00, 0.962, 0.854, 0.691, 0.500, 0.309, 0.146, 0.038]],
+    [
+        {"warmup_steps": 2, "t_total": 10, "warmup_multiplier": 0.1},
+        [0.1, 0.55, 1.00, 0.962, 0.854, 0.691, 0.500, 0.309, 0.146, 0.038],
+    ],
+    [
+        {"warmup_steps": 2, "t_total": 10, "warmup_multiplier": 0.1, "end_lr": 0.309},
+        [0.1, 0.55, 1.00, 0.962, 0.854, 0.691, 0.500, 0.309, 0.309, 0.309],
+    ],
+]
+
+
+class TestLRSCHEDULER(unittest.TestCase):
+
+    @parameterized.expand(TEST_CASE_LRSCHEDULER)
+    def test_shape(self, input_param, expected_lr):
+        net = SchedulerTestNet()
+        optimizer = torch.optim.Adam(net.parameters(), lr=1.0)
+        scheduler = WarmupCosineSchedule(optimizer, **input_param)
+        self.assertEqual(len([scheduler.get_last_lr()[0]]), 1)
+        lrs_1 = []
+        for _ in range(input_param["t_total"]):
+            lrs_1.append(float(f"{scheduler.get_last_lr()[0]:.3f}"))
+            optimizer.step()
+            scheduler.step()
+        for a, b in zip(lrs_1, expected_lr):
+            self.assertEqual(a, b, msg=f"LR is wrong ! expected {b}, got {a}")
+
+    def test_error(self):
+        """Should fail because warmup_multiplier is outside 0..1"""
+        net = SchedulerTestNet()
+        optimizer = torch.optim.Adam(net.parameters(), lr=1.0)
+        with self.assertRaises(ValueError):
+            WarmupCosineSchedule(optimizer, warmup_steps=2, t_total=10, warmup_multiplier=-1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/optimizers/test_optim_novograd.py

@@ -0,0 +1,100 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import torch
+from parameterized import parameterized
+from torch.autograd import Variable
+
+from monai.optimizers import Novograd
+
+
+def build_test_cases(data):
+    [weight, bias, input] = data
+    weight = Variable(weight, requires_grad=True)
+    bias = Variable(bias, requires_grad=True)
+    input = Variable(input)
+
+    default_params = {"lr": 1e-3, "amsgrad": False, "grad_averaging": False, "weight_decay": 0}
+
+    test_case_same_param = [{"params": [weight, bias]}]
+    test_case_diff_param = [
+        {"params": [weight]},
+        {"params": [bias], "lr": 1e-2, "amsgrad": True, "grad_averaging": True, "weight_decay": 0.1},
+    ]
+
+    test_cases = [
+        [test_case_same_param, default_params, weight, bias, input],
+        [test_case_diff_param, default_params, weight, bias, input],
+    ]
+    return test_cases
+
+
+TEST_CASES_ALL = build_test_cases([torch.randn(10, 5), torch.randn(10), torch.randn(5)])  # normal parameters
+
+TEST_CASES_ALL += build_test_cases(  # non-contiguous parameters
+    [torch.randn(10, 5, 2)[..., 0], torch.randn(10, 2)[..., 0], torch.randn(5)]
+)
+
+if torch.cuda.is_available():
+    TEST_CASES_ALL += build_test_cases(  # gpu parameters
+        [torch.randn(10, 5).cuda(), torch.randn(10).cuda(), torch.randn(5).cuda()]
+    )
+if torch.cuda.device_count() > 1:
+    TEST_CASES_ALL += build_test_cases(  # multi-gpu parameters
+        [torch.randn(10, 5).cuda(0), torch.randn(10).cuda(1), torch.randn(5).cuda(0)]
+    )
+
+
+class TestNovograd(unittest.TestCase):
+    """
+    This class takes `Pytorch's test_optim function:
+    https://github.com/pytorch/pytorch/blob/v1.9.0/test/test_optim.py for reference.
+
+    """
+
+    @parameterized.expand(TEST_CASES_ALL)
+    def test_step(self, specify_param, default_param, weight, bias, input):
+        optimizer = Novograd(specify_param, **default_param)
+
+        def fn():
+            optimizer.zero_grad()
+            y = weight.mv(input)
+            if y.is_cuda and bias.is_cuda and y.get_device() != bias.get_device():
+                y = y.cuda(bias.get_device())
+            loss = (y + bias).pow(2).sum()
+            loss.backward()
+            return loss
+
+        initial_value = fn().item()
+        for _ in range(100):
+            optimizer.step(fn)
+        self.assertLess(fn().item(), initial_value)
+
+    def test_ill_arg(self):
+        param = {"params": [Variable(torch.randn(10), requires_grad=True)]}
+        with self.assertRaisesRegex(ValueError, "Invalid learning rate: -1"):
+            Novograd(param, lr=-1)
+        with self.assertRaisesRegex(ValueError, "Invalid epsilon value: -1"):
+            Novograd(param, eps=-1)
+        with self.assertRaisesRegex(ValueError, "Invalid beta parameter at index 0: 1.0"):
+            Novograd(param, betas=(1.0, 0.98))
+        with self.assertRaisesRegex(ValueError, "Invalid beta parameter at index 1: -1"):
+            Novograd(param, betas=(0.9, -1))
+        with self.assertRaisesRegex(ValueError, "Invalid weight_decay value: -1"):
+            Novograd(param, weight_decay=-1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/profile_subclass/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/profile_subclass/cprofile_profiling.py

@@ -0,0 +1,29 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Profiling MetaTensor
+"""
+
+from __future__ import annotations
+
+import cProfile
+
+import torch
+
+from monai.data.meta_tensor import MetaTensor
+
+if __name__ == "__main__":
+    n_chan = 3
+    for hwd in (10, 200):
+        shape = (n_chan, hwd, hwd, hwd)
+        a = MetaTensor(torch.rand(shape), meta={"affine": torch.eye(4) * 2, "fname": "something1"})
+        b = MetaTensor(torch.rand(shape), meta={"affine": torch.eye(4) * 3, "fname": "something2"})
+        cProfile.run("c = a + b", filename=f"out_{hwd}.prof")

tests/profile_subclass/min_classes.py

@@ -0,0 +1,30 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Minimal subclassing as baselines
+Adapted from https://github.com/pytorch/pytorch/tree/v1.11.0/benchmarks/overrides_benchmark
+"""
+
+from __future__ import annotations
+
+import torch
+
+__all__ = ["SubTensor", "SubWithTorchFunc"]
+
+
+class SubTensor(torch.Tensor):
+    pass
+
+
+class SubWithTorchFunc(torch.Tensor):
+
+    def __torch_function__(self, func, types, args=(), kwargs=None):
+        return super().__torch_function__(func, types, args, {} if kwargs is None else kwargs)

tests/profile_subclass/profiling.py

@@ -0,0 +1,73 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Comparing torch.Tensor, SubTensor, SubWithTorchFunc, MetaTensor
+Adapted from https://github.com/pytorch/pytorch/tree/v1.11.0/benchmarks/overrides_benchmark
+"""
+from __future__ import annotations
+
+import argparse
+
+import torch
+from min_classes import SubTensor, SubWithTorchFunc
+
+from monai.data import MetaTensor
+from monai.utils.profiling import PerfContext
+
+NUM_REPEATS = 1000
+NUM_REPEAT_OF_REPEATS = 1000
+
+
+def bench(t1, t2):
+    bench_times = []
+    for _ in range(NUM_REPEAT_OF_REPEATS):
+        with PerfContext() as pc:
+            for _ in range(NUM_REPEATS):
+                torch.add(t1, t2)
+        bench_times.append(pc.total_time)
+
+    bench_time_min = float(torch.min(torch.Tensor(bench_times))) / NUM_REPEATS
+    bench_time_avg = float(torch.sum(torch.Tensor(bench_times))) / (NUM_REPEATS * NUM_REPEAT_OF_REPEATS)
+    bench_time_med = float(torch.median(torch.Tensor(bench_times))) / NUM_REPEATS
+    bench_std = float(torch.std(torch.Tensor(bench_times))) / NUM_REPEATS
+    return bench_time_min, bench_time_avg, bench_time_med, bench_std
+
+
+def main():
+    global NUM_REPEATS
+    global NUM_REPEAT_OF_REPEATS
+
+    parser = argparse.ArgumentParser(description="Run the __torch_function__ benchmarks.")
+    parser.add_argument(
+        "--nreps", "-n", type=int, default=NUM_REPEATS, help="The number of repeats for one measurement."
+    )
+    parser.add_argument("--nrepreps", "-m", type=int, default=NUM_REPEAT_OF_REPEATS, help="The number of measurements.")
+    args = parser.parse_args()
+
+    NUM_REPEATS = args.nreps
+    NUM_REPEAT_OF_REPEATS = args.nrepreps
+
+    types = torch.Tensor, SubTensor, SubWithTorchFunc, MetaTensor
+
+    for t in types:
+        tensor_1 = t(1)
+        tensor_2 = t(2)
+
+        b_min, b_avg, b_med, b_std = bench(tensor_1, tensor_2)
+        print(
+            "Type {} time (microseconds):  min: {}, avg: {}, median: {}, and std {}.".format(
+                t.__name__, (10**6 * b_min), (10**6) * b_avg, (10**6) * b_med, (10**6) * b_std
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

tests/profile_subclass/pyspy_profiling.py

@@ -0,0 +1,41 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+To be used with py-spy, comparing torch.Tensor, SubTensor, SubWithTorchFunc, MetaTensor
+Adapted from https://github.com/pytorch/pytorch/tree/v1.11.0/benchmarks/overrides_benchmark
+"""
+from __future__ import annotations
+
+import argparse
+
+import torch
+from min_classes import SubTensor, SubWithTorchFunc  # noqa: F401
+
+from monai.data import MetaTensor  # noqa: F401
+
+Tensor = torch.Tensor
+
+NUM_REPEATS = 1000000
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the torch.add for a given class a given number of times.")
+    parser.add_argument("tensor_class", metavar="TensorClass", type=str, help="The class to benchmark.")
+    parser.add_argument("--nreps", "-n", type=int, default=NUM_REPEATS, help="The number of repeats.")
+    args = parser.parse_args()
+
+    TensorClass = globals()[args.tensor_class]
+    NUM_REPEATS = args.nreps
+
+    t1 = TensorClass(1)
+    t2 = TensorClass(2)
+
+    for _ in range(NUM_REPEATS):
+        torch.add(t1, t2)

tests/transforms/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/transforms/compose/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.