monai-weekly 1.5.dev2505__py3-none-any.whl → 1.5.dev2507__py3-none-any.whl

tests/integration/test_integration_autorunner.py

@@ -0,0 +1,201 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+
+import nibabel as nib
+import numpy as np
+import torch
+
+from monai.apps.auto3dseg import AutoRunner
+from monai.bundle.config_parser import ConfigParser
+from monai.data import create_test_image_3d
+from monai.utils import optional_import
+from tests.test_utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
+
+_, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
+_, has_nni = optional_import("nni")
+
+num_images_perfold = max(torch.cuda.device_count(), 4)
+num_images_per_batch = 2
+
+sim_datalist: dict[str, list[dict]] = {
+    "testing": [{"image": f"ts_image__{idx:03d}.nii.gz"} for idx in range(num_images_perfold)],
+    "training": [
+        {
+            "fold": f,
+            "image": f"tr_image_{(f * num_images_perfold + idx):03d}.nii.gz",
+            "label": f"tr_label_{(f * num_images_perfold + idx):03d}.nii.gz",
+        }
+        for f in range(num_images_per_batch + 1)
+        for idx in range(num_images_perfold)
+    ],
+}
+
+train_param = (
+    {
+        "num_images_per_batch": num_images_per_batch,
+        "num_epochs": 2,
+        "num_epochs_per_validation": 1,
+        "num_warmup_epochs": 1,
+        "use_pretrain": False,
+        "pretrained_path": "",
+        "num_steps_per_image": 1,
+    }
+    if torch.cuda.is_available()
+    else {}
+)
+
+pred_param = {"files_slices": slice(0, 1), "mode": "mean", "sigmoid": True}
+
+
+@skip_if_quick
+@SkipIfBeforePyTorchVersion((1, 11, 1))  # for mem_get_info
+@unittest.skipIf(not has_tb, "no tensorboard summary writer")
+class TestAutoRunner(unittest.TestCase):
+    def setUp(self) -> None:
+        self.test_dir = tempfile.TemporaryDirectory()
+        test_path = self.test_dir.name
+
+        sim_dataroot = os.path.join(test_path, "dataroot")
+        if not os.path.isdir(sim_dataroot):
+            os.makedirs(sim_dataroot)
+
+        # Generate a fake dataset
+        for d in sim_datalist["testing"] + sim_datalist["training"]:
+            im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1)
+            nib_image = nib.Nifti1Image(im, affine=np.eye(4))
+            image_fpath = os.path.join(sim_dataroot, d["image"])
+            nib.save(nib_image, image_fpath)
+
+            if "label" in d:
+                nib_image = nib.Nifti1Image(seg, affine=np.eye(4))
+                label_fpath = os.path.join(sim_dataroot, d["label"])
+                nib.save(nib_image, label_fpath)
+
+        sim_json_datalist = os.path.join(sim_dataroot, "sim_input.json")
+        ConfigParser.export_config_file(sim_datalist, sim_json_datalist)
+
+        data_src_cfg = os.path.join(test_path, "data_src_cfg.yaml")
+        data_src = {
+            "name": "sim_data",
+            "task": "segmentation",
+            "modality": "MRI",
+            "datalist": sim_json_datalist,
+            "dataroot": sim_dataroot,
+            "multigpu": False,
+            "class_names": ["label_class"],
+        }
+
+        ConfigParser.export_config_file(data_src, data_src_cfg)
+        self.data_src_cfg = data_src_cfg
+        self.test_path = test_path
+
+    @skip_if_no_cuda
+    def test_autorunner(self) -> None:
+        work_dir = os.path.join(self.test_path, "work_dir")
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            templates_path_or_url=get_testing_algo_template_path(),
+            allow_skip=False,
+        )
+        runner.set_training_params(train_param)  # 2 epochs
+        runner.set_num_fold(1)
+        with skip_if_downloading_fails():
+            runner.run()
+
+    @skip_if_no_cuda
+    def test_autorunner_ensemble(self) -> None:
+        work_dir = os.path.join(self.test_path, "work_dir")
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            templates_path_or_url=get_testing_algo_template_path(),
+            allow_skip=False,
+        )
+        runner.set_training_params(train_param)  # 2 epochs
+        runner.set_ensemble_method("AlgoEnsembleBestByFold")
+        runner.set_num_fold(1)
+        with skip_if_downloading_fails():
+            runner.run()
+
+    @skip_if_no_cuda
+    def test_autorunner_gpu_customization(self) -> None:
+        work_dir = os.path.join(self.test_path, "work_dir")
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            templates_path_or_url=get_testing_algo_template_path(),
+            allow_skip=False,
+        )
+        gpu_customization_specs = {
+            "universal": {"num_trials": 1, "range_num_images_per_batch": [1, 2], "range_num_sw_batch_size": [1, 2]}
+        }
+        runner.set_gpu_customization(gpu_customization=True, gpu_customization_specs=gpu_customization_specs)
+        runner.set_training_params(train_param)  # 2 epochs
+        runner.set_num_fold(1)
+        with skip_if_downloading_fails():
+            runner.run()
+
+    @skip_if_no_cuda
+    @unittest.skipIf(not has_nni, "nni required")
+    def test_autorunner_hpo(self) -> None:
+        work_dir = os.path.join(self.test_path, "work_dir")
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            hpo=True,
+            ensemble=False,
+            templates_path_or_url=get_testing_algo_template_path(),
+            allow_skip=False,
+        )
+        hpo_param = {
+            "num_epochs_per_validation": train_param["num_epochs_per_validation"],
+            "num_images_per_batch": train_param["num_images_per_batch"],
+            "num_epochs": train_param["num_epochs"],
+            "num_warmup_epochs": train_param["num_warmup_epochs"],
+            "use_pretrain": train_param["use_pretrain"],
+            "pretrained_path": train_param["pretrained_path"],
+            # below are to shorten the time for dints
+            "training#num_epochs_per_validation": train_param["num_epochs_per_validation"],
+            "training#num_images_per_batch": train_param["num_images_per_batch"],
+            "training#num_epochs": train_param["num_epochs"],
+            "training#num_warmup_epochs": train_param["num_warmup_epochs"],
+            "searching#num_epochs_per_validation": train_param["num_epochs_per_validation"],
+            "searching#num_images_per_batch": train_param["num_images_per_batch"],
+            "searching#num_epochs": train_param["num_epochs"],
+            "searching#num_warmup_epochs": train_param["num_warmup_epochs"],
+            "nni_dry_run": True,
+        }
+        search_space = {"learning_rate": {"_type": "choice", "_value": [0.0001, 0.001, 0.01, 0.1]}}
+        runner.set_num_fold(1)
+        runner.set_nni_search_space(search_space)
+        runner.set_hpo_params(params=hpo_param)
+        with skip_if_downloading_fails():
+            runner.run()
+
+    def tearDown(self) -> None:
+        self.test_dir.cleanup()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/integration/test_integration_bundle_run.py

@@ -0,0 +1,240 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from glob import glob
+from pathlib import Path
+
+import nibabel as nib
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.bundle import ConfigParser
+from monai.bundle.utils import DEFAULT_HANDLERS_ID
+from monai.transforms import LoadImage
+from monai.utils import path_to_uri
+from tests.test_utils import command_line_tests
+
+TESTS_PATH = Path(__file__).parents[1]
+TEST_CASE_1 = [os.path.join(TESTS_PATH, "testing_data", "inference.json"), (128, 128, 128)]
+
+TEST_CASE_2 = [os.path.join(TESTS_PATH, "testing_data", "inference.yaml"), (128, 128, 128)]
+
+
+class _Runnable42:
+    def __init__(self, val=1):
+        self.val = val
+
+    def run(self):
+        assert self.val == 42  # defined in `TestBundleRun.test_tiny``
+        return self.val
+
+
+class _Runnable43:
+    def __init__(self, func):
+        self.func = func
+
+    def run(self):
+        self.func()
+
+
+class TestBundleRun(unittest.TestCase):
+    def setUp(self):
+        self.data_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.data_dir)
+
+    def test_tiny(self):
+        config_file = os.path.join(self.data_dir, "tiny_config.json")
+        meta_file = os.path.join(self.data_dir, "tiny_meta.json")
+        with open(config_file, "w") as f:
+            json.dump(
+                {
+                    "trainer": {"_target_": "tests.integration.test_integration_bundle_run._Runnable42", "val": 42},
+                    # keep this test case to cover the "run_id" arg
+                    "training": "$@trainer.run()",
+                },
+                f,
+            )
+        with open(meta_file, "w") as f:
+            json.dump(
+                {"version": "0.1.0", "monai_version": "1.1.0", "pytorch_version": "1.13.1", "numpy_version": "1.22.2"},
+                f,
+            )
+        cmd = ["coverage", "run", "-m", "monai.bundle"]
+        # test both CLI entry "run" and "run_workflow"
+        command_line_tests(cmd + ["run", "training", "--config_file", config_file, "--meta_file", meta_file])
+        command_line_tests(
+            cmd + ["run_workflow", "--run_id", "training", "--config_file", config_file, "--meta_file", meta_file]
+        )
+        with self.assertRaises(RuntimeError):
+            # test wrong run_id="run"
+            command_line_tests(cmd + ["run", "run", "--config_file", config_file])
+        # test missing meta file
+        self.assertIn("ERROR", command_line_tests(cmd + ["run", "training", "--config_file", config_file]))
+
+    def test_scripts_fold(self):
+        # test scripts directory has been added to Python search directories automatically
+        config_file = os.path.join(self.data_dir, "tiny_config.json")
+        meta_file = os.path.join(self.data_dir, "tiny_meta.json")
+        scripts_dir = os.path.join(self.data_dir, "scripts")
+        script_file = os.path.join(scripts_dir, "test_scripts_fold.py")
+        init_file = os.path.join(scripts_dir, "__init__.py")
+
+        with open(config_file, "w") as f:
+            json.dump(
+                {
+                    "imports": ["$import scripts"],
+                    "trainer": {
+                        "_target_": "tests.integration.test_integration_bundle_run._Runnable43",
+                        "func": "$scripts.tiny_test",
+                    },
+                    # keep this test case to cover the "run_id" arg
+                    "training": "$@trainer.run()",
+                },
+                f,
+            )
+        with open(meta_file, "w") as f:
+            json.dump(
+                {"version": "0.1.0", "monai_version": "1.1.0", "pytorch_version": "1.13.1", "numpy_version": "1.22.2"},
+                f,
+            )
+
+        os.mkdir(scripts_dir)
+        script_file_lines = ["def tiny_test():\n", "    print('successfully added scripts fold!') \n"]
+        init_file_line = "from .test_scripts_fold import tiny_test\n"
+        with open(script_file, "w") as f:
+            f.writelines(script_file_lines)
+            f.close()
+        with open(init_file, "w") as f:
+            f.write(init_file_line)
+            f.close()
+
+        cmd = ["coverage", "run", "-m", "monai.bundle"]
+        # test both CLI entry "run" and "run_workflow"
+        expected_condition = "successfully added scripts fold!"
+        command_run = cmd + ["run", "training", "--config_file", config_file, "--meta_file", meta_file]
+        completed_process = subprocess.run(command_run, check=True, capture_output=True, text=True)
+        output = repr(completed_process.stdout).replace("\\n", "\n").replace("\\t", "\t")  # Get the captured output
+
+        self.assertIn(expected_condition, output)
+        command_run_workflow = cmd + [
+            "run_workflow",
+            "--run_id",
+            "training",
+            "--config_file",
+            config_file,
+            "--meta_file",
+            meta_file,
+        ]
+        completed_process = subprocess.run(command_run_workflow, check=True, capture_output=True, text=True)
+        output = repr(completed_process.stdout).replace("\\n", "\n").replace("\\t", "\t")  # Get the captured output
+        self.assertIn(expected_condition, output)
+
+        # test missing meta file
+        self.assertIn("ERROR", command_line_tests(cmd + ["run", "training", "--config_file", config_file]))
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_shape(self, config_file, expected_shape):
+        test_image = np.random.rand(*expected_shape)
+        tempdir = self.data_dir
+        filename = os.path.join(tempdir, "image.nii")
+        nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename)
+
+        # generate default args in a JSON file
+        logging_conf = os.path.join(TESTS_PATH, "testing_data", "logging.conf")
+        def_args = {"config_file": "will be replaced by `config_file` arg", "logging_file": logging_conf}
+        def_args_file = os.path.join(tempdir, "def_args.json")
+        ConfigParser.export_config_file(config=def_args, filepath=def_args_file)
+
+        meta = {"datalist": [{"image": filename}], "window": (96, 96, 96)}
+        # test YAML file
+        meta_file = os.path.join(tempdir, "meta.yaml")
+        ConfigParser.export_config_file(config=meta, filepath=meta_file, fmt="yaml")
+
+        # test MLFlow settings
+        settings = {
+            "handlers_id": DEFAULT_HANDLERS_ID,
+            "configs": {
+                "no_epoch": True,  # test override config in the settings file
+                "evaluator": {
+                    "_target_": "MLFlowHandler",
+                    "tracking_uri": "$monai.utils.path_to_uri(@output_dir) + '/mlflow_override1'",
+                    "iteration_log": "@no_epoch",
+                },
+            },
+        }
+        settings_file = os.path.join(tempdir, "mlflow.json")
+        ConfigParser.export_config_file(config=settings, filepath=settings_file, fmt="json")
+
+        # test override with file, up case postfix
+        overridefile1 = os.path.join(tempdir, "override1.JSON")
+        with open(overridefile1, "w") as f:
+            # test override with part of the overriding file
+            json.dump({"move_net": "$@network_def.to(@device)"}, f)
+        os.makedirs(os.path.join(tempdir, "jsons"), exist_ok=True)
+        overridefile2 = os.path.join(tempdir, "jsons/override2.JSON")
+        with open(overridefile2, "w") as f:
+            # test override with the whole overriding file
+            json.dump("Dataset", f)
+
+        if sys.platform == "win32":
+            override = "--network $@network_def.to(@device) --dataset#_target_ Dataset"
+        else:
+            override = f"--network %{overridefile1}#move_net --dataset#_target_ %{overridefile2}"
+        device = "$torch.device('cuda:0')" if torch.cuda.is_available() else "$torch.device('cpu')"
+        # test with `monai.bundle` as CLI entry directly
+        cmd = "-m monai.bundle run --postprocessing#transforms#2#output_postfix seg"
+        cmd += f" {override} --no_epoch False --output_dir {tempdir} --device {device}"
+        la = ["coverage", "run"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file]
+        test_env = os.environ.copy()
+        print(f"CUDA_VISIBLE_DEVICES in {__file__}", test_env.get("CUDA_VISIBLE_DEVICES"))
+        command_line_tests(la + ["--args_file", def_args_file] + ["--tracking", settings_file])
+        loader = LoadImage(image_only=True)
+        self.assertTupleEqual(loader(os.path.join(tempdir, "image", "image_seg.nii.gz")).shape, expected_shape)
+        self.assertTrue(os.path.exists(f"{tempdir}/mlflow_override1"))
+
+        tracking_uri = path_to_uri(tempdir) + "/mlflow_override2"  # test override experiment management configs
+        # here test the script with `google fire` tool as CLI
+        cmd = "-m fire monai.bundle.scripts run --tracking mlflow --evaluator#amp False"
+        cmd += f" --tracking_uri {tracking_uri} {override} --output_dir {tempdir} --device {device}"
+        la = ["coverage", "run"] + cmd.split(" ") + ["--meta_file", meta_file] + ["--config_file", config_file]
+        command_line_tests(la)
+        self.assertTupleEqual(loader(os.path.join(tempdir, "image", "image_trans.nii.gz")).shape, expected_shape)
+        self.assertTrue(os.path.exists(f"{tempdir}/mlflow_override2"))
+        # test the saved execution configs
+        self.assertTrue(len(glob(f"{tempdir}/config_*.json")), 2)
+
+    def test_customized_workflow(self):
+        expected_shape = (64, 64, 64)
+        test_image = np.random.rand(*expected_shape)
+        filename = os.path.join(self.data_dir, "image.nii")
+        nib.save(nib.Nifti1Image(test_image, np.eye(4)), filename)
+
+        cmd = "-m fire monai.bundle.scripts run_workflow --workflow_name tests.nonconfig_workflow.NonConfigWorkflow"
+        cmd += f" --filename {filename} --output_dir {self.data_dir}"
+        command_line_tests(["coverage", "run"] + cmd.split(" "))
+        loader = LoadImage(image_only=True)
+        self.assertTupleEqual(loader(os.path.join(self.data_dir, "image", "image_seg.nii.gz")).shape, expected_shape)
+
+
+if __name__ == "__main__":
+    unittest.main()