monai-weekly 1.5.dev2506__py3-none-any.whl → 1.5.dev2508__py3-none-any.whl

tests/integration/test_integration_classification_2d.py

@@ -0,0 +1,282 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import unittest
+import warnings
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader
+
+import monai
+from monai.apps import download_and_extract
+from monai.data import decollate_batch
+from monai.metrics import ROCAUCMetric
+from monai.networks import eval_mode
+from monai.networks.nets import DenseNet121
+from monai.transforms import (
+    Activations,
+    AsDiscrete,
+    Compose,
+    EnsureChannelFirst,
+    LoadImage,
+    RandFlip,
+    RandRotate,
+    RandZoom,
+    ScaleIntensity,
+    Transpose,
+)
+from monai.utils import set_determinism
+from tests.test_utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick, testing_data_config
+from tests.testing_data.integration_answers import test_integration_value
+
+TASK = "integration_classification_2d"
+
+
+class MedNISTDataset(torch.utils.data.Dataset):
+    def __init__(self, image_files, labels, transforms):
+        self.image_files = image_files
+        self.labels = labels
+        self.transforms = transforms
+
+    def __len__(self):
+        return len(self.image_files)
+
+    def __getitem__(self, index):
+        return self.transforms(self.image_files[index]), self.labels[index]
+
+
+def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0", num_workers=10):
+    monai.config.print_config()
+    # define transforms for image and classification
+    train_transforms = Compose(
+        [
+            LoadImage(image_only=True, simple_keys=True),
+            EnsureChannelFirst(channel_dim="no_channel"),
+            Transpose(indices=[0, 2, 1]),
+            ScaleIntensity(),
+            RandRotate(range_x=np.pi / 12, prob=0.5, keep_size=True, dtype=np.float64),
+            RandFlip(spatial_axis=0, prob=0.5),
+            RandZoom(min_zoom=0.9, max_zoom=1.1, prob=0.5),
+        ]
+    )
+    train_transforms.set_random_state(1234)
+    val_transforms = Compose(
+        [
+            LoadImage(image_only=True, simple_keys=True),
+            EnsureChannelFirst(channel_dim="no_channel"),
+            Transpose(indices=[0, 2, 1]),
+            ScaleIntensity(),
+        ]
+    )
+    y_pred_trans = Compose([Activations(softmax=True)])
+    y_trans = AsDiscrete(to_onehot=len(np.unique(train_y)))
+    auc_metric = ROCAUCMetric()
+
+    # create train, val data loaders
+    train_ds = MedNISTDataset(train_x, train_y, train_transforms)
+    train_loader = DataLoader(train_ds, batch_size=300, shuffle=True, num_workers=num_workers)
+
+    val_ds = MedNISTDataset(val_x, val_y, val_transforms)
+    val_loader = DataLoader(val_ds, batch_size=300, num_workers=num_workers)
+
+    model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(train_y))).to(device)
+    loss_function = torch.nn.CrossEntropyLoss()
+    optimizer = torch.optim.Adam(model.parameters(), 1e-5)
+    epoch_num = 1
+    val_interval = 1
+
+    # start training validation
+    best_metric = -1
+    best_metric_epoch = -1
+    epoch_loss_values = []
+    metric_values = []
+    model_filename = os.path.join(root_dir, "best_metric_model.pth")
+    for epoch in range(epoch_num):
+        print("-" * 10)
+        print(f"Epoch {epoch + 1}/{epoch_num}")
+        model.train()
+        epoch_loss = 0
+        step = 0
+        for batch_data in train_loader:
+            step += 1
+            inputs, labels = batch_data[0].to(device), batch_data[1].to(device)
+            optimizer.zero_grad()
+            outputs = model(inputs)
+            loss = loss_function(outputs, labels)
+            loss.backward()
+            optimizer.step()
+            epoch_loss += loss.item()
+        epoch_loss /= step
+        epoch_loss_values.append(epoch_loss)
+        print(f"epoch {epoch + 1} average loss:{epoch_loss:0.4f}")
+
+        if (epoch + 1) % val_interval == 0:
+            with eval_mode(model):
+                y_pred = torch.tensor([], dtype=torch.float32, device=device)
+                y = torch.tensor([], dtype=torch.long, device=device)
+                for val_data in val_loader:
+                    val_images, val_labels = (val_data[0].to(device), val_data[1].to(device))
+                    y_pred = torch.cat([y_pred, model(val_images)], dim=0)
+                    y = torch.cat([y, val_labels], dim=0)
+
+                # compute accuracy
+                acc_value = torch.eq(y_pred.argmax(dim=1), y)
+                acc_metric = acc_value.sum().item() / len(acc_value)
+                # decollate prediction and label and execute post processing
+                y_pred = [y_pred_trans(i) for i in decollate_batch(y_pred)]
+                y = [y_trans(i) for i in decollate_batch(y, detach=False)]
+                # compute AUC
+                auc_metric(y_pred, y)
+                auc_value = auc_metric.aggregate()
+                auc_metric.reset()
+                metric_values.append(auc_value)
+                if auc_value > best_metric:
+                    best_metric = auc_value
+                    best_metric_epoch = epoch + 1
+                    torch.save(model.state_dict(), model_filename)
+                    print("saved new best metric model")
+                print(
+                    f"current epoch {epoch + 1} current AUC: {auc_value:0.4f} "
+                    f"current accuracy: {acc_metric:0.4f} best AUC: {best_metric:0.4f} at epoch {best_metric_epoch}"
+                )
+    print(f"train completed, best_metric: {best_metric:0.4f}  at epoch: {best_metric_epoch}")
+    return epoch_loss_values, best_metric, best_metric_epoch
+
+
+def run_inference_test(root_dir, test_x, test_y, device="cuda:0", num_workers=10):
+    # define transforms for image and classification
+    val_transforms = Compose(
+        [LoadImage(image_only=True), EnsureChannelFirst(channel_dim="no_channel"), ScaleIntensity()]
+    )
+    val_ds = MedNISTDataset(test_x, test_y, val_transforms)
+    val_loader = DataLoader(val_ds, batch_size=300, num_workers=num_workers)
+
+    model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(test_y))).to(device)
+
+    model_filename = os.path.join(root_dir, "best_metric_model.pth")
+    model.load_state_dict(torch.load(model_filename))
+    y_true = []
+    y_pred = []
+    with eval_mode(model):
+        for test_data in val_loader:
+            test_images, test_labels = test_data[0].to(device), test_data[1].to(device)
+            pred = model(test_images).argmax(dim=1)
+            for i in range(len(pred)):
+                y_true.append(test_labels[i].item())
+                y_pred.append(pred[i].item())
+    tps = [np.sum((np.asarray(y_true) == idx) & (np.asarray(y_pred) == idx)) for idx in np.unique(test_y)]
+    return tps
+
+
+@skip_if_quick
+class IntegrationClassification2D(DistTestCase):
+    def setUp(self):
+        set_determinism(seed=0)
+        self.data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../testing_data")
+        data_dir = os.path.join(self.data_dir, "MedNIST")
+        dataset_file = os.path.join(self.data_dir, "MedNIST.tar.gz")
+
+        if not os.path.exists(data_dir):
+            with skip_if_downloading_fails():
+                data_spec = testing_data_config("images", "mednist")
+                download_and_extract(
+                    data_spec["url"],
+                    dataset_file,
+                    self.data_dir,
+                    hash_val=data_spec["hash_val"],
+                    hash_type=data_spec["hash_type"],
+                )
+
+        assert os.path.exists(data_dir)
+
+        class_names = sorted(x for x in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, x)))
+        image_files = [
+            [os.path.join(data_dir, class_name, x) for x in sorted(os.listdir(os.path.join(data_dir, class_name)))]
+            for class_name in class_names
+        ]
+        image_file_list, image_classes = [], []
+        for i, _ in enumerate(class_names):
+            image_file_list.extend(image_files[i])
+            image_classes.extend([i] * len(image_files[i]))
+
+        # split train, val, test
+        valid_frac, test_frac = 0.1, 0.1
+        self.train_x, self.train_y = [], []
+        self.val_x, self.val_y = [], []
+        self.test_x, self.test_y = [], []
+        for i in range(len(image_classes)):
+            rann = np.random.random()
+            if rann < valid_frac:
+                self.val_x.append(image_file_list[i])
+                self.val_y.append(image_classes[i])
+            elif rann < test_frac + valid_frac:
+                self.test_x.append(image_file_list[i])
+                self.test_y.append(image_classes[i])
+            else:
+                self.train_x.append(image_file_list[i])
+                self.train_y.append(image_classes[i])
+
+        self.device = "cuda:0" if torch.cuda.is_available() else "cpu:0"
+
+    def tearDown(self):
+        set_determinism(seed=None)
+        try:
+            os.remove(os.path.join(self.data_dir, "best_metric_model.pth"))
+        except FileNotFoundError:
+            warnings.warn("not found best_metric_model.pth, training skipped?")
+            pass
+
+    def train_and_infer(self, idx=0):
+        results = []
+        if not os.path.exists(os.path.join(self.data_dir, "MedNIST")):
+            # skip test if no MedNIST dataset
+            return results
+
+        set_determinism(seed=0)
+        losses, best_metric, best_metric_epoch = run_training_test(
+            self.data_dir, self.train_x, self.train_y, self.val_x, self.val_y, device=self.device
+        )
+        infer_metric = run_inference_test(self.data_dir, self.test_x, self.test_y, device=self.device)
+
+        print(f"integration_classification_2d {losses}")
+        print("best metric", best_metric)
+        print("infer metric", infer_metric)
+        # check training properties
+        self.assertTrue(test_integration_value(TASK, key="losses", data=losses, rtol=1e-2))
+        self.assertTrue(test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-4))
+        np.testing.assert_allclose(best_metric_epoch, 4)
+        model_file = os.path.join(self.data_dir, "best_metric_model.pth")
+        self.assertTrue(os.path.exists(model_file))
+        # check inference properties
+        self.assertTrue(test_integration_value(TASK, key="infer_prop", data=np.asarray(infer_metric), rtol=1))
+        results.extend(losses)
+        results.append(best_metric)
+        results.extend(infer_metric)
+        return results
+
+    def test_training(self):
+        repeated = []
+        for i in range(1):
+            results = self.train_and_infer(i)
+            repeated.append(results)
+        np.testing.assert_allclose(repeated[0], repeated[1])
+
+    @TimedCall(seconds=2000, skip_timing=not torch.cuda.is_available(), force_quit=False, daemon=False)
+    def test_timing(self):
+        self.train_and_infer()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/integration/test_integration_determinism.py

@@ -0,0 +1,95 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, Dataset
+
+from monai.data import create_test_image_2d
+from monai.losses import DiceLoss
+from monai.networks.nets import UNet
+from monai.transforms import Compose, EnsureChannelFirst, RandRotate90, RandSpatialCrop, ScaleIntensity
+from monai.utils import set_determinism
+from tests.test_utils import DistTestCase, TimedCall
+
+
+def run_test(batch_size=64, train_steps=200, device="cuda:0"):
+    class _TestBatch(Dataset):
+        def __init__(self, transforms):
+            self.transforms = transforms
+
+        def __getitem__(self, _unused_id):
+            im, seg = create_test_image_2d(128, 128, noise_max=1, num_objs=4, num_seg_classes=1)
+            seed = np.random.randint(2147483647)
+            self.transforms.set_random_state(seed=seed)
+            im = self.transforms(im)
+            self.transforms.set_random_state(seed=seed)
+            seg = self.transforms(seg)
+            return im, seg
+
+        def __len__(self):
+            return train_steps
+
+    net = UNet(
+        spatial_dims=2, in_channels=1, out_channels=1, channels=(4, 8, 16, 32), strides=(2, 2, 2), num_res_units=2
+    ).to(device)
+
+    loss = DiceLoss(sigmoid=True)
+    opt = torch.optim.Adam(net.parameters(), 1e-2)
+    train_transforms = Compose(
+        [
+            EnsureChannelFirst(channel_dim="no_channel"),
+            ScaleIntensity(),
+            RandSpatialCrop((96, 96), random_size=False),
+            RandRotate90(),
+        ]
+    )
+
+    src = DataLoader(_TestBatch(train_transforms), batch_size=batch_size, shuffle=True)
+
+    net.train()
+    epoch_loss = 0
+    step = 0
+    for img, seg in src:
+        step += 1
+        opt.zero_grad()
+        output = net(img.to(device))
+        step_loss = loss(output, seg.to(device))
+        step_loss.backward()
+        opt.step()
+        epoch_loss += step_loss.item()
+    epoch_loss /= step
+
+    return epoch_loss, step
+
+
+class TestDeterminism(DistTestCase):
+    def setUp(self):
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu:0")
+
+    def tearDown(self):
+        set_determinism(seed=None)
+
+    @TimedCall(seconds=150, skip_timing=not torch.cuda.is_available())
+    def test_training(self):
+        set_determinism(seed=0)
+        loss, step = run_test(device=self.device)
+        print(f"Deterministic loss {loss} at training step {step}")
+        np.testing.assert_allclose(step, 4)
+        np.testing.assert_allclose(loss, 0.536134, rtol=1e-4)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/integration/test_integration_fast_train.py

@@ -0,0 +1,231 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import math
+import os
+import shutil
+import tempfile
+import time
+import unittest
+from glob import glob
+
+import nibabel as nib
+import numpy as np
+import torch
+
+import monai
+from monai.data import CacheDataset, ThreadDataLoader, create_test_image_3d, decollate_batch
+from monai.inferers import sliding_window_inference
+from monai.losses import DiceCELoss
+from monai.metrics import DiceMetric
+from monai.networks.layers import Norm
+from monai.networks.nets import UNet
+from monai.optimizers import Novograd
+from monai.transforms import (
+    AsDiscrete,
+    Compose,
+    CropForegroundd,
+    EnsureChannelFirstd,
+    FgBgToIndicesd,
+    LoadImaged,
+    RandAffined,
+    RandAxisFlipd,
+    RandCropByPosNegLabeld,
+    RandFlipd,
+    RandGaussianNoised,
+    RandRotate90d,
+    RandRotated,
+    RandStdShiftIntensityd,
+    RandZoomd,
+    ScaleIntensityd,
+    Spacingd,
+    ToDeviced,
+)
+from monai.utils import set_determinism
+from tests.test_utils import DistTestCase, TimedCall, skip_if_no_cuda, skip_if_quick
+
+
+@skip_if_no_cuda
+@skip_if_quick
+class IntegrationFastTrain(DistTestCase):
+    def setUp(self):
+        set_determinism(seed=0)
+        monai.config.print_config()
+
+        self.data_dir = tempfile.mkdtemp()
+        for i in range(41):
+            im, seg = create_test_image_3d(128, 128, 128, num_seg_classes=1, channel_dim=-1)
+            n = nib.Nifti1Image(im, np.eye(4))
+            nib.save(n, os.path.join(self.data_dir, f"img{i:d}.nii.gz"))
+            n = nib.Nifti1Image(seg, np.eye(4))
+            nib.save(n, os.path.join(self.data_dir, f"seg{i:d}.nii.gz"))
+
+    def tearDown(self):
+        set_determinism(seed=None)
+        shutil.rmtree(self.data_dir)
+
+    # test the fast training speed is as expected
+    @TimedCall(seconds=100, daemon=False, force_quit=False)
+    def test_train_timing(self):
+        images = sorted(glob(os.path.join(self.data_dir, "img*.nii.gz")))
+        segs = sorted(glob(os.path.join(self.data_dir, "seg*.nii.gz")))
+        train_files = [{"image": img, "label": seg} for img, seg in zip(images[:32], segs[:32])]
+        val_files = [{"image": img, "label": seg} for img, seg in zip(images[-9:], segs[-9:])]
+
+        device = torch.device("cuda:0")
+        # define transforms for train and validation
+        train_transforms = Compose(
+            [
+                LoadImaged(keys=["image", "label"]),
+                EnsureChannelFirstd(keys=["image", "label"]),
+                Spacingd(keys=["image", "label"], pixdim=(1.0, 1.0, 1.0), mode=("bilinear", "nearest")),
+                ScaleIntensityd(keys="image"),
+                CropForegroundd(keys=["image", "label"], source_key="image"),
+                # pre-compute foreground and background indexes
+                # and cache them to accelerate training
+                FgBgToIndicesd(keys="label", fg_postfix="_fg", bg_postfix="_bg"),
+                # move the data to GPU and cache to avoid CPU -> GPU sync in every epoch
+                ToDeviced(keys=["image", "label"], device=device),
+                # randomly crop out patch samples from big
+                # image based on pos / neg ratio
+                # the image centers of negative samples
+                # must be in valid image area
+                RandCropByPosNegLabeld(
+                    keys=["image", "label"],
+                    label_key="label",
+                    spatial_size=(64, 64, 64),
+                    pos=1,
+                    neg=1,
+                    num_samples=4,
+                    fg_indices_key="label_fg",
+                    bg_indices_key="label_bg",
+                ),
+                RandFlipd(keys=["image", "label"], prob=0.5, spatial_axis=[1, 2]),
+                RandAxisFlipd(keys=["image", "label"], prob=0.5),
+                RandRotate90d(keys=["image", "label"], prob=0.5, spatial_axes=(1, 2)),
+                RandZoomd(keys=["image", "label"], prob=0.5, min_zoom=0.8, max_zoom=1.2, keep_size=True),
+                RandRotated(
+                    keys=["image", "label"],
+                    prob=0.5,
+                    range_x=np.pi / 4,
+                    mode=("bilinear", "nearest"),
+                    align_corners=True,
+                    dtype=np.float64,
+                ),
+                RandAffined(keys=["image", "label"], prob=0.5, rotate_range=np.pi / 2, mode=("bilinear", "nearest")),
+                RandGaussianNoised(keys="image", prob=0.5),
+                RandStdShiftIntensityd(keys="image", prob=0.5, factors=0.05, nonzero=True),
+            ]
+        )
+
+        val_transforms = Compose(
+            [
+                LoadImaged(keys=["image", "label"]),
+                EnsureChannelFirstd(keys=["image", "label"]),
+                Spacingd(keys=["image", "label"], pixdim=(1.0, 1.0, 1.0), mode=("bilinear", "nearest")),
+                ScaleIntensityd(keys="image"),
+                CropForegroundd(keys=["image", "label"], source_key="image"),
+                # move the data to GPU and cache to avoid CPU -> GPU sync in every epoch
+                ToDeviced(keys=["image", "label"], device=device),
+            ]
+        )
+
+        max_epochs = 5
+        learning_rate = 2e-4
+        val_interval = 1  # do validation for every epoch
+
+        # set CacheDataset, ThreadDataLoader and DiceCE loss for MONAI fast training
+        train_ds = CacheDataset(data=train_files, transform=train_transforms, cache_rate=1.0, num_workers=8)
+        val_ds = CacheDataset(data=val_files, transform=val_transforms, cache_rate=1.0, runtime_cache=True)
+        # disable multi-workers because `ThreadDataLoader` works with multi-threads
+        train_loader = ThreadDataLoader(train_ds, num_workers=0, batch_size=4, shuffle=True)
+        val_loader = ThreadDataLoader(val_ds, num_workers=0, batch_size=1)
+
+        loss_function = DiceCELoss(to_onehot_y=True, softmax=True, squared_pred=True, batch=True)
+        model = UNet(
+            spatial_dims=3,
+            in_channels=1,
+            out_channels=2,
+            channels=(16, 32, 64, 128, 256),
+            strides=(2, 2, 2, 2),
+            num_res_units=2,
+            norm=Norm.BATCH,
+        ).to(device)
+
+        # Novograd paper suggests to use a bigger LR than Adam,
+        # because Adam does normalization by element-wise second moments
+        optimizer = Novograd(model.parameters(), learning_rate * 10)
+        scaler = torch.cuda.amp.GradScaler()
+
+        post_pred = AsDiscrete(argmax=True, to_onehot=2)
+        post_label = AsDiscrete(to_onehot=2)
+
+        dice_metric = DiceMetric(include_background=True, reduction="mean", get_not_nans=False)
+
+        best_metric = -1
+        total_start = time.time()
+        for epoch in range(max_epochs):
+            epoch_start = time.time()
+            print("-" * 10)
+            print(f"epoch {epoch + 1}/{max_epochs}")
+            model.train()
+            epoch_loss = 0
+            step = 0
+            for batch_data in train_loader:
+                step_start = time.time()
+                step += 1
+                optimizer.zero_grad()
+                # set AMP for training
+                with torch.cuda.amp.autocast():
+                    outputs = model(batch_data["image"])
+                    loss = loss_function(outputs, batch_data["label"])
+                scaler.scale(loss).backward()
+                scaler.step(optimizer)
+                scaler.update()
+                epoch_loss += loss.item()
+                epoch_len = math.ceil(len(train_ds) / train_loader.batch_size)
+                print(
+                    f"{step}/{epoch_len}, train_loss: {loss.item():.4f}" f" step time: {(time.time() - step_start):.4f}"
+                )
+            epoch_loss /= step
+            print(f"epoch {epoch + 1} average loss: {epoch_loss:.4f}")
+
+            if (epoch + 1) % val_interval == 0:
+                model.eval()
+                with torch.no_grad():
+                    for val_data in val_loader:
+                        roi_size = (96, 96, 96)
+                        sw_batch_size = 4
+                        # set AMP for validation
+                        with torch.cuda.amp.autocast():
+                            val_outputs = sliding_window_inference(val_data["image"], roi_size, sw_batch_size, model)
+
+                        val_outputs = [post_pred(i) for i in decollate_batch(val_outputs)]
+                        val_labels = [post_label(i) for i in decollate_batch(val_data["label"])]
+                        dice_metric(y_pred=val_outputs, y=val_labels)
+
+                    metric = dice_metric.aggregate().item()
+                    dice_metric.reset()
+                    if metric > best_metric:
+                        best_metric = metric
+                    print(f"epoch: {epoch + 1} current mean dice: {metric:.4f}, best mean dice: {best_metric:.4f}")
+            print(f"time consuming of epoch {epoch + 1} is: {(time.time() - epoch_start):.4f}")
+
+        total_time = time.time() - total_start
+        print(f"train completed, best_metric: {best_metric:.4f} total time: {total_time:.4f}")
+        # test expected metrics
+        self.assertGreater(best_metric, 0.95)
+
+
+if __name__ == "__main__":
+    unittest.main()