spacr 0.5.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

spacr/resources/MEDIAR/train_tools/data_utils/datasetter.py

@@ -1,208 +0,0 @@
-from torch.utils.data import DataLoader
-from monai.data import Dataset
-import pickle
-
-from .transforms import (
-    train_transforms,
-    public_transforms,
-    valid_transforms,
-    tuning_transforms,
-    unlabeled_transforms,
-)
-from .utils import split_train_valid, path_decoder
-
-DATA_LABEL_DICT_PICKLE_FILE = "./train_tools/data_utils/custom/modalities.pkl"
-
-__all__ = [
-    "get_dataloaders_labeled",
-    "get_dataloaders_public",
-    "get_dataloaders_unlabeled",
-]
-
-
-def get_dataloaders_labeled(
-    root,
-    mapping_file,
-    mapping_file_tuning,
-    join_mapping_file=None,
-    valid_portion=0.0,
-    batch_size=8,
-    amplified=False,
-    relabel=False,
-):
-    """Set DataLoaders for labeled datasets.
-
-    Args:
-        root (str): root directory
-        mapping_file (str): json file for mapping dataset
-        valid_portion (float, optional): portion of valid datasets. Defaults to 0.1.
-        batch_size (int, optional): batch size. Defaults to 8.
-        shuffle (bool, optional): shuffles dataloader. Defaults to True.
-        num_workers (int, optional): number of workers for each datalaoder. Defaults to 5.
-
-    Returns:
-        dict: dictionary of data loaders.
-    """
-
-    # Get list of data dictionaries from decoded paths
-    data_dicts = path_decoder(root, mapping_file)
-    tuning_dicts = path_decoder(root, mapping_file_tuning, no_label=True)
-
-    if amplified:
-        with open(DATA_LABEL_DICT_PICKLE_FILE, "rb") as f:
-            data_label_dict = pickle.load(f)
-
-        data_point_dict = {}
-
-        for label, data_lst in data_label_dict.items():
-            data_point_dict[label] = []
-
-            for d_idx in data_lst:
-                try:
-                    data_point_dict[label].append(data_dicts[d_idx])
-                except:
-                    print(label, d_idx)
-
-        data_dicts = []
-
-        for label, data_points in data_point_dict.items():
-            len_data_points = len(data_points)
-
-            if len_data_points >= 50:
-                data_dicts += data_points
-            else:
-                for i in range(50):
-                    data_dicts.append(data_points[i % len_data_points])
-
-    data_transforms = train_transforms
-
-    if join_mapping_file is not None:
-        data_dicts += path_decoder(root, join_mapping_file)
-        data_transforms = public_transforms
-
-    if relabel:
-        for elem in data_dicts:
-            cell_idx = int(elem["label"].split("_label.tiff")[0].split("_")[-1])
-            if cell_idx in range(340, 499):
-                new_label = elem["label"].replace(
-                    "/data/CellSeg/Official/Train_Labeled/labels/",
-                    "/CellSeg/pretrained_train_ext/",
-                )
-                elem["label"] = new_label
-
-    # Split datasets as Train/Valid
-    train_dicts, valid_dicts = split_train_valid(
-        data_dicts, valid_portion=valid_portion
-    )
-
-    # Obtain datasets with transforms
-    trainset = Dataset(train_dicts, transform=data_transforms)
-    validset = Dataset(valid_dicts, transform=valid_transforms)
-    tuningset = Dataset(tuning_dicts, transform=tuning_transforms)
-
-    # Set dataloader for Trainset
-    train_loader = DataLoader(
-        trainset, batch_size=batch_size, shuffle=True, num_workers=5
-    )
-
-    # Set dataloader for Validset (Batch size is fixed as 1)
-    valid_loader = DataLoader(validset, batch_size=1, shuffle=False,)
-
-    # Set dataloader for Tuningset (Batch size is fixed as 1)
-    tuning_loader = DataLoader(tuningset, batch_size=1, shuffle=False)
-
-    # Form dataloaders as dictionary
-    dataloaders = {
-        "train": train_loader,
-        "valid": valid_loader,
-        "tuning": tuning_loader,
-    }
-
-    return dataloaders
-
-
-def get_dataloaders_public(
-    root, mapping_file, valid_portion=0.0, batch_size=8,
-):
-    """Set DataLoaders for labeled datasets.
-
-    Args:
-        root (str): root directory
-        mapping_file (str): json file for mapping dataset
-        valid_portion (float, optional): portion of valid datasets. Defaults to 0.1.
-        batch_size (int, optional): batch size. Defaults to 8.
-        shuffle (bool, optional): shuffles dataloader. Defaults to True.
-
-    Returns:
-        dict: dictionary of data loaders.
-    """
-
-    # Get list of data dictionaries from decoded paths
-    data_dicts = path_decoder(root, mapping_file)
-
-    # Split datasets as Train/Valid
-    train_dicts, _ = split_train_valid(data_dicts, valid_portion=valid_portion)
-
-    trainset = Dataset(train_dicts, transform=public_transforms)
-    # Set dataloader for Trainset
-    train_loader = DataLoader(
-        trainset, batch_size=batch_size, shuffle=True, num_workers=5
-    )
-
-    # Form dataloaders as dictionary
-    dataloaders = {
-        "public": train_loader,
-    }
-
-    return dataloaders
-
-
-def get_dataloaders_unlabeled(
-    root, mapping_file, batch_size=8, shuffle=True, num_workers=5,
-):
-    """Set dataloaders for unlabeled dataset."""
-    # Get list of data dictionaries from decoded paths
-    unlabeled_dicts = path_decoder(root, mapping_file, no_label=True, unlabeled=True)
-
-    # Obtain datasets with transforms
-    unlabeled_dicts, _ = split_train_valid(unlabeled_dicts, valid_portion=0)
-    unlabeled_set = Dataset(unlabeled_dicts, transform=unlabeled_transforms)
-
-    # Set dataloader for Unlabeled dataset
-    unlabeled_loader = DataLoader(
-        unlabeled_set, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers
-    )
-
-    dataloaders = {
-        "unlabeled": unlabeled_loader,
-    }
-
-    return dataloaders
-
-
-def get_dataloaders_unlabeled_psuedo(
-    root, mapping_file, batch_size=8, shuffle=True, num_workers=5,
-):
-
-    # Get list of data dictionaries from decoded paths
-    unlabeled_psuedo_dicts = path_decoder(
-        root, mapping_file, no_label=False, unlabeled=True
-    )
-
-    # Obtain datasets with transforms
-    unlabeled_psuedo_dicts, _ = split_train_valid(
-        unlabeled_psuedo_dicts, valid_portion=0
-    )
-    unlabeled_psuedo_set = Dataset(unlabeled_psuedo_dicts, transform=train_transforms)
-
-    # Set dataloader for Unlabeled dataset
-    unlabeled_psuedo_loader = DataLoader(
-        unlabeled_psuedo_set,
-        batch_size=batch_size,
-        shuffle=shuffle,
-        num_workers=num_workers,
-    )
-
-    dataloaders = {"unlabeled": unlabeled_psuedo_loader}
-
-    return dataloaders

spacr/resources/MEDIAR/train_tools/data_utils/transforms.py

@@ -1,148 +0,0 @@
-from .custom import *
-
-from monai.transforms import *
-
-__all__ = [
-    "train_transforms",
-    "public_transforms",
-    "valid_transforms",
-    "tuning_transforms",
-    "unlabeled_transforms",
-]
-
-train_transforms = Compose(
-    [
-        # >>> Load and refine data --- img: (H, W, 3); label: (H, W)
-        CustomLoadImaged(keys=["img", "label"], image_only=True),
-        CustomNormalizeImaged(
-            keys=["img"],
-            allow_missing_keys=True,
-            channel_wise=False,
-            percentiles=[0.0, 99.5],
-        ),
-        EnsureChannelFirstd(keys=["img", "label"], channel_dim=-1),
-        RemoveRepeatedChanneld(keys=["label"], repeats=3),  # label: (H, W)
-        ScaleIntensityd(keys=["img"], allow_missing_keys=True),  # Do not scale label
-        # >>> Spatial transforms
-        RandZoomd(
-            keys=["img", "label"],
-            prob=0.5,
-            min_zoom=0.25,
-            max_zoom=1.5,
-            mode=["area", "nearest"],
-            keep_size=False,
-        ),
-        SpatialPadd(keys=["img", "label"], spatial_size=512),
-        RandSpatialCropd(keys=["img", "label"], roi_size=512, random_size=False),
-        RandAxisFlipd(keys=["img", "label"], prob=0.5),
-        RandRotate90d(keys=["img", "label"], prob=0.5, spatial_axes=[0, 1]),
-        IntensityDiversification(keys=["img", "label"], allow_missing_keys=True),
-        # # >>> Intensity transforms
-        RandGaussianNoised(keys=["img"], prob=0.25, mean=0, std=0.1),
-        RandAdjustContrastd(keys=["img"], prob=0.25, gamma=(1, 2)),
-        RandGaussianSmoothd(keys=["img"], prob=0.25, sigma_x=(1, 2)),
-        RandHistogramShiftd(keys=["img"], prob=0.25, num_control_points=3),
-        RandGaussianSharpend(keys=["img"], prob=0.25),
-        EnsureTyped(keys=["img", "label"]),
-    ]
-)
-
-
-public_transforms = Compose(
-    [
-        CustomLoadImaged(keys=["img", "label"], image_only=True),
-        BoundaryExclusion(keys=["label"]),
-        CustomNormalizeImaged(
-            keys=["img"],
-            allow_missing_keys=True,
-            channel_wise=False,
-            percentiles=[0.0, 99.5],
-        ),
-        EnsureChannelFirstd(keys=["img", "label"], channel_dim=-1),
-        RemoveRepeatedChanneld(keys=["label"], repeats=3),  # label: (H, W)
-        ScaleIntensityd(keys=["img"], allow_missing_keys=True),  # Do not scale label
-        # >>> Spatial transforms
-        SpatialPadd(keys=["img", "label"], spatial_size=512),
-        RandSpatialCropd(keys=["img", "label"], roi_size=512, random_size=False),
-        RandAxisFlipd(keys=["img", "label"], prob=0.5),
-        RandRotate90d(keys=["img", "label"], prob=0.5, spatial_axes=[0, 1]),
-        Rotate90d(k=1, keys=["label"], spatial_axes=(0, 1)),
-        Flipd(keys=["label"], spatial_axis=0),
-        EnsureTyped(keys=["img", "label"]),
-    ]
-)
-
-
-valid_transforms = Compose(
-    [
-        CustomLoadImaged(keys=["img", "label"], allow_missing_keys=True, image_only=True),
-        CustomNormalizeImaged(
-            keys=["img"],
-            allow_missing_keys=True,
-            channel_wise=False,
-            percentiles=[0.0, 99.5],
-        ),
-        EnsureChannelFirstd(keys=["img", "label"], allow_missing_keys=True, channel_dim=-1),
-        RemoveRepeatedChanneld(keys=["label"], repeats=3),
-        ScaleIntensityd(keys=["img"], allow_missing_keys=True),
-        EnsureTyped(keys=["img", "label"], allow_missing_keys=True),
-    ]
-)
-
-tuning_transforms = Compose(
-    [
-        CustomLoadImaged(keys=["img"], image_only=True),
-        CustomNormalizeImaged(
-            keys=["img"],
-            allow_missing_keys=True,
-            channel_wise=False,
-            percentiles=[0.0, 99.5],
-        ),
-        EnsureChannelFirstd(keys=["img"], channel_dim=-1),
-        ScaleIntensityd(keys=["img"]),
-        EnsureTyped(keys=["img"]),
-    ]
-)
-
-unlabeled_transforms = Compose(
-    [
-        # >>> Load and refine data --- img: (H, W, 3); label: (H, W)
-        CustomLoadImaged(keys=["img"], image_only=True),
-        CustomNormalizeImaged(
-            keys=["img"],
-            allow_missing_keys=True,
-            channel_wise=False,
-            percentiles=[0.0, 99.5],
-        ),
-        EnsureChannelFirstd(keys=["img"], channel_dim=-1),
-        RandZoomd(
-            keys=["img"],
-            prob=0.5,
-            min_zoom=0.25,
-            max_zoom=1.25,
-            mode=["area"],
-            keep_size=False,
-        ),
-        ScaleIntensityd(keys=["img"], allow_missing_keys=True),  # Do not scale label
-        # >>> Spatial transforms
-        SpatialPadd(keys=["img"], spatial_size=512),
-        RandSpatialCropd(keys=["img"], roi_size=512, random_size=False),
-        EnsureTyped(keys=["img"]),
-    ]
-)
-
-
-def get_pred_transforms():
-    """Prediction preprocessing"""
-    pred_transforms = Compose(
-        [
-            # >>> Load and refine data
-            CustomLoadImage(image_only=True),
-            CustomNormalizeImage(channel_wise=False, percentiles=[0.0, 99.5]),
-            EnsureChannelFirst(channel_dim=-1),  # image: (3, H, W)
-            ScaleIntensity(),
-            EnsureType(data_type="tensor"),
-        ]
-    )
-
-    return pred_transforms

spacr/resources/MEDIAR/train_tools/data_utils/utils.py

@@ -1,84 +0,0 @@
-import os
-import json
-import numpy as np
-
-__all__ = ["split_train_valid", "path_decoder"]
-
-
-def split_train_valid(data_dicts, valid_portion=0.1):
-    """Split train/validata data according to the given proportion"""
-
-    train_dicts, valid_dicts = data_dicts, []
-    if valid_portion > 0:
-
-        # Obtain & shuffle data indices
-        num_data_dicts = len(data_dicts)
-        indices = np.arange(num_data_dicts)
-        np.random.shuffle(indices)
-
-        # Divide train/valid indices by the proportion
-        valid_size = int(num_data_dicts * valid_portion)
-        train_indices = indices[valid_size:]
-        valid_indices = indices[:valid_size]
-
-        # Assign data dicts by split indices
-        train_dicts = [data_dicts[idx] for idx in train_indices]
-        valid_dicts = [data_dicts[idx] for idx in valid_indices]
-
-    print(
-        "\n(DataLoaded) Training data size: %d, Validation data size: %d\n"
-        % (len(train_dicts), len(valid_dicts))
-    )
-
-    return train_dicts, valid_dicts
-
-
-def path_decoder(root, mapping_file, no_label=False, unlabeled=False):
-    """Decode img/label file paths from root & mapping directory.
-
-    Args:
-        root (str):
-        mapping_file (str): json file containing image & label file paths.
-        no_label (bool, optional): whether to include "label" key. Defaults to False.
-
-    Returns:
-        list: list of dictionary. (ex. [{"img": img_path, "label": label_path}, ...])
-    """
-
-    data_dicts = []
-
-    with open(mapping_file, "r") as file:
-        data = json.load(file)
-
-        for map_key in data.keys():
-
-            # If no_label, assign "img" key only
-            if no_label:
-                data_dict_item = [
-                    {"img": os.path.join(root, elem["img"]),} for elem in data[map_key]
-                ]
-
-            # If label exists, assign both "img" and "label" keys
-            else:
-                data_dict_item = [
-                    {
-                        "img": os.path.join(root, elem["img"]),
-                        "label": os.path.join(root, elem["label"]),
-                    }
-                    for elem in data[map_key]
-                ]
-
-            # Add refined datasets to be returned
-            data_dicts += data_dict_item
-
-    if unlabeled:
-        refined_data_dicts = []
-
-        # Exclude the corrupted image to prevent errror
-        for data_dict in data_dicts:
-            if "00504" not in data_dict["img"]:
-                refined_data_dicts.append(data_dict)
-
-        data_dicts = refined_data_dicts
-
-    return data_dicts

spacr/resources/MEDIAR/train_tools/measures.py

@@ -1,200 +0,0 @@
-"""
-Adapted from the following references:
-[1] https://github.com/JunMa11/NeurIPS-CellSeg/blob/main/baseline/compute_metric.py
-[2] https://github.com/stardist/stardist/blob/master/stardist/matching.py
-
-"""
-
-import numpy as np
-from skimage import segmentation
-from scipy.optimize import linear_sum_assignment
-from numba import jit
-
-__all__ = ["evaluate_f1_score_cellseg", "evaluate_f1_score"]
-
-
-def evaluate_f1_score_cellseg(masks_true, masks_pred, threshold=0.5):
-    """
-    Get confusion elements for cell segmentation results.
-    Boundary pixels are not considered during evaluation.
-    """
-
-    if np.prod(masks_true.shape) < (5000 * 5000):
-        masks_true = _remove_boundary_cells(masks_true.astype(np.int32))
-        masks_pred = _remove_boundary_cells(masks_pred.astype(np.int32))
-
-        tp, fp, fn = get_confusion(masks_true, masks_pred, threshold)
-
-    # Compute by Patch-based way for large images
-    else:
-        H, W = masks_true.shape
-        roi_size = 2000
-
-        # Get patch grid by roi_size
-        if H % roi_size != 0:
-            n_H = H // roi_size + 1
-            new_H = roi_size * n_H
-        else:
-            n_H = H // roi_size
-            new_H = H
-
-        if W % roi_size != 0:
-            n_W = W // roi_size + 1
-            new_W = roi_size * n_W
-        else:
-            n_W = W // roi_size
-            new_W = W
-
-        # Allocate values on the grid
-        gt_pad = np.zeros((new_H, new_W), dtype=masks_true.dtype)
-        pred_pad = np.zeros((new_H, new_W), dtype=masks_true.dtype)
-        gt_pad[:H, :W] = masks_true
-        pred_pad[:H, :W] = masks_pred
-
-        tp, fp, fn = 0, 0, 0
-
-        # Calculate confusion elements for each patch
-        for i in range(n_H):
-            for j in range(n_W):
-                gt_roi = _remove_boundary_cells(
-                    gt_pad[
-                        roi_size * i : roi_size * (i + 1),
-                        roi_size * j : roi_size * (j + 1),
-                    ]
-                )
-                pred_roi = _remove_boundary_cells(
-                    pred_pad[
-                        roi_size * i : roi_size * (i + 1),
-                        roi_size * j : roi_size * (j + 1),
-                    ]
-                )
-                tp_i, fp_i, fn_i = get_confusion(gt_roi, pred_roi, threshold)
-                tp += tp_i
-                fp += fp_i
-                fn += fn_i
-
-    # Calculate f1 score
-    precision, recall, f1_score = evaluate_f1_score(tp, fp, fn)
-
-    return precision, recall, f1_score
-
-
-def evaluate_f1_score(tp, fp, fn):
-    """Evaluate F1-score for the given confusion elements"""
-
-    # Do not Compute on trivial results
-    if tp == 0:
-        precision, recall, f1_score = 0, 0, 0
-
-    else:
-        precision = tp / (tp + fp)
-        recall = tp / (tp + fn)
-        f1_score = 2 * (precision * recall) / (precision + recall)
-
-    return precision, recall, f1_score
-
-
-def _remove_boundary_cells(mask):
-    """Remove cells on the boundary from the mask"""
-
-    # Identify boundary cells
-    W, H = mask.shape
-    bd = np.ones((W, H))
-    bd[2 : W - 2, 2 : H - 2] = 0
-    bd_cells = np.unique(mask * bd)
-
-    # Remove cells on the boundary
-    for i in bd_cells[1:]:
-        mask[mask == i] = 0
-
-    # Allocate labels as sequential manner
-    new_label, _, _ = segmentation.relabel_sequential(mask)
-
-    return new_label
-
-
-def get_confusion(masks_true, masks_pred, threshold=0.5):
-    """Calculate confusion matrix elements: (TP, FP, FN)"""
-    num_gt_instances = np.max(masks_true)
-    num_pred_instances = np.max(masks_pred)
-
-    if num_pred_instances == 0:
-        print("No segmentation results!")
-        tp, fp, fn = 0, 0, 0
-
-    else:
-        # Calculate IoU and exclude background label (0)
-        iou = _get_iou(masks_true, masks_pred)
-        iou = iou[1:, 1:]
-
-        # Calculate true positives
-        tp = _get_true_positive(iou, threshold)
-        fp = num_pred_instances - tp
-        fn = num_gt_instances - tp
-
-    return tp, fp, fn
-
-
-def _get_true_positive(iou, threshold=0.5):
-    """Get true positive (TP) pixels at the given threshold"""
-
-    # Number of instances to be matched
-    num_matched = min(iou.shape[0], iou.shape[1])
-
-    # Find optimal matching by using IoU as tie-breaker
-    costs = -(iou >= threshold).astype(np.float) - iou / (2 * num_matched)
-    matched_gt_label, matched_pred_label = linear_sum_assignment(costs)
-
-    # Consider as the same instance only if the IoU is above the threshold
-    match_ok = iou[matched_gt_label, matched_pred_label] >= threshold
-    tp = match_ok.sum()
-
-    return tp
-
-
-def _get_iou(masks_true, masks_pred):
-    """Get the iou between masks_true and masks_pred"""
-
-    # Get overlap matrix (GT Instances Num, Pred Instance Num)
-    overlap = _label_overlap(masks_true, masks_pred)
-
-    # Predicted instance pixels
-    n_pixels_pred = np.sum(overlap, axis=0, keepdims=True)
-
-    # GT instance pixels
-    n_pixels_true = np.sum(overlap, axis=1, keepdims=True)
-
-    # Calculate intersection of union (IoU)
-    union = n_pixels_pred + n_pixels_true - overlap
-    iou = overlap / union
-
-    # Ensure numerical values
-    iou[np.isnan(iou)] = 0.0
-
-    return iou
-
-
-@jit(nopython=True)
-def _label_overlap(x, y):
-    """Get pixel overlaps between two masks
-
-    Parameters
-    ------------
-    x, y (np array; dtype int): 0=NO masks; 1,2... are mask labels
-
-    Returns
-    ------------
-    overlap (np array; dtype int): Overlaps of size [x.max()+1, y.max()+1]
-    """
-
-    # Make as 1D array
-    x, y = x.ravel(), y.ravel()
-
-    # Preallocate a Contact Map matrix
-    overlap = np.zeros((1 + x.max(), 1 + y.max()), dtype=np.uint)
-
-    # Calculate the number of shared pixels for each label
-    for i in range(len(x)):
-        overlap[x[i], y[i]] += 1
-
-    return overlap