dsa-helpers 1.1.5.dev4__tar.gz → 1.1.5.dev6__tar.gz

{dsa_helpers-1.1.5.dev4/dsa_helpers.egg-info → dsa_helpers-1.1.5.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsa_helpers
-Version: 1.1.5.dev4
+Version: 1.1.5.dev6
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -28,6 +28,7 @@ Requires-Dist: datasets
 Requires-Dist: torchvision
 Requires-Dist: large-image
 Requires-Dist: rasterio
+Requires-Dist: colorama
 Dynamic: license-file
 
 # Digital Slide Archive (DSA) Helpers

{dsa_helpers-1.1.5.dev4 → dsa_helpers-1.1.5.dev6}/dsa_helpers/__init__.py

@@ -3,7 +3,7 @@ from .imread import imread
 from .imwrite import imwrite
 
 # Version of the dsa-helpers package
-__version__ = "1.1.5.dev4"
+__version__ = "1.1.5.dev6"
 
 # To avoid slow downs, do not allow from dsa_helpers import * to import anything.
 __all__ = []

{dsa_helpers-1.1.5.dev4 → dsa_helpers-1.1.5.dev6}/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py

@@ -1,11 +1,14 @@
 from torch.utils.data import Dataset
 import pandas as pd
 from PIL import Image
-import numpy as np
+from colorama import Style, Fore
 
 
 class SegFormerSegmentationDataset(Dataset):
-    """PyTorch dataset class for semantic segmentation using HuggingFaces SegFormer
+    """DEPRECATED: see
+    dsa_helpers.ml.segformer_semantic_segmentation.datasets.SegFormerSegmentationDataset
+
+    PyTorch dataset class for semantic segmentation using HuggingFaces SegFormer
     model.
 
     """
@@ -17,6 +20,11 @@ class SegFormerSegmentationDataset(Dataset):
             df (pandas.DataFrame): A pandas DataFrame with columns "fp" and "mask_fp".
 
         """
+        print(Fore.RED)
+        print(
+            "This is deprecated, please import from dsa_helpers.ml.segformer_semantic_segmentation.datasets"
+        )
+        print(Style.RESET_ALL)
         self.image_files = df["fp"].tolist()
         self.mask_files = df["mask_fp"].tolist()
         self.df = df  # store potential metadata

{dsa_helpers-1.1.5.dev4 → dsa_helpers-1.1.5.dev6}/dsa_helpers/ml/datasets/utils.py

@@ -2,6 +2,7 @@
 import pandas as pd
 from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
 from datasets import Dataset, Features, Image
+from colorama import Style, Fore
 
 
 def dataset_generator(dataset):
@@ -13,7 +14,10 @@ def dataset_generator(dataset):
 def create_segformer_segmentation_dataset(
     df: pd.DataFrame | str, low_memory: bool = True, transforms=None
 ):
-    """Create a SegFormer segmentation dataset from a DataFrame.
+    """DEPRECATED: see
+    dsa_helpers.ml.segformer_semantic_segmentation.datasets.create_segformer_segmentation_dataset
+
+    Create a SegFormer segmentation dataset from a DataFrame.
 
     Args:
         df (pd.DataFrame | str): A pandas DataFrame with columns "fp" and "mask_fp" or a
@@ -26,6 +30,11 @@ def create_segformer_segmentation_dataset(
         A Dataset object to be used for HuggingFaces SegFormer model training.
 
     """
+    print(Fore.RED)
+    print(
+        "This is deprecated, please import from dsa_helpers.ml.segformer_semantic_segmentation.datasets"
+    )
+    print(Style.RESET_ALL)
     if isinstance(df, str):
         df = pd.read_csv(df, low_memory=low_memory)
 

dsa_helpers-1.1.5.dev6/dsa_helpers/ml/evaluate.py

@@ -0,0 +1,307 @@
+"""Functions for evaluating datasets with machine learning models.
+
+- evaluate_semantic_segmentation_segformer_group: Evaluate a segformer
+  trainer model on a specific dataset provided as a dataframe, grouped
+  by a specific column.
+- per_class_dice_on_dataset: Per class DICE coefficient on a set of
+    images.
+
+"""
+
+import torch
+from torch import nn
+from tqdm import tqdm
+from transformers.trainer import Trainer
+import pandas as pd
+import numpy as np
+from .datasets.utils import create_segformer_segmentation_dataset
+from .transforms.segformer_transforms import val_transforms
+from ..tile_utils import merge_tile_masks
+
+
+def evaluate_semantic_segmentation_segformer_group(
+    trainer: Trainer,
+    df: pd.DataFrame,
+    group_col: str,
+    batch_size: int = 16,
+    tile_size: int = 512,
+    background_label: int | None = None,
+) -> float:
+    """Evaluate a segformer trainer model on a specific dataset provided
+    as a dataframe, grouped by a specific column. For a unique value of
+    the column it will predict on the images for that group and then
+    merge the predictions into a single multipolygon. The mean IoU will
+    be calculated agains the ground truth masks for that group. The
+    reported mean IoU will be the average of all the mean IoUs for each
+    group.
+
+    Args:
+        trainer (Trainer): The trainer object with the model to evaluate.
+        df (pandas.DataFrame): The dataframe with the tile metadata.
+            Must have the fp column (file path), x column (x coordinate),
+            y column (y coordinate), and group_col.
+        group_col (str): The column to group the tiles by.
+        batch_size (int): The batch size to use for prediction. Note that
+            this is not the batch size use for predictions, rather it is
+            how many masks will be stored in memory. Batch size used
+            for predictions is specified on the trainer input. Default is
+            16.
+        tile_size (int): The size of the tiles. Default is 512.
+        background_label (int | None): The label to use for the background
+            class, which will be ignored. If None then no label will be
+            ignored. Default is None.
+
+    Returns:
+        float: The mean IoU across all groups.
+
+    """
+    ious = []
+
+    # Predict on the images for each group separately.
+    groups = list(df[group_col].unique())
+    for group in tqdm(groups, desc="Evaluating groups"):
+        # Get the tiles in this group.
+        group_df = df[df[group_col] == group]
+
+        pred_tile_list = []
+        true_tile_list = []
+
+        # Predict on the group in batches.
+        for i in range(0, len(group_df), batch_size):
+            # Create dataset for this batch.
+            batch_df = group_df.iloc[i : i + batch_size]
+            x_list = batch_df["x"].tolist()
+            y_list = batch_df["y"].tolist()
+
+            # Create dataset for this batch.
+            group_dataset = create_segformer_segmentation_dataset(
+                batch_df, transforms=val_transforms
+            )
+
+            # Predict on the batch.
+            out = trainer.predict(group_dataset)
+            preds = out[0]
+            true = out[1]
+
+            # Reshape the prediction logits to tile size.
+            preds = torch.from_numpy(preds).cpu()
+
+            preds = nn.functional.interpolate(
+                preds,
+                size=(tile_size, tile_size),
+                mode="bilinear",
+                align_corners=False,
+            ).argmax(
+                dim=1
+            )  # logits -> class predictions
+
+            preds = preds.detach().numpy()
+
+            # Append to the tile list.
+            pred_tile_list.extend(
+                [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
+            )
+
+            true_tile_list.extend(
+                [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
+            )
+
+        # Merge the tile masks.
+        pred_gdf = merge_tile_masks(
+            pred_tile_list, background_label=background_label
+        )
+        true_gdf = merge_tile_masks(
+            true_tile_list, background_label=background_label
+        )
+
+        # Get the unique labels in either the prediction or the ground truth.
+        labels = set(pred_gdf["label"].unique()) | set(
+            true_gdf["label"].unique()
+        )
+
+        # Calculate the IoU for each label.
+        ious = []
+
+        for label in labels:
+            pred_label_gdf = pred_gdf[pred_gdf["label"] == label]
+            true_label_gdf = true_gdf[true_gdf["label"] == label]
+
+            # If either the prediction or the ground truth is empty, the IoU is 0.
+            if pred_label_gdf.empty or true_label_gdf.empty:
+                ious.append(0)
+            else:
+                # Calculate the IoU.
+                geom1 = pred_label_gdf.iloc[0]["geometry"]
+                geom2 = true_label_gdf.iloc[0]["geometry"]
+
+                intersection_area = geom1.intersection(geom2).area
+                union = geom1.union(geom2).area
+
+                ious.append(intersection_area / union)
+
+    # Return the average of the ious.
+    return float(sum(ious) / len(ious))
+
+
+# def evaluate_semantic_segmentation_segformer_by_wsi(
+#     model: nn.Module,
+#     data: pd.DataFrame | str,
+#     group_col: str,
+#     label2id: dict[str, int],
+#     batch_size: int = 16,
+#     tile_size: int = 512,
+#     background_label: int | None = None,
+#     device: torch.device | None = None,
+# ) -> float:
+#     """Evaluate a segformer trainer model on a specific dataset provided
+#     as a dataframe, grouped by a specific column. For a unique value of
+#     the column it will predict on the images for that group and then
+#     merge the predictions into a single multipolygon. The mean IoU will
+#     be calculated agains the ground truth masks for that group. The
+#     reported mean IoU will be the average of all the mean IoUs for each
+#     group.
+
+#     Args:
+#         model (nn.Module): The model to evaluate.
+#         data (pd.DataFrame | str): The dataframe or path to a csv file
+#             containing the data to evaluate on. The dataframe must have
+#             the "fp" (filepath to image) and "mask_fp" (filepath to
+#             mask) columns. The column matching the "group_col" parameter
+#             will be used to group tiles into groups. The "x" and "y"
+#             parameters will be used for stiching the images back to
+#             their original size.
+#         group_col (str): The column to group the tiles by.
+#         batch_size (int): The batch size to use for prediction. Note that
+#             this is not the batch size use for predictions, rather it is
+#             how many masks will be stored in memory. Batch size used
+#             for predictions is specified on the trainer input. Default is
+#             16.
+#         label2id (dict): A dictionary mapping labels to their integer
+#             representations.
+#         tile_size (int): The size of the tiles. Default is 512.
+#         background_label (int | None): The label to use for the background
+#             class, which will be ignored. If None then no label will be
+#             ignored. Default is None.
+#         device (torch.device | None, optional): The device to use for
+#             evaluation. If None, the device will be "cuda" if available,
+#             otherwise "cpu". Default is None.
+
+#     Returns:
+#         float: The mean IoU across all groups.
+
+#     """
+#     # Read the data.
+#     if isinstance(data, str):
+#         data = pd.read_csv(data)
+
+#     ious = []
+
+#     if device is None:
+#         device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+#     # Move the model to the device.
+#     model.to(device)
+#     model.eval()  # model should be in evaluation mode
+
+#     # Predict on the images for each group separately.
+#     groups = list(data[group_col].unique())
+#     for group in tqdm(groups, desc="Evaluating groups"):
+#         # Get the tiles in this group.
+#         group_df = data[data[group_col] == group]
+
+#         pred_tile_list = []
+#         true_tile_list = []
+
+#         # Predict on the group in batches.
+#         for i in range(0, len(group_df), batch_size):
+#             # Create dataset for this batch.
+#             batch_df = group_df.iloc[i : i + batch_size]
+#             x_list = batch_df["x"].tolist()
+#             y_list = batch_df["y"].tolist()
+
+#             # Create dataset for this batch.
+#             dataset = create_segformer_segmentation_dataset(
+#                 batch_df, transforms=val_transforms
+#             )
+
+#             # Get the inputs.
+#             inputs = np.array(dataset[:]["pixel_values"])
+#             inputs = torch.tensor(inputs, dtype=torch.float32)
+#             inputs = inputs.to(device)
+
+#             true = np.array(dataset[:]["labels"])
+
+#             # Predict on the batch.
+#             with torch.no_grad():
+#                 out = model(inputs)
+#                 preds = out.logits
+
+#                 # Reshape the prediction logits to tile size.
+#                 preds = torch.from_numpy(preds)
+
+#                 preds = nn.functional.interpolate(
+#                     preds,
+#                     size=(tile_size, tile_size),
+#                     mode="bilinear",
+#                     align_corners=False,
+#                 ).argmax(
+#                     dim=1
+#                 )  # logits -> class predictions
+
+#                 preds = preds.detach().cpu().numpy()
+
+#                 # Append to the tile list.
+#                 pred_tile_list.extend(
+#                     [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
+#                 )
+
+#                 true_tile_list.extend(
+#                     [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
+#                 )
+
+#         # Merge the tile masks.
+#         pred_gdf = merge_tile_masks(
+#             pred_tile_list, background_label=background_label
+#         )
+#         true_gdf = merge_tile_masks(
+#             true_tile_list, background_label=background_label
+#         )
+
+#         # Calculate the Dice for each label.
+#         metrics = {label: [] for label in label2id}
+
+#         for label, integer in label2id.items():
+#             if integer == background_label:
+#                 continue  # don't calculate dice for background
+
+#             # For this integer, get the prediction and ground truth.
+#             pred_label_gdf = pred_gdf[pred_gdf["label"] == integer]
+#             true_label_gdf = true_gdf[true_gdf["label"] == integer]
+
+#             # If both are empty, the dice is 1.
+#             if pred_label_gdf.empty and true_label_gdf.empty:
+#                 metrics[label].append(1)
+#             else:
+#                 # Calculate the intersection and denominator.
+#                 intersection = pred_label_gdf.geometry.intersection(
+#                     true_label_gdf.geometry
+#                 ).area.sum()
+
+#                 # The denominator is the sum area of the two polygons.
+#                 denominator = (
+#                     pred_label_gdf.geometry.area.sum()
+#                     + true_label_gdf.geometry.area.sum()
+#                 )
+
+#                 # The dice is 2 * intersection / denominator.
+#                 metrics[label].append(float(2 * intersection / denominator))
+
+#     # Return the dice as means and standard deviations.
+#     metrics = {
+#         label: {
+#             "mean": float(np.mean(metrics[label])),
+#             "std": float(np.std(metrics[label])),
+#         }
+#         for label in label2id
+#     }
+#     return metrics

dsa_helpers-1.1.5.dev6/dsa_helpers/ml/segformer_semantic_segmentation/datasets.py

@@ -0,0 +1,119 @@
+from torch.utils.data import Dataset
+import pandas as pd
+from PIL import Image
+from datasets import Dataset, Features, Image, Value
+
+
+class SegFormerSegmentationDataset(Dataset):
+    """PyTorch dataset class for semantic segmentation using
+    HuggingFaces SegFormer model.
+
+    """
+
+    def __init__(
+        self, df: pd.DataFrame, extras: bool = False, group: str = "wsi_name"
+    ):
+        """Initiate an instance of the segmentation dataset.
+
+        Args:
+            df (pandas.DataFrame): Must have columns "fp" and "mask_fp".
+            extras (bool, optional): Whether to include additional
+                columns "x", "y" and "group" in the dataset. Default
+                is False.
+            group (str | None, optional): Additional column to add when
+                extras is True. Default is "wsi_name".
+
+        """
+        self.image_files = df["fp"].tolist()
+        self.mask_files = df["mask_fp"].tolist()
+        self.extras = extras
+
+        if extras:
+            self.x = df["x"].tolist()
+            self.y = df["y"].tolist()
+            self.group = df[group].tolist()
+
+    def __len__(self):
+        return len(self.image_files)
+
+    def __getitem__(self, idx):
+        # Get the filepath to the image and its mask.
+        image_path = self.image_files[idx]
+        mask_path = self.mask_files[idx]
+
+        if self.extras:
+            return {
+                "pixel_values": Image.open(image_path),
+                "label": Image.open(mask_path),
+                "x": self.x[idx],
+                "y": self.y[idx],
+                "group": self.group[idx],
+            }
+        else:
+            return {
+                "pixel_values": Image.open(image_path),
+                "label": Image.open(mask_path),
+            }
+
+
+def dataset_generator(dataset):
+    """Yield a dataset."""
+    for item in dataset:
+        yield item
+
+
+def create_segformer_segmentation_dataset(
+    df: pd.DataFrame | str,
+    low_memory: bool = True,
+    transforms=None,
+    extras: bool = False,
+    group: str | None = None,
+) -> SegFormerSegmentationDataset:
+    """Create a SegFormer segmentation dataset from a DataFrame.
+
+    Args:
+        df (pandas.DataFrame): Must have columns "fp" and "mask_fp".
+            Additionally, it can have columns "x", "y". If these are not
+            present, then they will be set to 0.
+        low_memory (bool, optional): Whether to read the CSV file in low
+            memory mode. Default is True.
+        transforms: Function to add transforms to the images.
+        group (str | None, optional): Additional column to pass to
+            the model, if None it will be an empty string. Default
+            is None.
+
+    Returns:
+        (SegFormerSegmentationDataset): A Dataset object to be used for
+        HuggingFaces SegFormer model training.
+
+    """
+    if isinstance(df, str):
+        df = pd.read_csv(df, low_memory=low_memory)
+
+    dataset = SegFormerSegmentationDataset(df, group=group, extras=extras)
+
+    if extras:
+        features = Features(
+            {
+                "pixel_values": Image(),
+                "label": Image(),
+                "x": Value("int32"),
+                "y": Value("int32"),
+                "group": Value("string"),
+            }
+        )
+    else:
+        features = Features(
+            {
+                "pixel_values": Image(),
+                "label": Image(),
+            }
+        )
+    dataset = Dataset.from_generator(
+        generator=lambda: dataset_generator(dataset), features=features
+    )
+
+    if transforms is not None:
+        dataset.set_transform(transforms)
+
+    return dataset

dsa_helpers-1.1.5.dev6/dsa_helpers/ml/segformer_semantic_segmentation/evaluate.py

@@ -0,0 +1,126 @@
+from torch import nn
+import pandas as pd
+import numpy as np
+import torch
+
+from .transforms import val_transforms
+from .datasets import create_segformer_segmentation_dataset
+
+
+def per_class_dice_on_dataset(
+    model: nn.Module,
+    data: pd.DataFrame | str,
+    label2id: dict[str, int],
+    batch_size: int = 16,
+    device: torch.device | None = None,
+    tile_size: int = 512,
+    tqdm_notebook: bool = False,
+):
+    """Calculate the DICE coefficient for a given model and set of
+    images. The model must be a semantic segmentation model that outputs
+    an object with the logits attribute. The dataset must be iterable
+    and return a dictionary with the keys "pixel_values" and "labels".
+    The pixel values should be in the format the model expects for the
+    input.
+
+    Args:
+        model (nn.Module): The model to evaluate.
+        data (pd.DataFrame | str): The dataframe or path to a csv file
+            containing the data to evaluate on. The dataframe must have
+            the "fp" (filepath to image) and "mask_fp" (filepath to
+            mask) columns.
+        label2id (dict): A dictionary mapping labels to their integer
+            representations.
+        batch_size (int, optional): The batch size to use for evaluation.
+            Default is 16.
+        device (torch.device | None, optional): The device to use for
+            evaluation. If None, the device will be "cuda" if available,
+            otherwise "cpu". Default is None.
+        tile_size (int, optional): The size of the tiles to use for
+            evaluation. Default is 512.
+        tqdm_notebook (bool, optional): Whether to use a tqdm notebook.
+            Default is False.
+
+    Returns:
+        dict: A dictionary with the keys "mean_dice" and the DICE
+            coefficient for each label.
+    """
+    if tqdm_notebook:
+        from tqdm.notebook import tqdm
+    else:
+        from tqdm import tqdm
+
+    if isinstance(data, str):
+        data = pd.read_csv(data)
+
+    # Create the dataset object.
+    dataset = create_segformer_segmentation_dataset(
+        data, transforms=val_transforms
+    )
+
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        print(f"Using device: {device}")
+
+    # Move the model to the device.
+    model.to(device)
+    model.eval()  # model should be in evaluation mode
+
+    # Number of images in the dataset.
+    n = len(dataset)
+
+    # Initialize the intersection and denominator for each label.
+    intersection = {label: 0 for label in label2id}
+    denominator = {label: 0 for label in label2id}
+
+    batches = list(range(0, n, batch_size))
+
+    for i in tqdm(batches):
+        # Get the batch of images and labels.
+        batch = dataset[i : i + batch_size]
+
+        inputs = np.array(batch["pixel_values"])
+        labels = np.array(batch["labels"])
+        inputs = torch.tensor(inputs, dtype=torch.float32)
+        labels = np.array(labels)
+
+        with torch.no_grad():
+            inputs = inputs.to(device)
+            outputs = model(inputs)
+            logits = outputs.logits
+
+            # Get the logits out, resizing them to the original tile size.
+            logits = torch.nn.functional.interpolate(
+                logits,
+                size=tile_size,
+                mode="bilinear",
+            )
+
+            # Get predicted class labels for each pixel.
+            masks = torch.argmax(logits, dim=1).detach().cpu().numpy()
+
+            # Flatten both masks and labels.
+            masks = masks.flatten()
+            labels = labels.flatten()
+
+            for label, integer in label2id.items():
+                # Calculate where both mask and labels are equal to the integer.
+                gt_mask = labels == integer
+                pred_mask = masks == integer
+                intersection[label] += np.sum(gt_mask & pred_mask)
+                denominator[label] += np.sum(gt_mask) + np.sum(pred_mask)
+
+    # Calculate the dice for each class.
+    metrics = {}
+    for label in label2id:
+        denominator_value = denominator[label]
+
+        if denominator_value:
+            metrics[f"{label}_dice"] = float(
+                2 * intersection[label] / denominator[label]
+            )
+        else:
+            metrics[f"{label}_dice"] = 1
+
+    # Calculate the mean dice.
+    return metrics