dsa-helpers 1.1.4.dev1__tar.gz → 1.1.5.dev0__tar.gz

{dsa_helpers-1.1.4.dev1/dsa_helpers.egg-info → dsa_helpers-1.1.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsa_helpers
-Version: 1.1.4.dev1
+Version: 1.1.5.dev0
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>

{dsa_helpers-1.1.4.dev1 → dsa_helpers-1.1.5.dev0}/dsa_helpers/__init__.py

@@ -3,7 +3,7 @@ from .imread import imread
 from .imwrite import imwrite
 
 # Version of the dsa-helpers package
-__version__ = "1.1.4.dev1"
+__version__ = "1.1.5.dev0"
 
 # To avoid slow downs, do not allow from dsa_helpers import * to import anything.
 __all__ = []

{dsa_helpers-1.1.4.dev1 → dsa_helpers-1.1.5.dev0}/dsa_helpers/girder_utils.py

@@ -10,11 +10,11 @@ Functions:
 - get_roi_images: Get regions of interest (ROIs) as images from DSA annotations
 - post_annotation: Post a new annotation to the DSA
 - remove_overlapping_annotations: Remove overlapping regions from elements
-
+- upload_dir_to_dsa: Upload a local directory to a DSA item
 """
 
 from girder_client import GirderClient, HttpError
-import pickle
+import pickle, shutil, tempfile
 import numpy as np
 import pandas as pd
 from pathlib import Path
@@ -878,3 +878,29 @@ def post_annotation(gc: GirderClient, item_id: str, annotation: dict) -> dict:
         parameters={"itemId": item_id},
         json=annotation,
     )
+
+
+def upload_dir_to_dsa(
+    gc: GirderClient,
+    checkpoint_dir: str,
+    item_id: str,
+):
+    """Upload a local directory to a DSA item.
+
+    Args:
+        gc (girder_client.GirderClient): The girder client.
+        checkpoint_dir (str): The path to the local directory to upload.
+        item_id (str): The ID of the DSA item to upload the directory to.
+
+    """
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_dir = Path(temp_dir)
+
+        # Zip the directory.
+        shutil.make_archive(temp_dir / "checkpoint", "zip", checkpoint_dir)
+
+        # Upload the zip file to the DSA item.
+        _ = gc.uploadFileToItem(
+            item_id,
+            str(temp_dir / "checkpoint.zip"),
+        )

dsa_helpers-1.1.5.dev0/dsa_helpers/ml/evaluate.py

@@ -0,0 +1,425 @@
+"""Functions for evaluating datasets with machine learning models.
+
+- evaluate_semantic_segmentation_segformer_group: Evaluate a segformer
+  trainer model on a specific dataset provided as a dataframe, grouped
+  by a specific column.
+- per_class_dice_on_dataset: Per class DICE coefficient on a set of
+    images.
+
+"""
+
+import torch
+from torch import nn
+from tqdm import tqdm
+from transformers.trainer import Trainer
+import pandas as pd
+import numpy as np
+from .datasets.utils import create_segformer_segmentation_dataset
+from .transforms.segformer_transforms import val_transforms
+from ..tile_utils import merge_tile_masks
+
+
+def evaluate_semantic_segmentation_segformer_group(
+    trainer: Trainer,
+    df: pd.DataFrame,
+    group_col: str,
+    batch_size: int = 16,
+    tile_size: int = 512,
+    background_label: int | None = None,
+) -> float:
+    """Evaluate a segformer trainer model on a specific dataset provided
+    as a dataframe, grouped by a specific column. For a unique value of
+    the column it will predict on the images for that group and then
+    merge the predictions into a single multipolygon. The mean IoU will
+    be calculated agains the ground truth masks for that group. The
+    reported mean IoU will be the average of all the mean IoUs for each
+    group.
+
+    Args:
+        trainer (Trainer): The trainer object with the model to evaluate.
+        df (pandas.DataFrame): The dataframe with the tile metadata.
+            Must have the fp column (file path), x column (x coordinate),
+            y column (y coordinate), and group_col.
+        group_col (str): The column to group the tiles by.
+        batch_size (int): The batch size to use for prediction. Note that
+            this is not the batch size use for predictions, rather it is
+            how many masks will be stored in memory. Batch size used
+            for predictions is specified on the trainer input. Default is
+            16.
+        tile_size (int): The size of the tiles. Default is 512.
+        background_label (int | None): The label to use for the background
+            class, which will be ignored. If None then no label will be
+            ignored. Default is None.
+
+    Returns:
+        float: The mean IoU across all groups.
+
+    """
+    ious = []
+
+    # Predict on the images for each group separately.
+    groups = list(df[group_col].unique())
+    for group in tqdm(groups, desc="Evaluating groups"):
+        # Get the tiles in this group.
+        group_df = df[df[group_col] == group]
+
+        pred_tile_list = []
+        true_tile_list = []
+
+        # Predict on the group in batches.
+        for i in range(0, len(group_df), batch_size):
+            # Create dataset for this batch.
+            batch_df = group_df.iloc[i : i + batch_size]
+            x_list = batch_df["x"].tolist()
+            y_list = batch_df["y"].tolist()
+
+            # Create dataset for this batch.
+            group_dataset = create_segformer_segmentation_dataset(
+                batch_df, transforms=val_transforms
+            )
+
+            # Predict on the batch.
+            out = trainer.predict(group_dataset)
+            preds = out[0]
+            true = out[1]
+
+            # Reshape the prediction logits to tile size.
+            preds = torch.from_numpy(preds).cpu()
+
+            preds = nn.functional.interpolate(
+                preds,
+                size=(tile_size, tile_size),
+                mode="bilinear",
+                align_corners=False,
+            ).argmax(
+                dim=1
+            )  # logits -> class predictions
+
+            preds = preds.detach().numpy()
+
+            # Append to the tile list.
+            pred_tile_list.extend(
+                [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
+            )
+
+            true_tile_list.extend(
+                [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
+            )
+
+        # Merge the tile masks.
+        pred_gdf = merge_tile_masks(
+            pred_tile_list, background_label=background_label
+        )
+        true_gdf = merge_tile_masks(
+            true_tile_list, background_label=background_label
+        )
+
+        # Get the unique labels in either the prediction or the ground truth.
+        labels = set(pred_gdf["label"].unique()) | set(
+            true_gdf["label"].unique()
+        )
+
+        # Calculate the IoU for each label.
+        ious = []
+
+        for label in labels:
+            pred_label_gdf = pred_gdf[pred_gdf["label"] == label]
+            true_label_gdf = true_gdf[true_gdf["label"] == label]
+
+            # If either the prediction or the ground truth is empty, the IoU is 0.
+            if pred_label_gdf.empty or true_label_gdf.empty:
+                ious.append(0)
+            else:
+                # Calculate the IoU.
+                geom1 = pred_label_gdf.iloc[0]["geometry"]
+                geom2 = true_label_gdf.iloc[0]["geometry"]
+
+                intersection_area = geom1.intersection(geom2).area
+                union = geom1.union(geom2).area
+
+                ious.append(intersection_area / union)
+
+    # Return the average of the ious.
+    return float(sum(ious) / len(ious))
+
+
+def per_class_dice_on_dataset(
+    model: nn.Module,
+    data: pd.DataFrame | str,
+    label2id: dict[str, int],
+    batch_size: int = 16,
+    device: torch.device | None = None,
+    tile_size: int = 512,
+    tqdm_notebook: bool = False,
+):
+    """Calculate the DICE coefficient for a given model and set of
+    images. The model must be a semantic segmentation model that outputs
+    an object with the logits attribute. The dataset must be iterable
+    and return a dictionary with the keys "pixel_values" and "labels".
+    The pixel values should be in the format the model expects for the
+    input.
+
+    Args:
+        model (nn.Module): The model to evaluate.
+        data (pd.DataFrame | str): The dataframe or path to a csv file
+            containing the data to evaluate on. The dataframe must have
+            the "fp" (filepath to image) and "mask_fp" (filepath to
+            mask) columns.
+        label2id (dict): A dictionary mapping labels to their integer
+            representations.
+        batch_size (int, optional): The batch size to use for evaluation.
+            Default is 16.
+        device (torch.device | None, optional): The device to use for
+            evaluation. If None, the device will be "cuda" if available,
+            otherwise "cpu". Default is None.
+        tile_size (int, optional): The size of the tiles to use for
+            evaluation. Default is 512.
+        tqdm_notebook (bool, optional): Whether to use a tqdm notebook.
+            Default is False.
+
+    Returns:
+        dict: A dictionary with the keys "mean_dice" and the DICE
+            coefficient for each label.
+    """
+    if tqdm_notebook:
+        from tqdm.notebook import tqdm
+    else:
+        from tqdm import tqdm
+
+    if isinstance(data, str):
+        data = pd.read_csv(data)
+
+    # Create the dataset object.
+    dataset = create_segformer_segmentation_dataset(
+        data, transforms=val_transforms
+    )
+
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Move the model to the device.
+    model.to(device)
+    model.eval()  # model should be in evaluation mode
+
+    # Number of images in the dataset.
+    n = len(dataset)
+
+    # Initialize the intersection and denominator for each label.
+    intersection = {label: 0 for label in label2id}
+    denominator = {label: 0 for label in label2id}
+
+    batches = list(range(0, n, batch_size))
+
+    for i in tqdm(batches):
+        # Get the batch of images and labels.
+        batch = dataset[i : i + batch_size]
+
+        inputs = np.array(batch["pixel_values"])
+        labels = np.array(batch["labels"])
+        inputs = torch.tensor(inputs, dtype=torch.float32)
+        labels = np.array(labels)
+
+        with torch.no_grad():
+            inputs = inputs.to(device)
+            outputs = model(inputs)
+            logits = outputs.logits
+
+            # Get the logits out, resizing them to the original tile size.
+            logits = torch.nn.functional.interpolate(
+                logits,
+                size=tile_size,
+                mode="bilinear",
+            )
+
+            # Get predicted class labels for each pixel.
+            masks = torch.argmax(logits, dim=1).detach().cpu().numpy()
+
+            # Flatten both masks and labels.
+            masks = masks.flatten()
+            labels = labels.flatten()
+
+            for label, integer in label2id.items():
+                # Calculate where both mask and labels are equal to the integer.
+                gt_mask = labels == integer
+                pred_mask = masks == integer
+                intersection[label] += np.sum(gt_mask & pred_mask)
+                denominator[label] += np.sum(gt_mask) + np.sum(pred_mask)
+
+    # Calculate the dice for each class.
+    metrics = {}
+    for label in label2id:
+        denominator_value = denominator[label]
+
+        if denominator_value:
+            metrics[f"{label}_dice"] = float(
+                2 * intersection[label] / denominator[label]
+            )
+        else:
+            metrics[f"{label}_dice"] = 1
+
+    # Calculate the mean dice.
+    return metrics
+
+
+def evaluate_semantic_segmentation_segformer_by_wsi(
+    model: nn.Module,
+    data: pd.DataFrame | str,
+    group_col: str,
+    label2id: dict[str, int],
+    batch_size: int = 16,
+    tile_size: int = 512,
+    background_label: int | None = None,
+    device: torch.device | None = None,
+) -> float:
+    """Evaluate a segformer trainer model on a specific dataset provided
+    as a dataframe, grouped by a specific column. For a unique value of
+    the column it will predict on the images for that group and then
+    merge the predictions into a single multipolygon. The mean IoU will
+    be calculated agains the ground truth masks for that group. The
+    reported mean IoU will be the average of all the mean IoUs for each
+    group.
+
+    Args:
+        model (nn.Module): The model to evaluate.
+        data (pd.DataFrame | str): The dataframe or path to a csv file
+            containing the data to evaluate on. The dataframe must have
+            the "fp" (filepath to image) and "mask_fp" (filepath to
+            mask) columns. The column matching the "group_col" parameter
+            will be used to group tiles into groups. The "x" and "y"
+            parameters will be used for stiching the images back to
+            their original size.
+        group_col (str): The column to group the tiles by.
+        batch_size (int): The batch size to use for prediction. Note that
+            this is not the batch size use for predictions, rather it is
+            how many masks will be stored in memory. Batch size used
+            for predictions is specified on the trainer input. Default is
+            16.
+        label2id (dict): A dictionary mapping labels to their integer
+            representations.
+        tile_size (int): The size of the tiles. Default is 512.
+        background_label (int | None): The label to use for the background
+            class, which will be ignored. If None then no label will be
+            ignored. Default is None.
+        device (torch.device | None, optional): The device to use for
+            evaluation. If None, the device will be "cuda" if available,
+            otherwise "cpu". Default is None.
+
+    Returns:
+        float: The mean IoU across all groups.
+
+    """
+    # Read the data.
+    if isinstance(data, str):
+        data = pd.read_csv(data)
+
+    ious = []
+
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Move the model to the device.
+    model.to(device)
+    model.eval()  # model should be in evaluation mode
+
+    # Predict on the images for each group separately.
+    groups = list(data[group_col].unique())
+    for group in tqdm(groups, desc="Evaluating groups"):
+        # Get the tiles in this group.
+        group_df = data[data[group_col] == group]
+
+        pred_tile_list = []
+        true_tile_list = []
+
+        # Predict on the group in batches.
+        for i in range(0, len(group_df), batch_size):
+            # Create dataset for this batch.
+            batch_df = group_df.iloc[i : i + batch_size]
+            x_list = batch_df["x"].tolist()
+            y_list = batch_df["y"].tolist()
+
+            # Create dataset for this batch.
+            dataset = create_segformer_segmentation_dataset(
+                batch_df, transforms=val_transforms
+            )
+
+            # Get the inputs.
+            inputs = np.array(dataset[:]["pixel_values"])
+            inputs = torch.tensor(inputs, dtype=torch.float32)
+            inputs = inputs.to(device)
+
+            true = np.array(dataset[:]["labels"])
+
+            # Predict on the batch.
+            with torch.no_grad():
+                out = model(inputs)
+                preds = out.logits
+
+                # Reshape the prediction logits to tile size.
+                preds = torch.from_numpy(preds)
+
+                preds = nn.functional.interpolate(
+                    preds,
+                    size=(tile_size, tile_size),
+                    mode="bilinear",
+                    align_corners=False,
+                ).argmax(
+                    dim=1
+                )  # logits -> class predictions
+
+                preds = preds.detach().cpu().numpy()
+
+                # Append to the tile list.
+                pred_tile_list.extend(
+                    [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
+                )
+
+                true_tile_list.extend(
+                    [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
+                )
+
+        # Merge the tile masks.
+        pred_gdf = merge_tile_masks(
+            pred_tile_list, background_label=background_label
+        )
+        true_gdf = merge_tile_masks(
+            true_tile_list, background_label=background_label
+        )
+
+        # Calculate the Dice for each label.
+        metrics = {label: [] for label in label2id}
+
+        for label, integer in label2id.items():
+            if integer == background_label:
+                continue  # don't calculate dice for background
+
+            # For this integer, get the prediction and ground truth.
+            pred_label_gdf = pred_gdf[pred_gdf["label"] == integer]
+            true_label_gdf = true_gdf[true_gdf["label"] == integer]
+
+            # If both are empty, the dice is 1.
+            if pred_label_gdf.empty and true_label_gdf.empty:
+                metrics[label].append(1)
+            else:
+                # Calculate the intersection and denominator.
+                intersection = pred_label_gdf.geometry.intersection(
+                    true_label_gdf.geometry
+                ).area.sum()
+
+                # The denominator is the sum area of the two polygons.
+                denominator = (
+                    pred_label_gdf.geometry.area.sum()
+                    + true_label_gdf.geometry.area.sum()
+                )
+
+                # The dice is 2 * intersection / denominator.
+                metrics[label].append(float(2 * intersection / denominator))
+
+    # Return the dice as means and standard deviations.
+    metrics = {
+        label: {
+            "mean": float(np.mean(metrics[label])),
+            "std": float(np.std(metrics[label])),
+        }
+        for label in label2id
+    }
+    return metrics

dsa_helpers-1.1.5.dev0/dsa_helpers/ml/segformer_semantic_segmentation/train.py

@@ -0,0 +1,285 @@
+import pandas as pd
+from transformers import (
+    SegformerForSemanticSegmentation,
+    TrainingArguments,
+    Trainer,
+)
+import numpy as np
+from pathlib import Path
+
+import torch
+from torch import nn
+
+from ..datasets.utils import create_segformer_segmentation_dataset
+from ..transforms.segformer_transforms import (
+    train_transforms,
+    val_transforms,
+)
+from ..callbacks import MetricsLoggerCallback
+from ..evaluate import per_class_dice_on_dataset
+
+
+def train_segformer_semantic_segmentation_model(
+    save_dir: str,
+    train_data: pd.DataFrame | str,
+    val_data: pd.DataFrame | str,
+    label2id: dict[str, int],
+    device: torch.device | None = None,
+    test_data: pd.DataFrame | str | None = None,
+    model_checkpoint: str | None = None,
+    learning_rate: float = 1e-4,
+    epochs: int = 10,
+    batch_size: int = 16,
+    eval_accumulation_steps: int = 100,
+    max_val_size: int | None = 500,
+    random_state: int = 42,
+    tile_size: int = 512,
+    tqdm_notebook: bool = False,
+):
+    """Workflow for training a SegFormer semantic segmentation model.
+
+    Args:
+        save_dir (str): Directory where model checkpoint directory will
+            be saved to.
+        train_data (pd.DataFrame | str): Train data, should be a pandas
+            dataframe or a path to a csv file containing the train data.
+            Must have the "fp" and "mask_fp" columns, which are the
+            filepaths to the images and masks, respectively.
+        val_data (pd.DataFrame | str): See train_data, but for the
+            validation data.
+        label2id (dict): A dictionary mapping the label names to their
+            corresponding integer ids.
+        device (torch.device | None, optional): Device to run the model
+            on. If None, will use "cuda" if available, otherwise "cpu".
+            Defaults to None.
+        test_data (pd.DataFrame | str | None, optional): See train_data,
+            but for the test data. Defaults to None.
+        model_checkpoint (str | None, optional): A local path to a model
+            checkpoint to load the model from. If None it will use the
+            default weights from huggingface: "nvidia/mit-b0". Defaults
+            to None.
+        learning_rate (float, optional): Learning rate for the model.
+            Defaults to 1e-4.
+        epochs (int, optional): Number of epochs to train the model.
+            Defaults to 10.
+        batch_size (int, optional): Batch size for the model.
+            Defaults to 16.
+        eval_accumulation_steps (int, optional): Number of steps to
+            accumulate the validation metrics over. Defaults to 100.
+        max_val_size (int | None, optional): Maximum number of
+            validation samples to use. If None, all validation samples
+            will be used. Defaults to None. Note that there is a bug
+            that this value must be kept short otherwise a cuda memory
+            error will occur.
+        random_state (int, optional): Random state for the validation
+            split. Defaults to 42.
+        tile_size (int, optional): Size of the tiles to use for the model.
+            Defaults to 512.
+        tqdm_notebook (bool, optional): Whether to use a tqdm notebook.
+            Defaults to False.
+
+    Returns:
+        tuple: A tuple containing the trainer and the results dictionary.
+
+    """
+    # Read the dataframes if they are strings.
+    if isinstance(train_data, str):
+        train_data = pd.read_csv(train_data)
+    if isinstance(val_data, str):
+        val_data = pd.read_csv(val_data)
+    if test_data is not None and isinstance(test_data, str):
+        test_data = pd.read_csv(test_data)
+
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Load the default pretrained model.
+    label2id = {k: int(v) for k, v in label2id.items()}
+    id2label = {int(v): k for k, v in label2id.items()}
+
+    if model_checkpoint is None:
+        model = SegformerForSemanticSegmentation.from_pretrained(
+            "nvidia/mit-b0", id2label=id2label, label2id=label2id
+        ).to(device)
+    else:
+        model = SegformerForSemanticSegmentation.from_pretrained(
+            model_checkpoint,
+            id2label=id2label,
+            label2id=label2id,
+            local_files_only=True,
+            ignore_mismatched_sizes=True,
+        ).to(device)
+
+    # Training arguments.
+    training_args = TrainingArguments(
+        str(save_dir),
+        learning_rate=learning_rate,
+        num_train_epochs=epochs,
+        per_device_train_batch_size=batch_size,
+        per_device_eval_batch_size=batch_size,
+        eval_accumulation_steps=eval_accumulation_steps,
+        eval_strategy="epoch",
+        save_strategy="epoch",
+        logging_strategy="epoch",
+        load_best_model_at_end=True,
+        save_total_limit=1,
+    )
+
+    # Create datasets for the model.
+    train_dataset = create_segformer_segmentation_dataset(
+        train_data, transforms=train_transforms
+    )
+
+    val_dataset = create_segformer_segmentation_dataset(
+        (
+            val_data
+            if max_val_size is None
+            else val_data.sample(n=max_val_size, random_state=random_state)
+        ),
+        transforms=val_transforms,
+    )
+
+    def compute_metrics(eval_pred):
+        """Function for computing DICE coefficient metrics for each
+        class, as well as the mean DICE and weighted mean DICE (maybe).
+        """
+        logits, labels = eval_pred
+
+        # Convert logits to tensor.
+        logits = torch.from_numpy(logits)
+
+        # Scale the logits back to the shape of the labels.
+        logits = nn.functional.interpolate(
+            logits,
+            size=tile_size,
+            mode="bilinear",
+            align_corners=False,
+        )
+
+        # Turn the logits to predictions.
+        pred = logits.argmax(dim=1)
+
+        # Calculate the Dice for each class.
+        intersection = {label: 0 for label in label2id}
+        denominator = {label: 0 for label in label2id}
+
+        # For easy calculations, convert pred to a numpy.
+        pred = pred.numpy()
+
+        # Flatten both arrays.
+        pred = pred.flatten()
+        labels = labels.flatten()
+
+        # Loop through each index.
+        for label, integer in label2id.items():
+            pred_mask = pred == integer
+            gt_mask = labels == integer
+
+            # Calculate the interesection.
+            intersection[label] += np.sum(gt_mask & pred_mask)
+            denominator[label] += np.sum(gt_mask) + np.sum(pred_mask)
+
+        # Calculate the Dice for each label.
+        metrics = {}
+        for label in label2id:
+            denominator_value = denominator[label]
+
+            if denominator_value:
+                metrics[label] = float(
+                    2 * intersection[label] / denominator_value
+                )
+            else:
+                metrics[label] = 1
+
+        # Add the mean Dice.
+        metrics["mean_dice"] = float(np.mean(list(metrics.values())))
+        return metrics
+
+    # Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset,
+        compute_metrics=compute_metrics,
+        callbacks=[MetricsLoggerCallback],  # Add the custom callback here
+    )
+
+    # Train the model.
+    _ = trainer.train()
+
+    # Rename the checkpoint to not include a step number.
+    save_dir = Path(save_dir)
+    for p in save_dir.iterdir():
+        if p.is_dir() and p.name.startswith("checkpoint-"):
+            p.rename(save_dir / "checkpoint")
+
+    results = {"train": {}, "val": {}}
+
+    for log in trainer.state.log_history[:-1]:
+        epoch = log["epoch"]
+
+        if epoch % 1:
+            continue  # only track end of each epoch
+
+        # Check if this is tracking train or validation.
+        if "eval_loss" in log:
+            data = results["val"]
+        else:
+            data = results["train"]
+
+        for k, v in log.items():
+            if k.startswith("eval_"):
+                k = k[5:]
+
+            if k not in data:
+                data[k] = []
+
+            data[k].append(v)
+
+    # For each of the datasets provided, get the metrics.
+    model = trainer.model
+
+    dataset = create_segformer_segmentation_dataset(
+        train_data, transforms=val_transforms
+    )
+    results["train"]["metrics"] = per_class_dice_on_dataset(
+        model,
+        dataset,
+        label2id,
+        batch_size=batch_size,
+        device=device,
+        tile_size=tile_size,
+        tqdm_notebook=tqdm_notebook,
+    )
+
+    dataset = create_segformer_segmentation_dataset(
+        val_data, transforms=val_transforms
+    )
+    results["val"]["metrics"] = per_class_dice_on_dataset(
+        model,
+        dataset,
+        label2id,
+        batch_size=batch_size,
+        device=device,
+        tile_size=tile_size,
+        tqdm_notebook=tqdm_notebook,
+    )
+
+    if test_data is not None:
+        results["test"] = {}
+
+        dataset = create_segformer_segmentation_dataset(
+            test_data, transforms=val_transforms
+        )
+        results["test"]["metrics"] = per_class_dice_on_dataset(
+            model,
+            dataset,
+            label2id,
+            batch_size=batch_size,
+            device=device,
+            tile_size=tile_size,
+            tqdm_notebook=tqdm_notebook,
+        )
+
+    return trainer, results

{dsa_helpers-1.1.4.dev1 → dsa_helpers-1.1.5.dev0/dsa_helpers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsa_helpers
-Version: 1.1.4.dev1
+Version: 1.1.5.dev0
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>

{dsa_helpers-1.1.4.dev1 → dsa_helpers-1.1.5.dev0}/dsa_helpers.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ dsa_helpers/ml/object_detection/tile_image.py
 dsa_helpers/ml/object_detection/utils.py
 dsa_helpers/ml/segformer_semantic_segmentation/__init__.py
 dsa_helpers/ml/segformer_semantic_segmentation/inference.py
+dsa_helpers/ml/segformer_semantic_segmentation/train.py
 dsa_helpers/ml/transforms/__init__.py
 dsa_helpers/ml/transforms/segformer_transforms.py
 tests/docker-compose.yaml

{dsa_helpers-1.1.4.dev1 → dsa_helpers-1.1.5.dev0}/pyproject.toml

@@ -37,7 +37,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 keywords = ["dsa", "girder", "girder_client"]
-version = "v1.1.4.dev1"
+version = "v1.1.5.dev0"
 
 [project.urls]
 Repository = "https://github.com/Gutman-Lab/dsa-python-utils"

dsa_helpers-1.1.4.dev1/dsa_helpers/ml/evaluate.py

@@ -1,136 +0,0 @@
-"""This module contains functions for evaluating model performance on 
-datasets."""
-
-import torch
-from torch import nn
-from tqdm import tqdm
-from transformers.trainer import Trainer
-import pandas as pd
-
-from .datasets.utils import create_segformer_segmentation_dataset
-from .transforms.segformer_transforms import val_transforms
-from ..tile_utils import merge_tile_masks
-
-
-def evaluate_semantic_segmentation_segformer_group(
-    trainer: Trainer,
-    df: pd.DataFrame,
-    group_col: str,
-    batch_size: int = 16,
-    tile_size: int = 512,
-    background_label: int | None = None,
-) -> float:
-    """Evaluate a segformer trainer model on a specific dataset provided
-    as a dataframe, grouped by a specific column. For a unique value of
-    the column it will predict on the images for that group and then
-    merge the predictions into a single multipolygon. The mean IoU will
-    be calculated agains the ground truth masks for that group. The
-    reported mean IoU will be the average of all the mean IoUs for each
-    group.
-
-    Args:
-        trainer (Trainer): The trainer object with the model to evaluate.
-        df (pandas.DataFrame): The dataframe with the tile metadata.
-            Must have the fp column (file path), x column (x coordinate),
-            y column (y coordinate), and group_col.
-        group_col (str): The column to group the tiles by.
-        batch_size (int): The batch size to use for prediction. Note that
-            this is not the batch size use for predictions, rather it is
-            how many masks will be stored in memory. Batch size used
-            for predictions is specified on the trainer input. Default is
-            16.
-        tile_size (int): The size of the tiles. Default is 512.
-        background_label (int | None): The label to use for the background
-            class, which will be ignored. If None then no label will be
-            ignored. Default is None.
-
-    Returns:
-        float: The mean IoU across all groups.
-
-    """
-    ious = []
-
-    # Predict on the images for each group separately.
-    groups = list(df[group_col].unique())
-    for group in tqdm(groups, desc="Evaluating groups"):
-        # Get the tiles in this group.
-        group_df = df[df[group_col] == group]
-
-        pred_tile_list = []
-        true_tile_list = []
-
-        # Predict on the group in batches.
-        for i in range(0, len(group_df), batch_size):
-            # Create dataset for this batch.
-            batch_df = group_df.iloc[i : i + batch_size]
-            x_list = batch_df["x"].tolist()
-            y_list = batch_df["y"].tolist()
-
-            # Create dataset for this batch.
-            group_dataset = create_segformer_segmentation_dataset(
-                batch_df, transforms=val_transforms
-            )
-
-            # Predict on the batch.
-            out = trainer.predict(group_dataset)
-            preds = out[0]
-            true = out[1]
-
-            # Reshape the prediction logits to tile size.
-            preds = torch.from_numpy(preds).cpu()
-
-            preds = nn.functional.interpolate(
-                preds,
-                size=(tile_size, tile_size),
-                mode="bilinear",
-                align_corners=False,
-            ).argmax(
-                dim=1
-            )  # logits -> class predictions
-
-            preds = preds.detach().numpy()
-
-            # Append to the tile list.
-            pred_tile_list.extend(
-                [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
-            )
-
-            true_tile_list.extend(
-                [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
-            )
-
-        # Merge the tile masks.
-        pred_gdf = merge_tile_masks(
-            pred_tile_list, background_label=background_label
-        )
-        true_gdf = merge_tile_masks(
-            true_tile_list, background_label=background_label
-        )
-
-        # Get the unique labels in either the prediction or the ground truth.
-        labels = set(pred_gdf["label"].unique()) | set(
-            true_gdf["label"].unique()
-        )
-
-        # Calculate the IoU for each label.
-        ious = []
-
-        for label in labels:
-            pred_label_gdf = pred_gdf[pred_gdf["label"] == label]
-            true_label_gdf = true_gdf[true_gdf["label"] == label]
-
-            # If either the prediction or the ground truth is empty, the IoU is 0.
-            if pred_label_gdf.empty or true_label_gdf.empty:
-                ious.append(0)
-            else:
-                # Calculate the IoU.
-                geom1 = pred_label_gdf.iloc[0]["geometry"]
-                geom2 = true_label_gdf.iloc[0]["geometry"]
-
-                intersection_area = geom1.intersection(geom2).area
-                union = geom1.union(geom2).area
-
-                ious.append(intersection_area / union)
-
-    # Return the average of the ious.
-    return float(sum(ious) / len(ious))