dsa-helpers 1.dev0__tar.gz → 1.0.1.dev0__tar.gz

{dsa_helpers-1.dev0/dsa_helpers.egg-info → dsa_helpers-1.0.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 1.dev0
+Version: 1.0.1.dev0
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0}/dsa_helpers/ml/__init__.py

@@ -5,6 +5,7 @@ from . import (
     transforms,
     segformer_semantic_segmentation,
     callbacks,
+    evaluate,
 )
 
 # Modules that are imported from * notation.
@@ -15,4 +16,5 @@ __all__ = [
     "transforms",
     "segformer_semantic_segmentation",
     "callbacks",
+    "evaluate",
 ]

dsa_helpers-1.0.1.dev0/dsa_helpers/ml/evaluate.py

@@ -0,0 +1,136 @@
+"""This module contains functions for evaluating model performance on 
+datasets."""
+
+import torch
+from torch import nn
+from tqdm import tqdm
+from transformers.trainer import Trainer
+import pandas as pd
+
+from .datasets.utils import create_segformer_segmentation_dataset
+from .transforms.segformer_transforms import val_transforms
+from ..tile_utils import merge_tile_masks
+
+
+def evaluate_semantic_segmentation_segformer_group(
+    trainer: Trainer,
+    df: pd.DataFrame,
+    group_col: str,
+    batch_size: int = 16,
+    tile_size: int = 512,
+    background_label: int | None = None,
+) -> float:
+    """Evaluate a segformer trainer model on a specific dataset provided
+    as a dataframe, grouped by a specific column. For a unique value of
+    the column it will predict on the images for that group and then
+    merge the predictions into a single multipolygon. The mean IoU will
+    be calculated agains the ground truth masks for that group. The
+    reported mean IoU will be the average of all the mean IoUs for each
+    group.
+
+    Args:
+        trainer (Trainer): The trainer object with the model to evaluate.
+        df (pandas.DataFrame): The dataframe with the tile metadata.
+            Must have the fp column (file path), x column (x coordinate),
+            y column (y coordinate), and group_col.
+        group_col (str): The column to group the tiles by.
+        batch_size (int): The batch size to use for prediction. Note that
+            this is not the batch size use for predictions, rather it is
+            how many masks will be stored in memory. Batch size used
+            for predictions is specified on the trainer input. Default is
+            16.
+        tile_size (int): The size of the tiles. Default is 512.
+        background_label (int | None): The label to use for the background
+            class, which will be ignored. If None then no label will be
+            ignored. Default is None.
+
+    Returns:
+        float: The mean IoU across all groups.
+
+    """
+    ious = []
+
+    # Predict on the images for each group separately.
+    groups = list(df[group_col].unique())
+    for group in tqdm(groups, desc="Evaluating groups"):
+        # Get the tiles in this group.
+        group_df = df[df[group_col] == group]
+
+        pred_tile_list = []
+        true_tile_list = []
+
+        # Predict on the group in batches.
+        for i in range(0, len(group_df), batch_size):
+            # Create dataset for this batch.
+            batch_df = group_df.iloc[i : i + batch_size]
+            x_list = batch_df["x"].tolist()
+            y_list = batch_df["y"].tolist()
+
+            # Create dataset for this batch.
+            group_dataset = create_segformer_segmentation_dataset(
+                batch_df, transforms=val_transforms
+            )
+
+            # Predict on the batch.
+            out = trainer.predict(group_dataset)
+            preds = out[0]
+            true = out[1]
+
+            # Reshape the prediction logits to tile size.
+            preds = torch.from_numpy(preds).cpu()
+
+            preds = nn.functional.interpolate(
+                preds,
+                size=(tile_size, tile_size),
+                mode="bilinear",
+                align_corners=False,
+            ).argmax(
+                dim=1
+            )  # logits -> class predictions
+
+            preds = preds.detach().numpy()
+
+            # Append to the tile list.
+            pred_tile_list.extend(
+                [(pred, x, y) for x, y, pred in zip(x_list, y_list, preds)]
+            )
+
+            true_tile_list.extend(
+                [(mask, x, y) for x, y, mask in zip(x_list, y_list, true)]
+            )
+
+        # Merge the tile masks.
+        pred_gdf = merge_tile_masks(
+            pred_tile_list, background_label=background_label
+        )
+        true_gdf = merge_tile_masks(
+            true_tile_list, background_label=background_label
+        )
+
+        # Get the unique labels in either the prediction or the ground truth.
+        labels = set(pred_gdf["label"].unique()) | set(
+            true_gdf["label"].unique()
+        )
+
+        # Calculate the IoU for each label.
+        ious = []
+
+        for label in labels:
+            pred_label_gdf = pred_gdf[pred_gdf["label"] == label]
+            true_label_gdf = true_gdf[true_gdf["label"] == label]
+
+            # If either the prediction or the ground truth is empty, the IoU is 0.
+            if pred_label_gdf.empty or true_label_gdf.empty:
+                ious.append(0)
+            else:
+                # Calculate the IoU.
+                geom1 = pred_label_gdf.iloc[0]["geometry"]
+                geom2 = true_label_gdf.iloc[0]["geometry"]
+
+                intersection_area = geom1.intersection(geom2).area
+                union = geom1.union(geom2).area
+
+                ious.append(intersection_area / union)
+
+    # Return the average of the ious.
+    return float(sum(ious) / len(ious))

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0}/dsa_helpers/tile_utils.py

@@ -8,7 +8,9 @@ from geopandas import GeoDataFrame
 from .utils import mask_to_shapely
 
 
-def merge_tile_masks(tile_list: list, buffer: int = 1) -> GeoDataFrame:
+def merge_tile_masks(
+    tile_list: list, buffer: int = 1, background_label: int | None = 0
+) -> GeoDataFrame:
     """
     Merge the tile masks into a single mask for a large image.
 
@@ -17,6 +19,9 @@ def merge_tile_masks(tile_list: list, buffer: int = 1) -> GeoDataFrame:
             (1) fp or array for mask, (2) x-coordinate for tile, (3)
             y-coordinate for tile.
         buffer (int): Buffer to apply to the polygons before dissolving.
+        background_label (int | None): Label value of the background class,
+            which is ignored. Default is 0. If None then all labels are
+            considered.
 
     Returns:
         GeoDataFrame: GeoDataFrame with the merged mask.
@@ -30,7 +35,9 @@ def merge_tile_masks(tile_list: list, buffer: int = 1) -> GeoDataFrame:
 
         # Process the mask by converting it to polygons.
         polygons_and_labels.extend(
-            mask_to_shapely(tile, x_offset=x, y_offset=y)
+            mask_to_shapely(
+                tile, x_offset=x, y_offset=y, background_label=background_label
+            )
         )
 
     # Convert polygons and labels into a GeoDataFrame.
@@ -45,4 +52,9 @@ def merge_tile_masks(tile_list: list, buffer: int = 1) -> GeoDataFrame:
     # Remove the buffer.
     gdf["geometry"] = gdf["geometry"].buffer(-buffer)
 
+    # Check that all polygons are valid.
+    for i, row in gdf.iterrows():
+        if not row["geometry"].is_valid:
+            gdf.loc[i, "geometry"] = row["geometry"].buffer(0)
+
     return gdf

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0}/dsa_helpers/utils.py

@@ -10,7 +10,7 @@ def mask_to_shapely(
     mask: str | np.ndarray,
     x_offset: int = 0,
     y_offset: int = 0,
-    background_label: int = 0,
+    background_label: int | None = 0,
     min_area: int = 0,
 ) -> list[Polygon, int]:
     """
@@ -20,9 +20,12 @@ def mask_to_shapely(
     Args:
         mask (str | np.ndarray): Path to the mask image or the mask.
         x_offset (int): Offset to add to x coordinates of polygons.
+            Default is 0.
         y_offset (int): Offset to add to y coordinates of polygons.
-        background_label (int): Label value of the background class,
-            which is ignored.
+            Default is 0.
+        background_label (int | None): Label value of the background class,
+            which is ignored. Default is 0. If None then all labels are
+            considered.
 
     Returns:
         list[Polygon, int]: List of polygons and their corresponding

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0/dsa_helpers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 1.dev0
+Version: 1.0.1.dev0
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0}/dsa_helpers.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ dsa_helpers/dash/header.py
 dsa_helpers/dash/header_callbacks.py
 dsa_helpers/ml/__init__.py
 dsa_helpers/ml/callbacks.py
+dsa_helpers/ml/evaluate.py
 dsa_helpers/ml/metrics.py
 dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py
 dsa_helpers/ml/datasets/__init__.py

{dsa_helpers-1.dev0 → dsa_helpers-1.0.1.dev0}/pyproject.toml

@@ -35,7 +35,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 keywords = ["dsa", "girder", "girder_client"]
-version = "v1dev0"
+version = "v1.0.1dev0"
 
 [project.urls]
 Repository = "https://github.com/Gutman-Lab/dsa-python-utils"