dsa-helpers 0.4.dev2__py3-none-any.whl → 0.4.dev4__py3-none-any.whl

dsa_helpers/__init__.py

@@ -5,4 +5,4 @@ from .imwrite import imwrite
 # Modules that should be available.
 # from . import girder_utils, image_utils, dash
 
-__all__ = ["girder_utils", "image_utils", "dash"]
+__all__ = ["girder_utils", "image_utils", "dash", "ml"]

dsa_helpers/ml/__init__.py

@@ -1,2 +1,2 @@
 # Modules that can be imported.
-__all__ = ["object_detection"]
+__all__ = ["object_detection", "datasets", "metrics", "transforms"]

dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py

@@ -0,0 +1,33 @@
+from torch.utils.data import Dataset
+import pandas as pd
+from PIL import Image
+import numpy as np
+
+
+class SegFormerSegmentationDataset(Dataset):
+    """PyTorch dataset class for semantic segmentation using HuggingFaces SegFormer
+    model.
+
+    """
+
+    def __init__(self, df: pd.DataFrame):
+        """Initiate an instance of the segmentation dataset.
+
+        Args:
+            df (pandas.DataFrame): A pandas DataFrame with columns "fp" and "mask_fp".
+
+        """
+        self.image_files = df["fp"].tolist()
+        self.mask_files = df["mask_fp"].tolist()
+        self.df = df  # store potential metadata
+
+    def __len__(self):
+        return len(self.image_files)
+
+    def __getitem__(self, idx):
+        # Get the filepath to the image and its mask.
+        image_path = self.image_files[idx]
+        mask_path = self.mask_files[idx]
+
+        # Return image & mask as PIL images in a dictionary.
+        return {"pixel_values": Image.open(image_path), "label": Image.open(mask_path)}

dsa_helpers/ml/datasets/__init__.py

@@ -0,0 +1,3 @@
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+
+__all__ = ["SegFormerSegmentationDataset", "utils"]

dsa_helpers/ml/datasets/utils.py

@@ -0,0 +1,42 @@
+# Utility functions, usualy for creating Datasets in specific formats.
+import pandas as pd
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+from datasets import Dataset, Features, Image
+
+
+def dataset_generator(dataset):
+    """Yield a dataset."""
+    for item in dataset:
+        yield item
+
+
+def create_segformer_segmentation_dataset(
+    df: pd.DataFrame | str, low_memory: bool = True, transforms=None
+):
+    """Create a SegFormer segmentation dataset from a DataFrame.
+
+    Args:
+        df (pd.DataFrame | str): A pandas DataFrame with columns "fp" and "mask_fp" or a
+            path to a CSV file.
+        low_memory (bool): Whether to read the CSV file in low memory mode.
+        transforms: A function that takes in a batch of samples (dictionary with
+            pixel_values and label keys) and returns a transformed batch.
+
+    Returns:
+        A Dataset object to be used for HuggingFaces SegFormer model training.
+
+    """
+    if isinstance(df, str):
+        df = pd.read_csv(df, low_memory=low_memory)
+
+    dataset = SegFormerSegmentationDataset(df)
+
+    dataset = Dataset.from_generator(
+        generator=lambda: dataset_generator(dataset),
+        features=Features({"pixel_values": Image(), "label": Image()}),  # Example shape
+    )
+
+    if transforms is not None:
+        dataset = dataset.set_transform(transforms)
+
+    return dataset

dsa_helpers/ml/metrics/__init__.py

@@ -0,0 +1,3 @@
+from .binary_dice_coefficient_metric import binary_dice_coefficient_metric
+
+__all__ = ["binary_dice_coefficient_metric"]

dsa_helpers/ml/metrics/binary_dice_coefficient_metric.py

@@ -0,0 +1,22 @@
+import torch
+from torch import nn
+from ...utils import binary_dice_coefficient
+
+
+def binary_dice_coefficient_metric(eval_pred):
+    with torch.no_grad():
+        logits, labels = eval_pred  # assumed both are numpy
+
+        logits_tensor = torch.from_numpy(logits)
+
+        # Scale the logits to the size of the label.
+        logits_tensor = nn.functional.interpolate(
+            logits_tensor,
+            size=labels.shape[-2:],
+            mode="bilinear",
+            align_corners=False,
+        ).argmax(dim=1)
+
+        pred_labels = logits_tensor.detach().cpu().numpy()
+
+        return {"dice_coefficient": binary_dice_coefficient(pred_labels, labels)}

dsa_helpers/ml/transforms/__init__.py

@@ -0,0 +1 @@
+__all__ = ["segformer_transforms"]

dsa_helpers/ml/transforms/segformer_transforms.py

@@ -0,0 +1,23 @@
+from transformers import SegformerImageProcessor
+from torchvision.transforms import ColorJitter
+
+
+def train_transforms(example_batch):
+    """Default transforms for training images."""
+    processor = SegformerImageProcessor()  # required for using SegFormer model.
+    jitter = ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)
+
+    images = [jitter(x) for x in example_batch["pixel_values"]]
+    labels = [x for x in example_batch["label"]]
+    inputs = processor(images, labels)
+    return inputs
+
+
+def val_transforms(example_batch):
+    """Default transforms for validation images."""
+    processor = SegformerImageProcessor()  # required for using SegFormer model.
+
+    images = [x for x in example_batch["pixel_values"]]
+    labels = [x for x in example_batch["label"]]
+    inputs = processor(images, labels)
+    return inputs

dsa_helpers/mongo_utils.py

@@ -89,10 +89,8 @@ def add_many_to_collection(
 
     operations = []
 
-    for item in items:
-        operations.append(
-            pymongo.UpdateOne({"_id": item[key]}, {"$set": item}, upsert=True)
-        )
+    for _id, item in items.items():
+        operations.append(pymongo.UpdateOne({"_id": _id}, {"$set": item}, upsert=True))
 
     for chunk in chunks(operations):
         _ = mongo_collection.bulk_write(chunk)

dsa_helpers/utils.py

@@ -0,0 +1,28 @@
+# Basic utility functions.
+import numpy as np
+
+
+def binary_dice_coefficient(mask1: np.ndarray, mask2: np.ndarray) -> float:
+    """Calculate the dice coefficient between two binary masks.
+
+    Args:
+        mask1 (np.ndarray): A binary mask.
+        mask2 (np.ndarray): A binary mask.
+
+    Returns:
+        float: The dice coefficient between the two masks.
+
+    """
+    # Convert the masks to binary.
+    mask1 = mask1 > 0
+    mask2 = mask2 > 0
+
+    # Calculate dice.
+    intersection = np.sum(mask1 * mask2)
+
+    sum_masks = np.sum(mask1) + np.sum(mask2)
+
+    if sum_masks == 0:
+        return 1.0  # Both masks are empty
+
+    return 2.0 * intersection / sum_masks

{dsa_helpers-0.4.dev2.dist-info → dsa_helpers-0.4.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 0.4.dev2
+Version: 0.4.dev4
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -22,6 +22,10 @@ Requires-Dist: dash
 Requires-Dist: dash-bootstrap-components
 Requires-Dist: shapely
 Requires-Dist: geopandas
+Requires-Dist: torch
+Requires-Dist: datasets
+Requires-Dist: transformers
+Requires-Dist: torchvision
 
 # Digital Slide Archive (DSA) Helpers
 Digital Slide Archive Python utility library.
@@ -36,9 +40,11 @@ This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 1. Install requirements for building and distributing package:
 ```
 $ python3 -m pip install --upgrade build  # for building package
-$ python3 -m pip install --upgrade build  # for uploading to PyPI
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
 ```
 2. When ready to build:
     - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
     - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
 3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

dsa_helpers-0.4.dev4.dist-info/RECORD

@@ -0,0 +1,28 @@
+dsa_helpers/__init__.py,sha256=-fAJdwc5u8G-Bbfkc1yh_zDktLXuPfQBoI8KqgVyCOw,216
+dsa_helpers/girder_utils.py,sha256=wzxJ3f_sjuCxfoP4V5KlbSy-B3ej0MEDxVWM6PxCprQ,11377
+dsa_helpers/image_utils.py,sha256=6VUteE_0MfFoRNHt4E2v7pb8XoSjbBxwdhNCk9Gzmgs,1757
+dsa_helpers/imread.py,sha256=jPr6hO4TOtGphENJQWLLtTaQ377yAwLe4F9jSAehJXQ,885
+dsa_helpers/imwrite.py,sha256=eQI8VGn7M35_85QMPrNqGgS66iQn9YboCS50tigAE8s,1159
+dsa_helpers/login.py,sha256=-QyVLW9fKywiPUQCPWxF1BIsufAOqMizDM062nUbupA,1147
+dsa_helpers/mongo_utils.py,sha256=x-66p8DQHYA-mhLnAFKaTcDea43dlSD6bYVf9nQiJds,4679
+dsa_helpers/utils.py,sha256=6_IcUOJHmBZDjVIMbsjEJ4tEpisj48QxvjdyL6OZu88,667
+dsa_helpers/dash/__init__.py,sha256=tNGwvi7Rgr_cSYT7qbbB3Y0mlBZ0pFEA5WFPXjQpugE,21
+dsa_helpers/dash/header.py,sha256=6AsI4npuxhE2_zOZf8OKlkkEkXJze_-fSvMA3OoMucM,4898
+dsa_helpers/dash/header_callbacks.py,sha256=tpf2PU8cgGsecQTWCByxq-Q_v_wUoRikozxRn3brB_4,3256
+dsa_helpers/ml/__init__.py,sha256=AsT-mCmtyYaL7QgHpko1ZOYCbV9Q07PSCs4IpgzME98,100
+dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py,sha256=KVMYQx60OTOslKv0SDKsJbhf8zWkJMG0Ux65mPONJWo,993
+dsa_helpers/ml/datasets/__init__.py,sha256=NZu-iv5uw9VnUIAR5KGOcWrOsAGdqEwO2PrKoWVJdHo,124
+dsa_helpers/ml/datasets/utils.py,sha256=DJedFRav1QPt9_9O8bpTv_oughmfQltHad4IlzOgfwo,1380
+dsa_helpers/ml/metrics/__init__.py,sha256=UemU9cjAi1fGEmurFWbJ8cOoJ8ybzyyQAVmB-joyNi4,121
+dsa_helpers/ml/metrics/binary_dice_coefficient_metric.py,sha256=AA29jG8qnGGvREXSdJba8SVursaNTrLopiJnHZs9EZ4,662
+dsa_helpers/ml/object_detection/__init__.py,sha256=StdNYGjQ3dcJfr5uvUCdWi_XiZpkKMqWNL2EbHBh4Pk,140
+dsa_helpers/ml/object_detection/get_pearce_roi_images.py,sha256=bHV-GpCcm_t8_RaA-TQs3M8E2zI2Q6rrEc0etPV5PGI,4466
+dsa_helpers/ml/object_detection/tile_image.py,sha256=GR6kmhtWVlZOmmoybyw_amew9UdZCj68PWL80kqkdQ4,4492
+dsa_helpers/ml/object_detection/utils.py,sha256=OOwljZTinMHk106-GS2qYXcQTrH7OwO7v9G9SBfTu9E,3273
+dsa_helpers/ml/transforms/__init__.py,sha256=O5pTx287amnPwNhXHY6wT0q5YQxryA3WFm-QRYlm9Ls,35
+dsa_helpers/ml/transforms/segformer_transforms.py,sha256=jn8Mi1GN2dUG7qeEnVnjgsgfIdF_mDGIxvNv_M8pSH8,853
+dsa_helpers-0.4.dev4.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+dsa_helpers-0.4.dev4.dist-info/METADATA,sha256=y7EoghsThMVnyBpDLW9Cm6BaZixytUGN_GBajCRyhHI,2100
+dsa_helpers-0.4.dev4.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+dsa_helpers-0.4.dev4.dist-info/top_level.txt,sha256=BmAEDVvDa1MJlSUFHWzx73UKec3o1K6dFUd9tHwgHGo,12
+dsa_helpers-0.4.dev4.dist-info/RECORD,,

dsa_helpers-0.4.dev2.dist-info/RECORD

@@ -1,20 +0,0 @@
-dsa_helpers/__init__.py,sha256=LriYvuDNhcz1kCSk4hqAVIZAmk2qFDPrdZmA2LK8vkU,210
-dsa_helpers/girder_utils.py,sha256=wzxJ3f_sjuCxfoP4V5KlbSy-B3ej0MEDxVWM6PxCprQ,11377
-dsa_helpers/image_utils.py,sha256=6VUteE_0MfFoRNHt4E2v7pb8XoSjbBxwdhNCk9Gzmgs,1757
-dsa_helpers/imread.py,sha256=jPr6hO4TOtGphENJQWLLtTaQ377yAwLe4F9jSAehJXQ,885
-dsa_helpers/imwrite.py,sha256=eQI8VGn7M35_85QMPrNqGgS66iQn9YboCS50tigAE8s,1159
-dsa_helpers/login.py,sha256=-QyVLW9fKywiPUQCPWxF1BIsufAOqMizDM062nUbupA,1147
-dsa_helpers/mongo_utils.py,sha256=Y3vkhGXpGAbPoDIcv6dZfVoHgmIW64AB6EDMkgMFc5w,4694
-dsa_helpers/dash/__init__.py,sha256=tNGwvi7Rgr_cSYT7qbbB3Y0mlBZ0pFEA5WFPXjQpugE,21
-dsa_helpers/dash/header.py,sha256=6AsI4npuxhE2_zOZf8OKlkkEkXJze_-fSvMA3OoMucM,4898
-dsa_helpers/dash/header_callbacks.py,sha256=tpf2PU8cgGsecQTWCByxq-Q_v_wUoRikozxRn3brB_4,3256
-dsa_helpers/ml/__init__.py,sha256=0LbEDNSy6jGcUnAglJQDC_3FSNKinB2Cr561UglWIUs,63
-dsa_helpers/ml/object_detection/__init__.py,sha256=StdNYGjQ3dcJfr5uvUCdWi_XiZpkKMqWNL2EbHBh4Pk,140
-dsa_helpers/ml/object_detection/get_pearce_roi_images.py,sha256=bHV-GpCcm_t8_RaA-TQs3M8E2zI2Q6rrEc0etPV5PGI,4466
-dsa_helpers/ml/object_detection/tile_image.py,sha256=GR6kmhtWVlZOmmoybyw_amew9UdZCj68PWL80kqkdQ4,4492
-dsa_helpers/ml/object_detection/utils.py,sha256=OOwljZTinMHk106-GS2qYXcQTrH7OwO7v9G9SBfTu9E,3273
-dsa_helpers-0.4.dev2.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
-dsa_helpers-0.4.dev2.dist-info/METADATA,sha256=MrL5fGz0rfDcslkjzQJOj0OgZzF9e26gBtsu6FBQCoU,1814
-dsa_helpers-0.4.dev2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-dsa_helpers-0.4.dev2.dist-info/top_level.txt,sha256=BmAEDVvDa1MJlSUFHWzx73UKec3o1K6dFUd9tHwgHGo,12
-dsa_helpers-0.4.dev2.dist-info/RECORD,,