dsa-helpers 0.4.dev3__tar.gz → 0.4.dev5__tar.gz

{dsa_helpers-0.4.dev3/dsa_helpers.egg-info → dsa_helpers-0.4.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 0.4.dev3
+Version: 0.4.dev5
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -22,6 +22,10 @@ Requires-Dist: dash
 Requires-Dist: dash_bootstrap_components
 Requires-Dist: shapely
 Requires-Dist: geopandas
+Requires-Dist: torch
+Requires-Dist: datasets
+Requires-Dist: transformers
+Requires-Dist: torchvision
 
 # Digital Slide Archive (DSA) Helpers
 Digital Slide Archive Python utility library.
@@ -36,9 +40,11 @@ This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 1. Install requirements for building and distributing package:
 ```
 $ python3 -m pip install --upgrade build  # for building package
-$ python3 -m pip install --upgrade build  # for uploading to PyPI
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
 ```
 2. When ready to build:
     - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
     - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
 3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/README.md

@@ -11,9 +11,11 @@ This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 1. Install requirements for building and distributing package:
 ```
 $ python3 -m pip install --upgrade build  # for building package
-$ python3 -m pip install --upgrade build  # for uploading to PyPI
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
 ```
 2. When ready to build:
     - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
     - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
-3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/dsa_helpers/__init__.py

@@ -5,4 +5,4 @@ from .imwrite import imwrite
 # Modules that should be available.
 # from . import girder_utils, image_utils, dash
 
-__all__ = ["girder_utils", "image_utils", "dash"]
+__all__ = ["girder_utils", "image_utils", "dash", "ml"]

dsa_helpers-0.4.dev5/dsa_helpers/ml/__init__.py

@@ -0,0 +1,2 @@
+# Modules that can be imported.
+__all__ = ["object_detection", "datasets", "metrics", "transforms"]

dsa_helpers-0.4.dev5/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py

@@ -0,0 +1,33 @@
+from torch.utils.data import Dataset
+import pandas as pd
+from PIL import Image
+import numpy as np
+
+
+class SegFormerSegmentationDataset(Dataset):
+    """PyTorch dataset class for semantic segmentation using HuggingFaces SegFormer
+    model.
+
+    """
+
+    def __init__(self, df: pd.DataFrame):
+        """Initiate an instance of the segmentation dataset.
+
+        Args:
+            df (pandas.DataFrame): A pandas DataFrame with columns "fp" and "mask_fp".
+
+        """
+        self.image_files = df["fp"].tolist()
+        self.mask_files = df["mask_fp"].tolist()
+        self.df = df  # store potential metadata
+
+    def __len__(self):
+        return len(self.image_files)
+
+    def __getitem__(self, idx):
+        # Get the filepath to the image and its mask.
+        image_path = self.image_files[idx]
+        mask_path = self.mask_files[idx]
+
+        # Return image & mask as PIL images in a dictionary.
+        return {"pixel_values": Image.open(image_path), "label": Image.open(mask_path)}

dsa_helpers-0.4.dev5/dsa_helpers/ml/datasets/__init__.py

@@ -0,0 +1,3 @@
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+
+__all__ = ["SegFormerSegmentationDataset", "utils"]

dsa_helpers-0.4.dev5/dsa_helpers/ml/datasets/utils.py

@@ -0,0 +1,42 @@
+# Utility functions, usualy for creating Datasets in specific formats.
+import pandas as pd
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+from datasets import Dataset, Features, Image
+
+
+def dataset_generator(dataset):
+    """Yield a dataset."""
+    for item in dataset:
+        yield item
+
+
+def create_segformer_segmentation_dataset(
+    df: pd.DataFrame | str, low_memory: bool = True, transforms=None
+):
+    """Create a SegFormer segmentation dataset from a DataFrame.
+
+    Args:
+        df (pd.DataFrame | str): A pandas DataFrame with columns "fp" and "mask_fp" or a
+            path to a CSV file.
+        low_memory (bool): Whether to read the CSV file in low memory mode.
+        transforms: A function that takes in a batch of samples (dictionary with
+            pixel_values and label keys) and returns a transformed batch.
+
+    Returns:
+        A Dataset object to be used for HuggingFaces SegFormer model training.
+
+    """
+    if isinstance(df, str):
+        df = pd.read_csv(df, low_memory=low_memory)
+
+    dataset = SegFormerSegmentationDataset(df)
+
+    dataset = Dataset.from_generator(
+        generator=lambda: dataset_generator(dataset),
+        features=Features({"pixel_values": Image(), "label": Image()}),  # Example shape
+    )
+
+    if transforms is not None:
+        dataset.set_transform(transforms)
+
+    return dataset

dsa_helpers-0.4.dev5/dsa_helpers/ml/metrics/__init__.py

@@ -0,0 +1,3 @@
+from .binary_dice_coefficient_metric import binary_dice_coefficient_metric
+
+__all__ = ["binary_dice_coefficient_metric"]

dsa_helpers-0.4.dev5/dsa_helpers/ml/metrics/binary_dice_coefficient_metric.py

@@ -0,0 +1,22 @@
+import torch
+from torch import nn
+from ...utils import binary_dice_coefficient
+
+
+def binary_dice_coefficient_metric(eval_pred):
+    with torch.no_grad():
+        logits, labels = eval_pred  # assumed both are numpy
+
+        logits_tensor = torch.from_numpy(logits)
+
+        # Scale the logits to the size of the label.
+        logits_tensor = nn.functional.interpolate(
+            logits_tensor,
+            size=labels.shape[-2:],
+            mode="bilinear",
+            align_corners=False,
+        ).argmax(dim=1)
+
+        pred_labels = logits_tensor.detach().cpu().numpy()
+
+        return {"dice_coefficient": binary_dice_coefficient(pred_labels, labels)}

dsa_helpers-0.4.dev5/dsa_helpers/ml/transforms/__init__.py

@@ -0,0 +1 @@
+__all__ = ["segformer_transforms"]

dsa_helpers-0.4.dev5/dsa_helpers/ml/transforms/segformer_transforms.py

@@ -0,0 +1,23 @@
+from transformers import SegformerImageProcessor
+from torchvision.transforms import ColorJitter
+
+
+def train_transforms(example_batch):
+    """Default transforms for training images."""
+    processor = SegformerImageProcessor()  # required for using SegFormer model.
+    jitter = ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)
+
+    images = [jitter(x) for x in example_batch["pixel_values"]]
+    labels = [x for x in example_batch["label"]]
+    inputs = processor(images, labels)
+    return inputs
+
+
+def val_transforms(example_batch):
+    """Default transforms for validation images."""
+    processor = SegformerImageProcessor()  # required for using SegFormer model.
+
+    images = [x for x in example_batch["pixel_values"]]
+    labels = [x for x in example_batch["label"]]
+    inputs = processor(images, labels)
+    return inputs

dsa_helpers-0.4.dev5/dsa_helpers/utils.py

@@ -0,0 +1,28 @@
+# Basic utility functions.
+import numpy as np
+
+
+def binary_dice_coefficient(mask1: np.ndarray, mask2: np.ndarray) -> float:
+    """Calculate the dice coefficient between two binary masks.
+
+    Args:
+        mask1 (np.ndarray): A binary mask.
+        mask2 (np.ndarray): A binary mask.
+
+    Returns:
+        float: The dice coefficient between the two masks.
+
+    """
+    # Convert the masks to binary.
+    mask1 = mask1 > 0
+    mask2 = mask2 > 0
+
+    # Calculate dice.
+    intersection = np.sum(mask1 * mask2)
+
+    sum_masks = np.sum(mask1) + np.sum(mask2)
+
+    if sum_masks == 0:
+        return 1.0  # Both masks are empty
+
+    return 2.0 * intersection / sum_masks

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5/dsa_helpers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 0.4.dev3
+Version: 0.4.dev5
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -22,6 +22,10 @@ Requires-Dist: dash
 Requires-Dist: dash_bootstrap_components
 Requires-Dist: shapely
 Requires-Dist: geopandas
+Requires-Dist: torch
+Requires-Dist: datasets
+Requires-Dist: transformers
+Requires-Dist: torchvision
 
 # Digital Slide Archive (DSA) Helpers
 Digital Slide Archive Python utility library.
@@ -36,9 +40,11 @@ This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 1. Install requirements for building and distributing package:
 ```
 $ python3 -m pip install --upgrade build  # for building package
-$ python3 -m pip install --upgrade build  # for uploading to PyPI
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
 ```
 2. When ready to build:
     - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
     - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
 3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/dsa_helpers.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ LICENSE
 README.md
 app.py
 pyproject.toml
-readthedocs.yaml
 requirements.txt
 dsa_helpers/__init__.py
 dsa_helpers/girder_utils.py
@@ -12,6 +11,7 @@ dsa_helpers/imread.py
 dsa_helpers/imwrite.py
 dsa_helpers/login.py
 dsa_helpers/mongo_utils.py
+dsa_helpers/utils.py
 dsa_helpers.egg-info/PKG-INFO
 dsa_helpers.egg-info/SOURCES.txt
 dsa_helpers.egg-info/dependency_links.txt
@@ -21,8 +21,18 @@ dsa_helpers/dash/__init__.py
 dsa_helpers/dash/header.py
 dsa_helpers/dash/header_callbacks.py
 dsa_helpers/ml/__init__.py
+dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py
+dsa_helpers/ml/datasets/__init__.py
+dsa_helpers/ml/datasets/utils.py
+dsa_helpers/ml/metrics/__init__.py
+dsa_helpers/ml/metrics/binary_dice_coefficient_metric.py
 dsa_helpers/ml/object_detection/__init__.py
 dsa_helpers/ml/object_detection/get_pearce_roi_images.py
 dsa_helpers/ml/object_detection/tile_image.py
 dsa_helpers/ml/object_detection/utils.py
-tests/test.ipynb
+dsa_helpers/ml/transforms/__init__.py
+dsa_helpers/ml/transforms/segformer_transforms.py
+tests/docker-compose.yaml
+tests/test.ipynb
+tests/app/Dockerfile
+tests/app/app.py

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/dsa_helpers.egg-info/requires.txt

@@ -9,3 +9,7 @@ dash
 dash_bootstrap_components
 shapely
 geopandas
+torch
+datasets
+transformers
+torchvision

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/pyproject.toml

@@ -14,7 +14,11 @@ dependencies = [
     'dash',
     'dash_bootstrap_components',
     'shapely',
-    'geopandas'
+    'geopandas',
+    "torch",
+    "datasets",
+    "transformers",
+    "torchvision"
 ]
 name = "dsa_helpers"
 authors = [
@@ -31,7 +35,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 keywords = ["dsa", "girder", "girder_client"]
-version = "v0.4dev3"
+version = "v0.4dev5"
 
 [project.urls]
 Repository = "https://github.com/Gutman-Lab/dsa-python-utils"

{dsa_helpers-0.4.dev3 → dsa_helpers-0.4.dev5}/requirements.txt

@@ -8,4 +8,8 @@ jupyter
 pillow
 dash
 dash_bootstrap_components
-python-dotenv
+python-dotenv
+torch
+datasets
+transformers
+torchvision

dsa_helpers-0.4.dev5/tests/app/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.11-slim-buster
+
+ENV PYTHONUNBUFFERED True
+
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
+
+WORKDIR /app
+
+COPY requirements.txt /app
+
+RUN pip install --upgrade pip
+
+RUN python -m pip install pymongo==4.9.1
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY dsa_helpers-0.4.dev1-py3-none-any.whl /app
+RUN pip install dsa_helpers-0.4.dev1-py3-none-any.whl

dsa_helpers-0.4.dev5/tests/app/app.py

@@ -0,0 +1,55 @@
+from dash import html, Dash, callback, Output, Input, State, no_update
+import dash_bootstrap_components as dbc
+from dsa_helpers.dash.header import get_header
+from os import getenv
+from dash_ag_grid import AgGrid
+
+table = AgGrid(
+    id="table",
+    columnDefs=[],
+    rowData=[],
+)
+
+app = Dash(
+    __name__,
+    external_stylesheets=[
+        dbc.themes.BOOTSTRAP,
+    ],
+    title="Test Application",
+)
+
+app.layout = html.Div(
+    [
+        get_header(
+            getenv("DSA_API_URL"),
+            title="Test Application",
+            store_id="user-store",
+        ),
+        dbc.Row(
+            [dbc.Col(dbc.Button("Get items", id="get-item-btn"), width="auto")],
+            justify="start",
+        ),
+        table
+    ]
+)
+
+
+@callback(
+    [
+        Output("table", "columnDefs"),
+        Output("table", "rowData"),
+    ],
+    Input("get-item-btn", "n_clicks"),
+    prevent_initial_call=True,
+)
+def get_items(n_clicks):
+    """Get the items."""
+    
+    
+
+if __name__ == "__main__":
+    app.run(
+        host="0.0.0.0",
+        port=8060,
+        debug=True,
+    )

dsa_helpers-0.4.dev5/tests/docker-compose.yaml

@@ -0,0 +1,31 @@
+volumes:
+  mongodb-data-volume:
+
+services:
+  mongodb:
+    image: mongo:7.0.6-jammy
+    restart: always
+    platform: linux/amd64
+    container_name: mongodb
+    volumes:
+      - mongodb-data-volume:/data/db
+    ports:
+      - 27017:27017
+    env_file: .env
+    command: [ "--quiet", "--maxConns", "1000" ]
+    logging:
+      options:
+        max-size: "10M"
+        max-file: "5"
+        
+  dash:
+    build: app/
+    platform: linux/amd64
+    volumes:
+      - ./app:/app
+    ports:
+      - 8060:8060
+    env_file: .env
+    command: python app.py
+    depends_on:
+      - mongodb