dsa-helpers 0.3.dev1__tar.gz → 0.4__tar.gz

{dsa_helpers-0.3.dev1 → dsa_helpers-0.4}/.gitignore

@@ -7,4 +7,5 @@ dsa_utils/_version.py
 
 *.pyc
 *.egg-info
-dist/
+dist/
+.env

dsa_helpers-0.4/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: dsa_helpers
+Version: 0.4
+Summary: Utility functions for working with the DSA girder client.
+Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
+Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
+Project-URL: Repository, https://github.com/Gutman-Lab/dsa-python-utils
+Keywords: dsa,girder,girder_client
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: girder-client
+Requires-Dist: numpy
+Requires-Dist: setuptools
+Requires-Dist: opencv-python
+Requires-Dist: pandas
+Requires-Dist: pymongo
+Requires-Dist: pillow
+Requires-Dist: dash
+Requires-Dist: dash_bootstrap_components
+Requires-Dist: shapely
+Requires-Dist: geopandas
+Requires-Dist: torch
+Requires-Dist: datasets
+Requires-Dist: transformers
+Requires-Dist: torchvision
+
+# Digital Slide Archive (DSA) Helpers
+Digital Slide Archive Python utility library.
+
+This library is available for installation through [Python Package Index (PyPI)](https://pypi.org/).
+
+This library was tested using Python version 3.11.8 and uses the dependencies described in requirements.txt.
+
+This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
+
+## Instructions for Development
+1. Install requirements for building and distributing package:
+```
+$ python3 -m pip install --upgrade build  # for building package
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
+```
+2. When ready to build:
+    - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
+    - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
+3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

dsa_helpers-0.4/README.md

@@ -0,0 +1,21 @@
+# Digital Slide Archive (DSA) Helpers
+Digital Slide Archive Python utility library.
+
+This library is available for installation through [Python Package Index (PyPI)](https://pypi.org/).
+
+This library was tested using Python version 3.11.8 and uses the dependencies described in requirements.txt.
+
+This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
+
+## Instructions for Development
+1. Install requirements for building and distributing package:
+```
+$ python3 -m pip install --upgrade build  # for building package
+$ python3 -m pip install --upgrade twine  # for uploading to PyPI
+```
+2. When ready to build:
+    - Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
+    - Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
+3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
+4. Upload to PyPI using twine: ```python3 -m twine upload --repository pypi dist/*```
+    * You can specify specific files to upload, the line above pushes everything in dist directory.

dsa_helpers-0.4/app.py

@@ -0,0 +1,14 @@
+# A test application.
+from dash import Dash, html
+import dash_bootstrap_components as dbc
+from dsa_helpers.dash.header import get_header
+import os
+
+os.environ["DSA_API_URL"] = "https://megabrain.neurology.emory.edu/api/v1"
+
+app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+
+app.layout = html.Div([get_header("Test")])
+
+if __name__ == "__main__":
+    app.run_server(debug=True)

{dsa_helpers-0.3.dev1 → dsa_helpers-0.4}/dsa_helpers/__init__.py

@@ -1,7 +1,8 @@
 # Shadow imports.
-from .login import login
 from .imread import imread
 from .imwrite import imwrite
 
 # Modules that should be available.
-from . import girder_utils, image_utils, dash
+# from . import girder_utils, image_utils, dash
+
+__all__ = ["girder_utils", "image_utils", "dash", "ml"]

{dsa_helpers-0.3.dev1 → dsa_helpers-0.4}/dsa_helpers/dash/header.py

@@ -1,24 +1,37 @@
 from dash import html, dcc
 import dash_bootstrap_components as dbc
-from . import header_callbacks
+from .header_callbacks import get_callbacks
 
 
-def get_header(title: str = "App", config: dict | None = None) -> html.Div:
+def get_header(
+    dsa_api_url: str,
+    title: str = "App",
+    store_id: str = "user-store",
+    config: dict | None = None,
+) -> html.Div:
     """Get the Dash html component that contains the header of the application
     with login capabilities.
 
     Args:
-        title (str): The title of the application.
-        config (dict): The configuration of the component.
-
-            backgroundColor
-            titleColor
-            fontColor
+        dsa_api_url (str): The URL of the DSA API.
+        title (str, optional): The title of the application. Defaults to "App".
+        config (dict, optional): The configuration of the component. Defaults to None.
+            If None the default configuration is:
+            {
+                "backgroundColor": "#012169",
+                "titleColor": "#d9d9d6",
+                "fontColor": "#f2a900",
+            }
 
     Returns:
         html.Div: The header component.
 
     """
+    if config is None:
+        config = {}
+
+    get_callbacks(dsa_api_url, store_id)
+
     login_modal = dbc.Modal(
         [
             dbc.ModalHeader("Log in"),
@@ -28,7 +41,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                         "Login or email", style={"margin": 5, "fontWeight": "bold"}
                     ),
                     dbc.Input(
-                        id="login",
+                        id=f"{store_id}=login",
                         type="text",
                         placeholder="Enter login",
                         style={"margin": 5},
@@ -38,7 +51,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                         style={"margin": 5, "marginTop": 15, "fontWeight": "bold"},
                     ),
                     dbc.Input(
-                        id="password",
+                        id=f"{store_id}=password",
                         type="password",
                         placeholder="Enter password",
                         style={"margin": 5},
@@ -46,7 +59,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                     html.Div(
                         "Login failed.",
                         hidden=True,
-                        id="login-failed",
+                        id=f"{store_id}=login-failed",
                         style={"color": "red", "fontWeight": "bold", "margin": 10},
                     ),
                 ],
@@ -57,7 +70,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                         dbc.Col(
                             dbc.Button(
                                 "Close",
-                                id="close-login-modal",
+                                id=f"{store_id}=close-login-modal",
                                 className="me-1",
                                 color="light",
                             )
@@ -65,7 +78,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                         dbc.Col(
                             dbc.Button(
                                 "Login",
-                                id="log-in-btn",
+                                id=f"{store_id}=log-in-btn",
                                 className="me-1",
                                 color="primary",
                             )
@@ -75,7 +88,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
             ),
         ],
         is_open=False,
-        id="login-modal",
+        id=f"{store_id}=login-modal",
     )
 
     logout_modal = dbc.Modal(
@@ -84,7 +97,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                 [
                     dbc.Button(
                         "Log out",
-                        id="logout-btn",
+                        id=f"{store_id}=logout-btn",
                         color="danger",
                         className="me-1",
                     ),
@@ -92,12 +105,9 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
             )
         ],
         is_open=False,
-        id="logout-modal",
+        id=f"{store_id}=logout-modal",
     )
 
-    if config is None:
-        config = {}
-
     return html.Div(
         [
             dbc.Row(
@@ -116,7 +126,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
                     dbc.Col(
                         dbc.Button(
                             "Log in",
-                            id="login-btn",
+                            id=f"{store_id}=login-btn",
                             color=config.get("backgroundColor", "#012169"),
                             style={
                                 "color": config.get("fontColor", "#f2a900"),
@@ -132,7 +142,7 @@ def get_header(title: str = "App", config: dict | None = None) -> html.Div:
             ),
             login_modal,
             logout_modal,
-            dcc.Store(id="user-store", storage_type="local", data={}),
+            dcc.Store(id=store_id, storage_type="local", data={}),
         ],
         style={
             "backgroundColor": config.get("backgroundColor", "#012169"),

dsa_helpers-0.4/dsa_helpers/dash/header_callbacks.py

@@ -0,0 +1,99 @@
+# Callbacks for the header component.
+from dash import callback, Output, Input, State, no_update
+from girder_client import GirderClient
+
+
+def get_callbacks(dsa_api_url: str, store_id: str):
+    """Setup the callbacks to the DSA API url of interest and the proper store id."""
+
+    @callback(Output(f"{store_id}=login-btn", "children"), Input(store_id, "data"))
+    def check_user_store(data):
+        # Check if the user store has user info or if no one is logged in.
+        return data["user"] if len(data) else "Log in"
+
+    @callback(
+        [
+            Output(f"{store_id}=login-modal", "is_open", allow_duplicate=True),
+            Output(f"{store_id}=logout-modal", "is_open", allow_duplicate=True),
+        ],
+        [
+            Input(f"{store_id}=login-btn", "n_clicks"),
+            State(f"{store_id}=login-btn", "children"),
+        ],
+        prevent_initial_call=True,
+    )
+    def open_login_modal(n_clicks, children):
+        # Open login / logout modal.
+        if n_clicks:
+            if children == "Log in":
+                return True, False
+            else:
+                return False, True
+
+        return False, False
+
+    @callback(
+        [
+            Output(store_id, "data"),
+            Output(f"{store_id}=login-failed", "hidden", allow_duplicate=True),
+            Output(f"{store_id}=login-modal", "is_open", allow_duplicate=True),
+            Output(f"{store_id}=login", "value", allow_duplicate=True),
+            Output(f"{store_id}=password", "value", allow_duplicate=True),
+        ],
+        [
+            Input(f"{store_id}=log-in-btn", "n_clicks"),
+            State(f"{store_id}=login", "value"),
+            State(f"{store_id}=password", "value"),
+        ],
+        prevent_initial_call=True,
+    )
+    def login(n_clicks, login, password):
+        # Try to login.
+        gc = GirderClient(apiUrl=dsa_api_url)
+
+        try:
+            _ = gc.authenticate(username=login, password=password)
+
+            response = gc.get("token/session")
+
+            user = gc.get("user/me")["login"]
+
+            return {"user": user, "token": response["token"]}, True, False, "", ""
+        except:
+            return (
+                {},
+                False,
+                True,
+                no_update,
+                no_update,
+            )
+
+    @callback(
+        [
+            Output(f"{store_id}=login-modal", "is_open", allow_duplicate=True),
+            Output(f"{store_id}=login", "value", allow_duplicate=True),
+            Output(f"{store_id}=password", "value", allow_duplicate=True),
+            Output(f"{store_id}=login-failed", "hidden", allow_duplicate=True),
+        ],
+        Input(f"{store_id}=close-login-modal", "n_clicks"),
+        prevent_initial_call=True,
+    )
+    def close_login_modal(n_clicks):
+        if n_clicks:
+            return False, "", "", True
+
+        return False, "", "", True
+
+    @callback(
+        [
+            Output(store_id, "data", allow_duplicate=True),
+            Output(f"{store_id}=logout-modal", "is_open", allow_duplicate=True),
+        ],
+        Input(f"{store_id}=logout-btn", "n_clicks"),
+        prevent_initial_call=True,
+    )
+    def logout(n_clicks):
+        if n_clicks:
+            return {}, False
+
+        return no_update, False

{dsa_helpers-0.3.dev1 → dsa_helpers-0.4}/dsa_helpers/girder_utils.py

@@ -1,5 +1,5 @@
 # Functions using the girder client.
-from girder_client import GirderClient
+from girder_client import GirderClient, HttpError
 import pickle
 import numpy as np
 import pandas as pd
@@ -285,3 +285,62 @@ def get_rectangle_element_coords(element):
     corner_coords = np.array(corner_coords, dtype=np.int32)
 
     return corner_coords
+
+
+def login(
+    api_url: str,
+    login_or_email: str | None = None,
+    password: str | None = None,
+    api_key: str | None = None,
+) -> GirderClient:
+    """Authenticate a girder client with the given credentials or interactively
+    if none is given.
+
+    Args:
+        api_url (str): The DSA girder API url.
+        login_or_email (str | None): The login or email. Defaults to None.
+        password (str | None): Password for login / email. Defaults to None.
+        api_key (str | None): The api key to authenticate with. Defaults to None.
+
+    Returns:
+        girder_client.GirderClient: The authenticated girder client.
+
+    """
+    gc = GirderClient(apiUrl=api_url)
+
+    if api_key is None:
+        if login_or_email is None:
+            _ = gc.authenticate(interactive=True)
+        elif password is None:
+            _ = gc.authenticate(username=login_or_email, interactive=True)
+        else:
+            _ = gc.authenticate(username=login_or_email, password=password)
+    else:
+        _ = gc.authenticate(apiKey=api_key)
+
+    return gc
+
+
+def get_items(gc: GirderClient, parend_id: str) -> list[dict]:
+    """Get the items in a parent location recursively.
+
+    Args:
+        gc (girder_client.GirderClient): The authenticated girder client.
+        parend_id (str): The parent id to start the search (folder / collection).
+
+    Returns:
+        list[dict]: The list of items.
+
+    """
+    params = {"type": "folder", "limit": 0, "offset": 0, "sort": "_id", "sortdir": 1}
+
+    request_url = f"resource/{parend_id}/items"
+
+    try:
+        items = gc.get(request_url, parameters=params)
+    except HttpError:
+        params["type"] = "collection"
+
+        items = gc.get(request_url, parameters=params)
+
+    return items

dsa_helpers-0.4/dsa_helpers/ml/__init__.py

@@ -0,0 +1,2 @@
+# Modules that can be imported.
+__all__ = ["object_detection", "datasets", "metrics", "transforms"]

dsa_helpers-0.4/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py

@@ -0,0 +1,33 @@
+from torch.utils.data import Dataset
+import pandas as pd
+from PIL import Image
+import numpy as np
+
+
+class SegFormerSegmentationDataset(Dataset):
+    """PyTorch dataset class for semantic segmentation using HuggingFaces SegFormer
+    model.
+
+    """
+
+    def __init__(self, df: pd.DataFrame):
+        """Initiate an instance of the segmentation dataset.
+
+        Args:
+            df (pandas.DataFrame): A pandas DataFrame with columns "fp" and "mask_fp".
+
+        """
+        self.image_files = df["fp"].tolist()
+        self.mask_files = df["mask_fp"].tolist()
+        self.df = df  # store potential metadata
+
+    def __len__(self):
+        return len(self.image_files)
+
+    def __getitem__(self, idx):
+        # Get the filepath to the image and its mask.
+        image_path = self.image_files[idx]
+        mask_path = self.mask_files[idx]
+
+        # Return image & mask as PIL images in a dictionary.
+        return {"pixel_values": Image.open(image_path), "label": Image.open(mask_path)}

dsa_helpers-0.4/dsa_helpers/ml/datasets/__init__.py

@@ -0,0 +1,3 @@
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+
+__all__ = ["SegFormerSegmentationDataset", "utils"]

dsa_helpers-0.4/dsa_helpers/ml/datasets/utils.py

@@ -0,0 +1,42 @@
+# Utility functions, usualy for creating Datasets in specific formats.
+import pandas as pd
+from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
+from datasets import Dataset, Features, Image
+
+
+def dataset_generator(dataset):
+    """Yield a dataset."""
+    for item in dataset:
+        yield item
+
+
+def create_segformer_segmentation_dataset(
+    df: pd.DataFrame | str, low_memory: bool = True, transforms=None
+):
+    """Create a SegFormer segmentation dataset from a DataFrame.
+
+    Args:
+        df (pd.DataFrame | str): A pandas DataFrame with columns "fp" and "mask_fp" or a
+            path to a CSV file.
+        low_memory (bool): Whether to read the CSV file in low memory mode.
+        transforms: A function that takes in a batch of samples (dictionary with
+            pixel_values and label keys) and returns a transformed batch.
+
+    Returns:
+        A Dataset object to be used for HuggingFaces SegFormer model training.
+
+    """
+    if isinstance(df, str):
+        df = pd.read_csv(df, low_memory=low_memory)
+
+    dataset = SegFormerSegmentationDataset(df)
+
+    dataset = Dataset.from_generator(
+        generator=lambda: dataset_generator(dataset),
+        features=Features({"pixel_values": Image(), "label": Image()}),  # Example shape
+    )
+
+    if transforms is not None:
+        dataset.set_transform(transforms)
+
+    return dataset

dsa_helpers-0.4/dsa_helpers/ml/metrics/__init__.py

@@ -0,0 +1,3 @@
+from .binary_dice_coefficient_metric import binary_dice_coefficient_metric
+
+__all__ = ["binary_dice_coefficient_metric"]

dsa_helpers-0.4/dsa_helpers/ml/metrics/binary_dice_coefficient_metric.py

@@ -0,0 +1,24 @@
+import torch
+from torch import nn
+from ...utils import binary_dice_coefficient
+
+
+def binary_dice_coefficient_metric(eval_pred):
+    with torch.no_grad():
+        logits, labels = eval_pred  # assumed both are numpy
+
+        logits_tensor = torch.from_numpy(logits).cpu()
+        labels_tensor = torch.from_numpy(labels).cpu()
+
+        # Scale the logits to the size of the label.
+        logits_tensor = nn.functional.interpolate(
+            logits_tensor,
+            size=labels.shape[-2:],
+            mode="bilinear",
+            align_corners=False,
+        ).argmax(dim=1)
+
+        pred_labels = logits_tensor.detach().numpy()
+        labels = labels_tensor.numpy()
+
+        return {"dice_coefficient": binary_dice_coefficient(pred_labels, labels)}

dsa_helpers-0.4/dsa_helpers/ml/object_detection/__init__.py

@@ -0,0 +1,5 @@
+# Shadow import functions.
+from .get_pearce_roi_images import get_pearce_roi_images
+from .tile_image import tile_image
+
+__all__ = ["utils"]

dsa_helpers-0.4/dsa_helpers/ml/object_detection/get_pearce_roi_images.py

@@ -0,0 +1,136 @@
+from girder_client import GirderClient
+import pandas as pd
+from pathlib import Path
+
+from ...girder_utils import get_region, get_item_large_image_metadata
+from ... import imwrite
+
+
+def get_pearce_roi_images(
+    gc: GirderClient,
+    item_id: str,
+    class_map: dict[str, int],
+    save_dir: str = ".",
+    mag: float | None = None,
+) -> pd.DataFrame:
+    """Download the smallestl bounding box around an ROI annotations and its
+    accompanying object detection annotations. The format used to determine ROIs
+    and boxes is based on the convention used by Dr. Thomas Pearce in his
+    OpenSeadragon applications. Labels are saved as text files in the YOLO format.
+
+    Args:
+        gc (girder_client.GirderClient): The Girder client to use.
+        item_id (str): The item ID of the WSI.
+        class_map (dict[str, int]): A dictionary mapping class names to integers.
+        save_dir (str, optional): The directory to save the images and labels.
+            Defaults to ".".
+        mag (float | None, optional): The magnification to download the ROI at.
+            If None, the scan magnification is used. Defaults to None.
+
+    Returns:
+        pandas.DataFrame: A DataFrame containing metadata for each ROI
+
+    """
+    # Create directories to save images and labels.
+    img_dir = Path(save_dir) / "images"
+    img_dir.mkdir(parents=True, exist_ok=True)
+
+    label_dir = Path(save_dir) / "labels"
+    label_dir.mkdir(parents=True, exist_ok=True)
+
+    # Get all the annotation documents with name starting with "ROI".
+    ann_docs_ids = []
+
+    for ann_doc in gc.get(
+        f"annotation?itemId={item_id}&limit=0&offset=0&sort=lowerName&sortdir=1"
+    ):
+        if ann_doc.get("annotation", {}).get("name", "").startswith("ROI"):
+            ann_docs_ids.append(ann_doc["_id"])
+
+    if not len(ann_docs_ids):
+        # No annotation documents, return None.
+        return
+
+    # Track metadata for each ROI.
+    roi_metadata = []
+
+    # Loop through each document.
+    for ann_doc_id in ann_docs_ids:
+        # Get the full annotation document.
+        ann_doc = gc.get(f"annotation/{ann_doc_id}")
+
+        # Separate the ROI element and the other elements.
+        roi_el = None
+        other_els = []
+
+        for element in ann_doc.get("annotation", {}).get("elements", []):
+            if element.get("user", {}).get("role") == "ROI":
+                roi_el = element
+            else:
+                other_els.append(element)
+
+        if roi_el is None:
+            raise Exception("No ROI element found in annotation document.")
+
+        # Download the ROI image.
+        xc, yc = roi_el["center"][:2]
+        roi_w, roi_h = int(roi_el["width"]), int(roi_el["height"])
+
+        # Get the top left coordinate of the ROI.
+        roi_left = int(xc - roi_w / 2)
+        roi_top = int(yc - roi_h / 2)
+
+        # Get scale factor to go from scan magnification to mag.
+        scan_mag = get_item_large_image_metadata(gc, item_id)["magnification"]
+
+        if mag is None:
+            mag = scan_mag
+
+        roi_img = get_region(
+            gc, item_id, left=roi_left, top=roi_top, width=roi_w, height=roi_h, mag=mag
+        )
+
+        fn = f"itemId-{item_id}-x{roi_left}y{roi_top}x{roi_left+roi_w}y{roi_top+roi_h}"
+
+        img_fp = img_dir / f"{fn}.png"
+        imwrite(img_fp, roi_img)
+
+        # Convert the boxes to YOLO format.
+        labels = ""
+
+        for element in other_els:
+            el_class = element["user"]["class"]
+
+            if el_class not in class_map:
+                continue
+
+            xc, yc = element["center"][:2]
+            xc, yc = int(xc), int(yc)
+
+            # Subtract the left and top of the ROI to get the correct coordinates.
+            xc = xc - roi_left
+            yc = yc - roi_top
+
+            # Normalize by the ROI width and height.
+            xc /= roi_w
+            yc /= roi_h
+
+            w = element["width"] / roi_w
+            h = element["height"] / roi_h
+
+            labels += f"{class_map[el_class]} {xc:.4f} {yc:.4f} {w:.4f} {h:.4f}\n"
+
+        # Save the label to a text file.
+        label_fp = label_dir / f"{fn}.txt"
+
+        with open(label_fp, "w") as fh:
+            fh.write(labels.strip())
+
+        # Append all the metadata to the list.
+        roi_metadata.append(
+            [item_id, img_fp, roi_left, roi_top, roi_w, roi_h, mag, scan_mag]
+        )
+
+    return pd.DataFrame(
+        roi_metadata, columns=["item_id", "fp", "x", "y", "w", "h", "mag", "scan_mag"]
+    )