dsa-helpers 0.6.dev0__tar.gz → 0.8__tar.gz

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/.gitignore

@@ -8,4 +8,5 @@ dist/
 .env
 
 # Ignore data directory created for testing the package.
-tests/data/
+tests/data/
+tests/models/

{dsa_helpers-0.6.dev0/dsa_helpers.egg-info → dsa_helpers-0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 0.6.dev0
+Version: 0.8
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -26,6 +26,7 @@ Requires-Dist: transformers[torch]
 Requires-Dist: torch
 Requires-Dist: datasets
 Requires-Dist: torchvision
+Requires-Dist: large-image
 
 # Digital Slide Archive (DSA) Helpers
 Digital Slide Archive Python utility library.
@@ -36,6 +37,9 @@ This library was tested using Python version 3.11.8 and uses the dependencies de
 
 This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 
+## Extra dependencies
+If you are getting a large image error, then install large image using the instructions provided in their [GitHub page](https://github.com/girder/large_image#:~:text=pip%20install%20large%2Dimage%5Ball%5D%20%2D%2Dfind%2Dlinks%20https%3A//girder.github.io/large_image_wheels).
+
 ## Instructions for Development
 1. Install requirements for building and distributing package:
 ```

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/README.md

@@ -7,6 +7,9 @@ This library was tested using Python version 3.11.8 and uses the dependencies de
 
 This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 
+## Extra dependencies
+If you are getting a large image error, then install large image using the instructions provided in their [GitHub page](https://github.com/girder/large_image#:~:text=pip%20install%20large%2Dimage%5Ball%5D%20%2D%2Dfind%2Dlinks%20https%3A//girder.github.io/large_image_wheels).
+
 ## Instructions for Development
 1. Install requirements for building and distributing package:
 ```

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/dsa_helpers/girder_utils.py

@@ -63,7 +63,9 @@ def get_thumbnail(
                 elif len(fill) == 4:
                     fill = f"rgba({fill[0]},{fill[1]},{fill[2]},{fill[3]})"
 
-            params.extend([f"width={width}", f"height={height}", f"fill={fill}"])
+            params.extend(
+                [f"width={width}", f"height={height}", f"fill={fill}"]
+            )
         elif width is not None:
             params.append(f"width={width}")
         elif height is not None:
@@ -246,7 +248,10 @@ def _rotate_point_list(point_list, rotation, center=(0, 0)):
         y = point[1] - center[1]
 
         point_list_rotated.append(
-            (int(x * cos - y * sin + center[0]), int(x * sin + y * cos + center[1]))
+            (
+                int(x * cos - y * sin + center[0]),
+                int(x * sin + y * cos + center[1]),
+            )
         )
 
     return point_list_rotated
@@ -274,12 +279,19 @@ def get_rectangle_element_coords(element):
     x_max = center_x + w // 2
     y_min = center_y - h // 2
     y_max = center_y + h // 2
-    corner_coords = [(x_min, y_min), (x_max, y_min), (x_max, y_max), (x_min, y_max)]
+    corner_coords = [
+        (x_min, y_min),
+        (x_max, y_min),
+        (x_max, y_max),
+        (x_min, y_max),
+    ]
 
     # if there is rotation rotate
     if element["rotation"]:
         corner_coords = _rotate_point_list(
-            corner_coords, rotation=element["rotation"], center=(center_x, center_y)
+            corner_coords,
+            rotation=element["rotation"],
+            center=(center_x, center_y),
         )
 
     corner_coords = np.array(corner_coords, dtype=np.int32)
@@ -332,7 +344,13 @@ def get_items(gc: GirderClient, parend_id: str) -> list[dict]:
         list[dict]: The list of items.
 
     """
-    params = {"type": "folder", "limit": 0, "offset": 0, "sort": "_id", "sortdir": 1}
+    params = {
+        "type": "folder",
+        "limit": 0,
+        "offset": 0,
+        "sort": "_id",
+        "sortdir": 1,
+    }
 
     request_url = f"resource/{parend_id}/items"
 
@@ -344,3 +362,128 @@ def get_items(gc: GirderClient, parend_id: str) -> list[dict]:
         items = gc.get(request_url, parameters=params)
 
     return items
+
+
+def get_roi_with_yolo_labels_from_single_doc(
+    gc: GirderClient,
+    ann_doc: dict,
+    roi_element_group: str,
+    group_map: dict[str, int],
+    mag: float | None = None,
+    rgb_fill: tuple[int, int, int] = (114, 114, 114),
+) -> tuple[np.ndarray, str]:
+    """Get the ROI image with the YOLO labels given a DSA annotation document.
+
+    Args:
+        gc (girder_client.GirderClient): Authenticated girder client.
+        ann_doc (dict): Annotation document metadata.
+        roi_element_group (str): The group name of the ROI element.
+        group_map (dict[str, int]): Mapping of group names to group IDs.
+        mag (float, optional): The magnification to use for the ROI. Defaults to
+            None which uses the default magnification.
+        rgb_fill (tuple[int, int, int], optional): The RGB color to fill the ROI
+            with. Defaults to (114, 114, 114).
+
+    Returns:
+        tuple[np.ndarray, str]: The ROI image and YOLO labels data as string.
+
+    Raises:
+        ValueError: If multiple ROI elements are found in the document.
+        ValueError: If ROI element type is not polyline.
+
+    """
+    roi_element = None
+    box_elements = []
+
+    # Loop through all elements in the annotation document.
+    for element in ann_doc.get("annotation", {}).get("elements", []):
+        if element.get("group") == roi_element_group:
+            # Append the ROI element, there can't more than one.
+            if roi_element is not None:
+                raise ValueError("Multiple ROI elements found in the document.")
+
+            roi_element = element
+        elif element.get("group") in group_map:
+            # Append the box elements.
+            box_elements.append(element)
+
+    # Get the large image metadata.
+    large_image_metadata = get_item_large_image_metadata(gc, ann_doc["itemId"])
+    scan_mag = large_image_metadata["magnification"]
+
+    # Specify magnification to use.
+    if mag is None:
+        mag = scan_mag
+
+    # Calculate the factor to convert from scan mag to desired mag.
+    sf = mag / scan_mag
+
+    # Get the ROI image.
+    if roi_element.get("type") == "polyline":
+        # Grab the smallest bounding box.
+        points = np.array(roi_element["points"])[:, :2]  # remove z-axis
+        roi_x1, roi_y1 = points.min(axis=0)
+        roi_x2, roi_y2 = points.max(axis=0)
+
+        img = get_region(
+            gc,
+            ann_doc["itemId"],
+            roi_x1,
+            roi_y1,
+            roi_x2 - roi_x1,
+            roi_y2 - roi_y1,
+            mag=mag,
+        )[
+            :, :, :3
+        ]  # make sure it is RGB
+
+        # Shift the points to be relative to smallest bounding box.
+        points -= [roi_x1, roi_y1]
+
+        # Create a mask and draw the ROI filled on it (handles rotated ROIs).
+        mask = np.zeros(img.shape[:2], dtype=np.uint8)
+        mask = cv.drawContours(
+            mask, [(points * sf).astype(np.int32)], 0, 1, cv.FILLED
+        )
+
+        # Gray out regions outside of ROI.
+        img[np.where(mask == 0)] = rgb_fill
+    else:
+        raise ValueError("Only polyline ROIs are supported.")
+
+    roi_h, roi_w = img.shape[:2]
+
+    # Convert the box elements into YOLO format.
+    labels_data = ""
+
+    for element in box_elements:
+        if element.get("group") in group_map:
+            if element.get("type") != "rectangle":
+                print(f"Skipping element: {element.get('type')} not supported.")
+                continue
+
+            group_id = group_map[element.get("group")]
+
+            # Get the center of box and shift to be relative to ROI.
+            x_center, y_center = (
+                np.array(element["center"])[:2] - [roi_x1, roi_y1]
+            ) * sf
+
+            # Normalize the coordinates.
+            x_center /= roi_w
+            y_center /= roi_h
+
+            # Scale the box width and height and then normalize.
+            width, height = (
+                element["width"] * sf / roi_w,
+                element["height"] * sf / roi_h,
+            )
+
+            labels_data += f"{group_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n"
+        else:
+            print(
+                f"Skipping element: group {element.get('group')} not found in group map."
+            )
+
+    # Return the ROI image and labels data.
+    return img, labels_data.strip()

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/dsa_helpers/ml/__init__.py

@@ -5,6 +5,7 @@ from . import (
     metrics,
     transforms,
     segformer_semantic_segmentation,
+    tiling,
 )
 
 # Modules that are imported from * notation.
@@ -14,4 +15,5 @@ __all__ = [
     "metrics",
     "transforms",
     "segformer_semantic_segmentation",
+    "tiling",
 ]

dsa_helpers-0.8/dsa_helpers/ml/tiling/__init__.py

@@ -0,0 +1,5 @@
+from .semantic_seg_tile_wsi_with_dsa_annotations import (
+    semantic_seg_tile_wsi_with_dsa_annotations,
+)
+
+__all__ = ["semantic_seg_tile_wsi_with_dsa_annotations"]

dsa_helpers-0.8/dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py

@@ -0,0 +1,225 @@
+import large_image
+import cv2 as cv
+import numpy as np
+from pathlib import Path
+from ...imwrite import imwrite
+from multiprocessing import Pool
+from tqdm import tqdm
+
+
+def _process_tile(
+    ts,
+    prepend_name,
+    x,
+    y,
+    scan_tile_size,
+    tile_size,
+    mag,
+    contours,
+    sf,
+    img_dir,
+    mask_dir,
+):
+    fn = f"{prepend_name}x{x}y{y}.png"
+
+    # Get the tile.
+    tile_img = ts.getRegion(
+        region={
+            "left": x,
+            "top": y,
+            "right": x + scan_tile_size,
+            "bottom": y + scan_tile_size,
+        },
+        format=large_image.constants.TILE_FORMAT_NUMPY,
+        scale={"magnification": mag},
+    )[0][:, :, :3]
+
+    if tile_img.shape[:2] != (tile_size, tile_size):
+        # Pad the image with zeroes to make it the desired size.
+        tile_img = cv.copyMakeBorder(
+            tile_img,
+            0,
+            tile_size - tile_img.shape[0],
+            0,
+            tile_size - tile_img.shape[1],
+            cv.BORDER_CONSTANT,
+            value=0,
+        )
+
+    # Create a blank tile mask.
+    tile_mask = np.zeros(tile_img.shape[:2], dtype=np.uint8)
+
+    # Loop through contours to draw on tile mask.
+    for idx, contour_dict in contours.items():
+        # Create a mask for the specific label.
+        label_mask = np.zeros(tile_mask.shape, dtype=np.uint8)
+
+        points = contour_dict["points"].copy()
+        holes = contour_dict["holes"].copy()
+
+        # For points and hole:
+        # (1) Shift by top left of the current tile.
+        # (2) Scale by be desired magnification.
+        points = [((point - [x, y]) * sf).astype(int) for point in points]
+
+        corrected_holes = []
+
+        for hole in holes:
+            if len(hole):
+                corrected_holes.append(((hole - [x, y]) * sf).astype(int))
+
+        # Use points to draw contours as positive.
+        label_mask = cv.drawContours(label_mask, points, -1, 1, cv.FILLED)
+
+        # Draw in the holes as background class zero.
+        label_mask = cv.drawContours(label_mask, holes, -1, 0, cv.FILLED)
+
+        # Apply the label to tile mask where label mask is positive.
+        tile_mask[label_mask == 1] = idx
+
+    # Save the image and mask.
+    img_fp = img_dir / fn
+    imwrite(img_fp, tile_img)
+    imwrite(mask_dir / fn, tile_mask, grayscale=True)
+
+    return str(img_fp), x, y
+
+
+def semantic_seg_tile_wsi_with_dsa_annotations(
+    wsi_fp: str,
+    annotation_docs: list[dict],
+    label2idx: dict,
+    save_dir: str,
+    tile_size: int,
+    mag: float | None = None,
+    prepend_name: str = "",
+    nproc: int = 1,
+) -> list[str]:
+    """Tile a WSI with semantic segmentation label masks created from
+    DSA annotations.
+
+    Args:
+        wsi_fp (str): file path to WSI.
+        annotation_docs (list[dict]): list of annotation documents.
+        label2idx (dict): mapping of label names to integer indices.
+        save_dir (str): directory to save the tiled images.
+        tile_size (int): size of the tile images at desired magnificaiton.
+        mag (float | None, optional): magnification level of the WSI.
+            If None, the function will use the default magnification
+            level. Defaults to None.
+        prepend_name (str, optional): prepend name to the created tile
+            images and masks. Defaults to "".
+        nproc (int, optional): number of processes to use for tiling.
+
+    Returns:
+        list[str]: A list of tuples: (tile file path, x, y coordinates
+            of tile).
+
+    """
+    # Format annotation documents for drawing on tile masks.
+    contours = {idx: {"points": [], "holes": []} for idx in label2idx.values()}
+
+    for ann_doc in annotation_docs:
+        # Loop through the elements of the annotation document.
+        elements = ann_doc["annotation"].get("elements", [])
+
+        for element in elements:
+            # Check if element has a label.
+            label = element.get("label", {}).get("value")
+
+            if label not in label2idx:
+                # Skip this element if it is not in the label2idx mapping.
+                continue
+
+            idx = label2idx[label]
+
+            element_type = element["type"]
+
+            if element_type == "polyline":
+                points = element.get("points", [])
+
+                if len(points):
+                    contours[idx]["points"].append(
+                        np.array(points, dtype=int)[:, :2]
+                    )
+
+                holes = element.get("holes", [])
+
+                if len(holes):
+                    for hole in holes:
+                        contours[idx]["holes"].append(
+                            np.array(hole, dtype=int)[:, :2]
+                        )
+            elif element_type == "rectangle":
+                # Convert the rectangle coordinates to polygon ones.
+                x1, y1 = element["center"][:2]
+                x2 = x1 + element["width"]
+                y2 = y1 + element["height"]
+
+                contours[idx]["points"].append(
+                    np.array(
+                        [[x1, y1], [x2, y1], [x2, y2], [x1, y2]], dtype=int
+                    )
+                )
+
+    # Create directory to save tile images and masks.
+    save_dir = Path(save_dir)
+    img_dir = save_dir / "images"
+    mask_dir = save_dir / "masks"
+    img_dir.mkdir(parents=True, exist_ok=True)
+    mask_dir.mkdir(exist_ok=True)
+
+    # Get the tile source of WSI.
+    ts = large_image.getTileSource(str(wsi_fp))
+
+    # Get the metadata of the WSI.
+    metadata = ts.getMetadata()
+
+    wsi_w, wsi_h = metadata["sizeX"], metadata["sizeY"]
+
+    # Get the magnification of the WSI.
+    wsi_mag = metadata["magnification"]
+
+    # Calculate the tile size at full resolution (1 if mag is None or the same as scan mag).
+    if mag is None:
+        scan_tile_size = tile_size
+        mag = wsi_mag
+        sf = 1
+    else:
+        sf = mag / wsi_mag  # scan mag -> desired mag
+        scan_tile_size = int(tile_size / sf)
+
+    # Get a list of x,y coordinates for the tile top left corners.
+    xys = [
+        (x, y)
+        for x in range(0, wsi_w, scan_tile_size)
+        for y in range(0, wsi_h, scan_tile_size)
+    ]
+
+    with Pool(nproc) as pool:
+        jobs = [
+            pool.apply_async(
+                _process_tile,
+                (
+                    ts,
+                    prepend_name,
+                    xy[0],
+                    xy[1],
+                    scan_tile_size,
+                    tile_size,
+                    mag,
+                    contours,
+                    sf,
+                    img_dir,
+                    mask_dir,
+                ),
+            )
+            for xy in xys
+        ]
+
+        tile_info = []
+
+        for job in tqdm(jobs, desc="Tiling..."):
+            tile_info.append(job.get())
+
+        return tile_info

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/dsa_helpers/mongo_utils.py

@@ -9,6 +9,7 @@ from typing import Union
 
 
 def get_mongo_client(
+    protocol: str | None = None,
     host: str | None = None,
     username: str | None = None,
     password: str | None = None,
@@ -20,6 +21,9 @@ def get_mongo_client(
     not present it sets them to default values.
 
     Args:
+        protocol (str, optional): The protocol of the mongo server. If None it
+            look for the environment variable "MONGO_PROTOCOL". If this is
+            missing it will default to "mongodb".
         host (str, optional): The host name of the mongo server. If None it will
             look for the environment variable "MONGO_HOST_NAME". If this is
             missing it will default to "mongodb".
@@ -39,6 +43,8 @@ def get_mongo_client(
         The mongo client.
 
     """
+    if protocol is None:
+        protocol = getenv("MONGO_PROTOCOL", "mongodb")
     if username is None:
         username = getenv("MONGO_INITDB_ROOT_USERNAME", "docker")
     if password is None:
@@ -48,7 +54,7 @@ def get_mongo_client(
     if port is None:
         port = int(getenv("MONGO_HOST_PORT", 27017))
 
-    return pymongo.MongoClient(f"mongodb://{username}:{password}@{host}:{port}")
+    return pymongo.MongoClient(f"{protocol}://{username}:{password}@{host}:{port}")
 
 
 def chunks(lst: list, n=500):

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8/dsa_helpers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dsa_helpers
-Version: 0.6.dev0
+Version: 0.8
 Summary: Utility functions for working with the DSA girder client.
 Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
 Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -26,6 +26,7 @@ Requires-Dist: transformers[torch]
 Requires-Dist: torch
 Requires-Dist: datasets
 Requires-Dist: torchvision
+Requires-Dist: large-image
 
 # Digital Slide Archive (DSA) Helpers
 Digital Slide Archive Python utility library.
@@ -36,6 +37,9 @@ This library was tested using Python version 3.11.8 and uses the dependencies de
 
 This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
 
+## Extra dependencies
+If you are getting a large image error, then install large image using the instructions provided in their [GitHub page](https://github.com/girder/large_image#:~:text=pip%20install%20large%2Dimage%5Ball%5D%20%2D%2Dfind%2Dlinks%20https%3A//girder.github.io/large_image_wheels).
+
 ## Instructions for Development
 1. Install requirements for building and distributing package:
 ```

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/dsa_helpers.egg-info/SOURCES.txt

@@ -32,6 +32,8 @@ dsa_helpers/ml/object_detection/tile_image.py
 dsa_helpers/ml/object_detection/utils.py
 dsa_helpers/ml/segformer_semantic_segmentation/__init__.py
 dsa_helpers/ml/segformer_semantic_segmentation/inference.py
+dsa_helpers/ml/tiling/__init__.py
+dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py
 dsa_helpers/ml/transforms/__init__.py
 dsa_helpers/ml/transforms/segformer_transforms.py
 tests/docker-compose.yaml

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/dsa_helpers.egg-info/requires.txt

@@ -13,3 +13,4 @@ transformers[torch]
 torch
 datasets
 torchvision
+large-image

{dsa_helpers-0.6.dev0 → dsa_helpers-0.8}/pyproject.toml

@@ -18,7 +18,8 @@ dependencies = [
     "transformers[torch]",
     "torch",
     "datasets",
-    "torchvision"
+    "torchvision",
+    "large-image"
 ]
 name = "dsa_helpers"
 authors = [
@@ -35,7 +36,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 keywords = ["dsa", "girder", "girder_client"]
-version = "v0.6dev"
+version = "v0.8"
 
 [project.urls]
 Repository = "https://github.com/Gutman-Lab/dsa-python-utils"