dsa-helpers 0.9.dev1__tar.gz → 1.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. dsa_helpers-1.dev0/.readthedocs.yaml +16 -0
  2. {dsa_helpers-0.9.dev1/dsa_helpers.egg-info → dsa_helpers-1.dev0}/PKG-INFO +1 -1
  3. dsa_helpers-1.dev0/docs/requirements.txt +16 -0
  4. dsa_helpers-1.dev0/docs/source/conf.py +42 -0
  5. dsa_helpers-1.dev0/docs/source/index.rst +18 -0
  6. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/__init__.py +0 -2
  7. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/header.py +14 -10
  8. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/girder_utils.py +26 -21
  9. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/__init__.py +2 -2
  10. dsa_helpers-1.dev0/dsa_helpers/ml/callbacks.py +55 -0
  11. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/__init__.py +1 -1
  12. dsa_helpers-1.dev0/dsa_helpers/ml/metrics.py +123 -0
  13. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/__init__.py +1 -1
  14. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/segformer_semantic_segmentation/__init__.py +1 -1
  15. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/segformer_semantic_segmentation/inference.py +4 -2
  16. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0/dsa_helpers.egg-info}/PKG-INFO +1 -1
  17. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/SOURCES.txt +5 -2
  18. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/pyproject.toml +1 -1
  19. dsa_helpers-0.9.dev1/dsa_helpers/ml/metrics.py +0 -117
  20. dsa_helpers-0.9.dev1/dsa_helpers/ml/tiling/__init__.py +0 -5
  21. dsa_helpers-0.9.dev1/dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py +0 -225
  22. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/.gitignore +0 -0
  23. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/LICENSE +0 -0
  24. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/README.md +0 -0
  25. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/app.py +0 -0
  26. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/__init__.py +0 -0
  27. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/dash_paperdragon_utils.py +0 -0
  28. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/header_callbacks.py +0 -0
  29. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/imread.py +0 -0
  30. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/imwrite.py +0 -0
  31. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py +0 -0
  32. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/utils.py +0 -0
  33. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/get_pearce_roi_images.py +0 -0
  34. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/tile_image.py +0 -0
  35. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/utils.py +0 -0
  36. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/transforms/__init__.py +0 -0
  37. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/transforms/segformer_transforms.py +0 -0
  38. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/mongo_utils.py +0 -0
  39. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/tile_utils.py +0 -0
  40. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/tiling.py +0 -0
  41. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/utils.py +0 -0
  42. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/dependency_links.txt +0 -0
  43. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/requires.txt +0 -0
  44. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/top_level.txt +0 -0
  45. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/setup.cfg +0 -0
  46. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/app/Dockerfile +0 -0
  47. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/app/app.py +0 -0
  48. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/docker-compose.yaml +0 -0
  49. {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/test.ipynb +0 -0
@@ -0,0 +1,16 @@
1
+ version: 2
2
+
3
+ build:
4
+ os: "ubuntu-24.04"
5
+ tools:
6
+ python: "3.12"
7
+ jobs:
8
+ pre_build:
9
+ - sphinx-apidoc -o docs/source dsa_helpers/
10
+
11
+ python:
12
+ install:
13
+ - requirements: docs/requirements.txt
14
+
15
+ sphinx:
16
+ configuration: docs/source/conf.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dsa_helpers
3
- Version: 0.9.dev1
3
+ Version: 1.dev0
4
4
  Summary: Utility functions for working with the DSA girder client.
5
5
  Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
6
6
  Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -0,0 +1,16 @@
1
+ girder-client
2
+ numpy
3
+ setuptools
4
+ opencv-python
5
+ pandas
6
+ pymongo
7
+ pillow
8
+ dash
9
+ dash-bootstrap-components
10
+ shapely
11
+ geopandas
12
+ transformers[torch]
13
+ datasets
14
+ torchvision
15
+ large-image
16
+ sphinx_rtd_theme
@@ -0,0 +1,42 @@
1
+ # Configuration file for the Sphinx documentation builder.
2
+ #
3
+ # For the full list of built-in configuration values, see the documentation:
4
+ # https://www.sphinx-doc.org/en/master/usage/configuration.html
5
+
6
+ # -- Path setup --------------------------------------------------------------
7
+
8
+ # If extensions (or modules to document with autodoc) are in another directory,
9
+ # add these directories to sys.path here. If the directory is relative to the
10
+ # documentation root, use os.path.abspath to make it absolute, like shown here.
11
+ #
12
+ import os
13
+ import sys
14
+
15
+ sys.path.insert(0, os.path.abspath("../.."))
16
+
17
+ # -- Project information -----------------------------------------------------
18
+ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
19
+
20
+ project = "DSA Helpers"
21
+ copyright = "2024, Juan Carlos Vizcarra"
22
+ author = "Juan Carlos Vizcarra"
23
+ release = "00.00.01"
24
+
25
+ # -- General configuration ---------------------------------------------------
26
+ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
27
+
28
+ extensions = [
29
+ "sphinx.ext.autodoc",
30
+ "sphinx.ext.viewcode",
31
+ "sphinx.ext.napoleon",
32
+ ]
33
+
34
+ templates_path = ["_templates"]
35
+ exclude_patterns = []
36
+
37
+
38
+ # -- Options for HTML output -------------------------------------------------
39
+ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
40
+
41
+ html_theme = "sphinx_rtd_theme"
42
+ html_static_path = ["_static"]
@@ -0,0 +1,18 @@
1
+ .. DSA Helpers documentation master file, created by
2
+ sphinx-quickstart on Fri Dec 6 12:07:31 2024.
3
+ You can adapt this file completely to your liking, but it should at least
4
+ contain the root `toctree` directive.
5
+
6
+ DSA Helpers documentation
7
+ =========================
8
+
9
+ Add your content using ``reStructuredText`` syntax. See the
10
+ `reStructuredText <https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html>`_
11
+ documentation for details.
12
+
13
+
14
+ .. toctree::
15
+ :maxdepth: 2
16
+ :caption: Contents:
17
+
18
+ modules
@@ -16,8 +16,6 @@ __all__ = [
16
16
  "girder_utils",
17
17
  "dash",
18
18
  "ml",
19
- "imread",
20
- "imwrite",
21
19
  "mongo_utils",
22
20
  "tiling",
23
21
  "utils",
@@ -15,13 +15,8 @@ def get_header(
15
15
  Args:
16
16
  dsa_api_url (str): The URL of the DSA API.
17
17
  title (str, optional): The title of the application. Defaults to "App".
18
- config (dict, optional): The configuration of the component. Defaults to None.
19
- If None the default configuration is:
20
- {
21
- "backgroundColor": "#012169",
22
- "titleColor": "#d9d9d6",
23
- "fontColor": "#f2a900",
24
- }
18
+ config (dict, optional): The configuration of the component.
19
+ Defaults to None which uses default settings.
25
20
 
26
21
  Returns:
27
22
  html.Div: The header component.
@@ -38,7 +33,8 @@ def get_header(
38
33
  dbc.ModalBody(
39
34
  [
40
35
  html.Div(
41
- "Login or email", style={"margin": 5, "fontWeight": "bold"}
36
+ "Login or email",
37
+ style={"margin": 5, "fontWeight": "bold"},
42
38
  ),
43
39
  dbc.Input(
44
40
  id=f"{store_id}=login",
@@ -48,7 +44,11 @@ def get_header(
48
44
  ),
49
45
  html.Div(
50
46
  "Password",
51
- style={"margin": 5, "marginTop": 15, "fontWeight": "bold"},
47
+ style={
48
+ "margin": 5,
49
+ "marginTop": 15,
50
+ "fontWeight": "bold",
51
+ },
52
52
  ),
53
53
  dbc.Input(
54
54
  id=f"{store_id}=password",
@@ -60,7 +60,11 @@ def get_header(
60
60
  "Login failed.",
61
61
  hidden=True,
62
62
  id=f"{store_id}=login-failed",
63
- style={"color": "red", "fontWeight": "bold", "margin": 10},
63
+ style={
64
+ "color": "red",
65
+ "fontWeight": "bold",
66
+ "margin": 10,
67
+ },
64
68
  ),
65
69
  ],
66
70
  ),
@@ -88,19 +88,19 @@ def get_region(
88
88
  height: int,
89
89
  mag: float | None = None,
90
90
  ) -> np.ndarray:
91
- """Get a region of the image for an item. Note that the output image might
92
- not be in the shape (width, height) if left + width or top + height exceeds
93
- the image size. The outpuzerositem id.
91
+ """Get image region from DSA. Note that the output image might not
92
+ be in the shape (width, height) if left + width or top + height
93
+ exceeds the image size.
94
+
95
+ Args:
96
+ gc (girder_client.GirderClient): The authenticated girder client.
97
+ item_id (str): The item id.
94
98
  left (int): The left coordinate.
95
99
  top (int): The top coordinate.
96
100
  width (int): The width of the region.
97
101
  height (int): The height of the region.
98
- mag (float, optional): The magnification. Defaults to None which returns
99
- the image at scan magnification. Using a mag lower than the scan
100
- magnification will result in an ouptut image smaller than the
101
- width and height. Similarly, using a mag higher than the scan
102
- magnification will result in an output image larger than the width
103
- and height.
102
+ mag (float, optional): The magnification. Defaults to None which
103
+ returns the image at the scan magnification.
104
104
 
105
105
  Returns:
106
106
  np.ndarray: The region of the image.
@@ -258,17 +258,16 @@ def _rotate_point_list(point_list, rotation, center=(0, 0)):
258
258
  return point_list_rotated
259
259
 
260
260
 
261
- def get_rectangle_element_coords(element):
261
+ def get_rectangle_element_coords(element: dict) -> np.ndarray:
262
262
  """Get the corner coordinate from a rectangle HistomicsUI element, can handle rotated elements.
263
263
  Adapted from: https://github.com/DigitalSlideArchive/HistomicsTK/blob/master/histomicstk/annotations_and_masks/annotation_and_mask_utils.py
264
- INPUTS
265
- ------
266
- element : dict
267
- rectangle element, in HistomicsUI format
268
- RETURN
269
- ------
270
- corner_coords : array
271
- array of shape [4, 2] for the four corners of the rectangle in (x, y) format
264
+
265
+ Args:
266
+ element (dict): rectangle element from HistomicsUI annotation.
267
+
268
+ Returns:
269
+ corner_coords (np.ndarray): 4x2 array of corner coordinates of rectangle.
270
+
272
271
  """
273
272
  # element is a dict so prevent referencing
274
273
  element = deepcopy(element)
@@ -401,7 +400,9 @@ def get_roi_with_yolo_labels_from_single_doc(
401
400
  if element.get("group") == roi_element_group:
402
401
  # Append the ROI element, there can't more than one.
403
402
  if roi_element is not None:
404
- raise ValueError("Multiple ROI elements found in the document.")
403
+ raise ValueError(
404
+ "Multiple ROI elements found in the document."
405
+ )
405
406
 
406
407
  roi_element = element
407
408
  elif element.get("group") in group_map:
@@ -460,7 +461,9 @@ def get_roi_with_yolo_labels_from_single_doc(
460
461
  for element in box_elements:
461
462
  if element.get("group") in group_map:
462
463
  if element.get("type") != "rectangle":
463
- print(f"Skipping element: {element.get('type')} not supported.")
464
+ print(
465
+ f"Skipping element: {element.get('type')} not supported."
466
+ )
464
467
  continue
465
468
 
466
469
  group_id = group_map[element.get("group")]
@@ -523,7 +526,9 @@ def post_annotations_from_gdf(
523
526
  # Process the dataframe.
524
527
  for _, row in gdf.iterrows():
525
528
  # Flatten multipolygon to list of polygons.
526
- list_of_polygons = list(row["geometry"].geoms) # assuming multipolygons
529
+ list_of_polygons = list(
530
+ row["geometry"].geoms
531
+ ) # assuming multipolygons
527
532
  label = row["label"]
528
533
 
529
534
  for poly in list_of_polygons:
@@ -4,7 +4,7 @@ from . import (
4
4
  datasets,
5
5
  transforms,
6
6
  segformer_semantic_segmentation,
7
- tiling,
7
+ callbacks,
8
8
  )
9
9
 
10
10
  # Modules that are imported from * notation.
@@ -14,5 +14,5 @@ __all__ = [
14
14
  "metrics",
15
15
  "transforms",
16
16
  "segformer_semantic_segmentation",
17
- "tiling",
17
+ "callbacks",
18
18
  ]
@@ -0,0 +1,55 @@
1
+ """Provides custom callbacks for the Hugging Face Trainer class."""
2
+
3
+ from transformers import TrainerCallback
4
+ from pathlib import Path
5
+ import pandas as pd
6
+
7
+
8
+ class MetricsLoggerCallback(TrainerCallback):
9
+ def _load_current_file(self, output_dir):
10
+ output_dir = Path(output_dir)
11
+ metrics_file = output_dir / "metrics.csv"
12
+
13
+ return (
14
+ pd.read_csv(metrics_file)
15
+ if metrics_file.exists()
16
+ else pd.DataFrame()
17
+ )
18
+
19
+ def on_log(self, args, state, control, logs=None, **kwargs):
20
+ """
21
+ Triggered during training logging (e.g., for training loss).
22
+ """
23
+ if "loss" in logs: # Ensure training loss is available
24
+ metrics_df = self._load_current_file(args.output_dir)
25
+
26
+ # Append the epoch and the training loss.
27
+ loss = logs["loss"]
28
+ epoch = state.epoch
29
+
30
+ metrics_df = pd.concat(
31
+ [
32
+ metrics_df,
33
+ pd.DataFrame([{"epoch": epoch, "train_loss": loss}]),
34
+ ],
35
+ ignore_index=True,
36
+ )
37
+
38
+ # Save it back to file.
39
+ metrics_df.to_csv(
40
+ Path(args.output_dir) / "metrics.csv", index=False
41
+ )
42
+
43
+ def on_evaluate(self, args, state, control, metrics, **kwargs):
44
+ """
45
+ This is triggered at the end of evaluation (including at the end of each epoch).
46
+ """
47
+ # Get the current metrics file.
48
+ metrics_df = self._load_current_file(args.output_dir)
49
+
50
+ # Add metrics to the last row of the current metrics file.
51
+ for k, v in metrics.items():
52
+ metrics_df.at[metrics_df.index[-1], k] = v
53
+
54
+ # Save the updated metrics file.
55
+ metrics_df.to_csv(Path(args.output_dir) / "metrics.csv", index=False)
@@ -1,3 +1,3 @@
1
1
  from .SegFormerSegmentationDataset import SegFormerSegmentationDataset
2
2
 
3
- __all__ = ["SegFormerSegmentationDataset", "utils"]
3
+ __all__ = ["utils"]
@@ -0,0 +1,123 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Metric functions.
3
+
4
+ This module contains functions used to evaluate the performance of
5
+ machine learning models.
6
+
7
+ """
8
+ import torch
9
+ from torch import nn
10
+ import numpy as np
11
+
12
+
13
+ def binary_dice_coefficient(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
14
+ """Calculate the dice coefficient for binary masks. Developed for use
15
+ in compute_metrics parameters of transformers Trainer class, for
16
+ SegFormer senantic segmentation model of two classes.
17
+
18
+ Args:
19
+ eval_pred (tuple[np.ndarray, np.ndarray]): A tuple containing the
20
+ predicted and true masks.
21
+
22
+ Returns:
23
+ dict: A dictionary containing the dice coefficient.
24
+
25
+ """
26
+ with torch.no_grad():
27
+ logits, labels = eval_pred
28
+
29
+ # Convert to tensors.
30
+ logits = torch.from_numpy(logits).cpu()
31
+
32
+ # Resize the logits to be the same shape as the labels.
33
+ logits = nn.functional.interpolate(
34
+ logits,
35
+ size=labels.shape[-2:],
36
+ mode="bilinear",
37
+ align_corners=False,
38
+ ).argmax(dim=1)
39
+
40
+ preds = logits.detach().numpy()
41
+
42
+ # Convert the masks to binary.
43
+ preds = preds > 0
44
+ labels = labels > 0
45
+
46
+ # Calculate dice.
47
+ intersection = np.sum(preds * labels)
48
+ union = np.sum(preds) + np.sum(labels)
49
+
50
+ if union == 0:
51
+ dice = 1.0 # Both masks are empty
52
+ else:
53
+ dice = 2.0 * intersection / union
54
+
55
+ return {"dice_coefficient": dice}
56
+
57
+
58
+ def mean_iou(label2idx: dict):
59
+ """Return the compute metric function for mean intersection over
60
+ union for a multi-class semantic segmentation task.
61
+
62
+ Args:
63
+ label2idx (dict): A dictionary mapping class labels to indices.
64
+
65
+ Returns:
66
+ function: The compute_metrics function.
67
+
68
+ """
69
+ idx2label = {v: k for k, v in label2idx.items()}
70
+
71
+ def compute_metrics(eval_pred):
72
+ """Compute the mean intersection over union (IoU) for semantic
73
+ segmentation. Also returns the IoU for each class."""
74
+ with torch.no_grad():
75
+ logits, labels = eval_pred
76
+
77
+ # Convert logits to tensor.
78
+ logits_tensor = torch.from_numpy(logits).cpu()
79
+ labels_tensor = torch.from_numpy(labels).cpu()
80
+
81
+ # From logits get the number of classes in the dataset.
82
+ num_classes = logits.shape[1]
83
+
84
+ # Scale the logits back to the shape of the labels.
85
+ logits_tensor = nn.functional.interpolate(
86
+ logits_tensor,
87
+ size=labels.shape[-2:],
88
+ mode="bilinear",
89
+ align_corners=False,
90
+ )
91
+
92
+ # Turn the logits to predictions.
93
+ pred = logits_tensor.argmax(dim=1)
94
+
95
+ # Calculate the IoU for each class.
96
+ ious = []
97
+
98
+ metrics = {}
99
+
100
+ for cls in range(num_classes):
101
+ pred_mask = pred == cls
102
+ labels_mask = labels_tensor == cls
103
+
104
+ intersection = (
105
+ torch.logical_and(pred_mask, labels_mask).sum().item()
106
+ )
107
+ union = torch.logical_or(pred_mask, labels_mask).sum().item()
108
+
109
+ if union == 0:
110
+ iou = float("nan") # or 1.0
111
+ else:
112
+ iou = intersection / union
113
+
114
+ metrics[f"{idx2label[cls]}_iou"] = iou
115
+
116
+ ious.append(iou)
117
+
118
+ mean_iou = np.nanmean(ious) # Use NumPy for NaN handling
119
+ metrics["mean_iou"] = mean_iou
120
+
121
+ return metrics
122
+
123
+ return compute_metrics
@@ -2,4 +2,4 @@
2
2
  from .get_pearce_roi_images import get_pearce_roi_images
3
3
  from .tile_image import tile_image
4
4
 
5
- __all__ = ["utils", "get_pearce_roi_images", "tile_image"]
5
+ __all__ = ["utils"]
@@ -1,3 +1,3 @@
1
1
  from .inference import inference
2
2
 
3
- __all__ = ["inference"]
3
+ __all__ = []
@@ -10,7 +10,7 @@ import torch
10
10
  import torch.nn as nn
11
11
 
12
12
  from ... import imread
13
- from ...image_utils import tile_image
13
+ from ...tiling import tile_image
14
14
 
15
15
 
16
16
  def inference(
@@ -115,7 +115,9 @@ def inference(
115
115
  predicted_mask = predicted_mask[:h, :w]
116
116
 
117
117
  # Extract the contours of the mask.
118
- contours = cv.findContours(predicted_mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)[0]
118
+ contours = cv.findContours(
119
+ predicted_mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE
120
+ )[0]
119
121
 
120
122
  # Return the mask.
121
123
  return predicted_mask, contours
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dsa_helpers
3
- Version: 0.9.dev1
3
+ Version: 1.dev0
4
4
  Summary: Utility functions for working with the DSA girder client.
5
5
  Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
6
6
  Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
@@ -1,8 +1,12 @@
1
1
  .gitignore
2
+ .readthedocs.yaml
2
3
  LICENSE
3
4
  README.md
4
5
  app.py
5
6
  pyproject.toml
7
+ docs/requirements.txt
8
+ docs/source/conf.py
9
+ docs/source/index.rst
6
10
  dsa_helpers/__init__.py
7
11
  dsa_helpers/girder_utils.py
8
12
  dsa_helpers/imread.py
@@ -21,6 +25,7 @@ dsa_helpers/dash/dash_paperdragon_utils.py
21
25
  dsa_helpers/dash/header.py
22
26
  dsa_helpers/dash/header_callbacks.py
23
27
  dsa_helpers/ml/__init__.py
28
+ dsa_helpers/ml/callbacks.py
24
29
  dsa_helpers/ml/metrics.py
25
30
  dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py
26
31
  dsa_helpers/ml/datasets/__init__.py
@@ -31,8 +36,6 @@ dsa_helpers/ml/object_detection/tile_image.py
31
36
  dsa_helpers/ml/object_detection/utils.py
32
37
  dsa_helpers/ml/segformer_semantic_segmentation/__init__.py
33
38
  dsa_helpers/ml/segformer_semantic_segmentation/inference.py
34
- dsa_helpers/ml/tiling/__init__.py
35
- dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py
36
39
  dsa_helpers/ml/transforms/__init__.py
37
40
  dsa_helpers/ml/transforms/segformer_transforms.py
38
41
  tests/docker-compose.yaml
@@ -35,7 +35,7 @@ classifiers = [
35
35
  "License :: OSI Approved :: MIT License",
36
36
  ]
37
37
  keywords = ["dsa", "girder", "girder_client"]
38
- version = "v0.9dev1"
38
+ version = "v1dev0"
39
39
 
40
40
  [project.urls]
41
41
  Repository = "https://github.com/Gutman-Lab/dsa-python-utils"
@@ -1,117 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- """Metric functions.
3
-
4
- This module contains functions used to evaluate the performance of
5
- machine learning models.
6
-
7
- """
8
- import torch
9
- from torch import nn
10
- import numpy as np
11
-
12
-
13
- def binary_dice_coefficient(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
14
- """Calculate the dice coefficient for binary masks. Developed for use
15
- in compute_metrics parameters of transformers Trainer class, for
16
- SegFormer senantic segmentation model of two classes.
17
-
18
- Args:
19
- eval_pred (tuple[np.ndarray, np.ndarray]): A tuple containing the
20
- predicted and true masks.
21
-
22
- Returns:
23
- dict: A dictionary containing the dice coefficient.
24
-
25
- """
26
- with torch.no_grad():
27
- logits, labels = eval_pred
28
-
29
- # Convert to tensors.
30
- logits = torch.from_numpy(logits).cpu()
31
-
32
- # Resize the logits to be the same shape as the labels.
33
- logits = nn.functional.interpolate(
34
- logits,
35
- size=labels.shape[-2:],
36
- mode="bilinear",
37
- align_corners=False,
38
- ).argmax(dim=1)
39
-
40
- preds = logits.detach().numpy()
41
-
42
- # Convert the masks to binary.
43
- preds = preds > 0
44
- labels = labels > 0
45
-
46
- # Calculate dice.
47
- intersection = np.sum(preds * labels)
48
- union = np.sum(preds) + np.sum(labels)
49
-
50
- if union == 0:
51
- dice = 1.0 # Both masks are empty
52
- else:
53
- dice = 2.0 * intersection / union
54
-
55
- return {"dice_coefficient": dice}
56
-
57
-
58
- def mean_iou(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
59
- """Compute the mean intersection over union (IoU) for semantic
60
- segmentation. Also returns the IoU for each class.
61
-
62
- Args:
63
- eval_pred (tuple[numpy.ndarray, numpy.ndarray]): Tuple of
64
- logits and labels.
65
-
66
- Returns:
67
- dict: Dictionary of metrics.
68
-
69
- """
70
- with torch.no_grad():
71
- logits, labels = eval_pred
72
-
73
- # Convert logits to tensor.
74
- logits_tensor = torch.from_numpy(logits).cpu()
75
- labels_tensor = torch.from_numpy(labels).cpu()
76
-
77
- # From logits get the number of classes in the dataset.
78
- num_classes = logits.shape[1]
79
-
80
- # Scale the logits back to the shape of the labels.
81
- logits_tensor = nn.functional.interpolate(
82
- logits_tensor,
83
- size=labels.shape[-2:],
84
- mode="bilinear",
85
- align_corners=False,
86
- )
87
-
88
- # Turn the logits to predictions.
89
- pred = logits_tensor.argmax(dim=1)
90
-
91
- # Calculate the IoU for each class.
92
- ious = []
93
-
94
- metrics = {}
95
-
96
- for cls in range(num_classes):
97
- pred_mask = pred == cls
98
- labels_mask = labels_tensor == cls
99
-
100
- intersection = (
101
- torch.logical_and(pred_mask, labels_mask).sum().item()
102
- )
103
- union = torch.logical_or(pred_mask, labels_mask).sum().item()
104
-
105
- if union == 0:
106
- iou = float("nan") # or 1.0
107
- else:
108
- iou = intersection / union
109
-
110
- metrics[f"label {cls} IoU"] = iou
111
-
112
- ious.append(iou)
113
-
114
- mean_iou = np.nanmean(ious) # Use NumPy for NaN handling
115
- metrics["mean_iou"] = mean_iou
116
-
117
- return metrics
@@ -1,5 +0,0 @@
1
- from .semantic_seg_tile_wsi_with_dsa_annotations import (
2
- semantic_seg_tile_wsi_with_dsa_annotations,
3
- )
4
-
5
- __all__ = ["semantic_seg_tile_wsi_with_dsa_annotations"]
@@ -1,225 +0,0 @@
1
- import large_image
2
- import cv2 as cv
3
- import numpy as np
4
- from pathlib import Path
5
- from ...imwrite import imwrite
6
- from multiprocessing import Pool
7
- from tqdm import tqdm
8
-
9
-
10
- def _process_tile(
11
- ts,
12
- prepend_name,
13
- x,
14
- y,
15
- scan_tile_size,
16
- tile_size,
17
- mag,
18
- contours,
19
- sf,
20
- img_dir,
21
- mask_dir,
22
- ):
23
- fn = f"{prepend_name}x{x}y{y}.png"
24
-
25
- # Get the tile.
26
- tile_img = ts.getRegion(
27
- region={
28
- "left": x,
29
- "top": y,
30
- "right": x + scan_tile_size,
31
- "bottom": y + scan_tile_size,
32
- },
33
- format=large_image.constants.TILE_FORMAT_NUMPY,
34
- scale={"magnification": mag},
35
- )[0][:, :, :3]
36
-
37
- if tile_img.shape[:2] != (tile_size, tile_size):
38
- # Pad the image with zeroes to make it the desired size.
39
- tile_img = cv.copyMakeBorder(
40
- tile_img,
41
- 0,
42
- tile_size - tile_img.shape[0],
43
- 0,
44
- tile_size - tile_img.shape[1],
45
- cv.BORDER_CONSTANT,
46
- value=0,
47
- )
48
-
49
- # Create a blank tile mask.
50
- tile_mask = np.zeros(tile_img.shape[:2], dtype=np.uint8)
51
-
52
- # Loop through contours to draw on tile mask.
53
- for idx, contour_dict in contours.items():
54
- # Create a mask for the specific label.
55
- label_mask = np.zeros(tile_mask.shape, dtype=np.uint8)
56
-
57
- points = contour_dict["points"].copy()
58
- holes = contour_dict["holes"].copy()
59
-
60
- # For points and hole:
61
- # (1) Shift by top left of the current tile.
62
- # (2) Scale by be desired magnification.
63
- points = [((point - [x, y]) * sf).astype(int) for point in points]
64
-
65
- corrected_holes = []
66
-
67
- for hole in holes:
68
- if len(hole):
69
- corrected_holes.append(((hole - [x, y]) * sf).astype(int))
70
-
71
- # Use points to draw contours as positive.
72
- label_mask = cv.drawContours(label_mask, points, -1, 1, cv.FILLED)
73
-
74
- # Draw in the holes as background class zero.
75
- label_mask = cv.drawContours(label_mask, holes, -1, 0, cv.FILLED)
76
-
77
- # Apply the label to tile mask where label mask is positive.
78
- tile_mask[label_mask == 1] = idx
79
-
80
- # Save the image and mask.
81
- img_fp = img_dir / fn
82
- imwrite(img_fp, tile_img)
83
- imwrite(mask_dir / fn, tile_mask, grayscale=True)
84
-
85
- return str(img_fp), x, y
86
-
87
-
88
- def semantic_seg_tile_wsi_with_dsa_annotations(
89
- wsi_fp: str,
90
- annotation_docs: list[dict],
91
- label2idx: dict,
92
- save_dir: str,
93
- tile_size: int,
94
- mag: float | None = None,
95
- prepend_name: str = "",
96
- nproc: int = 1,
97
- ) -> list[str]:
98
- """Tile a WSI with semantic segmentation label masks created from
99
- DSA annotations.
100
-
101
- Args:
102
- wsi_fp (str): file path to WSI.
103
- annotation_docs (list[dict]): list of annotation documents.
104
- label2idx (dict): mapping of label names to integer indices.
105
- save_dir (str): directory to save the tiled images.
106
- tile_size (int): size of the tile images at desired magnificaiton.
107
- mag (float | None, optional): magnification level of the WSI.
108
- If None, the function will use the default magnification
109
- level. Defaults to None.
110
- prepend_name (str, optional): prepend name to the created tile
111
- images and masks. Defaults to "".
112
- nproc (int, optional): number of processes to use for tiling.
113
-
114
- Returns:
115
- list[str]: A list of tuples: (tile file path, x, y coordinates
116
- of tile).
117
-
118
- """
119
- # Format annotation documents for drawing on tile masks.
120
- contours = {idx: {"points": [], "holes": []} for idx in label2idx.values()}
121
-
122
- for ann_doc in annotation_docs:
123
- # Loop through the elements of the annotation document.
124
- elements = ann_doc["annotation"].get("elements", [])
125
-
126
- for element in elements:
127
- # Check if element has a label.
128
- label = element.get("label", {}).get("value")
129
-
130
- if label not in label2idx:
131
- # Skip this element if it is not in the label2idx mapping.
132
- continue
133
-
134
- idx = label2idx[label]
135
-
136
- element_type = element["type"]
137
-
138
- if element_type == "polyline":
139
- points = element.get("points", [])
140
-
141
- if len(points):
142
- contours[idx]["points"].append(
143
- np.array(points, dtype=int)[:, :2]
144
- )
145
-
146
- holes = element.get("holes", [])
147
-
148
- if len(holes):
149
- for hole in holes:
150
- contours[idx]["holes"].append(
151
- np.array(hole, dtype=int)[:, :2]
152
- )
153
- elif element_type == "rectangle":
154
- # Convert the rectangle coordinates to polygon ones.
155
- x1, y1 = element["center"][:2]
156
- x2 = x1 + element["width"]
157
- y2 = y1 + element["height"]
158
-
159
- contours[idx]["points"].append(
160
- np.array(
161
- [[x1, y1], [x2, y1], [x2, y2], [x1, y2]], dtype=int
162
- )
163
- )
164
-
165
- # Create directory to save tile images and masks.
166
- save_dir = Path(save_dir)
167
- img_dir = save_dir / "images"
168
- mask_dir = save_dir / "masks"
169
- img_dir.mkdir(parents=True, exist_ok=True)
170
- mask_dir.mkdir(exist_ok=True)
171
-
172
- # Get the tile source of WSI.
173
- ts = large_image.getTileSource(str(wsi_fp))
174
-
175
- # Get the metadata of the WSI.
176
- metadata = ts.getMetadata()
177
-
178
- wsi_w, wsi_h = metadata["sizeX"], metadata["sizeY"]
179
-
180
- # Get the magnification of the WSI.
181
- wsi_mag = metadata["magnification"]
182
-
183
- # Calculate the tile size at full resolution (1 if mag is None or the same as scan mag).
184
- if mag is None:
185
- scan_tile_size = tile_size
186
- mag = wsi_mag
187
- sf = 1
188
- else:
189
- sf = mag / wsi_mag # scan mag -> desired mag
190
- scan_tile_size = int(tile_size / sf)
191
-
192
- # Get a list of x,y coordinates for the tile top left corners.
193
- xys = [
194
- (x, y)
195
- for x in range(0, wsi_w, scan_tile_size)
196
- for y in range(0, wsi_h, scan_tile_size)
197
- ]
198
-
199
- with Pool(nproc) as pool:
200
- jobs = [
201
- pool.apply_async(
202
- _process_tile,
203
- (
204
- ts,
205
- prepend_name,
206
- xy[0],
207
- xy[1],
208
- scan_tile_size,
209
- tile_size,
210
- mag,
211
- contours,
212
- sf,
213
- img_dir,
214
- mask_dir,
215
- ),
216
- )
217
- for xy in xys
218
- ]
219
-
220
- tile_info = []
221
-
222
- for job in tqdm(jobs, desc="Tiling..."):
223
- tile_info.append(job.get())
224
-
225
- return tile_info
File without changes
File without changes
File without changes
File without changes
File without changes