dsa-helpers 0.9.dev1__tar.gz → 1.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsa_helpers-1.dev0/.readthedocs.yaml +16 -0
- {dsa_helpers-0.9.dev1/dsa_helpers.egg-info → dsa_helpers-1.dev0}/PKG-INFO +1 -1
- dsa_helpers-1.dev0/docs/requirements.txt +16 -0
- dsa_helpers-1.dev0/docs/source/conf.py +42 -0
- dsa_helpers-1.dev0/docs/source/index.rst +18 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/__init__.py +0 -2
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/header.py +14 -10
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/girder_utils.py +26 -21
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/__init__.py +2 -2
- dsa_helpers-1.dev0/dsa_helpers/ml/callbacks.py +55 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/__init__.py +1 -1
- dsa_helpers-1.dev0/dsa_helpers/ml/metrics.py +123 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/__init__.py +1 -1
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/segformer_semantic_segmentation/__init__.py +1 -1
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/segformer_semantic_segmentation/inference.py +4 -2
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0/dsa_helpers.egg-info}/PKG-INFO +1 -1
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/SOURCES.txt +5 -2
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/pyproject.toml +1 -1
- dsa_helpers-0.9.dev1/dsa_helpers/ml/metrics.py +0 -117
- dsa_helpers-0.9.dev1/dsa_helpers/ml/tiling/__init__.py +0 -5
- dsa_helpers-0.9.dev1/dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py +0 -225
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/.gitignore +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/LICENSE +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/README.md +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/app.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/__init__.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/dash_paperdragon_utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/dash/header_callbacks.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/imread.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/imwrite.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/get_pearce_roi_images.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/tile_image.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/transforms/__init__.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/transforms/segformer_transforms.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/mongo_utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/tile_utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/tiling.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/utils.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/dependency_links.txt +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/requires.txt +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers.egg-info/top_level.txt +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/setup.cfg +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/app/Dockerfile +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/app/app.py +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/docker-compose.yaml +0 -0
- {dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/tests/test.ipynb +0 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
|
|
3
|
+
build:
|
|
4
|
+
os: "ubuntu-24.04"
|
|
5
|
+
tools:
|
|
6
|
+
python: "3.12"
|
|
7
|
+
jobs:
|
|
8
|
+
pre_build:
|
|
9
|
+
- sphinx-apidoc -o docs/source dsa_helpers/
|
|
10
|
+
|
|
11
|
+
python:
|
|
12
|
+
install:
|
|
13
|
+
- requirements: docs/requirements.txt
|
|
14
|
+
|
|
15
|
+
sphinx:
|
|
16
|
+
configuration: docs/source/conf.py
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Configuration file for the Sphinx documentation builder.
|
|
2
|
+
#
|
|
3
|
+
# For the full list of built-in configuration values, see the documentation:
|
|
4
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
|
5
|
+
|
|
6
|
+
# -- Path setup --------------------------------------------------------------
|
|
7
|
+
|
|
8
|
+
# If extensions (or modules to document with autodoc) are in another directory,
|
|
9
|
+
# add these directories to sys.path here. If the directory is relative to the
|
|
10
|
+
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
|
11
|
+
#
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, os.path.abspath("../.."))
|
|
16
|
+
|
|
17
|
+
# -- Project information -----------------------------------------------------
|
|
18
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
|
19
|
+
|
|
20
|
+
project = "DSA Helpers"
|
|
21
|
+
copyright = "2024, Juan Carlos Vizcarra"
|
|
22
|
+
author = "Juan Carlos Vizcarra"
|
|
23
|
+
release = "00.00.01"
|
|
24
|
+
|
|
25
|
+
# -- General configuration ---------------------------------------------------
|
|
26
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
|
27
|
+
|
|
28
|
+
extensions = [
|
|
29
|
+
"sphinx.ext.autodoc",
|
|
30
|
+
"sphinx.ext.viewcode",
|
|
31
|
+
"sphinx.ext.napoleon",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
templates_path = ["_templates"]
|
|
35
|
+
exclude_patterns = []
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# -- Options for HTML output -------------------------------------------------
|
|
39
|
+
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
|
40
|
+
|
|
41
|
+
html_theme = "sphinx_rtd_theme"
|
|
42
|
+
html_static_path = ["_static"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
.. DSA Helpers documentation master file, created by
|
|
2
|
+
sphinx-quickstart on Fri Dec 6 12:07:31 2024.
|
|
3
|
+
You can adapt this file completely to your liking, but it should at least
|
|
4
|
+
contain the root `toctree` directive.
|
|
5
|
+
|
|
6
|
+
DSA Helpers documentation
|
|
7
|
+
=========================
|
|
8
|
+
|
|
9
|
+
Add your content using ``reStructuredText`` syntax. See the
|
|
10
|
+
`reStructuredText <https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html>`_
|
|
11
|
+
documentation for details.
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
.. toctree::
|
|
15
|
+
:maxdepth: 2
|
|
16
|
+
:caption: Contents:
|
|
17
|
+
|
|
18
|
+
modules
|
|
@@ -15,13 +15,8 @@ def get_header(
|
|
|
15
15
|
Args:
|
|
16
16
|
dsa_api_url (str): The URL of the DSA API.
|
|
17
17
|
title (str, optional): The title of the application. Defaults to "App".
|
|
18
|
-
config (dict, optional): The configuration of the component.
|
|
19
|
-
|
|
20
|
-
{
|
|
21
|
-
"backgroundColor": "#012169",
|
|
22
|
-
"titleColor": "#d9d9d6",
|
|
23
|
-
"fontColor": "#f2a900",
|
|
24
|
-
}
|
|
18
|
+
config (dict, optional): The configuration of the component.
|
|
19
|
+
Defaults to None which uses default settings.
|
|
25
20
|
|
|
26
21
|
Returns:
|
|
27
22
|
html.Div: The header component.
|
|
@@ -38,7 +33,8 @@ def get_header(
|
|
|
38
33
|
dbc.ModalBody(
|
|
39
34
|
[
|
|
40
35
|
html.Div(
|
|
41
|
-
"Login or email",
|
|
36
|
+
"Login or email",
|
|
37
|
+
style={"margin": 5, "fontWeight": "bold"},
|
|
42
38
|
),
|
|
43
39
|
dbc.Input(
|
|
44
40
|
id=f"{store_id}=login",
|
|
@@ -48,7 +44,11 @@ def get_header(
|
|
|
48
44
|
),
|
|
49
45
|
html.Div(
|
|
50
46
|
"Password",
|
|
51
|
-
style={
|
|
47
|
+
style={
|
|
48
|
+
"margin": 5,
|
|
49
|
+
"marginTop": 15,
|
|
50
|
+
"fontWeight": "bold",
|
|
51
|
+
},
|
|
52
52
|
),
|
|
53
53
|
dbc.Input(
|
|
54
54
|
id=f"{store_id}=password",
|
|
@@ -60,7 +60,11 @@ def get_header(
|
|
|
60
60
|
"Login failed.",
|
|
61
61
|
hidden=True,
|
|
62
62
|
id=f"{store_id}=login-failed",
|
|
63
|
-
style={
|
|
63
|
+
style={
|
|
64
|
+
"color": "red",
|
|
65
|
+
"fontWeight": "bold",
|
|
66
|
+
"margin": 10,
|
|
67
|
+
},
|
|
64
68
|
),
|
|
65
69
|
],
|
|
66
70
|
),
|
|
@@ -88,19 +88,19 @@ def get_region(
|
|
|
88
88
|
height: int,
|
|
89
89
|
mag: float | None = None,
|
|
90
90
|
) -> np.ndarray:
|
|
91
|
-
"""Get
|
|
92
|
-
|
|
93
|
-
the image size.
|
|
91
|
+
"""Get image region from DSA. Note that the output image might not
|
|
92
|
+
be in the shape (width, height) if left + width or top + height
|
|
93
|
+
exceeds the image size.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
gc (girder_client.GirderClient): The authenticated girder client.
|
|
97
|
+
item_id (str): The item id.
|
|
94
98
|
left (int): The left coordinate.
|
|
95
99
|
top (int): The top coordinate.
|
|
96
100
|
width (int): The width of the region.
|
|
97
101
|
height (int): The height of the region.
|
|
98
|
-
mag (float, optional): The magnification. Defaults to None which
|
|
99
|
-
the image at scan magnification.
|
|
100
|
-
magnification will result in an ouptut image smaller than the
|
|
101
|
-
width and height. Similarly, using a mag higher than the scan
|
|
102
|
-
magnification will result in an output image larger than the width
|
|
103
|
-
and height.
|
|
102
|
+
mag (float, optional): The magnification. Defaults to None which
|
|
103
|
+
returns the image at the scan magnification.
|
|
104
104
|
|
|
105
105
|
Returns:
|
|
106
106
|
np.ndarray: The region of the image.
|
|
@@ -258,17 +258,16 @@ def _rotate_point_list(point_list, rotation, center=(0, 0)):
|
|
|
258
258
|
return point_list_rotated
|
|
259
259
|
|
|
260
260
|
|
|
261
|
-
def get_rectangle_element_coords(element):
|
|
261
|
+
def get_rectangle_element_coords(element: dict) -> np.ndarray:
|
|
262
262
|
"""Get the corner coordinate from a rectangle HistomicsUI element, can handle rotated elements.
|
|
263
263
|
Adapted from: https://github.com/DigitalSlideArchive/HistomicsTK/blob/master/histomicstk/annotations_and_masks/annotation_and_mask_utils.py
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
array of shape [4, 2] for the four corners of the rectangle in (x, y) format
|
|
264
|
+
|
|
265
|
+
Args:
|
|
266
|
+
element (dict): rectangle element from HistomicsUI annotation.
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
corner_coords (np.ndarray): 4x2 array of corner coordinates of rectangle.
|
|
270
|
+
|
|
272
271
|
"""
|
|
273
272
|
# element is a dict so prevent referencing
|
|
274
273
|
element = deepcopy(element)
|
|
@@ -401,7 +400,9 @@ def get_roi_with_yolo_labels_from_single_doc(
|
|
|
401
400
|
if element.get("group") == roi_element_group:
|
|
402
401
|
# Append the ROI element, there can't more than one.
|
|
403
402
|
if roi_element is not None:
|
|
404
|
-
raise ValueError(
|
|
403
|
+
raise ValueError(
|
|
404
|
+
"Multiple ROI elements found in the document."
|
|
405
|
+
)
|
|
405
406
|
|
|
406
407
|
roi_element = element
|
|
407
408
|
elif element.get("group") in group_map:
|
|
@@ -460,7 +461,9 @@ def get_roi_with_yolo_labels_from_single_doc(
|
|
|
460
461
|
for element in box_elements:
|
|
461
462
|
if element.get("group") in group_map:
|
|
462
463
|
if element.get("type") != "rectangle":
|
|
463
|
-
print(
|
|
464
|
+
print(
|
|
465
|
+
f"Skipping element: {element.get('type')} not supported."
|
|
466
|
+
)
|
|
464
467
|
continue
|
|
465
468
|
|
|
466
469
|
group_id = group_map[element.get("group")]
|
|
@@ -523,7 +526,9 @@ def post_annotations_from_gdf(
|
|
|
523
526
|
# Process the dataframe.
|
|
524
527
|
for _, row in gdf.iterrows():
|
|
525
528
|
# Flatten multipolygon to list of polygons.
|
|
526
|
-
list_of_polygons = list(
|
|
529
|
+
list_of_polygons = list(
|
|
530
|
+
row["geometry"].geoms
|
|
531
|
+
) # assuming multipolygons
|
|
527
532
|
label = row["label"]
|
|
528
533
|
|
|
529
534
|
for poly in list_of_polygons:
|
|
@@ -4,7 +4,7 @@ from . import (
|
|
|
4
4
|
datasets,
|
|
5
5
|
transforms,
|
|
6
6
|
segformer_semantic_segmentation,
|
|
7
|
-
|
|
7
|
+
callbacks,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
10
|
# Modules that are imported from * notation.
|
|
@@ -14,5 +14,5 @@ __all__ = [
|
|
|
14
14
|
"metrics",
|
|
15
15
|
"transforms",
|
|
16
16
|
"segformer_semantic_segmentation",
|
|
17
|
-
"
|
|
17
|
+
"callbacks",
|
|
18
18
|
]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Provides custom callbacks for the Hugging Face Trainer class."""
|
|
2
|
+
|
|
3
|
+
from transformers import TrainerCallback
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MetricsLoggerCallback(TrainerCallback):
|
|
9
|
+
def _load_current_file(self, output_dir):
|
|
10
|
+
output_dir = Path(output_dir)
|
|
11
|
+
metrics_file = output_dir / "metrics.csv"
|
|
12
|
+
|
|
13
|
+
return (
|
|
14
|
+
pd.read_csv(metrics_file)
|
|
15
|
+
if metrics_file.exists()
|
|
16
|
+
else pd.DataFrame()
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
|
20
|
+
"""
|
|
21
|
+
Triggered during training logging (e.g., for training loss).
|
|
22
|
+
"""
|
|
23
|
+
if "loss" in logs: # Ensure training loss is available
|
|
24
|
+
metrics_df = self._load_current_file(args.output_dir)
|
|
25
|
+
|
|
26
|
+
# Append the epoch and the training loss.
|
|
27
|
+
loss = logs["loss"]
|
|
28
|
+
epoch = state.epoch
|
|
29
|
+
|
|
30
|
+
metrics_df = pd.concat(
|
|
31
|
+
[
|
|
32
|
+
metrics_df,
|
|
33
|
+
pd.DataFrame([{"epoch": epoch, "train_loss": loss}]),
|
|
34
|
+
],
|
|
35
|
+
ignore_index=True,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# Save it back to file.
|
|
39
|
+
metrics_df.to_csv(
|
|
40
|
+
Path(args.output_dir) / "metrics.csv", index=False
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def on_evaluate(self, args, state, control, metrics, **kwargs):
|
|
44
|
+
"""
|
|
45
|
+
This is triggered at the end of evaluation (including at the end of each epoch).
|
|
46
|
+
"""
|
|
47
|
+
# Get the current metrics file.
|
|
48
|
+
metrics_df = self._load_current_file(args.output_dir)
|
|
49
|
+
|
|
50
|
+
# Add metrics to the last row of the current metrics file.
|
|
51
|
+
for k, v in metrics.items():
|
|
52
|
+
metrics_df.at[metrics_df.index[-1], k] = v
|
|
53
|
+
|
|
54
|
+
# Save the updated metrics file.
|
|
55
|
+
metrics_df.to_csv(Path(args.output_dir) / "metrics.csv", index=False)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Metric functions.
|
|
3
|
+
|
|
4
|
+
This module contains functions used to evaluate the performance of
|
|
5
|
+
machine learning models.
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
import torch
|
|
9
|
+
from torch import nn
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def binary_dice_coefficient(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
|
|
14
|
+
"""Calculate the dice coefficient for binary masks. Developed for use
|
|
15
|
+
in compute_metrics parameters of transformers Trainer class, for
|
|
16
|
+
SegFormer senantic segmentation model of two classes.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
eval_pred (tuple[np.ndarray, np.ndarray]): A tuple containing the
|
|
20
|
+
predicted and true masks.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
dict: A dictionary containing the dice coefficient.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
with torch.no_grad():
|
|
27
|
+
logits, labels = eval_pred
|
|
28
|
+
|
|
29
|
+
# Convert to tensors.
|
|
30
|
+
logits = torch.from_numpy(logits).cpu()
|
|
31
|
+
|
|
32
|
+
# Resize the logits to be the same shape as the labels.
|
|
33
|
+
logits = nn.functional.interpolate(
|
|
34
|
+
logits,
|
|
35
|
+
size=labels.shape[-2:],
|
|
36
|
+
mode="bilinear",
|
|
37
|
+
align_corners=False,
|
|
38
|
+
).argmax(dim=1)
|
|
39
|
+
|
|
40
|
+
preds = logits.detach().numpy()
|
|
41
|
+
|
|
42
|
+
# Convert the masks to binary.
|
|
43
|
+
preds = preds > 0
|
|
44
|
+
labels = labels > 0
|
|
45
|
+
|
|
46
|
+
# Calculate dice.
|
|
47
|
+
intersection = np.sum(preds * labels)
|
|
48
|
+
union = np.sum(preds) + np.sum(labels)
|
|
49
|
+
|
|
50
|
+
if union == 0:
|
|
51
|
+
dice = 1.0 # Both masks are empty
|
|
52
|
+
else:
|
|
53
|
+
dice = 2.0 * intersection / union
|
|
54
|
+
|
|
55
|
+
return {"dice_coefficient": dice}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def mean_iou(label2idx: dict):
|
|
59
|
+
"""Return the compute metric function for mean intersection over
|
|
60
|
+
union for a multi-class semantic segmentation task.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
label2idx (dict): A dictionary mapping class labels to indices.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
function: The compute_metrics function.
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
idx2label = {v: k for k, v in label2idx.items()}
|
|
70
|
+
|
|
71
|
+
def compute_metrics(eval_pred):
|
|
72
|
+
"""Compute the mean intersection over union (IoU) for semantic
|
|
73
|
+
segmentation. Also returns the IoU for each class."""
|
|
74
|
+
with torch.no_grad():
|
|
75
|
+
logits, labels = eval_pred
|
|
76
|
+
|
|
77
|
+
# Convert logits to tensor.
|
|
78
|
+
logits_tensor = torch.from_numpy(logits).cpu()
|
|
79
|
+
labels_tensor = torch.from_numpy(labels).cpu()
|
|
80
|
+
|
|
81
|
+
# From logits get the number of classes in the dataset.
|
|
82
|
+
num_classes = logits.shape[1]
|
|
83
|
+
|
|
84
|
+
# Scale the logits back to the shape of the labels.
|
|
85
|
+
logits_tensor = nn.functional.interpolate(
|
|
86
|
+
logits_tensor,
|
|
87
|
+
size=labels.shape[-2:],
|
|
88
|
+
mode="bilinear",
|
|
89
|
+
align_corners=False,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Turn the logits to predictions.
|
|
93
|
+
pred = logits_tensor.argmax(dim=1)
|
|
94
|
+
|
|
95
|
+
# Calculate the IoU for each class.
|
|
96
|
+
ious = []
|
|
97
|
+
|
|
98
|
+
metrics = {}
|
|
99
|
+
|
|
100
|
+
for cls in range(num_classes):
|
|
101
|
+
pred_mask = pred == cls
|
|
102
|
+
labels_mask = labels_tensor == cls
|
|
103
|
+
|
|
104
|
+
intersection = (
|
|
105
|
+
torch.logical_and(pred_mask, labels_mask).sum().item()
|
|
106
|
+
)
|
|
107
|
+
union = torch.logical_or(pred_mask, labels_mask).sum().item()
|
|
108
|
+
|
|
109
|
+
if union == 0:
|
|
110
|
+
iou = float("nan") # or 1.0
|
|
111
|
+
else:
|
|
112
|
+
iou = intersection / union
|
|
113
|
+
|
|
114
|
+
metrics[f"{idx2label[cls]}_iou"] = iou
|
|
115
|
+
|
|
116
|
+
ious.append(iou)
|
|
117
|
+
|
|
118
|
+
mean_iou = np.nanmean(ious) # Use NumPy for NaN handling
|
|
119
|
+
metrics["mean_iou"] = mean_iou
|
|
120
|
+
|
|
121
|
+
return metrics
|
|
122
|
+
|
|
123
|
+
return compute_metrics
|
|
@@ -10,7 +10,7 @@ import torch
|
|
|
10
10
|
import torch.nn as nn
|
|
11
11
|
|
|
12
12
|
from ... import imread
|
|
13
|
-
from ...
|
|
13
|
+
from ...tiling import tile_image
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def inference(
|
|
@@ -115,7 +115,9 @@ def inference(
|
|
|
115
115
|
predicted_mask = predicted_mask[:h, :w]
|
|
116
116
|
|
|
117
117
|
# Extract the contours of the mask.
|
|
118
|
-
contours = cv.findContours(
|
|
118
|
+
contours = cv.findContours(
|
|
119
|
+
predicted_mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE
|
|
120
|
+
)[0]
|
|
119
121
|
|
|
120
122
|
# Return the mask.
|
|
121
123
|
return predicted_mask, contours
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
.gitignore
|
|
2
|
+
.readthedocs.yaml
|
|
2
3
|
LICENSE
|
|
3
4
|
README.md
|
|
4
5
|
app.py
|
|
5
6
|
pyproject.toml
|
|
7
|
+
docs/requirements.txt
|
|
8
|
+
docs/source/conf.py
|
|
9
|
+
docs/source/index.rst
|
|
6
10
|
dsa_helpers/__init__.py
|
|
7
11
|
dsa_helpers/girder_utils.py
|
|
8
12
|
dsa_helpers/imread.py
|
|
@@ -21,6 +25,7 @@ dsa_helpers/dash/dash_paperdragon_utils.py
|
|
|
21
25
|
dsa_helpers/dash/header.py
|
|
22
26
|
dsa_helpers/dash/header_callbacks.py
|
|
23
27
|
dsa_helpers/ml/__init__.py
|
|
28
|
+
dsa_helpers/ml/callbacks.py
|
|
24
29
|
dsa_helpers/ml/metrics.py
|
|
25
30
|
dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py
|
|
26
31
|
dsa_helpers/ml/datasets/__init__.py
|
|
@@ -31,8 +36,6 @@ dsa_helpers/ml/object_detection/tile_image.py
|
|
|
31
36
|
dsa_helpers/ml/object_detection/utils.py
|
|
32
37
|
dsa_helpers/ml/segformer_semantic_segmentation/__init__.py
|
|
33
38
|
dsa_helpers/ml/segformer_semantic_segmentation/inference.py
|
|
34
|
-
dsa_helpers/ml/tiling/__init__.py
|
|
35
|
-
dsa_helpers/ml/tiling/semantic_seg_tile_wsi_with_dsa_annotations.py
|
|
36
39
|
dsa_helpers/ml/transforms/__init__.py
|
|
37
40
|
dsa_helpers/ml/transforms/segformer_transforms.py
|
|
38
41
|
tests/docker-compose.yaml
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""Metric functions.
|
|
3
|
-
|
|
4
|
-
This module contains functions used to evaluate the performance of
|
|
5
|
-
machine learning models.
|
|
6
|
-
|
|
7
|
-
"""
|
|
8
|
-
import torch
|
|
9
|
-
from torch import nn
|
|
10
|
-
import numpy as np
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def binary_dice_coefficient(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
|
|
14
|
-
"""Calculate the dice coefficient for binary masks. Developed for use
|
|
15
|
-
in compute_metrics parameters of transformers Trainer class, for
|
|
16
|
-
SegFormer senantic segmentation model of two classes.
|
|
17
|
-
|
|
18
|
-
Args:
|
|
19
|
-
eval_pred (tuple[np.ndarray, np.ndarray]): A tuple containing the
|
|
20
|
-
predicted and true masks.
|
|
21
|
-
|
|
22
|
-
Returns:
|
|
23
|
-
dict: A dictionary containing the dice coefficient.
|
|
24
|
-
|
|
25
|
-
"""
|
|
26
|
-
with torch.no_grad():
|
|
27
|
-
logits, labels = eval_pred
|
|
28
|
-
|
|
29
|
-
# Convert to tensors.
|
|
30
|
-
logits = torch.from_numpy(logits).cpu()
|
|
31
|
-
|
|
32
|
-
# Resize the logits to be the same shape as the labels.
|
|
33
|
-
logits = nn.functional.interpolate(
|
|
34
|
-
logits,
|
|
35
|
-
size=labels.shape[-2:],
|
|
36
|
-
mode="bilinear",
|
|
37
|
-
align_corners=False,
|
|
38
|
-
).argmax(dim=1)
|
|
39
|
-
|
|
40
|
-
preds = logits.detach().numpy()
|
|
41
|
-
|
|
42
|
-
# Convert the masks to binary.
|
|
43
|
-
preds = preds > 0
|
|
44
|
-
labels = labels > 0
|
|
45
|
-
|
|
46
|
-
# Calculate dice.
|
|
47
|
-
intersection = np.sum(preds * labels)
|
|
48
|
-
union = np.sum(preds) + np.sum(labels)
|
|
49
|
-
|
|
50
|
-
if union == 0:
|
|
51
|
-
dice = 1.0 # Both masks are empty
|
|
52
|
-
else:
|
|
53
|
-
dice = 2.0 * intersection / union
|
|
54
|
-
|
|
55
|
-
return {"dice_coefficient": dice}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def mean_iou(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict:
|
|
59
|
-
"""Compute the mean intersection over union (IoU) for semantic
|
|
60
|
-
segmentation. Also returns the IoU for each class.
|
|
61
|
-
|
|
62
|
-
Args:
|
|
63
|
-
eval_pred (tuple[numpy.ndarray, numpy.ndarray]): Tuple of
|
|
64
|
-
logits and labels.
|
|
65
|
-
|
|
66
|
-
Returns:
|
|
67
|
-
dict: Dictionary of metrics.
|
|
68
|
-
|
|
69
|
-
"""
|
|
70
|
-
with torch.no_grad():
|
|
71
|
-
logits, labels = eval_pred
|
|
72
|
-
|
|
73
|
-
# Convert logits to tensor.
|
|
74
|
-
logits_tensor = torch.from_numpy(logits).cpu()
|
|
75
|
-
labels_tensor = torch.from_numpy(labels).cpu()
|
|
76
|
-
|
|
77
|
-
# From logits get the number of classes in the dataset.
|
|
78
|
-
num_classes = logits.shape[1]
|
|
79
|
-
|
|
80
|
-
# Scale the logits back to the shape of the labels.
|
|
81
|
-
logits_tensor = nn.functional.interpolate(
|
|
82
|
-
logits_tensor,
|
|
83
|
-
size=labels.shape[-2:],
|
|
84
|
-
mode="bilinear",
|
|
85
|
-
align_corners=False,
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
# Turn the logits to predictions.
|
|
89
|
-
pred = logits_tensor.argmax(dim=1)
|
|
90
|
-
|
|
91
|
-
# Calculate the IoU for each class.
|
|
92
|
-
ious = []
|
|
93
|
-
|
|
94
|
-
metrics = {}
|
|
95
|
-
|
|
96
|
-
for cls in range(num_classes):
|
|
97
|
-
pred_mask = pred == cls
|
|
98
|
-
labels_mask = labels_tensor == cls
|
|
99
|
-
|
|
100
|
-
intersection = (
|
|
101
|
-
torch.logical_and(pred_mask, labels_mask).sum().item()
|
|
102
|
-
)
|
|
103
|
-
union = torch.logical_or(pred_mask, labels_mask).sum().item()
|
|
104
|
-
|
|
105
|
-
if union == 0:
|
|
106
|
-
iou = float("nan") # or 1.0
|
|
107
|
-
else:
|
|
108
|
-
iou = intersection / union
|
|
109
|
-
|
|
110
|
-
metrics[f"label {cls} IoU"] = iou
|
|
111
|
-
|
|
112
|
-
ious.append(iou)
|
|
113
|
-
|
|
114
|
-
mean_iou = np.nanmean(ious) # Use NumPy for NaN handling
|
|
115
|
-
metrics["mean_iou"] = mean_iou
|
|
116
|
-
|
|
117
|
-
return metrics
|
|
@@ -1,225 +0,0 @@
|
|
|
1
|
-
import large_image
|
|
2
|
-
import cv2 as cv
|
|
3
|
-
import numpy as np
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from ...imwrite import imwrite
|
|
6
|
-
from multiprocessing import Pool
|
|
7
|
-
from tqdm import tqdm
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def _process_tile(
|
|
11
|
-
ts,
|
|
12
|
-
prepend_name,
|
|
13
|
-
x,
|
|
14
|
-
y,
|
|
15
|
-
scan_tile_size,
|
|
16
|
-
tile_size,
|
|
17
|
-
mag,
|
|
18
|
-
contours,
|
|
19
|
-
sf,
|
|
20
|
-
img_dir,
|
|
21
|
-
mask_dir,
|
|
22
|
-
):
|
|
23
|
-
fn = f"{prepend_name}x{x}y{y}.png"
|
|
24
|
-
|
|
25
|
-
# Get the tile.
|
|
26
|
-
tile_img = ts.getRegion(
|
|
27
|
-
region={
|
|
28
|
-
"left": x,
|
|
29
|
-
"top": y,
|
|
30
|
-
"right": x + scan_tile_size,
|
|
31
|
-
"bottom": y + scan_tile_size,
|
|
32
|
-
},
|
|
33
|
-
format=large_image.constants.TILE_FORMAT_NUMPY,
|
|
34
|
-
scale={"magnification": mag},
|
|
35
|
-
)[0][:, :, :3]
|
|
36
|
-
|
|
37
|
-
if tile_img.shape[:2] != (tile_size, tile_size):
|
|
38
|
-
# Pad the image with zeroes to make it the desired size.
|
|
39
|
-
tile_img = cv.copyMakeBorder(
|
|
40
|
-
tile_img,
|
|
41
|
-
0,
|
|
42
|
-
tile_size - tile_img.shape[0],
|
|
43
|
-
0,
|
|
44
|
-
tile_size - tile_img.shape[1],
|
|
45
|
-
cv.BORDER_CONSTANT,
|
|
46
|
-
value=0,
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
# Create a blank tile mask.
|
|
50
|
-
tile_mask = np.zeros(tile_img.shape[:2], dtype=np.uint8)
|
|
51
|
-
|
|
52
|
-
# Loop through contours to draw on tile mask.
|
|
53
|
-
for idx, contour_dict in contours.items():
|
|
54
|
-
# Create a mask for the specific label.
|
|
55
|
-
label_mask = np.zeros(tile_mask.shape, dtype=np.uint8)
|
|
56
|
-
|
|
57
|
-
points = contour_dict["points"].copy()
|
|
58
|
-
holes = contour_dict["holes"].copy()
|
|
59
|
-
|
|
60
|
-
# For points and hole:
|
|
61
|
-
# (1) Shift by top left of the current tile.
|
|
62
|
-
# (2) Scale by be desired magnification.
|
|
63
|
-
points = [((point - [x, y]) * sf).astype(int) for point in points]
|
|
64
|
-
|
|
65
|
-
corrected_holes = []
|
|
66
|
-
|
|
67
|
-
for hole in holes:
|
|
68
|
-
if len(hole):
|
|
69
|
-
corrected_holes.append(((hole - [x, y]) * sf).astype(int))
|
|
70
|
-
|
|
71
|
-
# Use points to draw contours as positive.
|
|
72
|
-
label_mask = cv.drawContours(label_mask, points, -1, 1, cv.FILLED)
|
|
73
|
-
|
|
74
|
-
# Draw in the holes as background class zero.
|
|
75
|
-
label_mask = cv.drawContours(label_mask, holes, -1, 0, cv.FILLED)
|
|
76
|
-
|
|
77
|
-
# Apply the label to tile mask where label mask is positive.
|
|
78
|
-
tile_mask[label_mask == 1] = idx
|
|
79
|
-
|
|
80
|
-
# Save the image and mask.
|
|
81
|
-
img_fp = img_dir / fn
|
|
82
|
-
imwrite(img_fp, tile_img)
|
|
83
|
-
imwrite(mask_dir / fn, tile_mask, grayscale=True)
|
|
84
|
-
|
|
85
|
-
return str(img_fp), x, y
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def semantic_seg_tile_wsi_with_dsa_annotations(
|
|
89
|
-
wsi_fp: str,
|
|
90
|
-
annotation_docs: list[dict],
|
|
91
|
-
label2idx: dict,
|
|
92
|
-
save_dir: str,
|
|
93
|
-
tile_size: int,
|
|
94
|
-
mag: float | None = None,
|
|
95
|
-
prepend_name: str = "",
|
|
96
|
-
nproc: int = 1,
|
|
97
|
-
) -> list[str]:
|
|
98
|
-
"""Tile a WSI with semantic segmentation label masks created from
|
|
99
|
-
DSA annotations.
|
|
100
|
-
|
|
101
|
-
Args:
|
|
102
|
-
wsi_fp (str): file path to WSI.
|
|
103
|
-
annotation_docs (list[dict]): list of annotation documents.
|
|
104
|
-
label2idx (dict): mapping of label names to integer indices.
|
|
105
|
-
save_dir (str): directory to save the tiled images.
|
|
106
|
-
tile_size (int): size of the tile images at desired magnificaiton.
|
|
107
|
-
mag (float | None, optional): magnification level of the WSI.
|
|
108
|
-
If None, the function will use the default magnification
|
|
109
|
-
level. Defaults to None.
|
|
110
|
-
prepend_name (str, optional): prepend name to the created tile
|
|
111
|
-
images and masks. Defaults to "".
|
|
112
|
-
nproc (int, optional): number of processes to use for tiling.
|
|
113
|
-
|
|
114
|
-
Returns:
|
|
115
|
-
list[str]: A list of tuples: (tile file path, x, y coordinates
|
|
116
|
-
of tile).
|
|
117
|
-
|
|
118
|
-
"""
|
|
119
|
-
# Format annotation documents for drawing on tile masks.
|
|
120
|
-
contours = {idx: {"points": [], "holes": []} for idx in label2idx.values()}
|
|
121
|
-
|
|
122
|
-
for ann_doc in annotation_docs:
|
|
123
|
-
# Loop through the elements of the annotation document.
|
|
124
|
-
elements = ann_doc["annotation"].get("elements", [])
|
|
125
|
-
|
|
126
|
-
for element in elements:
|
|
127
|
-
# Check if element has a label.
|
|
128
|
-
label = element.get("label", {}).get("value")
|
|
129
|
-
|
|
130
|
-
if label not in label2idx:
|
|
131
|
-
# Skip this element if it is not in the label2idx mapping.
|
|
132
|
-
continue
|
|
133
|
-
|
|
134
|
-
idx = label2idx[label]
|
|
135
|
-
|
|
136
|
-
element_type = element["type"]
|
|
137
|
-
|
|
138
|
-
if element_type == "polyline":
|
|
139
|
-
points = element.get("points", [])
|
|
140
|
-
|
|
141
|
-
if len(points):
|
|
142
|
-
contours[idx]["points"].append(
|
|
143
|
-
np.array(points, dtype=int)[:, :2]
|
|
144
|
-
)
|
|
145
|
-
|
|
146
|
-
holes = element.get("holes", [])
|
|
147
|
-
|
|
148
|
-
if len(holes):
|
|
149
|
-
for hole in holes:
|
|
150
|
-
contours[idx]["holes"].append(
|
|
151
|
-
np.array(hole, dtype=int)[:, :2]
|
|
152
|
-
)
|
|
153
|
-
elif element_type == "rectangle":
|
|
154
|
-
# Convert the rectangle coordinates to polygon ones.
|
|
155
|
-
x1, y1 = element["center"][:2]
|
|
156
|
-
x2 = x1 + element["width"]
|
|
157
|
-
y2 = y1 + element["height"]
|
|
158
|
-
|
|
159
|
-
contours[idx]["points"].append(
|
|
160
|
-
np.array(
|
|
161
|
-
[[x1, y1], [x2, y1], [x2, y2], [x1, y2]], dtype=int
|
|
162
|
-
)
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
# Create directory to save tile images and masks.
|
|
166
|
-
save_dir = Path(save_dir)
|
|
167
|
-
img_dir = save_dir / "images"
|
|
168
|
-
mask_dir = save_dir / "masks"
|
|
169
|
-
img_dir.mkdir(parents=True, exist_ok=True)
|
|
170
|
-
mask_dir.mkdir(exist_ok=True)
|
|
171
|
-
|
|
172
|
-
# Get the tile source of WSI.
|
|
173
|
-
ts = large_image.getTileSource(str(wsi_fp))
|
|
174
|
-
|
|
175
|
-
# Get the metadata of the WSI.
|
|
176
|
-
metadata = ts.getMetadata()
|
|
177
|
-
|
|
178
|
-
wsi_w, wsi_h = metadata["sizeX"], metadata["sizeY"]
|
|
179
|
-
|
|
180
|
-
# Get the magnification of the WSI.
|
|
181
|
-
wsi_mag = metadata["magnification"]
|
|
182
|
-
|
|
183
|
-
# Calculate the tile size at full resolution (1 if mag is None or the same as scan mag).
|
|
184
|
-
if mag is None:
|
|
185
|
-
scan_tile_size = tile_size
|
|
186
|
-
mag = wsi_mag
|
|
187
|
-
sf = 1
|
|
188
|
-
else:
|
|
189
|
-
sf = mag / wsi_mag # scan mag -> desired mag
|
|
190
|
-
scan_tile_size = int(tile_size / sf)
|
|
191
|
-
|
|
192
|
-
# Get a list of x,y coordinates for the tile top left corners.
|
|
193
|
-
xys = [
|
|
194
|
-
(x, y)
|
|
195
|
-
for x in range(0, wsi_w, scan_tile_size)
|
|
196
|
-
for y in range(0, wsi_h, scan_tile_size)
|
|
197
|
-
]
|
|
198
|
-
|
|
199
|
-
with Pool(nproc) as pool:
|
|
200
|
-
jobs = [
|
|
201
|
-
pool.apply_async(
|
|
202
|
-
_process_tile,
|
|
203
|
-
(
|
|
204
|
-
ts,
|
|
205
|
-
prepend_name,
|
|
206
|
-
xy[0],
|
|
207
|
-
xy[1],
|
|
208
|
-
scan_tile_size,
|
|
209
|
-
tile_size,
|
|
210
|
-
mag,
|
|
211
|
-
contours,
|
|
212
|
-
sf,
|
|
213
|
-
img_dir,
|
|
214
|
-
mask_dir,
|
|
215
|
-
),
|
|
216
|
-
)
|
|
217
|
-
for xy in xys
|
|
218
|
-
]
|
|
219
|
-
|
|
220
|
-
tile_info = []
|
|
221
|
-
|
|
222
|
-
for job in tqdm(jobs, desc="Tiling..."):
|
|
223
|
-
tile_info.append(job.get())
|
|
224
|
-
|
|
225
|
-
return tile_info
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/datasets/SegFormerSegmentationDataset.py
RENAMED
|
File without changes
|
|
File without changes
|
{dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/object_detection/get_pearce_roi_images.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dsa_helpers-0.9.dev1 → dsa_helpers-1.dev0}/dsa_helpers/ml/transforms/segformer_transforms.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|