dsa-helpers 0.3.dev1__tar.gz → 0.4.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/.gitignore +2 -1
- {dsa_helpers-0.3.dev1/dsa_helpers.egg-info → dsa_helpers-0.4.dev0}/PKG-INFO +16 -1
- dsa_helpers-0.4.dev0/README.md +19 -0
- dsa_helpers-0.4.dev0/app.py +14 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/__init__.py +3 -2
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/girder_utils.py +60 -1
- dsa_helpers-0.4.dev0/dsa_helpers/ml/__init__.py +2 -0
- dsa_helpers-0.4.dev0/dsa_helpers/ml/object_detection/__init__.py +5 -0
- dsa_helpers-0.4.dev0/dsa_helpers/ml/object_detection/get_pearce_roi_images.py +136 -0
- dsa_helpers-0.4.dev0/dsa_helpers/ml/object_detection/tile_image.py +137 -0
- dsa_helpers-0.4.dev0/dsa_helpers/ml/object_detection/utils.py +104 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/mongo_utils.py +50 -33
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0/dsa_helpers.egg-info}/PKG-INFO +16 -1
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers.egg-info/SOURCES.txt +8 -2
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers.egg-info/requires.txt +2 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/pyproject.toml +4 -2
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/requirements.txt +2 -1
- dsa_helpers-0.4.dev0/tests/test.ipynb +268 -0
- dsa_helpers-0.3.dev1/README.md +0 -6
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/LICENSE +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/dash/__init__.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/dash/header.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/dash/header_callbacks.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/image_utils.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/imread.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/imwrite.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers/login.py +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers.egg-info/dependency_links.txt +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/dsa_helpers.egg-info/top_level.txt +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/readthedocs.yaml +0 -0
- {dsa_helpers-0.3.dev1 → dsa_helpers-0.4.dev0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dsa_helpers
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.dev0
|
|
4
4
|
Summary: Utility functions for working with the DSA girder client.
|
|
5
5
|
Author-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
|
|
6
6
|
Maintainer-email: "Juan C. Vizcarra" <jvizcar@emory.edu>
|
|
@@ -20,6 +20,8 @@ Requires-Dist: pymongo
|
|
|
20
20
|
Requires-Dist: pillow
|
|
21
21
|
Requires-Dist: dash
|
|
22
22
|
Requires-Dist: dash_bootstrap_components
|
|
23
|
+
Requires-Dist: shapely
|
|
24
|
+
Requires-Dist: geopandas
|
|
23
25
|
|
|
24
26
|
# Digital Slide Archive (DSA) Helpers
|
|
25
27
|
Digital Slide Archive Python utility library.
|
|
@@ -27,3 +29,16 @@ Digital Slide Archive Python utility library.
|
|
|
27
29
|
This library is available for installation through [Python Package Index (PyPI)](https://pypi.org/).
|
|
28
30
|
|
|
29
31
|
This library was tested using Python version 3.11.8 and uses the dependencies described in requirements.txt.
|
|
32
|
+
|
|
33
|
+
This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
|
|
34
|
+
|
|
35
|
+
## Instructions for Development
|
|
36
|
+
1. Install requirements for building and distributing package:
|
|
37
|
+
```
|
|
38
|
+
$ python3 -m pip install --upgrade build # for building package
|
|
39
|
+
$ python3 -m pip install --upgrade build # for uploading to PyPI
|
|
40
|
+
```
|
|
41
|
+
2. When ready to build:
|
|
42
|
+
- Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
|
|
43
|
+
- Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
|
|
44
|
+
3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Digital Slide Archive (DSA) Helpers
|
|
2
|
+
Digital Slide Archive Python utility library.
|
|
3
|
+
|
|
4
|
+
This library is available for installation through [Python Package Index (PyPI)](https://pypi.org/).
|
|
5
|
+
|
|
6
|
+
This library was tested using Python version 3.11.8 and uses the dependencies described in requirements.txt.
|
|
7
|
+
|
|
8
|
+
This Python PyPI package is found [here](https://pypi.org/project/dsa-helpers/).
|
|
9
|
+
|
|
10
|
+
## Instructions for Development
|
|
11
|
+
1. Install requirements for building and distributing package:
|
|
12
|
+
```
|
|
13
|
+
$ python3 -m pip install --upgrade build # for building package
|
|
14
|
+
$ python3 -m pip install --upgrade build # for uploading to PyPI
|
|
15
|
+
```
|
|
16
|
+
2. When ready to build:
|
|
17
|
+
- Modify "pyproject.toml" file, change the "version" key to a different value than a version already pushed.
|
|
18
|
+
- Run ```$ python3 -m build```. This will create a dist directory and put your new wheel and tar distribution files there.
|
|
19
|
+
3. To install locally for development: ```$ pip3 install dist/dsa_helpers-*.whl```, choosing the wheel version you want to test.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# A test application.
|
|
2
|
+
from dash import Dash, html
|
|
3
|
+
import dash_bootstrap_components as dbc
|
|
4
|
+
from dsa_helpers.dash.header import get_header
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
os.environ["DSA_API_URL"] = "https://megabrain.neurology.emory.edu/api/v1"
|
|
8
|
+
|
|
9
|
+
app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
|
10
|
+
|
|
11
|
+
app.layout = html.Div([get_header("Test")])
|
|
12
|
+
|
|
13
|
+
if __name__ == "__main__":
|
|
14
|
+
app.run_server(debug=True)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# Shadow imports.
|
|
2
|
-
from .login import login
|
|
3
2
|
from .imread import imread
|
|
4
3
|
from .imwrite import imwrite
|
|
5
4
|
|
|
6
5
|
# Modules that should be available.
|
|
7
|
-
from . import girder_utils, image_utils, dash
|
|
6
|
+
# from . import girder_utils, image_utils, dash
|
|
7
|
+
|
|
8
|
+
__all__ = ["girder_utils", "image_utils", "dash"]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Functions using the girder client.
|
|
2
|
-
from girder_client import GirderClient
|
|
2
|
+
from girder_client import GirderClient, HttpError
|
|
3
3
|
import pickle
|
|
4
4
|
import numpy as np
|
|
5
5
|
import pandas as pd
|
|
@@ -285,3 +285,62 @@ def get_rectangle_element_coords(element):
|
|
|
285
285
|
corner_coords = np.array(corner_coords, dtype=np.int32)
|
|
286
286
|
|
|
287
287
|
return corner_coords
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def login(
|
|
291
|
+
api_url: str,
|
|
292
|
+
login_or_email: str | None = None,
|
|
293
|
+
password: str | None = None,
|
|
294
|
+
api_key: str | None = None,
|
|
295
|
+
) -> GirderClient:
|
|
296
|
+
"""Authenticate a girder client with the given credentials or interactively
|
|
297
|
+
if none is given.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
api_url (str): The DSA girder API url.
|
|
301
|
+
login_or_email (str | None): The login or email. Defaults to None.
|
|
302
|
+
password (str | None): Password for login / email. Defaults to None.
|
|
303
|
+
api_key (str | None): The api key to authenticate with. Defaults to None.
|
|
304
|
+
|
|
305
|
+
Returns:
|
|
306
|
+
girder_client.GirderClient: The authenticated girder client.
|
|
307
|
+
|
|
308
|
+
"""
|
|
309
|
+
gc = GirderClient(apiUrl=api_url)
|
|
310
|
+
|
|
311
|
+
if api_key is None:
|
|
312
|
+
if login_or_email is None:
|
|
313
|
+
_ = gc.authenticate(interactive=True)
|
|
314
|
+
elif password is None:
|
|
315
|
+
_ = gc.authenticate(username=login_or_email, interactive=True)
|
|
316
|
+
else:
|
|
317
|
+
_ = gc.authenticate(username=login_or_email, password=password)
|
|
318
|
+
else:
|
|
319
|
+
_ = gc.authenticate(apiKey=api_key)
|
|
320
|
+
|
|
321
|
+
return gc
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def get_items(gc: GirderClient, parend_id: str) -> list[dict]:
|
|
325
|
+
"""Get the items in a parent location recursively.
|
|
326
|
+
|
|
327
|
+
Args:
|
|
328
|
+
gc (girder_client.GirderClient): The authenticated girder client.
|
|
329
|
+
parend_id (str): The parent id to start the search (folder / collection).
|
|
330
|
+
|
|
331
|
+
Returns:
|
|
332
|
+
list[dict]: The list of items.
|
|
333
|
+
|
|
334
|
+
"""
|
|
335
|
+
params = {"type": "folder", "limit": 0, "offset": 0, "sort": "_id", "sortdir": 1}
|
|
336
|
+
|
|
337
|
+
request_url = f"resource/{parend_id}/items"
|
|
338
|
+
|
|
339
|
+
try:
|
|
340
|
+
items = gc.get(request_url, parameters=params)
|
|
341
|
+
except HttpError:
|
|
342
|
+
params["type"] = "collection"
|
|
343
|
+
|
|
344
|
+
items = gc.get(request_url, parameters=params)
|
|
345
|
+
|
|
346
|
+
return items
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
from girder_client import GirderClient
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ...girder_utils import get_region, get_item_large_image_metadata
|
|
6
|
+
from ... import imwrite
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_pearce_roi_images(
|
|
10
|
+
gc: GirderClient,
|
|
11
|
+
item_id: str,
|
|
12
|
+
class_map: dict[str, int],
|
|
13
|
+
save_dir: str = ".",
|
|
14
|
+
mag: float | None = None,
|
|
15
|
+
) -> pd.DataFrame:
|
|
16
|
+
"""Download the smallestl bounding box around an ROI annotations and its
|
|
17
|
+
accompanying object detection annotations. The format used to determine ROIs
|
|
18
|
+
and boxes is based on the convention used by Dr. Thomas Pearce in his
|
|
19
|
+
OpenSeadragon applications. Labels are saved as text files in the YOLO format.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
gc (girder_client.GirderClient): The Girder client to use.
|
|
23
|
+
item_id (str): The item ID of the WSI.
|
|
24
|
+
class_map (dict[str, int]): A dictionary mapping class names to integers.
|
|
25
|
+
save_dir (str, optional): The directory to save the images and labels.
|
|
26
|
+
Defaults to ".".
|
|
27
|
+
mag (float | None, optional): The magnification to download the ROI at.
|
|
28
|
+
If None, the scan magnification is used. Defaults to None.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
pandas.DataFrame: A DataFrame containing metadata for each ROI
|
|
32
|
+
|
|
33
|
+
"""
|
|
34
|
+
# Create directories to save images and labels.
|
|
35
|
+
img_dir = Path(save_dir) / "images"
|
|
36
|
+
img_dir.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
|
|
38
|
+
label_dir = Path(save_dir) / "labels"
|
|
39
|
+
label_dir.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
|
|
41
|
+
# Get all the annotation documents with name starting with "ROI".
|
|
42
|
+
ann_docs_ids = []
|
|
43
|
+
|
|
44
|
+
for ann_doc in gc.get(
|
|
45
|
+
f"annotation?itemId={item_id}&limit=0&offset=0&sort=lowerName&sortdir=1"
|
|
46
|
+
):
|
|
47
|
+
if ann_doc.get("annotation", {}).get("name", "").startswith("ROI"):
|
|
48
|
+
ann_docs_ids.append(ann_doc["_id"])
|
|
49
|
+
|
|
50
|
+
if not len(ann_docs_ids):
|
|
51
|
+
# No annotation documents, return None.
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
# Track metadata for each ROI.
|
|
55
|
+
roi_metadata = []
|
|
56
|
+
|
|
57
|
+
# Loop through each document.
|
|
58
|
+
for ann_doc_id in ann_docs_ids:
|
|
59
|
+
# Get the full annotation document.
|
|
60
|
+
ann_doc = gc.get(f"annotation/{ann_doc_id}")
|
|
61
|
+
|
|
62
|
+
# Separate the ROI element and the other elements.
|
|
63
|
+
roi_el = None
|
|
64
|
+
other_els = []
|
|
65
|
+
|
|
66
|
+
for element in ann_doc.get("annotation", {}).get("elements", []):
|
|
67
|
+
if element.get("user", {}).get("role") == "ROI":
|
|
68
|
+
roi_el = element
|
|
69
|
+
else:
|
|
70
|
+
other_els.append(element)
|
|
71
|
+
|
|
72
|
+
if roi_el is None:
|
|
73
|
+
raise Exception("No ROI element found in annotation document.")
|
|
74
|
+
|
|
75
|
+
# Download the ROI image.
|
|
76
|
+
xc, yc = roi_el["center"][:2]
|
|
77
|
+
roi_w, roi_h = int(roi_el["width"]), int(roi_el["height"])
|
|
78
|
+
|
|
79
|
+
# Get the top left coordinate of the ROI.
|
|
80
|
+
roi_left = int(xc - roi_w / 2)
|
|
81
|
+
roi_top = int(yc - roi_h / 2)
|
|
82
|
+
|
|
83
|
+
# Get scale factor to go from scan magnification to mag.
|
|
84
|
+
scan_mag = get_item_large_image_metadata(gc, item_id)["magnification"]
|
|
85
|
+
|
|
86
|
+
if mag is None:
|
|
87
|
+
mag = scan_mag
|
|
88
|
+
|
|
89
|
+
roi_img = get_region(
|
|
90
|
+
gc, item_id, left=roi_left, top=roi_top, width=roi_w, height=roi_h, mag=mag
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
fn = f"itemId-{item_id}-x{roi_left}y{roi_top}x{roi_left+roi_w}y{roi_top+roi_h}"
|
|
94
|
+
|
|
95
|
+
img_fp = img_dir / f"{fn}.png"
|
|
96
|
+
imwrite(img_fp, roi_img)
|
|
97
|
+
|
|
98
|
+
# Convert the boxes to YOLO format.
|
|
99
|
+
labels = ""
|
|
100
|
+
|
|
101
|
+
for element in other_els:
|
|
102
|
+
el_class = element["user"]["class"]
|
|
103
|
+
|
|
104
|
+
if el_class not in class_map:
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
xc, yc = element["center"][:2]
|
|
108
|
+
xc, yc = int(xc), int(yc)
|
|
109
|
+
|
|
110
|
+
# Subtract the left and top of the ROI to get the correct coordinates.
|
|
111
|
+
xc = xc - roi_left
|
|
112
|
+
yc = yc - roi_top
|
|
113
|
+
|
|
114
|
+
# Normalize by the ROI width and height.
|
|
115
|
+
xc /= roi_w
|
|
116
|
+
yc /= roi_h
|
|
117
|
+
|
|
118
|
+
w = element["width"] / roi_w
|
|
119
|
+
h = element["height"] / roi_h
|
|
120
|
+
|
|
121
|
+
labels += f"{class_map[el_class]} {xc:.4f} {yc:.4f} {w:.4f} {h:.4f}\n"
|
|
122
|
+
|
|
123
|
+
# Save the label to a text file.
|
|
124
|
+
label_fp = label_dir / f"{fn}.txt"
|
|
125
|
+
|
|
126
|
+
with open(label_fp, "w") as fh:
|
|
127
|
+
fh.write(labels.strip())
|
|
128
|
+
|
|
129
|
+
# Append all the metadata to the list.
|
|
130
|
+
roi_metadata.append(
|
|
131
|
+
[item_id, img_fp, roi_left, roi_top, roi_w, roi_h, mag, scan_mag]
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
return pd.DataFrame(
|
|
135
|
+
roi_metadata, columns=["item_id", "fp", "x", "y", "w", "h", "mag", "scan_mag"]
|
|
136
|
+
)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from geopandas import GeoDataFrame
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import cv2 as cv
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from ... import imread, imwrite
|
|
8
|
+
from .utils import read_yolo_label, corners_to_polygon
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def tile_image(
|
|
12
|
+
fp: str,
|
|
13
|
+
label_fp: str | None = None,
|
|
14
|
+
save_dir: str = ".",
|
|
15
|
+
tile_size: int = 1280,
|
|
16
|
+
stride: int = None,
|
|
17
|
+
boundary_thr: float = 0.2,
|
|
18
|
+
fill: tuple[int, int, int] = (114, 114, 114),
|
|
19
|
+
box_thr: float = 0.2,
|
|
20
|
+
) -> pd.DataFrame:
|
|
21
|
+
"""Tile an ROI image with labels.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
fp: Image filepath, should be in an '/images/ directory'.
|
|
25
|
+
save_dir: Location to save images and labels.
|
|
26
|
+
tile_size: Size of tile, uses square tiles only.
|
|
27
|
+
stride: Stride to use when tiling, if None then it is set equal to
|
|
28
|
+
tile_size (no overlap between tiles).
|
|
29
|
+
boundary_thr: If ROI has a boundary (for rotated ROIs) then a tile must
|
|
30
|
+
have sufficient area in boundary to be included (0.0 - 1.0).
|
|
31
|
+
fill: RGB when padding image.
|
|
32
|
+
box_thr: Area threshold of box that must be in a tile.
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
Metadata of tiles saved.
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
# Read the image.
|
|
39
|
+
img = imread(fp)
|
|
40
|
+
h, w = img.shape[:2]
|
|
41
|
+
|
|
42
|
+
# Get the name of the file.
|
|
43
|
+
name = Path(fp).stem
|
|
44
|
+
|
|
45
|
+
# Get the label if it exists.
|
|
46
|
+
labels = []
|
|
47
|
+
|
|
48
|
+
if label_fp is not None:
|
|
49
|
+
labels = read_yolo_label(label_fp, im_shape=(w, h), convert=True)
|
|
50
|
+
|
|
51
|
+
# Convert the labels into a GeoDataFrame.
|
|
52
|
+
label_df = []
|
|
53
|
+
|
|
54
|
+
for box in labels:
|
|
55
|
+
label = box[0]
|
|
56
|
+
x1, y1, x2, y2 = box[1:5].astype(int)
|
|
57
|
+
|
|
58
|
+
label_df.append([label, x1, y1, x2, y2, corners_to_polygon(x1, y1, x2, y2)])
|
|
59
|
+
|
|
60
|
+
label_df = GeoDataFrame(
|
|
61
|
+
label_df, columns=["label", "x1", "y1", "x2", "y2", "geometry"]
|
|
62
|
+
)
|
|
63
|
+
label_areas = label_df.area
|
|
64
|
+
|
|
65
|
+
# Depracated code for rotated ROIs.
|
|
66
|
+
roi = corners_to_polygon(0, 0, w, h)
|
|
67
|
+
|
|
68
|
+
# Make the directories to save images and label files.
|
|
69
|
+
img_dir = Path(save_dir) / "images"
|
|
70
|
+
img_dir.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
|
|
72
|
+
label_dir = Path(save_dir) / "labels"
|
|
73
|
+
label_dir.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
|
|
75
|
+
# Default stride is no-overlap.
|
|
76
|
+
if stride is None:
|
|
77
|
+
stride = tile_size # default behavior - no overlap
|
|
78
|
+
|
|
79
|
+
# Pad the image to avoid getting tiles not of the right size.
|
|
80
|
+
img = cv.copyMakeBorder(
|
|
81
|
+
img, 0, tile_size, 0, tile_size, cv.BORDER_CONSTANT, value=fill
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# Get the top left corner of each tile.
|
|
85
|
+
xys = list(((x, y) for x in range(0, w, stride) for y in range(0, h, stride)))
|
|
86
|
+
|
|
87
|
+
tile_df = [] # track tile data.
|
|
88
|
+
|
|
89
|
+
# Pre-calculate the number of pixels in tile that must be in ROI to include.
|
|
90
|
+
intersection_thr = tile_size**2 * boundary_thr
|
|
91
|
+
|
|
92
|
+
# loop through each tile coordinate
|
|
93
|
+
for xy in xys:
|
|
94
|
+
# Check if this tile is sufficiently in the boundary.
|
|
95
|
+
x1, y1 = xy
|
|
96
|
+
x2, y2 = x1 + tile_size, y1 + tile_size
|
|
97
|
+
|
|
98
|
+
tile_pol = corners_to_polygon(x1, y1, x2, y2)
|
|
99
|
+
intersection = roi.intersection(tile_pol).area
|
|
100
|
+
|
|
101
|
+
if intersection > intersection_thr:
|
|
102
|
+
# Get the tile image.
|
|
103
|
+
tile = img[y1:y2, x1:x2]
|
|
104
|
+
|
|
105
|
+
# Create a name for the tile image / label.
|
|
106
|
+
fn = f"{name}_x{x1}y{y1}x{x2}y{y2}"
|
|
107
|
+
|
|
108
|
+
img_fp = img_dir / f"{fn}.png"
|
|
109
|
+
|
|
110
|
+
tile_df.append([img_fp, x1, y1, x2, y2])
|
|
111
|
+
|
|
112
|
+
imwrite(str(img_fp), tile)
|
|
113
|
+
|
|
114
|
+
# Find all boxes that intersect
|
|
115
|
+
label_intersection = label_df.geometry.intersection(tile_pol).area
|
|
116
|
+
|
|
117
|
+
tile_boxes = label_df[label_intersection / label_areas > box_thr]
|
|
118
|
+
|
|
119
|
+
# save these as normalized labels, threshold the box edges.
|
|
120
|
+
if len(tile_boxes):
|
|
121
|
+
labels = ""
|
|
122
|
+
|
|
123
|
+
for _, r in tile_boxes.iterrows():
|
|
124
|
+
# Shift coordinates to be relative to this tile.
|
|
125
|
+
xx1 = np.clip(r.x1 - x1, 0, tile_size) / tile_size
|
|
126
|
+
yy1 = np.clip(r.y1 - y1, 0, tile_size) / tile_size
|
|
127
|
+
xx2 = np.clip(r.x2 - x1, 0, tile_size) / tile_size
|
|
128
|
+
yy2 = np.clip(r.y2 - y1, 0, tile_size) / tile_size
|
|
129
|
+
|
|
130
|
+
xc, yc = (xx1 + xx2) / 2, (yy1 + yy2) / 2
|
|
131
|
+
bw, bh = xx2 - xx1, yy2 - yy1
|
|
132
|
+
labels += f"{r.label} {xc:.4f} {yc:.4f} {bw:.4f} {bh:.4f}\n"
|
|
133
|
+
|
|
134
|
+
with open(str(label_dir / f"{fn}.txt"), "w") as fh:
|
|
135
|
+
fh.write(labels.strip())
|
|
136
|
+
|
|
137
|
+
return pd.DataFrame(tile_df, columns=["fp", "x1", "y1", "x2", "y2"])
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from shapely.geometry import Polygon
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def convert_box_type(box):
|
|
6
|
+
"""Convert a box type from YOLO format (x-center, y-center, box-width, box-height) to (x1, y1, x2, y2) where point 1 is the
|
|
7
|
+
top left corner of box and point 2 is the bottom right corner
|
|
8
|
+
|
|
9
|
+
INPUT
|
|
10
|
+
-----
|
|
11
|
+
box : array
|
|
12
|
+
[N, 4], each row a point and the format being (x-center, y-center, box-width, box-height)
|
|
13
|
+
|
|
14
|
+
RETURN
|
|
15
|
+
------
|
|
16
|
+
new_box : array
|
|
17
|
+
[N, 4] each row a point and the format x1, y1, x2, y2
|
|
18
|
+
|
|
19
|
+
"""
|
|
20
|
+
# get half the box height and width
|
|
21
|
+
half_bw = box[:, 2] / 2
|
|
22
|
+
half_bh = box[:, 3] / 2
|
|
23
|
+
|
|
24
|
+
new_box = np.zeros(box.shape, dtype=box.dtype)
|
|
25
|
+
new_box[:, 0] = box[:, 0] - half_bw
|
|
26
|
+
new_box[:, 1] = box[:, 1] - half_bh
|
|
27
|
+
new_box[:, 2] = box[:, 0] + half_bw
|
|
28
|
+
new_box[:, 3] = box[:, 1] + half_bh
|
|
29
|
+
|
|
30
|
+
return new_box
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def read_yolo_label(filepath, im_shape=None, shift=None, convert=False):
|
|
34
|
+
"""Read a yolo label text file. It may contain a confidence value for the labels or not, will handle both cases
|
|
35
|
+
|
|
36
|
+
INPUTS
|
|
37
|
+
------
|
|
38
|
+
filepath : str
|
|
39
|
+
the path of the text file
|
|
40
|
+
im_shape : tuple or int (default: None)
|
|
41
|
+
image width and height corresponding to the label, if an int it is assumed both are the same. Will scale coordinates
|
|
42
|
+
to int values instead of normalized if given
|
|
43
|
+
shift : tuple or int (default: None)
|
|
44
|
+
shift value in the x and y direction, if int it is assumed to be the same in both. These values will be subtracted and applied
|
|
45
|
+
after scaling if needed
|
|
46
|
+
convert : bool (default: False)
|
|
47
|
+
If True, convert the output boxes from yolo format (label, x-center, y-center, width, height, conf) to (label, x1, y1, x2, y2, conf)
|
|
48
|
+
where point 1 is the top left corner of box and point 2 is the bottom corner of box
|
|
49
|
+
|
|
50
|
+
RETURN
|
|
51
|
+
------
|
|
52
|
+
coords : array
|
|
53
|
+
coordinates array, [N, 4 or 5] depending if confidence was in input file
|
|
54
|
+
|
|
55
|
+
"""
|
|
56
|
+
coords = []
|
|
57
|
+
|
|
58
|
+
with open(filepath, "r") as fh:
|
|
59
|
+
for line in fh.readlines():
|
|
60
|
+
if len(line):
|
|
61
|
+
coords.append([float(ln) for ln in line.strip().split(" ")])
|
|
62
|
+
|
|
63
|
+
coords = np.array(coords)
|
|
64
|
+
|
|
65
|
+
# scale coords if needed
|
|
66
|
+
if im_shape is not None:
|
|
67
|
+
if isinstance(im_shape, int):
|
|
68
|
+
w, h = im_shape, im_shape
|
|
69
|
+
else:
|
|
70
|
+
w, h = im_shape[:2]
|
|
71
|
+
|
|
72
|
+
coords[:, 1] *= w
|
|
73
|
+
coords[:, 3] *= w
|
|
74
|
+
coords[:, 2] *= h
|
|
75
|
+
coords[:, 4] *= h
|
|
76
|
+
|
|
77
|
+
# shift coords
|
|
78
|
+
if shift is not None:
|
|
79
|
+
if isinstance(shift, int):
|
|
80
|
+
x_shift, y_shift = shift, shift
|
|
81
|
+
else:
|
|
82
|
+
x_shift, y_shift = shift[:2]
|
|
83
|
+
|
|
84
|
+
coords[:, 1] -= x_shift
|
|
85
|
+
coords[:, 2] -= y_shift
|
|
86
|
+
|
|
87
|
+
if convert:
|
|
88
|
+
coords[:, 1:5] = convert_box_type(coords[:, 1:5])
|
|
89
|
+
|
|
90
|
+
return coords
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def corners_to_polygon(x1: int, y1: int, x2: int, y2: int) -> Polygon:
|
|
94
|
+
"""Return a Polygon from shapely with the box coordinates given the top left and bottom right corners of a
|
|
95
|
+
rectangle (can be rotated).
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
x1, y1, x2, y2: Coordinates of the top left corner (point 1) and the bottom right corner (point 2) of a box.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
Shapely polygon object of the box.
|
|
102
|
+
|
|
103
|
+
"""
|
|
104
|
+
return Polygon([(x1, y1), (x2, y1), (x2, y2), (x1, y2)])
|
|
@@ -8,33 +8,47 @@ from PIL import Image
|
|
|
8
8
|
from typing import Union
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
def
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
11
|
+
def get_mongo_client(
|
|
12
|
+
host: str | None = None,
|
|
13
|
+
username: str | None = None,
|
|
14
|
+
password: str | None = None,
|
|
15
|
+
port: int | None = None,
|
|
16
|
+
) -> pymongo.MongoClient:
|
|
17
|
+
"""Get a mongo client by providing the key parts of the URL. By default
|
|
18
|
+
when the parameters are not passed, it sets them to default values first
|
|
19
|
+
by looking at key environment variables. If the environment variables are
|
|
20
|
+
not present it sets them to default values.
|
|
21
|
+
|
|
18
22
|
Args:
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
23
|
+
host (str, optional): The host name of the mongo server. If None it will
|
|
24
|
+
look for the environment variable "MONGO_HOST_NAME". If this is
|
|
25
|
+
missing it will default to "mongodb".
|
|
26
|
+
username (str, optional): The username to connect to the mongo server.
|
|
27
|
+
If None it will look for the environment variable
|
|
28
|
+
"MONGO_INITDB_ROOT_USERNAME". If this is missing it will default to
|
|
29
|
+
"docker".
|
|
30
|
+
password (str, optional): The password to connect to the mongo server.
|
|
31
|
+
If None it will look for the environment variable
|
|
32
|
+
"MONGO_INITDB_ROOT_PASSWORD". If this is missing it will default to
|
|
33
|
+
"docker".
|
|
34
|
+
port (int, optional): The port number of the mongo server. If None it
|
|
35
|
+
will look for the environment variable "MONGO_HOST_PORT". If this is
|
|
36
|
+
missing it will default to 27017.
|
|
37
|
+
|
|
22
38
|
Returns:
|
|
23
|
-
pymongo.
|
|
24
|
-
|
|
39
|
+
pymongo.MongoClient: The mongo client.
|
|
40
|
+
|
|
25
41
|
"""
|
|
26
|
-
if
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
database = getenv("MONGODB_DB")
|
|
42
|
+
if username is None:
|
|
43
|
+
username = getenv("MONGO_INITDB_ROOT_USERNAME", "docker")
|
|
44
|
+
if password is None:
|
|
45
|
+
password = getenv("MONGO_INITDB_ROOT_PASSWORD", "docker")
|
|
46
|
+
if host is None:
|
|
47
|
+
host = getenv("MONGO_HOST_NAME", "mongodb")
|
|
48
|
+
if port is None:
|
|
49
|
+
port = int(getenv("MONGO_HOST_PORT", 27017))
|
|
35
50
|
|
|
36
|
-
|
|
37
|
-
return mc[database]
|
|
51
|
+
return pymongo.MongoClient(f"mongodb://{username}:{password}@{host}:{port}")
|
|
38
52
|
|
|
39
53
|
|
|
40
54
|
def chunks(lst: list, n=500):
|
|
@@ -50,20 +64,21 @@ def chunks(lst: list, n=500):
|
|
|
50
64
|
"""
|
|
51
65
|
for i in range(0, len(lst), n):
|
|
52
66
|
yield lst[i : i + n]
|
|
53
|
-
|
|
54
|
-
|
|
67
|
+
|
|
68
|
+
|
|
55
69
|
def add_many_to_collection(
|
|
56
70
|
mongo_collection: pymongo.collection.Collection,
|
|
57
|
-
items: dict | list,
|
|
71
|
+
items: dict | list,
|
|
72
|
+
key: str = "_id",
|
|
58
73
|
) -> dict[str | dict]:
|
|
59
74
|
"""Add items to a mongo collection. For this project we always add items with a unique
|
|
60
75
|
user key.
|
|
61
|
-
|
|
76
|
+
|
|
62
77
|
Args:
|
|
63
78
|
mongo_collection (pymongo.collection.Collection): The mongo collection to add to.
|
|
64
79
|
items (dict | list): The items to add to the collection.
|
|
65
80
|
key (str): The key to use to identify the items.
|
|
66
|
-
|
|
81
|
+
|
|
67
82
|
Returns:
|
|
68
83
|
dict[str | dict]: The items added to the collection.
|
|
69
84
|
|
|
@@ -85,7 +100,9 @@ def add_many_to_collection(
|
|
|
85
100
|
return items
|
|
86
101
|
|
|
87
102
|
|
|
88
|
-
def get_img_from_db(
|
|
103
|
+
def get_img_from_db(
|
|
104
|
+
mongo_db: pymongo.database.Database, img_id: str
|
|
105
|
+
) -> Union[np.ndarray, None]:
|
|
89
106
|
"""Get an image from the database by its location id.
|
|
90
107
|
|
|
91
108
|
Args:
|
|
@@ -111,8 +128,8 @@ def get_img_from_db(mongo_db: pymongo.database.Database, img_id: str) -> Union[n
|
|
|
111
128
|
src_img = np.array(Image.open(byte_io))
|
|
112
129
|
|
|
113
130
|
return src_img
|
|
114
|
-
|
|
115
|
-
|
|
131
|
+
|
|
132
|
+
|
|
116
133
|
def add_img_to_db(
|
|
117
134
|
mongo_db: pymongo.database.Database,
|
|
118
135
|
img: Image.Image | np.ndarray,
|
|
@@ -122,7 +139,7 @@ def add_img_to_db(
|
|
|
122
139
|
Args:
|
|
123
140
|
mongo_db (pymongo.database.Database): The mongo database.
|
|
124
141
|
img (np.ndarray or PIL.Image.Image): The image.
|
|
125
|
-
|
|
142
|
+
|
|
126
143
|
Returns:
|
|
127
144
|
str: The image id in the database.
|
|
128
145
|
|
|
@@ -144,4 +161,4 @@ def add_img_to_db(
|
|
|
144
161
|
# Save the byte data to MongoDB using GridFS
|
|
145
162
|
file_id = fs.put(byte_data)
|
|
146
163
|
|
|
147
|
-
return file_id
|
|
164
|
+
return file_id
|