mapreader 1.3.1__tar.gz → 1.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapreader-1.3.1 → mapreader-1.3.2}/PKG-INFO +4 -3
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/__init__.py +5 -0
- mapreader-1.3.2/mapreader/__main__.py +6 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/_version.py +3 -3
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/annotator.py +33 -27
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/classifier.py +5 -5
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/load_annotations.py +9 -2
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/sheet_downloader.py +1 -1
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/images.py +4 -3
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/post_process.py +40 -23
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/PKG-INFO +4 -3
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/SOURCES.txt +1 -0
- mapreader-1.3.2/mapreader.egg-info/entry_points.txt +2 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/requires.txt +3 -2
- {mapreader-1.3.1 → mapreader-1.3.2}/setup.py +4 -3
- {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_annotator.py +4 -4
- mapreader-1.3.2/tests/test_post_processing.py +171 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_sheet_downloader.py +0 -6
- mapreader-1.3.1/mapreader.egg-info/entry_points.txt +0 -2
- mapreader-1.3.1/tests/test_post_processing.py +0 -152
- {mapreader-1.3.1 → mapreader-1.3.2}/LICENSE +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/README.md +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/utils.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/custom_models.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/datasets.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/data_structures.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/downloader.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/downloader_utils.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/tile_loading.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/tile_merging.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/geo_utils.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/loader.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/process.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/deepsolo_runner.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/dptext_detr_runner.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/__init__.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/compute_and_save_stats.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/slice_parallel.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/dependency_links.txt +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/not-zip-safe +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/top_level.txt +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/setup.cfg +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_geo_pipeline.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_import.py +0 -0
- {mapreader-1.3.1 → mapreader-1.3.2}/versioneer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mapreader
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.2
|
|
4
4
|
Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
|
|
5
5
|
Home-page: https://github.com/Living-with-machines/MapReader
|
|
6
6
|
Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
|
|
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
License-File: LICENSE
|
|
30
30
|
Requires-Dist: matplotlib<4.0.0,>=3.5.0
|
|
31
31
|
Requires-Dist: numpy<2.0.0,>=1.21.5
|
|
32
|
-
Requires-Dist: pandas
|
|
32
|
+
Requires-Dist: pandas>=2.0.0
|
|
33
33
|
Requires-Dist: pyproj<4.0.0,>=3.2.0
|
|
34
34
|
Requires-Dist: azure-storage-blob<13.0.0,>=12.9.0
|
|
35
35
|
Requires-Dist: aiohttp<4.0.0,>=3.8.1
|
|
@@ -53,10 +53,11 @@ Requires-Dist: simplekml<2.0.0,>=1.3.6
|
|
|
53
53
|
Requires-Dist: versioneer>=0.28
|
|
54
54
|
Requires-Dist: tqdm<5.0.0
|
|
55
55
|
Requires-Dist: torchinfo<2.0.0
|
|
56
|
-
Requires-Dist: openpyxl<4.0.0
|
|
56
|
+
Requires-Dist: openpyxl<4.0.0,>=3.1.2
|
|
57
57
|
Requires-Dist: geopandas<1.0.0
|
|
58
58
|
Requires-Dist: pyogrio>=0.7.2
|
|
59
59
|
Requires-Dist: cartopy>=0.22.0
|
|
60
|
+
Requires-Dist: joblib>=1.4.0
|
|
60
61
|
Provides-Extra: dev
|
|
61
62
|
Requires-Dist: pytest<9.0.0; extra == "dev"
|
|
62
63
|
Requires-Dist: pytest-cov<6.0.0,>=4.1.0; extra == "dev"
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2024-05-
|
|
11
|
+
"date": "2024-05-15T11:10:51+0100",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "1.3.
|
|
14
|
+
"full-revisionid": "282b8733c2c83412b979c2c234adda1432691862",
|
|
15
|
+
"version": "1.3.2"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -27,7 +27,7 @@ _CENTER_LAYOUT = widgets.Layout(
|
|
|
27
27
|
)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
class Annotator
|
|
30
|
+
class Annotator:
|
|
31
31
|
"""
|
|
32
32
|
Annotator class for annotating patches with labels.
|
|
33
33
|
|
|
@@ -227,28 +227,29 @@ class Annotator(pd.DataFrame):
|
|
|
227
227
|
delimiter=delimiter,
|
|
228
228
|
)
|
|
229
229
|
|
|
230
|
-
# initiate as a DataFrame
|
|
231
|
-
super().__init__(patch_df)
|
|
232
|
-
|
|
233
230
|
## pixel_bounds = x0, y0, x1, y1
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
231
|
+
patch_df["min_x"] = patch_df["pixel_bounds"].apply(lambda x: x[0])
|
|
232
|
+
patch_df["min_y"] = patch_df["pixel_bounds"].apply(lambda x: x[1])
|
|
233
|
+
patch_df["max_x"] = patch_df["pixel_bounds"].apply(lambda x: x[2])
|
|
234
|
+
patch_df["max_y"] = patch_df["pixel_bounds"].apply(lambda x: x[3])
|
|
238
235
|
|
|
239
236
|
# Sort by sortby column if provided
|
|
240
237
|
if isinstance(sortby, str):
|
|
241
|
-
if sortby in
|
|
238
|
+
if sortby in patch_df.columns:
|
|
242
239
|
self._sortby = sortby
|
|
243
240
|
self._ascending = ascending
|
|
244
241
|
else:
|
|
245
|
-
raise ValueError(
|
|
242
|
+
raise ValueError(
|
|
243
|
+
f"[ERROR] {sortby} is not a column in the patch DataFrame."
|
|
244
|
+
)
|
|
246
245
|
elif sortby is not None:
|
|
247
246
|
raise ValueError("[ERROR] ``sortby`` must be a string or None.")
|
|
248
247
|
else:
|
|
249
248
|
self._sortby = None
|
|
250
249
|
self._ascending = True
|
|
251
250
|
|
|
251
|
+
self.patch_df = patch_df
|
|
252
|
+
|
|
252
253
|
self._labels = labels
|
|
253
254
|
self.label_col = label_col
|
|
254
255
|
self.patch_paths_col = patch_paths_col
|
|
@@ -299,6 +300,9 @@ class Annotator(pd.DataFrame):
|
|
|
299
300
|
# Setup queue
|
|
300
301
|
self._queue = []
|
|
301
302
|
|
|
303
|
+
def __len__(self):
|
|
304
|
+
return len(self.patch_df)
|
|
305
|
+
|
|
302
306
|
@staticmethod
|
|
303
307
|
def _load_dataframes(
|
|
304
308
|
patch_paths: str | None = None,
|
|
@@ -499,7 +503,7 @@ class Annotator(pd.DataFrame):
|
|
|
499
503
|
|
|
500
504
|
return True
|
|
501
505
|
|
|
502
|
-
queue_df = self.copy(deep=True)
|
|
506
|
+
queue_df = self.patch_df.copy(deep=True)
|
|
503
507
|
queue_df = queue_df[queue_df[self.label_col].isna()] # only unlabelled
|
|
504
508
|
queue_df["eligible"] = queue_df.apply(check_eligibility, axis=1)
|
|
505
509
|
|
|
@@ -562,20 +566,20 @@ class Annotator(pd.DataFrame):
|
|
|
562
566
|
|
|
563
567
|
ix = self._queue[self.current_index]
|
|
564
568
|
|
|
565
|
-
min_x = self.at[ix, "min_x"]
|
|
566
|
-
min_y = self.at[ix, "min_y"]
|
|
569
|
+
min_x = self.patch_df.at[ix, "min_x"]
|
|
570
|
+
min_y = self.patch_df.at[ix, "min_y"]
|
|
567
571
|
|
|
568
572
|
# cannot assume all patches are same size
|
|
569
573
|
try:
|
|
570
|
-
height, width, _ = self.at[ix, "shape"]
|
|
574
|
+
height, width, _ = self.patch_df.at[ix, "shape"]
|
|
571
575
|
except KeyError:
|
|
572
|
-
im_path = self.at[ix, self.patch_paths_col]
|
|
576
|
+
im_path = self.patch_df.at[ix, self.patch_paths_col]
|
|
573
577
|
im = Image.open(im_path)
|
|
574
578
|
height = im.height
|
|
575
579
|
width = im.width
|
|
576
580
|
|
|
577
|
-
current_parent = self.at[ix, "parent_id"]
|
|
578
|
-
parent_frame = self.query(f"parent_id=='{current_parent}'")
|
|
581
|
+
current_parent = self.patch_df.at[ix, "parent_id"]
|
|
582
|
+
parent_frame = self.patch_df.query(f"parent_id=='{current_parent}'")
|
|
579
583
|
|
|
580
584
|
deltas = list(range(-self.surrounding, self.surrounding + 1))
|
|
581
585
|
y_and_x = list(
|
|
@@ -770,7 +774,7 @@ class Annotator(pd.DataFrame):
|
|
|
770
774
|
|
|
771
775
|
ix = self._queue[self.current_index]
|
|
772
776
|
|
|
773
|
-
img_path = self.at[ix, self.patch_paths_col]
|
|
777
|
+
img_path = self.patch_df.at[ix, self.patch_paths_col]
|
|
774
778
|
|
|
775
779
|
self.render()
|
|
776
780
|
return self.previous_index, self.current_index, img_path
|
|
@@ -794,7 +798,7 @@ class Annotator(pd.DataFrame):
|
|
|
794
798
|
|
|
795
799
|
ix = self._queue[self.current_index]
|
|
796
800
|
|
|
797
|
-
img_path = self.at[ix, self.patch_paths_col]
|
|
801
|
+
img_path = self.patch_df.at[ix, self.patch_paths_col]
|
|
798
802
|
|
|
799
803
|
self.render()
|
|
800
804
|
return self.previous_index, self.current_index, img_path
|
|
@@ -826,7 +830,7 @@ class Annotator(pd.DataFrame):
|
|
|
826
830
|
# disable skip button when at last example
|
|
827
831
|
button.disabled = self.current_index >= len(self) - 1
|
|
828
832
|
elif button.description != "submit":
|
|
829
|
-
if self.at[ix, self.label_col] == button.description:
|
|
833
|
+
if self.patch_df.at[ix, self.label_col] == button.description:
|
|
830
834
|
button.icon = "check"
|
|
831
835
|
else:
|
|
832
836
|
button.icon = ""
|
|
@@ -842,8 +846,8 @@ class Annotator(pd.DataFrame):
|
|
|
842
846
|
else:
|
|
843
847
|
display(image.convert("RGB"))
|
|
844
848
|
add_ins = []
|
|
845
|
-
if "url" in self.loc[ix].keys():
|
|
846
|
-
url = self.at[ix, "url"]
|
|
849
|
+
if "url" in self.patch_df.loc[ix].keys():
|
|
850
|
+
url = self.patch_df.at[ix, "url"]
|
|
847
851
|
text = f'<p><a href="{url}" target="_blank">Click to see entire map.</a></p>'
|
|
848
852
|
add_ins += [widgets.HTML(text)]
|
|
849
853
|
|
|
@@ -881,7 +885,7 @@ class Annotator(pd.DataFrame):
|
|
|
881
885
|
PIL.Image
|
|
882
886
|
A PIL.Image object of the image at the given index.
|
|
883
887
|
"""
|
|
884
|
-
image_path = self.at[ix, self.patch_paths_col]
|
|
888
|
+
image_path = self.patch_df.at[ix, self.patch_paths_col]
|
|
885
889
|
image = Image.open(image_path)
|
|
886
890
|
|
|
887
891
|
if self.resize_to is not None:
|
|
@@ -907,7 +911,7 @@ class Annotator(pd.DataFrame):
|
|
|
907
911
|
"""
|
|
908
912
|
# ix = self.iloc[self.current_index].name
|
|
909
913
|
ix = self._queue[self.current_index]
|
|
910
|
-
self.at[ix, self.label_col] = annotation
|
|
914
|
+
self.patch_df.at[ix, self.label_col] = annotation
|
|
911
915
|
if self.auto_save:
|
|
912
916
|
self._auto_save()
|
|
913
917
|
self._next_example()
|
|
@@ -950,7 +954,9 @@ class Annotator(pd.DataFrame):
|
|
|
950
954
|
A dataframe containing the labelled images and their associated
|
|
951
955
|
label index.
|
|
952
956
|
"""
|
|
953
|
-
filtered_df = self[self[self.label_col].notna()].copy(
|
|
957
|
+
filtered_df = self.patch_df[self.patch_df[self.label_col].notna()].copy(
|
|
958
|
+
deep=True
|
|
959
|
+
)
|
|
954
960
|
|
|
955
961
|
# force image_id to be index (incase of integer index)
|
|
956
962
|
# TODO: Force all indices to be integers so this is not needed
|
|
@@ -973,8 +979,8 @@ class Annotator(pd.DataFrame):
|
|
|
973
979
|
|
|
974
980
|
@property
|
|
975
981
|
def filtered(self) -> pd.DataFrame:
|
|
976
|
-
_filter = ~self[self.label_col].isna()
|
|
977
|
-
return self[_filter]
|
|
982
|
+
_filter = ~self.patch_df[self.label_col].isna()
|
|
983
|
+
return self.patch_df[_filter]
|
|
978
984
|
|
|
979
985
|
def render_complete(self):
|
|
980
986
|
"""
|
|
@@ -1367,7 +1367,7 @@ Use ``add_criterion`` to define one."
|
|
|
1367
1367
|
def _initialize_model(
|
|
1368
1368
|
self,
|
|
1369
1369
|
model_name: str,
|
|
1370
|
-
|
|
1370
|
+
weights: str | None = "DEFAULT",
|
|
1371
1371
|
last_layer_num_classes: str | int | None = "default",
|
|
1372
1372
|
) -> tuple[Any, int, bool]:
|
|
1373
1373
|
"""
|
|
@@ -1378,8 +1378,9 @@ Use ``add_criterion`` to define one."
|
|
|
1378
1378
|
----------
|
|
1379
1379
|
model_name : str
|
|
1380
1380
|
Name of a PyTorch model. See https://pytorch.org/vision/0.8/models.html for options.
|
|
1381
|
-
|
|
1382
|
-
|
|
1381
|
+
weights : str, optional
|
|
1382
|
+
Weights to load into the model. If ``"DEFAULT"``, loads the default weights for the chosen model.
|
|
1383
|
+
By default, ``"DEFAULT"``.
|
|
1383
1384
|
last_layer_num_classes : str or int, optional
|
|
1384
1385
|
Number of elements in the last layer. If ``"default"``, sets it to
|
|
1385
1386
|
the number of classes. By default, ``"default"``.
|
|
@@ -1408,8 +1409,7 @@ Use ``add_criterion`` to define one."
|
|
|
1408
1409
|
|
|
1409
1410
|
# Initialize these variables which will be set in this if statement.
|
|
1410
1411
|
# Each of these variables is model specific.
|
|
1411
|
-
model_dw = models.
|
|
1412
|
-
model_dw = model_dw(pretrained)
|
|
1412
|
+
model_dw = models.get_model(model_name, weights=weights)
|
|
1413
1413
|
input_size = (224, 224)
|
|
1414
1414
|
is_inception = False
|
|
1415
1415
|
|
|
@@ -395,7 +395,9 @@ Please check your image paths and update them if necessary.'
|
|
|
395
395
|
)
|
|
396
396
|
iter_ids.append(annots2review.iloc[image_idx].name)
|
|
397
397
|
# Add to reviewed
|
|
398
|
-
self.reviewed =
|
|
398
|
+
self.reviewed = pd.concat(
|
|
399
|
+
[self.reviewed, annots2review.iloc[image_idx : image_idx + 1]]
|
|
400
|
+
)
|
|
399
401
|
try:
|
|
400
402
|
self.reviewed.drop_duplicates(subset=[deduplicate_col])
|
|
401
403
|
except Exception:
|
|
@@ -416,9 +418,14 @@ Please check your image paths and update them if necessary.'
|
|
|
416
418
|
]:
|
|
417
419
|
list_input_ids = user_input_ids.split(",")
|
|
418
420
|
print(
|
|
419
|
-
f"[INFO] Options for labels
|
|
421
|
+
f"[INFO] Options for labels:{list(self.annotations[self.label_col].unique())}"
|
|
420
422
|
)
|
|
421
423
|
input_label = input("Enter new label: ")
|
|
424
|
+
if input_label not in list(self.annotations[self.label_col].unique()):
|
|
425
|
+
print(
|
|
426
|
+
f'[ERROR] Label "{input_label}" not found in the annotations. Please enter a valid label.'
|
|
427
|
+
)
|
|
428
|
+
continue
|
|
422
429
|
|
|
423
430
|
for input_id in list_input_ids:
|
|
424
431
|
input_id = int(input_id)
|
|
@@ -1289,7 +1289,6 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
|
|
|
1289
1289
|
pass
|
|
1290
1290
|
|
|
1291
1291
|
for feature in features:
|
|
1292
|
-
text_id = feature["wfs_id_no"]
|
|
1293
1292
|
coords = np.array(feature["geometry"]["coordinates"][0][0])
|
|
1294
1293
|
|
|
1295
1294
|
# Plot coordinates
|
|
@@ -1302,6 +1301,7 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
|
|
|
1302
1301
|
)
|
|
1303
1302
|
|
|
1304
1303
|
if add_id:
|
|
1304
|
+
text_id = feature["wfs_id_no"]
|
|
1305
1305
|
plt.text(
|
|
1306
1306
|
np.mean(coords[:, 0]) - 0.15,
|
|
1307
1307
|
np.mean(coords[:, 1]) - 0.05,
|
|
@@ -1392,7 +1392,6 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
|
|
|
1392
1392
|
min_x, min_y, max_x, max_y = self.patches[patch]["pixel_bounds"]
|
|
1393
1393
|
if width != max_x - min_x:
|
|
1394
1394
|
width = max_x - min_x
|
|
1395
|
-
img = img.crop((0, 0, width, height))
|
|
1396
1395
|
if height != max_y - min_y:
|
|
1397
1396
|
height = max_y - min_y
|
|
1398
1397
|
img = img.crop((0, 0, width, height))
|
|
@@ -1400,22 +1399,24 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
|
|
|
1400
1399
|
bands = img.getbands()
|
|
1401
1400
|
|
|
1402
1401
|
if calc_mean:
|
|
1403
|
-
if
|
|
1402
|
+
if "mean_pixel" in patch_keys:
|
|
1404
1403
|
calc_mean = False
|
|
1405
1404
|
if calc_std:
|
|
1406
|
-
if
|
|
1405
|
+
if "std_pixel" in patch_keys:
|
|
1407
1406
|
calc_std = False
|
|
1408
1407
|
|
|
1409
1408
|
img_stat = ImageStat.Stat(img)
|
|
1410
1409
|
|
|
1411
1410
|
if calc_mean:
|
|
1412
1411
|
img_mean = img_stat.mean
|
|
1412
|
+
self.patches[patch]["mean_pixel"] = np.mean(img_mean) / 255
|
|
1413
1413
|
for i, band in enumerate(bands):
|
|
1414
1414
|
# Calculate mean pixel values
|
|
1415
1415
|
self.patches[patch][f"mean_pixel_{band}"] = img_mean[i] / 255
|
|
1416
1416
|
|
|
1417
1417
|
if calc_std:
|
|
1418
1418
|
img_std = img_stat.stddev
|
|
1419
|
+
self.patches[patch]["std_pixel"] = np.mean(img_std) / 255
|
|
1419
1420
|
for i, band in enumerate(bands):
|
|
1420
1421
|
# Calculate std pixel values
|
|
1421
1422
|
self.patches[patch][f"std_pixel_{band}"] = img_std[i] / 255
|
|
@@ -8,8 +8,8 @@ import pandas as pd
|
|
|
8
8
|
from tqdm import tqdm
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
class
|
|
12
|
-
"""A class for
|
|
11
|
+
class PostProcessor:
|
|
12
|
+
"""A class for post-processing predictions on patches using the surrounding context.
|
|
13
13
|
|
|
14
14
|
Parameters
|
|
15
15
|
----------
|
|
@@ -56,12 +56,16 @@ class PatchDataFrame(pd.DataFrame):
|
|
|
56
56
|
else:
|
|
57
57
|
patch_df[["min_x", "min_y", "max_x", "max_y"]] = [*patch_df["pixel_bounds"]]
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
# set the patch_df attribute
|
|
60
|
+
self.patch_df = patch_df
|
|
60
61
|
|
|
61
62
|
self.labels_map = labels_map
|
|
62
63
|
self._label_patches = None
|
|
63
64
|
self.context = {}
|
|
64
65
|
|
|
66
|
+
def __len__(self):
|
|
67
|
+
return len(self.patch_df)
|
|
68
|
+
|
|
65
69
|
def get_context(
|
|
66
70
|
self,
|
|
67
71
|
labels: str | list,
|
|
@@ -75,7 +79,9 @@ class PatchDataFrame(pd.DataFrame):
|
|
|
75
79
|
"""
|
|
76
80
|
if isinstance(labels, str):
|
|
77
81
|
labels = [labels]
|
|
78
|
-
self._label_patches = self[
|
|
82
|
+
self._label_patches = self.patch_df[
|
|
83
|
+
self.patch_df["predicted_label"].isin(labels)
|
|
84
|
+
]
|
|
79
85
|
|
|
80
86
|
for id in tqdm(self._label_patches.index):
|
|
81
87
|
if id not in self.context:
|
|
@@ -89,28 +95,36 @@ class PatchDataFrame(pd.DataFrame):
|
|
|
89
95
|
id,
|
|
90
96
|
):
|
|
91
97
|
"""Get the context of the patch with the specified index."""
|
|
92
|
-
parent_id = self.loc[id, "parent_id"]
|
|
93
|
-
min_x = self.loc[id, "min_x"]
|
|
94
|
-
min_y = self.loc[id, "min_y"]
|
|
95
|
-
max_x = self.loc[id, "max_x"]
|
|
96
|
-
max_y = self.loc[id, "max_y"]
|
|
98
|
+
parent_id = self.patch_df.loc[id, "parent_id"]
|
|
99
|
+
min_x = self.patch_df.loc[id, "min_x"]
|
|
100
|
+
min_y = self.patch_df.loc[id, "min_y"]
|
|
101
|
+
max_x = self.patch_df.loc[id, "max_x"]
|
|
102
|
+
max_y = self.patch_df.loc[id, "max_y"]
|
|
97
103
|
|
|
98
104
|
context_grid = [
|
|
99
105
|
*product(
|
|
100
|
-
[
|
|
101
|
-
|
|
106
|
+
[
|
|
107
|
+
(self.patch_df["min_x"], min_x),
|
|
108
|
+
(min_x, max_x),
|
|
109
|
+
(max_x, self.patch_df["max_x"]),
|
|
110
|
+
],
|
|
111
|
+
[
|
|
112
|
+
(self.patch_df["min_y"], min_y),
|
|
113
|
+
(min_y, max_y),
|
|
114
|
+
(max_y, self.patch_df["max_y"]),
|
|
115
|
+
],
|
|
102
116
|
)
|
|
103
117
|
]
|
|
104
118
|
# reshape to min_x, min_y, max_x, max_y
|
|
105
119
|
context_grid = [(x[0][0], x[1][0], x[0][1], x[1][1]) for x in context_grid]
|
|
106
120
|
|
|
107
121
|
context_list = [
|
|
108
|
-
self[
|
|
109
|
-
(self["min_x"] == context_loc[0])
|
|
110
|
-
& (self["min_y"] == context_loc[1])
|
|
111
|
-
& (self["max_x"] == context_loc[2])
|
|
112
|
-
& (self["max_y"] == context_loc[3])
|
|
113
|
-
& (self["parent_id"] == parent_id)
|
|
122
|
+
self.patch_df[
|
|
123
|
+
(self.patch_df["min_x"] == context_loc[0])
|
|
124
|
+
& (self.patch_df["min_y"] == context_loc[1])
|
|
125
|
+
& (self.patch_df["max_x"] == context_loc[2])
|
|
126
|
+
& (self.patch_df["max_y"] == context_loc[3])
|
|
127
|
+
& (self.patch_df["parent_id"] == parent_id)
|
|
114
128
|
]
|
|
115
129
|
for context_loc in context_grid
|
|
116
130
|
]
|
|
@@ -163,7 +177,10 @@ class PatchDataFrame(pd.DataFrame):
|
|
|
163
177
|
"""Update the predictions of the patch with the specified index."""
|
|
164
178
|
context_list = self.context[id]
|
|
165
179
|
|
|
166
|
-
context_df = self[self.index.isin(context_list)]
|
|
180
|
+
context_df = self.patch_df[self.patch_df.index.isin(context_list)].copy(
|
|
181
|
+
deep=True
|
|
182
|
+
)
|
|
183
|
+
|
|
167
184
|
# drop central patch from context
|
|
168
185
|
context_df.drop(index=id, inplace=True)
|
|
169
186
|
|
|
@@ -172,11 +189,11 @@ class PatchDataFrame(pd.DataFrame):
|
|
|
172
189
|
|
|
173
190
|
prefix = "" if inplace else "new_"
|
|
174
191
|
if (not any(context_df["predicted_label"].isin(labels))) & (
|
|
175
|
-
self.loc[id, "conf"] < conf
|
|
192
|
+
self.patch_df.loc[id, "conf"] < conf
|
|
176
193
|
):
|
|
177
|
-
self.loc[id, f"{prefix}predicted_label"] = remap[
|
|
178
|
-
self.loc[id, "predicted_label"]
|
|
194
|
+
self.patch_df.loc[id, f"{prefix}predicted_label"] = remap[
|
|
195
|
+
self.patch_df.loc[id, "predicted_label"]
|
|
179
196
|
]
|
|
180
|
-
self.loc[id, f"{prefix}pred"] = label_index_dict[
|
|
181
|
-
self.loc[id, f"{prefix}predicted_label"]
|
|
197
|
+
self.patch_df.loc[id, f"{prefix}pred"] = label_index_dict[
|
|
198
|
+
self.patch_df.loc[id, f"{prefix}predicted_label"]
|
|
182
199
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mapreader
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.2
|
|
4
4
|
Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
|
|
5
5
|
Home-page: https://github.com/Living-with-machines/MapReader
|
|
6
6
|
Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
|
|
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
License-File: LICENSE
|
|
30
30
|
Requires-Dist: matplotlib<4.0.0,>=3.5.0
|
|
31
31
|
Requires-Dist: numpy<2.0.0,>=1.21.5
|
|
32
|
-
Requires-Dist: pandas
|
|
32
|
+
Requires-Dist: pandas>=2.0.0
|
|
33
33
|
Requires-Dist: pyproj<4.0.0,>=3.2.0
|
|
34
34
|
Requires-Dist: azure-storage-blob<13.0.0,>=12.9.0
|
|
35
35
|
Requires-Dist: aiohttp<4.0.0,>=3.8.1
|
|
@@ -53,10 +53,11 @@ Requires-Dist: simplekml<2.0.0,>=1.3.6
|
|
|
53
53
|
Requires-Dist: versioneer>=0.28
|
|
54
54
|
Requires-Dist: tqdm<5.0.0
|
|
55
55
|
Requires-Dist: torchinfo<2.0.0
|
|
56
|
-
Requires-Dist: openpyxl<4.0.0
|
|
56
|
+
Requires-Dist: openpyxl<4.0.0,>=3.1.2
|
|
57
57
|
Requires-Dist: geopandas<1.0.0
|
|
58
58
|
Requires-Dist: pyogrio>=0.7.2
|
|
59
59
|
Requires-Dist: cartopy>=0.22.0
|
|
60
|
+
Requires-Dist: joblib>=1.4.0
|
|
60
61
|
Provides-Extra: dev
|
|
61
62
|
Requires-Dist: pytest<9.0.0; extra == "dev"
|
|
62
63
|
Requires-Dist: pytest-cov<6.0.0,>=4.1.0; extra == "dev"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
matplotlib<4.0.0,>=3.5.0
|
|
2
2
|
numpy<2.0.0,>=1.21.5
|
|
3
|
-
pandas
|
|
3
|
+
pandas>=2.0.0
|
|
4
4
|
pyproj<4.0.0,>=3.2.0
|
|
5
5
|
azure-storage-blob<13.0.0,>=12.9.0
|
|
6
6
|
aiohttp<4.0.0,>=3.8.1
|
|
@@ -24,10 +24,11 @@ simplekml<2.0.0,>=1.3.6
|
|
|
24
24
|
versioneer>=0.28
|
|
25
25
|
tqdm<5.0.0
|
|
26
26
|
torchinfo<2.0.0
|
|
27
|
-
openpyxl<4.0.0
|
|
27
|
+
openpyxl<4.0.0,>=3.1.2
|
|
28
28
|
geopandas<1.0.0
|
|
29
29
|
pyogrio>=0.7.2
|
|
30
30
|
cartopy>=0.22.0
|
|
31
|
+
joblib>=1.4.0
|
|
31
32
|
|
|
32
33
|
[dev]
|
|
33
34
|
pytest<9.0.0
|
|
@@ -30,7 +30,7 @@ setuptools.setup(
|
|
|
30
30
|
install_requires=[
|
|
31
31
|
"matplotlib>=3.5.0,<4.0.0",
|
|
32
32
|
"numpy>=1.21.5,<2.0.0",
|
|
33
|
-
"pandas>=
|
|
33
|
+
"pandas>=2.0.0",
|
|
34
34
|
"pyproj>=3.2.0,<4.0.0",
|
|
35
35
|
"azure-storage-blob>=12.9.0,<13.0.0",
|
|
36
36
|
"aiohttp>=3.8.1,<4.0.0",
|
|
@@ -54,10 +54,11 @@ setuptools.setup(
|
|
|
54
54
|
"versioneer>=0.28",
|
|
55
55
|
"tqdm<5.0.0",
|
|
56
56
|
"torchinfo<2.0.0",
|
|
57
|
-
"openpyxl
|
|
57
|
+
"openpyxl>=3.1.2,<4.0.0",
|
|
58
58
|
"geopandas<1.0.0",
|
|
59
59
|
"pyogrio>=0.7.2",
|
|
60
60
|
"cartopy>=0.22.0",
|
|
61
|
+
"joblib>=1.4.0",
|
|
61
62
|
],
|
|
62
63
|
extras_require={
|
|
63
64
|
"dev": [
|
|
@@ -89,7 +90,7 @@ setuptools.setup(
|
|
|
89
90
|
],
|
|
90
91
|
entry_points={
|
|
91
92
|
"console_scripts": [
|
|
92
|
-
"mapreader = mapreader
|
|
93
|
+
"mapreader = mapreader:print_version",
|
|
93
94
|
],
|
|
94
95
|
},
|
|
95
96
|
)
|
|
@@ -35,7 +35,7 @@ def test_init_with_dfs(load_dfs):
|
|
|
35
35
|
auto_save=False,
|
|
36
36
|
)
|
|
37
37
|
assert len(annotator) == 9
|
|
38
|
-
assert isinstance(annotator.iloc[0]["coordinates"], tuple)
|
|
38
|
+
assert isinstance(annotator.patch_df.iloc[0]["coordinates"], tuple)
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def test_init_with_csvs(load_dfs):
|
|
@@ -48,7 +48,7 @@ def test_init_with_csvs(load_dfs):
|
|
|
48
48
|
auto_save=False,
|
|
49
49
|
)
|
|
50
50
|
assert len(annotator) == 9
|
|
51
|
-
assert isinstance(annotator.iloc[0]["coordinates"], tuple)
|
|
51
|
+
assert isinstance(annotator.patch_df.iloc[0]["coordinates"], tuple)
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
def test_init_with_fpaths(load_dfs, sample_dir):
|
|
@@ -62,7 +62,7 @@ def test_init_with_fpaths(load_dfs, sample_dir):
|
|
|
62
62
|
auto_save=False,
|
|
63
63
|
)
|
|
64
64
|
assert len(annotator) == 9
|
|
65
|
-
assert "mean_pixel_R" in annotator.columns
|
|
65
|
+
assert "mean_pixel_R" in annotator.patch_df.columns
|
|
66
66
|
|
|
67
67
|
|
|
68
68
|
def test_init_with_fpaths_tsv(load_dfs, sample_dir):
|
|
@@ -77,7 +77,7 @@ def test_init_with_fpaths_tsv(load_dfs, sample_dir):
|
|
|
77
77
|
delimiter="\t",
|
|
78
78
|
)
|
|
79
79
|
assert len(annotator) == 9
|
|
80
|
-
assert "mean_pixel_R" in annotator.columns
|
|
80
|
+
assert "mean_pixel_R" in annotator.patch_df.columns
|
|
81
81
|
|
|
82
82
|
|
|
83
83
|
def test_no_labels(load_dfs):
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from mapreader.process.post_process import PostProcessor
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def sample_dir():
|
|
13
|
+
return Path(__file__).resolve().parent / "sample_files"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.fixture
|
|
17
|
+
def patch_df(sample_dir):
|
|
18
|
+
return pd.read_csv(f"{sample_dir}/post_processing_patch_df.csv", index_col=0)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.fixture
|
|
22
|
+
def labels_map():
|
|
23
|
+
return {0: "no", 1: "railspace", 2: "building", 3: "railspace&building"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_init(labels_map, patch_df):
|
|
27
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
28
|
+
assert isinstance(patches, PostProcessor)
|
|
29
|
+
assert len(patches) == 81
|
|
30
|
+
assert patches.labels_map == labels_map
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_init_errors(patch_df, labels_map):
|
|
34
|
+
with pytest.raises(ValueError, match="must contain the following columns"):
|
|
35
|
+
PostProcessor(patch_df.drop(columns=["parent_id", "pred"]), labels_map)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_init_imaged_id_col(patch_df, labels_map):
|
|
39
|
+
# e.g. if you have integer index
|
|
40
|
+
patches = PostProcessor(patch_df.reset_index(drop=False), labels_map)
|
|
41
|
+
assert isinstance(patches, PostProcessor)
|
|
42
|
+
assert patches.patch_df.index.name == "image_id"
|
|
43
|
+
|
|
44
|
+
patch_df["image_id"] = patch_df.index
|
|
45
|
+
patches = PostProcessor(patch_df, labels_map)
|
|
46
|
+
assert isinstance(patches, PostProcessor)
|
|
47
|
+
assert patches.patch_df.index.name == "image_id"
|
|
48
|
+
assert len(patches) == 81
|
|
49
|
+
assert "image_id" in patches.patch_df.columns
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_get_context(patch_df, labels_map):
|
|
53
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
54
|
+
# labels as str
|
|
55
|
+
patches.get_context("railspace")
|
|
56
|
+
assert len(patches.context) == 10
|
|
57
|
+
# labels as list
|
|
58
|
+
patches.get_context(["railspace", "railspace&building"])
|
|
59
|
+
assert len(patches.context) == 13
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_update_preds_railspace(patch_df, labels_map):
|
|
63
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
64
|
+
patches.get_context(["railspace"])
|
|
65
|
+
remap = {"railspace": "no"}
|
|
66
|
+
patches.update_preds(remap)
|
|
67
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
|
|
68
|
+
assert (
|
|
69
|
+
patches.patch_df.loc[
|
|
70
|
+
"patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
71
|
+
]
|
|
72
|
+
== "no"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
76
|
+
patches.update_preds(remap, conf=0.8)
|
|
77
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 2
|
|
78
|
+
assert (
|
|
79
|
+
patches.patch_df.loc[
|
|
80
|
+
"patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
81
|
+
]
|
|
82
|
+
== "no"
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
86
|
+
patches.update_preds(remap, conf=1) # all conf == 1 should remain unchanged
|
|
87
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 3
|
|
88
|
+
assert (
|
|
89
|
+
patches.patch_df.loc[
|
|
90
|
+
"patch-1-4-2-5-#cropped_74488689.png#.png", "new_predicted_label"
|
|
91
|
+
]
|
|
92
|
+
== "no"
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_update_preds_railspace_railspace_building(patch_df, labels_map):
|
|
97
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
98
|
+
patches.get_context(["railspace", "railspace&building"])
|
|
99
|
+
remap = {"railspace": "no", "railspace&building": "building"}
|
|
100
|
+
patches.update_preds(remap)
|
|
101
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
|
|
102
|
+
assert (
|
|
103
|
+
patches.patch_df.loc[
|
|
104
|
+
"patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
105
|
+
]
|
|
106
|
+
== "building"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
110
|
+
patches.update_preds(remap, conf=0.8)
|
|
111
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 2
|
|
112
|
+
assert (
|
|
113
|
+
patches.patch_df.loc[
|
|
114
|
+
"patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
115
|
+
]
|
|
116
|
+
== "no"
|
|
117
|
+
)
|
|
118
|
+
assert (
|
|
119
|
+
patches.patch_df.loc[
|
|
120
|
+
"patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
121
|
+
]
|
|
122
|
+
== "building"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_update_preds_inplace(patch_df, labels_map):
|
|
127
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
128
|
+
patches.get_context(["railspace"])
|
|
129
|
+
remap = {"railspace": "no"}
|
|
130
|
+
patches.update_preds(remap, inplace=True)
|
|
131
|
+
assert "new_predicted_label" not in patches.patch_df.columns
|
|
132
|
+
assert (
|
|
133
|
+
patches.patch_df.loc[
|
|
134
|
+
"patch-4-7-5-8-#cropped_74488689.png#.png", "predicted_label"
|
|
135
|
+
]
|
|
136
|
+
== "no"
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_update_preds_new_label(patch_df, labels_map):
|
|
141
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
142
|
+
patches.get_context(["railspace"])
|
|
143
|
+
remap = {"railspace": "new"}
|
|
144
|
+
patches.update_preds(remap)
|
|
145
|
+
assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
|
|
146
|
+
assert (
|
|
147
|
+
patches.patch_df.loc[
|
|
148
|
+
"patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"
|
|
149
|
+
]
|
|
150
|
+
== "new"
|
|
151
|
+
)
|
|
152
|
+
assert (
|
|
153
|
+
patches.patch_df.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_pred"]
|
|
154
|
+
== 4
|
|
155
|
+
)
|
|
156
|
+
assert patches.labels_map[4] == "new"
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def test_update_preds_errors(patch_df, labels_map):
|
|
160
|
+
patches = PostProcessor(patch_df, labels_map=labels_map)
|
|
161
|
+
remap = {"railspace": "no"}
|
|
162
|
+
with pytest.raises(ValueError, match="run `get_context` first"):
|
|
163
|
+
patches.update_preds(remap)
|
|
164
|
+
|
|
165
|
+
patches.get_context(["fake"])
|
|
166
|
+
with pytest.raises(ValueError, match="No patches to update"):
|
|
167
|
+
patches.update_preds(remap)
|
|
168
|
+
|
|
169
|
+
patches.get_context(["railspace"])
|
|
170
|
+
with pytest.raises(ValueError, match="must specify a remap"):
|
|
171
|
+
patches.update_preds(remap={"fake": "no"})
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
-
import re
|
|
5
4
|
from ast import literal_eval
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
|
|
@@ -504,11 +503,6 @@ def test_download_same_image_names(sheet_downloader, tmp_path, capfd):
|
|
|
504
503
|
sd.download_map_sheets_by_wfs_ids(
|
|
505
504
|
[107, 116], maps_path, metadata_fname, overwrite=True
|
|
506
505
|
) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
|
|
507
|
-
out, _ = capfd.readouterr()
|
|
508
|
-
assert re.search(
|
|
509
|
-
r"\[INFO\] Downloaded \".*\/test_maps\/map_101603986.png\"\n\[INFO\] Downloaded \".*\/test_maps\/map_101603986_1.png\"\n$",
|
|
510
|
-
out,
|
|
511
|
-
)
|
|
512
506
|
assert os.path.exists(f"{maps_path}/map_101603986.png")
|
|
513
507
|
assert os.path.exists(f"{maps_path}/map_101603986_1.png")
|
|
514
508
|
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import pandas as pd
|
|
6
|
-
import pytest
|
|
7
|
-
|
|
8
|
-
from mapreader.process.post_process import PatchDataFrame
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@pytest.fixture
|
|
12
|
-
def sample_dir():
|
|
13
|
-
return Path(__file__).resolve().parent / "sample_files"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
@pytest.fixture
|
|
17
|
-
def patch_df(sample_dir):
|
|
18
|
-
return pd.read_csv(f"{sample_dir}/post_processing_patch_df.csv", index_col=0)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@pytest.fixture
|
|
22
|
-
def labels_map():
|
|
23
|
-
return {0: "no", 1: "railspace", 2: "building", 3: "railspace&building"}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def test_init(labels_map, patch_df):
|
|
27
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
28
|
-
assert isinstance(patches, PatchDataFrame)
|
|
29
|
-
assert len(patches) == 81
|
|
30
|
-
assert patches.labels_map == labels_map
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def test_init_errors(patch_df, labels_map):
|
|
34
|
-
with pytest.raises(ValueError, match="must contain the following columns"):
|
|
35
|
-
PatchDataFrame(patch_df.drop(columns=["parent_id", "pred"]), labels_map)
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def test_init_imaged_id_col(patch_df, labels_map):
|
|
39
|
-
# e.g. if you have integer index
|
|
40
|
-
patches = PatchDataFrame(patch_df.reset_index(drop=False), labels_map)
|
|
41
|
-
assert isinstance(patches, PatchDataFrame)
|
|
42
|
-
assert patches.index.name == "image_id"
|
|
43
|
-
|
|
44
|
-
patch_df["image_id"] = patch_df.index
|
|
45
|
-
patches = PatchDataFrame(patch_df, labels_map)
|
|
46
|
-
assert isinstance(patches, PatchDataFrame)
|
|
47
|
-
assert patches.index.name == "image_id"
|
|
48
|
-
assert len(patches) == 81
|
|
49
|
-
assert "image_id" in patches.columns
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def test_get_context(patch_df, labels_map):
|
|
53
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
54
|
-
# labels as str
|
|
55
|
-
patches.get_context("railspace")
|
|
56
|
-
assert len(patches.context) == 10
|
|
57
|
-
# labels as list
|
|
58
|
-
patches.get_context(["railspace", "railspace&building"])
|
|
59
|
-
assert len(patches.context) == 13
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def test_update_preds_railspace(patch_df, labels_map):
|
|
63
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
64
|
-
patches.get_context(["railspace"])
|
|
65
|
-
remap = {"railspace": "no"}
|
|
66
|
-
patches.update_preds(remap)
|
|
67
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 1
|
|
68
|
-
assert (
|
|
69
|
-
patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
70
|
-
== "no"
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
74
|
-
patches.update_preds(remap, conf=0.8)
|
|
75
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 2
|
|
76
|
-
assert (
|
|
77
|
-
patches.loc["patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
78
|
-
== "no"
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
82
|
-
patches.update_preds(remap, conf=1) # all conf == 1 should remain unchanged
|
|
83
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 3
|
|
84
|
-
assert (
|
|
85
|
-
patches.loc["patch-1-4-2-5-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
86
|
-
== "no"
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
def test_update_preds_railspace_railspace_building(patch_df, labels_map):
|
|
91
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
92
|
-
patches.get_context(["railspace", "railspace&building"])
|
|
93
|
-
remap = {"railspace": "no", "railspace&building": "building"}
|
|
94
|
-
patches.update_preds(remap)
|
|
95
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 1
|
|
96
|
-
assert (
|
|
97
|
-
patches.loc["patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
98
|
-
== "building"
|
|
99
|
-
)
|
|
100
|
-
|
|
101
|
-
patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
|
|
102
|
-
patches.update_preds(remap, conf=0.8)
|
|
103
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 2
|
|
104
|
-
assert (
|
|
105
|
-
patches.loc["patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
106
|
-
== "no"
|
|
107
|
-
)
|
|
108
|
-
assert (
|
|
109
|
-
patches.loc["patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
110
|
-
== "building"
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
def test_update_preds_inplace(patch_df, labels_map):
|
|
115
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
116
|
-
patches.get_context(["railspace"])
|
|
117
|
-
remap = {"railspace": "no"}
|
|
118
|
-
patches.update_preds(remap, inplace=True)
|
|
119
|
-
assert "new_predicted_label" not in patches.columns
|
|
120
|
-
assert (
|
|
121
|
-
patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "predicted_label"]
|
|
122
|
-
== "no"
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def test_update_preds_new_label(patch_df, labels_map):
|
|
127
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
128
|
-
patches.get_context(["railspace"])
|
|
129
|
-
remap = {"railspace": "new"}
|
|
130
|
-
patches.update_preds(remap)
|
|
131
|
-
assert len(patches[patches["new_predicted_label"].notna()]) == 1
|
|
132
|
-
assert (
|
|
133
|
-
patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"]
|
|
134
|
-
== "new"
|
|
135
|
-
)
|
|
136
|
-
assert patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_pred"] == 4
|
|
137
|
-
assert patches.labels_map[4] == "new"
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
def test_update_preds_errors(patch_df, labels_map):
|
|
141
|
-
patches = PatchDataFrame(patch_df, labels_map=labels_map)
|
|
142
|
-
remap = {"railspace": "no"}
|
|
143
|
-
with pytest.raises(ValueError, match="run `get_context` first"):
|
|
144
|
-
patches.update_preds(remap)
|
|
145
|
-
|
|
146
|
-
patches.get_context(["fake"])
|
|
147
|
-
with pytest.raises(ValueError, match="No patches to update"):
|
|
148
|
-
patches.update_preds(remap)
|
|
149
|
-
|
|
150
|
-
patches.get_context(["railspace"])
|
|
151
|
-
with pytest.raises(ValueError, match="must specify a remap"):
|
|
152
|
-
patches.update_preds(remap={"fake": "no"})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|