mapreader 1.3.1__tar.gz → 1.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {mapreader-1.3.1 → mapreader-1.3.2}/PKG-INFO +4 -3
  2. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/__init__.py +5 -0
  3. mapreader-1.3.2/mapreader/__main__.py +6 -0
  4. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/_version.py +3 -3
  5. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/annotator.py +33 -27
  6. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/classifier.py +5 -5
  7. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/load_annotations.py +9 -2
  8. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/sheet_downloader.py +1 -1
  9. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/images.py +4 -3
  10. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/post_process.py +40 -23
  11. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/PKG-INFO +4 -3
  12. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/SOURCES.txt +1 -0
  13. mapreader-1.3.2/mapreader.egg-info/entry_points.txt +2 -0
  14. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/requires.txt +3 -2
  15. {mapreader-1.3.1 → mapreader-1.3.2}/setup.py +4 -3
  16. {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_annotator.py +4 -4
  17. mapreader-1.3.2/tests/test_post_processing.py +171 -0
  18. {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_sheet_downloader.py +0 -6
  19. mapreader-1.3.1/mapreader.egg-info/entry_points.txt +0 -2
  20. mapreader-1.3.1/tests/test_post_processing.py +0 -152
  21. {mapreader-1.3.1 → mapreader-1.3.2}/LICENSE +0 -0
  22. {mapreader-1.3.1 → mapreader-1.3.2}/README.md +0 -0
  23. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/__init__.py +0 -0
  24. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/annotate/utils.py +0 -0
  25. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/__init__.py +0 -0
  26. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/custom_models.py +0 -0
  27. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/classify/datasets.py +0 -0
  28. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/__init__.py +0 -0
  29. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/data_structures.py +0 -0
  30. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/downloader.py +0 -0
  31. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/downloader_utils.py +0 -0
  32. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/tile_loading.py +0 -0
  33. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/download/tile_merging.py +0 -0
  34. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/__init__.py +0 -0
  35. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/geo_utils.py +0 -0
  36. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/load/loader.py +0 -0
  37. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/__init__.py +0 -0
  38. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/process/process.py +0 -0
  39. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/__init__.py +0 -0
  40. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/deepsolo_runner.py +0 -0
  41. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/spot_text/dptext_detr_runner.py +0 -0
  42. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/__init__.py +0 -0
  43. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/compute_and_save_stats.py +0 -0
  44. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader/utils/slice_parallel.py +0 -0
  45. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/dependency_links.txt +0 -0
  46. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/not-zip-safe +0 -0
  47. {mapreader-1.3.1 → mapreader-1.3.2}/mapreader.egg-info/top_level.txt +0 -0
  48. {mapreader-1.3.1 → mapreader-1.3.2}/setup.cfg +0 -0
  49. {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_geo_pipeline.py +0 -0
  50. {mapreader-1.3.1 → mapreader-1.3.2}/tests/test_import.py +0 -0
  51. {mapreader-1.3.1 → mapreader-1.3.2}/versioneer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mapreader
3
- Version: 1.3.1
3
+ Version: 1.3.2
4
4
  Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
5
5
  Home-page: https://github.com/Living-with-machines/MapReader
6
6
  Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
29
29
  License-File: LICENSE
30
30
  Requires-Dist: matplotlib<4.0.0,>=3.5.0
31
31
  Requires-Dist: numpy<2.0.0,>=1.21.5
32
- Requires-Dist: pandas<2.0.0,>=1.3.4
32
+ Requires-Dist: pandas>=2.0.0
33
33
  Requires-Dist: pyproj<4.0.0,>=3.2.0
34
34
  Requires-Dist: azure-storage-blob<13.0.0,>=12.9.0
35
35
  Requires-Dist: aiohttp<4.0.0,>=3.8.1
@@ -53,10 +53,11 @@ Requires-Dist: simplekml<2.0.0,>=1.3.6
53
53
  Requires-Dist: versioneer>=0.28
54
54
  Requires-Dist: tqdm<5.0.0
55
55
  Requires-Dist: torchinfo<2.0.0
56
- Requires-Dist: openpyxl<4.0.0
56
+ Requires-Dist: openpyxl<4.0.0,>=3.1.2
57
57
  Requires-Dist: geopandas<1.0.0
58
58
  Requires-Dist: pyogrio>=0.7.2
59
59
  Requires-Dist: cartopy>=0.22.0
60
+ Requires-Dist: joblib>=1.4.0
60
61
  Provides-Extra: dev
61
62
  Requires-Dist: pytest<9.0.0; extra == "dev"
62
63
  Requires-Dist: pytest-cov<6.0.0,>=4.1.0; extra == "dev"
@@ -31,3 +31,8 @@ from . import _version
31
31
  __version__ = _version.get_versions()["version"]
32
32
 
33
33
  from mapreader.load import geo_utils
34
+
35
+ import mapreader
36
+
37
+ def print_version():
38
+ print(mapreader.__version__)
@@ -0,0 +1,6 @@
1
+ from __future__ import annotations
2
+
3
+ from . import print_version
4
+
5
+ if __name__ == "__main__":
6
+ print_version()
@@ -8,11 +8,11 @@ import json
8
8
 
9
9
  version_json = '''
10
10
  {
11
- "date": "2024-05-03T13:32:40+0100",
11
+ "date": "2024-05-15T11:10:51+0100",
12
12
  "dirty": false,
13
13
  "error": null,
14
- "full-revisionid": "3d7ab10b0f7bc0833e122a65a4d4bd608ca62fcd",
15
- "version": "1.3.1"
14
+ "full-revisionid": "282b8733c2c83412b979c2c234adda1432691862",
15
+ "version": "1.3.2"
16
16
  }
17
17
  ''' # END VERSION_JSON
18
18
 
@@ -27,7 +27,7 @@ _CENTER_LAYOUT = widgets.Layout(
27
27
  )
28
28
 
29
29
 
30
- class Annotator(pd.DataFrame):
30
+ class Annotator:
31
31
  """
32
32
  Annotator class for annotating patches with labels.
33
33
 
@@ -227,28 +227,29 @@ class Annotator(pd.DataFrame):
227
227
  delimiter=delimiter,
228
228
  )
229
229
 
230
- # initiate as a DataFrame
231
- super().__init__(patch_df)
232
-
233
230
  ## pixel_bounds = x0, y0, x1, y1
234
- self["min_x"] = self.pixel_bounds.apply(lambda x: x[0])
235
- self["min_y"] = self.pixel_bounds.apply(lambda x: x[1])
236
- self["max_x"] = self.pixel_bounds.apply(lambda x: x[2])
237
- self["max_y"] = self.pixel_bounds.apply(lambda x: x[3])
231
+ patch_df["min_x"] = patch_df["pixel_bounds"].apply(lambda x: x[0])
232
+ patch_df["min_y"] = patch_df["pixel_bounds"].apply(lambda x: x[1])
233
+ patch_df["max_x"] = patch_df["pixel_bounds"].apply(lambda x: x[2])
234
+ patch_df["max_y"] = patch_df["pixel_bounds"].apply(lambda x: x[3])
238
235
 
239
236
  # Sort by sortby column if provided
240
237
  if isinstance(sortby, str):
241
- if sortby in self.columns:
238
+ if sortby in patch_df.columns:
242
239
  self._sortby = sortby
243
240
  self._ascending = ascending
244
241
  else:
245
- raise ValueError(f"[ERROR] {sortby} is not a column in the DataFrame.")
242
+ raise ValueError(
243
+ f"[ERROR] {sortby} is not a column in the patch DataFrame."
244
+ )
246
245
  elif sortby is not None:
247
246
  raise ValueError("[ERROR] ``sortby`` must be a string or None.")
248
247
  else:
249
248
  self._sortby = None
250
249
  self._ascending = True
251
250
 
251
+ self.patch_df = patch_df
252
+
252
253
  self._labels = labels
253
254
  self.label_col = label_col
254
255
  self.patch_paths_col = patch_paths_col
@@ -299,6 +300,9 @@ class Annotator(pd.DataFrame):
299
300
  # Setup queue
300
301
  self._queue = []
301
302
 
303
+ def __len__(self):
304
+ return len(self.patch_df)
305
+
302
306
  @staticmethod
303
307
  def _load_dataframes(
304
308
  patch_paths: str | None = None,
@@ -499,7 +503,7 @@ class Annotator(pd.DataFrame):
499
503
 
500
504
  return True
501
505
 
502
- queue_df = self.copy(deep=True)
506
+ queue_df = self.patch_df.copy(deep=True)
503
507
  queue_df = queue_df[queue_df[self.label_col].isna()] # only unlabelled
504
508
  queue_df["eligible"] = queue_df.apply(check_eligibility, axis=1)
505
509
 
@@ -562,20 +566,20 @@ class Annotator(pd.DataFrame):
562
566
 
563
567
  ix = self._queue[self.current_index]
564
568
 
565
- min_x = self.at[ix, "min_x"]
566
- min_y = self.at[ix, "min_y"]
569
+ min_x = self.patch_df.at[ix, "min_x"]
570
+ min_y = self.patch_df.at[ix, "min_y"]
567
571
 
568
572
  # cannot assume all patches are same size
569
573
  try:
570
- height, width, _ = self.at[ix, "shape"]
574
+ height, width, _ = self.patch_df.at[ix, "shape"]
571
575
  except KeyError:
572
- im_path = self.at[ix, self.patch_paths_col]
576
+ im_path = self.patch_df.at[ix, self.patch_paths_col]
573
577
  im = Image.open(im_path)
574
578
  height = im.height
575
579
  width = im.width
576
580
 
577
- current_parent = self.at[ix, "parent_id"]
578
- parent_frame = self.query(f"parent_id=='{current_parent}'")
581
+ current_parent = self.patch_df.at[ix, "parent_id"]
582
+ parent_frame = self.patch_df.query(f"parent_id=='{current_parent}'")
579
583
 
580
584
  deltas = list(range(-self.surrounding, self.surrounding + 1))
581
585
  y_and_x = list(
@@ -770,7 +774,7 @@ class Annotator(pd.DataFrame):
770
774
 
771
775
  ix = self._queue[self.current_index]
772
776
 
773
- img_path = self.at[ix, self.patch_paths_col]
777
+ img_path = self.patch_df.at[ix, self.patch_paths_col]
774
778
 
775
779
  self.render()
776
780
  return self.previous_index, self.current_index, img_path
@@ -794,7 +798,7 @@ class Annotator(pd.DataFrame):
794
798
 
795
799
  ix = self._queue[self.current_index]
796
800
 
797
- img_path = self.at[ix, self.patch_paths_col]
801
+ img_path = self.patch_df.at[ix, self.patch_paths_col]
798
802
 
799
803
  self.render()
800
804
  return self.previous_index, self.current_index, img_path
@@ -826,7 +830,7 @@ class Annotator(pd.DataFrame):
826
830
  # disable skip button when at last example
827
831
  button.disabled = self.current_index >= len(self) - 1
828
832
  elif button.description != "submit":
829
- if self.at[ix, self.label_col] == button.description:
833
+ if self.patch_df.at[ix, self.label_col] == button.description:
830
834
  button.icon = "check"
831
835
  else:
832
836
  button.icon = ""
@@ -842,8 +846,8 @@ class Annotator(pd.DataFrame):
842
846
  else:
843
847
  display(image.convert("RGB"))
844
848
  add_ins = []
845
- if "url" in self.loc[ix].keys():
846
- url = self.at[ix, "url"]
849
+ if "url" in self.patch_df.loc[ix].keys():
850
+ url = self.patch_df.at[ix, "url"]
847
851
  text = f'<p><a href="{url}" target="_blank">Click to see entire map.</a></p>'
848
852
  add_ins += [widgets.HTML(text)]
849
853
 
@@ -881,7 +885,7 @@ class Annotator(pd.DataFrame):
881
885
  PIL.Image
882
886
  A PIL.Image object of the image at the given index.
883
887
  """
884
- image_path = self.at[ix, self.patch_paths_col]
888
+ image_path = self.patch_df.at[ix, self.patch_paths_col]
885
889
  image = Image.open(image_path)
886
890
 
887
891
  if self.resize_to is not None:
@@ -907,7 +911,7 @@ class Annotator(pd.DataFrame):
907
911
  """
908
912
  # ix = self.iloc[self.current_index].name
909
913
  ix = self._queue[self.current_index]
910
- self.at[ix, self.label_col] = annotation
914
+ self.patch_df.at[ix, self.label_col] = annotation
911
915
  if self.auto_save:
912
916
  self._auto_save()
913
917
  self._next_example()
@@ -950,7 +954,9 @@ class Annotator(pd.DataFrame):
950
954
  A dataframe containing the labelled images and their associated
951
955
  label index.
952
956
  """
953
- filtered_df = self[self[self.label_col].notna()].copy(deep=True)
957
+ filtered_df = self.patch_df[self.patch_df[self.label_col].notna()].copy(
958
+ deep=True
959
+ )
954
960
 
955
961
  # force image_id to be index (incase of integer index)
956
962
  # TODO: Force all indices to be integers so this is not needed
@@ -973,8 +979,8 @@ class Annotator(pd.DataFrame):
973
979
 
974
980
  @property
975
981
  def filtered(self) -> pd.DataFrame:
976
- _filter = ~self[self.label_col].isna()
977
- return self[_filter]
982
+ _filter = ~self.patch_df[self.label_col].isna()
983
+ return self.patch_df[_filter]
978
984
 
979
985
  def render_complete(self):
980
986
  """
@@ -1367,7 +1367,7 @@ Use ``add_criterion`` to define one."
1367
1367
  def _initialize_model(
1368
1368
  self,
1369
1369
  model_name: str,
1370
- pretrained: bool | None = True,
1370
+ weights: str | None = "DEFAULT",
1371
1371
  last_layer_num_classes: str | int | None = "default",
1372
1372
  ) -> tuple[Any, int, bool]:
1373
1373
  """
@@ -1378,8 +1378,9 @@ Use ``add_criterion`` to define one."
1378
1378
  ----------
1379
1379
  model_name : str
1380
1380
  Name of a PyTorch model. See https://pytorch.org/vision/0.8/models.html for options.
1381
- pretrained : bool, optional
1382
- Use pretrained version, by default ``True``
1381
+ weights : str, optional
1382
+ Weights to load into the model. If ``"DEFAULT"``, loads the default weights for the chosen model.
1383
+ By default, ``"DEFAULT"``.
1383
1384
  last_layer_num_classes : str or int, optional
1384
1385
  Number of elements in the last layer. If ``"default"``, sets it to
1385
1386
  the number of classes. By default, ``"default"``.
@@ -1408,8 +1409,7 @@ Use ``add_criterion`` to define one."
1408
1409
 
1409
1410
  # Initialize these variables which will be set in this if statement.
1410
1411
  # Each of these variables is model specific.
1411
- model_dw = models.__getattribute__(model_name)
1412
- model_dw = model_dw(pretrained)
1412
+ model_dw = models.get_model(model_name, weights=weights)
1413
1413
  input_size = (224, 224)
1414
1414
  is_inception = False
1415
1415
 
@@ -395,7 +395,9 @@ Please check your image paths and update them if necessary.'
395
395
  )
396
396
  iter_ids.append(annots2review.iloc[image_idx].name)
397
397
  # Add to reviewed
398
- self.reviewed = self.reviewed.append(annots2review.iloc[image_idx])
398
+ self.reviewed = pd.concat(
399
+ [self.reviewed, annots2review.iloc[image_idx : image_idx + 1]]
400
+ )
399
401
  try:
400
402
  self.reviewed.drop_duplicates(subset=[deduplicate_col])
401
403
  except Exception:
@@ -416,9 +418,14 @@ Please check your image paths and update them if necessary.'
416
418
  ]:
417
419
  list_input_ids = user_input_ids.split(",")
418
420
  print(
419
- f"[INFO] Options for labels (or create a new label):{list(self.annotations[self.label_col].unique())}"
421
+ f"[INFO] Options for labels:{list(self.annotations[self.label_col].unique())}"
420
422
  )
421
423
  input_label = input("Enter new label: ")
424
+ if input_label not in list(self.annotations[self.label_col].unique()):
425
+ print(
426
+ f'[ERROR] Label "{input_label}" not found in the annotations. Please enter a valid label.'
427
+ )
428
+ continue
422
429
 
423
430
  for input_id in list_input_ids:
424
431
  input_id = int(input_id)
@@ -1289,7 +1289,6 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
1289
1289
  pass
1290
1290
 
1291
1291
  for feature in features:
1292
- text_id = feature["wfs_id_no"]
1293
1292
  coords = np.array(feature["geometry"]["coordinates"][0][0])
1294
1293
 
1295
1294
  # Plot coordinates
@@ -1302,6 +1301,7 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
1302
1301
  )
1303
1302
 
1304
1303
  if add_id:
1304
+ text_id = feature["wfs_id_no"]
1305
1305
  plt.text(
1306
1306
  np.mean(coords[:, 0]) - 0.15,
1307
1307
  np.mean(coords[:, 1]) - 0.05,
@@ -1392,7 +1392,6 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
1392
1392
  min_x, min_y, max_x, max_y = self.patches[patch]["pixel_bounds"]
1393
1393
  if width != max_x - min_x:
1394
1394
  width = max_x - min_x
1395
- img = img.crop((0, 0, width, height))
1396
1395
  if height != max_y - min_y:
1397
1396
  height = max_y - min_y
1398
1397
  img = img.crop((0, 0, width, height))
@@ -1400,22 +1399,24 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
1400
1399
  bands = img.getbands()
1401
1400
 
1402
1401
  if calc_mean:
1403
- if all(f"mean_pixel_{band}" in patch_keys for band in bands):
1402
+ if "mean_pixel" in patch_keys:
1404
1403
  calc_mean = False
1405
1404
  if calc_std:
1406
- if all(f"std_pixel_{band}" in patch_keys for band in bands):
1405
+ if "std_pixel" in patch_keys:
1407
1406
  calc_std = False
1408
1407
 
1409
1408
  img_stat = ImageStat.Stat(img)
1410
1409
 
1411
1410
  if calc_mean:
1412
1411
  img_mean = img_stat.mean
1412
+ self.patches[patch]["mean_pixel"] = np.mean(img_mean) / 255
1413
1413
  for i, band in enumerate(bands):
1414
1414
  # Calculate mean pixel values
1415
1415
  self.patches[patch][f"mean_pixel_{band}"] = img_mean[i] / 255
1416
1416
 
1417
1417
  if calc_std:
1418
1418
  img_std = img_stat.stddev
1419
+ self.patches[patch]["std_pixel"] = np.mean(img_std) / 255
1419
1420
  for i, band in enumerate(bands):
1420
1421
  # Calculate std pixel values
1421
1422
  self.patches[patch][f"std_pixel_{band}"] = img_std[i] / 255
@@ -8,8 +8,8 @@ import pandas as pd
8
8
  from tqdm import tqdm
9
9
 
10
10
 
11
- class PatchDataFrame(pd.DataFrame):
12
- """A class for storing patch dataframes.
11
+ class PostProcessor:
12
+ """A class for post-processing predictions on patches using the surrounding context.
13
13
 
14
14
  Parameters
15
15
  ----------
@@ -56,12 +56,16 @@ class PatchDataFrame(pd.DataFrame):
56
56
  else:
57
57
  patch_df[["min_x", "min_y", "max_x", "max_y"]] = [*patch_df["pixel_bounds"]]
58
58
 
59
- super().__init__(patch_df)
59
+ # set the patch_df attribute
60
+ self.patch_df = patch_df
60
61
 
61
62
  self.labels_map = labels_map
62
63
  self._label_patches = None
63
64
  self.context = {}
64
65
 
66
+ def __len__(self):
67
+ return len(self.patch_df)
68
+
65
69
  def get_context(
66
70
  self,
67
71
  labels: str | list,
@@ -75,7 +79,9 @@ class PatchDataFrame(pd.DataFrame):
75
79
  """
76
80
  if isinstance(labels, str):
77
81
  labels = [labels]
78
- self._label_patches = self[self["predicted_label"].isin(labels)]
82
+ self._label_patches = self.patch_df[
83
+ self.patch_df["predicted_label"].isin(labels)
84
+ ]
79
85
 
80
86
  for id in tqdm(self._label_patches.index):
81
87
  if id not in self.context:
@@ -89,28 +95,36 @@ class PatchDataFrame(pd.DataFrame):
89
95
  id,
90
96
  ):
91
97
  """Get the context of the patch with the specified index."""
92
- parent_id = self.loc[id, "parent_id"]
93
- min_x = self.loc[id, "min_x"]
94
- min_y = self.loc[id, "min_y"]
95
- max_x = self.loc[id, "max_x"]
96
- max_y = self.loc[id, "max_y"]
98
+ parent_id = self.patch_df.loc[id, "parent_id"]
99
+ min_x = self.patch_df.loc[id, "min_x"]
100
+ min_y = self.patch_df.loc[id, "min_y"]
101
+ max_x = self.patch_df.loc[id, "max_x"]
102
+ max_y = self.patch_df.loc[id, "max_y"]
97
103
 
98
104
  context_grid = [
99
105
  *product(
100
- [(self["min_x"], min_x), (min_x, max_x), (max_x, self["max_x"])],
101
- [(self["min_y"], min_y), (min_y, max_y), (max_y, self["max_y"])],
106
+ [
107
+ (self.patch_df["min_x"], min_x),
108
+ (min_x, max_x),
109
+ (max_x, self.patch_df["max_x"]),
110
+ ],
111
+ [
112
+ (self.patch_df["min_y"], min_y),
113
+ (min_y, max_y),
114
+ (max_y, self.patch_df["max_y"]),
115
+ ],
102
116
  )
103
117
  ]
104
118
  # reshape to min_x, min_y, max_x, max_y
105
119
  context_grid = [(x[0][0], x[1][0], x[0][1], x[1][1]) for x in context_grid]
106
120
 
107
121
  context_list = [
108
- self[
109
- (self["min_x"] == context_loc[0])
110
- & (self["min_y"] == context_loc[1])
111
- & (self["max_x"] == context_loc[2])
112
- & (self["max_y"] == context_loc[3])
113
- & (self["parent_id"] == parent_id)
122
+ self.patch_df[
123
+ (self.patch_df["min_x"] == context_loc[0])
124
+ & (self.patch_df["min_y"] == context_loc[1])
125
+ & (self.patch_df["max_x"] == context_loc[2])
126
+ & (self.patch_df["max_y"] == context_loc[3])
127
+ & (self.patch_df["parent_id"] == parent_id)
114
128
  ]
115
129
  for context_loc in context_grid
116
130
  ]
@@ -163,7 +177,10 @@ class PatchDataFrame(pd.DataFrame):
163
177
  """Update the predictions of the patch with the specified index."""
164
178
  context_list = self.context[id]
165
179
 
166
- context_df = self[self.index.isin(context_list)]
180
+ context_df = self.patch_df[self.patch_df.index.isin(context_list)].copy(
181
+ deep=True
182
+ )
183
+
167
184
  # drop central patch from context
168
185
  context_df.drop(index=id, inplace=True)
169
186
 
@@ -172,11 +189,11 @@ class PatchDataFrame(pd.DataFrame):
172
189
 
173
190
  prefix = "" if inplace else "new_"
174
191
  if (not any(context_df["predicted_label"].isin(labels))) & (
175
- self.loc[id, "conf"] < conf
192
+ self.patch_df.loc[id, "conf"] < conf
176
193
  ):
177
- self.loc[id, f"{prefix}predicted_label"] = remap[
178
- self.loc[id, "predicted_label"]
194
+ self.patch_df.loc[id, f"{prefix}predicted_label"] = remap[
195
+ self.patch_df.loc[id, "predicted_label"]
179
196
  ]
180
- self.loc[id, f"{prefix}pred"] = label_index_dict[
181
- self.loc[id, f"{prefix}predicted_label"]
197
+ self.patch_df.loc[id, f"{prefix}pred"] = label_index_dict[
198
+ self.patch_df.loc[id, f"{prefix}predicted_label"]
182
199
  ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mapreader
3
- Version: 1.3.1
3
+ Version: 1.3.2
4
4
  Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
5
5
  Home-page: https://github.com/Living-with-machines/MapReader
6
6
  Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
29
29
  License-File: LICENSE
30
30
  Requires-Dist: matplotlib<4.0.0,>=3.5.0
31
31
  Requires-Dist: numpy<2.0.0,>=1.21.5
32
- Requires-Dist: pandas<2.0.0,>=1.3.4
32
+ Requires-Dist: pandas>=2.0.0
33
33
  Requires-Dist: pyproj<4.0.0,>=3.2.0
34
34
  Requires-Dist: azure-storage-blob<13.0.0,>=12.9.0
35
35
  Requires-Dist: aiohttp<4.0.0,>=3.8.1
@@ -53,10 +53,11 @@ Requires-Dist: simplekml<2.0.0,>=1.3.6
53
53
  Requires-Dist: versioneer>=0.28
54
54
  Requires-Dist: tqdm<5.0.0
55
55
  Requires-Dist: torchinfo<2.0.0
56
- Requires-Dist: openpyxl<4.0.0
56
+ Requires-Dist: openpyxl<4.0.0,>=3.1.2
57
57
  Requires-Dist: geopandas<1.0.0
58
58
  Requires-Dist: pyogrio>=0.7.2
59
59
  Requires-Dist: cartopy>=0.22.0
60
+ Requires-Dist: joblib>=1.4.0
60
61
  Provides-Extra: dev
61
62
  Requires-Dist: pytest<9.0.0; extra == "dev"
62
63
  Requires-Dist: pytest-cov<6.0.0,>=4.1.0; extra == "dev"
@@ -4,6 +4,7 @@ setup.cfg
4
4
  setup.py
5
5
  versioneer.py
6
6
  mapreader/__init__.py
7
+ mapreader/__main__.py
7
8
  mapreader/_version.py
8
9
  mapreader.egg-info/PKG-INFO
9
10
  mapreader.egg-info/SOURCES.txt
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ mapreader = mapreader:print_version
@@ -1,6 +1,6 @@
1
1
  matplotlib<4.0.0,>=3.5.0
2
2
  numpy<2.0.0,>=1.21.5
3
- pandas<2.0.0,>=1.3.4
3
+ pandas>=2.0.0
4
4
  pyproj<4.0.0,>=3.2.0
5
5
  azure-storage-blob<13.0.0,>=12.9.0
6
6
  aiohttp<4.0.0,>=3.8.1
@@ -24,10 +24,11 @@ simplekml<2.0.0,>=1.3.6
24
24
  versioneer>=0.28
25
25
  tqdm<5.0.0
26
26
  torchinfo<2.0.0
27
- openpyxl<4.0.0
27
+ openpyxl<4.0.0,>=3.1.2
28
28
  geopandas<1.0.0
29
29
  pyogrio>=0.7.2
30
30
  cartopy>=0.22.0
31
+ joblib>=1.4.0
31
32
 
32
33
  [dev]
33
34
  pytest<9.0.0
@@ -30,7 +30,7 @@ setuptools.setup(
30
30
  install_requires=[
31
31
  "matplotlib>=3.5.0,<4.0.0",
32
32
  "numpy>=1.21.5,<2.0.0",
33
- "pandas>=1.3.4,<2.0.0",
33
+ "pandas>=2.0.0",
34
34
  "pyproj>=3.2.0,<4.0.0",
35
35
  "azure-storage-blob>=12.9.0,<13.0.0",
36
36
  "aiohttp>=3.8.1,<4.0.0",
@@ -54,10 +54,11 @@ setuptools.setup(
54
54
  "versioneer>=0.28",
55
55
  "tqdm<5.0.0",
56
56
  "torchinfo<2.0.0",
57
- "openpyxl<4.0.0",
57
+ "openpyxl>=3.1.2,<4.0.0",
58
58
  "geopandas<1.0.0",
59
59
  "pyogrio>=0.7.2",
60
60
  "cartopy>=0.22.0",
61
+ "joblib>=1.4.0",
61
62
  ],
62
63
  extras_require={
63
64
  "dev": [
@@ -89,7 +90,7 @@ setuptools.setup(
89
90
  ],
90
91
  entry_points={
91
92
  "console_scripts": [
92
- "mapreader = mapreader.mapreader:main",
93
+ "mapreader = mapreader:print_version",
93
94
  ],
94
95
  },
95
96
  )
@@ -35,7 +35,7 @@ def test_init_with_dfs(load_dfs):
35
35
  auto_save=False,
36
36
  )
37
37
  assert len(annotator) == 9
38
- assert isinstance(annotator.iloc[0]["coordinates"], tuple)
38
+ assert isinstance(annotator.patch_df.iloc[0]["coordinates"], tuple)
39
39
 
40
40
 
41
41
  def test_init_with_csvs(load_dfs):
@@ -48,7 +48,7 @@ def test_init_with_csvs(load_dfs):
48
48
  auto_save=False,
49
49
  )
50
50
  assert len(annotator) == 9
51
- assert isinstance(annotator.iloc[0]["coordinates"], tuple)
51
+ assert isinstance(annotator.patch_df.iloc[0]["coordinates"], tuple)
52
52
 
53
53
 
54
54
  def test_init_with_fpaths(load_dfs, sample_dir):
@@ -62,7 +62,7 @@ def test_init_with_fpaths(load_dfs, sample_dir):
62
62
  auto_save=False,
63
63
  )
64
64
  assert len(annotator) == 9
65
- assert "mean_pixel_R" in annotator.columns
65
+ assert "mean_pixel_R" in annotator.patch_df.columns
66
66
 
67
67
 
68
68
  def test_init_with_fpaths_tsv(load_dfs, sample_dir):
@@ -77,7 +77,7 @@ def test_init_with_fpaths_tsv(load_dfs, sample_dir):
77
77
  delimiter="\t",
78
78
  )
79
79
  assert len(annotator) == 9
80
- assert "mean_pixel_R" in annotator.columns
80
+ assert "mean_pixel_R" in annotator.patch_df.columns
81
81
 
82
82
 
83
83
  def test_no_labels(load_dfs):
@@ -0,0 +1,171 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ import pandas as pd
6
+ import pytest
7
+
8
+ from mapreader.process.post_process import PostProcessor
9
+
10
+
11
+ @pytest.fixture
12
+ def sample_dir():
13
+ return Path(__file__).resolve().parent / "sample_files"
14
+
15
+
16
+ @pytest.fixture
17
+ def patch_df(sample_dir):
18
+ return pd.read_csv(f"{sample_dir}/post_processing_patch_df.csv", index_col=0)
19
+
20
+
21
+ @pytest.fixture
22
+ def labels_map():
23
+ return {0: "no", 1: "railspace", 2: "building", 3: "railspace&building"}
24
+
25
+
26
+ def test_init(labels_map, patch_df):
27
+ patches = PostProcessor(patch_df, labels_map=labels_map)
28
+ assert isinstance(patches, PostProcessor)
29
+ assert len(patches) == 81
30
+ assert patches.labels_map == labels_map
31
+
32
+
33
+ def test_init_errors(patch_df, labels_map):
34
+ with pytest.raises(ValueError, match="must contain the following columns"):
35
+ PostProcessor(patch_df.drop(columns=["parent_id", "pred"]), labels_map)
36
+
37
+
38
+ def test_init_imaged_id_col(patch_df, labels_map):
39
+ # e.g. if you have integer index
40
+ patches = PostProcessor(patch_df.reset_index(drop=False), labels_map)
41
+ assert isinstance(patches, PostProcessor)
42
+ assert patches.patch_df.index.name == "image_id"
43
+
44
+ patch_df["image_id"] = patch_df.index
45
+ patches = PostProcessor(patch_df, labels_map)
46
+ assert isinstance(patches, PostProcessor)
47
+ assert patches.patch_df.index.name == "image_id"
48
+ assert len(patches) == 81
49
+ assert "image_id" in patches.patch_df.columns
50
+
51
+
52
+ def test_get_context(patch_df, labels_map):
53
+ patches = PostProcessor(patch_df, labels_map=labels_map)
54
+ # labels as str
55
+ patches.get_context("railspace")
56
+ assert len(patches.context) == 10
57
+ # labels as list
58
+ patches.get_context(["railspace", "railspace&building"])
59
+ assert len(patches.context) == 13
60
+
61
+
62
+ def test_update_preds_railspace(patch_df, labels_map):
63
+ patches = PostProcessor(patch_df, labels_map=labels_map)
64
+ patches.get_context(["railspace"])
65
+ remap = {"railspace": "no"}
66
+ patches.update_preds(remap)
67
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
68
+ assert (
69
+ patches.patch_df.loc[
70
+ "patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"
71
+ ]
72
+ == "no"
73
+ )
74
+
75
+ patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
76
+ patches.update_preds(remap, conf=0.8)
77
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 2
78
+ assert (
79
+ patches.patch_df.loc[
80
+ "patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"
81
+ ]
82
+ == "no"
83
+ )
84
+
85
+ patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
86
+ patches.update_preds(remap, conf=1) # all conf == 1 should remain unchanged
87
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 3
88
+ assert (
89
+ patches.patch_df.loc[
90
+ "patch-1-4-2-5-#cropped_74488689.png#.png", "new_predicted_label"
91
+ ]
92
+ == "no"
93
+ )
94
+
95
+
96
+ def test_update_preds_railspace_railspace_building(patch_df, labels_map):
97
+ patches = PostProcessor(patch_df, labels_map=labels_map)
98
+ patches.get_context(["railspace", "railspace&building"])
99
+ remap = {"railspace": "no", "railspace&building": "building"}
100
+ patches.update_preds(remap)
101
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
102
+ assert (
103
+ patches.patch_df.loc[
104
+ "patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"
105
+ ]
106
+ == "building"
107
+ )
108
+
109
+ patches.patch_df.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
110
+ patches.update_preds(remap, conf=0.8)
111
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 2
112
+ assert (
113
+ patches.patch_df.loc[
114
+ "patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"
115
+ ]
116
+ == "no"
117
+ )
118
+ assert (
119
+ patches.patch_df.loc[
120
+ "patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"
121
+ ]
122
+ == "building"
123
+ )
124
+
125
+
126
+ def test_update_preds_inplace(patch_df, labels_map):
127
+ patches = PostProcessor(patch_df, labels_map=labels_map)
128
+ patches.get_context(["railspace"])
129
+ remap = {"railspace": "no"}
130
+ patches.update_preds(remap, inplace=True)
131
+ assert "new_predicted_label" not in patches.patch_df.columns
132
+ assert (
133
+ patches.patch_df.loc[
134
+ "patch-4-7-5-8-#cropped_74488689.png#.png", "predicted_label"
135
+ ]
136
+ == "no"
137
+ )
138
+
139
+
140
+ def test_update_preds_new_label(patch_df, labels_map):
141
+ patches = PostProcessor(patch_df, labels_map=labels_map)
142
+ patches.get_context(["railspace"])
143
+ remap = {"railspace": "new"}
144
+ patches.update_preds(remap)
145
+ assert len(patches.patch_df[patches.patch_df["new_predicted_label"].notna()]) == 1
146
+ assert (
147
+ patches.patch_df.loc[
148
+ "patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"
149
+ ]
150
+ == "new"
151
+ )
152
+ assert (
153
+ patches.patch_df.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_pred"]
154
+ == 4
155
+ )
156
+ assert patches.labels_map[4] == "new"
157
+
158
+
159
+ def test_update_preds_errors(patch_df, labels_map):
160
+ patches = PostProcessor(patch_df, labels_map=labels_map)
161
+ remap = {"railspace": "no"}
162
+ with pytest.raises(ValueError, match="run `get_context` first"):
163
+ patches.update_preds(remap)
164
+
165
+ patches.get_context(["fake"])
166
+ with pytest.raises(ValueError, match="No patches to update"):
167
+ patches.update_preds(remap)
168
+
169
+ patches.get_context(["railspace"])
170
+ with pytest.raises(ValueError, match="must specify a remap"):
171
+ patches.update_preds(remap={"fake": "no"})
@@ -1,7 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import os
4
- import re
5
4
  from ast import literal_eval
6
5
  from pathlib import Path
7
6
 
@@ -504,11 +503,6 @@ def test_download_same_image_names(sheet_downloader, tmp_path, capfd):
504
503
  sd.download_map_sheets_by_wfs_ids(
505
504
  [107, 116], maps_path, metadata_fname, overwrite=True
506
505
  ) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
507
- out, _ = capfd.readouterr()
508
- assert re.search(
509
- r"\[INFO\] Downloaded \".*\/test_maps\/map_101603986.png\"\n\[INFO\] Downloaded \".*\/test_maps\/map_101603986_1.png\"\n$",
510
- out,
511
- )
512
506
  assert os.path.exists(f"{maps_path}/map_101603986.png")
513
507
  assert os.path.exists(f"{maps_path}/map_101603986_1.png")
514
508
  assert os.path.exists(f"{maps_path}/{metadata_fname}")
@@ -1,2 +0,0 @@
1
- [console_scripts]
2
- mapreader = mapreader.mapreader:main
@@ -1,152 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from pathlib import Path
4
-
5
- import pandas as pd
6
- import pytest
7
-
8
- from mapreader.process.post_process import PatchDataFrame
9
-
10
-
11
- @pytest.fixture
12
- def sample_dir():
13
- return Path(__file__).resolve().parent / "sample_files"
14
-
15
-
16
- @pytest.fixture
17
- def patch_df(sample_dir):
18
- return pd.read_csv(f"{sample_dir}/post_processing_patch_df.csv", index_col=0)
19
-
20
-
21
- @pytest.fixture
22
- def labels_map():
23
- return {0: "no", 1: "railspace", 2: "building", 3: "railspace&building"}
24
-
25
-
26
- def test_init(labels_map, patch_df):
27
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
28
- assert isinstance(patches, PatchDataFrame)
29
- assert len(patches) == 81
30
- assert patches.labels_map == labels_map
31
-
32
-
33
- def test_init_errors(patch_df, labels_map):
34
- with pytest.raises(ValueError, match="must contain the following columns"):
35
- PatchDataFrame(patch_df.drop(columns=["parent_id", "pred"]), labels_map)
36
-
37
-
38
- def test_init_imaged_id_col(patch_df, labels_map):
39
- # e.g. if you have integer index
40
- patches = PatchDataFrame(patch_df.reset_index(drop=False), labels_map)
41
- assert isinstance(patches, PatchDataFrame)
42
- assert patches.index.name == "image_id"
43
-
44
- patch_df["image_id"] = patch_df.index
45
- patches = PatchDataFrame(patch_df, labels_map)
46
- assert isinstance(patches, PatchDataFrame)
47
- assert patches.index.name == "image_id"
48
- assert len(patches) == 81
49
- assert "image_id" in patches.columns
50
-
51
-
52
- def test_get_context(patch_df, labels_map):
53
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
54
- # labels as str
55
- patches.get_context("railspace")
56
- assert len(patches.context) == 10
57
- # labels as list
58
- patches.get_context(["railspace", "railspace&building"])
59
- assert len(patches.context) == 13
60
-
61
-
62
- def test_update_preds_railspace(patch_df, labels_map):
63
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
64
- patches.get_context(["railspace"])
65
- remap = {"railspace": "no"}
66
- patches.update_preds(remap)
67
- assert len(patches[patches["new_predicted_label"].notna()]) == 1
68
- assert (
69
- patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"]
70
- == "no"
71
- )
72
-
73
- patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
74
- patches.update_preds(remap, conf=0.8)
75
- assert len(patches[patches["new_predicted_label"].notna()]) == 2
76
- assert (
77
- patches.loc["patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"]
78
- == "no"
79
- )
80
-
81
- patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
82
- patches.update_preds(remap, conf=1) # all conf == 1 should remain unchanged
83
- assert len(patches[patches["new_predicted_label"].notna()]) == 3
84
- assert (
85
- patches.loc["patch-1-4-2-5-#cropped_74488689.png#.png", "new_predicted_label"]
86
- == "no"
87
- )
88
-
89
-
90
- def test_update_preds_railspace_railspace_building(patch_df, labels_map):
91
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
92
- patches.get_context(["railspace", "railspace&building"])
93
- remap = {"railspace": "no", "railspace&building": "building"}
94
- patches.update_preds(remap)
95
- assert len(patches[patches["new_predicted_label"].notna()]) == 1
96
- assert (
97
- patches.loc["patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"]
98
- == "building"
99
- )
100
-
101
- patches.drop(columns=["new_predicted_label", "new_pred"], inplace=True)
102
- patches.update_preds(remap, conf=0.8)
103
- assert len(patches[patches["new_predicted_label"].notna()]) == 2
104
- assert (
105
- patches.loc["patch-1-7-2-8-#cropped_74488689.png#.png", "new_predicted_label"]
106
- == "no"
107
- )
108
- assert (
109
- patches.loc["patch-7-7-8-8-#cropped_74488689.png#.png", "new_predicted_label"]
110
- == "building"
111
- )
112
-
113
-
114
- def test_update_preds_inplace(patch_df, labels_map):
115
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
116
- patches.get_context(["railspace"])
117
- remap = {"railspace": "no"}
118
- patches.update_preds(remap, inplace=True)
119
- assert "new_predicted_label" not in patches.columns
120
- assert (
121
- patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "predicted_label"]
122
- == "no"
123
- )
124
-
125
-
126
- def test_update_preds_new_label(patch_df, labels_map):
127
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
128
- patches.get_context(["railspace"])
129
- remap = {"railspace": "new"}
130
- patches.update_preds(remap)
131
- assert len(patches[patches["new_predicted_label"].notna()]) == 1
132
- assert (
133
- patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_predicted_label"]
134
- == "new"
135
- )
136
- assert patches.loc["patch-4-7-5-8-#cropped_74488689.png#.png", "new_pred"] == 4
137
- assert patches.labels_map[4] == "new"
138
-
139
-
140
- def test_update_preds_errors(patch_df, labels_map):
141
- patches = PatchDataFrame(patch_df, labels_map=labels_map)
142
- remap = {"railspace": "no"}
143
- with pytest.raises(ValueError, match="run `get_context` first"):
144
- patches.update_preds(remap)
145
-
146
- patches.get_context(["fake"])
147
- with pytest.raises(ValueError, match="No patches to update"):
148
- patches.update_preds(remap)
149
-
150
- patches.get_context(["railspace"])
151
- with pytest.raises(ValueError, match="must specify a remap"):
152
- patches.update_preds(remap={"fake": "no"})
File without changes
File without changes
File without changes
File without changes