mapreader 1.3.0__tar.gz → 1.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {mapreader-1.3.0 → mapreader-1.3.1}/PKG-INFO +1 -1
  2. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/_version.py +3 -3
  3. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/sheet_downloader.py +73 -26
  4. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/tile_merging.py +22 -8
  5. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/PKG-INFO +1 -1
  6. {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_sheet_downloader.py +158 -51
  7. {mapreader-1.3.0 → mapreader-1.3.1}/LICENSE +0 -0
  8. {mapreader-1.3.0 → mapreader-1.3.1}/README.md +0 -0
  9. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/__init__.py +0 -0
  10. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/__init__.py +0 -0
  11. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/annotator.py +0 -0
  12. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/utils.py +0 -0
  13. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/__init__.py +0 -0
  14. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/classifier.py +0 -0
  15. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/custom_models.py +0 -0
  16. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/datasets.py +0 -0
  17. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/load_annotations.py +0 -0
  18. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/__init__.py +0 -0
  19. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/data_structures.py +0 -0
  20. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/downloader.py +0 -0
  21. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/downloader_utils.py +0 -0
  22. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/tile_loading.py +0 -0
  23. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/__init__.py +0 -0
  24. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/geo_utils.py +0 -0
  25. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/images.py +0 -0
  26. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/loader.py +0 -0
  27. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/__init__.py +0 -0
  28. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/post_process.py +0 -0
  29. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/process.py +0 -0
  30. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/__init__.py +0 -0
  31. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/deepsolo_runner.py +0 -0
  32. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/dptext_detr_runner.py +0 -0
  33. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/__init__.py +0 -0
  34. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/compute_and_save_stats.py +0 -0
  35. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/slice_parallel.py +0 -0
  36. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/SOURCES.txt +0 -0
  37. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/dependency_links.txt +0 -0
  38. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/entry_points.txt +0 -0
  39. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/not-zip-safe +0 -0
  40. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/requires.txt +0 -0
  41. {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/top_level.txt +0 -0
  42. {mapreader-1.3.0 → mapreader-1.3.1}/setup.cfg +0 -0
  43. {mapreader-1.3.0 → mapreader-1.3.1}/setup.py +0 -0
  44. {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_annotator.py +0 -0
  45. {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_geo_pipeline.py +0 -0
  46. {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_import.py +0 -0
  47. {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_post_processing.py +0 -0
  48. {mapreader-1.3.0 → mapreader-1.3.1}/versioneer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mapreader
3
- Version: 1.3.0
3
+ Version: 1.3.1
4
4
  Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
5
5
  Home-page: https://github.com/Living-with-machines/MapReader
6
6
  Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
@@ -8,11 +8,11 @@ import json
8
8
 
9
9
  version_json = '''
10
10
  {
11
- "date": "2024-04-29T15:52:44+0100",
11
+ "date": "2024-05-03T13:32:40+0100",
12
12
  "dirty": false,
13
13
  "error": null,
14
- "full-revisionid": "f256ae9b73ae9dd14d8bd72b61a86088f8f1905a",
15
- "version": "1.3.0"
14
+ "full-revisionid": "3d7ab10b0f7bc0833e122a65a4d4bd608ca62fcd",
15
+ "version": "1.3.1"
16
16
  }
17
17
  ''' # END VERSION_JSON
18
18
 
@@ -533,7 +533,7 @@ class SheetDownloader:
533
533
  """
534
534
  self.merger = TileMerger(output_folder=f"{path_save}/")
535
535
 
536
- def _check_map_sheet_exists(self, feature: dict) -> bool:
536
+ def _check_map_sheet_exists(self, feature: dict, metadata_fname) -> bool:
537
537
  """
538
538
  Checks if a map sheet is already saved.
539
539
 
@@ -544,51 +544,87 @@ class SheetDownloader:
544
544
  Returns
545
545
  -------
546
546
  bool
547
- True if file exists, False if not.
547
+ img_path if file exists, False if not.
548
548
  """
549
- map_name = str("map_" + feature["properties"]["IMAGE"])
550
549
  path_save = self.merger.output_folder
551
- if os.path.exists(f"{path_save}{map_name}.png"):
550
+
551
+ try:
552
+ # get image id with same coords in metadata
553
+ existing_metadata_df = pd.read_csv(
554
+ f"{path_save}{metadata_fname}", sep=",", index_col=0
555
+ )
556
+ except FileNotFoundError:
557
+ return False
558
+
559
+ polygon = get_polygon_from_grid_bb(feature["grid_bb"])
560
+ if str(polygon.bounds) in existing_metadata_df["coordinates"].values:
561
+ image_id = existing_metadata_df[
562
+ existing_metadata_df["coordinates"] == str(polygon.bounds)
563
+ ].iloc[0]["name"]
564
+ else:
565
+ return False # coordinates not in metadata means image doesn't exist
566
+
567
+ if os.path.exists(f"{path_save}{image_id}"):
552
568
  try:
553
- mpimg.imread(f"{path_save}{map_name}.png")
554
- print(
555
- f'[INFO] "{path_save}{map_name}.png" already exists. Skipping download.'
556
- )
557
- return True
569
+ # check image is valid
570
+ mpimg.imread(f"{path_save}{image_id}")
571
+ return image_id
558
572
  except OSError:
559
573
  return False
560
574
  return False
561
575
 
562
- def _download_map(self, feature: dict, download_in_parallel: bool = True) -> bool:
576
+ def _download_map(
577
+ self,
578
+ feature: dict,
579
+ existing_id: str | bool,
580
+ download_in_parallel: bool = True,
581
+ overwrite: bool = False,
582
+ ) -> str | bool:
563
583
  """
564
584
  Downloads a single map sheet and saves as png file.
565
585
 
566
586
  Parameters
567
587
  ----------
568
588
  feature : dict
589
+ The feature for which to download the map sheet.
590
+ existing_id : str | bool
591
+ The existing image id if the map sheet already exists.
592
+ download_in_parallel : bool, optional
593
+ Whether to download tiles in parallel, by default ``True``.
594
+ overwrite : bool, optional
595
+ Whether to overwrite existing maps, by default ``False``.
569
596
 
570
597
  Returns
571
598
  -------
572
- bool
573
- True if map was downloaded successfully, False if not.
599
+ str or bool
600
+ image path if map was downloaded successfully, False if not.
574
601
  """
575
- map_name = str("map_" + feature["properties"]["IMAGE"])
576
602
  self.downloader.download_tiles(
577
603
  feature["grid_bb"], download_in_parallel=download_in_parallel
578
604
  )
579
- success = self.merger.merge(feature["grid_bb"], map_name)
580
- if success:
581
- print(f'[INFO] Downloaded "{map_name}.png"')
605
+
606
+ if existing_id is False:
607
+ map_name = f"map_{feature['properties']['IMAGE']}"
608
+ else:
609
+ map_name = existing_id[:-4] # remove file extension (assuming .png)
610
+
611
+ img_path = self.merger.merge(
612
+ feature["grid_bb"], file_name=map_name, overwrite=overwrite
613
+ )
614
+
615
+ if img_path is not False:
616
+ print(f'[INFO] Downloaded "{img_path}"')
582
617
  else:
583
- print(f'[WARNING] Download of "{map_name}.png" was unsuccessful.')
618
+ print(f'[WARNING] Download of "{img_path}" was unsuccessful.')
584
619
 
585
620
  shutil.rmtree(DEFAULT_TEMP_FOLDER)
586
- return success
621
+ return img_path
587
622
 
588
623
  def _save_metadata(
589
624
  self,
590
625
  feature: dict,
591
626
  out_filepath: str,
627
+ img_path: str,
592
628
  metadata_to_save: dict | None = None,
593
629
  **kwargs: dict | None,
594
630
  ) -> None:
@@ -602,6 +638,8 @@ class SheetDownloader:
602
638
  The feature for which to extract the metadata from
603
639
  out_filepath : str
604
640
  The path to save metadata csv.
641
+ img_path : str
642
+ The path to the downloaded map sheet.
605
643
  metadata_to_save : dict, optional
606
644
  A dictionary containing column names (str) and metadata keys (str or list) to save to metadata csv.
607
645
  Multilayer keys should be passed as a list, i.e. ["key1","key2"] will search for ``self.features[i]["key1"]["key2"]``.
@@ -631,7 +669,7 @@ class SheetDownloader:
631
669
  metadata_dict = {col: None for col in metadata_cols}
632
670
 
633
671
  # get default metadata
634
- metadata_dict["name"] = str("map_" + feature["properties"]["IMAGE"] + ".png")
672
+ metadata_dict["name"] = os.path.basename(img_path)
635
673
  metadata_dict["url"] = str(feature["properties"]["IMAGEURL"])
636
674
  if not self.published_dates:
637
675
  date_col = kwargs.get("date_col", None)
@@ -705,16 +743,25 @@ class SheetDownloader:
705
743
  """
706
744
 
707
745
  for feature in tqdm(features):
708
- if not overwrite:
709
- if self._check_map_sheet_exists(feature):
710
- continue
711
- success = self._download_map(
712
- feature, download_in_parallel=download_in_parallel
746
+ existing_id = self._check_map_sheet_exists(feature, metadata_fname)
747
+ if (
748
+ not overwrite and existing_id is not False
749
+ ): # if map already exists and overwrite is False then skip
750
+ print(f'[INFO] "{existing_id}" already exists. Skipping download.')
751
+ continue
752
+ img_path = self._download_map(
753
+ feature,
754
+ existing_id,
755
+ download_in_parallel=download_in_parallel,
756
+ overwrite=overwrite,
713
757
  )
714
- if success:
758
+ if img_path is not False:
715
759
  metadata_path = f"{path_save}/{metadata_fname}"
716
760
  self._save_metadata(
717
- feature=feature, out_filepath=metadata_path, **kwargs
761
+ feature=feature,
762
+ out_filepath=metadata_path,
763
+ img_path=img_path,
764
+ **kwargs,
718
765
  )
719
766
 
720
767
  def download_all_map_sheets(
@@ -142,7 +142,12 @@ class TileMerger:
142
142
  tile_size = img_size[0]
143
143
  return tile_size
144
144
 
145
- def merge(self, grid_bb: GridBoundingBox, file_name: str | None = None) -> bool:
145
+ def merge(
146
+ self,
147
+ grid_bb: GridBoundingBox,
148
+ file_name: str | None = None,
149
+ overwrite: bool = False,
150
+ ) -> str | bool:
146
151
  """Merges cells contained within GridBoundingBox.
147
152
 
148
153
  Parameters
@@ -151,11 +156,13 @@ class TileMerger:
151
156
  GridBoundingBox containing tiles to merge
152
157
  file_name : Union[str, None], optional
153
158
  Name to use when saving map
154
-
159
+ If None, default name will be used, by default None
160
+ overwrite : bool, optional
161
+ Whether or not to overwrite existing files, by default False
155
162
  Returns
156
163
  -------
157
- bool
158
- True if file has successfully downloaded, False if not.
164
+ str or bool
165
+ out path if file has successfully downloaded, False if not.
159
166
  """
160
167
  os.makedirs(self.output_folder, exist_ok=True)
161
168
 
@@ -191,11 +198,18 @@ class TileMerger:
191
198
  file_name = self._get_output_name(grid_bb)
192
199
 
193
200
  out_path = f"{self.output_folder}{file_name}.{self.img_output_format[0]}"
201
+ if not overwrite:
202
+ i = 1
203
+ while os.path.exists(out_path):
204
+ out_path = (
205
+ f"{self.output_folder}{file_name}_{i}.{self.img_output_format[0]}"
206
+ )
207
+ i += 1
194
208
  merged_image.save(out_path, self.img_output_format[1])
195
- success = True if os.path.exists(out_path) else False
196
- if success:
197
- logger.info(f"Merge successful! The image has been stored at '{out_path}'")
198
- else:
209
+ success = out_path if os.path.exists(out_path) else False
210
+ if success is False:
199
211
  logger.warning(f"Merge unsuccessful! '{out_path}' not saved.")
212
+ else:
213
+ logger.info(f"Merge successful! The image has been stored at '{out_path}'")
200
214
 
201
215
  return success
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mapreader
3
- Version: 1.3.0
3
+ Version: 1.3.1
4
4
  Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
5
5
  Home-page: https://github.com/Living-with-machines/MapReader
6
6
  Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
@@ -1,16 +1,20 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import os
4
+ import re
4
5
  from ast import literal_eval
5
6
  from pathlib import Path
6
7
 
7
8
  import pandas as pd
8
9
  import pytest
10
+ from PIL import Image
9
11
  from pytest import approx
10
12
  from shapely.geometry import LineString, MultiPolygon, Polygon
11
13
 
12
14
  from mapreader import SheetDownloader
13
15
  from mapreader.download.data_structures import GridBoundingBox
16
+ from mapreader.download.tile_loading import TileDownloader
17
+ from mapreader.download.tile_merging import TileMerger
14
18
 
15
19
 
16
20
  @pytest.fixture
@@ -20,14 +24,14 @@ def sample_dir():
20
24
 
21
25
  @pytest.fixture
22
26
  def sheet_downloader(sample_dir):
23
- test_json = f"{sample_dir}/test_json.json" # contains 4 features
27
+ test_json = f"{sample_dir}/test_json.json" # contains 6 features
24
28
  download_url = "https://geo.nls.uk/maps/os/1inch_2nd_ed/{z}/{x}/{y}.png"
25
29
  return SheetDownloader(test_json, download_url)
26
30
 
27
31
 
28
32
  def test_init(sheet_downloader):
29
33
  sd = sheet_downloader
30
- assert len(sd) == 4
34
+ assert len(sd) == 6
31
35
  assert sd.crs == "EPSG:4326"
32
36
 
33
37
 
@@ -121,7 +125,7 @@ def test_get_minmax_latlon(sheet_downloader, capfd):
121
125
  out, _ = capfd.readouterr()
122
126
  assert (
123
127
  out
124
- == "[INFO] Min lat: 51.49344796, max lat: 54.2089733 \n[INFO] Min lon: -2.62837527, max lon: -0.16093917\n"
128
+ == "[INFO] Min lat: 51.49344796, max lat: 54.2089733 \n[INFO] Min lon: -4.7682, max lon: -0.16093917\n"
125
129
  )
126
130
 
127
131
 
@@ -280,17 +284,49 @@ def test_query_by_string_key_errors(sheet_downloader):
280
284
  # download
281
285
 
282
286
 
283
- def test_download_all(sheet_downloader, tmp_path):
287
+ @pytest.fixture(scope="function")
288
+ def mock_response(monkeypatch):
289
+ def mock_download_tiles(self, *args, **kwargs):
290
+ os.makedirs(self.temp_folder, exist_ok=True)
291
+ return
292
+
293
+ monkeypatch.setattr(TileDownloader, "download_tiles", mock_download_tiles)
294
+
295
+ def mock_merge(self, *args, **kwargs):
296
+ os.makedirs(self.output_folder, exist_ok=True)
297
+
298
+ merged_image = Image.new("RGBA", (10, 10))
299
+
300
+ if kwargs["file_name"] is None:
301
+ file_name = self._get_output_name(kwargs["grid_bb"])
302
+ else:
303
+ file_name = kwargs["file_name"]
304
+
305
+ out_path = f"{self.output_folder}{file_name}.{self.img_output_format[0]}"
306
+ if not kwargs["overwrite"]:
307
+ i = 1
308
+ while os.path.exists(out_path):
309
+ out_path = (
310
+ f"{self.output_folder}{file_name}_{i}.{self.img_output_format[0]}"
311
+ )
312
+ i += 1
313
+ merged_image.save(out_path, self.img_output_format[1])
314
+ return out_path
315
+
316
+ monkeypatch.setattr(TileMerger, "merge", mock_merge)
317
+
318
+
319
+ def test_download_all(sheet_downloader, tmp_path, mock_response):
284
320
  sd = sheet_downloader
285
- sd.get_grid_bb(10)
286
- assert sd.grid_bbs is True
287
- maps_path = tmp_path / "test_maps/"
321
+ # zoom level 14
322
+ sd.get_grid_bb(14)
323
+ maps_path = tmp_path / "test_maps_14/"
288
324
  metadata_fname = "test_metadata.csv"
289
325
  sd.download_all_map_sheets(maps_path, metadata_fname)
290
326
  assert os.path.exists(f"{maps_path}/map_102352861.png")
291
327
  assert os.path.exists(f"{maps_path}/{metadata_fname}")
292
328
  df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
293
- assert len(df) == 4
329
+ assert len(df) == 6
294
330
  assert list(df.columns) == [
295
331
  "name",
296
332
  "url",
@@ -299,25 +335,32 @@ def test_download_all(sheet_downloader, tmp_path):
299
335
  "published_date",
300
336
  "grid_bb",
301
337
  ]
302
- assert list(df["name"]) == [
303
- "map_101602026.png",
304
- "map_101602038.png",
305
- "map_102352861.png",
306
- "map_91617032.png",
307
- ]
338
+ assert all(
339
+ name in list(df["name"])
340
+ for name in [
341
+ "map_101602026.png",
342
+ "map_101602038.png",
343
+ "map_102352861.png",
344
+ "map_91617032.png",
345
+ "map_101603986.png",
346
+ "map_101603986_1.png",
347
+ ]
348
+ )
308
349
  # test coords
309
350
  assert literal_eval(df.loc[0, "coordinates"]) == approx(
310
- (-1.0546875, 53.33087298301705, -0.703125, 53.54030739150021), rel=1e-6
351
+ (-0.98876953125, 53.448806835427575, -0.90087890625, 53.48804553605621),
352
+ rel=1e-6,
311
353
  )
312
-
313
- sd.get_grid_bb(14)
314
- maps_path = tmp_path / "test_maps_14/"
354
+ # zoom level 17
355
+ sd.get_grid_bb(17)
356
+ assert sd.grid_bbs is True
357
+ maps_path = tmp_path / "test_maps_17/"
315
358
  metadata_fname = "test_metadata.csv"
316
359
  sd.download_all_map_sheets(maps_path, metadata_fname)
317
360
  assert os.path.exists(f"{maps_path}/map_102352861.png")
318
361
  assert os.path.exists(f"{maps_path}/{metadata_fname}")
319
362
  df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
320
- assert len(df) == 4
363
+ assert len(df) == 6
321
364
  assert list(df.columns) == [
322
365
  "name",
323
366
  "url",
@@ -326,32 +369,35 @@ def test_download_all(sheet_downloader, tmp_path):
326
369
  "published_date",
327
370
  "grid_bb",
328
371
  ]
329
- assert list(df["name"]) == [
330
- "map_101602026.png",
331
- "map_101602038.png",
332
- "map_102352861.png",
333
- "map_91617032.png",
334
- ]
335
- # test coords
336
- assert literal_eval(df.loc[0, "coordinates"]) == approx(
337
- (-0.98876953125, 53.448806835427575, -0.90087890625, 53.48804553605621),
338
- rel=1e-6,
372
+ print(list(df["name"]))
373
+ assert all(
374
+ name in list(df["name"])
375
+ for name in [
376
+ "map_101602026.png",
377
+ "map_101602038.png",
378
+ "map_102352861.png",
379
+ "map_91617032.png",
380
+ "map_101603986.png",
381
+ "map_101603986_1.png",
382
+ ]
339
383
  )
340
384
 
341
385
 
342
386
  def test_download_all_kwargs(sheet_downloader, tmp_path):
343
387
  sd = sheet_downloader
344
- sd.get_grid_bb(10)
345
- maps_path = tmp_path / "test_maps/"
388
+ # zoom level 14
389
+ sd.get_grid_bb(14)
390
+ maps_path = tmp_path / "test_maps_14/"
346
391
  metadata_fname = "test_metadata.csv"
347
392
  kwargs = {
348
393
  "metadata_to_save": {"test1": ["properties", "test"], "test2": "id"},
349
394
  "date_col": "test_date",
350
395
  }
351
396
  sd.download_all_map_sheets(maps_path, metadata_fname, **kwargs)
397
+ assert os.path.exists(f"{maps_path}/map_102352861.png")
352
398
  assert os.path.exists(f"{maps_path}/{metadata_fname}")
353
399
  df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
354
- assert len(df) == 4
400
+ assert len(df) == 6
355
401
  assert list(df.columns) == [
356
402
  "name",
357
403
  "url",
@@ -362,12 +408,22 @@ def test_download_all_kwargs(sheet_downloader, tmp_path):
362
408
  "test1",
363
409
  "test2",
364
410
  ]
365
- assert list(df["name"]) == [
366
- "map_101602026.png",
367
- "map_101602038.png",
368
- "map_102352861.png",
369
- "map_91617032.png",
370
- ]
411
+ assert all(
412
+ name in list(df["name"])
413
+ for name in [
414
+ "map_101602026.png",
415
+ "map_101602038.png",
416
+ "map_102352861.png",
417
+ "map_91617032.png",
418
+ "map_101603986.png",
419
+ "map_101603986_1.png",
420
+ ]
421
+ )
422
+ # test coords
423
+ assert literal_eval(df.loc[0, "coordinates"]) == approx(
424
+ (-0.98876953125, 53.448806835427575, -0.90087890625, 53.48804553605621),
425
+ rel=1e-6,
426
+ )
371
427
  assert df.loc[3, "published_date"] == 2021
372
428
  assert df.loc[3, "test1"] == "test"
373
429
  assert df.loc[3, "test2"] == "Six_Inch_GB_WFS.132"
@@ -375,7 +431,7 @@ def test_download_all_kwargs(sheet_downloader, tmp_path):
375
431
 
376
432
  def test_download_by_wfs_ids(sheet_downloader, tmp_path):
377
433
  sd = sheet_downloader
378
- sd.get_grid_bb(10)
434
+ sd.get_grid_bb(14)
379
435
  maps_path = tmp_path / "test_maps/"
380
436
  metadata_fname = "test_metadata.csv"
381
437
  sd.download_map_sheets_by_wfs_ids(
@@ -411,9 +467,60 @@ def test_download_by_wfs_ids(sheet_downloader, tmp_path):
411
467
  assert df.loc[1, "name"] == "map_101602038.png"
412
468
 
413
469
 
470
+ def test_download_same_image_names(sheet_downloader, tmp_path, capfd):
471
+ sd = sheet_downloader
472
+ sd.get_grid_bb(14)
473
+ maps_path = tmp_path / "test_maps/"
474
+ metadata_fname = "test_metadata.csv"
475
+ sd.download_map_sheets_by_wfs_ids(
476
+ [107, 116], maps_path, metadata_fname
477
+ ) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
478
+ assert os.path.exists(f"{maps_path}/map_101603986.png")
479
+ assert os.path.exists(f"{maps_path}/map_101603986_1.png")
480
+ assert os.path.exists(f"{maps_path}/{metadata_fname}")
481
+ df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
482
+ assert len(df) == 2
483
+ assert list(df.columns) == [
484
+ "name",
485
+ "url",
486
+ "coordinates",
487
+ "crs",
488
+ "published_date",
489
+ "grid_bb",
490
+ ]
491
+ assert df.loc[0, "name"] == "map_101603986.png"
492
+ assert df.loc[1, "name"] == "map_101603986_1.png"
493
+
494
+ # run again, nothing should happen
495
+ sd.download_map_sheets_by_wfs_ids([107, 116], maps_path, metadata_fname)
496
+ out, _ = capfd.readouterr()
497
+ assert out.endswith(
498
+ '[INFO] "map_101603986.png" already exists. Skipping download.\n[INFO] "map_101603986_1.png" already exists. Skipping download.\n'
499
+ )
500
+ df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
501
+ assert len(df) == 2
502
+
503
+ # now overwrite them, but check we don't add new ones (_2, _3 etc.)
504
+ sd.download_map_sheets_by_wfs_ids(
505
+ [107, 116], maps_path, metadata_fname, overwrite=True
506
+ ) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
507
+ out, _ = capfd.readouterr()
508
+ assert re.search(
509
+ r"\[INFO\] Downloaded \".*\/test_maps\/map_101603986.png\"\n\[INFO\] Downloaded \".*\/test_maps\/map_101603986_1.png\"\n$",
510
+ out,
511
+ )
512
+ assert os.path.exists(f"{maps_path}/map_101603986.png")
513
+ assert os.path.exists(f"{maps_path}/map_101603986_1.png")
514
+ assert os.path.exists(f"{maps_path}/{metadata_fname}")
515
+ df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
516
+ assert len(df) == 2
517
+ assert df.loc[0, "name"] == "map_101603986.png"
518
+ assert df.loc[1, "name"] == "map_101603986_1.png"
519
+
520
+
414
521
  def test_download_by_wfs_ids_errors(sheet_downloader, tmp_path):
415
522
  sd = sheet_downloader
416
- sd.get_grid_bb(10)
523
+ sd.get_grid_bb(14)
417
524
  maps_path = tmp_path / "test_maps/"
418
525
  metadata_fname = "test_metadata.csv"
419
526
  with pytest.raises(ValueError, match="as int or list of ints"):
@@ -427,7 +534,7 @@ def test_download_by_wfs_ids_errors(sheet_downloader, tmp_path):
427
534
 
428
535
  def test_download_by_polygon(sheet_downloader, tmp_path):
429
536
  sd = sheet_downloader
430
- sd.get_grid_bb(10)
537
+ sd.get_grid_bb(14)
431
538
  polygon = Polygon(
432
539
  [
433
540
  [-0.98078243, 53.45664144],
@@ -467,7 +574,7 @@ def test_download_by_polygon(sheet_downloader, tmp_path):
467
574
 
468
575
  def test_download_by_polygon_errors(sheet_downloader, tmp_path):
469
576
  sd = sheet_downloader
470
- sd.get_grid_bb(10)
577
+ sd.get_grid_bb(14)
471
578
  polygon = Polygon([[0, 1], [1, 2], [2, 3], [3, 4], [0, 1]])
472
579
  maps_path = tmp_path / "test_maps/"
473
580
  metadata_fname = "test_metadata.csv"
@@ -483,7 +590,7 @@ def test_download_by_polygon_errors(sheet_downloader, tmp_path):
483
590
 
484
591
  def test_download_by_coords(sheet_downloader, tmp_path):
485
592
  sd = sheet_downloader
486
- sd.get_grid_bb(10)
593
+ sd.get_grid_bb(14)
487
594
  maps_path = tmp_path / "test_maps/"
488
595
  metadata_fname = "test_metadata.csv"
489
596
  sd.download_map_sheets_by_coordinates((-0.99, 53.43), maps_path, metadata_fname)
@@ -504,7 +611,7 @@ def test_download_by_coords(sheet_downloader, tmp_path):
504
611
 
505
612
  def test_download_by_coords_errors(sheet_downloader, tmp_path):
506
613
  sd = sheet_downloader
507
- sd.get_grid_bb(10)
614
+ sd.get_grid_bb(14)
508
615
  maps_path = tmp_path / "test_maps/"
509
616
  metadata_fname = "test_metadata.csv"
510
617
  with pytest.raises(ValueError, match="out of map metadata bounds"):
@@ -513,7 +620,7 @@ def test_download_by_coords_errors(sheet_downloader, tmp_path):
513
620
 
514
621
  def test_download_by_line(sheet_downloader, tmp_path):
515
622
  sd = sheet_downloader
516
- sd.get_grid_bb(10)
623
+ sd.get_grid_bb(14)
517
624
  maps_path = tmp_path / "test_maps/"
518
625
  metadata_fname = "test_metadata.csv"
519
626
  line = LineString([(-0.99, 53.43), (-0.93, 53.46)])
@@ -535,7 +642,7 @@ def test_download_by_line(sheet_downloader, tmp_path):
535
642
 
536
643
  def test_download_by_line_errors(sheet_downloader, tmp_path):
537
644
  sd = sheet_downloader
538
- sd.get_grid_bb(10)
645
+ sd.get_grid_bb(14)
539
646
  maps_path = tmp_path / "test_maps/"
540
647
  metadata_fname = "test_metadata.csv"
541
648
  line = LineString([(0, 1), (2, 3)])
@@ -545,7 +652,7 @@ def test_download_by_line_errors(sheet_downloader, tmp_path):
545
652
 
546
653
  def test_download_by_string(sheet_downloader, tmp_path):
547
654
  sd = sheet_downloader
548
- sd.get_grid_bb(10)
655
+ sd.get_grid_bb(14)
549
656
  maps_path = tmp_path / "test_maps/"
550
657
  metadata_fname = "test_metadata.csv"
551
658
  sd.download_map_sheets_by_string(
@@ -585,7 +692,7 @@ def test_download_by_string(sheet_downloader, tmp_path):
585
692
 
586
693
  def test_download_by_string_value_errors(sheet_downloader, tmp_path):
587
694
  sd = sheet_downloader
588
- sd.get_grid_bb(10)
695
+ sd.get_grid_bb(14)
589
696
  maps_path = tmp_path / "test_maps/"
590
697
  metadata_fname = "test_metadata.csv"
591
698
  with pytest.raises(ValueError, match="pass ``string`` as a string"):
@@ -598,7 +705,7 @@ def test_download_by_string_value_errors(sheet_downloader, tmp_path):
598
705
 
599
706
  def test_download_by_string_key_errors(sheet_downloader, tmp_path):
600
707
  sd = sheet_downloader
601
- sd.get_grid_bb(10)
708
+ sd.get_grid_bb(14)
602
709
  maps_path = tmp_path / "test_maps/"
603
710
  metadata_fname = "test_metadata.csv"
604
711
  with pytest.raises(KeyError, match="not found in features dictionary"):
@@ -609,7 +716,7 @@ def test_download_by_string_key_errors(sheet_downloader, tmp_path):
609
716
 
610
717
  def test_download_by_queries(sheet_downloader, tmp_path):
611
718
  sd = sheet_downloader
612
- sd.get_grid_bb(10)
719
+ sd.get_grid_bb(14)
613
720
  maps_path = tmp_path / "test_maps/"
614
721
  metadata_fname = "test_metadata.csv"
615
722
  sd.query_map_sheets_by_wfs_ids([16320, 132]) # features[0] and [3]
@@ -634,7 +741,7 @@ def test_download_by_queries(sheet_downloader, tmp_path):
634
741
 
635
742
  def test_download_by_queries_errors(sheet_downloader, tmp_path):
636
743
  sd = sheet_downloader
637
- sd.get_grid_bb(10)
744
+ sd.get_grid_bb(14)
638
745
  maps_path = tmp_path / "test_maps/"
639
746
  metadata_fname = "test_metadata.csv"
640
747
  with pytest.raises(ValueError, match="No query results"):
File without changes
File without changes
File without changes
File without changes
File without changes