mapreader 1.3.0__tar.gz → 1.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapreader-1.3.0 → mapreader-1.3.1}/PKG-INFO +1 -1
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/_version.py +3 -3
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/sheet_downloader.py +73 -26
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/tile_merging.py +22 -8
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/PKG-INFO +1 -1
- {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_sheet_downloader.py +158 -51
- {mapreader-1.3.0 → mapreader-1.3.1}/LICENSE +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/README.md +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/annotator.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/annotate/utils.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/classifier.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/custom_models.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/datasets.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/classify/load_annotations.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/data_structures.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/downloader.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/downloader_utils.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/download/tile_loading.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/geo_utils.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/images.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/load/loader.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/post_process.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/process/process.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/deepsolo_runner.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/spot_text/dptext_detr_runner.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/__init__.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/compute_and_save_stats.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader/utils/slice_parallel.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/SOURCES.txt +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/dependency_links.txt +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/entry_points.txt +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/not-zip-safe +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/requires.txt +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/mapreader.egg-info/top_level.txt +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/setup.cfg +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/setup.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_annotator.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_geo_pipeline.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_import.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/tests/test_post_processing.py +0 -0
- {mapreader-1.3.0 → mapreader-1.3.1}/versioneer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mapreader
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.1
|
|
4
4
|
Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
|
|
5
5
|
Home-page: https://github.com/Living-with-machines/MapReader
|
|
6
6
|
Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2024-
|
|
11
|
+
"date": "2024-05-03T13:32:40+0100",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "1.3.
|
|
14
|
+
"full-revisionid": "3d7ab10b0f7bc0833e122a65a4d4bd608ca62fcd",
|
|
15
|
+
"version": "1.3.1"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -533,7 +533,7 @@ class SheetDownloader:
|
|
|
533
533
|
"""
|
|
534
534
|
self.merger = TileMerger(output_folder=f"{path_save}/")
|
|
535
535
|
|
|
536
|
-
def _check_map_sheet_exists(self, feature: dict) -> bool:
|
|
536
|
+
def _check_map_sheet_exists(self, feature: dict, metadata_fname) -> bool:
|
|
537
537
|
"""
|
|
538
538
|
Checks if a map sheet is already saved.
|
|
539
539
|
|
|
@@ -544,51 +544,87 @@ class SheetDownloader:
|
|
|
544
544
|
Returns
|
|
545
545
|
-------
|
|
546
546
|
bool
|
|
547
|
-
|
|
547
|
+
img_path if file exists, False if not.
|
|
548
548
|
"""
|
|
549
|
-
map_name = str("map_" + feature["properties"]["IMAGE"])
|
|
550
549
|
path_save = self.merger.output_folder
|
|
551
|
-
|
|
550
|
+
|
|
551
|
+
try:
|
|
552
|
+
# get image id with same coords in metadata
|
|
553
|
+
existing_metadata_df = pd.read_csv(
|
|
554
|
+
f"{path_save}{metadata_fname}", sep=",", index_col=0
|
|
555
|
+
)
|
|
556
|
+
except FileNotFoundError:
|
|
557
|
+
return False
|
|
558
|
+
|
|
559
|
+
polygon = get_polygon_from_grid_bb(feature["grid_bb"])
|
|
560
|
+
if str(polygon.bounds) in existing_metadata_df["coordinates"].values:
|
|
561
|
+
image_id = existing_metadata_df[
|
|
562
|
+
existing_metadata_df["coordinates"] == str(polygon.bounds)
|
|
563
|
+
].iloc[0]["name"]
|
|
564
|
+
else:
|
|
565
|
+
return False # coordinates not in metadata means image doesn't exist
|
|
566
|
+
|
|
567
|
+
if os.path.exists(f"{path_save}{image_id}"):
|
|
552
568
|
try:
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
)
|
|
557
|
-
return True
|
|
569
|
+
# check image is valid
|
|
570
|
+
mpimg.imread(f"{path_save}{image_id}")
|
|
571
|
+
return image_id
|
|
558
572
|
except OSError:
|
|
559
573
|
return False
|
|
560
574
|
return False
|
|
561
575
|
|
|
562
|
-
def _download_map(
|
|
576
|
+
def _download_map(
|
|
577
|
+
self,
|
|
578
|
+
feature: dict,
|
|
579
|
+
existing_id: str | bool,
|
|
580
|
+
download_in_parallel: bool = True,
|
|
581
|
+
overwrite: bool = False,
|
|
582
|
+
) -> str | bool:
|
|
563
583
|
"""
|
|
564
584
|
Downloads a single map sheet and saves as png file.
|
|
565
585
|
|
|
566
586
|
Parameters
|
|
567
587
|
----------
|
|
568
588
|
feature : dict
|
|
589
|
+
The feature for which to download the map sheet.
|
|
590
|
+
existing_id : str | bool
|
|
591
|
+
The existing image id if the map sheet already exists.
|
|
592
|
+
download_in_parallel : bool, optional
|
|
593
|
+
Whether to download tiles in parallel, by default ``True``.
|
|
594
|
+
overwrite : bool, optional
|
|
595
|
+
Whether to overwrite existing maps, by default ``False``.
|
|
569
596
|
|
|
570
597
|
Returns
|
|
571
598
|
-------
|
|
572
|
-
bool
|
|
573
|
-
|
|
599
|
+
str or bool
|
|
600
|
+
image path if map was downloaded successfully, False if not.
|
|
574
601
|
"""
|
|
575
|
-
map_name = str("map_" + feature["properties"]["IMAGE"])
|
|
576
602
|
self.downloader.download_tiles(
|
|
577
603
|
feature["grid_bb"], download_in_parallel=download_in_parallel
|
|
578
604
|
)
|
|
579
|
-
|
|
580
|
-
if
|
|
581
|
-
|
|
605
|
+
|
|
606
|
+
if existing_id is False:
|
|
607
|
+
map_name = f"map_{feature['properties']['IMAGE']}"
|
|
608
|
+
else:
|
|
609
|
+
map_name = existing_id[:-4] # remove file extension (assuming .png)
|
|
610
|
+
|
|
611
|
+
img_path = self.merger.merge(
|
|
612
|
+
feature["grid_bb"], file_name=map_name, overwrite=overwrite
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
if img_path is not False:
|
|
616
|
+
print(f'[INFO] Downloaded "{img_path}"')
|
|
582
617
|
else:
|
|
583
|
-
print(f'[WARNING] Download of "{
|
|
618
|
+
print(f'[WARNING] Download of "{img_path}" was unsuccessful.')
|
|
584
619
|
|
|
585
620
|
shutil.rmtree(DEFAULT_TEMP_FOLDER)
|
|
586
|
-
return
|
|
621
|
+
return img_path
|
|
587
622
|
|
|
588
623
|
def _save_metadata(
|
|
589
624
|
self,
|
|
590
625
|
feature: dict,
|
|
591
626
|
out_filepath: str,
|
|
627
|
+
img_path: str,
|
|
592
628
|
metadata_to_save: dict | None = None,
|
|
593
629
|
**kwargs: dict | None,
|
|
594
630
|
) -> None:
|
|
@@ -602,6 +638,8 @@ class SheetDownloader:
|
|
|
602
638
|
The feature for which to extract the metadata from
|
|
603
639
|
out_filepath : str
|
|
604
640
|
The path to save metadata csv.
|
|
641
|
+
img_path : str
|
|
642
|
+
The path to the downloaded map sheet.
|
|
605
643
|
metadata_to_save : dict, optional
|
|
606
644
|
A dictionary containing column names (str) and metadata keys (str or list) to save to metadata csv.
|
|
607
645
|
Multilayer keys should be passed as a list, i.e. ["key1","key2"] will search for ``self.features[i]["key1"]["key2"]``.
|
|
@@ -631,7 +669,7 @@ class SheetDownloader:
|
|
|
631
669
|
metadata_dict = {col: None for col in metadata_cols}
|
|
632
670
|
|
|
633
671
|
# get default metadata
|
|
634
|
-
metadata_dict["name"] =
|
|
672
|
+
metadata_dict["name"] = os.path.basename(img_path)
|
|
635
673
|
metadata_dict["url"] = str(feature["properties"]["IMAGEURL"])
|
|
636
674
|
if not self.published_dates:
|
|
637
675
|
date_col = kwargs.get("date_col", None)
|
|
@@ -705,16 +743,25 @@ class SheetDownloader:
|
|
|
705
743
|
"""
|
|
706
744
|
|
|
707
745
|
for feature in tqdm(features):
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
746
|
+
existing_id = self._check_map_sheet_exists(feature, metadata_fname)
|
|
747
|
+
if (
|
|
748
|
+
not overwrite and existing_id is not False
|
|
749
|
+
): # if map already exists and overwrite is False then skip
|
|
750
|
+
print(f'[INFO] "{existing_id}" already exists. Skipping download.')
|
|
751
|
+
continue
|
|
752
|
+
img_path = self._download_map(
|
|
753
|
+
feature,
|
|
754
|
+
existing_id,
|
|
755
|
+
download_in_parallel=download_in_parallel,
|
|
756
|
+
overwrite=overwrite,
|
|
713
757
|
)
|
|
714
|
-
if
|
|
758
|
+
if img_path is not False:
|
|
715
759
|
metadata_path = f"{path_save}/{metadata_fname}"
|
|
716
760
|
self._save_metadata(
|
|
717
|
-
feature=feature,
|
|
761
|
+
feature=feature,
|
|
762
|
+
out_filepath=metadata_path,
|
|
763
|
+
img_path=img_path,
|
|
764
|
+
**kwargs,
|
|
718
765
|
)
|
|
719
766
|
|
|
720
767
|
def download_all_map_sheets(
|
|
@@ -142,7 +142,12 @@ class TileMerger:
|
|
|
142
142
|
tile_size = img_size[0]
|
|
143
143
|
return tile_size
|
|
144
144
|
|
|
145
|
-
def merge(
|
|
145
|
+
def merge(
|
|
146
|
+
self,
|
|
147
|
+
grid_bb: GridBoundingBox,
|
|
148
|
+
file_name: str | None = None,
|
|
149
|
+
overwrite: bool = False,
|
|
150
|
+
) -> str | bool:
|
|
146
151
|
"""Merges cells contained within GridBoundingBox.
|
|
147
152
|
|
|
148
153
|
Parameters
|
|
@@ -151,11 +156,13 @@ class TileMerger:
|
|
|
151
156
|
GridBoundingBox containing tiles to merge
|
|
152
157
|
file_name : Union[str, None], optional
|
|
153
158
|
Name to use when saving map
|
|
154
|
-
|
|
159
|
+
If None, default name will be used, by default None
|
|
160
|
+
overwrite : bool, optional
|
|
161
|
+
Whether or not to overwrite existing files, by default False
|
|
155
162
|
Returns
|
|
156
163
|
-------
|
|
157
|
-
bool
|
|
158
|
-
|
|
164
|
+
str or bool
|
|
165
|
+
out path if file has successfully downloaded, False if not.
|
|
159
166
|
"""
|
|
160
167
|
os.makedirs(self.output_folder, exist_ok=True)
|
|
161
168
|
|
|
@@ -191,11 +198,18 @@ class TileMerger:
|
|
|
191
198
|
file_name = self._get_output_name(grid_bb)
|
|
192
199
|
|
|
193
200
|
out_path = f"{self.output_folder}{file_name}.{self.img_output_format[0]}"
|
|
201
|
+
if not overwrite:
|
|
202
|
+
i = 1
|
|
203
|
+
while os.path.exists(out_path):
|
|
204
|
+
out_path = (
|
|
205
|
+
f"{self.output_folder}{file_name}_{i}.{self.img_output_format[0]}"
|
|
206
|
+
)
|
|
207
|
+
i += 1
|
|
194
208
|
merged_image.save(out_path, self.img_output_format[1])
|
|
195
|
-
success =
|
|
196
|
-
if success:
|
|
197
|
-
logger.info(f"Merge successful! The image has been stored at '{out_path}'")
|
|
198
|
-
else:
|
|
209
|
+
success = out_path if os.path.exists(out_path) else False
|
|
210
|
+
if success is False:
|
|
199
211
|
logger.warning(f"Merge unsuccessful! '{out_path}' not saved.")
|
|
212
|
+
else:
|
|
213
|
+
logger.info(f"Merge successful! The image has been stored at '{out_path}'")
|
|
200
214
|
|
|
201
215
|
return success
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mapreader
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.1
|
|
4
4
|
Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
|
|
5
5
|
Home-page: https://github.com/Living-with-machines/MapReader
|
|
6
6
|
Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
|
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
import re
|
|
4
5
|
from ast import literal_eval
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
7
8
|
import pandas as pd
|
|
8
9
|
import pytest
|
|
10
|
+
from PIL import Image
|
|
9
11
|
from pytest import approx
|
|
10
12
|
from shapely.geometry import LineString, MultiPolygon, Polygon
|
|
11
13
|
|
|
12
14
|
from mapreader import SheetDownloader
|
|
13
15
|
from mapreader.download.data_structures import GridBoundingBox
|
|
16
|
+
from mapreader.download.tile_loading import TileDownloader
|
|
17
|
+
from mapreader.download.tile_merging import TileMerger
|
|
14
18
|
|
|
15
19
|
|
|
16
20
|
@pytest.fixture
|
|
@@ -20,14 +24,14 @@ def sample_dir():
|
|
|
20
24
|
|
|
21
25
|
@pytest.fixture
|
|
22
26
|
def sheet_downloader(sample_dir):
|
|
23
|
-
test_json = f"{sample_dir}/test_json.json" # contains
|
|
27
|
+
test_json = f"{sample_dir}/test_json.json" # contains 6 features
|
|
24
28
|
download_url = "https://geo.nls.uk/maps/os/1inch_2nd_ed/{z}/{x}/{y}.png"
|
|
25
29
|
return SheetDownloader(test_json, download_url)
|
|
26
30
|
|
|
27
31
|
|
|
28
32
|
def test_init(sheet_downloader):
|
|
29
33
|
sd = sheet_downloader
|
|
30
|
-
assert len(sd) ==
|
|
34
|
+
assert len(sd) == 6
|
|
31
35
|
assert sd.crs == "EPSG:4326"
|
|
32
36
|
|
|
33
37
|
|
|
@@ -121,7 +125,7 @@ def test_get_minmax_latlon(sheet_downloader, capfd):
|
|
|
121
125
|
out, _ = capfd.readouterr()
|
|
122
126
|
assert (
|
|
123
127
|
out
|
|
124
|
-
== "[INFO] Min lat: 51.49344796, max lat: 54.2089733 \n[INFO] Min lon: -
|
|
128
|
+
== "[INFO] Min lat: 51.49344796, max lat: 54.2089733 \n[INFO] Min lon: -4.7682, max lon: -0.16093917\n"
|
|
125
129
|
)
|
|
126
130
|
|
|
127
131
|
|
|
@@ -280,17 +284,49 @@ def test_query_by_string_key_errors(sheet_downloader):
|
|
|
280
284
|
# download
|
|
281
285
|
|
|
282
286
|
|
|
283
|
-
|
|
287
|
+
@pytest.fixture(scope="function")
|
|
288
|
+
def mock_response(monkeypatch):
|
|
289
|
+
def mock_download_tiles(self, *args, **kwargs):
|
|
290
|
+
os.makedirs(self.temp_folder, exist_ok=True)
|
|
291
|
+
return
|
|
292
|
+
|
|
293
|
+
monkeypatch.setattr(TileDownloader, "download_tiles", mock_download_tiles)
|
|
294
|
+
|
|
295
|
+
def mock_merge(self, *args, **kwargs):
|
|
296
|
+
os.makedirs(self.output_folder, exist_ok=True)
|
|
297
|
+
|
|
298
|
+
merged_image = Image.new("RGBA", (10, 10))
|
|
299
|
+
|
|
300
|
+
if kwargs["file_name"] is None:
|
|
301
|
+
file_name = self._get_output_name(kwargs["grid_bb"])
|
|
302
|
+
else:
|
|
303
|
+
file_name = kwargs["file_name"]
|
|
304
|
+
|
|
305
|
+
out_path = f"{self.output_folder}{file_name}.{self.img_output_format[0]}"
|
|
306
|
+
if not kwargs["overwrite"]:
|
|
307
|
+
i = 1
|
|
308
|
+
while os.path.exists(out_path):
|
|
309
|
+
out_path = (
|
|
310
|
+
f"{self.output_folder}{file_name}_{i}.{self.img_output_format[0]}"
|
|
311
|
+
)
|
|
312
|
+
i += 1
|
|
313
|
+
merged_image.save(out_path, self.img_output_format[1])
|
|
314
|
+
return out_path
|
|
315
|
+
|
|
316
|
+
monkeypatch.setattr(TileMerger, "merge", mock_merge)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def test_download_all(sheet_downloader, tmp_path, mock_response):
|
|
284
320
|
sd = sheet_downloader
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
maps_path = tmp_path / "
|
|
321
|
+
# zoom level 14
|
|
322
|
+
sd.get_grid_bb(14)
|
|
323
|
+
maps_path = tmp_path / "test_maps_14/"
|
|
288
324
|
metadata_fname = "test_metadata.csv"
|
|
289
325
|
sd.download_all_map_sheets(maps_path, metadata_fname)
|
|
290
326
|
assert os.path.exists(f"{maps_path}/map_102352861.png")
|
|
291
327
|
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
292
328
|
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
293
|
-
assert len(df) ==
|
|
329
|
+
assert len(df) == 6
|
|
294
330
|
assert list(df.columns) == [
|
|
295
331
|
"name",
|
|
296
332
|
"url",
|
|
@@ -299,25 +335,32 @@ def test_download_all(sheet_downloader, tmp_path):
|
|
|
299
335
|
"published_date",
|
|
300
336
|
"grid_bb",
|
|
301
337
|
]
|
|
302
|
-
assert
|
|
303
|
-
"
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
338
|
+
assert all(
|
|
339
|
+
name in list(df["name"])
|
|
340
|
+
for name in [
|
|
341
|
+
"map_101602026.png",
|
|
342
|
+
"map_101602038.png",
|
|
343
|
+
"map_102352861.png",
|
|
344
|
+
"map_91617032.png",
|
|
345
|
+
"map_101603986.png",
|
|
346
|
+
"map_101603986_1.png",
|
|
347
|
+
]
|
|
348
|
+
)
|
|
308
349
|
# test coords
|
|
309
350
|
assert literal_eval(df.loc[0, "coordinates"]) == approx(
|
|
310
|
-
(-
|
|
351
|
+
(-0.98876953125, 53.448806835427575, -0.90087890625, 53.48804553605621),
|
|
352
|
+
rel=1e-6,
|
|
311
353
|
)
|
|
312
|
-
|
|
313
|
-
sd.get_grid_bb(
|
|
314
|
-
|
|
354
|
+
# zoom level 17
|
|
355
|
+
sd.get_grid_bb(17)
|
|
356
|
+
assert sd.grid_bbs is True
|
|
357
|
+
maps_path = tmp_path / "test_maps_17/"
|
|
315
358
|
metadata_fname = "test_metadata.csv"
|
|
316
359
|
sd.download_all_map_sheets(maps_path, metadata_fname)
|
|
317
360
|
assert os.path.exists(f"{maps_path}/map_102352861.png")
|
|
318
361
|
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
319
362
|
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
320
|
-
assert len(df) ==
|
|
363
|
+
assert len(df) == 6
|
|
321
364
|
assert list(df.columns) == [
|
|
322
365
|
"name",
|
|
323
366
|
"url",
|
|
@@ -326,32 +369,35 @@ def test_download_all(sheet_downloader, tmp_path):
|
|
|
326
369
|
"published_date",
|
|
327
370
|
"grid_bb",
|
|
328
371
|
]
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
"
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
372
|
+
print(list(df["name"]))
|
|
373
|
+
assert all(
|
|
374
|
+
name in list(df["name"])
|
|
375
|
+
for name in [
|
|
376
|
+
"map_101602026.png",
|
|
377
|
+
"map_101602038.png",
|
|
378
|
+
"map_102352861.png",
|
|
379
|
+
"map_91617032.png",
|
|
380
|
+
"map_101603986.png",
|
|
381
|
+
"map_101603986_1.png",
|
|
382
|
+
]
|
|
339
383
|
)
|
|
340
384
|
|
|
341
385
|
|
|
342
386
|
def test_download_all_kwargs(sheet_downloader, tmp_path):
|
|
343
387
|
sd = sheet_downloader
|
|
344
|
-
|
|
345
|
-
|
|
388
|
+
# zoom level 14
|
|
389
|
+
sd.get_grid_bb(14)
|
|
390
|
+
maps_path = tmp_path / "test_maps_14/"
|
|
346
391
|
metadata_fname = "test_metadata.csv"
|
|
347
392
|
kwargs = {
|
|
348
393
|
"metadata_to_save": {"test1": ["properties", "test"], "test2": "id"},
|
|
349
394
|
"date_col": "test_date",
|
|
350
395
|
}
|
|
351
396
|
sd.download_all_map_sheets(maps_path, metadata_fname, **kwargs)
|
|
397
|
+
assert os.path.exists(f"{maps_path}/map_102352861.png")
|
|
352
398
|
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
353
399
|
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
354
|
-
assert len(df) ==
|
|
400
|
+
assert len(df) == 6
|
|
355
401
|
assert list(df.columns) == [
|
|
356
402
|
"name",
|
|
357
403
|
"url",
|
|
@@ -362,12 +408,22 @@ def test_download_all_kwargs(sheet_downloader, tmp_path):
|
|
|
362
408
|
"test1",
|
|
363
409
|
"test2",
|
|
364
410
|
]
|
|
365
|
-
assert
|
|
366
|
-
"
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
411
|
+
assert all(
|
|
412
|
+
name in list(df["name"])
|
|
413
|
+
for name in [
|
|
414
|
+
"map_101602026.png",
|
|
415
|
+
"map_101602038.png",
|
|
416
|
+
"map_102352861.png",
|
|
417
|
+
"map_91617032.png",
|
|
418
|
+
"map_101603986.png",
|
|
419
|
+
"map_101603986_1.png",
|
|
420
|
+
]
|
|
421
|
+
)
|
|
422
|
+
# test coords
|
|
423
|
+
assert literal_eval(df.loc[0, "coordinates"]) == approx(
|
|
424
|
+
(-0.98876953125, 53.448806835427575, -0.90087890625, 53.48804553605621),
|
|
425
|
+
rel=1e-6,
|
|
426
|
+
)
|
|
371
427
|
assert df.loc[3, "published_date"] == 2021
|
|
372
428
|
assert df.loc[3, "test1"] == "test"
|
|
373
429
|
assert df.loc[3, "test2"] == "Six_Inch_GB_WFS.132"
|
|
@@ -375,7 +431,7 @@ def test_download_all_kwargs(sheet_downloader, tmp_path):
|
|
|
375
431
|
|
|
376
432
|
def test_download_by_wfs_ids(sheet_downloader, tmp_path):
|
|
377
433
|
sd = sheet_downloader
|
|
378
|
-
sd.get_grid_bb(
|
|
434
|
+
sd.get_grid_bb(14)
|
|
379
435
|
maps_path = tmp_path / "test_maps/"
|
|
380
436
|
metadata_fname = "test_metadata.csv"
|
|
381
437
|
sd.download_map_sheets_by_wfs_ids(
|
|
@@ -411,9 +467,60 @@ def test_download_by_wfs_ids(sheet_downloader, tmp_path):
|
|
|
411
467
|
assert df.loc[1, "name"] == "map_101602038.png"
|
|
412
468
|
|
|
413
469
|
|
|
470
|
+
def test_download_same_image_names(sheet_downloader, tmp_path, capfd):
|
|
471
|
+
sd = sheet_downloader
|
|
472
|
+
sd.get_grid_bb(14)
|
|
473
|
+
maps_path = tmp_path / "test_maps/"
|
|
474
|
+
metadata_fname = "test_metadata.csv"
|
|
475
|
+
sd.download_map_sheets_by_wfs_ids(
|
|
476
|
+
[107, 116], maps_path, metadata_fname
|
|
477
|
+
) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
|
|
478
|
+
assert os.path.exists(f"{maps_path}/map_101603986.png")
|
|
479
|
+
assert os.path.exists(f"{maps_path}/map_101603986_1.png")
|
|
480
|
+
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
481
|
+
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
482
|
+
assert len(df) == 2
|
|
483
|
+
assert list(df.columns) == [
|
|
484
|
+
"name",
|
|
485
|
+
"url",
|
|
486
|
+
"coordinates",
|
|
487
|
+
"crs",
|
|
488
|
+
"published_date",
|
|
489
|
+
"grid_bb",
|
|
490
|
+
]
|
|
491
|
+
assert df.loc[0, "name"] == "map_101603986.png"
|
|
492
|
+
assert df.loc[1, "name"] == "map_101603986_1.png"
|
|
493
|
+
|
|
494
|
+
# run again, nothing should happen
|
|
495
|
+
sd.download_map_sheets_by_wfs_ids([107, 116], maps_path, metadata_fname)
|
|
496
|
+
out, _ = capfd.readouterr()
|
|
497
|
+
assert out.endswith(
|
|
498
|
+
'[INFO] "map_101603986.png" already exists. Skipping download.\n[INFO] "map_101603986_1.png" already exists. Skipping download.\n'
|
|
499
|
+
)
|
|
500
|
+
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
501
|
+
assert len(df) == 2
|
|
502
|
+
|
|
503
|
+
# now overwrite them, but check we don't add new ones (_2, _3 etc.)
|
|
504
|
+
sd.download_map_sheets_by_wfs_ids(
|
|
505
|
+
[107, 116], maps_path, metadata_fname, overwrite=True
|
|
506
|
+
) # 107 and 116 both refer to https://maps.nls.uk/view/101603986
|
|
507
|
+
out, _ = capfd.readouterr()
|
|
508
|
+
assert re.search(
|
|
509
|
+
r"\[INFO\] Downloaded \".*\/test_maps\/map_101603986.png\"\n\[INFO\] Downloaded \".*\/test_maps\/map_101603986_1.png\"\n$",
|
|
510
|
+
out,
|
|
511
|
+
)
|
|
512
|
+
assert os.path.exists(f"{maps_path}/map_101603986.png")
|
|
513
|
+
assert os.path.exists(f"{maps_path}/map_101603986_1.png")
|
|
514
|
+
assert os.path.exists(f"{maps_path}/{metadata_fname}")
|
|
515
|
+
df = pd.read_csv(f"{maps_path}/{metadata_fname}", sep=",", index_col=0)
|
|
516
|
+
assert len(df) == 2
|
|
517
|
+
assert df.loc[0, "name"] == "map_101603986.png"
|
|
518
|
+
assert df.loc[1, "name"] == "map_101603986_1.png"
|
|
519
|
+
|
|
520
|
+
|
|
414
521
|
def test_download_by_wfs_ids_errors(sheet_downloader, tmp_path):
|
|
415
522
|
sd = sheet_downloader
|
|
416
|
-
sd.get_grid_bb(
|
|
523
|
+
sd.get_grid_bb(14)
|
|
417
524
|
maps_path = tmp_path / "test_maps/"
|
|
418
525
|
metadata_fname = "test_metadata.csv"
|
|
419
526
|
with pytest.raises(ValueError, match="as int or list of ints"):
|
|
@@ -427,7 +534,7 @@ def test_download_by_wfs_ids_errors(sheet_downloader, tmp_path):
|
|
|
427
534
|
|
|
428
535
|
def test_download_by_polygon(sheet_downloader, tmp_path):
|
|
429
536
|
sd = sheet_downloader
|
|
430
|
-
sd.get_grid_bb(
|
|
537
|
+
sd.get_grid_bb(14)
|
|
431
538
|
polygon = Polygon(
|
|
432
539
|
[
|
|
433
540
|
[-0.98078243, 53.45664144],
|
|
@@ -467,7 +574,7 @@ def test_download_by_polygon(sheet_downloader, tmp_path):
|
|
|
467
574
|
|
|
468
575
|
def test_download_by_polygon_errors(sheet_downloader, tmp_path):
|
|
469
576
|
sd = sheet_downloader
|
|
470
|
-
sd.get_grid_bb(
|
|
577
|
+
sd.get_grid_bb(14)
|
|
471
578
|
polygon = Polygon([[0, 1], [1, 2], [2, 3], [3, 4], [0, 1]])
|
|
472
579
|
maps_path = tmp_path / "test_maps/"
|
|
473
580
|
metadata_fname = "test_metadata.csv"
|
|
@@ -483,7 +590,7 @@ def test_download_by_polygon_errors(sheet_downloader, tmp_path):
|
|
|
483
590
|
|
|
484
591
|
def test_download_by_coords(sheet_downloader, tmp_path):
|
|
485
592
|
sd = sheet_downloader
|
|
486
|
-
sd.get_grid_bb(
|
|
593
|
+
sd.get_grid_bb(14)
|
|
487
594
|
maps_path = tmp_path / "test_maps/"
|
|
488
595
|
metadata_fname = "test_metadata.csv"
|
|
489
596
|
sd.download_map_sheets_by_coordinates((-0.99, 53.43), maps_path, metadata_fname)
|
|
@@ -504,7 +611,7 @@ def test_download_by_coords(sheet_downloader, tmp_path):
|
|
|
504
611
|
|
|
505
612
|
def test_download_by_coords_errors(sheet_downloader, tmp_path):
|
|
506
613
|
sd = sheet_downloader
|
|
507
|
-
sd.get_grid_bb(
|
|
614
|
+
sd.get_grid_bb(14)
|
|
508
615
|
maps_path = tmp_path / "test_maps/"
|
|
509
616
|
metadata_fname = "test_metadata.csv"
|
|
510
617
|
with pytest.raises(ValueError, match="out of map metadata bounds"):
|
|
@@ -513,7 +620,7 @@ def test_download_by_coords_errors(sheet_downloader, tmp_path):
|
|
|
513
620
|
|
|
514
621
|
def test_download_by_line(sheet_downloader, tmp_path):
|
|
515
622
|
sd = sheet_downloader
|
|
516
|
-
sd.get_grid_bb(
|
|
623
|
+
sd.get_grid_bb(14)
|
|
517
624
|
maps_path = tmp_path / "test_maps/"
|
|
518
625
|
metadata_fname = "test_metadata.csv"
|
|
519
626
|
line = LineString([(-0.99, 53.43), (-0.93, 53.46)])
|
|
@@ -535,7 +642,7 @@ def test_download_by_line(sheet_downloader, tmp_path):
|
|
|
535
642
|
|
|
536
643
|
def test_download_by_line_errors(sheet_downloader, tmp_path):
|
|
537
644
|
sd = sheet_downloader
|
|
538
|
-
sd.get_grid_bb(
|
|
645
|
+
sd.get_grid_bb(14)
|
|
539
646
|
maps_path = tmp_path / "test_maps/"
|
|
540
647
|
metadata_fname = "test_metadata.csv"
|
|
541
648
|
line = LineString([(0, 1), (2, 3)])
|
|
@@ -545,7 +652,7 @@ def test_download_by_line_errors(sheet_downloader, tmp_path):
|
|
|
545
652
|
|
|
546
653
|
def test_download_by_string(sheet_downloader, tmp_path):
|
|
547
654
|
sd = sheet_downloader
|
|
548
|
-
sd.get_grid_bb(
|
|
655
|
+
sd.get_grid_bb(14)
|
|
549
656
|
maps_path = tmp_path / "test_maps/"
|
|
550
657
|
metadata_fname = "test_metadata.csv"
|
|
551
658
|
sd.download_map_sheets_by_string(
|
|
@@ -585,7 +692,7 @@ def test_download_by_string(sheet_downloader, tmp_path):
|
|
|
585
692
|
|
|
586
693
|
def test_download_by_string_value_errors(sheet_downloader, tmp_path):
|
|
587
694
|
sd = sheet_downloader
|
|
588
|
-
sd.get_grid_bb(
|
|
695
|
+
sd.get_grid_bb(14)
|
|
589
696
|
maps_path = tmp_path / "test_maps/"
|
|
590
697
|
metadata_fname = "test_metadata.csv"
|
|
591
698
|
with pytest.raises(ValueError, match="pass ``string`` as a string"):
|
|
@@ -598,7 +705,7 @@ def test_download_by_string_value_errors(sheet_downloader, tmp_path):
|
|
|
598
705
|
|
|
599
706
|
def test_download_by_string_key_errors(sheet_downloader, tmp_path):
|
|
600
707
|
sd = sheet_downloader
|
|
601
|
-
sd.get_grid_bb(
|
|
708
|
+
sd.get_grid_bb(14)
|
|
602
709
|
maps_path = tmp_path / "test_maps/"
|
|
603
710
|
metadata_fname = "test_metadata.csv"
|
|
604
711
|
with pytest.raises(KeyError, match="not found in features dictionary"):
|
|
@@ -609,7 +716,7 @@ def test_download_by_string_key_errors(sheet_downloader, tmp_path):
|
|
|
609
716
|
|
|
610
717
|
def test_download_by_queries(sheet_downloader, tmp_path):
|
|
611
718
|
sd = sheet_downloader
|
|
612
|
-
sd.get_grid_bb(
|
|
719
|
+
sd.get_grid_bb(14)
|
|
613
720
|
maps_path = tmp_path / "test_maps/"
|
|
614
721
|
metadata_fname = "test_metadata.csv"
|
|
615
722
|
sd.query_map_sheets_by_wfs_ids([16320, 132]) # features[0] and [3]
|
|
@@ -634,7 +741,7 @@ def test_download_by_queries(sheet_downloader, tmp_path):
|
|
|
634
741
|
|
|
635
742
|
def test_download_by_queries_errors(sheet_downloader, tmp_path):
|
|
636
743
|
sd = sheet_downloader
|
|
637
|
-
sd.get_grid_bb(
|
|
744
|
+
sd.get_grid_bb(14)
|
|
638
745
|
maps_path = tmp_path / "test_maps/"
|
|
639
746
|
metadata_fname = "test_metadata.csv"
|
|
640
747
|
with pytest.raises(ValueError, match="No query results"):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|