ne-loader 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ne_loader-0.3.2 → ne_loader-0.3.3}/PKG-INFO +4 -2
- {ne_loader-0.3.2 → ne_loader-0.3.3}/README.md +2 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/pyproject.toml +2 -2
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader/cacher.py +3 -4
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader/error_handler.py +3 -3
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader/map_loader.py +24 -20
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/PKG-INFO +4 -2
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/SOURCES.txt +1 -0
- ne_loader-0.3.3/tests/test_download_ne_data.py +127 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/tests/test_map_loader.py +5 -2
- {ne_loader-0.3.2 → ne_loader-0.3.3}/setup.cfg +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader/__init__.py +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader/cli.py +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/dependency_links.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/entry_points.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/requires.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/src/ne_loader.egg-info/top_level.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.3.3}/tests/test_error_handler.py +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ne-loader
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: A simple loader for Natural Earth map data.
|
|
5
5
|
Author-email: Eric Tunn <erictunn@icloud.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/erictunn/ne-loader
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
Requires-Dist: geopandas>=0.12
|
|
10
10
|
Requires-Dist: platformdirs>=4
|
|
@@ -12,6 +12,8 @@ Requires-Dist: requests>=2.25
|
|
|
12
12
|
|
|
13
13
|
# NE Loader
|
|
14
14
|
|
|
15
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
16
|
+
|
|
15
17
|
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
16
18
|
|
|
17
19
|
## Features
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# NE Loader
|
|
2
2
|
|
|
3
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
4
|
+
|
|
3
5
|
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
4
6
|
|
|
5
7
|
## Features
|
|
@@ -7,14 +7,14 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "ne-loader"
|
|
10
|
-
version = "0.3.
|
|
10
|
+
version = "0.3.3"
|
|
11
11
|
description = "A simple loader for Natural Earth map data."
|
|
12
12
|
authors = [
|
|
13
13
|
{ name = "Eric Tunn", email = "erictunn@icloud.com" }
|
|
14
14
|
]
|
|
15
15
|
readme = "README.md"
|
|
16
16
|
license = { file = "LICENSE" }
|
|
17
|
-
requires-python = ">=3.
|
|
17
|
+
requires-python = ">=3.10"
|
|
18
18
|
dependencies = [
|
|
19
19
|
"geopandas>=0.12",
|
|
20
20
|
"platformdirs>=4",
|
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Optional, Union
|
|
6
5
|
|
|
7
6
|
from platformdirs import user_cache_dir
|
|
8
7
|
|
|
9
8
|
|
|
10
|
-
PathLike =
|
|
9
|
+
PathLike = str | Path
|
|
11
10
|
|
|
12
11
|
|
|
13
|
-
def get_cache_dir(path_override:
|
|
12
|
+
def get_cache_dir(path_override: PathLike | None = None) -> Path:
|
|
14
13
|
"""Return the directory used to cache Natural Earth downloads.
|
|
15
14
|
|
|
16
15
|
Cache directory in order of precedence:
|
|
@@ -27,7 +26,7 @@ def get_cache_dir(path_override: Optional[PathLike] = None) -> Path:
|
|
|
27
26
|
if path_override:
|
|
28
27
|
return Path(path_override).expanduser()
|
|
29
28
|
|
|
30
|
-
env_path:
|
|
29
|
+
env_path: str | None = os.getenv("NATURAL_EARTH_CACHE_DIR")
|
|
31
30
|
if env_path:
|
|
32
31
|
return Path(env_path).expanduser()
|
|
33
32
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Helpers for error handling."""
|
|
2
2
|
|
|
3
|
-
from typing import Literal, NoReturn,
|
|
3
|
+
from typing import Literal, NoReturn, overload
|
|
4
4
|
|
|
5
5
|
ErrorMode = Literal["ignore", "raise", "return"]
|
|
6
6
|
|
|
@@ -20,7 +20,7 @@ def error_handler(error: Exception, error_mode: Literal["raise"]) -> NoReturn: .
|
|
|
20
20
|
def error_handler(
|
|
21
21
|
error: Exception,
|
|
22
22
|
error_mode: ErrorMode,
|
|
23
|
-
) ->
|
|
23
|
+
) -> Exception | None:
|
|
24
24
|
"""Provide consistent error handling based on standardised error mode.
|
|
25
25
|
|
|
26
26
|
Args:
|
|
@@ -32,7 +32,7 @@ def error_handler(
|
|
|
32
32
|
ValueError: if error mode is invalid, raises a ValueError.
|
|
33
33
|
|
|
34
34
|
Returns:
|
|
35
|
-
|
|
35
|
+
Exception | None: depending on mode, can return None (ignore),
|
|
36
36
|
the error object (return) or raise the error again (raise).
|
|
37
37
|
|
|
38
38
|
"""
|
|
@@ -5,7 +5,7 @@ import logging
|
|
|
5
5
|
import shutil
|
|
6
6
|
import zipfile
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Literal,
|
|
8
|
+
from typing import Literal, overload
|
|
9
9
|
|
|
10
10
|
import geopandas as gpd
|
|
11
11
|
import requests
|
|
@@ -54,10 +54,10 @@ def get_natural_earth(
|
|
|
54
54
|
name: str,
|
|
55
55
|
res: Resolution = "10m",
|
|
56
56
|
*,
|
|
57
|
-
dir_override:
|
|
57
|
+
dir_override: PathLike | None = None,
|
|
58
58
|
error_mode: Literal["ignore"],
|
|
59
|
-
user_logger:
|
|
60
|
-
) ->
|
|
59
|
+
user_logger: logging.Logger | None = None,
|
|
60
|
+
) -> gpd.GeoDataFrame | None: ...
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
@overload
|
|
@@ -66,9 +66,9 @@ def get_natural_earth(
|
|
|
66
66
|
name: str,
|
|
67
67
|
res: Resolution = "10m",
|
|
68
68
|
*,
|
|
69
|
-
dir_override:
|
|
69
|
+
dir_override: PathLike | None = None,
|
|
70
70
|
error_mode: Literal["raise"] = "raise",
|
|
71
|
-
user_logger:
|
|
71
|
+
user_logger: logging.Logger | None = None,
|
|
72
72
|
) -> gpd.GeoDataFrame: ...
|
|
73
73
|
|
|
74
74
|
|
|
@@ -78,10 +78,10 @@ def get_natural_earth(
|
|
|
78
78
|
name: str,
|
|
79
79
|
res: Resolution = "10m",
|
|
80
80
|
*,
|
|
81
|
-
dir_override:
|
|
81
|
+
dir_override: PathLike | None = None,
|
|
82
82
|
error_mode: Literal["return"],
|
|
83
|
-
user_logger:
|
|
84
|
-
) ->
|
|
83
|
+
user_logger: logging.Logger | None = None,
|
|
84
|
+
) -> gpd.GeoDataFrame | Exception: ...
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def get_natural_earth(
|
|
@@ -89,10 +89,10 @@ def get_natural_earth(
|
|
|
89
89
|
name: str,
|
|
90
90
|
res: Resolution = "10m",
|
|
91
91
|
*,
|
|
92
|
-
dir_override:
|
|
92
|
+
dir_override: PathLike | None = None,
|
|
93
93
|
error_mode: ErrorMode = "raise",
|
|
94
|
-
user_logger:
|
|
95
|
-
) ->
|
|
94
|
+
user_logger: logging.Logger | None = None,
|
|
95
|
+
) -> gpd.GeoDataFrame | Exception | None:
|
|
96
96
|
"""Download, cache, and load a Natural Earth vector dataset.
|
|
97
97
|
|
|
98
98
|
Args:
|
|
@@ -129,7 +129,7 @@ def get_natural_earth(
|
|
|
129
129
|
extract_dir: Path = build_ne_extract_dir(data_dir, name, res)
|
|
130
130
|
shp_file: Path = build_ne_shp_path(data_dir, name, res)
|
|
131
131
|
|
|
132
|
-
|
|
132
|
+
download_ne_data(
|
|
133
133
|
url=url,
|
|
134
134
|
extract_dir=extract_dir,
|
|
135
135
|
name=name,
|
|
@@ -148,7 +148,7 @@ def get_natural_earth(
|
|
|
148
148
|
return error_handler(error, error_mode)
|
|
149
149
|
|
|
150
150
|
|
|
151
|
-
def
|
|
151
|
+
def download_ne_data(
|
|
152
152
|
url: str,
|
|
153
153
|
extract_dir: Path,
|
|
154
154
|
name: str,
|
|
@@ -178,7 +178,7 @@ def _download_ne_data(
|
|
|
178
178
|
|
|
179
179
|
except requests.exceptions.HTTPError as error:
|
|
180
180
|
logger.error(
|
|
181
|
-
"ne-loader/
|
|
181
|
+
"ne-loader/download_ne_data(): "
|
|
182
182
|
"A HTTP error occurred while attempting to fetch data: %s\n"
|
|
183
183
|
"This may cause an error when attempting to load the data.",
|
|
184
184
|
error,
|
|
@@ -186,7 +186,7 @@ def _download_ne_data(
|
|
|
186
186
|
raise
|
|
187
187
|
except requests.exceptions.RequestException as error:
|
|
188
188
|
logger.error(
|
|
189
|
-
"ne-loader/
|
|
189
|
+
"ne-loader/download_ne_data(): "
|
|
190
190
|
"A request error occurred while attempting to fetch data: %s\n"
|
|
191
191
|
"This may cause an error when attempting to load the data.",
|
|
192
192
|
error,
|
|
@@ -196,10 +196,12 @@ def _download_ne_data(
|
|
|
196
196
|
finally:
|
|
197
197
|
with contextlib.suppress(FileNotFoundError):
|
|
198
198
|
zip_path.unlink()
|
|
199
|
-
|
|
200
|
-
|
|
199
|
+
|
|
200
|
+
expected_extract_dir = build_ne_filename(name, res, suffix="")
|
|
201
|
+
if not shp_file.exists() and extract_dir.name == expected_extract_dir:
|
|
201
202
|
shutil.rmtree(extract_dir, ignore_errors=True)
|
|
202
203
|
|
|
204
|
+
|
|
203
205
|
def validate_res(res: str) -> None:
|
|
204
206
|
"""Validate the resolution against "10m", "50m", "110m".
|
|
205
207
|
|
|
@@ -211,5 +213,7 @@ def validate_res(res: str) -> None:
|
|
|
211
213
|
|
|
212
214
|
"""
|
|
213
215
|
if res not in ("10m", "50m", "110m"):
|
|
214
|
-
raise ValueError(
|
|
215
|
-
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Invalid resolution: {res}.\n"
|
|
218
|
+
'Resolution must be one of ("10m", "50m", "110m")'
|
|
219
|
+
)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ne-loader
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: A simple loader for Natural Earth map data.
|
|
5
5
|
Author-email: Eric Tunn <erictunn@icloud.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/erictunn/ne-loader
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
Requires-Dist: geopandas>=0.12
|
|
10
10
|
Requires-Dist: platformdirs>=4
|
|
@@ -12,6 +12,8 @@ Requires-Dist: requests>=2.25
|
|
|
12
12
|
|
|
13
13
|
# NE Loader
|
|
14
14
|
|
|
15
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
16
|
+
|
|
15
17
|
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
16
18
|
|
|
17
19
|
## Features
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Tests for natural earth data downloader functions."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import logging
|
|
5
|
+
import zipfile
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from ne_loader.map_loader import build_ne_filename, download_ne_data, Resolution
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _mock_zip_bytes(name: str, res: str) -> bytes:
|
|
16
|
+
"""Build an in-memory Natural Earth zip file containing one shapefile."""
|
|
17
|
+
buffer = io.BytesIO()
|
|
18
|
+
with zipfile.ZipFile(buffer, "w") as zip_file:
|
|
19
|
+
zip_file.writestr(
|
|
20
|
+
build_ne_filename(name, res, suffix=".shp"),
|
|
21
|
+
b"mock shp content",
|
|
22
|
+
)
|
|
23
|
+
return buffer.getvalue()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MockResponse:
|
|
27
|
+
"""Mock successful response object for the downloader's requests.get call."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, data: bytes) -> None:
|
|
30
|
+
"""Store response bytes for chunked streaming."""
|
|
31
|
+
self._data = data
|
|
32
|
+
|
|
33
|
+
def raise_for_status(self) -> None:
|
|
34
|
+
"""Match requests.Response.raise_for_status for a successful response."""
|
|
35
|
+
return None
|
|
36
|
+
|
|
37
|
+
def iter_content(self, chunk_size: int = 8192) -> Iterator[bytes]:
|
|
38
|
+
"""Yield response bytes in the same shape as requests.Response."""
|
|
39
|
+
for start in range(0, len(self._data), chunk_size):
|
|
40
|
+
yield self._data[start : start + chunk_size]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
url = "https://example.org/fake.zip"
|
|
45
|
+
name = "admin_0_countries"
|
|
46
|
+
res: Resolution = "10m"
|
|
47
|
+
logger = logging.getLogger("testing")
|
|
48
|
+
|
|
49
|
+
def test_download_ne_data_success(
|
|
50
|
+
tmp_path: Path,
|
|
51
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Verify download_ne_data writes the extracted shapefile on success."""
|
|
54
|
+
mocked_zip = _mock_zip_bytes(name, res)
|
|
55
|
+
|
|
56
|
+
def mock_get(request_url: str, stream: bool, timeout: int) -> MockResponse:
|
|
57
|
+
"""Return a fake response and verify the downloader's request options."""
|
|
58
|
+
assert request_url == url
|
|
59
|
+
assert stream is True
|
|
60
|
+
assert timeout == 10
|
|
61
|
+
return MockResponse(mocked_zip)
|
|
62
|
+
|
|
63
|
+
monkeypatch.setattr(requests, "get", mock_get)
|
|
64
|
+
base = tmp_path / "ne-cache"
|
|
65
|
+
base.mkdir()
|
|
66
|
+
extract_dir = base / build_ne_filename(name, res, suffix="")
|
|
67
|
+
zip_path = base / build_ne_filename(name, res)
|
|
68
|
+
shp_file = extract_dir / build_ne_filename(name, res, suffix=".shp")
|
|
69
|
+
|
|
70
|
+
download_ne_data(
|
|
71
|
+
url=url,
|
|
72
|
+
extract_dir=extract_dir,
|
|
73
|
+
name=name,
|
|
74
|
+
res=res,
|
|
75
|
+
zip_path=zip_path,
|
|
76
|
+
shp_file=shp_file,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
assert shp_file.exists()
|
|
81
|
+
assert extract_dir.exists()
|
|
82
|
+
assert not zip_path.exists()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class BadResponse:
|
|
86
|
+
"""Mock unsuccessful response for the downloader's requests.get call."""
|
|
87
|
+
|
|
88
|
+
def raise_for_status(self) -> None:
|
|
89
|
+
"""Mock the unsuccessful response."""
|
|
90
|
+
raise requests.exceptions.HTTPError("404 Client Error")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_download_ne_data_failure(
|
|
94
|
+
tmp_path: Path,
|
|
95
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Tests that download_ne_data leaves no artifacts upon HTTP error."""
|
|
98
|
+
|
|
99
|
+
def mock_get(request_url: str, stream: bool, timeout: int) -> BadResponse:
|
|
100
|
+
"""Return a fake response and verify the downloader's request options."""
|
|
101
|
+
assert request_url == url
|
|
102
|
+
assert stream is True
|
|
103
|
+
assert timeout == 10
|
|
104
|
+
return BadResponse()
|
|
105
|
+
|
|
106
|
+
monkeypatch.setattr(requests, "get", mock_get)
|
|
107
|
+
|
|
108
|
+
base = tmp_path / "ne-cache"
|
|
109
|
+
base.mkdir()
|
|
110
|
+
extract_dir = base / build_ne_filename(name, res, suffix="")
|
|
111
|
+
zip_path = base / build_ne_filename(name, res)
|
|
112
|
+
shp_file = extract_dir / build_ne_filename(name, res, suffix=".shp")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
with pytest.raises(requests.exceptions.HTTPError):
|
|
116
|
+
download_ne_data(
|
|
117
|
+
url=url,
|
|
118
|
+
extract_dir=extract_dir,
|
|
119
|
+
name=name,
|
|
120
|
+
res=res,
|
|
121
|
+
zip_path=zip_path,
|
|
122
|
+
shp_file=shp_file,
|
|
123
|
+
logger=logger,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
assert not zip_path.exists()
|
|
127
|
+
assert not extract_dir.exists()
|
|
@@ -51,6 +51,9 @@ def test_build_ne_shp_path() -> None:
|
|
|
51
51
|
|
|
52
52
|
def test_validate_res() -> None:
|
|
53
53
|
"""Tests that validate_res(res=kaboom) raises the correct ValueError."""
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
expected_message = (
|
|
55
|
+
'Invalid resolution: kaboom.\n'
|
|
56
|
+
'Resolution must be one of ("10m", "50m", "110m")'
|
|
57
|
+
)
|
|
58
|
+
with pytest.raises(ValueError, match=re.escape(expected_message)):
|
|
56
59
|
validate_res("kaboom")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|