ne-loader 0.3.2__tar.gz → 0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ne_loader-0.3.2 → ne_loader-0.4}/PKG-INFO +39 -2
- ne_loader-0.4/README.md +89 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/pyproject.toml +7 -3
- ne_loader-0.4/src/ne_loader/__init__.py +8 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader/cacher.py +3 -4
- ne_loader-0.4/src/ne_loader/cli.py +71 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader/error_handler.py +4 -4
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader/map_loader.py +26 -22
- ne_loader-0.4/src/ne_loader/py.typed +0 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/PKG-INFO +39 -2
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/SOURCES.txt +3 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/requires.txt +1 -0
- ne_loader-0.4/tests/test_cli.py +49 -0
- ne_loader-0.4/tests/test_download_ne_data.py +127 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/tests/test_map_loader.py +5 -2
- ne_loader-0.3.2/README.md +0 -53
- ne_loader-0.3.2/src/ne_loader/__init__.py +0 -4
- ne_loader-0.3.2/src/ne_loader/cli.py +0 -38
- {ne_loader-0.3.2 → ne_loader-0.4}/setup.cfg +0 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/dependency_links.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/entry_points.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/src/ne_loader.egg-info/top_level.txt +0 -0
- {ne_loader-0.3.2 → ne_loader-0.4}/tests/test_error_handler.py +0 -0
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ne-loader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4
|
|
4
4
|
Summary: A simple loader for Natural Earth map data.
|
|
5
5
|
Author-email: Eric Tunn <erictunn@icloud.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/erictunn/ne-loader
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
Requires-Dist: geopandas>=0.12
|
|
10
10
|
Requires-Dist: platformdirs>=4
|
|
11
11
|
Requires-Dist: requests>=2.25
|
|
12
|
+
Requires-Dist: click>=8.4
|
|
12
13
|
|
|
13
14
|
# NE Loader
|
|
14
15
|
|
|
16
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
17
|
+
|
|
15
18
|
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
16
19
|
|
|
17
20
|
## Features
|
|
@@ -54,6 +57,40 @@ To set an environment override for the NE data save path:
|
|
|
54
57
|
export NATURAL_EARTH_CACHE_DIR="..."
|
|
55
58
|
```
|
|
56
59
|
|
|
60
|
+
Some commands include:
|
|
61
|
+
List cached datasets.
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
ne-loader list
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Query where the cached datasets are.
|
|
68
|
+
If you change NATURAL_EARTH_CACHE_DIR, the command shows the active cache path for the current environment.
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
ne-loader where
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Download a Natural Earth Dataset. Default resolution is 10m.
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
ne-loader download {category} {name} --res {res}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Example usage to download 10m_admin_0_countries
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
ne-loader download cultural admin_0_countries
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Delete either one dataset or all, from current cache directory.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
ne-loader rm {dataset}
|
|
90
|
+
# or
|
|
91
|
+
ne-loader rm --all
|
|
92
|
+
```
|
|
93
|
+
|
|
57
94
|
## Tests
|
|
58
95
|
|
|
59
96
|
```bash
|
ne_loader-0.4/README.md
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# NE Loader
|
|
2
|
+
|
|
3
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- Download and cache Natural Earth datasets
|
|
10
|
+
- Load shapefiles as GeoDataFrames
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install ne-loader
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Usage
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from ne_loader import map_loader
|
|
22
|
+
world = map_loader.get_natural_earth('cultural', 'admin_0_countries')
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Errors are raised by default. To return the caught exception instead:
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
result = map_loader.get_natural_earth(
|
|
29
|
+
'cultural',
|
|
30
|
+
'admin_0_countries',
|
|
31
|
+
error_mode='return',
|
|
32
|
+
)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## CLI
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
ne-loader --help
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
To set an environment override for the NE data save path:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
export NATURAL_EARTH_CACHE_DIR="..."
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Some commands include:
|
|
48
|
+
List cached datasets.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
ne-loader list
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Query where the cached datasets are.
|
|
55
|
+
If you change NATURAL_EARTH_CACHE_DIR, the command shows the active cache path for the current environment.
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
ne-loader where
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Download a Natural Earth Dataset. Default resolution is 10m.
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
ne-loader download {category} {name} --res {res}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Example usage to download 10m_admin_0_countries
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
ne-loader download cultural admin_0_countries
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Delete either one dataset or all, from current cache directory.
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
ne-loader rm {dataset}
|
|
77
|
+
# or
|
|
78
|
+
ne-loader rm --all
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Tests
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
python3 -m pytest tests
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
MIT
|
|
@@ -5,20 +5,24 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[tool.setuptools.packages.find]
|
|
6
6
|
where = ["src"]
|
|
7
7
|
|
|
8
|
+
[tool.setuptools.package-data]
|
|
9
|
+
ne_loader = ["py.typed"]
|
|
10
|
+
|
|
8
11
|
[project]
|
|
9
12
|
name = "ne-loader"
|
|
10
|
-
version = "0.
|
|
13
|
+
version = "0.4"
|
|
11
14
|
description = "A simple loader for Natural Earth map data."
|
|
12
15
|
authors = [
|
|
13
16
|
{ name = "Eric Tunn", email = "erictunn@icloud.com" }
|
|
14
17
|
]
|
|
15
18
|
readme = "README.md"
|
|
16
19
|
license = { file = "LICENSE" }
|
|
17
|
-
requires-python = ">=3.
|
|
20
|
+
requires-python = ">=3.10"
|
|
18
21
|
dependencies = [
|
|
19
22
|
"geopandas>=0.12",
|
|
20
23
|
"platformdirs>=4",
|
|
21
|
-
"requests>=2.25"
|
|
24
|
+
"requests>=2.25",
|
|
25
|
+
"click>=8.4"
|
|
22
26
|
]
|
|
23
27
|
|
|
24
28
|
[project.urls]
|
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Optional, Union
|
|
6
5
|
|
|
7
6
|
from platformdirs import user_cache_dir
|
|
8
7
|
|
|
9
8
|
|
|
10
|
-
PathLike =
|
|
9
|
+
PathLike = str | Path
|
|
11
10
|
|
|
12
11
|
|
|
13
|
-
def get_cache_dir(path_override:
|
|
12
|
+
def get_cache_dir(path_override: PathLike | None = None) -> Path:
|
|
14
13
|
"""Return the directory used to cache Natural Earth downloads.
|
|
15
14
|
|
|
16
15
|
Cache directory in order of precedence:
|
|
@@ -27,7 +26,7 @@ def get_cache_dir(path_override: Optional[PathLike] = None) -> Path:
|
|
|
27
26
|
if path_override:
|
|
28
27
|
return Path(path_override).expanduser()
|
|
29
28
|
|
|
30
|
-
env_path:
|
|
29
|
+
env_path: str | None = os.getenv("NATURAL_EARTH_CACHE_DIR")
|
|
31
30
|
if env_path:
|
|
32
31
|
return Path(env_path).expanduser()
|
|
33
32
|
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Provides basic command line functionality."""
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
from . import map_loader
|
|
7
|
+
from .cacher import get_cache_dir
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@click.group()
|
|
11
|
+
def main() -> None:
|
|
12
|
+
"""Entry point for the ne-loader CLI."""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@main.command("download")
|
|
16
|
+
@click.argument("category")
|
|
17
|
+
@click.argument("name")
|
|
18
|
+
@click.option("--res", default="10m", help="NE dataset resolution (default: 10m)")
|
|
19
|
+
def cli_get_natural_earth(category: str, name: str, res: map_loader.Resolution) -> None:
|
|
20
|
+
"""Download a Natural Earth dataset and load it into GeoPandas."""
|
|
21
|
+
map_loader.get_natural_earth(category, name, res)
|
|
22
|
+
|
|
23
|
+
@main.command("where")
|
|
24
|
+
def cli_where_cache() -> None:
|
|
25
|
+
"""Locate the cache directory."""
|
|
26
|
+
cache_dir = get_cache_dir()
|
|
27
|
+
click.echo(cache_dir)
|
|
28
|
+
|
|
29
|
+
@main.command("list")
|
|
30
|
+
def cli_list_cached_files() -> None:
|
|
31
|
+
"""List all cached NE files within the cache dir."""
|
|
32
|
+
cache_dir = get_cache_dir()
|
|
33
|
+
sub_folders = [f.name for f in cache_dir.iterdir() if f.is_dir()]
|
|
34
|
+
click.echo(", ".join(sub_folders))
|
|
35
|
+
|
|
36
|
+
@main.command("rm")
|
|
37
|
+
@click.argument("dataset", required=False)
|
|
38
|
+
@click.option(
|
|
39
|
+
"--all",
|
|
40
|
+
"all_",
|
|
41
|
+
is_flag=True,
|
|
42
|
+
default=False,
|
|
43
|
+
help="Remove the entire cache directory instead of a single dataset.",
|
|
44
|
+
)
|
|
45
|
+
def cli_remove_cached_file(dataset: str | None, all_: bool) -> None:
|
|
46
|
+
"""Remove a specific cached dataset, or the entire cache when --all is used."""
|
|
47
|
+
cache_dir = get_cache_dir()
|
|
48
|
+
|
|
49
|
+
if all_:
|
|
50
|
+
if click.confirm(
|
|
51
|
+
"Are you sure you want to delete all cached Natural Earth datasets?",
|
|
52
|
+
default=False,
|
|
53
|
+
):
|
|
54
|
+
shutil.rmtree(cache_dir, ignore_errors=True)
|
|
55
|
+
click.echo("Removed all cached datasets.")
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
if not dataset:
|
|
59
|
+
raise click.UsageError("Specify a dataset name to remove or use --all to" \
|
|
60
|
+
"clear the whole cache.")
|
|
61
|
+
|
|
62
|
+
dataset_dir = cache_dir / dataset
|
|
63
|
+
if not dataset_dir.exists():
|
|
64
|
+
raise click.ClickException(f"Cached dataset not found: {dataset}")
|
|
65
|
+
|
|
66
|
+
if click.confirm(
|
|
67
|
+
f"Are you sure you want to delete the cached dataset '{dataset}'?",
|
|
68
|
+
default=False,
|
|
69
|
+
):
|
|
70
|
+
shutil.rmtree(dataset_dir, ignore_errors=True)
|
|
71
|
+
click.echo(f"Removed cached dataset: {dataset}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Helpers for error handling."""
|
|
2
2
|
|
|
3
|
-
from typing import Literal, NoReturn,
|
|
3
|
+
from typing import Literal, NoReturn, overload
|
|
4
4
|
|
|
5
5
|
ErrorMode = Literal["ignore", "raise", "return"]
|
|
6
6
|
|
|
@@ -20,7 +20,7 @@ def error_handler(error: Exception, error_mode: Literal["raise"]) -> NoReturn: .
|
|
|
20
20
|
def error_handler(
|
|
21
21
|
error: Exception,
|
|
22
22
|
error_mode: ErrorMode,
|
|
23
|
-
) ->
|
|
23
|
+
) -> Exception | None:
|
|
24
24
|
"""Provide consistent error handling based on standardised error mode.
|
|
25
25
|
|
|
26
26
|
Args:
|
|
@@ -32,7 +32,7 @@ def error_handler(
|
|
|
32
32
|
ValueError: if error mode is invalid, raises a ValueError.
|
|
33
33
|
|
|
34
34
|
Returns:
|
|
35
|
-
|
|
35
|
+
Exception | None: depending on mode, can return None (ignore),
|
|
36
36
|
the error object (return) or raise the error again (raise).
|
|
37
37
|
|
|
38
38
|
"""
|
|
@@ -45,7 +45,7 @@ def error_handler(
|
|
|
45
45
|
raise error
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def validate_error_mode(error_mode:
|
|
48
|
+
def validate_error_mode(error_mode: ErrorMode) -> None:
|
|
49
49
|
"""Validate error_mode.
|
|
50
50
|
|
|
51
51
|
Args:
|
|
@@ -5,7 +5,7 @@ import logging
|
|
|
5
5
|
import shutil
|
|
6
6
|
import zipfile
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Literal,
|
|
8
|
+
from typing import Literal, overload
|
|
9
9
|
|
|
10
10
|
import geopandas as gpd
|
|
11
11
|
import requests
|
|
@@ -54,10 +54,10 @@ def get_natural_earth(
|
|
|
54
54
|
name: str,
|
|
55
55
|
res: Resolution = "10m",
|
|
56
56
|
*,
|
|
57
|
-
dir_override:
|
|
57
|
+
dir_override: PathLike | None = None,
|
|
58
58
|
error_mode: Literal["ignore"],
|
|
59
|
-
user_logger:
|
|
60
|
-
) ->
|
|
59
|
+
user_logger: logging.Logger | None = None,
|
|
60
|
+
) -> gpd.GeoDataFrame | None: ...
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
@overload
|
|
@@ -66,9 +66,9 @@ def get_natural_earth(
|
|
|
66
66
|
name: str,
|
|
67
67
|
res: Resolution = "10m",
|
|
68
68
|
*,
|
|
69
|
-
dir_override:
|
|
69
|
+
dir_override: PathLike | None = None,
|
|
70
70
|
error_mode: Literal["raise"] = "raise",
|
|
71
|
-
user_logger:
|
|
71
|
+
user_logger: logging.Logger | None = None,
|
|
72
72
|
) -> gpd.GeoDataFrame: ...
|
|
73
73
|
|
|
74
74
|
|
|
@@ -78,10 +78,10 @@ def get_natural_earth(
|
|
|
78
78
|
name: str,
|
|
79
79
|
res: Resolution = "10m",
|
|
80
80
|
*,
|
|
81
|
-
dir_override:
|
|
81
|
+
dir_override: PathLike | None = None,
|
|
82
82
|
error_mode: Literal["return"],
|
|
83
|
-
user_logger:
|
|
84
|
-
) ->
|
|
83
|
+
user_logger: logging.Logger | None = None,
|
|
84
|
+
) -> gpd.GeoDataFrame | Exception: ...
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def get_natural_earth(
|
|
@@ -89,10 +89,10 @@ def get_natural_earth(
|
|
|
89
89
|
name: str,
|
|
90
90
|
res: Resolution = "10m",
|
|
91
91
|
*,
|
|
92
|
-
dir_override:
|
|
92
|
+
dir_override: PathLike | None = None,
|
|
93
93
|
error_mode: ErrorMode = "raise",
|
|
94
|
-
user_logger:
|
|
95
|
-
) ->
|
|
94
|
+
user_logger: logging.Logger | None = None,
|
|
95
|
+
) -> gpd.GeoDataFrame | Exception | None:
|
|
96
96
|
"""Download, cache, and load a Natural Earth vector dataset.
|
|
97
97
|
|
|
98
98
|
Args:
|
|
@@ -129,7 +129,7 @@ def get_natural_earth(
|
|
|
129
129
|
extract_dir: Path = build_ne_extract_dir(data_dir, name, res)
|
|
130
130
|
shp_file: Path = build_ne_shp_path(data_dir, name, res)
|
|
131
131
|
|
|
132
|
-
|
|
132
|
+
download_ne_data(
|
|
133
133
|
url=url,
|
|
134
134
|
extract_dir=extract_dir,
|
|
135
135
|
name=name,
|
|
@@ -142,13 +142,13 @@ def get_natural_earth(
|
|
|
142
142
|
return gpd.read_file(shp_file)
|
|
143
143
|
except Exception as error:
|
|
144
144
|
logger.error(
|
|
145
|
-
"ne-loader: error caught fetching data with get_natural_earth()
|
|
145
|
+
"ne-loader: error caught fetching data with get_natural_earth():%s",
|
|
146
146
|
error,
|
|
147
147
|
)
|
|
148
148
|
return error_handler(error, error_mode)
|
|
149
149
|
|
|
150
150
|
|
|
151
|
-
def
|
|
151
|
+
def download_ne_data(
|
|
152
152
|
url: str,
|
|
153
153
|
extract_dir: Path,
|
|
154
154
|
name: str,
|
|
@@ -178,7 +178,7 @@ def _download_ne_data(
|
|
|
178
178
|
|
|
179
179
|
except requests.exceptions.HTTPError as error:
|
|
180
180
|
logger.error(
|
|
181
|
-
"ne-loader/
|
|
181
|
+
"ne-loader/download_ne_data(): "
|
|
182
182
|
"A HTTP error occurred while attempting to fetch data: %s\n"
|
|
183
183
|
"This may cause an error when attempting to load the data.",
|
|
184
184
|
error,
|
|
@@ -186,7 +186,7 @@ def _download_ne_data(
|
|
|
186
186
|
raise
|
|
187
187
|
except requests.exceptions.RequestException as error:
|
|
188
188
|
logger.error(
|
|
189
|
-
"ne-loader/
|
|
189
|
+
"ne-loader/download_ne_data(): "
|
|
190
190
|
"A request error occurred while attempting to fetch data: %s\n"
|
|
191
191
|
"This may cause an error when attempting to load the data.",
|
|
192
192
|
error,
|
|
@@ -196,11 +196,13 @@ def _download_ne_data(
|
|
|
196
196
|
finally:
|
|
197
197
|
with contextlib.suppress(FileNotFoundError):
|
|
198
198
|
zip_path.unlink()
|
|
199
|
-
|
|
200
|
-
|
|
199
|
+
|
|
200
|
+
expected_extract_dir = build_ne_filename(name, res, suffix="")
|
|
201
|
+
if not shp_file.exists() and extract_dir.name == expected_extract_dir:
|
|
201
202
|
shutil.rmtree(extract_dir, ignore_errors=True)
|
|
202
203
|
|
|
203
|
-
|
|
204
|
+
|
|
205
|
+
def validate_res(res: Resolution) -> None:
|
|
204
206
|
"""Validate the resolution against "10m", "50m", "110m".
|
|
205
207
|
|
|
206
208
|
Args:
|
|
@@ -211,5 +213,7 @@ def validate_res(res: str) -> None:
|
|
|
211
213
|
|
|
212
214
|
"""
|
|
213
215
|
if res not in ("10m", "50m", "110m"):
|
|
214
|
-
raise ValueError(
|
|
215
|
-
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Invalid resolution: {res}.\n"
|
|
218
|
+
'Resolution must be one of ("10m", "50m", "110m")'
|
|
219
|
+
)
|
|
File without changes
|
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ne-loader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4
|
|
4
4
|
Summary: A simple loader for Natural Earth map data.
|
|
5
5
|
Author-email: Eric Tunn <erictunn@icloud.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/erictunn/ne-loader
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
Requires-Dist: geopandas>=0.12
|
|
10
10
|
Requires-Dist: platformdirs>=4
|
|
11
11
|
Requires-Dist: requests>=2.25
|
|
12
|
+
Requires-Dist: click>=8.4
|
|
12
13
|
|
|
13
14
|
# NE Loader
|
|
14
15
|
|
|
16
|
+
[](https://github.com/erictunn/NE_loader_package/actions/workflows/ci.yml)
|
|
17
|
+
|
|
15
18
|
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
16
19
|
|
|
17
20
|
## Features
|
|
@@ -54,6 +57,40 @@ To set an environment override for the NE data save path:
|
|
|
54
57
|
export NATURAL_EARTH_CACHE_DIR="..."
|
|
55
58
|
```
|
|
56
59
|
|
|
60
|
+
Some commands include:
|
|
61
|
+
List cached datasets.
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
ne-loader list
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Query where the cached datasets are.
|
|
68
|
+
If you change NATURAL_EARTH_CACHE_DIR, the command shows the active cache path for the current environment.
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
ne-loader where
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Download a Natural Earth Dataset. Default resolution is 10m.
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
ne-loader download {category} {name} --res {res}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Example usage to download 10m_admin_0_countries
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
ne-loader download cultural admin_0_countries
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Delete either one dataset or all, from current cache directory.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
ne-loader rm {dataset}
|
|
90
|
+
# or
|
|
91
|
+
ne-loader rm --all
|
|
92
|
+
```
|
|
93
|
+
|
|
57
94
|
## Tests
|
|
58
95
|
|
|
59
96
|
```bash
|
|
@@ -5,11 +5,14 @@ src/ne_loader/cacher.py
|
|
|
5
5
|
src/ne_loader/cli.py
|
|
6
6
|
src/ne_loader/error_handler.py
|
|
7
7
|
src/ne_loader/map_loader.py
|
|
8
|
+
src/ne_loader/py.typed
|
|
8
9
|
src/ne_loader.egg-info/PKG-INFO
|
|
9
10
|
src/ne_loader.egg-info/SOURCES.txt
|
|
10
11
|
src/ne_loader.egg-info/dependency_links.txt
|
|
11
12
|
src/ne_loader.egg-info/entry_points.txt
|
|
12
13
|
src/ne_loader.egg-info/requires.txt
|
|
13
14
|
src/ne_loader.egg-info/top_level.txt
|
|
15
|
+
tests/test_cli.py
|
|
16
|
+
tests/test_download_ne_data.py
|
|
14
17
|
tests/test_error_handler.py
|
|
15
18
|
tests/test_map_loader.py
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Minimal tests for the CLI entrypoint."""
|
|
2
|
+
|
|
3
|
+
from click.testing import CliRunner
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
import ne_loader.cli as cli
|
|
7
|
+
from ne_loader.map_loader import Resolution
|
|
8
|
+
from ne_loader.cacher import get_cache_dir
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_cli_delegates_to_get_natural_earth(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
12
|
+
"""Ensure the CLI forwards arguments to map_loader.get_natural_earth."""
|
|
13
|
+
called = {}
|
|
14
|
+
|
|
15
|
+
def fake_get(category: str, name: str, res: Resolution):
|
|
16
|
+
called["args"] = (category, name, res)
|
|
17
|
+
|
|
18
|
+
monkeypatch.setattr(cli.map_loader, "get_natural_earth", fake_get)
|
|
19
|
+
|
|
20
|
+
runner = CliRunner()
|
|
21
|
+
result = runner.invoke(
|
|
22
|
+
cli.main,
|
|
23
|
+
["download", "Cultural", "admin_0_countries", "--res", "50m"],
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
assert result.exit_code == 0
|
|
27
|
+
assert called["args"] == ("Cultural", "admin_0_countries", "50m")
|
|
28
|
+
|
|
29
|
+
def test_cli_where_errors() -> None:
|
|
30
|
+
"""Ensure CLI where command returns without error."""
|
|
31
|
+
runner = CliRunner()
|
|
32
|
+
result = runner.invoke(
|
|
33
|
+
cli.main,
|
|
34
|
+
["where"],
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
assert result.exit_code == 0
|
|
38
|
+
assert result.exception is None
|
|
39
|
+
|
|
40
|
+
def test_cli_where_output() -> None:
|
|
41
|
+
"""Ensure where command output is the same as get_cache_dir()."""
|
|
42
|
+
true_cache_dir = str(get_cache_dir()).strip()
|
|
43
|
+
|
|
44
|
+
runner = CliRunner()
|
|
45
|
+
result = runner.invoke(
|
|
46
|
+
cli.main,
|
|
47
|
+
["where"],
|
|
48
|
+
)
|
|
49
|
+
assert true_cache_dir == str(result.output).strip()
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Tests for natural earth data downloader functions."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import logging
|
|
5
|
+
import zipfile
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from ne_loader.map_loader import build_ne_filename, download_ne_data, Resolution
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _mock_zip_bytes(name: str, res: str) -> bytes:
|
|
16
|
+
"""Build an in-memory Natural Earth zip file containing one shapefile."""
|
|
17
|
+
buffer = io.BytesIO()
|
|
18
|
+
with zipfile.ZipFile(buffer, "w") as zip_file:
|
|
19
|
+
zip_file.writestr(
|
|
20
|
+
build_ne_filename(name, res, suffix=".shp"),
|
|
21
|
+
b"mock shp content",
|
|
22
|
+
)
|
|
23
|
+
return buffer.getvalue()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MockResponse:
|
|
27
|
+
"""Mock successful response object for the downloader's requests.get call."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, data: bytes) -> None:
|
|
30
|
+
"""Store response bytes for chunked streaming."""
|
|
31
|
+
self._data = data
|
|
32
|
+
|
|
33
|
+
def raise_for_status(self) -> None:
|
|
34
|
+
"""Match requests.Response.raise_for_status for a successful response."""
|
|
35
|
+
return None
|
|
36
|
+
|
|
37
|
+
def iter_content(self, chunk_size: int = 8192) -> Iterator[bytes]:
|
|
38
|
+
"""Yield response bytes in the same shape as requests.Response."""
|
|
39
|
+
for start in range(0, len(self._data), chunk_size):
|
|
40
|
+
yield self._data[start : start + chunk_size]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
url = "https://example.org/fake.zip"
|
|
45
|
+
name = "admin_0_countries"
|
|
46
|
+
res: Resolution = "10m"
|
|
47
|
+
logger = logging.getLogger("testing")
|
|
48
|
+
|
|
49
|
+
def test_download_ne_data_success(
|
|
50
|
+
tmp_path: Path,
|
|
51
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Verify download_ne_data writes the extracted shapefile on success."""
|
|
54
|
+
mocked_zip = _mock_zip_bytes(name, res)
|
|
55
|
+
|
|
56
|
+
def mock_get(request_url: str, stream: bool, timeout: int) -> MockResponse:
|
|
57
|
+
"""Return a fake response and verify the downloader's request options."""
|
|
58
|
+
assert request_url == url
|
|
59
|
+
assert stream is True
|
|
60
|
+
assert timeout == 10
|
|
61
|
+
return MockResponse(mocked_zip)
|
|
62
|
+
|
|
63
|
+
monkeypatch.setattr(requests, "get", mock_get)
|
|
64
|
+
base = tmp_path / "ne-cache"
|
|
65
|
+
base.mkdir()
|
|
66
|
+
extract_dir = base / build_ne_filename(name, res, suffix="")
|
|
67
|
+
zip_path = base / build_ne_filename(name, res)
|
|
68
|
+
shp_file = extract_dir / build_ne_filename(name, res, suffix=".shp")
|
|
69
|
+
|
|
70
|
+
download_ne_data(
|
|
71
|
+
url=url,
|
|
72
|
+
extract_dir=extract_dir,
|
|
73
|
+
name=name,
|
|
74
|
+
res=res,
|
|
75
|
+
zip_path=zip_path,
|
|
76
|
+
shp_file=shp_file,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
assert shp_file.exists()
|
|
81
|
+
assert extract_dir.exists()
|
|
82
|
+
assert not zip_path.exists()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class BadResponse:
|
|
86
|
+
"""Mock unsuccessful response for the downloader's requests.get call."""
|
|
87
|
+
|
|
88
|
+
def raise_for_status(self) -> None:
|
|
89
|
+
"""Mock the unsuccessful response."""
|
|
90
|
+
raise requests.exceptions.HTTPError("404 Client Error")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_download_ne_data_failure(
|
|
94
|
+
tmp_path: Path,
|
|
95
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Tests that download_ne_data leaves no artifacts upon HTTP error."""
|
|
98
|
+
|
|
99
|
+
def mock_get(request_url: str, stream: bool, timeout: int) -> BadResponse:
|
|
100
|
+
"""Return a fake response and verify the downloader's request options."""
|
|
101
|
+
assert request_url == url
|
|
102
|
+
assert stream is True
|
|
103
|
+
assert timeout == 10
|
|
104
|
+
return BadResponse()
|
|
105
|
+
|
|
106
|
+
monkeypatch.setattr(requests, "get", mock_get)
|
|
107
|
+
|
|
108
|
+
base = tmp_path / "ne-cache"
|
|
109
|
+
base.mkdir()
|
|
110
|
+
extract_dir = base / build_ne_filename(name, res, suffix="")
|
|
111
|
+
zip_path = base / build_ne_filename(name, res)
|
|
112
|
+
shp_file = extract_dir / build_ne_filename(name, res, suffix=".shp")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
with pytest.raises(requests.exceptions.HTTPError):
|
|
116
|
+
download_ne_data(
|
|
117
|
+
url=url,
|
|
118
|
+
extract_dir=extract_dir,
|
|
119
|
+
name=name,
|
|
120
|
+
res=res,
|
|
121
|
+
zip_path=zip_path,
|
|
122
|
+
shp_file=shp_file,
|
|
123
|
+
logger=logger,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
assert not zip_path.exists()
|
|
127
|
+
assert not extract_dir.exists()
|
|
@@ -51,6 +51,9 @@ def test_build_ne_shp_path() -> None:
|
|
|
51
51
|
|
|
52
52
|
def test_validate_res() -> None:
|
|
53
53
|
"""Tests that validate_res(res=kaboom) raises the correct ValueError."""
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
expected_message = (
|
|
55
|
+
'Invalid resolution: kaboom.\n'
|
|
56
|
+
'Resolution must be one of ("10m", "50m", "110m")'
|
|
57
|
+
)
|
|
58
|
+
with pytest.raises(ValueError, match=re.escape(expected_message)):
|
|
56
59
|
validate_res("kaboom")
|
ne_loader-0.3.2/README.md
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
# NE Loader
|
|
2
|
-
|
|
3
|
-
A simple, robust Python package to download and load Natural Earth map data using GeoPandas.
|
|
4
|
-
|
|
5
|
-
## Features
|
|
6
|
-
|
|
7
|
-
- Download and cache Natural Earth datasets
|
|
8
|
-
- Load shapefiles as GeoDataFrames
|
|
9
|
-
|
|
10
|
-
## Installation
|
|
11
|
-
|
|
12
|
-
```bash
|
|
13
|
-
pip install ne-loader
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
## Usage
|
|
17
|
-
|
|
18
|
-
```python
|
|
19
|
-
from ne_loader import map_loader
|
|
20
|
-
world = map_loader.get_natural_earth('cultural', 'admin_0_countries')
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
Errors are raised by default. To return the caught exception instead:
|
|
24
|
-
|
|
25
|
-
```python
|
|
26
|
-
result = map_loader.get_natural_earth(
|
|
27
|
-
'cultural',
|
|
28
|
-
'admin_0_countries',
|
|
29
|
-
error_mode='return',
|
|
30
|
-
)
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## CLI
|
|
34
|
-
|
|
35
|
-
```bash
|
|
36
|
-
ne-loader --help
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
To set an environment override for the NE data save path:
|
|
40
|
-
|
|
41
|
-
```bash
|
|
42
|
-
export NATURAL_EARTH_CACHE_DIR="..."
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
## Tests
|
|
46
|
-
|
|
47
|
-
```bash
|
|
48
|
-
python3 -m pytest tests
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
## License
|
|
52
|
-
|
|
53
|
-
MIT
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
"""Provides basic command line functionality."""
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
|
|
5
|
-
import geopandas as gpd
|
|
6
|
-
|
|
7
|
-
from . import map_loader
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def main() -> None:
|
|
11
|
-
"""Run the Natural Earth loader command-line interface."""
|
|
12
|
-
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
|
13
|
-
description="Download and load Natural Earth data."
|
|
14
|
-
)
|
|
15
|
-
parser.add_argument(
|
|
16
|
-
"category",
|
|
17
|
-
choices=["cultural", "physical"],
|
|
18
|
-
help="Data category",
|
|
19
|
-
)
|
|
20
|
-
parser.add_argument("name", help="Dataset name (e.g., admin_0_countries)")
|
|
21
|
-
parser.add_argument("--res", default="10m", help="Resolution (default: 10m)")
|
|
22
|
-
parser.add_argument("--out", help="Output file to save as GeoJSON (optional)")
|
|
23
|
-
args: argparse.Namespace = parser.parse_args()
|
|
24
|
-
|
|
25
|
-
gdf: gpd.GeoDataFrame = map_loader.get_natural_earth(
|
|
26
|
-
category=args.category,
|
|
27
|
-
name=args.name,
|
|
28
|
-
res=args.res,
|
|
29
|
-
)
|
|
30
|
-
if args.out:
|
|
31
|
-
gdf.to_file(args.out, driver="GeoJSON")
|
|
32
|
-
print(f"Saved to {args.out}")
|
|
33
|
-
else:
|
|
34
|
-
print(gdf.head())
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
if __name__ == "__main__":
|
|
38
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|