overturemaps 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {overturemaps-2.0.0 → overturemaps-2.1.0}/PKG-INFO +1 -1
- {overturemaps-2.0.0 → overturemaps-2.1.0}/SPEC.md +8 -6
- {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/python-api.md +16 -0
- overturemaps-2.1.0/overturemaps/_http.py +86 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/core.py +155 -36
- {overturemaps-2.0.0 → overturemaps-2.1.0}/pyproject.toml +1 -1
- overturemaps-2.1.0/tests/conftest.py +53 -0
- overturemaps-2.1.0/tests/fake_stac.py +34 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_changelog.py +21 -1
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_download.py +29 -0
- overturemaps-2.1.0/tests/test_core.py +479 -0
- overturemaps-2.1.0/tests/test_http.py +168 -0
- overturemaps-2.0.0/overturemaps/_http.py +0 -23
- overturemaps-2.0.0/tests/test_core.py +0 -163
- overturemaps-2.0.0/tests/test_http.py +0 -56
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/dependabot.yml +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/build-binaries.yml +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/publish-pypi.yml +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/release.yml +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/test-run.yml +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/.gitignore +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/CODEOWNERS +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/CONTRIBUTING.md +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/LICENSE +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/README.md +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/benchmarks/bench_count_rows.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/benchmarks/bench_geojson_writer.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/cli.md +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/performance.md +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/screenshot.png +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/justfile +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/__init__.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/__main__.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/changelog.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/cli.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/examples/geopandas_example.ipynb +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/models.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/releases.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/state.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/writers.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/pytest.ini +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/__init__.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_gers.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_releases.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_copy.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_gers.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_main.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_models.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_releases.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_state.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_writers.py +0 -0
- {overturemaps-2.0.0 → overturemaps-2.1.0}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: overturemaps
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Python tools for interacting with Overture Maps (overturemaps.org) data.
|
|
5
5
|
Project-URL: Homepage, https://overturemaps.org
|
|
6
6
|
Project-URL: Source, https://github.com/OvertureMaps/overturemaps-py
|
|
@@ -44,13 +44,14 @@ Models (models.py) ← pure data classes, no I/O
|
|
|
44
44
|
|
|
45
45
|
**Default path (STAC-accelerated)**:
|
|
46
46
|
1. Fetch `https://stac.overturemaps.org/catalog.json` (in-process cached).
|
|
47
|
-
2. Download `/{release}/collections.parquet` (
|
|
47
|
+
2. Download `/{release}/collections.parquet` (in-process cached per release, spatially filtered).
|
|
48
48
|
3. Extract the list of Parquet file paths that intersect the query bbox.
|
|
49
49
|
4. Open those files as a `pyarrow.dataset.Dataset` with row-level bbox predicate pushdown.
|
|
50
50
|
5. Stream `RecordBatch` objects through writer.
|
|
51
51
|
|
|
52
52
|
**Fallback path (no STAC)**:
|
|
53
53
|
- `--no-stac` flag skips step 2–3; opens the full S3 partition directly.
|
|
54
|
+
- If the index can't be read (after retrying HTTP 429/5xx) or lists no files for the type, warns with `StacIndexWarning` and opens the full S3 partition.
|
|
54
55
|
|
|
55
56
|
**GERS path**:
|
|
56
57
|
1. Fetch STAC catalog → registry manifest (`[filename, max_id]` sorted list).
|
|
@@ -140,14 +141,14 @@ def record_batch_reader(
|
|
|
140
141
|
release: str | None = None,
|
|
141
142
|
connect_timeout: int | None = None,
|
|
142
143
|
request_timeout: int | None = None,
|
|
143
|
-
stac: bool =
|
|
144
|
+
stac: bool = True,
|
|
144
145
|
) -> pyarrow.RecordBatchReader | None
|
|
145
146
|
```
|
|
146
147
|
|
|
147
|
-
- Returns a streaming `RecordBatchReader`;
|
|
148
|
+
- Returns a streaming `RecordBatchReader`; it is empty if no data matches and `None` on read errors.
|
|
148
149
|
- `bbox=None` queries globally (large data warning applies at CLI layer, not here).
|
|
149
150
|
- `release=None` resolves to latest release.
|
|
150
|
-
- When `stac=True
|
|
151
|
+
- When `stac=True` (default), uses STAC spatial index to minimize file scanning.
|
|
151
152
|
|
|
152
153
|
### 4.2 `get_all_overture_types`
|
|
153
154
|
|
|
@@ -389,7 +390,8 @@ State file path = `{output_path}.state` (e.g. `boston.geojson.state`).
|
|
|
389
390
|
### 10.2 Reliability
|
|
390
391
|
|
|
391
392
|
- **Anonymous S3**: No credentials to rotate, expire, or leak.
|
|
392
|
-
- **STAC cache**: In-process only; no disk cache. TTL = process lifetime.
|
|
393
|
+
- **STAC cache**: Catalog and per-release file index. In-process only; no disk cache. TTL = process lifetime.
|
|
394
|
+
- **STAC index retries**: Downloads of a release's file index retry HTTP 429 and 5xx 3 times with exponential backoff, honouring `Retry-After` (at most 30 s per wait). Other STAC requests aren't retried.
|
|
393
395
|
- **Changelog fallback**: STAC stub failure is silent; direct S3 scan always attempted.
|
|
394
396
|
- **Empty batch filtering**: Zero-row batches never passed to writers.
|
|
395
397
|
- **State load failures**: Always `None`, never exception. Stale state = staleness check reports "update available", not a crash.
|
|
@@ -538,7 +540,7 @@ State file path = `{output_path}.state` (e.g. `boston.geojson.state`).
|
|
|
538
540
|
|
|
539
541
|
- **Validate at CLI boundary**: bbox, release string, GERS UUID, format choice. Library functions trust their callers.
|
|
540
542
|
- **Fail fast on invalid type**: `type` not in `type_theme_map` should raise `KeyError` immediately, not silently produce an empty result.
|
|
541
|
-
- **No silent swallowing** except: `load_state` (missing/corrupt state is non-fatal by design), `release_exists` (network errors = False), STAC fallback (fallback is the contract).
|
|
543
|
+
- **No silent swallowing** except: `load_state` (missing/corrupt state is non-fatal by design), `release_exists` (network errors = False), STAC fallback (fallback is the contract; the file-index fallback warns with `StacIndexWarning`).
|
|
542
544
|
- **`RuntimeError` for unrecoverable catalog failures** (e.g. no releases found).
|
|
543
545
|
|
|
544
546
|
### 14.3 Output discipline
|
|
@@ -31,6 +31,22 @@ gdf = geodataframe("building", bbox=bbox)
|
|
|
31
31
|
print(gdf.head())
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
+
## STAC index
|
|
35
|
+
|
|
36
|
+
When you pass a `bbox`, `record_batch_reader`, `geodataframe` and `overturemaps.core.count_rows` use the release's [STAC index](https://stac.overturemaps.org/) to read only the files that intersect it. The index is downloaded once per release and reused by later queries in the same process. If its download is rate limited (HTTP 429) or hits a server error (5xx), it's retried with backoff.
|
|
37
|
+
|
|
38
|
+
If the index still can't be read, or lists no files for the type, the query issues a `StacIndexWarning` and falls back to scanning every file of the type. Results are the same, but the scan can take several minutes. To fail instead:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import warnings
|
|
42
|
+
|
|
43
|
+
from overturemaps.core import StacIndexWarning
|
|
44
|
+
|
|
45
|
+
warnings.simplefilter("error", StacIndexWarning)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Pass `stac=False` to skip the index and scan every file.
|
|
49
|
+
|
|
34
50
|
## Writing formats
|
|
35
51
|
|
|
36
52
|
Use `get_writer` and `copy` from `overturemaps.writers` to write data to GeoJSON, GeoJSONSeq, or GeoParquet without using the CLI.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""HTTP requests that identify the Overture Maps client."""
|
|
2
|
+
|
|
3
|
+
import platform
|
|
4
|
+
import random
|
|
5
|
+
import time
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from email.utils import parsedate_to_datetime
|
|
8
|
+
from importlib.metadata import version
|
|
9
|
+
from typing import Optional
|
|
10
|
+
from urllib.error import HTTPError
|
|
11
|
+
from urllib.request import Request, urlopen
|
|
12
|
+
|
|
13
|
+
DEFAULT_TIMEOUT = 30
|
|
14
|
+
# Retries for callers that opt in: HTTP 429 (rate limited) and 5xx responses are
|
|
15
|
+
# retried this many times, with exponential backoff.
|
|
16
|
+
MAX_RETRIES = 3
|
|
17
|
+
RETRY_BACKOFF = 1.0
|
|
18
|
+
# Upper bound for a single wait, including one requested by a Retry-After header.
|
|
19
|
+
MAX_RETRY_DELAY = 30.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _user_agent() -> str:
|
|
23
|
+
python_version = platform.python_version()
|
|
24
|
+
system = platform.system()
|
|
25
|
+
return f"overturemaps/{version('overturemaps')} Python/{python_version} ({system})"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _is_retryable(error: HTTPError) -> bool:
|
|
29
|
+
return error.code == 429 or 500 <= error.code < 600
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _retry_after(error: HTTPError) -> Optional[float]:
|
|
33
|
+
"""Return the wait in seconds requested by a Retry-After header, if any.
|
|
34
|
+
|
|
35
|
+
The header holds either a number of seconds or an HTTP date.
|
|
36
|
+
"""
|
|
37
|
+
value = error.headers.get("Retry-After") if error.headers is not None else None
|
|
38
|
+
if not value:
|
|
39
|
+
return None
|
|
40
|
+
value = value.strip()
|
|
41
|
+
if value.isdigit():
|
|
42
|
+
return float(value)
|
|
43
|
+
try:
|
|
44
|
+
retry_at = parsedate_to_datetime(value)
|
|
45
|
+
except (TypeError, ValueError):
|
|
46
|
+
return None
|
|
47
|
+
if retry_at.tzinfo is None:
|
|
48
|
+
retry_at = retry_at.replace(tzinfo=timezone.utc)
|
|
49
|
+
return max(0.0, (retry_at - datetime.now(timezone.utc)).total_seconds())
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _retry_delay(error: HTTPError, attempt: int) -> float:
|
|
53
|
+
delay = _retry_after(error)
|
|
54
|
+
if delay is None:
|
|
55
|
+
delay = RETRY_BACKOFF * 2**attempt + random.uniform(0, RETRY_BACKOFF)
|
|
56
|
+
return min(delay, MAX_RETRY_DELAY)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def open_url(
|
|
60
|
+
url: str,
|
|
61
|
+
timeout: Optional[int] = None,
|
|
62
|
+
accept: Optional[str] = None,
|
|
63
|
+
retries: int = 0,
|
|
64
|
+
):
|
|
65
|
+
"""Open a URL with an application User-Agent instead of urllib's default.
|
|
66
|
+
|
|
67
|
+
HTTP 429 and 5xx responses are retried up to ``retries`` times (none by
|
|
68
|
+
default) with exponential backoff, honouring the server's Retry-After header.
|
|
69
|
+
"""
|
|
70
|
+
headers = {"User-Agent": _user_agent()}
|
|
71
|
+
if accept:
|
|
72
|
+
headers["Accept"] = accept
|
|
73
|
+
request = Request(url, headers=headers)
|
|
74
|
+
attempt = 0
|
|
75
|
+
while True:
|
|
76
|
+
try:
|
|
77
|
+
return urlopen(
|
|
78
|
+
request, timeout=timeout if timeout is not None else DEFAULT_TIMEOUT
|
|
79
|
+
)
|
|
80
|
+
except HTTPError as error:
|
|
81
|
+
if attempt >= retries or not _is_retryable(error):
|
|
82
|
+
raise
|
|
83
|
+
delay = _retry_delay(error, attempt)
|
|
84
|
+
error.close()
|
|
85
|
+
time.sleep(delay)
|
|
86
|
+
attempt += 1
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import json
|
|
3
3
|
import sys
|
|
4
|
+
import threading
|
|
5
|
+
import warnings
|
|
6
|
+
from concurrent.futures import Future
|
|
4
7
|
from typing import List, Optional, Tuple
|
|
5
8
|
from urllib.parse import urlparse
|
|
6
9
|
|
|
@@ -10,14 +13,28 @@ import pyarrow.dataset as ds
|
|
|
10
13
|
import pyarrow.fs as fs
|
|
11
14
|
import pyarrow.parquet as pq
|
|
12
15
|
|
|
13
|
-
from ._http import open_url
|
|
16
|
+
from ._http import MAX_RETRIES, open_url
|
|
14
17
|
from .models import BBox
|
|
15
18
|
|
|
16
19
|
STAC_CATALOG_URL = "https://stac.overturemaps.org/catalog.json"
|
|
20
|
+
STAC_INDEX_URL_TEMPLATE = "https://stac.overturemaps.org/{release}/collections.parquet"
|
|
17
21
|
|
|
18
22
|
# Cache for STAC catalog to avoid repeated network calls
|
|
19
23
|
_cached_stac_catalog = None
|
|
20
24
|
|
|
25
|
+
# Cache for each release's STAC file index. A published release's index doesn't
|
|
26
|
+
# change, so one download per release serves every bbox query in the process.
|
|
27
|
+
_cached_stac_indexes: dict[str, pa.Table] = {}
|
|
28
|
+
# Index downloads in progress, so concurrent queries for a release share one.
|
|
29
|
+
_stac_index_downloads: dict[str, Future] = {}
|
|
30
|
+
# Guards both dicts. Never held during a download.
|
|
31
|
+
_stac_index_lock = threading.Lock()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class StacIndexWarning(UserWarning):
|
|
35
|
+
"""The STAC file index couldn't be used, so a query is scanning every file of its type."""
|
|
36
|
+
|
|
37
|
+
|
|
21
38
|
# Allows for optional import of additional dependencies
|
|
22
39
|
try:
|
|
23
40
|
from geopandas import GeoDataFrame
|
|
@@ -142,23 +159,70 @@ def _coerce_bbox(
|
|
|
142
159
|
)
|
|
143
160
|
|
|
144
161
|
|
|
162
|
+
def _get_stac_index(release: str) -> pa.Table:
|
|
163
|
+
"""
|
|
164
|
+
Fetch and cache the STAC-geoparquet index of a release's data files.
|
|
165
|
+
|
|
166
|
+
Concurrent calls for a release share one download, and all get its exception
|
|
167
|
+
if it fails. Failed downloads aren't cached, so a later call tries again.
|
|
168
|
+
"""
|
|
169
|
+
stac_table = _cached_stac_indexes.get(release)
|
|
170
|
+
if stac_table is not None:
|
|
171
|
+
return stac_table
|
|
172
|
+
|
|
173
|
+
with _stac_index_lock:
|
|
174
|
+
stac_table = _cached_stac_indexes.get(release)
|
|
175
|
+
if stac_table is not None:
|
|
176
|
+
return stac_table
|
|
177
|
+
download = _stac_index_downloads.get(release)
|
|
178
|
+
is_downloader = download is None
|
|
179
|
+
if is_downloader:
|
|
180
|
+
download = _stac_index_downloads[release] = Future()
|
|
181
|
+
|
|
182
|
+
if not is_downloader:
|
|
183
|
+
return download.result()
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
stac_table = _download_stac_index(release)
|
|
187
|
+
except BaseException as e:
|
|
188
|
+
# Hand over every exception, KeyboardInterrupt included, so waiters
|
|
189
|
+
# never hang. Later calls start a new download.
|
|
190
|
+
with _stac_index_lock:
|
|
191
|
+
_stac_index_downloads.pop(release, None)
|
|
192
|
+
download.set_exception(e)
|
|
193
|
+
raise
|
|
194
|
+
with _stac_index_lock:
|
|
195
|
+
_cached_stac_indexes[release] = stac_table
|
|
196
|
+
_stac_index_downloads.pop(release, None)
|
|
197
|
+
download.set_result(stac_table)
|
|
198
|
+
return stac_table
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _download_stac_index(release: str) -> pa.Table:
|
|
202
|
+
stac_url = STAC_INDEX_URL_TEMPLATE.format(release=release)
|
|
203
|
+
# Arrow can't read HTTP URLs directly; read into memory first
|
|
204
|
+
with open_url(
|
|
205
|
+
stac_url, accept="application/octet-stream", retries=MAX_RETRIES
|
|
206
|
+
) as response:
|
|
207
|
+
data = response.read()
|
|
208
|
+
return pq.read_table(io.BytesIO(data))
|
|
209
|
+
|
|
210
|
+
|
|
145
211
|
def _get_files_from_stac(
|
|
146
212
|
theme: str, overture_type: str, bbox: BBox, release: str
|
|
147
213
|
) -> Optional[List[str]]:
|
|
148
214
|
"""
|
|
149
|
-
Returns a list of bucket/key paths using the STAC-geoparquet index
|
|
215
|
+
Returns a list of bucket/key paths using the STAC-geoparquet index, or None
|
|
216
|
+
after a StacIndexWarning if the index can't be read or lists no files of the type
|
|
150
217
|
"""
|
|
151
|
-
stac_url = f"https://stac.overturemaps.org/{release}/collections.parquet"
|
|
152
218
|
try:
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
pc.field("type") == "Feature"
|
|
161
|
-
)
|
|
219
|
+
stac_table = _get_stac_index(release)
|
|
220
|
+
|
|
221
|
+
type_items = stac_table.filter(_stac_type_filter(overture_type))
|
|
222
|
+
if type_items.num_rows == 0:
|
|
223
|
+
# Handled like an unreadable index, so the query scans the type's
|
|
224
|
+
# folder instead of reporting no data.
|
|
225
|
+
raise ValueError(f"the index lists no {overture_type} files")
|
|
162
226
|
|
|
163
227
|
xmin, ymin, xmax, ymax = bbox.as_tuple()
|
|
164
228
|
bbox_filter = (
|
|
@@ -168,29 +232,78 @@ def _get_files_from_stac(
|
|
|
168
232
|
& (pc.field("bbox", "ymax") > ymin)
|
|
169
233
|
)
|
|
170
234
|
|
|
171
|
-
|
|
172
|
-
table = stac_table.filter(combined_filter)
|
|
235
|
+
table = type_items.filter(bbox_filter)
|
|
173
236
|
|
|
174
237
|
if table.num_rows > 0:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
# clip out the "s3://" prefix
|
|
178
|
-
s3_paths = [
|
|
179
|
-
path["aws"]["alternate"]["s3"]["href"][len("s3://") :]
|
|
180
|
-
for path in file_paths
|
|
181
|
-
]
|
|
182
|
-
return s3_paths
|
|
238
|
+
return _s3_paths(table)
|
|
183
239
|
else:
|
|
184
240
|
print(
|
|
185
|
-
f"No data found for release {release} in query bbox {bbox.as_tuple()}."
|
|
241
|
+
f"No data found for release {release} in query bbox {bbox.as_tuple()}.",
|
|
242
|
+
file=sys.stderr,
|
|
186
243
|
)
|
|
187
244
|
return []
|
|
188
245
|
|
|
189
246
|
except Exception as e:
|
|
190
|
-
|
|
247
|
+
warnings.warn(
|
|
248
|
+
f"Could not use the STAC index for release {release} ({e}). Falling back "
|
|
249
|
+
f"to a full scan of s3://{_dataset_path(overture_type, release)}, which "
|
|
250
|
+
"reads metadata from every file and can take several minutes. "
|
|
251
|
+
"Results are not affected.",
|
|
252
|
+
StacIndexWarning,
|
|
253
|
+
stacklevel=_caller_stacklevel(),
|
|
254
|
+
)
|
|
191
255
|
return None
|
|
192
256
|
|
|
193
257
|
|
|
258
|
+
def _stac_type_filter(overture_type: str) -> pc.Expression:
|
|
259
|
+
"""Match the STAC index rows for the type's data files."""
|
|
260
|
+
return (pc.field("collection") == overture_type) & (pc.field("type") == "Feature")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _s3_paths(stac_items: pa.Table) -> List[str]:
|
|
264
|
+
"""Return the bucket/key paths of the data files in STAC index rows."""
|
|
265
|
+
# clip out the "s3://" prefix
|
|
266
|
+
return [
|
|
267
|
+
assets["aws"]["alternate"]["s3"]["href"][len("s3://") :]
|
|
268
|
+
for assets in stac_items.column("assets").to_pylist()
|
|
269
|
+
]
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _type_schema(
|
|
273
|
+
overture_type: str, release: str, connect_timeout=None, request_timeout=None
|
|
274
|
+
) -> pa.Schema:
|
|
275
|
+
"""
|
|
276
|
+
Read the type's schema from the first of its data files in the STAC index,
|
|
277
|
+
which is quicker than listing the type's folder. Only called after
|
|
278
|
+
_get_files_from_stac has found files of the type, so the index is cached.
|
|
279
|
+
"""
|
|
280
|
+
stac_items = _get_stac_index(release).filter(_stac_type_filter(overture_type))
|
|
281
|
+
dataset = ds.dataset(
|
|
282
|
+
_s3_paths(stac_items.slice(0, 1)),
|
|
283
|
+
filesystem=_s3_filesystem(connect_timeout, request_timeout),
|
|
284
|
+
)
|
|
285
|
+
return dataset.schema
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _caller_stacklevel() -> int:
|
|
289
|
+
"""Return the warnings stacklevel of the first caller outside this module."""
|
|
290
|
+
frame = sys._getframe(1)
|
|
291
|
+
level = 1
|
|
292
|
+
while frame.f_back is not None and frame.f_globals.get("__name__") == __name__:
|
|
293
|
+
frame = frame.f_back
|
|
294
|
+
level += 1
|
|
295
|
+
return level
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _s3_filesystem(connect_timeout=None, request_timeout=None) -> fs.S3FileSystem:
|
|
299
|
+
return fs.S3FileSystem(
|
|
300
|
+
anonymous=True,
|
|
301
|
+
region="us-west-2",
|
|
302
|
+
connect_timeout=connect_timeout,
|
|
303
|
+
request_timeout=request_timeout,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
194
307
|
def _record_batch_reader_from_dataset(
|
|
195
308
|
dataset: ds.Dataset,
|
|
196
309
|
filter_expr=None,
|
|
@@ -234,13 +347,15 @@ def _prepare_query(
|
|
|
234
347
|
release=None,
|
|
235
348
|
connect_timeout=None,
|
|
236
349
|
request_timeout=None,
|
|
237
|
-
stac=
|
|
350
|
+
stac=True,
|
|
238
351
|
) -> Optional[Tuple[ds.Dataset, Optional[pc.Expression]]]:
|
|
239
352
|
"""
|
|
240
353
|
Resolve the S3 dataset and filter expression for a given query.
|
|
241
354
|
|
|
242
355
|
Returns the dataset and filter expression ready for counting or streaming,
|
|
243
|
-
or None if STAC reports no files intersect the bbox.
|
|
356
|
+
or None if STAC reports no files intersect the bbox. If the STAC index
|
|
357
|
+
can't be read or lists no files of the type, warns with StacIndexWarning and
|
|
358
|
+
scans every file of the type.
|
|
244
359
|
"""
|
|
245
360
|
if release is None:
|
|
246
361
|
release = get_latest_release()
|
|
@@ -268,12 +383,7 @@ def _prepare_query(
|
|
|
268
383
|
|
|
269
384
|
dataset = ds.dataset(
|
|
270
385
|
intersecting_files if intersecting_files is not None else path,
|
|
271
|
-
filesystem=
|
|
272
|
-
anonymous=True,
|
|
273
|
-
region="us-west-2",
|
|
274
|
-
connect_timeout=connect_timeout,
|
|
275
|
-
request_timeout=request_timeout,
|
|
276
|
-
),
|
|
386
|
+
filesystem=_s3_filesystem(connect_timeout, request_timeout),
|
|
277
387
|
)
|
|
278
388
|
|
|
279
389
|
return dataset, filter_expr
|
|
@@ -285,7 +395,7 @@ def count_rows(
|
|
|
285
395
|
release=None,
|
|
286
396
|
connect_timeout=None,
|
|
287
397
|
request_timeout=None,
|
|
288
|
-
stac=
|
|
398
|
+
stac=True,
|
|
289
399
|
) -> int:
|
|
290
400
|
"""Return the number of rows matching the given parameters."""
|
|
291
401
|
result = _prepare_query(
|
|
@@ -304,14 +414,19 @@ def record_batch_reader(
|
|
|
304
414
|
release=None,
|
|
305
415
|
connect_timeout=None,
|
|
306
416
|
request_timeout=None,
|
|
307
|
-
stac=
|
|
417
|
+
stac=True,
|
|
308
418
|
) -> Optional[pa.RecordBatchReader]:
|
|
309
419
|
"""Return a pyarrow RecordBatchReader for the desired bounding box and s3 path, or None on error."""
|
|
420
|
+
if release is None:
|
|
421
|
+
release = get_latest_release()
|
|
310
422
|
result = _prepare_query(
|
|
311
423
|
overture_type, bbox, release, connect_timeout, request_timeout, stac
|
|
312
424
|
)
|
|
313
425
|
if result is None:
|
|
314
|
-
|
|
426
|
+
# No files intersect the bbox. Return an empty reader with the type's
|
|
427
|
+
# schema, the same result a full scan gives.
|
|
428
|
+
schema = _type_schema(overture_type, release, connect_timeout, request_timeout)
|
|
429
|
+
return pa.RecordBatchReader.from_batches(geoarrow_schema_adapter(schema), [])
|
|
315
430
|
dataset, filter_expr = result
|
|
316
431
|
return _record_batch_reader_from_dataset(dataset, filter_expr=filter_expr)
|
|
317
432
|
|
|
@@ -322,7 +437,7 @@ def geodataframe(
|
|
|
322
437
|
release: str = None,
|
|
323
438
|
connect_timeout: int = None,
|
|
324
439
|
request_timeout: int = None,
|
|
325
|
-
stac: bool =
|
|
440
|
+
stac: bool = True,
|
|
326
441
|
) -> GeoDataFrame:
|
|
327
442
|
"""
|
|
328
443
|
Loads geoparquet for specified type into a geopandas dataframe
|
|
@@ -331,8 +446,12 @@ def geodataframe(
|
|
|
331
446
|
----------
|
|
332
447
|
overture_type: type to load
|
|
333
448
|
bbox: optional bounding box for data fetch (xmin, ymin, xmax, ymax)
|
|
449
|
+
release: optional release version, defaults to the latest
|
|
334
450
|
connect_timeout: optional connection timeout in seconds
|
|
335
451
|
request_timeout: optional request timeout in seconds
|
|
452
|
+
stac: use the STAC index to read only the files that intersect the bbox
|
|
453
|
+
(default True). If the index can't be read or lists no files of the
|
|
454
|
+
type, a StacIndexWarning is issued and every file of the type is scanned.
|
|
336
455
|
|
|
337
456
|
Returns
|
|
338
457
|
-------
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "overturemaps"
|
|
3
3
|
# Release workflows stamp the tag's version into this placeholder before building.
|
|
4
|
-
version = "2.
|
|
4
|
+
version = "2.1.0"
|
|
5
5
|
description = "Python tools for interacting with Overture Maps (overturemaps.org) data."
|
|
6
6
|
authors = [
|
|
7
7
|
{name = "Jacob Wasserman", email = "jwasserman@meta.com"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Shared fixtures for STAC index tests that make no network requests."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import threading
|
|
5
|
+
from concurrent.futures import Future
|
|
6
|
+
|
|
7
|
+
import pyarrow as pa
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from overturemaps import core
|
|
11
|
+
|
|
12
|
+
from .fake_stac import stac_index_bytes
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture
|
|
16
|
+
def empty_stac_cache(monkeypatch):
|
|
17
|
+
"""Start with no cached or in-progress STAC index downloads."""
|
|
18
|
+
monkeypatch.setattr(core, "_cached_stac_indexes", {})
|
|
19
|
+
monkeypatch.setattr(core, "_stac_index_downloads", {})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.fixture
|
|
23
|
+
def stac_index(mocker, empty_stac_cache):
|
|
24
|
+
"""Serve the fake STAC index through a mocked open_url, starting from an empty cache."""
|
|
25
|
+
data = stac_index_bytes()
|
|
26
|
+
return mocker.patch(
|
|
27
|
+
"overturemaps.core.open_url", side_effect=lambda *args, **kwargs: io.BytesIO(data)
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def index_waiters(mocker):
|
|
33
|
+
"""A semaphore released each time a caller waits on another caller's index download."""
|
|
34
|
+
waiting = threading.Semaphore(0)
|
|
35
|
+
|
|
36
|
+
class WatchedDownload(Future):
|
|
37
|
+
def result(self, timeout=None):
|
|
38
|
+
waiting.release()
|
|
39
|
+
return super().result(timeout)
|
|
40
|
+
|
|
41
|
+
mocker.patch("overturemaps.core.Future", WatchedDownload)
|
|
42
|
+
return waiting
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.fixture
|
|
46
|
+
def s3_dataset(mocker):
|
|
47
|
+
"""Replace pyarrow dataset discovery so no S3 request is made."""
|
|
48
|
+
dataset = mocker.patch("overturemaps.core.ds.dataset")
|
|
49
|
+
dataset.return_value.count_rows.return_value = 42
|
|
50
|
+
dataset.return_value.schema = pa.schema(
|
|
51
|
+
[pa.field("id", pa.string()), pa.field("geometry", pa.binary())]
|
|
52
|
+
)
|
|
53
|
+
return dataset
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""A tiny fake STAC index for tests that make no network requests."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
|
|
5
|
+
import pyarrow as pa
|
|
6
|
+
import pyarrow.parquet as pq
|
|
7
|
+
|
|
8
|
+
RELEASE = "2026-10-21.0"
|
|
9
|
+
SF_BBOX = (-122.42, 37.77, -122.40, 37.785)
|
|
10
|
+
OCEAN_BBOX = (-40.0, 0.0, -39.9, 0.1)
|
|
11
|
+
WEST_FILE = f"overturemaps-us-west-2/release/{RELEASE}/theme=buildings/type=building/west.parquet"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def stac_index_bytes() -> bytes:
|
|
15
|
+
"""A tiny STAC index: two building files in North America and one global place file."""
|
|
16
|
+
|
|
17
|
+
def item(collection, path, bbox):
|
|
18
|
+
return {
|
|
19
|
+
"type": "Feature",
|
|
20
|
+
"collection": collection,
|
|
21
|
+
"bbox": dict(zip(("xmin", "ymin", "xmax", "ymax"), bbox)),
|
|
22
|
+
"assets": {"aws": {"alternate": {"s3": {"href": f"s3://{path}"}}}},
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
table = pa.Table.from_pylist(
|
|
26
|
+
[
|
|
27
|
+
item("building", WEST_FILE, (-125.0, 30.0, -110.0, 45.0)),
|
|
28
|
+
item("building", WEST_FILE.replace("west", "east"), (-80.0, 35.0, -70.0, 45.0)),
|
|
29
|
+
item("place", WEST_FILE.replace("building", "place"), (-180.0, -90.0, 180.0, 90.0)),
|
|
30
|
+
]
|
|
31
|
+
)
|
|
32
|
+
sink = io.BytesIO()
|
|
33
|
+
pq.write_table(table, sink)
|
|
34
|
+
return sink.getvalue()
|
|
@@ -1,11 +1,31 @@
|
|
|
1
1
|
"""Tests for the changelog module."""
|
|
2
2
|
|
|
3
|
+
from urllib.error import HTTPError
|
|
4
|
+
|
|
3
5
|
import pytest
|
|
4
|
-
from overturemaps.changelog import
|
|
6
|
+
from overturemaps.changelog import (
|
|
7
|
+
_get_changelog_files_from_stac,
|
|
8
|
+
query_changelog_ids,
|
|
9
|
+
summarize_changelog,
|
|
10
|
+
)
|
|
5
11
|
from overturemaps.models import BBox
|
|
6
12
|
from overturemaps.releases import get_latest_release
|
|
7
13
|
|
|
8
14
|
|
|
15
|
+
def test_changelog_index_download_is_not_retried(mocker):
|
|
16
|
+
sleep = mocker.patch("overturemaps._http.time.sleep")
|
|
17
|
+
urlopen = mocker.patch(
|
|
18
|
+
"overturemaps._http.urlopen",
|
|
19
|
+
side_effect=HTTPError("url", 429, "Too Many Requests", {}, None),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
bbox = BBox(xmin=-71.068, ymin=42.353, xmax=-71.058, ymax=42.363)
|
|
23
|
+
# None means the query falls back to scanning the changelog folder.
|
|
24
|
+
assert _get_changelog_files_from_stac("buildings", "building", bbox, "2026-10-21.0") is None
|
|
25
|
+
assert urlopen.call_count == 1
|
|
26
|
+
sleep.assert_not_called()
|
|
27
|
+
|
|
28
|
+
|
|
9
29
|
@pytest.mark.integration
|
|
10
30
|
def test_query_changelog_ids():
|
|
11
31
|
"""Test querying changelog IDs for a small bbox."""
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Tests for download CLI command."""
|
|
2
2
|
|
|
3
|
+
import json
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import pytest
|
|
@@ -11,6 +12,8 @@ from overturemaps.cli import (
|
|
|
11
12
|
cli,
|
|
12
13
|
)
|
|
13
14
|
|
|
15
|
+
from .fake_stac import OCEAN_BBOX, RELEASE
|
|
16
|
+
|
|
14
17
|
|
|
15
18
|
class _DummyWriter:
|
|
16
19
|
def __enter__(self):
|
|
@@ -255,3 +258,29 @@ def test_download_invalid_release_explains_retention_policy(monkeypatch):
|
|
|
255
258
|
assert "60 days" in result.output
|
|
256
259
|
assert "2026-03-18.0" in result.output
|
|
257
260
|
assert "docs.overturemaps.org/release-calendar" in result.output
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def test_download_empty_area_keeps_stdout_valid_geojson(monkeypatch, stac_index, s3_dataset):
|
|
264
|
+
"""When no files intersect the bbox, stdout holds only an empty FeatureCollection."""
|
|
265
|
+
monkeypatch.setattr(
|
|
266
|
+
"overturemaps.cli.get_available_releases", lambda: ([RELEASE], RELEASE)
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
result = CliRunner().invoke(
|
|
270
|
+
cli,
|
|
271
|
+
[
|
|
272
|
+
"download",
|
|
273
|
+
"-f",
|
|
274
|
+
"geojson",
|
|
275
|
+
"-t",
|
|
276
|
+
"building",
|
|
277
|
+
"-r",
|
|
278
|
+
RELEASE,
|
|
279
|
+
"--bbox",
|
|
280
|
+
",".join(str(v) for v in OCEAN_BBOX),
|
|
281
|
+
],
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
assert result.exit_code == 0, result.output
|
|
285
|
+
assert json.loads(result.stdout) == {"type": "FeatureCollection", "features": []}
|
|
286
|
+
assert "No data found" in result.stderr
|