overturemaps 2.0.0__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {overturemaps-2.0.0 → overturemaps-2.1.0}/PKG-INFO +1 -1
  2. {overturemaps-2.0.0 → overturemaps-2.1.0}/SPEC.md +8 -6
  3. {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/python-api.md +16 -0
  4. overturemaps-2.1.0/overturemaps/_http.py +86 -0
  5. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/core.py +155 -36
  6. {overturemaps-2.0.0 → overturemaps-2.1.0}/pyproject.toml +1 -1
  7. overturemaps-2.1.0/tests/conftest.py +53 -0
  8. overturemaps-2.1.0/tests/fake_stac.py +34 -0
  9. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_changelog.py +21 -1
  10. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_download.py +29 -0
  11. overturemaps-2.1.0/tests/test_core.py +479 -0
  12. overturemaps-2.1.0/tests/test_http.py +168 -0
  13. overturemaps-2.0.0/overturemaps/_http.py +0 -23
  14. overturemaps-2.0.0/tests/test_core.py +0 -163
  15. overturemaps-2.0.0/tests/test_http.py +0 -56
  16. {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/dependabot.yml +0 -0
  17. {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/build-binaries.yml +0 -0
  18. {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/publish-pypi.yml +0 -0
  19. {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/release.yml +0 -0
  20. {overturemaps-2.0.0 → overturemaps-2.1.0}/.github/workflows/test-run.yml +0 -0
  21. {overturemaps-2.0.0 → overturemaps-2.1.0}/.gitignore +0 -0
  22. {overturemaps-2.0.0 → overturemaps-2.1.0}/CODEOWNERS +0 -0
  23. {overturemaps-2.0.0 → overturemaps-2.1.0}/CONTRIBUTING.md +0 -0
  24. {overturemaps-2.0.0 → overturemaps-2.1.0}/LICENSE +0 -0
  25. {overturemaps-2.0.0 → overturemaps-2.1.0}/README.md +0 -0
  26. {overturemaps-2.0.0 → overturemaps-2.1.0}/benchmarks/bench_count_rows.py +0 -0
  27. {overturemaps-2.0.0 → overturemaps-2.1.0}/benchmarks/bench_geojson_writer.py +0 -0
  28. {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/cli.md +0 -0
  29. {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/performance.md +0 -0
  30. {overturemaps-2.0.0 → overturemaps-2.1.0}/docs/screenshot.png +0 -0
  31. {overturemaps-2.0.0 → overturemaps-2.1.0}/justfile +0 -0
  32. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/__init__.py +0 -0
  33. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/__main__.py +0 -0
  34. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/changelog.py +0 -0
  35. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/cli.py +0 -0
  36. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/examples/geopandas_example.ipynb +0 -0
  37. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/models.py +0 -0
  38. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/releases.py +0 -0
  39. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/state.py +0 -0
  40. {overturemaps-2.0.0 → overturemaps-2.1.0}/overturemaps/writers.py +0 -0
  41. {overturemaps-2.0.0 → overturemaps-2.1.0}/pytest.ini +0 -0
  42. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/__init__.py +0 -0
  43. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_gers.py +0 -0
  44. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_cli_releases.py +0 -0
  45. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_copy.py +0 -0
  46. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_gers.py +0 -0
  47. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_main.py +0 -0
  48. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_models.py +0 -0
  49. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_releases.py +0 -0
  50. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_state.py +0 -0
  51. {overturemaps-2.0.0 → overturemaps-2.1.0}/tests/test_writers.py +0 -0
  52. {overturemaps-2.0.0 → overturemaps-2.1.0}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: overturemaps
3
- Version: 2.0.0
3
+ Version: 2.1.0
4
4
  Summary: Python tools for interacting with Overture Maps (overturemaps.org) data.
5
5
  Project-URL: Homepage, https://overturemaps.org
6
6
  Project-URL: Source, https://github.com/OvertureMaps/overturemaps-py
@@ -44,13 +44,14 @@ Models (models.py) ← pure data classes, no I/O
44
44
 
45
45
  **Default path (STAC-accelerated)**:
46
46
  1. Fetch `https://stac.overturemaps.org/catalog.json` (in-process cached).
47
- 2. Download `/{release}/collections.parquet` (column-pruned, spatially filtered).
47
+ 2. Download `/{release}/collections.parquet` (in-process cached per release, spatially filtered).
48
48
  3. Extract the list of Parquet file paths that intersect the query bbox.
49
49
  4. Open those files as a `pyarrow.dataset.Dataset` with row-level bbox predicate pushdown.
50
50
  5. Stream `RecordBatch` objects through writer.
51
51
 
52
52
  **Fallback path (no STAC)**:
53
53
  - `--no-stac` flag skips step 2–3; opens the full S3 partition directly.
54
+ - If the index can't be read (after retrying HTTP 429/5xx) or lists no files for the type, warns with `StacIndexWarning` and opens the full S3 partition.
54
55
 
55
56
  **GERS path**:
56
57
  1. Fetch STAC catalog → registry manifest (`[filename, max_id]` sorted list).
@@ -140,14 +141,14 @@ def record_batch_reader(
140
141
  release: str | None = None,
141
142
  connect_timeout: int | None = None,
142
143
  request_timeout: int | None = None,
143
- stac: bool = False,
144
+ stac: bool = True,
144
145
  ) -> pyarrow.RecordBatchReader | None
145
146
  ```
146
147
 
147
- - Returns a streaming `RecordBatchReader`; returns `None` if no data matches.
148
+ - Returns a streaming `RecordBatchReader`; it is empty if no data matches and `None` on read errors.
148
149
  - `bbox=None` queries globally (large data warning applies at CLI layer, not here).
149
150
  - `release=None` resolves to latest release.
150
- - When `stac=True`, uses STAC spatial index to minimize file scanning.
151
+ - When `stac=True` (default), uses STAC spatial index to minimize file scanning.
151
152
 
152
153
  ### 4.2 `get_all_overture_types`
153
154
 
@@ -389,7 +390,8 @@ State file path = `{output_path}.state` (e.g. `boston.geojson.state`).
389
390
  ### 10.2 Reliability
390
391
 
391
392
  - **Anonymous S3**: No credentials to rotate, expire, or leak.
392
- - **STAC cache**: In-process only; no disk cache. TTL = process lifetime.
393
+ - **STAC cache**: Catalog and per-release file index. In-process only; no disk cache. TTL = process lifetime.
394
+ - **STAC index retries**: Downloads of a release's file index retry HTTP 429 and 5xx 3 times with exponential backoff, honouring `Retry-After` (at most 30 s per wait). Other STAC requests aren't retried.
393
395
  - **Changelog fallback**: STAC stub failure is silent; direct S3 scan always attempted.
394
396
  - **Empty batch filtering**: Zero-row batches never passed to writers.
395
397
  - **State load failures**: Always `None`, never exception. Stale state = staleness check reports "update available", not a crash.
@@ -538,7 +540,7 @@ State file path = `{output_path}.state` (e.g. `boston.geojson.state`).
538
540
 
539
541
  - **Validate at CLI boundary**: bbox, release string, GERS UUID, format choice. Library functions trust their callers.
540
542
  - **Fail fast on invalid type**: `type` not in `type_theme_map` should raise `KeyError` immediately, not silently produce an empty result.
541
- - **No silent swallowing** except: `load_state` (missing/corrupt state is non-fatal by design), `release_exists` (network errors = False), STAC fallback (fallback is the contract).
543
+ - **No silent swallowing** except: `load_state` (missing/corrupt state is non-fatal by design), `release_exists` (network errors = False), STAC fallback (fallback is the contract; the file-index fallback warns with `StacIndexWarning`).
542
544
  - **`RuntimeError` for unrecoverable catalog failures** (e.g. no releases found).
543
545
 
544
546
  ### 14.3 Output discipline
@@ -31,6 +31,22 @@ gdf = geodataframe("building", bbox=bbox)
31
31
  print(gdf.head())
32
32
  ```
33
33
 
34
+ ## STAC index
35
+
36
+ When you pass a `bbox`, `record_batch_reader`, `geodataframe` and `overturemaps.core.count_rows` use the release's [STAC index](https://stac.overturemaps.org/) to read only the files that intersect it. The index is downloaded once per release and reused by later queries in the same process. If its download is rate limited (HTTP 429) or hits a server error (5xx), it's retried with backoff.
37
+
38
+ If the index still can't be read, or lists no files for the type, the query issues a `StacIndexWarning` and falls back to scanning every file of the type. Results are the same, but the scan can take several minutes. To fail instead:
39
+
40
+ ```python
41
+ import warnings
42
+
43
+ from overturemaps.core import StacIndexWarning
44
+
45
+ warnings.simplefilter("error", StacIndexWarning)
46
+ ```
47
+
48
+ Pass `stac=False` to skip the index and scan every file.
49
+
34
50
  ## Writing formats
35
51
 
36
52
  Use `get_writer` and `copy` from `overturemaps.writers` to write data to GeoJSON, GeoJSONSeq, or GeoParquet without using the CLI.
@@ -0,0 +1,86 @@
1
+ """HTTP requests that identify the Overture Maps client."""
2
+
3
+ import platform
4
+ import random
5
+ import time
6
+ from datetime import datetime, timezone
7
+ from email.utils import parsedate_to_datetime
8
+ from importlib.metadata import version
9
+ from typing import Optional
10
+ from urllib.error import HTTPError
11
+ from urllib.request import Request, urlopen
12
+
13
+ DEFAULT_TIMEOUT = 30
14
+ # Retries for callers that opt in: HTTP 429 (rate limited) and 5xx responses are
15
+ # retried this many times, with exponential backoff.
16
+ MAX_RETRIES = 3
17
+ RETRY_BACKOFF = 1.0
18
+ # Upper bound for a single wait, including one requested by a Retry-After header.
19
+ MAX_RETRY_DELAY = 30.0
20
+
21
+
22
+ def _user_agent() -> str:
23
+ python_version = platform.python_version()
24
+ system = platform.system()
25
+ return f"overturemaps/{version('overturemaps')} Python/{python_version} ({system})"
26
+
27
+
28
+ def _is_retryable(error: HTTPError) -> bool:
29
+ return error.code == 429 or 500 <= error.code < 600
30
+
31
+
32
+ def _retry_after(error: HTTPError) -> Optional[float]:
33
+ """Return the wait in seconds requested by a Retry-After header, if any.
34
+
35
+ The header holds either a number of seconds or an HTTP date.
36
+ """
37
+ value = error.headers.get("Retry-After") if error.headers is not None else None
38
+ if not value:
39
+ return None
40
+ value = value.strip()
41
+ if value.isdigit():
42
+ return float(value)
43
+ try:
44
+ retry_at = parsedate_to_datetime(value)
45
+ except (TypeError, ValueError):
46
+ return None
47
+ if retry_at.tzinfo is None:
48
+ retry_at = retry_at.replace(tzinfo=timezone.utc)
49
+ return max(0.0, (retry_at - datetime.now(timezone.utc)).total_seconds())
50
+
51
+
52
+ def _retry_delay(error: HTTPError, attempt: int) -> float:
53
+ delay = _retry_after(error)
54
+ if delay is None:
55
+ delay = RETRY_BACKOFF * 2**attempt + random.uniform(0, RETRY_BACKOFF)
56
+ return min(delay, MAX_RETRY_DELAY)
57
+
58
+
59
+ def open_url(
60
+ url: str,
61
+ timeout: Optional[int] = None,
62
+ accept: Optional[str] = None,
63
+ retries: int = 0,
64
+ ):
65
+ """Open a URL with an application User-Agent instead of urllib's default.
66
+
67
+ HTTP 429 and 5xx responses are retried up to ``retries`` times (none by
68
+ default) with exponential backoff, honouring the server's Retry-After header.
69
+ """
70
+ headers = {"User-Agent": _user_agent()}
71
+ if accept:
72
+ headers["Accept"] = accept
73
+ request = Request(url, headers=headers)
74
+ attempt = 0
75
+ while True:
76
+ try:
77
+ return urlopen(
78
+ request, timeout=timeout if timeout is not None else DEFAULT_TIMEOUT
79
+ )
80
+ except HTTPError as error:
81
+ if attempt >= retries or not _is_retryable(error):
82
+ raise
83
+ delay = _retry_delay(error, attempt)
84
+ error.close()
85
+ time.sleep(delay)
86
+ attempt += 1
@@ -1,6 +1,9 @@
1
1
  import io
2
2
  import json
3
3
  import sys
4
+ import threading
5
+ import warnings
6
+ from concurrent.futures import Future
4
7
  from typing import List, Optional, Tuple
5
8
  from urllib.parse import urlparse
6
9
 
@@ -10,14 +13,28 @@ import pyarrow.dataset as ds
10
13
  import pyarrow.fs as fs
11
14
  import pyarrow.parquet as pq
12
15
 
13
- from ._http import open_url
16
+ from ._http import MAX_RETRIES, open_url
14
17
  from .models import BBox
15
18
 
16
19
  STAC_CATALOG_URL = "https://stac.overturemaps.org/catalog.json"
20
+ STAC_INDEX_URL_TEMPLATE = "https://stac.overturemaps.org/{release}/collections.parquet"
17
21
 
18
22
  # Cache for STAC catalog to avoid repeated network calls
19
23
  _cached_stac_catalog = None
20
24
 
25
+ # Cache for each release's STAC file index. A published release's index doesn't
26
+ # change, so one download per release serves every bbox query in the process.
27
+ _cached_stac_indexes: dict[str, pa.Table] = {}
28
+ # Index downloads in progress, so concurrent queries for a release share one.
29
+ _stac_index_downloads: dict[str, Future] = {}
30
+ # Guards both dicts. Never held during a download.
31
+ _stac_index_lock = threading.Lock()
32
+
33
+
34
+ class StacIndexWarning(UserWarning):
35
+ """The STAC file index couldn't be used, so a query is scanning every file of its type."""
36
+
37
+
21
38
  # Allows for optional import of additional dependencies
22
39
  try:
23
40
  from geopandas import GeoDataFrame
@@ -142,23 +159,70 @@ def _coerce_bbox(
142
159
  )
143
160
 
144
161
 
162
+ def _get_stac_index(release: str) -> pa.Table:
163
+ """
164
+ Fetch and cache the STAC-geoparquet index of a release's data files.
165
+
166
+ Concurrent calls for a release share one download, and all get its exception
167
+ if it fails. Failed downloads aren't cached, so a later call tries again.
168
+ """
169
+ stac_table = _cached_stac_indexes.get(release)
170
+ if stac_table is not None:
171
+ return stac_table
172
+
173
+ with _stac_index_lock:
174
+ stac_table = _cached_stac_indexes.get(release)
175
+ if stac_table is not None:
176
+ return stac_table
177
+ download = _stac_index_downloads.get(release)
178
+ is_downloader = download is None
179
+ if is_downloader:
180
+ download = _stac_index_downloads[release] = Future()
181
+
182
+ if not is_downloader:
183
+ return download.result()
184
+
185
+ try:
186
+ stac_table = _download_stac_index(release)
187
+ except BaseException as e:
188
+ # Hand over every exception, KeyboardInterrupt included, so waiters
189
+ # never hang. Later calls start a new download.
190
+ with _stac_index_lock:
191
+ _stac_index_downloads.pop(release, None)
192
+ download.set_exception(e)
193
+ raise
194
+ with _stac_index_lock:
195
+ _cached_stac_indexes[release] = stac_table
196
+ _stac_index_downloads.pop(release, None)
197
+ download.set_result(stac_table)
198
+ return stac_table
199
+
200
+
201
+ def _download_stac_index(release: str) -> pa.Table:
202
+ stac_url = STAC_INDEX_URL_TEMPLATE.format(release=release)
203
+ # Arrow can't read HTTP URLs directly; read into memory first
204
+ with open_url(
205
+ stac_url, accept="application/octet-stream", retries=MAX_RETRIES
206
+ ) as response:
207
+ data = response.read()
208
+ return pq.read_table(io.BytesIO(data))
209
+
210
+
145
211
  def _get_files_from_stac(
146
212
  theme: str, overture_type: str, bbox: BBox, release: str
147
213
  ) -> Optional[List[str]]:
148
214
  """
149
- Returns a list of bucket/key paths using the STAC-geoparquet index
215
+ Returns a list of bucket/key paths using the STAC-geoparquet index, or None
216
+ after a StacIndexWarning if the index can't be read or lists no files of the type
150
217
  """
151
- stac_url = f"https://stac.overturemaps.org/{release}/collections.parquet"
152
218
  try:
153
- # Arrow can't read HTTP URLs directly; read into memory first
154
- with open_url(stac_url, accept="application/octet-stream") as response:
155
- data = response.read()
156
- buffer = io.BytesIO(data)
157
- stac_table = pq.read_table(buffer)
158
-
159
- feature_type_filter = (pc.field("collection") == overture_type) & (
160
- pc.field("type") == "Feature"
161
- )
219
+ stac_table = _get_stac_index(release)
220
+
221
+ type_items = stac_table.filter(_stac_type_filter(overture_type))
222
+ if type_items.num_rows == 0:
223
+ # Handled like an unreadable index, so the query scans the type's
224
+ # folder instead of reporting no data.
225
+ raise ValueError(f"the index lists no {overture_type} files")
162
226
 
163
227
  xmin, ymin, xmax, ymax = bbox.as_tuple()
164
228
  bbox_filter = (
@@ -168,29 +232,78 @@ def _get_files_from_stac(
168
232
  & (pc.field("bbox", "ymax") > ymin)
169
233
  )
170
234
 
171
- combined_filter = feature_type_filter & bbox_filter
172
- table = stac_table.filter(combined_filter)
235
+ table = type_items.filter(bbox_filter)
173
236
 
174
237
  if table.num_rows > 0:
175
- file_paths = table.column("assets").to_pylist()
176
-
177
- # clip out the "s3://" prefix
178
- s3_paths = [
179
- path["aws"]["alternate"]["s3"]["href"][len("s3://") :]
180
- for path in file_paths
181
- ]
182
- return s3_paths
238
+ return _s3_paths(table)
183
239
  else:
184
240
  print(
185
- f"No data found for release {release} in query bbox {bbox.as_tuple()}."
241
+ f"No data found for release {release} in query bbox {bbox.as_tuple()}.",
242
+ file=sys.stderr,
186
243
  )
187
244
  return []
188
245
 
189
246
  except Exception as e:
190
- print(f"Error reading STAC index at {stac_url}: {e}")
247
+ warnings.warn(
248
+ f"Could not use the STAC index for release {release} ({e}). Falling back "
249
+ f"to a full scan of s3://{_dataset_path(overture_type, release)}, which "
250
+ "reads metadata from every file and can take several minutes. "
251
+ "Results are not affected.",
252
+ StacIndexWarning,
253
+ stacklevel=_caller_stacklevel(),
254
+ )
191
255
  return None
192
256
 
193
257
 
258
+ def _stac_type_filter(overture_type: str) -> pc.Expression:
259
+ """Match the STAC index rows for the type's data files."""
260
+ return (pc.field("collection") == overture_type) & (pc.field("type") == "Feature")
261
+
262
+
263
+ def _s3_paths(stac_items: pa.Table) -> List[str]:
264
+ """Return the bucket/key paths of the data files in STAC index rows."""
265
+ # clip out the "s3://" prefix
266
+ return [
267
+ assets["aws"]["alternate"]["s3"]["href"][len("s3://") :]
268
+ for assets in stac_items.column("assets").to_pylist()
269
+ ]
270
+
271
+
272
+ def _type_schema(
273
+ overture_type: str, release: str, connect_timeout=None, request_timeout=None
274
+ ) -> pa.Schema:
275
+ """
276
+ Read the type's schema from the first of its data files in the STAC index,
277
+ which is quicker than listing the type's folder. Only called after
278
+ _get_files_from_stac has found files of the type, so the index is cached.
279
+ """
280
+ stac_items = _get_stac_index(release).filter(_stac_type_filter(overture_type))
281
+ dataset = ds.dataset(
282
+ _s3_paths(stac_items.slice(0, 1)),
283
+ filesystem=_s3_filesystem(connect_timeout, request_timeout),
284
+ )
285
+ return dataset.schema
286
+
287
+
288
+ def _caller_stacklevel() -> int:
289
+ """Return the warnings stacklevel of the first caller outside this module."""
290
+ frame = sys._getframe(1)
291
+ level = 1
292
+ while frame.f_back is not None and frame.f_globals.get("__name__") == __name__:
293
+ frame = frame.f_back
294
+ level += 1
295
+ return level
296
+
297
+
298
+ def _s3_filesystem(connect_timeout=None, request_timeout=None) -> fs.S3FileSystem:
299
+ return fs.S3FileSystem(
300
+ anonymous=True,
301
+ region="us-west-2",
302
+ connect_timeout=connect_timeout,
303
+ request_timeout=request_timeout,
304
+ )
305
+
306
+
194
307
  def _record_batch_reader_from_dataset(
195
308
  dataset: ds.Dataset,
196
309
  filter_expr=None,
@@ -234,13 +347,15 @@ def _prepare_query(
234
347
  release=None,
235
348
  connect_timeout=None,
236
349
  request_timeout=None,
237
- stac=False,
350
+ stac=True,
238
351
  ) -> Optional[Tuple[ds.Dataset, Optional[pc.Expression]]]:
239
352
  """
240
353
  Resolve the S3 dataset and filter expression for a given query.
241
354
 
242
355
  Returns the dataset and filter expression ready for counting or streaming,
243
- or None if STAC reports no files intersect the bbox.
356
+ or None if STAC reports no files intersect the bbox. If the STAC index
357
+ can't be read or lists no files of the type, warns with StacIndexWarning and
358
+ scans every file of the type.
244
359
  """
245
360
  if release is None:
246
361
  release = get_latest_release()
@@ -268,12 +383,7 @@ def _prepare_query(
268
383
 
269
384
  dataset = ds.dataset(
270
385
  intersecting_files if intersecting_files is not None else path,
271
- filesystem=fs.S3FileSystem(
272
- anonymous=True,
273
- region="us-west-2",
274
- connect_timeout=connect_timeout,
275
- request_timeout=request_timeout,
276
- ),
386
+ filesystem=_s3_filesystem(connect_timeout, request_timeout),
277
387
  )
278
388
 
279
389
  return dataset, filter_expr
@@ -285,7 +395,7 @@ def count_rows(
285
395
  release=None,
286
396
  connect_timeout=None,
287
397
  request_timeout=None,
288
- stac=False,
398
+ stac=True,
289
399
  ) -> int:
290
400
  """Return the number of rows matching the given parameters."""
291
401
  result = _prepare_query(
@@ -304,14 +414,19 @@ def record_batch_reader(
304
414
  release=None,
305
415
  connect_timeout=None,
306
416
  request_timeout=None,
307
- stac=False,
417
+ stac=True,
308
418
  ) -> Optional[pa.RecordBatchReader]:
309
419
  """Return a pyarrow RecordBatchReader for the desired bounding box and s3 path, or None on error."""
420
+ if release is None:
421
+ release = get_latest_release()
310
422
  result = _prepare_query(
311
423
  overture_type, bbox, release, connect_timeout, request_timeout, stac
312
424
  )
313
425
  if result is None:
314
- return None
426
+ # No files intersect the bbox. Return an empty reader with the type's
427
+ # schema, the same result a full scan gives.
428
+ schema = _type_schema(overture_type, release, connect_timeout, request_timeout)
429
+ return pa.RecordBatchReader.from_batches(geoarrow_schema_adapter(schema), [])
315
430
  dataset, filter_expr = result
316
431
  return _record_batch_reader_from_dataset(dataset, filter_expr=filter_expr)
317
432
 
@@ -322,7 +437,7 @@ def geodataframe(
322
437
  release: str = None,
323
438
  connect_timeout: int = None,
324
439
  request_timeout: int = None,
325
- stac: bool = False,
440
+ stac: bool = True,
326
441
  ) -> GeoDataFrame:
327
442
  """
328
443
  Loads geoparquet for specified type into a geopandas dataframe
@@ -331,8 +446,12 @@ def geodataframe(
331
446
  ----------
332
447
  overture_type: type to load
333
448
  bbox: optional bounding box for data fetch (xmin, ymin, xmax, ymax)
449
+ release: optional release version, defaults to the latest
334
450
  connect_timeout: optional connection timeout in seconds
335
451
  request_timeout: optional request timeout in seconds
452
+ stac: use the STAC index to read only the files that intersect the bbox
453
+ (default True). If the index can't be read or lists no files of the
454
+ type, a StacIndexWarning is issued and every file of the type is scanned.
336
455
 
337
456
  Returns
338
457
  -------
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "overturemaps"
3
3
  # Release workflows stamp the tag's version into this placeholder before building.
4
- version = "2.0.0"
4
+ version = "2.1.0"
5
5
  description = "Python tools for interacting with Overture Maps (overturemaps.org) data."
6
6
  authors = [
7
7
  {name = "Jacob Wasserman", email = "jwasserman@meta.com"}
@@ -0,0 +1,53 @@
1
+ """Shared fixtures for STAC index tests that make no network requests."""
2
+
3
+ import io
4
+ import threading
5
+ from concurrent.futures import Future
6
+
7
+ import pyarrow as pa
8
+ import pytest
9
+
10
+ from overturemaps import core
11
+
12
+ from .fake_stac import stac_index_bytes
13
+
14
+
15
+ @pytest.fixture
16
+ def empty_stac_cache(monkeypatch):
17
+ """Start with no cached or in-progress STAC index downloads."""
18
+ monkeypatch.setattr(core, "_cached_stac_indexes", {})
19
+ monkeypatch.setattr(core, "_stac_index_downloads", {})
20
+
21
+
22
+ @pytest.fixture
23
+ def stac_index(mocker, empty_stac_cache):
24
+ """Serve the fake STAC index through a mocked open_url, starting from an empty cache."""
25
+ data = stac_index_bytes()
26
+ return mocker.patch(
27
+ "overturemaps.core.open_url", side_effect=lambda *args, **kwargs: io.BytesIO(data)
28
+ )
29
+
30
+
31
+ @pytest.fixture
32
+ def index_waiters(mocker):
33
+ """A semaphore released each time a caller waits on another caller's index download."""
34
+ waiting = threading.Semaphore(0)
35
+
36
+ class WatchedDownload(Future):
37
+ def result(self, timeout=None):
38
+ waiting.release()
39
+ return super().result(timeout)
40
+
41
+ mocker.patch("overturemaps.core.Future", WatchedDownload)
42
+ return waiting
43
+
44
+
45
+ @pytest.fixture
46
+ def s3_dataset(mocker):
47
+ """Replace pyarrow dataset discovery so no S3 request is made."""
48
+ dataset = mocker.patch("overturemaps.core.ds.dataset")
49
+ dataset.return_value.count_rows.return_value = 42
50
+ dataset.return_value.schema = pa.schema(
51
+ [pa.field("id", pa.string()), pa.field("geometry", pa.binary())]
52
+ )
53
+ return dataset
@@ -0,0 +1,34 @@
1
+ """A tiny fake STAC index for tests that make no network requests."""
2
+
3
+ import io
4
+
5
+ import pyarrow as pa
6
+ import pyarrow.parquet as pq
7
+
8
+ RELEASE = "2026-10-21.0"
9
+ SF_BBOX = (-122.42, 37.77, -122.40, 37.785)
10
+ OCEAN_BBOX = (-40.0, 0.0, -39.9, 0.1)
11
+ WEST_FILE = f"overturemaps-us-west-2/release/{RELEASE}/theme=buildings/type=building/west.parquet"
12
+
13
+
14
+ def stac_index_bytes() -> bytes:
15
+ """A tiny STAC index: two building files in North America and one global place file."""
16
+
17
+ def item(collection, path, bbox):
18
+ return {
19
+ "type": "Feature",
20
+ "collection": collection,
21
+ "bbox": dict(zip(("xmin", "ymin", "xmax", "ymax"), bbox)),
22
+ "assets": {"aws": {"alternate": {"s3": {"href": f"s3://{path}"}}}},
23
+ }
24
+
25
+ table = pa.Table.from_pylist(
26
+ [
27
+ item("building", WEST_FILE, (-125.0, 30.0, -110.0, 45.0)),
28
+ item("building", WEST_FILE.replace("west", "east"), (-80.0, 35.0, -70.0, 45.0)),
29
+ item("place", WEST_FILE.replace("building", "place"), (-180.0, -90.0, 180.0, 90.0)),
30
+ ]
31
+ )
32
+ sink = io.BytesIO()
33
+ pq.write_table(table, sink)
34
+ return sink.getvalue()
@@ -1,11 +1,31 @@
1
1
  """Tests for the changelog module."""
2
2
 
3
+ from urllib.error import HTTPError
4
+
3
5
  import pytest
4
- from overturemaps.changelog import query_changelog_ids, summarize_changelog
6
+ from overturemaps.changelog import (
7
+ _get_changelog_files_from_stac,
8
+ query_changelog_ids,
9
+ summarize_changelog,
10
+ )
5
11
  from overturemaps.models import BBox
6
12
  from overturemaps.releases import get_latest_release
7
13
 
8
14
 
15
+ def test_changelog_index_download_is_not_retried(mocker):
16
+ sleep = mocker.patch("overturemaps._http.time.sleep")
17
+ urlopen = mocker.patch(
18
+ "overturemaps._http.urlopen",
19
+ side_effect=HTTPError("url", 429, "Too Many Requests", {}, None),
20
+ )
21
+
22
+ bbox = BBox(xmin=-71.068, ymin=42.353, xmax=-71.058, ymax=42.363)
23
+ # None means the query falls back to scanning the changelog folder.
24
+ assert _get_changelog_files_from_stac("buildings", "building", bbox, "2026-10-21.0") is None
25
+ assert urlopen.call_count == 1
26
+ sleep.assert_not_called()
27
+
28
+
9
29
  @pytest.mark.integration
10
30
  def test_query_changelog_ids():
11
31
  """Test querying changelog IDs for a small bbox."""
@@ -1,5 +1,6 @@
1
1
  """Tests for download CLI command."""
2
2
 
3
+ import json
3
4
  from pathlib import Path
4
5
 
5
6
  import pytest
@@ -11,6 +12,8 @@ from overturemaps.cli import (
11
12
  cli,
12
13
  )
13
14
 
15
+ from .fake_stac import OCEAN_BBOX, RELEASE
16
+
14
17
 
15
18
  class _DummyWriter:
16
19
  def __enter__(self):
@@ -255,3 +258,29 @@ def test_download_invalid_release_explains_retention_policy(monkeypatch):
255
258
  assert "60 days" in result.output
256
259
  assert "2026-03-18.0" in result.output
257
260
  assert "docs.overturemaps.org/release-calendar" in result.output
261
+
262
+
263
+ def test_download_empty_area_keeps_stdout_valid_geojson(monkeypatch, stac_index, s3_dataset):
264
+ """When no files intersect the bbox, stdout holds only an empty FeatureCollection."""
265
+ monkeypatch.setattr(
266
+ "overturemaps.cli.get_available_releases", lambda: ([RELEASE], RELEASE)
267
+ )
268
+
269
+ result = CliRunner().invoke(
270
+ cli,
271
+ [
272
+ "download",
273
+ "-f",
274
+ "geojson",
275
+ "-t",
276
+ "building",
277
+ "-r",
278
+ RELEASE,
279
+ "--bbox",
280
+ ",".join(str(v) for v in OCEAN_BBOX),
281
+ ],
282
+ )
283
+
284
+ assert result.exit_code == 0, result.output
285
+ assert json.loads(result.stdout) == {"type": "FeatureCollection", "features": []}
286
+ assert "No data found" in result.stderr